diff --git a/README.md b/README.md index 67f7e61..7f69ab9 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@ pulp_group role pulp_publication role pulp_repository role pulp_user role +container_repositories module +container_remotes module +container_syncs module +container_distributions module ## Using this collection diff --git a/galaxy.yml b/galaxy.yml index 1d1c5ff..3a8c22f 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -2,7 +2,7 @@ namespace: stackhpc name: pulp description: > Roles and plugins Pulp repository server configuration -version: "0.6.0" +version: "0.7.0" readme: "README.md" authors: - "Piotr Parczewski" @@ -10,7 +10,7 @@ authors: - "Mark Goddard" - "Alex Welsh" dependencies: - "pulp.squeezer": "0.2.3" + "pulp.squeezer": "0.3.0" license: - "Apache-2.0" tags: diff --git a/plugins/modules/container_distributions.py b/plugins/modules/container_distributions.py new file mode 100644 index 0000000..5b5c8db --- /dev/null +++ b/plugins/modules/container_distributions.py @@ -0,0 +1,274 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2025, StackHPC +# Apache License, Version 2.0 (see LICENSE or http://www.apache.org/licenses/LICENSE-2.0) + +DOCUMENTATION = r""" +--- +module: container_distributions +short_description: Manage multiple container distributions of a pulp api server instance +description: + - "This performs CRUD operations on multiple container distributions in a pulp api server instance concurrently." +options: + distributions: + description: + - List of distributions to manage + type: list + elements: dict + suboptions: + name: + description: + - Name of the distribution + type: str + required: true + base_path: + description: + - Base path of the distribution + type: str + repository: + description: + - Repository name the distribution serves + type: str + version: + description: + - Repository version the distribution serves + type: int + content_guard: + description: + - Content guard to protect the distribution + type: str + private: + description: + - Whether the distribution is private + type: bool + state: + description: + - Desired state of the distribution + type: str + choices: ["present", "absent"] + default: present + required: true + concurrency: + description: + - Maximum number of concurrent operations + type: int + default: 10 +extends_documentation_fragment: + - pulp.squeezer.pulp +author: + - Alex Welsh (@alex-welsh) +""" + +EXAMPLES = r""" +- name: Create multiple container distributions + stackhpc.pulp.container_distributions: + pulp_url: https://pulp.example.org + username: admin + password: password + distributions: + - name: dist1 + base_path: dist1 + repository: repo1 + state: present + - name: dist2 + base_path: dist2 + repository: repo2 + private: true + state: present + +- name: Delete multiple container distributions + stackhpc.pulp.container_distributions: + pulp_url: https://pulp.example.org + username: admin + password: password + distributions: + - name: dist1 + state: absent + - name: dist2 + state: absent +""" + +RETURN = r""" + distributions: + description: List of container distribution results + type: list + returned: always + elements: dict + contains: + name: + description: Name of the distribution + type: str + distribution: + description: Distribution details (when applicable) + type: dict + changed: + description: Whether the distribution was changed + type: bool + failed: + description: Whether the operation failed + type: bool + msg: + description: Error message if failed + type: str + msg: + description: Summary of the overall operation failure + type: str + returned: on failure +""" + + +import traceback +import concurrent.futures + +from ansible_collections.pulp.squeezer.plugins.module_utils.pulp_glue import PulpAnsibleModule + +try: + from pulp_glue.container.context import ( + PulpContainerDistributionContext, + PulpContainerRepositoryContext, + ) + from pulp_glue.common.context import PulpContext + from pulp_glue.common.openapi import BasicAuthProvider + from pulp_glue.common import __version__ as pulp_glue_version + + PULP_GLUE_IMPORT_ERR = None +except ImportError: + PULP_GLUE_IMPORT_ERR = traceback.format_exc() + PulpContainerDistributionContext = None + PulpContext = None + BasicAuthProvider = None + pulp_glue_version = None + + +class PulpBatchDistributionAnsibleModule(PulpAnsibleModule): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def process_single_distribution(self, entity): + result = { + "name": entity["name"], + "changed": False, + "failed": False, + "msg": "", + } + try: + # Create a separate PulpContext for each thread to avoid correlation ID conflicts + auth_args = {} + if self.params["username"]: + auth_args["auth_provider"] = BasicAuthProvider( + username=self.params["username"], + password=self.params["password"], + ) + + pulp_ctx = PulpContext( + api_root="/pulp/", + api_kwargs=dict( + base_url=self.params["pulp_url"], + cert=self.params["user_cert"], + key=self.params["user_key"], + validate_certs=self.params["validate_certs"], + refresh_cache=self.params["refresh_api_cache"], + user_agent=f"Squeezer/{pulp_glue_version}", + **auth_args, + ), + background_tasks=False, + timeout=self.params["timeout"], + fake_mode=self.check_mode, + ) + + context = PulpContainerDistributionContext(pulp_ctx) + natural_key = {"name": entity["name"]} + desired_attributes = {} + + # Map fields + for key in ["base_path", "private", "content_guard"]: + if key in entity and entity[key] is not None: + desired_attributes[key] = entity[key] + + if "repository" in entity and entity["repository"]: + repo_ctx = PulpContainerRepositoryContext(pulp_ctx, entity={"name": entity["repository"]}) + if not repo_ctx.entity: + result["failed"] = True + result["msg"] = f"Repository '{entity['repository']}' not found." + return result + if "version" in entity and entity["version"] is not None: + repo_version = repo_ctx.get_version_context().find(number=entity["version"]) + if repo_version: + desired_attributes["repository_version"] = repo_version["pulp_href"] + else: + result["failed"] = True + result["msg"] = f"Repository version '{entity['version']}' not found for repository '{entity['repository']}'." + return result + else: + desired_attributes["repository"] = repo_ctx.entity["pulp_href"] + + state = entity.get("state", "present") + if state == "present": + desired_entity = desired_attributes + elif state == "absent": + desired_entity = None + else: + result["failed"] = True + result["msg"] = f"Invalid state '{state}'" + return result + + # Simulate the converge logic + context.entity = natural_key + changed, before, after = context.converge(desired_entity) + if changed: + result["changed"] = True + if after is not None: + result["distribution"] = after + except Exception as e: + result["failed"] = True + result["msg"] = traceback.format_exc() + return result + + def process_batch_distributions(self, distributions, concurrency=10): + results = [] + overall_changed = False + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(self.process_single_distribution, entity) for entity in distributions] + for future in concurrent.futures.as_completed(futures): + result = future.result() + if result["changed"]: + overall_changed = True + results.append(result) + + # Sort results by original order + results.sort(key=lambda x, m={e["name"]: i for i, e in enumerate(distributions)}: m[x["name"]]) + + if overall_changed: + self.set_changed() + self.set_result("distributions", results) + if any(r["failed"] for r in results): + self.fail_json(msg="One or more items failed", distributions=results) + + +def main(): + with PulpBatchDistributionAnsibleModule( + import_errors=[("pulp-glue", PULP_GLUE_IMPORT_ERR)], + argument_spec={ + "distributions": { + "type": "list", + "elements": "dict", + "options": { + "name": {"required": True, "type": "str"}, + "base_path": {"type": "str"}, + "repository": {"type": "str"}, + "version": {"type": "int"}, + "content_guard": {"type": "str"}, + "private": {"type": "bool"}, + "state": {"choices": ["present", "absent"], "default": "present", "type": "str"}, + }, + "required": True, + }, + "concurrency": {"type": "int", "default": 10}, + }, + ) as module: + module.process_batch_distributions(module.params["distributions"], module.params["concurrency"]) + + +if __name__ == "__main__": + main() diff --git a/plugins/modules/container_remotes.py b/plugins/modules/container_remotes.py new file mode 100644 index 0000000..b7aae64 --- /dev/null +++ b/plugins/modules/container_remotes.py @@ -0,0 +1,372 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2025, StackHPC +# Apache License, Version 2.0 (see LICENSE or http://www.apache.org/licenses/LICENSE-2.0) + +DOCUMENTATION = r""" +--- +module: container_remotes +short_description: Manage multiple container remotes of a pulp api server instance +description: + - "This performs CRUD operations on multiple container remotes in a pulp api server instance concurrently." +options: + remotes: + description: + - List of remotes to manage + type: list + elements: dict + suboptions: + name: + description: + - Name of the remote + type: str + required: true + upstream_name: + description: + - Name of the upstream repository + type: str + url: + description: + - URL of the remote + type: str + policy: + description: + - Whether downloads should be performed immediately, or lazy. + type: str + choices: + - immediate + - on_demand + - streamed + exclude_tags: + description: + - A list of tags to exclude during sync + type: list + elements: str + include_tags: + description: + - A list of tags to include during sync + type: list + elements: str + headers: + description: + - Headers to send with requests + type: list + elements: dict + remote_username: + description: + - Username for remote authentication + type: str + remote_password: + description: + - Password for remote authentication + type: str + ca_cert: + description: + - CA certificate for TLS validation + type: str + client_cert: + description: + - Client certificate for authentication + type: str + client_key: + description: + - Client key for authentication + type: str + tls_validation: + description: + - Whether to validate TLS certificates + type: bool + proxy_url: + description: + - Proxy URL + type: str + proxy_username: + description: + - Username for proxy authentication + type: str + proxy_password: + description: + - Password for proxy authentication + type: str + download_concurrency: + description: + - Number of concurrent downloads + type: int + rate_limit: + description: + - Rate limit for downloads + type: int + total_timeout: + description: + - Total timeout for operations + type: float + connect_timeout: + description: + - Connect timeout + type: float + sock_connect_timeout: + description: + - Socket connect timeout + type: float + sock_read_timeout: + description: + - Socket read timeout + type: float + max_retries: + description: + - Maximum number of retries + type: int + state: + description: + - Desired state of the remote + type: str + choices: ["present", "absent"] + default: present + required: true + concurrency: + description: + - Maximum number of concurrent operations + type: int + default: 10 +extends_documentation_fragment: + - pulp.squeezer.pulp +author: + - Alex Welsh (@alex-welsh) +""" + +EXAMPLES = r""" +- name: Create multiple container remotes + stackhpc.pulp.container_remotes: + pulp_url: https://pulp.example.org + username: admin + password: password + remotes: + - name: remote1 + upstream_name: upstream1 + url: https://registry.example.com/repo1 + policy: immediate + state: present + - name: remote2 + upstream_name: upstream2 + url: https://registry.example.com/repo2 + state: present + +- name: Delete multiple container remotes + stackhpc.pulp.container_remotes: + pulp_url: https://pulp.example.org + username: admin + password: password + remotes: + - name: remote1 + state: absent + - name: remote2 + state: absent +""" + +RETURN = r""" + remotes: + description: List of container remote results + type: list + returned: always + elements: dict + contains: + name: + description: Name of the remote + type: str + remote: + description: Remote details (when applicable) + type: dict + changed: + description: Whether the remote was changed + type: bool + failed: + description: Whether the operation failed + type: bool + msg: + description: Error message if failed + type: str + msg: + description: Summary of the overall operation failure + type: str + returned: on failure +""" + + +import traceback +import concurrent.futures + +from ansible_collections.pulp.squeezer.plugins.module_utils.pulp_glue import PulpAnsibleModule + +try: + from pulp_glue.container.context import PulpContainerRemoteContext + from pulp_glue.common.context import PulpContext + from pulp_glue.common.openapi import BasicAuthProvider + from pulp_glue.common import __version__ as pulp_glue_version + + PULP_GLUE_IMPORT_ERR = None +except ImportError: + PULP_GLUE_IMPORT_ERR = traceback.format_exc() + PulpContainerRemoteContext = None + PulpContext = None + BasicAuthProvider = None + pulp_glue_version = None + + +class PulpBatchRemoteAnsibleModule(PulpAnsibleModule): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def process_single_remote(self, remote_item): + result = { + "name": remote_item["name"], + "changed": False, + "failed": False, + "msg": "", + } + try: + # Create a separate PulpContext for each thread to avoid correlation ID conflicts + auth_args = {} + if self.params["username"]: + auth_args["auth_provider"] = BasicAuthProvider( + username=self.params["username"], + password=self.params["password"], + ) + + pulp_ctx = PulpContext( + api_root="/pulp/", + api_kwargs=dict( + base_url=self.params["pulp_url"], + cert=self.params["user_cert"], + key=self.params["user_key"], + validate_certs=self.params["validate_certs"], + refresh_cache=self.params["refresh_api_cache"], + user_agent=f"Squeezer/{pulp_glue_version}", + **auth_args, + ), + background_tasks=False, + timeout=self.params["timeout"], + fake_mode=self.check_mode, + ) + + context = PulpContainerRemoteContext(pulp_ctx) + natural_key = {"name": remote_item["name"]} + desired_attributes = {} + + # Collect desired attributes from remote_item + for key in [ + "upstream_name", "url", "policy", "exclude_tags", "include_tags", + "headers", "ca_cert", "client_cert", "client_key", "tls_validation", + "proxy_url", "proxy_username", "proxy_password", "download_concurrency", + "rate_limit", "total_timeout", "connect_timeout", "sock_connect_timeout", + "sock_read_timeout", "max_retries" + ]: + if key in remote_item and remote_item[key] is not None: + desired_attributes[key] = remote_item[key] + + # Handle auth + if "remote_username" in remote_item and remote_item["remote_username"] is not None: + desired_attributes["username"] = remote_item["remote_username"] + if "remote_password" in remote_item and remote_item["remote_password"] is not None: + desired_attributes["password"] = remote_item["remote_password"] + + state = remote_item.get("state", "present") + if state == "present": + desired_entity = desired_attributes + elif state == "absent": + desired_entity = None + else: + result["failed"] = True + result["msg"] = f"Invalid state '{state}'" + return result + + # Simulate the converge logic + context.entity = natural_key + changed, before, after = context.converge(desired_entity) + if changed: + result["changed"] = True + if after is not None: + # Sanitize sensitive data from the returned object + if "password" in after: + del after["password"] + if "proxy_password" in after: + del after["proxy_password"] + if "client_key" in after: + del after["client_key"] + result["remote"] = after + except Exception as e: + result["failed"] = True + result["msg"] = traceback.format_exc() + return result + + def process_batch_remotes(self, remotes, concurrency=10): + results = [] + overall_changed = False + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(self.process_single_remote, remote_item) for remote_item in remotes] + for future in concurrent.futures.as_completed(futures): + result = future.result() + if result["changed"]: + overall_changed = True + results.append(result) + + # Sort results by original order + results.sort(key=lambda x, m={r["name"]: i for i, r in enumerate(remotes)}: m[x["name"]]) + + if overall_changed: + self.set_changed() + self.set_result("remotes", results) + if any(r["failed"] for r in results): + self.fail_json(msg="One or more items failed", remotes=results) + + +def main(): + with PulpBatchRemoteAnsibleModule( + import_errors=[("pulp-glue", PULP_GLUE_IMPORT_ERR)], + argument_spec={ + "remotes": { + "type": "list", + "elements": "dict", + "required": True, + "options": { + "name": {"type": "str", "required": True}, + "upstream_name": {"type": "str"}, + "url": {"type": "str"}, + "policy": { + "type": "str", + "choices": ["immediate", "on_demand", "streamed"], + }, + "exclude_tags": {"type": "list", "elements": "str"}, + "include_tags": {"type": "list", "elements": "str"}, + "headers": {"type": "list", "elements": "dict"}, + "remote_username": {"type": "str"}, + "remote_password": {"type": "str", "no_log": True}, + "ca_cert": {"type": "str"}, + "client_cert": {"type": "str"}, + "client_key": {"type": "str", "no_log": True}, + "tls_validation": {"type": "bool"}, + "proxy_url": {"type": "str"}, + "proxy_username": {"type": "str"}, + "proxy_password": {"type": "str", "no_log": True}, + "download_concurrency": {"type": "int"}, + "rate_limit": {"type": "int"}, + "total_timeout": {"type": "float"}, + "connect_timeout": {"type": "float"}, + "sock_connect_timeout": {"type": "float"}, + "sock_read_timeout": {"type": "float"}, + "max_retries": {"type": "int"}, + "state": { + "type": "str", + "choices": ["present", "absent"], + "default": "present", + }, + }, + }, + "concurrency": {"type": "int", "default": 10}, + }, + ) as module: + module.process_batch_remotes(module.params["remotes"], module.params["concurrency"]) + + +if __name__ == "__main__": + main() diff --git a/plugins/modules/container_repositories.py b/plugins/modules/container_repositories.py new file mode 100644 index 0000000..5d27be6 --- /dev/null +++ b/plugins/modules/container_repositories.py @@ -0,0 +1,241 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2025, StackHPC +# Apache License, Version 2.0 (see LICENSE or http://www.apache.org/licenses/LICENSE-2.0) + +DOCUMENTATION = r""" +--- +module: container_repositories +short_description: Manage multiple container repositories of a pulp api server instance +description: + - "This performs CRUD operations on multiple container repositories in a pulp api server instance in a single call." +options: + repositories: + description: + - List of repositories to manage + type: list + elements: dict + required: true + suboptions: + name: + description: + - Name of the repository + type: str + required: true + description: + description: + - Description of the repository + type: str + state: + description: + - Desired state of the repository + type: str + choices: ["present", "absent"] + default: present + concurrency: + description: + - Maximum number of concurrent operations + type: int + default: 10 +extends_documentation_fragment: + - pulp.squeezer.pulp +author: + - Mark Goddard (@markgoddard) +""" + +EXAMPLES = r""" +- name: Create multiple container repositories + stackhpc.pulp.container_repositories: + pulp_url: https://pulp.example.org + username: admin + password: password + repositories: + - name: repo1 + description: A brand new repository + state: present + - name: repo2 + description: Another repository + state: present + +- name: Create multiple container repositories with custom concurrency + stackhpc.pulp.container_repositories: + pulp_url: https://pulp.example.org + username: admin + password: password + concurrency: 5 + repositories: + - name: repo1 + description: A brand new repository + - name: repo2 + description: Another repository + +- name: Delete multiple container repositories + stackhpc.pulp.container_repositories: + pulp_url: https://pulp.example.org + username: admin + password: password + repositories: + - name: repo1 + state: absent + - name: repo2 + state: absent +""" + +RETURN = r""" + repositories: + description: List of container repository results + type: list + returned: always + elements: dict + contains: + name: + description: Name of the repository + type: str + repository: + description: Repository details (when applicable) + type: dict + changed: + description: Whether the repository was changed + type: bool + failed: + description: Whether the operation failed + type: bool + msg: + description: Error message if failed + type: str + msg: + description: Summary of the overall operation failure + type: str + returned: on failure +""" + + +import traceback +import concurrent.futures + +from ansible_collections.pulp.squeezer.plugins.module_utils.pulp_glue import PulpAnsibleModule + +try: + from pulp_glue.container.context import PulpContainerRepositoryContext + from pulp_glue.common.context import PulpContext + from pulp_glue.common.openapi import BasicAuthProvider + from pulp_glue.common import __version__ as pulp_glue_version + PULP_GLUE_IMPORT_ERR = None +except ImportError: + PULP_GLUE_IMPORT_ERR = traceback.format_exc() + PulpContainerRepositoryContext = None + PulpContext = None + BasicAuthProvider = None + pulp_glue_version = None + + +class PulpBatchRepositoryAnsibleModule(PulpAnsibleModule): + def __init__(self, context_class, **kwargs): + super().__init__(**kwargs) + self.context_class = context_class + + def process_single_repository(self, entity): + result = { + "name": entity["name"], + "changed": False, + "failed": False, + "msg": "", + } + try: + # Create a separate PulpContext for each thread to avoid correlation ID conflicts + auth_args = {} + if self.params["username"]: + auth_args["auth_provider"] = BasicAuthProvider( + username=self.params["username"], + password=self.params["password"], + ) + + pulp_ctx = PulpContext( + api_root="/pulp/", + api_kwargs=dict( + base_url=self.params["pulp_url"], + cert=self.params["user_cert"], + key=self.params["user_key"], + validate_certs=self.params["validate_certs"], + refresh_cache=self.params["refresh_api_cache"], + user_agent=f"Squeezer/{pulp_glue_version}", + **auth_args, + ), + background_tasks=False, + timeout=self.params["timeout"], + fake_mode=self.check_mode, + ) + + context = self.context_class(pulp_ctx) + natural_key = {"name": entity["name"]} + desired_attributes = {} + if "description" in entity and entity["description"] is not None: + desired_attributes["description"] = entity["description"] + + state = entity.get("state", "present") + if state == "present": + desired_entity = desired_attributes + elif state == "absent": + desired_entity = None + else: + result["failed"] = True + result["msg"] = f"Invalid state '{state}'" + return result + + # Simulate the converge logic + context.entity = natural_key + changed, before, after = context.converge(desired_entity) + if changed: + result["changed"] = True + if after is not None: + result["repository"] = after + except Exception as e: + result["failed"] = True + result["msg"] = traceback.format_exc() + return result + + def process_batch_repositories(self, entities, concurrency=10): + results = [] + overall_changed = False + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(self.process_single_repository, entity) for entity in entities] + for future in concurrent.futures.as_completed(futures): + result = future.result() + if result["changed"]: + overall_changed = True + results.append(result) + + # Sort results by original order + results.sort(key=lambda x, m={e["name"]: i for i, e in enumerate(entities)}: m[x["name"]]) + + if overall_changed: + self.set_changed() + self.set_result("repositories", results) + if any(r["failed"] for r in results): + self.fail_json(msg="One or more items failed", repositories=results) + + +def main(): + with PulpBatchRepositoryAnsibleModule( + context_class=PulpContainerRepositoryContext, + import_errors=[("pulp-glue", PULP_GLUE_IMPORT_ERR)], + argument_spec={ + "repositories": { + "type": "list", + "elements": "dict", + "options": { + "name": {"required": True, "type": "str"}, + "description": {"type": "str"}, + "state": {"choices": ["present", "absent"], "default": "present"}, + }, + "required": True, + }, + "concurrency": {"type": "int", "default": 10}, + }, + ) as module: + module.process_batch_repositories(module.params["repositories"], module.params["concurrency"]) + + +if __name__ == "__main__": + main() diff --git a/plugins/modules/container_syncs.py b/plugins/modules/container_syncs.py new file mode 100644 index 0000000..4d17b84 --- /dev/null +++ b/plugins/modules/container_syncs.py @@ -0,0 +1,237 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +# Copyright: (c) 2025, StackHPC +# Apache License, Version 2.0 (see LICENSE or http://www.apache.org/licenses/LICENSE-2.0) + +DOCUMENTATION = r""" +--- +module: container_syncs +short_description: Synchronize multiple container remotes on a pulp server concurrently +description: + - "This module synchronizes multiple container remotes into repositories concurrently." + - "In check_mode this module assumes, nothing changed upstream." +options: + syncs: + description: + - List of sync operations to perform + type: list + elements: dict + suboptions: + remote: + description: + - Name of the remote to synchronize + type: str + required: false + repository: + description: + - Name of the repository + type: str + required: true + timeout: + description: + - Timeout for the sync operation + type: int + default: 3600 + required: true + concurrency: + description: + - Maximum number of concurrent sync operations + type: int + default: 10 +extends_documentation_fragment: + - pulp.squeezer.pulp +author: + - Alex Welsh (@alex-welsh) +""" + +EXAMPLES = r""" +- name: Sync multiple container remotes into repositories + stackhpc.pulp.container_syncs: + pulp_url: https://pulp.example.org + username: admin + password: password + syncs: + - repository: repo_1 + remote: remote_1 + - repository: repo_2 + remote: remote_2 + register: sync_results + +- name: Sync multiple repositories with custom concurrency + stackhpc.pulp.container_syncs: + pulp_url: https://pulp.example.org + username: admin + password: password + concurrency: 5 + syncs: + - repository: repo_1 + - repository: repo_2 + timeout: 7200 +""" + +RETURN = r""" + syncs: + description: List of sync operation results + type: list + returned: always + elements: dict + contains: + repository: + description: Name of the repository + type: str + repository_version: + description: Repository version after syncing + type: dict + changed: + description: Whether the sync changed the repository + type: bool + failed: + description: Whether the sync operation failed + type: bool + msg: + description: Error message if failed + type: str + msg: + description: Summary of the overall operation failure + type: str + returned: on failure +""" + + +import traceback +import concurrent.futures + +from ansible_collections.pulp.squeezer.plugins.module_utils.pulp_glue import PulpAnsibleModule + +try: + from pulp_glue.container.context import ( + PulpContainerRemoteContext, + PulpContainerRepositoryContext, + ) + from pulp_glue.common.context import PulpContext + from pulp_glue.common.openapi import BasicAuthProvider + from pulp_glue.common import __version__ as pulp_glue_version + + PULP_GLUE_IMPORT_ERR = None +except ImportError: + PULP_GLUE_IMPORT_ERR = traceback.format_exc() + PulpContainerRemoteContext = None + PulpContainerRepositoryContext = None + PulpContext = None + BasicAuthProvider = None + pulp_glue_version = None + + +class PulpBatchSyncAnsibleModule(PulpAnsibleModule): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def process_single_sync(self, sync_item): + result = { + "repository": sync_item["repository"], + "changed": False, + "failed": False, + "msg": "", + } + try: + # Create a separate PulpContext for each thread to avoid correlation ID conflicts + auth_args = {} + if self.params["username"]: + auth_args["auth_provider"] = BasicAuthProvider( + username=self.params["username"], + password=self.params["password"], + ) + + pulp_ctx = PulpContext( + api_root="/pulp/", + api_kwargs=dict( + base_url=self.params["pulp_url"], + cert=self.params["user_cert"], + key=self.params["user_key"], + validate_certs=self.params["validate_certs"], + refresh_cache=self.params["refresh_api_cache"], + user_agent=f"Squeezer/{pulp_glue_version}", + **auth_args, + ), + background_tasks=False, + timeout=sync_item.get("timeout", 3600), + fake_mode=self.check_mode, + ) + + repository_ctx = PulpContainerRepositoryContext( + pulp_ctx, entity={"name": sync_item["repository"]} + ) + repository = repository_ctx.entity + + payload = {} + remote_name = sync_item.get("remote") + if remote_name is None: + if repository.get("remote") is None: + raise Exception( + "No remote was specified and none preconfigured on the repository." + ) + else: + remote_ctx = PulpContainerRemoteContext( + pulp_ctx, entity={"name": remote_name} + ) + payload["remote"] = remote_ctx + + repository_version = repository.get("latest_version_href") + # In check_mode, assume nothing changed + if not self.check_mode: + sync_task = repository_ctx.sync(body=payload) + + if sync_task["created_resources"]: + result["changed"] = True + repository_version = sync_task["created_resources"][0] + + result["repository_version"] = repository_version + except Exception as e: + result["failed"] = True + result["msg"] = traceback.format_exc() + return result + + def process_batch_syncs(self, syncs, concurrency=10): + results = [] + overall_changed = False + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = [executor.submit(self.process_single_sync, sync_item) for sync_item in syncs] + for future in concurrent.futures.as_completed(futures): + result = future.result() + if result["changed"]: + overall_changed = True + results.append(result) + + # Sort results by original order + results.sort(key=lambda x, m={s["repository"]: i for i, s in enumerate(syncs)}: m[x["repository"]]) + + if overall_changed: + self.set_changed() + self.set_result("syncs", results) + if any(r["failed"] for r in results): + self.fail_json(msg="One or more items failed", syncs=results) + + +def main(): + with PulpBatchSyncAnsibleModule( + import_errors=[("pulp-glue", PULP_GLUE_IMPORT_ERR)], + argument_spec={ + "syncs": { + "type": "list", + "elements": "dict", + "options": { + "remote": {"type": "str"}, + "repository": {"required": True, "type": "str"}, + "timeout": {"type": "int", "default": 3600}, + }, + "required": True, + }, + "concurrency": {"type": "int", "default": 10}, + }, + ) as module: + module.process_batch_syncs(module.params["syncs"], module.params["concurrency"]) + + +if __name__ == "__main__": + main() diff --git a/roles/pulp_distribution/defaults/main.yml b/roles/pulp_distribution/defaults/main.yml index d5796c3..5bd6658 100644 --- a/roles/pulp_distribution/defaults/main.yml +++ b/roles/pulp_distribution/defaults/main.yml @@ -13,3 +13,6 @@ pulp_distribution_rpm: [] # publication. pulp_distribution_deb_skip_existing: false pulp_distribution_rpm_skip_existing: false + +pulp_distribution_concurrency: 10 +pulp_distribution_retries: 3 diff --git a/roles/pulp_distribution/tasks/container.yml b/roles/pulp_distribution/tasks/container.yml index 937d325..93880bf 100644 --- a/roles/pulp_distribution/tasks/container.yml +++ b/roles/pulp_distribution/tasks/container.yml @@ -1,15 +1,13 @@ --- - name: Ensure container distributions are defined - pulp.squeezer.container_distribution: + stackhpc.pulp.container_distributions: pulp_url: "{{ pulp_url }}" username: "{{ pulp_username }}" password: "{{ pulp_password }}" validate_certs: "{{ pulp_validate_certs | bool }}" - name: "{{ item.name }}" - base_path: "{{ item.base_path | default(omit) }}" - repository: "{{ item.repository | default(omit) }}" - version: "{{ item.version | default(omit) }}" - content_guard: "{{ item.content_guard | default(omit) }}" - private: "{{ item.private | default(omit) }}" - state: "{{ item.state }}" - with_items: "{{ pulp_distribution_container }}" + distributions: "{{ pulp_distribution_container | map('dict2items') | map('selectattr', 'key', 'in', ['name', 'base_path', 'repository', 'version', 'content_guard', 'private', 'state']) | map('items2dict') | list }}" + concurrency: "{{ pulp_distribution_concurrency }}" + register: pulp_distribution_container_distributions + until: "not ((pulp_distribution_container_distributions | default({})).get('distributions', []) | selectattr('failed') | list)" + retries: "{{ pulp_distribution_retries }}" + delay: 1 diff --git a/roles/pulp_repository/README.md b/roles/pulp_repository/README.md index 970afc1..bb7633b 100644 --- a/roles/pulp_repository/README.md +++ b/roles/pulp_repository/README.md @@ -14,6 +14,7 @@ Role variables * `pulp_repository_rpm_repos`: List of RPM repositories. Default is an empty list. * `pulp_repository_python_repos`: List of PyPI repositories. Default is an empty list. * `pulp_repository_deb_repos`: List of Deb respositories. Default is an empty list. +* `pulp_repository_concurrency`: Concurrency level for batch operations (currently only supported for container repositories). Default is 10. Example playbook ---------------- diff --git a/roles/pulp_repository/defaults/main.yml b/roles/pulp_repository/defaults/main.yml index f1ae7c5..ce7b00f 100644 --- a/roles/pulp_repository/defaults/main.yml +++ b/roles/pulp_repository/defaults/main.yml @@ -13,6 +13,7 @@ pulp_repository_retries: 3 pulp_repository_sync_retries: "{{ pulp_repository_retries }}" pulp_repository_remote_retries: "{{ pulp_repository_retries }}" pulp_repository_create_repository_retries: "{{ pulp_repository_retries }}" +pulp_repository_concurrency: 10 pulp_repository_container_repos_sync_retries: "{{ pulp_repository_sync_retries }}" pulp_repository_container_remotes_retries: "{{ pulp_repository_remote_retries }}" diff --git a/roles/pulp_repository/tasks/container.yml b/roles/pulp_repository/tasks/container.yml index 821b39a..b76d13e 100644 --- a/roles/pulp_repository/tasks/container.yml +++ b/roles/pulp_repository/tasks/container.yml @@ -1,69 +1,83 @@ --- - name: Setup container repositories - pulp.squeezer.container_repository: + stackhpc.pulp.container_repositories: pulp_url: "{{ pulp_url }}" username: "{{ pulp_username }}" password: "{{ pulp_password }}" validate_certs: "{{ pulp_validate_certs | bool }}" - name: "{{ pulp_repository_container_repos[repository_index].name }}" - state: "{{ pulp_repository_container_repos[repository_index].state }}" - loop: "{{ pulp_repository_container_repos | map(attribute='name') }}" - loop_control: - index_var: repository_index + repositories: "{{ pulp_repository_container_repos | map('dict2items') | map('selectattr', 'key', 'in', ['name', 'description', 'state']) | map('items2dict') | list }}" + concurrency: "{{ pulp_repository_concurrency }}" register: pulp_repository_container_repositories - until: "pulp_repository_container_repositories is not failed" + until: "not ((pulp_repository_container_repositories | default({})).get('repositories', []) | selectattr('failed') | list)" retries: "{{ pulp_repository_container_repositories_retries }}" delay: 1 +- name: Initialize container remotes list + set_fact: + container_remotes_list: [] + + # When state is absent, having values as None fails with a type error + # (even though it doesn't matter because it's getting deleted) + # Hence the rejectattr('value', 'none') +- name: Build container remotes list + set_fact: + container_remotes_list: "{{ container_remotes_list + [{ + 'name': item.name + '-remote', + 'upstream_name': item.get('upstream_name', item.name), + 'url': item.get('url'), + 'ca_cert': item.get('ca_cert'), + 'client_cert': item.get('client_cert'), + 'client_key': item.get('client_key'), + 'download_concurrency': item.get('download_concurrency'), + 'exclude_tags': item.get('exclude_tags'), + 'include_tags': item.get('include_tags'), + 'policy': item.get('policy'), + 'proxy_url': item.get('proxy_url'), + 'proxy_username': item.get('proxy_username'), + 'proxy_password': item.get('proxy_password'), + 'remote_username': item.get('remote_username'), + 'remote_password': item.get('remote_password'), + 'tls_validation': item.get('tls_validation'), + 'state': item.get('state') } | dict2items | rejectattr('value', 'none') | items2dict ] }}" + loop: "{{ (pulp_repository_container_repos | selectattr('state', 'equalto', 'absent') + pulp_repository_container_repos | selectattr('url', 'defined')) | unique }}" + loop_control: + loop_var: item + label: "{{ item.name }}" + - name: Setup container remotes - pulp.squeezer.container_remote: + stackhpc.pulp.container_remotes: pulp_url: "{{ pulp_url }}" username: "{{ pulp_username }}" password: "{{ pulp_password }}" validate_certs: "{{ pulp_validate_certs | bool }}" - name: "{{ pulp_repository_container_repos[repository_index].name }}-remote" - ca_cert: "{{ pulp_repository_container_repos[repository_index].ca_cert | default(omit) }}" - client_cert: "{{ pulp_repository_container_repos[repository_index].client_cert | default(omit) }}" - client_key: "{{ pulp_repository_container_repos[repository_index].client_key | default(omit) }}" - download_concurrency: "{{ pulp_repository_container_repos[repository_index].download_concurrency | default(omit) }}" - exclude_tags: "{{ pulp_repository_container_repos[repository_index].exclude_tags | default(omit) }}" - include_tags: "{{ pulp_repository_container_repos[repository_index].include_tags | default(omit) }}" - policy: "{{ pulp_repository_container_repos[repository_index].policy | default(omit) }}" - proxy_url: "{{ pulp_repository_container_repos[repository_index].proxy_url | default(omit) }}" - proxy_username: "{{ pulp_repository_container_repos[repository_index].proxy_username | default(omit) }}" - proxy_password: "{{ pulp_repository_container_repos[repository_index].proxy_password | default(omit) }}" - remote_username: "{{ pulp_repository_container_repos[repository_index].remote_username | default(omit) }}" - remote_password: "{{ pulp_repository_container_repos[repository_index].remote_password | default(omit) }}" - tls_validation: "{{ pulp_repository_container_repos[repository_index].tls_validation | default(omit) }}" - upstream_name: "{{ pulp_repository_container_repos[repository_index].upstream_name | default(pulp_repository_container_repos[repository_index].name) }}" - url: "{{ pulp_repository_container_repos[repository_index].url | default(omit) }}" - state: "{{ pulp_repository_container_repos[repository_index].state }}" - when: > - pulp_repository_container_repos[repository_index].state == "absent" or - pulp_repository_container_repos[repository_index].url is defined - loop: "{{ pulp_repository_container_repos | map(attribute='name') }}" - loop_control: - index_var: repository_index + remotes: "{{ container_remotes_list }}" + concurrency: "{{ pulp_repository_concurrency }}" register: pulp_repository_container_remotes - until: "pulp_repository_container_remotes is not failed" + until: "not ((pulp_repository_container_remotes | default({})).get('remotes', []) | selectattr('failed') | list)" retries: "{{ pulp_repository_container_remotes_retries }}" delay: 1 +- name: Initialize container syncs list + set_fact: + container_syncs_list: [] + +- name: Build container syncs list + set_fact: + container_syncs_list: "{{ container_syncs_list + [{'repository': item.name, 'remote': item.name + '-remote'}] }}" + loop: "{{ pulp_repository_container_repos | selectattr('url', 'defined') | selectattr('state', 'equalto', 'present') }}" + loop_control: + loop_var: item + label: "{{ item.name }}" + - name: Sync container remotes into repositories - pulp.squeezer.container_sync: + stackhpc.pulp.container_syncs: pulp_url: "{{ pulp_url }}" username: "{{ pulp_username }}" password: "{{ pulp_password }}" validate_certs: "{{ pulp_validate_certs | bool }}" - repository: "{{ pulp_repository_container_repos[repository_index].name }}" - remote: "{{ pulp_repository_container_repos[repository_index].name }}-remote" - when: - - pulp_repository_container_repos[repository_index].url is defined - - pulp_repository_container_repos[repository_index].state == "present" - loop: "{{ pulp_repository_container_repos | map(attribute='name') }}" - loop_control: - index_var: repository_index + syncs: "{{ container_syncs_list }}" + concurrency: "{{ pulp_repository_concurrency }}" register: pulp_repository_container_repos_sync - until: "pulp_repository_container_repos_sync is not failed" + until: "not ((pulp_repository_container_repos_sync | default({})).get('syncs', []) | selectattr('failed') | list)" retries: "{{ pulp_repository_container_repos_sync_retries }}" delay: 1 diff --git a/roles/pulp_repository/tasks/main.yml b/roles/pulp_repository/tasks/main.yml index 1302c9c..2d14ced 100644 --- a/roles/pulp_repository/tasks/main.yml +++ b/roles/pulp_repository/tasks/main.yml @@ -1,7 +1,20 @@ --- +- include_tasks: prereqs.yml + when: > + (pulp_repository_container_repos | length > 0) or + (pulp_repository_deb_repos | length > 0) or + (pulp_repository_python_repos | length > 0) or + (pulp_repository_rpm_repos | length > 0) + tags: + - prereqs + - container + - deb + - python + - rpm + - include_tasks: container.yml - tags: container when: pulp_repository_container_repos | length > 0 + tags: container - include_tasks: deb.yml when: pulp_repository_deb_repos | length > 0 @@ -12,5 +25,5 @@ tags: python - include_tasks: rpm.yml - tags: rpm when: pulp_repository_rpm_repos | length > 0 + tags: rpm diff --git a/roles/pulp_repository/tasks/prereqs.yml b/roles/pulp_repository/tasks/prereqs.yml new file mode 100644 index 0000000..fbc24a7 --- /dev/null +++ b/roles/pulp_repository/tasks/prereqs.yml @@ -0,0 +1,7 @@ +--- +- name: Ensure dependencies are installed + ansible.builtin.pip: + name: + - "pulp-glue==0.33.*" + - "pulp-glue-deb==0.3.*" + state: present diff --git a/tests/sanity/ignore-2.18.txt b/tests/sanity/ignore-2.18.txt new file mode 100644 index 0000000..2e8a488 --- /dev/null +++ b/tests/sanity/ignore-2.18.txt @@ -0,0 +1,4 @@ +plugins/modules/container_repositories.py validate-modules:missing-gplv3-license +plugins/modules/container_remotes.py validate-modules:missing-gplv3-license +plugins/modules/container_distributions.py validate-modules:missing-gplv3-license +plugins/modules/container_syncs.py validate-modules:missing-gplv3-license \ No newline at end of file diff --git a/tests/sanity/ignore-2.20.txt b/tests/sanity/ignore-2.20.txt new file mode 100644 index 0000000..2e8a488 --- /dev/null +++ b/tests/sanity/ignore-2.20.txt @@ -0,0 +1,4 @@ +plugins/modules/container_repositories.py validate-modules:missing-gplv3-license +plugins/modules/container_remotes.py validate-modules:missing-gplv3-license +plugins/modules/container_distributions.py validate-modules:missing-gplv3-license +plugins/modules/container_syncs.py validate-modules:missing-gplv3-license \ No newline at end of file diff --git a/tests/test_container_distributions.yml b/tests/test_container_distributions.yml new file mode 100644 index 0000000..245e18c --- /dev/null +++ b/tests/test_container_distributions.yml @@ -0,0 +1,142 @@ +--- +- name: Test container distributions module + gather_facts: false + hosts: localhost + vars: + pulp_url: http://localhost:8080 + pulp_username: admin + pulp_password: password + pulp_validate_certs: true + tasks: + - name: Create test repository + pulp.squeezer.container_repository: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + name: test_dist_repo + state: present + + - name: Create test content guard + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + operation_id: contentguards_core_content_redirect_create + body: + name: test_dist_redirect_guard + register: test_content_guard + + - name: Create test container distributions + stackhpc.pulp.container_distributions: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + distributions: + - name: test_dist_1 + base_path: test_dist_1 + repository: test_dist_repo + state: present + - name: test_dist_2 + base_path: test_dist_2 + repository: test_dist_repo + private: true + state: present + - name: test_dist_guarded + base_path: test_dist_guarded + repository: test_dist_repo + content_guard: test_dist_redirect_guard + state: present + register: create_result + + - name: Verify distribution creation + assert: + that: + - create_result.distributions | length == 3 + - create_result.distributions[0].name == "test_dist_1" + - create_result.distributions[0].distribution.base_path == "test_dist_1" + - create_result.distributions[0].changed == true + - create_result.distributions[1].name == "test_dist_2" + - create_result.distributions[1].distribution.private == true + - create_result.distributions[2].name == "test_dist_guarded" + - create_result.distributions[2].distribution.content_guard is search("/contentguards/") + + - name: Update container distributions + stackhpc.pulp.container_distributions: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + distributions: + - name: test_dist_1 + base_path: test_dist_1_updated + repository: test_dist_repo + state: present + register: update_result + + - name: Verify update + assert: + that: + - update_result.distributions[0].changed == true + - update_result.distributions[0].distribution.base_path == "test_dist_1_updated" + + - name: Update container distributions again + stackhpc.pulp.container_distributions: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + distributions: + - name: test_dist_1 + base_path: test_dist_1_updated + repository: test_dist_repo + state: present + register: idempotence_result + + - name: Verify idempotence + assert: + that: + - idempotence_result.distributions[0].changed == false + + - name: Delete test container distributions + stackhpc.pulp.container_distributions: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + distributions: + - name: test_dist_1 + state: absent + - name: test_dist_2 + state: absent + - name: test_dist_guarded + state: absent + register: delete_result + + - name: Verify distribution deletion + assert: + that: + - delete_result.distributions[0].changed == true + - delete_result.distributions[1].changed == true + - delete_result.distributions[2].changed == true + + - name: Clean up test content guard + pulp.squeezer.api_call: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + operation_id: contentguards_core_content_redirect_delete + parameters: + content_redirect_content_guard_href: "{{ test_content_guard.response.pulp_href }}" + + - name: Clean up test repository + pulp.squeezer.container_repository: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + name: test_dist_repo + state: absent diff --git a/tests/test_container_remotes.yml b/tests/test_container_remotes.yml new file mode 100644 index 0000000..5bd6698 --- /dev/null +++ b/tests/test_container_remotes.yml @@ -0,0 +1,117 @@ +--- +- name: Test container remotes module + gather_facts: false + hosts: localhost + vars: + pulp_url: http://localhost:8080 + pulp_username: admin + pulp_password: password + pulp_validate_certs: true + tasks: + - name: Create test container remotes + stackhpc.pulp.container_remotes: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + remotes: + - name: test_container_remote_1 + upstream_name: pulp/test-fixture-1 + url: "https://registry-1.docker.io" + policy: immediate + state: present + - name: test_container_remote_2 + upstream_name: pulp/test-fixture-1 + url: "https://registry-1.docker.io" + policy: on_demand + state: present + register: create_result + + - name: Verify remote creation + assert: + that: + - create_result.remotes | length == 2 + - create_result.remotes[0].name == "test_container_remote_1" + - create_result.remotes[0].remote.upstream_name == "pulp/test-fixture-1" + - create_result.remotes[0].remote.url == "https://registry-1.docker.io" + - create_result.remotes[0].remote.policy == "immediate" + - create_result.remotes[1].name == "test_container_remote_2" + - create_result.remotes[1].remote.upstream_name == "pulp/test-fixture-1" + - create_result.remotes[1].remote.policy == "on_demand" + + - name: Query remotes + stackhpc.pulp.container_remotes: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + remotes: + - name: test_container_remote_1 + - name: test_container_remote_2 + register: query_result + + - name: Verify query results + assert: + that: + - query_result.remotes | length == 2 + - query_result.remotes[0].name == "test_container_remote_1" + - query_result.remotes[1].name == "test_container_remote_2" + + - name: Update remote + stackhpc.pulp.container_remotes: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + remotes: + - name: test_container_remote_1 + policy: on_demand + include_tags: ["latest"] + exclude_tags: ["beta"] + register: update_result + + - name: Verify update + assert: + that: + - update_result.remotes[0].changed == true + - update_result.remotes[0].remote.policy == "on_demand" + - update_result.remotes[0].remote.include_tags == ["latest"] + - update_result.remotes[0].remote.exclude_tags == ["beta"] + + - name: Update remote again (no change expected) + stackhpc.pulp.container_remotes: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + remotes: + - name: test_container_remote_1 + policy: on_demand + include_tags: ["latest"] + exclude_tags: ["beta"] + register: update_again_result + + - name: Verify idempotence + assert: + that: + - update_again_result.remotes[0].changed == false + + - name: Delete test container remotes + stackhpc.pulp.container_remotes: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + remotes: + - name: test_container_remote_1 + state: absent + - name: test_container_remote_2 + state: absent + register: delete_result + + - name: Verify remote deletion + assert: + that: + - delete_result.remotes | length == 2 + - delete_result.remotes[0].failed == false + - delete_result.remotes[1].failed == false diff --git a/tests/test_container_repositories.yml b/tests/test_container_repositories.yml new file mode 100644 index 0000000..162b606 --- /dev/null +++ b/tests/test_container_repositories.yml @@ -0,0 +1,102 @@ +--- +- name: Test container repositories module + gather_facts: false + hosts: localhost + vars: + pulp_url: http://localhost:8080 + pulp_username: admin + pulp_password: password + pulp_validate_certs: true + tasks: + - name: Create test container repositories + stackhpc.pulp.container_repositories: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + repositories: + - name: test_container_repo_1 + description: Test repository 1 + state: present + - name: test_container_repo_2 + description: Test repository 2 + state: present + register: create_result + + - name: Verify repository creation + assert: + that: + - create_result.repositories | length == 2 + - create_result.repositories[0].name == "test_container_repo_1" + - create_result.repositories[0].repository.description == "Test repository 1" + - create_result.repositories[1].name == "test_container_repo_2" + - create_result.repositories[1].repository.description == "Test repository 2" + + - name: Query repositories + stackhpc.pulp.container_repositories: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + repositories: + - name: test_container_repo_1 + - name: test_container_repo_2 + register: query_result + + - name: Verify query results + assert: + that: + - query_result.repositories | length == 2 + + - name: Update repository description + stackhpc.pulp.container_repositories: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + repositories: + - name: test_container_repo_1 + description: "Updated description" + register: update_result + + - name: Verify update + assert: + that: + - update_result.repositories[0].changed == true + - update_result.repositories[0].repository.description == "Updated description" + + - name: Update repository again + stackhpc.pulp.container_repositories: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + repositories: + - name: test_container_repo_1 + description: "Updated description" + register: update_again_result + + - name: Verify idempotence + assert: + that: + - update_again_result.repositories[0].changed == false + + - name: Delete test container repositories + stackhpc.pulp.container_repositories: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + repositories: + - name: test_container_repo_1 + state: absent + - name: test_container_repo_2 + state: absent + register: delete_result + + - name: Verify repository deletion + assert: + that: + - delete_result.repositories | length == 2 + - delete_result.repositories[0].failed == false + - delete_result.repositories[1].failed == false diff --git a/tests/test_container_repository.yml b/tests/test_container_repository.yml index f6d7aba..743897c 100644 --- a/tests/test_container_repository.yml +++ b/tests/test_container_repository.yml @@ -105,7 +105,7 @@ - name: Assert that syncing from a URL that returns 404 is retried the correct number of times assert: that: - - failed_result.results[0].attempts == pulp_repository_container_repos_sync_retries + - failed_result.attempts == pulp_repository_container_repos_sync_retries - include_role: name: pulp_repository diff --git a/tests/test_container_syncs.yml b/tests/test_container_syncs.yml new file mode 100644 index 0000000..511d791 --- /dev/null +++ b/tests/test_container_syncs.yml @@ -0,0 +1,132 @@ +--- +- name: Test container syncs module + gather_facts: false + hosts: localhost + vars: + pulp_url: http://localhost:8080 + pulp_username: admin + pulp_password: password + pulp_validate_certs: true + tasks: + - name: Create test repository for sync + pulp.squeezer.container_repository: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + name: "{{ item.name }}" + state: "{{ item.state }}" + loop: + - name: test_sync_repo1 + state: present + - name: test_sync_repo2 + state: present + + - name: Create test remote for sync + pulp.squeezer.container_remote: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + name: "{{ item.name }}" + upstream_name: "{{ item.upstream_name }}" + url: "{{ item.url }}" + policy: "{{ item.policy }}" + state: "{{ item.state }}" + loop: + - name: test_sync_remote1 + upstream_name: pulp/test-fixture-1 + url: "https://registry-1.docker.io" + policy: immediate + state: present + - name: test_sync_remote2 + upstream_name: pulp/test-fixture-1 + url: "https://registry-1.docker.io" + policy: on_demand + state: present + + - name: Sync container repositories + stackhpc.pulp.container_syncs: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + syncs: + - repository: test_sync_repo1 + remote: test_sync_remote1 + - repository: test_sync_repo2 + remote: test_sync_remote2 + register: sync_result + + - name: Verify sync result + assert: + that: + - sync_result.syncs | length == 2 + - sync_result.syncs[0].repository == "test_sync_repo1" + - sync_result.syncs[0].failed == false + - sync_result.syncs[1].repository == "test_sync_repo2" + - sync_result.syncs[1].failed == false + + - name: Sync again (should not change) + stackhpc.pulp.container_syncs: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + syncs: + - repository: test_sync_repo1 + remote: test_sync_remote1 + - repository: test_sync_repo2 + remote: test_sync_remote2 + register: sync_again_result + + - name: Verify no change on second sync + assert: + that: + - sync_again_result.syncs[0].changed == false + - sync_again_result.syncs[0].failed == false + - sync_again_result.syncs[1].changed == false + - sync_again_result.syncs[1].failed == false + + - name: Test sync failure (non-existent remote) + stackhpc.pulp.container_syncs: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + syncs: + - repository: test_sync_repo1 + remote: non_existent_remote + register: failed_sync_result + ignore_errors: true + + - name: Verify failure reporting + assert: + that: + - failed_sync_result.msg is search("One or more items failed") + - failed_sync_result.syncs[0].failed == true + - failed_sync_result.syncs[0].msg is search("Could not find container remote with") + + - name: Clean up test remote + pulp.squeezer.container_remote: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + name: "{{ item }}" + state: absent + loop: + - test_sync_remote1 + - test_sync_remote2 + + - name: Clean up test repository + pulp.squeezer.container_repository: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + validate_certs: "{{ pulp_validate_certs }}" + name: "{{ item }}" + state: absent + loop: + - test_sync_repo1 + - test_sync_repo2