From 7d24c4bd3bd133a004355ee4edb97eeebd608a7a Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Wed, 10 Dec 2025 13:31:27 +0000 Subject: [PATCH 01/61] [major] Clean up old backup and restore code and implement Mongo backup and restore (#2011) Co-authored-by: Sanjay Prabhakar --- build/bin/copy-role-docs.sh | 1 - docs/playbooks/backup-restore.md | 599 ------------------ .../backup_restore/after_run_tasks.yml | 54 -- .../backup_restore/before_run_tasks.yml | 33 - .../backup_restore/check_backup_vars.yml | 196 ------ .../backup_restore/check_common_vars.yml | 41 -- .../backup_restore/check_restore_vars.yml | 208 ------ .../backup_restore/confirm_cluster_info.yml | 32 - .../copy_local_files_to_storage.yml | 24 - .../copy_pod_files_to_storage.yml | 53 -- .../copy_storage_files_to_local.yml | 24 - .../copy_storage_files_to_pod.yml | 76 --- .../backup_restore/create_cleanup_job.yml | 63 -- .../create_local_job_folder.yml | 21 - .../delete_storage_job_folder.yml | 15 - .../list_storage_job_folders.yml | 27 - .../rename_storage_job_folder.yml | 23 - .../backup_restore/restart_and_reconsiled.yml | 23 - .../backup_restore/update_job_status.yml | 156 ----- .../common_tasks/get_version_from_channel.yml | 1 + .../templates/backup_restore/backup.yml.j2 | 45 -- .../backup_restore/cleanup_job.sh.j2 | 64 -- .../backup_restore/cleanup_job.yml.j2 | 83 --- .../copy_cloud_files_job.yml.j2 | 95 --- .../templates/backup_restore/restore.yml.j2 | 36 -- ibm/mas_devops/common_vars/backup_restore.yml | 53 -- .../common_vars/cp4d_supported_versions.yml | 6 +- ibm/mas_devops/meta/runtime.yml | 10 + ibm/mas_devops/playbooks/br_core.yml | 76 --- ibm/mas_devops/playbooks/br_db2.yml | 80 --- ibm/mas_devops/playbooks/br_health.yml | 107 ---- ibm/mas_devops/playbooks/br_iot.yml | 101 --- ibm/mas_devops/playbooks/br_manage.yml | 97 --- ibm/mas_devops/playbooks/br_mongodb.yml | 74 --- ibm/mas_devops/playbooks/br_monitor.yml | 107 ---- ibm/mas_devops/playbooks/br_optimizer.yml | 107 ---- .../playbooks/br_visualinspection.yml | 88 --- .../playbooks/mirror_dependencies.yml | 24 + .../plugins/action/backup_mongo_instance.py | 365 +++++++++++ .../plugins/action/get_mongoce_info.py | 209 ++++++ .../action/get_mongodb_cr_to_restore.py | 75 +++ .../action/restore_mongoce_resources.py | 103 +++ .../action/verify_backup_restore_vars.py | 43 ++ .../plugins/action/verify_mongoce_version.py | 179 ++++++ .../plugins/action/verify_storage_class.py | 51 ++ .../plugins/action/wait_for_app_ready.py | 66 ++ ibm/mas_devops/plugins/filter/filters.py | 24 +- .../roles/aws_vpc/tasks/deprovision.yml | 4 +- ibm/mas_devops/roles/cp4d/README.md | 15 + .../cp4d/templates/catalog_sources/5.2.0.yml | 22 + .../config_maps/olm-utils-cm-5.2.0.yml.j2 | 575 +++++++++++++++++ ibm/mas_devops/roles/cp4d_service/README.md | 47 +- .../cp4d_service/tasks/wait/wait-ccs.yml | 9 +- .../tasks/wait/wait-opensearch.yml | 79 +++ ibm/mas_devops/roles/db2/README.md | 110 ---- .../roles/db2/tasks/after-backup-restore.yml | 11 - .../db2/tasks/backup/backup-database.yml | 163 ----- .../roles/db2/tasks/backup/main.yml | 59 -- .../roles/db2/tasks/before-backup-restore.yml | 147 ----- .../tasks/restore/copy-db2-backup-file.yml | 107 ---- .../roles/db2/tasks/restore/main.yml | 60 +- .../db2/tasks/restore/restore-database.yml | 291 --------- .../roles/db2/templates/tlsroute.yml.j2 | 2 +- .../kafka/tasks/provider/aws/uninstall.yml | 4 +- .../aws/utils/create-security-group.yml | 4 +- .../provider/aws/utils/create-subnet.yml | 2 +- .../roles/mongodb/defaults/main.yml | 8 + .../mongodb/tasks/providers/aws/install.yml | 8 +- .../backup-restore/after-backup-restore.yml | 11 - .../backup-restore/backup-database.yml | 441 +++---------- .../backup-restore/before-backup-restore.yml | 159 ----- .../backup-restore/create-role-user.sh.j2 | 69 ++ .../backup-restore/create-role-user.yml | 67 -- .../backup-restore/database-backup.sh.j2 | 81 +++ .../backup-restore/database-restore.sh.j2 | 73 +++ .../backup-restore/get-mongo-info.yml | 29 +- .../backup-restore/restore-database-patch.yml | 76 --- .../restore-database-perform.yml | 121 ---- .../backup-restore/restore-database.yml | 242 +++---- .../tasks/providers/community/backup.yml | 109 ++-- .../community/check-mongo-exists.yml | 2 +- .../community/controlled-upgrade.yml | 2 +- .../providers/community/install-mongo.yml | 36 +- .../tasks/providers/community/restore.yml | 194 ++++-- .../tasks/providers/community/uninstall.yml | 6 +- .../templates/community/0.12.0/cr.yml.j2 | 28 +- .../templates/community/0.7.9/cr.yml.j2 | 12 +- .../templates/community/0.8.3/cr.yml.j2 | 28 +- .../templates/community/0.9.0/cr.yml.j2 | 28 +- .../templates/community/admin-password.yml | 2 +- .../mongodb/templates/community/ca-cert.yml | 2 +- .../community/metrics-endpoint-secret.yml.j2 | 2 +- .../templates/community/mongo-hosts.yml.j2 | 2 +- .../templates/community/server-cert.yml | 2 +- .../templates/community/servicemonitor.yml.j2 | 8 +- .../roles/mongodb/templates/community/tls.yml | 2 +- .../tasks/providers/fyre/provision_fyre.yml | 4 +- .../roles/ocs/tasks/upgrade/main.yml | 2 +- .../roles/suite_app_backup/README.md | 98 +++ .../roles/suite_app_backup/defaults/main.yml | 3 + .../roles/suite_app_backup/tasks/main.yml | 18 + .../tasks/manage/backup-vars.yml | 1 + .../tasks/manage/pv-info.yml | 0 .../tasks/manage/restore-namespace.yml | 0 .../roles/suite_app_backup_restore/README.md | 223 ------- .../defaults/main.yml | 20 - .../tasks/backup-namespace.yml | 113 ---- .../tasks/backup-pv.yml | 85 --- .../tasks/get-app-info.yml | 114 ---- .../tasks/health/backup-vars.yml | 34 - .../tasks/health/backup-wsl.yml | 97 --- .../tasks/health/get-app-info.yml | 21 - .../tasks/health/get-wsl-info.yml | 80 --- .../tasks/health/restore-vars.yml | 8 - .../tasks/health/restore-wsl.yml | 127 ---- .../tasks/iot/backup-vars.yml | 29 - .../tasks/iot/restore-namespace.yml | 50 -- .../tasks/iot/restore-vars.yml | 6 - .../suite_app_backup_restore/tasks/main.yml | 97 --- .../tasks/manage/backup-vars.yml | 27 - .../tasks/manage/restore-vars.yml | 8 - .../tasks/monitor/backup-vars.yml | 42 -- .../tasks/monitor/restore-namespace.yml | 88 --- .../tasks/monitor/restore-vars.yml | 6 - .../tasks/optimizer/backup-vars.yml | 19 - .../tasks/optimizer/restore-vars.yml | 6 - .../tasks/restore-namespace.yml | 109 ---- .../tasks/restore-pv.yml | 85 --- .../tasks/visualinspection/backup-vars.yml | 27 - .../tasks/visualinspection/pv-info.yml | 41 -- .../tasks/visualinspection/restore-vars.yml | 8 - .../roles/suite_app_config/tasks/main.yml | 53 +- .../tasks/manage/post-config/main.yml | 2 +- .../roles/suite_app_install/tasks/main.yml | 24 +- .../roles/suite_app_verify/README.md | 35 - .../roles/suite_app_verify/defaults/main.yml | 11 - .../roles/suite_app_verify/meta/main.yml | 22 - .../roles/suite_app_verify/tasks/main.yml | 43 -- ibm/mas_devops/roles/suite_backup/README.md | 13 + .../roles/suite_backup/defaults/main.yml | 2 + .../roles/suite_backup/tasks/main.yml | 12 + .../roles/suite_backup_restore/README.md | 113 ---- .../suite_backup_restore/defaults/main.yml | 6 - .../tasks/backup-namespace.yml | 113 ---- .../tasks/backup-vars.yml | 40 -- .../tasks/get-suite-info.yml | 45 -- .../roles/suite_backup_restore/tasks/main.yml | 78 --- .../tasks/restore-namespace.yml | 120 ---- .../tasks/restore-vars.yml | 6 - mkdocs.yml | 3 - 150 files changed, 2845 insertions(+), 7191 deletions(-) delete mode 100644 docs/playbooks/backup-restore.md delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/create_local_job_folder.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/restart_and_reconsiled.yml delete mode 100644 ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml delete mode 100644 ibm/mas_devops/common_tasks/templates/backup_restore/backup.yml.j2 delete mode 100644 ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.sh.j2 delete mode 100644 ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.yml.j2 delete mode 100644 ibm/mas_devops/common_tasks/templates/backup_restore/copy_cloud_files_job.yml.j2 delete mode 100644 ibm/mas_devops/common_tasks/templates/backup_restore/restore.yml.j2 delete mode 100644 ibm/mas_devops/common_vars/backup_restore.yml delete mode 100644 ibm/mas_devops/playbooks/br_core.yml delete mode 100644 ibm/mas_devops/playbooks/br_db2.yml delete mode 100644 ibm/mas_devops/playbooks/br_health.yml delete mode 100644 ibm/mas_devops/playbooks/br_iot.yml delete mode 100644 ibm/mas_devops/playbooks/br_manage.yml delete mode 100644 ibm/mas_devops/playbooks/br_mongodb.yml delete mode 100644 ibm/mas_devops/playbooks/br_monitor.yml delete mode 100644 ibm/mas_devops/playbooks/br_optimizer.yml delete mode 100644 ibm/mas_devops/playbooks/br_visualinspection.yml create mode 100644 ibm/mas_devops/plugins/action/backup_mongo_instance.py create mode 100644 ibm/mas_devops/plugins/action/get_mongoce_info.py create mode 100644 ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py create mode 100644 ibm/mas_devops/plugins/action/restore_mongoce_resources.py create mode 100644 ibm/mas_devops/plugins/action/verify_backup_restore_vars.py create mode 100644 ibm/mas_devops/plugins/action/verify_mongoce_version.py create mode 100644 ibm/mas_devops/plugins/action/verify_storage_class.py create mode 100644 ibm/mas_devops/plugins/action/wait_for_app_ready.py create mode 100644 ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.2.0.yml create mode 100644 ibm/mas_devops/roles/cp4d/templates/config_maps/olm-utils-cm-5.2.0.yml.j2 create mode 100644 ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-opensearch.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/after-backup-restore.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/after-backup-restore.yml delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.sh.j2 delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml create mode 100644 ibm/mas_devops/roles/suite_app_backup/README.md create mode 100644 ibm/mas_devops/roles/suite_app_backup/defaults/main.yml create mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/main.yml create mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml rename ibm/mas_devops/roles/{suite_app_backup_restore => suite_app_backup}/tasks/manage/pv-info.yml (100%) rename ibm/mas_devops/roles/{suite_app_backup_restore => suite_app_backup}/tasks/manage/restore-namespace.yml (100%) delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/README.md delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/defaults/main.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-namespace.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-pv.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-wsl.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-app-info.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-wsl-info.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-wsl.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-namespace.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-namespace.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/restore-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/pv-info.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/restore-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_verify/README.md delete mode 100644 ibm/mas_devops/roles/suite_app_verify/defaults/main.yml delete mode 100644 ibm/mas_devops/roles/suite_app_verify/meta/main.yml delete mode 100644 ibm/mas_devops/roles/suite_app_verify/tasks/main.yml create mode 100644 ibm/mas_devops/roles/suite_backup/README.md create mode 100644 ibm/mas_devops/roles/suite_backup/defaults/main.yml create mode 100644 ibm/mas_devops/roles/suite_backup/tasks/main.yml delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/README.md delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/defaults/main.yml delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/tasks/backup-namespace.yml delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/tasks/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/tasks/get-suite-info.yml delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml delete mode 100644 ibm/mas_devops/roles/suite_backup_restore/tasks/restore-vars.yml diff --git a/build/bin/copy-role-docs.sh b/build/bin/copy-role-docs.sh index 6b6365578c..2965bdd87b 100644 --- a/build/bin/copy-role-docs.sh +++ b/build/bin/copy-role-docs.sh @@ -73,7 +73,6 @@ copyDoc suite_app_install copyDoc suite_app_uninstall copyDoc suite_app_upgrade copyDoc suite_app_rollback -copyDoc suite_app_verify copyDoc suite_backup_restore copyDoc suite_config copyDoc suite_db2_setup_for_manage diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md deleted file mode 100644 index 1a517befec..0000000000 --- a/docs/playbooks/backup-restore.md +++ /dev/null @@ -1,599 +0,0 @@ -Backup and Restore -=============================================================================== - -Overview -------------------------------------------------------------------------------- -MAS Devops Collection includes playbooks for backing up and restoring of the following MAS components and their dependencies: - -- [MongoDB](#backuprestore-for-mongodb) -- [Db2](#backuprestore-for-db2) -- [MAS Core](#backuprestore-for-mas-core) -- [Manage](#backuprestore-for-manage) -- [IoT](#backuprestore-for-iot) -- [Monitor](#backuprestore-for-monitor) -- [Health](#backuprestore-for-health) -- [Optimizer](#backuprestore-for-optimizer) -- [Visual Inspection](#backuprestore-for-visual-inspection) - - -Creation of both **full** and **incremental** backups are supported. The backup and restore Ansible roles can also be used individually, allowing you to build your own customized backup and restore playbook covering exactly what you need. For example, you can only [backup/restore Manage attachments](../roles/suite_app_backup_restore.md). - -!!! important - The backup and restore playbooks in this collection are still work in progress, they are not suitable for production use at this time. You may track development progress using the [Backup & Restore](https://github.com/ibm-mas/ansible-devops/issues?q=label%3A%22Backup+%26+Restore%22+) label in the Github repository. - - Production-ready backup and restore options are detailed in the [Backup and restore](https://www.ibm.com/docs/en/mas-cd/continuous-delivery?topic=administering-backing-up-restoring-maximo-application-suite) topic in the product documentation. - -Configuration - Storage -------------------------------------------------------------------------------- -You can save the backup files to a folder on your local file system by setting the following environment variables: - -| Envrionment variable | Required (Default Value) | Description | -| ------------------------------------ | -------------------------- | ----------- | -| MASBR_STORAGE_LOCAL_FOLDER | **Yes** | The local path to save the backup files | -| MASBR_LOCAL_TEMP_FOLDER | No (`/tmp/masbr`) | Local folder for saving the temporary backup/restore data, the data in this folder will be deleted after the backup/restore job completed. | - - -Configuration - Backup -------------------------------------------------------------------------------- - -| Envrionment variable | Required (Default Value) | Description | -| ------------------------------------ | ------------------------ | ----------- | -| MASBR_ACTION | **Yes** | Whether to run the playbook to perform a `backup` or a `restore` | -| MASBR_BACKUP_TYPE | No (`full`) | Set `full` or `incr` to indicate the playbook to create a **full** backup or **incremental** backup. | -| MASBR_BACKUP_FROM_VERSION | No | Set the full backup version to use in the incremental backup, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`). | - -The playbooks are switched to backup mode by setting `MASBR_ACTION` to `backup`. - -### Full Backups -If you set environment variable `MASBR_BACKUP_TYPE=full` or do not specify a value for this variable, the playbook will take a full backup. - -### Incremental Backups -You can set environment variable `MASBR_BACKUP_TYPE=incr` to indicate the playbook to take an incremental backup. - -!!! important - Only supports creating incremental backup for MonogDB, Db2 and persistent volume data. The playbook will always create a full backup for other type of data regardless of whether this variable be set to `incr`. - -The environment variable `MASBR_BACKUP_FROM_VERSION` is only valid if `MASBR_BACKUP_TYPE=incr`. It indicates which backup version that the incremental backup to based on. If you do not set a value for this variable, the playbook will try to find the latest Completed Full backup from the specified storage location, and then take an incremental backup based on it. - -!!! important - The backup files you specified by `MASBR_BACKUP_FROM_VERSION` must be a Full backup. And the component name and data types in the specified Full backup file must be same as the current incremental backup job. - - -Configuration - Restore -------------------------------------------------------------------------------- - -| Envrionment variable | Required (Default Value) | Description | -| ------------------------------------ | ------------------------ | ----------- | -| MASBR_ACTION | **Yes** | Whether to run the playbook to perform a `backup` or a `restore` | -| MASBR_RESTORE_FROM_VERSION | **Yes** | Set the backup version to use in the restore, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`) | -| MASBR_RESTORE_OVERWRITE | **Yes** | Set whether the restore should **overwrite** any existing data or if we should stop and **FAIL** if there is data detected in the directory. **WARNING:** This will overwrite all data when restoring! | - -The playbooks are switched to restore mode by setting `MASBR_ACTION` to `restore`. You **must** specify the `MASBR_RESTORE_FROM_VERSION` environment variable to indicate which version of the backup files to use. - -In the case of restoring from an incremental backup, the corresponding full backup will be restored first before continuing to restore the incremental backup. - - -Backup/Restore for MongoDB -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_mongodb` will invoke the role [mongodb](../roles/mongodb.md) to backup/restore the MongoDB databases. - -This playbook supports backing up and restoring databases for an in-cluster MongoDB CE instance. If you are using other MongoDB venders, such as IBM Cloud Databases for MongoDB, Amazon DocumentDB or MongoDB Altas Database, please refer to the corresponding vender's documentation for more information about their provided backup/restore service. - -### Environment Variables -- `MONGODB_NAMESPACE`: By default the backup and restore processes will use a namespace of `mongoce`, if you have customized the install of MongoDb CE you must set this environment variable to the appropriate namespace you wish to backup from/restore to. -- `MAS_INSTANCE_ID`: **Required**. This playbook supports backup/restore MongoDB databases that belong to a specific MAS instance, call the playbook multiple times with different values for `MAS_INSTANCE_ID` if you wish to back up multiple MAS instances that use the same MongoDB CE instance. -- `MAS_APP_ID`: **Optional**. By default, this playbook will backup all databases belonging to the specified MAS instance. You can backup the databases only belong to a specific MAS application by setting this environment variable to a supported MAS application id `core`, `manage`, `iot`, `monitor`, `health`, `optimizer` or `visualinspection`. - -### Examples -```bash -# Full backup all MongoDB data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_mongodb - -# Incremental backup all MongoDB data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_mongodb - -# Restore all MongoDB data for the dev1 instance -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_mongodb - -# Backup just the IoT MongoDB data for the dev2 instance -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev2 -export MAS_APP_ID=iot -ansible-playbook ibm.mas_devops.br_mongodb -``` - - -Backup/Restore for Db2 -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_db2` will invoke the role [db2](../roles/db2.md) to backup/restore a single Db2 instance. - -### Environment Variables - -- `DB2_INSTANCE_NAME`: **Required** This playbook only supports backing up specific Db2 instance at a time. If you want to backup all Db2 instances in the Db2 cluster, you need to run this playbook multiple times with different value of this environment variable. -- `MAS_INSTANCE_ID`: **Required** Set the instance ID for the MAS install. -- `MASBR_ACTION`: **Required** Set the action to be performed, `backup` or `restore`. -- `MASBR_STORAGE_LOCAL_FOLDER`: **Required** Set the local path to the directory to be used for backup and restore. -- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` - -### Examples -```bash -# Full backup for the db2w-shared Db2 instance -export MAS_INSTANCE_ID=dev -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_db2 - -# Incremental backup for the db2w-shared Db2 instance -export MAS_INSTANCE_ID=dev -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_db2 - -# Restore for the db2w-shared Db2 instance -export MAS_INSTANCE_ID=dev -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_db2 -``` - -Backup/Restore for MAS Core -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_core` will backup the following components that MAS Core depends on in order: - -| Component | Ansible Role | Data included | -| --------- | -------------------------------------------------------- | ---------------------------------- | -| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core | -| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | - - -### Environment Variables - -- `MAS_INSTANCE_ID` **Required**. The MAS instance ID to perform a backup for. - -### Examples -```bash -# Full backup all core data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_core - -# Incremental backup all core data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_core - -# Restore all core data for the dev1 instance -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_core -``` - - -Backup/Restore for Manage -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_manage` will backup the following components that Manage depends on in order: - -| Component | Role | Data included | -| --------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------ | -| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core | -| db2 | [db2](../roles/db2.md) | Db2 instance used by Manage | -| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | -| manage | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Manage namespace resources
Persistent volume data, such as attachments | - - -### Environment Variables - -- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `DB2_INSTANCE_NAME` **Optional**. When defined, this playbook will backup the Db2 instance used by Manage. DB2 role is skipped when environment variable is not defined.. -- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` - -### Examples - -```bash -# Full backup all manage data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role -ansible-playbook ibm.mas_devops.br_manage - -# Incremental backup all manage data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role -ansible-playbook ibm.mas_devops.br_manage - -# Restore all manage data for the dev1 instance and ws1 workspace -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role -ansible-playbook ibm.mas_devops.br_manage -``` - - -Backup/Restore for IoT -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_iot` will backup the following components that IoT depends on in order: - -| Component | Ansible Role | Data included | -| --------- | ---------------------------------------------------------------- | ------------------------------------------ | -| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core and IoT | -| db2 | [db2](../roles/db2.md) | Db2 instance used by IoT | -| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | -| iot | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | IoT namespace resources | - -### Environment Variables - -- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by IoT, you need to set the correct Db2 instance name for this environment variable. -- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` - -### Examples - -```bash -# Full backup all iot data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_iot - -# Incremental backup all iot data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_iot - -# Restore all iot data for the dev1 instance -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_iot - -``` - - -Backup/Restore for Monitor -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_monitor` will backup the following components that Monitor depends on in order: - -| Component | Ansible Role | Data included | -| --------- | ---------------------------------------------------------------- | --------------------------------------------------- | -| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core, IoT and Monitor | -| db2 | [db2](../roles/db2.md) | Db2 instance used by IoT and Monitor | -| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | -| iot | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | IoT namespace resources | -| monitor | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Monitor namespace resources | - - -### Environment Variables - -- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by IoT and Monitor, you need to set the correct Db2 instance name for this environment variable. -- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` - -### Examples - -```bash -# Full backup all monitor data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_monitor - -# Incremental backup all monitor data for the dev1 instance -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_monitor - -# Restore all monitor data for the dev1 instance -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=db2w-shared -ansible-playbook ibm.mas_devops.br_monitor -``` - - -Backup/Restore for Health -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_health` will backup the following components that Health depends on in order: - -| Component | Ansible Role | Data included | -| --------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------ | -| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core | -| db2 | [db2](../roles/db2.md) | Db2 instance used by Manage and Health | -| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | -| manage | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Manage namespace resources
Persistent volume data, such as attachments | -| health | [suite_backup_restore](../roles/suite_backup_restore.md) | Health namespace resources
Watson Studio project assets | - -### Environment Variables - -- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by Manage and Health, you need to set the correct Db2 instance name for this environment variable. - -### Examples - -```bash -# Full backup all health data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_health - -# Incremental backup all health data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_health - -# Restore all health data for the dev1 instance and ws1 workspace -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_health -``` - - -Backup/Restore for Optimizer -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_optimizer` will backup the following components that Optimizer depends on in order: - -| Component | Ansible Role | Data included | -| --------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------ | -| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core and Optimizer | -| db2 | [db2](../roles/db2.md) | Db2 instance used by Manage | -| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | -| manage | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Manage namespace resources
Persistent volume data, such as attachments | -| optimizer | [suite_backup_restore](../roles/suite_backup_restore.md) | Optimizer namespace resources | - -### Environment Variables - -- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by Manage, you need to set the correct Db2 instance name for this environment variable. -- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` - -### Examples - -```bash -# Full backup all optimizer data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_optimizer - -# Incremental backup all optimizer data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_optimizer - -# Restore all optimizer data for the dev1 instance and ws1 workspace -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -export DB2_INSTANCE_NAME=mas-dev1-ws1-manage -ansible-playbook ibm.mas_devops.br_optimizer -``` - - -Backup/Restore for Visual Inspection -------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_visualinspection` will backup the following components that Visual Inspection depends on in order: - -| Component | Ansible Role | Data included | -| ---------------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | -| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core and Visual Inspection | -| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | -| visualinspection | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Visual Inspection namespace resources
Persistent volume data, such as images and models | - -### Environment Variables - -- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. -- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. - -### Examples - -```bash -# Full backup all visual inspection data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -ansible-playbook ibm.mas_devops.br_visualinspection - -# Incremental backup all visual inspection data for the dev1 instance and ws1 workspace -export MASBR_ACTION=backup -export MASBR_BACKUP_TYPE=incr -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -ansible-playbook ibm.mas_devops.br_visualinspection - -# Restore all visual inspection data for the dev1 instance and ws1 workspace -export MASBR_ACTION=restore -export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup -export MASBR_RESTORE_FROM_VERSION=20240630132439 -export MAS_INSTANCE_ID=dev -export MAS_WORKSPACE_ID=ws1 -ansible-playbook ibm.mas_devops.br_visualinspection -``` - - -Reference -------------------------------------------------------------------------------- -### Directory Structure -No matter what kind of storage systems you choose, the folder structure created in the storage system is same. - -Below is the sample folder structure for saving backup jobs: - -``` -/backups/mongodb-main-full-20240621122530 -├── backup.yml -├── database -│ ├── mongodb-main-full-20240621122530.tar.gz -│ └── query.json -└── log - ├── mongodb-main-full-20240621122530-backup-log.tar.gz - └── mongodb-main-full-20240621122530-ansible-log.tar.gz - -/backups/core-main-full-20240621122530 -├── backup.yml -├── log -│ ├── core-main-full-20240621122530-ansible-log.tar.gz -│ └── core-main-full-20240621122530-namespace-log.tar.gz -└── namespace - └── core-main-full-20240621122530-namespace.tar.gz -``` - -- ``: the root folder is specified by `MASBR_STORAGE_LOCAL_FOLDER` or `MASBR_STORAGE_CLOUD_BUCKET` -- The backup playbooks will create a seperated backup job folder under the `backups` folder for each component. The backup job folder is named by following this format: `{BACKUP COMPONENT}-{INSTANCE ID}-{BACKUP TYPE}-{BACKUP VERSION}`. -- When using playbook to backup multiple components at once, all backup job folders will be assigned to the same backup version. In above example, the same backup version `20240621122530` for backing up `mongodb` and `core` components. -- `backup.yml`: keep the backup job information -- `database`: data type for database. This folder save the backup files of MongoDB database, Db2 database. -- `namespace`: data type for namespace resources. This folder save the exported namespace resources. -- `pv`: data type for persistent volume. This folder save the persistent volume data, e.g. the Manage attachments, VI images and models. -- `log`: this folder save all job running log files - -In addition to the backup jobs, we also save restore jobs in the specified storage location. For example: - -``` -/restores/mongodb-main-incr-20240622040201-20240622075501 -├── log -│ ├── mongodb-main-incr-20240622040201-20240622075501-ansible-log.tar.gz -│ └── mongodb-main-incr-20240622040201-20240622075501-restore-log.tar.gz -└── restore.yml - -/restores/core-main-incr-20240622040201-20240622075501 -├── log -│ ├── core-main-incr-20240622040201-20240622075501-ansible-log.tar.gz -│ └── core-main-incr-20240622040201-20240622075501-namespace-log.tar.gz -└── restore.yml -``` - -The restore playbooks will create a seperated restore job folder under the `restores` folder for each component. The restore job folder is named by following this format: `{BACKUP JOB NAME}-{RESTORE VERSION}`. - -- `restore.yml`: keep the restore job information -- `log`: this folder save all job running log files - -### Data Model -#### backup.yml -```yaml -kind: Backup -name: "core-main-incr-20240622040201" -version: "20240622040201" -type: "incr" -from: "core-main-full-20240621122530" -source: - domain: "source-cluster.mydomain.com" - suite: "8.11.11" - instance: "main" - workspace: "" -component: - name: "core" - instance: "main" - namespace: "mas-main-core" -data: - - seq: "1" - type: "namespace" - phase: "Completed" -status: - phase: "Completed" - startTimestamp: "2024-06-22T04:05:22" - completionTimestamp: "2024-06-22T04:06:04" - sentNotifications: - - type: "Slack" - channel: "#ansible-slack-dev" - timestamp: "2024-06-22T04:05:34" - phase: "InProgress" - - type: "Slack" - channel: "#ansible-slack-dev" - timestamp: "2024-06-22T04:06:10" - phase: "Completed" -``` - -#### restore.yml -```yaml -kind: Restore -name: "core-main-incr-20240622040201-20240622075501" -version: "20240622075501" -from: "core-main-incr-20240622040201" -target: - domain: "target-cluster.mydomain.com" -component: - name: "core" - instance: "main" - namespace: "mas-main-core" -data: - - seq: 1 - type: "namespace" - phase: "Completed" -status: - phase: "Completed" - startTimestamp: "2024-06-22T08:04:19" - completionTimestamp: "2024-06-22T08:04:33" -``` diff --git a/ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml b/ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml deleted file mode 100644 index f8ae4c50da..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/after_run_tasks.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -# After backup/restore component -# ------------------------------------------------------------------------- -- name: "After {{ masbr_job_type }} {{ masbr_job_component.name }}" - when: _component_after_task_path is defined and _component_after_task_path | length > 0 - include_tasks: "{{ _component_after_task_path }}" - -# Copy Ansible log file to storage location -# ----------------------------------------------------------------------------- -- name: "Set fact: Ansible log path" - set_fact: - masbr_ansible_log_path: "{{ lookup('env', 'ANSIBLE_LOG_PATH') }}" - masbr_ansible_log_name: "{{ masbr_job_name }}-ansible" - -- name: "Copy Ansible log file to storage location" - when: - - masbr_ansible_log_path is defined - - masbr_ansible_log_path | length > 0 - block: - - name: "Debug: Ansbile log path" - debug: - msg: "Ansible log path .................. {{ masbr_ansible_log_path }}" - - - name: "Create a tar.gz archive of Ansible log file" - shell: >- - mkdir -p {{ masbr_local_job_folder }}/log && - cp -f {{ masbr_ansible_log_path }} {{ masbr_local_job_folder }}/log/{{ masbr_ansible_log_name }}.log && - tar -czf {{ masbr_local_job_folder }}/log/{{ masbr_ansible_log_name }}-log.tar.gz - -C {{ masbr_local_job_folder }}/log {{ masbr_ansible_log_name }}.log - - - name: "Copy Ansible log file from local to storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "{{ masbr_job_type }}" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "log/{{ masbr_ansible_log_name }}-log.tar.gz" - dest_folder: "log" - -# Delete local job folder -# ----------------------------------------------------------------------------- -- name: "Delete local job folder" - file: - path: "{{ masbr_local_job_folder }}" - state: absent - -# Display summary of the running task results -# ----------------------------------------------------------------------------- -- name: "Summary" - debug: - msg: - - "Job name ........................... {{ masbr_job_name }}" - - "Job status ......................... {{ masbr_job_status.phase }}" - - "Job storage location ............... {{ masbr_storage_job_folder_final | default(masbr_storage_job_folder, true) }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml b/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml deleted file mode 100644 index ab977485f4..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/before_run_tasks.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -# Scenario 1 - when running a playbook: -# 1. playbook include this task -# 2. roles include this task -# Scenario 2 - when running a role: -# 1. the role include this task - -# Check common variables -# ----------------------------------------------------------------------------- -- name: "Check common variables" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/check_common_vars.yml" - -# Confirm cluster information -# ----------------------------------------------------------------------------- -- name: "Confirm the currently connected cluster information" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/confirm_cluster_info.yml" - -# Check common backup/restore variables -# ----------------------------------------------------------------------------- -- name: "Check common {{ _job_type }} variables" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/check_{{ _job_type }}_vars.yml" - -# Before backup/restore component -# ------------------------------------------------------------------------- -- name: "Before {{ _job_type }} {{ masbr_job_component.name }}" - when: _component_before_task_path is defined and _component_before_task_path | length > 0 - include_tasks: "{{ _component_before_task_path }}" - -# Set a flag to indicate these tasks are included -# ----------------------------------------------------------------------------- -- name: "Set fact: already included these tasks" - set_fact: - masbr_included_before_run_tasks: true diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml deleted file mode 100644 index 87ddb9b788..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/check_backup_vars.yml +++ /dev/null @@ -1,196 +0,0 @@ ---- -# Set below common job facts: -# masbr_task_type: backup, restore -# masbr_job_type: backup, restore -# masbr_job_name, masbr_job_name_final -# -# Set below backup job facts: -# masbr_backup_from -# masbr_backup_from_yaml - -# Backup environment variables -# ----------------------------------------------------------------------------- -- name: "Set fact: backup environment variables" - set_fact: - # Supported backup types: 'full', 'incr', 'delta' (Not support 'delta' by now) - masbr_backup_type: "{{ lookup('env', 'MASBR_BACKUP_TYPE') | default('full', true) }}" - - # Data type string separated by commas: e.g.'namespace,pv' - masbr_backup_data: "{{ lookup('env', 'MASBR_BACKUP_DATA') | default('', true) }}" - - # The version of the backup to create incremental backup based on - # only used when masbr_backup_type='incr' - masbr_backup_from_version: "{{ lookup('env', 'MASBR_BACKUP_FROM_VERSION') | default('', true) }}" - -# Check 'masbr_job_component' -# ----------------------------------------------------------------------------- -- name: "Fail if masbr_job_component is not provided" - assert: - that: - - masbr_job_component is defined - - ('name' in masbr_job_component) - - ('namespace' in masbr_job_component) - fail_msg: - - "masbr_job_component.name is required" - - "masbr_job_component.namespace is required" - -# Check 'masbr_job_data_list' -# ----------------------------------------------------------------------------- -- name: "Set fact: init masbr_job_data_list" - set_fact: - masbr_job_data_list: "{{ masbr_job_data_list | default([], true) }}" - -- name: "Set fact: specified backup data" - when: - - masbr_backup_data is defined - - masbr_backup_data | length > 0 - - (_ignore_masbr_backup_data is not defined) or (_ignore_masbr_backup_data is defined and not _ignore_masbr_backup_data) - block: - - name: "Set fact: reset masbr_job_data_specified" - set_fact: - masbr_job_data_specified: [] - - - name: "Get specified backup data" - set_fact: - masbr_job_data_specified: "{{ masbr_job_data_specified + [{ 'seq': (idx+1)|int, 'type': item|trim }] }}" - loop: "{{ masbr_backup_data | split(',') }}" - loop_control: - index_var: idx - - - name: "Set fact: override the default masbr_job_data_list" - set_fact: - masbr_job_data_list: "{{ masbr_job_data_specified }}" - -- name: "Set fact: set default phase to each backup data" - when: masbr_job_data_list is defined and masbr_job_data_list | length > 0 - block: - - name: "Set fact: reset masbr_job_data_init" - set_fact: - masbr_job_data_init: [] - - - name: "Set fact: set default phase to each backup data" - set_fact: - masbr_job_data_init: "{{ masbr_job_data_init + [ item | combine({ 'phase': 'New' }) ] }}" - loop: "{{ masbr_job_data_list }}" - - - name: "Set fact: backup data with default phase" - set_fact: - masbr_job_data_list: "{{ masbr_job_data_init }}" - -# Set 'masbr_task_type' -# ----------------------------------------------------------------------------- -- name: "Set fact: backup job variables" - set_fact: - masbr_task_type: "backup" - masbr_job_type: "backup" - -- name: "Set fact: job name include instance" - when: masbr_job_component.instance is defined and masbr_job_component.instance | length > 0 - set_fact: - # Format '---' - # 'mongodb-main-incr-20240509130354' - # 'db2-mas-main-masdev-manage-full-20240509130354' - # 'manage-ivt90x-01-full-20240509130354' - masbr_job_name_prefix: >- - {{ masbr_job_component.name }}-{{ masbr_job_component.instance }} - -- name: "Set fact: job name without instance" - when: masbr_job_component.instance is undefined or masbr_job_component.instance | length == 0 - set_fact: - masbr_job_name_prefix: "{{ masbr_job_component.name }}" - -- name: "Set fact: backup job name" - set_fact: - masbr_job_name: "{{ masbr_job_name_prefix }}-{{ masbr_backup_type }}-{{ masbr_job_version }}" - -- name: "Set fact: final backup job name" - set_fact: - # At this point, set it as the same value of masbr_job_name - masbr_job_name_final: "{{ masbr_job_name }}" - -# Create local job folder -# ----------------------------------------------------------------------------- -- name: "Create local job folder" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_local_job_folder.yml" - -# Check incremental backup -# ----------------------------------------------------------------------------- -- name: "Checks for incremental backup" - when: masbr_backup_type == "incr" - block: - # when 'masbr_backup_from_version' is not specified: find the latest full backup - - name: "Get the latest Full backup job name when masbr_backup_from_version not provided" - when: masbr_backup_from_version is not defined or masbr_backup_from_version | length == 0 - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" - vars: - masbr_ls_job_type: "backup" - masbr_ls_filter: "| grep -P '^{{ masbr_job_name_prefix }}-full-.*(? 0 - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" - vars: - masbr_ls_job_type: "backup" - masbr_ls_filter: "| grep '^{{ masbr_job_name_prefix }}-full-{{ masbr_backup_from_version }}$' | sort -r | head -1" - - - name: "Fail if not found any previous Full backup job" - assert: - that: masbr_ls_results is defined and masbr_ls_results | length == 1 - fail_msg: "Not found any previous Full backup job, please take a Full backup first!" - - - name: "Set fact: backup from job name" - set_fact: - masbr_backup_from: "{{ masbr_ls_results[0] }}" - - # Get backup from job information - - name: "Get backup from job information" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_local.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_backup_from }}" - masbr_cf_paths: - - src_file: "backup.yml" - dest_folder: "from" - - - name: "Set fact: backup from job information" - set_fact: - masbr_backup_from_yaml: "{{ lookup('file', masbr_local_job_folder + '/from/backup.yml') | from_yaml }}" - - - name: "Debug: backup from job information" - debug: - msg: "{{ masbr_backup_from_yaml }}" - - # The backup from should be Completed - - name: "Fail if the backup from is not Completed" - assert: - that: masbr_backup_from_yaml.status.phase == "Completed" - fail_msg: "The specified backup job is not Completed, please specify a Completed backup job." - - # The backup from job should has the same component and data as current job - - name: "Fail if the component name of backup from job is not same as current job" - assert: - that: masbr_backup_from_yaml.component.name == masbr_job_component.name - fail_msg: "The component name of backup from job is not same as current job" - - - name: "Set fact: data list difference" - set_fact: - masbr_job_data_list_differ: >- - {{ masbr_job_data_list | map(attribute='type') | - difference(masbr_backup_from_yaml.data | map(attribute='type')) }} - - - name: "Fail if the data list of backup from job does not cover current job" - assert: - that: masbr_job_data_list_differ | length == 0 - fail_msg: "The data list of backup from job does not cover current job: {{ masbr_job_data_list_differ }}" - -# Show backup job information -# ----------------------------------------------------------------------------- -- name: "Debug: backup job information" - debug: - msg: - - "Backup job name ....................... {{ masbr_job_name }}" - - "Backup type ........................... {{ masbr_backup_type }}" - - "Backup from ........................... {{ masbr_backup_from | default('', true) }}" - - "Backup component ...................... {{ masbr_job_component }}" - - "Backup data ........................... {{ masbr_job_data_list }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml deleted file mode 100644 index 1f3eb049a4..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/check_common_vars.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -# Load default variables -# ----------------------------------------------------------------------------- -- name: "Load common variables" - include_vars: "{{ role_path }}/../../common_vars/backup_restore.yml" - -- name: "Set fact: internal used common variables" - set_fact: - # ONLY FOR DEV - __masbr_dev_create_env_file: "{{ lookup('env', '__MASBR_DEV_CREATE_ENV_FILE') | default(false, true) | bool }}" - - # Temp folder in the Pod for backup/restore - masbr_pod_temp_folder: "/tmp/masbr" - - # Timestamp display format - masbr_timestamp_format: "%Y%m%d%H%M%S" - -- name: "Set fact: job lock file" - set_fact: - masbr_pod_lock_file: "{{masbr_pod_temp_folder}}/running.lock" - -# Get 'masbr_job_version' in below order: -# 1. get from input 'masbr_job_version' -# 2. if not set, get from env 'MASBR_JOB_VERSION' -# 3. For schedule, always create a new version. -- name: "Get job version from env" - when: masbr_job_version is not defined - set_fact: - masbr_job_version: "{{ lookup('env', 'MASBR_JOB_VERSION') | default(masbr_timestamp_format | strftime, true) }}" - -# Storage location -# ----------------------------------------------------------------------------- -- name: "Fail if masbr_storage_local_folder is not provided" - assert: - that: masbr_storage_local_folder is defined and masbr_storage_local_folder != "" - fail_msg: "MASBR_STORAGE_LOCAL_FOLDER is required" - -- name: "Debug: variables for local backup storage" - debug: - msg: - - "Local storage folder ............... {{ masbr_storage_local_folder }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml b/ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml deleted file mode 100644 index 49c5e1d22c..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/check_restore_vars.yml +++ /dev/null @@ -1,208 +0,0 @@ ---- -# Set below common job facts: -# masbr_task_type, masbr_job_type -# masbr_job_name, masbr_job_name_final -# -# Set below restore job facts: -# masbr_restore_from -# masbr_restore_basedon -# masbr_restore_from_yaml -# masbr_restore_from_incr: true|false -# masbr_restore_to_diff_domain: true|false -# masbr_restore_to_diff_instance: true|false - -# Restore environment variables -# ----------------------------------------------------------------------------- -- name: "Set fact: restore environment variables" - set_fact: - # The information of the backup to be restored from - # - masbr_restore_from_version: "{{ lookup('env', 'MASBR_RESTORE_FROM_VERSION') }}" - masbr_restore_overwrite: "{{ lookup('env', 'MASBR_RESTORE_OVERWRITE') }}" - - # Data type string separated by commas: e.g.'namespace,pv' - masbr_restore_data: "{{ lookup('env', 'MASBR_RESTORE_DATA') | default('', true) }}" - - # Also will restore the based on full backup when trying to restore from an incremental backup - # (Not used by now) - masbr_restore_include_basedon: "{{ lookup('env', 'MASBR_RESTORE_INCLUDE_BASEDON') | default(true, true) }}" - -- name: "Fail if masbr_restore_from_version is not provided" - assert: - that: masbr_restore_from_version is defined and masbr_restore_from_version != "" - fail_msg: "MASBR_RESTORE_FROM_VERSION is required for running restore job" - -- name: "Fail if masbr_restore_overwrite is not provided" - assert: - that: masbr_restore_overwrite is defined and masbr_restore_overwrite != "" - fail_msg: "MASBR_RESTORE_OVERWRITE is required for running restore job" - -# Check 'masbr_job_component' -# ----------------------------------------------------------------------------- -- name: "Fail if masbr_job_component is not provided" - assert: - that: - - masbr_job_component is defined - - ('name' in masbr_job_component) - - ('namespace' in masbr_job_component) - fail_msg: - - "masbr_job_component.name is required" - - "masbr_job_component.namespace is required" - -# Check 'masbr_job_data_list' -# ----------------------------------------------------------------------------- -- name: "Set fact: init masbr_job_data_list" - set_fact: - masbr_job_data_list: "{{ masbr_job_data_list | default([], true) }}" - -- name: "Set fact: specified restore data" - when: masbr_restore_data is defined and masbr_restore_data | length > 0 - block: - - name: "Set fact: reset masbr_job_data_specified" - set_fact: - masbr_job_data_specified: [] - - - name: "Get specified restore data" - set_fact: - masbr_job_data_specified: "{{ masbr_job_data_specified + [{ 'seq': (idx+1)|int, 'type': item|trim }] }}" - loop: "{{ masbr_restore_data | split(',') }}" - loop_control: - index_var: idx - - - name: "Set fact: override the default masbr_job_data_list" - set_fact: - masbr_job_data_list: "{{ masbr_job_data_specified }}" - -- name: "Set fact: set default phase to each restore data" - when: masbr_job_data_list is defined and masbr_job_data_list | length > 0 - block: - - name: "Set fact: reset masbr_job_data_init" - set_fact: - masbr_job_data_init: [] - - - name: "Set fact: set default phase to each restore data" - set_fact: - masbr_job_data_init: "{{ masbr_job_data_init + [ item | combine({ 'phase': 'New' }) ] }}" - loop: "{{ masbr_job_data_list }}" - - - name: "Set fact: restore data with default phase" - set_fact: - masbr_job_data_list: "{{ masbr_job_data_init }}" - -# Find restore-from job name -# ----------------------------------------------------------------------------- -- name: "Find the restore-from job name" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" - vars: - masbr_ls_job_type: "backup" - masbr_ls_filter: "| grep '^{{ masbr_job_component.name }}-.*-{{ masbr_restore_from_version }}$'" - -- name: "Fail if not found the restore-from job name" - assert: - that: masbr_ls_results is defined and masbr_ls_results | length == 1 - fail_msg: "Not found the job name specified by MASBR_RESTORE_FROM_VERSION" - -- name: "Set fact: restore-from job name" - set_fact: - masbr_restore_from: "{{ masbr_ls_results[0] }}" - -# Set restore job variables -# ----------------------------------------------------------------------------- -- name: "Set fact: restore job variables" - set_fact: - masbr_task_type: "restore" - masbr_job_type: "restore" - -- name: "Set fact: restore job name" - set_fact: - masbr_job_name: "{{ masbr_restore_from }}-{{ masbr_job_version }}" - -- name: "Set fact: final restore job name" - set_fact: - # At this point, set it as the same value of masbr_job_name - masbr_job_name_final: "{{ masbr_job_name }}" - -# Create local job folder -# ----------------------------------------------------------------------------- -- name: "Create local job folder" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/create_local_job_folder.yml" - -# Get restore-from job information -# ----------------------------------------------------------------------------- -- name: "Get restore-from job information" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_local.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_restore_from }}" - masbr_cf_paths: - - src_file: "backup.yml" - dest_folder: "from" - -- name: "Set fact: restore-from job yaml" - set_fact: - masbr_restore_from_yaml: "{{ lookup('file', masbr_local_job_folder + '/from/backup.yml') | from_yaml }}" - -- name: "Debug: restore-from job yaml" - debug: - msg: "{{ masbr_restore_from_yaml }}" - -# The restore-from job should be Completed -- name: "Fail if the restore-from job is not Completed" - assert: - that: masbr_restore_from_yaml.status.phase == "Completed" - fail_msg: "The specified backup job is not Completed, please specify a Completed Full backup job." - -- name: "Set fact: restore-from job variables" - set_fact: - # Whether restore from an incremental backup - masbr_restore_from_incr: "{{ true if masbr_restore_from_yaml.type == 'incr' else false }}" - - # Whether restore to different domain (Disaster Recovery) - masbr_restore_to_diff_domain: >- - {{ true if masbr_restore_from_yaml.source.domain != masbr_cluster_domain else false }} - - # Whether restore to different mas instance (Data Migration) - masbr_restore_to_diff_instance: >- - {{ true if (masbr_job_component.instance is defined and masbr_job_component.instance | length > 0 and - masbr_restore_from_yaml.source.instance != masbr_job_component.instance) else false }} - -# Trying to restore from an incremental backup, also need to check the existance of the based on full backup -# ----------------------------------------------------------------------------- -- name: "Check the existence of the based on full backup job" - when: masbr_restore_from_incr - block: - - name: "Fail if not found 'from' specified in this incremental backup job information" - assert: - that: masbr_restore_from_yaml.from is defined and masbr_restore_from_yaml.from | length > 0 - fail_msg: "Not found 'from' specified in this incremental backup job information" - - - name: "Set fact: the based on full backup job name" - set_fact: - masbr_restore_basedon: "{{ masbr_restore_from_yaml.from }}" - - - name: "Check the existence of the based on full backup job" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/list_storage_job_folders.yml" - vars: - masbr_ls_job_type: "backup" - masbr_ls_filter: "| grep {{ masbr_restore_basedon }}" - - - name: "Fail if not found the based on full backup job" - assert: - that: masbr_ls_results is defined and masbr_ls_results | length == 1 - fail_msg: >- - Not found the based on full backup job folder: - {{ masbr_storage_job_type_folder }}/{{ masbr_restore_basedon }} - -# Show restore job information -# ----------------------------------------------------------------------------- -- name: "Debug: restore job information" - debug: - msg: - - "Restore job name ....................... {{ masbr_job_name }}" - - "Restore from ........................... {{ masbr_restore_from }}" - - "Restore overwrite existing data ........ {{ masbr_restore_overwrite }}" - - "Restore component ...................... {{ masbr_job_component }}" - - "Restore data ........................... {{ masbr_job_data_list }}" - - "Restore from incremental backup ........ {{ masbr_restore_from_incr }}" - - "Restore to different domain ............ {{ masbr_restore_to_diff_domain }}" - - "Restore to different instance .......... {{ masbr_restore_to_diff_instance }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml b/ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml deleted file mode 100644 index 097623b698..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/confirm_cluster_info.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# Get cluster domain -# ----------------------------------------------------------------------------- -- name: "Get cluster domain" - kubernetes.core.k8s_info: - api_version: config.openshift.io/v1 - kind: DNS - name: cluster - register: _cluster_dns - -- name: "Set fact: cluster domain" - set_fact: - masbr_cluster_domain: "{{ _cluster_dns.resources[0].spec.baseDomain }}" - -- name: "Debug: cluster domain" - debug: - msg: "Cluster domain ........................ {{ masbr_cluster_domain }}" - -# Confirm the cluster information -# ----------------------------------------------------------------------------- -- name: "Confirm the connected cluster information" - when: masbr_confirm_cluster - block: - - name: "Get user input" - pause: - prompt: "\nCurrently connected to cluster:\n {{ masbr_cluster_domain }}\nProceed on this cluster? [yes/no]" - register: _confirm_cluster_info - - - name: "Cancel task" - when: not _confirm_cluster_info.user_input | bool - fail: - msg: "User chooses to cancel task" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml deleted file mode 100644 index 058043a510..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_local_files_to_storage.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Copy local job files to local storage -# ----------------------------------------------------------------------------- -- name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - -- name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - -- name: "Copy local job files to local storage job folder" - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - cp -rf {{ [masbr_local_job_folder, item.src_file] | path_join }} - {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - ls -lA {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - register: _local_copy_output - -- name: "Debug: copy local job files to local storage job folder" - debug: - msg: "{{ _local_copy_output | json_query('results[*].stdout_lines') }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml deleted file mode 100644 index 1d22b4d351..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_pod_files_to_storage.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# Copy files from pod to local storage -# ----------------------------------------------------------------------------- -- name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - -- name: "Debug: All PV variables" - debug: - msg: - - "Pod Name ................... {{ masbr_cf_pod_name }}" - - "Local storage job folder ......... {{ masbr_storage_job_folder }}" - - "Folder setup .................... {{ masbr_cf_paths }}" - - "Source Folder .................... {{ item.src_folder | default('null', true) }}" - - "Destination Folder ............... {{ [masbr_storage_job_folder, item.dest_folder] | path_join }}" - loop: "{{ masbr_cf_paths }}" - -# Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* -- name: "Copy files from pod folder to local storage folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_folder }} - {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - -# Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file -- name: "Copy file from pod to local storage folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_folder] | path_join }} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.src_file }} - {{ [masbr_storage_job_folder, item.dest_folder, item.src_file|basename] | path_join }} - loop: "{{ masbr_cf_paths }}" - -# Condition 3. src_file -> dest_file -- name: "Copy file from pod to local storage file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_file is defined and item.dest_file | length > 0 - shell: >- - mkdir -p {{ [masbr_storage_job_folder, item.dest_file|dirname] | path_join }} && - oc cp --retries=50 -c {{ masbr_cf_container_name }} - {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ [item.src_folder, item.src_file] | path_join }} - {{ [masbr_storage_job_folder, item.dest_file] | path_join }} - loop: "{{ masbr_cf_paths }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml deleted file mode 100644 index 363e4516ca..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_local.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Copy job files from local storage to local job folder -# ----------------------------------------------------------------------------- -- name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - -- name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - -- name: "Copy job files from local storage to local job folder" - shell: >- - mkdir -p {{ [masbr_local_job_folder, item.dest_folder] | path_join }} && - cp -rf {{ [masbr_storage_job_folder, item.src_file] | path_join }} - {{ [masbr_local_job_folder, item.dest_folder] | path_join }} && - ls -lA {{ [masbr_local_job_folder, item.dest_folder] | path_join }} - loop: "{{ masbr_cf_paths }}" - register: _local_copy_output - -- name: "Debug: copy job files from local storage to local job folder" - debug: - msg: "{{ _local_copy_output | json_query('results[*].stdout_lines') }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml b/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml deleted file mode 100644 index c9ff13ed7d..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/copy_storage_files_to_pod.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -# Copy files from local storage to pod -# ----------------------------------------------------------------------------- - -# Local storage job folder -- name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: >- - {{ masbr_storage_local_folder }}/{{ masbr_cf_job_type }}s/{{ masbr_cf_job_name }} - -- name: "Debug: All PV variables" - debug: - msg: - - "Local storage job folder ......... {{ masbr_storage_job_folder }}" - - "Overwrite existing data .......... {{ masbr_restore_overwrite }}" - - "Folder setup .................... {{ masbr_cf_paths }}" - - "Source Folder .................... {{ [masbr_storage_job_folder, item.src_folder | default(item.src_file, true)] | path_join }}" - - "Destination Folder ............... {{ item.dest_folder }}" - loop: "{{ masbr_cf_paths }}" - -- name: "Erase all existing data found in destination folders" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - - masbr_restore_overwrite is defined and masbr_restore_overwrite - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'rm -rf {{ item.dest_folder }}/*' - loop: "{{ masbr_cf_paths }}" - -- name: "Detect if there is any data in destination folders" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - - masbr_restore_overwrite is defined and masbr_restore_overwrite == False - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c '[ "$(ls -A {{ item.dest_folder }})" ] && { echo "{{ item.dest_folder }} is not empty!" && exit 1; } || echo "{{ item.dest_folder }} is empty!";' - loop: "{{ masbr_cf_paths }}" - - -# Condition 1. src_folder -> dest_folder: copy src_folder/* to dest_folder/* -# -# - exec into masbr_cf_pod_name/masbr_cf_container_name, create temp folder -# - cp from src_folder to temp folder inside masbr_cf_pod_name/masbr_cf_container_name -# - exec into masbr_cf_pod_name/masbr_cf_container_name, move item.temp_dest_folder to dest_folder and delete temp_dest_folder -- name: "Copy files from local storage folder to pod folder" - when: - - item.src_folder is defined and item.src_folder | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mkdir -p {{ [item.dest_folder, masbr_job_version] | path_join }}' \ - && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_folder] | path_join }}/. {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ [item.dest_folder, masbr_job_version] | path_join }} \ - && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f {{ [item.dest_folder, masbr_job_version] | path_join }}/* {{ item.dest_folder }} && rm -rf {{ [item.dest_folder, masbr_job_version] | path_join }}' - loop: "{{ masbr_cf_paths }}" - -# Condition 2. src_file -> dest_folder: copy src_file to dest_folder/src_file -- name: "Copy file from local storage folder to pod folder" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mkdir -p {{ item.dest_folder }}' \ - && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [masbr_storage_job_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ item.dest_folder }} - loop: "{{ masbr_cf_paths }}" - -# Condition 3. src_file -> dest_file -# - exec into masbr_cf_pod_name/masbr_cf_container_name, create temp folder -# - cp from src_folder to temp folder inside masbr_cf_pod_name/masbr_cf_container_name -# - exec into masbr_cf_pod_name/masbr_cf_container_name, move temp_dest_folder to item.dest_folder and delete temp_dest_folder -- name: "Copy file from local storage folder to pod file" - when: - - item.src_file is defined and item.src_file | length > 0 - - item.dest_folder is defined and item.dest_folder | length > 0 - shell: >- - oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mkdir -p {{ [item.dest_folder, masbr_job_version] | path_join }}' \ - && oc cp --retries=50 -c {{ masbr_cf_container_name }} {{ [temp_src_folder, item.src_file] | path_join }} {{ masbr_cf_namespace }}/{{ masbr_cf_pod_name }}:{{ [item.dest_folder, masbr_job_version] | path_join }} \ - && oc exec {{ masbr_cf_pod_name }} -c {{ masbr_cf_container_name }} -n {{ masbr_cf_namespace }} -- bash -c 'mv -f {{ [item.dest_folder, masbr_job_version] | path_join }}/{{ item.src_file|basename }} {{ item.dest_folder }} && rm -rf {{ [item.dest_folder, masbr_job_version] | path_join }}' diff --git a/ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml b/ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml deleted file mode 100644 index eacd6e9153..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/create_cleanup_job.yml +++ /dev/null @@ -1,63 +0,0 @@ ---- -# Check if cleanup Job exists -# ----------------------------------------------------------------------------- -- name: "Get cleanup Job" - kubernetes.core.k8s_info: - api_version: batch/v1 - kind: CronJob - name: masbr-cleanup - namespace: "{{ masbr_cleanup_namespace }}" - register: _cleanup_job_info - -- name: "Set fact: cleanup Job exists" - when: - - _cleanup_job_info is defined - - _cleanup_job_info.resources is defined - - _cleanup_job_info.resources | length == 1 - set_fact: - masbr_cleanup_job_exists: true - -# Create script configmap if cleanup Job not exists -# ----------------------------------------------------------------------------- -- name: "Create script configmap if cleanup Job not exists" - when: masbr_cleanup_job_exists is not defined - block: - - name: "Create cleanup script" - template: - src: "{{ role_path }}/../../common_tasks/templates/backup_restore/cleanup_job.sh.j2" - dest: "{{ masbr_local_job_folder }}/cleanup_job.sh" - - - name: "Get cleanup script content" - shell: > - cat {{ masbr_local_job_folder }}/cleanup_job.sh - register: _cleanup_sh_content - - - name: "Create configmap to save cleanup script" - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: ConfigMap - metadata: - name: masbr-cleanup - namespace: "{{ masbr_cleanup_namespace }}" - data: - script: "{{ _cleanup_sh_content.stdout }}" - wait: true - -# Create or update cleanup Job -# ----------------------------------------------------------------------------- -- name: "Set fact: cleanup Job variables" - set_fact: - masbr_cleanup_env: - - name: "MASBR_CLEANUP_TTL_SEC" - value: "{{ masbr_cleanup_ttl_sec }}" - masbr_cleanup_cmds: >- - oc get cm/masbr-cleanup -n {{ masbr_cleanup_namespace }} -o yaml | yq '.data.script' > /tmp/masbr-cleanup.sh && - chmod +x /tmp/masbr-cleanup.sh && - /tmp/masbr-cleanup.sh - -- name: "Create or update cleanup Job" - kubernetes.core.k8s: - apply: true - template: "{{ role_path }}/../../common_tasks/templates/backup_restore/cleanup_job.yml.j2" - state: present diff --git a/ibm/mas_devops/common_tasks/backup_restore/create_local_job_folder.yml b/ibm/mas_devops/common_tasks/backup_restore/create_local_job_folder.yml deleted file mode 100644 index 56a57ad033..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/create_local_job_folder.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -# Create local job folder if not exists -# ----------------------------------------------------------------------------- -- name: "Set fact: local job folder" - set_fact: - masbr_local_job_folder: "{{ masbr_local_temp_folder }}/{{ masbr_job_name }}" - -- name: "Debug: local job folder" - debug: - msg: "Local job folder ...................... {{ masbr_local_job_folder }}" - -- name: "Check if local job folder exists" - stat: - path: "{{ masbr_local_job_folder }}" - register: _file_stat_output - -- name: "Create local job folder if not exists" - when: not _file_stat_output.stat.exists - file: - path: "{{ masbr_local_job_folder }}" - state: directory diff --git a/ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml b/ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml deleted file mode 100644 index 60d6a71fa0..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/delete_storage_job_folder.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# Delete the job folder from local storage -# ----------------------------------------------------------------------------- -- name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name }}" - -- name: "Debug: local storage job folder" - debug: - msg: "Local storage job folder .......... {{ masbr_storage_job_folder }}" - -- name: "Delete the job folder from local storage" - command: rm -rf {{ masbr_storage_job_folder }} - args: - removes: "{{ masbr_storage_job_folder }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml b/ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml deleted file mode 100644 index 7115813af6..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/list_storage_job_folders.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -# List job folders in local storage -# ----------------------------------------------------------------------------- -- name: "Set fact: local storage job type folder" - set_fact: - masbr_storage_job_type_folder: "{{ masbr_storage_local_folder }}/{{ masbr_ls_job_type }}s" - -- name: "Debug: list job folders variables" - debug: - msg: - - "Search folder ...................... {{ masbr_storage_job_type_folder }}" - - "Search filter ...................... {{ masbr_ls_filter | default('', true) }}" - -- name: "List job folders in local storage" - changed_when: false - shell: >- - ls {{ masbr_storage_job_type_folder }} {{ masbr_ls_filter | default('') }}; - exit 0 - register: _ls_output - -- name: "Set fact: results of list job folders" - set_fact: - masbr_ls_results: "{{ _ls_output.stdout_lines }}" - -- name: "Debug: results of list job folders" - debug: - msg: "Results of list job folders ....... {{ masbr_ls_results }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml b/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml deleted file mode 100644 index fe869342e1..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/rename_storage_job_folder.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: "Set fact: final job folder name" - set_fact: - masbr_job_name_final: "{{ masbr_job_name }}-{{ masbr_job_status.phase }}" - -# Rename the job folder in local storage -# ----------------------------------------------------------------------------- -- name: "Set fact: local storage job folder" - set_fact: - masbr_storage_job_folder: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name }}" - masbr_storage_job_folder_final: "{{ masbr_storage_local_folder }}/{{ masbr_job_type }}s/{{ masbr_job_name_final }}" - -- name: "Debug: rename local storage job folder" - debug: - msg: - - "Source job folder .................. {{ masbr_storage_job_folder }}" - - "Dest job folder .................... {{ masbr_storage_job_folder_final }}" - -- name: "Rename the job folder in local storage" - command: mv {{ masbr_storage_job_folder }} {{ masbr_storage_job_folder_final }} - args: - removes: "{{ masbr_storage_job_folder }}" - creates: "{{ masbr_storage_job_folder_final }}" diff --git a/ibm/mas_devops/common_tasks/backup_restore/restart_and_reconsiled.yml b/ibm/mas_devops/common_tasks/backup_restore/restart_and_reconsiled.yml deleted file mode 100644 index ff022cdd72..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/restart_and_reconsiled.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: "Restart {{ _pod_keywords }} pod" - shell: > - oc get pods -n {{ _pod_namespace }} --no-headers=true | grep "{{ _pod_keywords }}" | awk '{print $1}' - | xargs oc delete pod -n {{ _pod_namespace }} - -- name: "Wait for {{ _pod_keywords }} pod to be ready (10s delay)" - shell: > - oc get pods -n {{ _pod_namespace }} --no-headers=true | grep "{{ _pod_keywords }}" - | grep -Evi "1/1|2/2|3/3|4/4|5/5|6/6|7/7|8/8|9/9|complete" | wc -l - register: _is_not_ready - until: _is_not_ready.stdout|int == 0 - retries: 30 - delay: 10 - -- name: "Wait for {{ _pod_keywords }} pod to be reconsiled (10s delay)" - shell: > - oc get pods -n {{ _pod_namespace }} --no-headers=true | grep "{{ _pod_keywords }}" | awk '{print $1}' - | xargs oc logs -c {{ _container_name }} -n {{ _pod_namespace }} | grep "ok=" | wc -l - register: _is_reconsiled - until: _is_reconsiled.stdout|int == 1 - retries: 60 - delay: 10 diff --git a/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml b/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml deleted file mode 100644 index 98f751b30f..0000000000 --- a/ibm/mas_devops/common_tasks/backup_restore/update_job_status.yml +++ /dev/null @@ -1,156 +0,0 @@ ---- -# Update job variables -# ----------------------------------------------------------------------------- -# Update 'masbr_job_component' -- name: "Update fact: masbr_job_component" - when: _job_component is defined - set_fact: - masbr_job_component: "{{ masbr_job_component | combine(_job_component) }}" - -# Update 'masbr_job_data_list' -- name: "Update fact: masbr_job_data_list" - when: - - _job_data_list is defined - - _job_data_list | length > 0 - block: - - name: "Update fact: masbr_job_data_list" - ansible.utils.update_fact: - updates: - - path: masbr_job_data_list[{{ item.seq|int -1 }}].phase - value: "{{ item.phase }}" - loop: "{{ _job_data_list }}" - register: _job_data_list_updated - - - name: "Set fact: masbr_job_data_list" - set_fact: - masbr_job_data_list: "{{ _job_data_list_updated.results[-1].masbr_job_data_list }}" - -# Get specified 'masbr_job_status.phase' -- name: "Get specified masbr_job_status.phase" - when: - - _job_status is defined - - _job_status.phase is defined - set_fact: - _job_status_: - phase: "{{ _job_status.phase }}" - -# Determine 'masbr_job_status.phase' based on 'masbr_job_data_list' -- name: "Determine masbr_job_status.phase based on masbr_job_data_list" - when: _job_status is not defined - block: - - name: "Get unique phases of all job data types" - set_fact: - masbr_job_data_phases: "{{ masbr_job_data_list | map(attribute='phase') | unique }}" - - - name: "Debug: unique phases of all job data types" - debug: - msg: "Job data phases ................... {{ masbr_job_data_phases }}" - - # masbr_job_data_phases: ['New'] - - name: "Set fact: masbr_job_status_phase ('New')" - when: - - ("New" in masbr_job_data_phases) - - masbr_job_data_phases | length == 1 - set_fact: - _job_status_: - phase: "New" - - # masbr_job_data_phases: ['InProgress'], ['InProgress', 'New'], ['Completed', 'InProgress', 'New'] - - name: "Set fact: masbr_job_status_phase ('InProgress')" - when: - - ("InProgress" in masbr_job_data_phases) - - ("Failed" not in masbr_job_data_phases) - set_fact: - _job_status_: - phase: "InProgress" - - # masbr_job_data_phases: ['Completed'] - - name: "Set fact: masbr_job_status_phase ('Completed')" - when: - - ("Completed" in masbr_job_data_phases) - - masbr_job_data_phases | length == 1 - set_fact: - _job_status_: - phase: "Completed" - - # masbr_job_data_phases: ['Failed', 'InProgress', 'New'] - - name: "Set fact: masbr_job_status_phase ('PartiallyFailed')" - when: - - ("Failed" in masbr_job_data_phases) - - masbr_job_data_phases | length > 1 - set_fact: - _job_status_: - phase: "PartiallyFailed" - - # masbr_job_data_phases: ['Failed'] - - name: "Set fact: masbr_job_status_phase ('Failed')" - when: - - ("Failed" in masbr_job_data_phases) - - masbr_job_data_phases | length == 1 - set_fact: - _job_status_: - phase: "Failed" - -# When Job status is "New" -- name: "Update fact: masbr_job_status.phase ('New')" - when: - - _job_status_ is defined - - _job_status_.phase is defined - - _job_status_.phase == "New" - set_fact: - masbr_job_status: - phase: "New" - startTimestamp: "{{ '%Y-%m-%dT%H:%M:%S' | strftime }}" - -# When Job status is "InProgress" -- name: "Update fact: masbr_job_status.phase ('InProgress')" - when: - - _job_status_ is defined - - _job_status_.phase is defined - - _job_status_.phase == "InProgress" - set_fact: - masbr_job_status: "{{ masbr_job_status | combine(_job_status_) }}" - -# When Job status is "Completed", "Failed" or "PartiallyFailed" -- name: "Update fact: masbr_job_status.phase ('Completed', 'Failed', 'PartiallyFailed')" - when: - - _job_status_ is defined - - _job_status_.phase is defined - - _job_status_.phase in ['Completed', 'Failed', 'PartiallyFailed'] - set_fact: - masbr_job_status: >- - {{ masbr_job_status | combine({ - 'phase': _job_status_.phase, - 'completionTimestamp': '%Y-%m-%dT%H:%M:%S' | strftime - }) }} - -# Create job file -# ----------------------------------------------------------------------------- -- name: "Debug: update job variables" - debug: - msg: - - "masbr_job_component .................... {{ masbr_job_component }}" - - "masbr_job_data_list .................... {{ masbr_job_data_list }}" - - "masbr_job_status ....................... {{ masbr_job_status }}" - -- name: "Create updated job file" - template: - src: "{{ role_path }}/../../common_tasks/templates/backup_restore/{{ masbr_job_type }}.yml.j2" - dest: "{{ masbr_local_job_folder }}/{{ masbr_job_type }}.yml" - -# Copy local job files to specified storage location -# ----------------------------------------------------------------------------- -- name: "Copy local job files to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "{{ masbr_job_type }}" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_type }}.yml" - dest_folder: "" - -# Append job final status to the job folder name -# ----------------------------------------------------------------------------- -- name: "Append job final status to the job folder name" - when: masbr_job_status.phase in ['PartiallyFailed', 'Failed'] - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/rename_storage_job_folder.yml" diff --git a/ibm/mas_devops/common_tasks/get_version_from_channel.yml b/ibm/mas_devops/common_tasks/get_version_from_channel.yml index c6c7c08388..37b909a752 100644 --- a/ibm/mas_devops/common_tasks/get_version_from_channel.yml +++ b/ibm/mas_devops/common_tasks/get_version_from_channel.yml @@ -49,6 +49,7 @@ ansible.builtin.assert: that: - op_version is defined + - op_version != None - op_version != '' fail_msg: "Could not find 'currentCSV' in from PackageManifest for {{ op_pm_name }}" diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/backup.yml.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/backup.yml.j2 deleted file mode 100644 index 03c2148b85..0000000000 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/backup.yml.j2 +++ /dev/null @@ -1,45 +0,0 @@ ---- -kind: Backup -name: "{{ masbr_job_name }}" -version: "{{ masbr_job_version }}" -type: "{{ masbr_backup_type }}" -{% if masbr_backup_from is defined and masbr_backup_from | length > 0 %} -from: "{{ masbr_backup_from }}" -{% endif %} -source: - domain: "{{ masbr_cluster_domain }}" - suite: "{{ mas_core_version | default('', true) }}" - instance: "{{ mas_instance_id | default('', true) }}" - workspace: "{{ mas_workspace_id | default('', true) }}" -{% if masbr_job_component is defined and masbr_job_component.items() %} -component: -{% for key, value in masbr_job_component.items() %} - {{ key }}: "{{ value }}" -{% endfor %} -{% endif %} -{% if masbr_job_data_list is defined and masbr_job_data_list | length > 0 %} -data: -{% for job_data in masbr_job_data_list %} - - seq: "{{ job_data.seq }}" - type: "{{ job_data.type }}" - phase: "{{ job_data.phase | default('New', true) }}" -{% endfor %} -{% endif %} -{% if masbr_backup_schedule is defined and masbr_backup_schedule | length > 0 %} -schedule: "{{ masbr_backup_schedule }}" -{% endif %} -status: - phase: "{{ masbr_job_status.phase | default('New', true) }}" - startTimestamp: "{{ masbr_job_status.startTimestamp | default('', true) }}" - completionTimestamp: "{{ masbr_job_status.completionTimestamp | default('', true) }}" -{% if masbr_job_status is defined - and masbr_job_status.sentNotifications is defined - and masbr_job_status.sentNotifications | length > 0 %} - sentNotifications: -{% for notification in masbr_job_status.sentNotifications %} - - type: "{{ notification.type }}" - channel: "{{ notification.channel }}" - timestamp: "{{ notification.timestamp }}" - phase: "{{ notification.phase }}" -{% endfor %} -{% endif %} diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.sh.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.sh.j2 deleted file mode 100644 index e07040d5be..0000000000 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.sh.j2 +++ /dev/null @@ -1,64 +0,0 @@ -namespace={{ masbr_cleanup_namespace }} -ttl=${MASBR_CLEANUP_TTL_SEC} - -current_time=$(date +%Y-%m-%dT%H:%M:%SZ) -current_ts=$(date +%s) - -echo "Start running cleanup job" -echo "Current time: ${current_time}" -echo "Current ts: ${current_ts}" -echo "TTL: ${ttl}s" - - -# Cleanup Jobs -job_names=($(oc get job -n ${namespace} --ignore-not-found=true --no-headers=true -l 'masbr-type in (backup,restore,schedule,copy)' | awk '{print $1}')) - -for job_name in ${job_names[@]}; do - echo "" - echo "Checking Job [ ${job_name} ] ..." - job_yaml=$(oc get job/${job_name} -n ${namespace} -o yaml) - job_complete=$(echo "${job_yaml}" | yq '.status.conditions.[] | select(.type == "Complete") | .status') - - if [[ "${job_complete}" == "True" ]]; then - job_complete_time=$(echo "${job_yaml}" | yq '.status.completionTime') - job_complete_ts=$(date +%s -d "${job_complete_time}") - echo "Job completion time: ${job_complete_time} (${job_complete_ts})" - - delta=$((current_ts - job_complete_ts)) - if [ ${delta} -gt ${ttl} ]; then - echo "Job exceed TTL (+$((delta - ttl))s)" - oc delete job/${job_name} -n ${namespace} - else - echo "Job not exceed TTL (-$((ttl - delta))s)" - fi - else - echo "Job not complete" - fi -done - - -# Cleanup ConfigMaps -cm_names=($(oc get cm -n ${namespace} --ignore-not-found=true --no-headers=true -l 'masbr-type in (backup,restore,schedule,copy)' | awk '{print $1}')) -for cm_name in ${cm_names[@]}; do - echo "" - echo "Checking ConfigMap [ ${cm_name} ] ..." - cm_yaml=$(oc get cm/${cm_name} -n ${namespace} -o yaml) - masbr_type=$(echo "${cm_yaml}" | yq '.metadata.labels.masbr-type') - - if [[ "${masbr_type}" == "schedule" ]]; then - if [[ "$(oc get cronjob -n ${namespace} --ignore-not-found=true --no-headers=true | grep ${cm_name} | wc -l)" == "0" ]]; then - echo "Not found related CronJob" - oc delete cm/${cm_name} -n ${namespace} - else - echo "Found related CronJob" - fi - - else - if [[ "$(oc get job -n ${namespace} --ignore-not-found=true --no-headers=true | grep ${cm_name} | wc -l)" == "0" ]]; then - echo "Not found related Job" - oc delete cm/${cm_name} -n ${namespace} - else - echo "Found related Job" - fi - fi -done diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.yml.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.yml.j2 deleted file mode 100644 index 144a9efb2c..0000000000 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/cleanup_job.yml.j2 +++ /dev/null @@ -1,83 +0,0 @@ ---- -kind: ServiceAccount -apiVersion: v1 -metadata: - name: "masbr-sa" - namespace: "{{ masbr_cleanup_namespace }}" - labels: - mas.ibm.com/masbr: "" - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: "masbr-{{ masbr_cleanup_namespace }}" - labels: - mas.ibm.com/masbr: "" -subjects: - - kind: ServiceAccount - name: "masbr-sa" - namespace: "{{ masbr_cleanup_namespace }}" -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin - ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: "masbr-network-policy" - namespace: "{{ masbr_cleanup_namespace }}" - labels: - mas.ibm.com/masbr: "" -spec: - podSelector: - matchLabels: - mas.ibm.com/masbr: "" - egress: - - {} - policyTypes: - - Egress - ---- -kind: CronJob -apiVersion: batch/v1 -metadata: - name: "masbr-cleanup" - namespace: "{{ masbr_cleanup_namespace }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "cleanup" -spec: - schedule: "{{ masbr_cleanup_schedule }}" -{% if masbr_job_timezone is defined and masbr_job_timezone | length > 0 %} - timeZone: "{{ masbr_job_timezone }}" -{% endif %} - successfulJobsHistoryLimit: 1 - jobTemplate: - spec: - backoffLimit: 1 - template: - metadata: - name: "masbr-cleanup" - labels: - mas.ibm.com/masbr: "" - masbr-type: "cleanup" - spec: - serviceAccountName: "masbr-sa" - containers: - - name: main - image: quay.io/ibmmas/cli:{{ masbr_mascli_image_tag }} -{% if masbr_mascli_image_pull_policy is defined and masbr_mascli_image_pull_policy | length > 0 %} - imagePullPolicy: "{{ masbr_mascli_image_pull_policy }}" -{% endif %} - command: - - sh - - '-c' - - >- - {{ masbr_cleanup_cmds }} -{% if masbr_cleanup_env is defined and masbr_cleanup_env | length > 0 %} - env: {{ masbr_cleanup_env }} -{% endif %} - restartPolicy: Never diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/copy_cloud_files_job.yml.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/copy_cloud_files_job.yml.j2 deleted file mode 100644 index d72c96c0d2..0000000000 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/copy_cloud_files_job.yml.j2 +++ /dev/null @@ -1,95 +0,0 @@ ---- -kind: NetworkPolicy -apiVersion: networking.k8s.io/v1 -metadata: - name: "masbr-network-policy" - namespace: "{{ masbr_cf_namespace }}" - labels: - mas.ibm.com/masbr: "" -spec: - podSelector: - matchLabels: - mas.ibm.com/masbr: "" - egress: - - {} - policyTypes: - - Egress - ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: "{{ masbr_cf_k8s_name }}" - namespace: "{{ masbr_cf_namespace }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "copy" - masbr-job: "{{ masbr_job_name }}" -spec: - backoffLimit: 1 - template: - metadata: - name: "{{ masbr_cf_k8s_name }}" - labels: - mas.ibm.com/masbr: "" - masbr-type: "copy" - masbr-job: "{{ masbr_job_name }}" - spec: -{% if masbr_cf_affinity is defined and masbr_cf_affinity %} - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: statefulset.kubernetes.io/pod-name - operator: In - values: - - "{{ masbr_cf_pod_name }}" - topologyKey: kubernetes.io/hostname -{% endif %} -{% if masbr_cf_service_account_name is defined and masbr_cf_service_account_name | length > 0 %} - serviceAccountName: "{{ masbr_cf_service_account_name }}" -{% endif %} -{% if masbr_cf_service_account is defined and masbr_cf_service_account | length > 0 %} - serviceAccount: "{{ masbr_cf_service_account }}" -{% endif %} -{% if masbr_cf_pod_security_context is defined and masbr_cf_pod_security_context | length > 0 %} - securityContext: {{ masbr_cf_pod_security_context }} -{% endif %} - containers: - - name: main - image: quay.io/ibmmas/cli:{{ masbr_mascli_image_tag }} -{% if masbr_mascli_image_pull_policy is defined and masbr_mascli_image_pull_policy | length > 0 %} - imagePullPolicy: "{{ masbr_mascli_image_pull_policy }}" -{% endif %} - command: - - sh - - '-c' - - >- - {{ masbr_cf_cmds }} -{% if masbr_cf_env is defined and masbr_cf_env | length > 0 %} - env: {{ masbr_cf_env }} -{% endif %} - volumeMounts: - - name: tmp - mountPath: /tmp - - name: data-volume - mountPath: "{{ masbr_cf_pvc_mount_path }}" -{% if masbr_cf_pvc_sub_path is defined and masbr_cf_pvc_sub_path | length > 0 %} - subPath: "{{ masbr_cf_pvc_sub_path }}" -{% endif %} - - name: cm-volume - mountPath: /mnt/configmap -{% if masbr_cf_container_security_context is defined and masbr_cf_container_security_context | length > 0 %} - securityContext: {{ masbr_cf_container_security_context }} -{% endif %} - restartPolicy: Never - volumes: - - name: tmp - emptyDir: {} - - name: data-volume - persistentVolumeClaim: - claimName: "{{ masbr_cf_pvc_name }}" - - name: cm-volume - configMap: - name: "{{ masbr_cf_k8s_name }}" diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/restore.yml.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/restore.yml.j2 deleted file mode 100644 index 8ad544b404..0000000000 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/restore.yml.j2 +++ /dev/null @@ -1,36 +0,0 @@ ---- -kind: Restore -name: "{{ masbr_job_name }}" -version: "{{ masbr_job_version }}" -from: "{{ masbr_restore_from }}" -target: - domain: "{{ masbr_cluster_domain }}" -{% if masbr_job_component is defined and masbr_job_component.items() %} -component: -{% for key, value in masbr_job_component.items() %} - {{ key }}: "{{ value }}" -{% endfor %} -{% endif %} -{% if masbr_job_data_list is defined and masbr_job_data_list | length > 0 %} -data: -{% for job_data in masbr_job_data_list %} - - seq: {{ job_data.seq }} - type: "{{ job_data.type }}" - phase: "{{ job_data.phase | default('New', true) }}" -{% endfor %} -{% endif %} -status: - phase: "{{ masbr_job_status.phase | default('New', true) }}" - startTimestamp: "{{ masbr_job_status.startTimestamp | default('', true) }}" - completionTimestamp: "{{ masbr_job_status.completionTimestamp | default('', true) }}" -{% if masbr_job_status is defined - and masbr_job_status.sentNotifications is defined - and masbr_job_status.sentNotifications | length > 0 %} - sentNotifications: -{% for notification in masbr_job_status.sentNotifications %} - - type: "{{ notification.type }}" - channel: "{{ notification.channel }}" - timestamp: "{{ notification.timestamp }}" - phase: "{{ notification.phase }}" -{% endfor %} -{% endif %} diff --git a/ibm/mas_devops/common_vars/backup_restore.yml b/ibm/mas_devops/common_vars/backup_restore.yml deleted file mode 100644 index 10c54bb15f..0000000000 --- a/ibm/mas_devops/common_vars/backup_restore.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# Job management -# ----------------------------------------------------------------------------- -# Whether to confirm the currently connected cluster before run tasks -masbr_confirm_cluster: "{{ lookup('env', 'MASBR_CONFIRM_CLUSTER') | default(false, true) | bool }}" - -# Copy file timeout in seconds (default timeout is 12 hours: 3600 * 12) -masbr_copy_timeout_sec: "{{ lookup('env', 'MASBR_COPY_TIMEOUT_SEC') | default(43200, true) | int }}" - -# Whether to allow multiple backup/restore jobs to run simultaneously -masbr_allow_multi_jobs: "{{ lookup('env', 'MASBR_ALLOW_MULTI_JOBS') | default(true, true) | bool }}" - -# Cron expression of cleanup Job (default to run at 1:00 every day) -# https://en.wikipedia.org/wiki/Cron -masbr_cleanup_schedule: "{{ lookup('env', 'MASBR_CLEANUP_SCHEDULE') | default('0 1 * * *', true) }}" - -# The completed Jobs that exceed this time-to-live in seconds will be deleted (default ttl is 1 week: 3600 * 24 * 7) -masbr_cleanup_ttl_sec: "{{ lookup('env', 'MASBR_CLEANUP_TTL_SEC') | default('604800', true) }}" - -# Time zone of CronJob -# https://en.wikipedia.org/wiki/List_of_tz_database_time_zones -masbr_job_timezone: "{{ lookup('env', 'MASBR_JOB_TIMEZONE') | default('', true) }}" - -# Docker image tag -# ----------------------------------------------------------------------------- -masbr_mascli_image_tag: "{{ lookup('env', 'MASBR_MASCLI_IMAGE_TAG') | default('latest', true) }}" -masbr_mascli_image_pull_policy: "{{ lookup('env', 'MASBR_MASCLI_IMAGE_PULL_POLICY') | default('', true) }}" - -# Storage variables -# ----------------------------------------------------------------------------- -# Local temp folder for backup/restore -masbr_local_temp_folder: "{{ lookup('env', 'MASBR_LOCAL_TEMP_FOLDER') | default('/tmp/masbr', true) }}" -masbr_storage_local_folder: "{{ lookup('env', 'MASBR_STORAGE_LOCAL_FOLDER') }}" - -# Notification variables -# ----------------------------------------------------------------------------- -# Supported notification levels: -# - 'verbose': send notifications when job in all phases 'InProgress', 'Completed', 'Failed', 'PartiallyFailed' -# - 'info': send job final results 'Completed', 'Failed', 'PartiallyFailed' -# - 'failure': sent notifications only when job in the phase 'Failed', 'PartiallyFailed' -masbr_notification_levels: - verbose: - - "InProgress" - - "Completed" - - "Failed" - - "PartiallyFailed" - info: - - "Completed" - - "Failed" - - "PartiallyFailed" - failure: - - "Failed" - - "PartiallyFailed" diff --git a/ibm/mas_devops/common_vars/cp4d_supported_versions.yml b/ibm/mas_devops/common_vars/cp4d_supported_versions.yml index eaa8fb14ef..553872aee4 100644 --- a/ibm/mas_devops/common_vars/cp4d_supported_versions.yml +++ b/ibm/mas_devops/common_vars/cp4d_supported_versions.yml @@ -1,5 +1,7 @@ cpd_supported_versions: - "5.1.3" + - "5.2.0" -# Extract the cpd minor version as there are specific steps to run depending on the cpd minor version defined -cpd_minor_version: "{{ cpd_product_version | regex_search('(?<=)(.*)(?=..)') }}" +cpd_minor_version: "{{ cpd_product_version | regex_search('(?<=)(.*)(?=..)') }}" # extract the cpd minor version as there are specific steps to run depending on the cpd minor version defined +cpd_52_or_higher: "{{ cpd_minor_version is version('5.2','>=') }}" +cpd_51: "{{ cpd_minor_version is version('5.1') }}" diff --git a/ibm/mas_devops/meta/runtime.yml b/ibm/mas_devops/meta/runtime.yml index 4d0c9225e1..4492c39ab0 100644 --- a/ibm/mas_devops/meta/runtime.yml +++ b/ibm/mas_devops/meta/runtime.yml @@ -11,3 +11,13 @@ action_groups: - verify_core_version - verify_subscriptions - verify_workloads + - verify_storage_class + - backup_mongo_instance + - get_mongoce_info + - verify_backup_restore_vars + - get_mongodb_cr_to_restore + - restore_mongoce_resources + - verify_mongoce_version + - wait_for_app_ready + - wait_for_conditions + diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml deleted file mode 100644 index 1039ab6e15..0000000000 --- a/ibm/mas_devops/playbooks/br_core.yml +++ /dev/null @@ -1,76 +0,0 @@ -- name: "Backup/Restore MAS Core" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "core" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-core" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "MAS_INSTANCE_ID is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "MASBR_ACTION is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "core" - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml deleted file mode 100644 index d285c59175..0000000000 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ /dev/null @@ -1,80 +0,0 @@ -- name: "Backup/Restore Db2 for MAS" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - db2_instance_id: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "db2" - instance: "{{ db2_instance_id }}" - namespace: "{{ db2_namespace }}" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "MAS_INSTANCE_ID is required" - - - name: "Fail if db2_instance_id is not provided" - assert: - that: db2_instance_id is defined and db2_instance_id != "" - fail_msg: "DB2_INSTANCE_NAME is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "MASBR_ACTION is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - # Run backup/restore tasks locally - # ------------------------------------------------------------------------- - - name: "Db2 {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" diff --git a/ibm/mas_devops/playbooks/br_health.yml b/ibm/mas_devops/playbooks/br_health.yml deleted file mode 100644 index e82b22fe2b..0000000000 --- a/ibm/mas_devops/playbooks/br_health.yml +++ /dev/null @@ -1,107 +0,0 @@ -- name: "Backup/Restore Maximo Health" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - db2_instance_id: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" - db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "health" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-manage" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "MAS_INSTANCE_ID is required" - - - name: "Fail if mas_workspace_id is not provided" - assert: - that: mas_workspace_id is defined and mas_workspace_id != "" - fail_msg: "MAS_WORKSPACE_ID is required" - - - name: "Fail if db2_instance_id is not provided" - assert: - that: db2_instance_id is defined and db2_instance_id != "" - fail_msg: "DB2_INSTANCE_NAME is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "MASBR_ACTION is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "health" - - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - - - name: "Manage namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "manage" - - - name: "Health namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "health" diff --git a/ibm/mas_devops/playbooks/br_iot.yml b/ibm/mas_devops/playbooks/br_iot.yml deleted file mode 100644 index 1d5ebdd9e3..0000000000 --- a/ibm/mas_devops/playbooks/br_iot.yml +++ /dev/null @@ -1,101 +0,0 @@ -- name: "Backup/Restore IoT tool" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - db2_instance_id: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" - db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "iot" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-iot" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - - - name: "Fail if mas_workspace_id is not provided" - assert: - that: mas_workspace_id is defined and mas_workspace_id != "" - fail_msg: "mas_workspace_id is required" - - - name: "Fail if db2_instance_id is not provided" - assert: - that: db2_instance_id is defined and db2_instance_id != "" - fail_msg: "db2_instance_id is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "iot" - - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - - - name: "IoT namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "iot" diff --git a/ibm/mas_devops/playbooks/br_manage.yml b/ibm/mas_devops/playbooks/br_manage.yml deleted file mode 100644 index 96f364b62f..0000000000 --- a/ibm/mas_devops/playbooks/br_manage.yml +++ /dev/null @@ -1,97 +0,0 @@ -- name: "Backup/Restore Maximo Manage" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - db2_instance_id: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" - db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "manage" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-manage" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - - - name: "Fail if mas_workspace_id is not provided" - assert: - that: mas_workspace_id is defined and mas_workspace_id != "" - fail_msg: "mas_workspace_id is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "manage" - - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" - when: db2_instance_id is defined and db2_instance_id != "" - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - - - name: "Manage namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "manage" diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml deleted file mode 100644 index 9fa399eae7..0000000000 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ /dev/null @@ -1,74 +0,0 @@ -- name: "Backup/Restore MongoDB for MAS" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" - mongodb_provider: "{{ lookup('env', 'MONGODB_PROVIDER') | default('community', true) }}" - mongodb_namespace: "{{ lookup('env', 'MONGODB_NAMESPACE') | default('mongoce', true) }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "mongodb" - instance: "{{ mas_instance_id }}" - namespace: "{{ mongodb_namespace }}" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" diff --git a/ibm/mas_devops/playbooks/br_monitor.yml b/ibm/mas_devops/playbooks/br_monitor.yml deleted file mode 100644 index d3835d1c9b..0000000000 --- a/ibm/mas_devops/playbooks/br_monitor.yml +++ /dev/null @@ -1,107 +0,0 @@ -- name: "Backup/Restore Maximo Monitor" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - db2_instance_id: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" - db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "monitor" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-monitor" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - - - name: "Fail if mas_workspace_id is not provided" - assert: - that: mas_workspace_id is defined and mas_workspace_id != "" - fail_msg: "mas_workspace_id is required" - - - name: "Fail if db2_instance_id is not provided" - assert: - that: db2_instance_id is defined and db2_instance_id != "" - fail_msg: "db2_instance_id is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "monitor" - - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - - - name: "IoT namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "iot" - - - name: "Monitor namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "monitor" diff --git a/ibm/mas_devops/playbooks/br_optimizer.yml b/ibm/mas_devops/playbooks/br_optimizer.yml deleted file mode 100644 index cece75a761..0000000000 --- a/ibm/mas_devops/playbooks/br_optimizer.yml +++ /dev/null @@ -1,107 +0,0 @@ -- name: "Backup/Restore Maximo Optimizer" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - db2_instance_id: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" - db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "optimizer" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-optimizer" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - - - name: "Fail if mas_workspace_id is not provided" - assert: - that: mas_workspace_id is defined and mas_workspace_id != "" - fail_msg: "mas_workspace_id is required" - - - name: "Fail if db2_instance_id is not provided" - assert: - that: db2_instance_id is defined and db2_instance_id != "" - fail_msg: "db2_instance_id is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "optimizer" - - - name: "Db2: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.db2 - vars: - db2_action: "{{ masbr_action }}" - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - - - name: "Manage namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "manage" - - - name: "Optimizer namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "optimizer" diff --git a/ibm/mas_devops/playbooks/br_visualinspection.yml b/ibm/mas_devops/playbooks/br_visualinspection.yml deleted file mode 100644 index 4023ad752f..0000000000 --- a/ibm/mas_devops/playbooks/br_visualinspection.yml +++ /dev/null @@ -1,88 +0,0 @@ -- name: "Backup/Restore Maximo Visual Inspection" - hosts: localhost - any_errors_fatal: true - - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - - # Define what action to perform - masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - - # Define what to backup/restore - masbr_job_component: - name: "visualinspection" - instance: "{{ mas_instance_id }}" - namespace: "mas-{{ mas_instance_id }}-visualinspection" - - # Configure path to backup_restore tasks - role_path: "{{ [playbook_dir, '../common_tasks/backup_restore'] | path_join }}" - - pre_tasks: - # Display the notice that this is still a work in progress - # ------------------------------------------------------------------------- - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * The backup and restore playbooks in this collection are still * - * work in progress, they are not suitable for production use at * - * this time. * - * * - * You may track development progress using the Backup & Restore * - * label in the GitHub repository: * - * * - * https://ibm.biz/BdGnfb * - * * - * Production-ready backup and restore options are detailed in the * - * Backup and restore topic in the product documentation: * - * * - * https://ibm.biz/BdGnf3 * - * * - ********************************************************************* - - # Check for required environment variables - # ------------------------------------------------------------------------- - - name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - - - name: "Fail if mas_workspace_id is not provided" - assert: - that: mas_workspace_id is defined and mas_workspace_id != "" - fail_msg: "mas_workspace_id is required" - - - name: "Fail if masbr_action is not set to backup|restore" - assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" - - # Common checks before run tasks - # ------------------------------------------------------------------------- - - name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - tasks: - - name: "MongoDB: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mas_app_id: "visualinspection" - - - name: "MAS Core namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_backup_restore - - - name: "Visual Inspection namespace: {{ masbr_action }}" - include_role: - name: ibm.mas_devops.suite_app_backup_restore - vars: - mas_app_id: "visualinspection" diff --git a/ibm/mas_devops/playbooks/mirror_dependencies.yml b/ibm/mas_devops/playbooks/mirror_dependencies.yml index 91f8df1bda..b496e22325 100644 --- a/ibm/mas_devops/playbooks/mirror_dependencies.yml +++ b/ibm/mas_devops/playbooks/mirror_dependencies.yml @@ -35,6 +35,8 @@ # ------------------------------------------------------------------------- cpd_product_version: "{{ lookup('env', 'CPD_PRODUCT_VERSION') }}" mirror_cp4d: "{{ lookup('env', 'MIRROR_CP4D') | default ('False', True) | bool }}" + cpd_52_or_higher: "{{ cpd_product_version is defined and cpd_product_version is version('5.2.0','>=') | bool }}" + cpd_51: "{{ cpd_product_version is defined and cpd_product_version is version('5.1.3','==') | bool }}" # 7. Watson Studio Local # ------------------------------------------------------------------------- @@ -378,6 +380,28 @@ manifest_name: ibm-ccs manifest_version: "{{ mas_catalog_metadata.wsl_version }}" + # 6.4 CP4D Platform - PostgreSQL dependency (only for CPD 5.2+) + # ------------------------------------------------------------------------- + - role: ibm.mas_devops.mirror_case_prepare + when: + - mirror_cp4d + - mirror_mode != "from-filesystem" + - cpd_52_or_higher + vars: + case_name: ibm-cloud-native-postgresql + case_version: "{{ mas_catalog_metadata.postgress_version }}" + exclude_images: [] + ibmpak_skip_dependencies: true + + - role: ibm.mas_devops.mirror_images + when: + - mirror_cp4d + - mirror_mode != "from-filesystem" + - cpd_52_or_higher + vars: + manifest_name: ibm-cloud-native-postgresql + manifest_version: "{{ mas_catalog_metadata.postgress_version }}" + # 7.1 IBM Watson Studio dependency - ibm-datarefinery # ------------------------------------------------------------------------- - role: ibm.mas_devops.mirror_case_prepare diff --git a/ibm/mas_devops/plugins/action/backup_mongo_instance.py b/ibm/mas_devops/plugins/action/backup_mongo_instance.py new file mode 100644 index 0000000000..d1974e5bb9 --- /dev/null +++ b/ibm/mas_devops/plugins/action/backup_mongo_instance.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +import yaml +import os + +from mas.devops.ocp import getCR, getSecret + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +display = Display() + +def display_information(mongoDBCommunityCR : dict): + display.v(f"MongoCE instance name .......................... {mongoDBCommunityCR['metadata']['name']}") + display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") + display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") + +def get_tlscertkey_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str: + if 'security' in mongoDBCommunityCR['spec']: + if 'tls' in mongoDBCommunityCR['spec']['security']: + if 'certificateKeySecretRef' in mongoDBCommunityCR['spec']['security']['tls']: + return mongoDBCommunityCR['spec']['security']['tls']['certificateKeySecretRef']['name'] + else: + return None + +def get_usersecrets_from_mongoce(mongoDBCommunityCR : dict) -> list: + user_secrets = [] + for user in mongoDBCommunityCR['spec'].get('users', []): + if 'passwordSecretRef' in user: + user_secrets.append(user['passwordSecretRef']['name']) + if 'scramCredentialsSecretName' in user: + user_secrets.append(f"{user['scramCredentialsSecretName']}-scram-credentials") + return user_secrets + +def get_prometheus_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str: + if 'prometheus' in mongoDBCommunityCR['spec']: + if 'passwordSecretRef' in mongoDBCommunityCR['spec']['prometheus']: + return mongoDBCommunityCR['spec']['prometheus']['passwordSecretRef']['name'] + else: + return None + +def getMongoVersionFromCR(mongoCR: dict) -> str: + """ + Get MongoDB version from MongoDB Community CR + """ + if 'spec' in mongoCR: + if 'version' in mongoCR['spec']: + return mongoCR['spec']['version'] + return "" + +def copyContentsToYamlFile(file_path: str, content: dict) -> bool: + """ + Write dictionary content to a YAML file + """ + try: + with open(file_path, 'w') as yaml_file: + yaml.dump(content, yaml_file, default_flow_style=False) + return True + except Exception as e: + display.v(f"Error writing to YAML file {file_path}: {e}") + return False + +def filterMongoCR(crData: dict) -> dict: + """ + Filter out unnecessary fields from a Custom Resource + """ + metadata_fields_to_remove = [ + 'annotations', + 'creationTimestamp', + 'generation', + 'resourceVersion', + 'selfLink', + 'uid', + 'managedFields' + ] + filteredCR = crData.copy() + if 'metadata' in filteredCR: + for field in metadata_fields_to_remove: + if field in filteredCR['metadata']: + del filteredCR['metadata'][field] + # remove status field + if 'status' in filteredCR: + del filteredCR['status'] + + # remove replicaSetHorizons field from spec if exists + if 'spec' in filteredCR: + if 'replicaSetHorizons' in filteredCR['spec']: + del filteredCR['spec']['replicaSetHorizons'] + + return filteredCR + +def filterResourceData(data: dict) -> dict: + """ + filter metadata from Resource data and create minimal dict + """ + metadata_fields_to_remove = [ + 'annotations', + 'creationTimestamp', + 'generation', + 'resourceVersion', + 'selfLink', + 'uid', + 'managedFields' + ] + filteredCopy = data.copy() + if 'metadata' in filteredCopy: + for field in metadata_fields_to_remove: + if field in filteredCopy['metadata']: + del filteredCopy['metadata'][field] + + if 'status' in filteredCopy: + del filteredCopy['status'] + + return filteredCopy + +def backupSecret(dynClient: DynamicClient, namespace: str, secret_name: str, backup_path: str) -> bool: + """ + Backup a Secret to a YAML file + """ + secret = getSecret(dynClient, namespace, secret_name) + if secret: + secret_file_path = f"{backup_path}/{secret_name}.yaml" + filtered_secret = filterResourceData(secret) + if copyContentsToYamlFile(secret_file_path, filtered_secret): + display.v(f"Successfully backed up Secret '{secret_name}' to '{secret_file_path}'") + return True + else: + display.v(f"Failed to back up Secret '{secret_name}' to '{secret_file_path}'") + return False + else: + display.v(f"Secret '{secret_name}' not found in namespace '{namespace}', skipping backup") + return False + +def isMongoRunning(mongoCR: dict) -> bool: + """ + Check if MongoDB Community instance is running + return True if running, else False + """ + display.v(f"Checking if MongoDB Community instance is in 'Running' state") + if 'status' in mongoCR: + if 'phase' in mongoCR['status']: + if mongoCR['status']['phase'] == 'Running': + display.v(f"MongoDB Community instance is in 'Running' state") + return True + display.v(f"MongoDB Community instance is not in 'Running' state") + return False + +def getMongoceCR(dynClient: DynamicClient, mongodb_instance_name: str, mongodb_namespace: str) -> dict: + """ + Check if MongoDB Community instance exists + return cr if exists, else return empty dict + """ + display.v(f"Checking if MongoDB Community instance '{mongodb_instance_name}' exists in namespace '{mongodb_namespace}'") + mongodbCR = getCR( + dynClient=dynClient, + cr_api_version="mongodbcommunity.mongodb.com/v1", + cr_kind="MongoDBCommunity", + cr_name=mongodb_instance_name, + namespace=mongodb_namespace + ) + if mongodbCR: + return mongodbCR.to_dict() + else: + return {} + +def backupIssuersInNamespace(dynClient: DynamicClient, namespace: str, backup_path: str) -> bool: + """ + Backup all Issuers in a namespace + """ + display.v(f"Backing up Issuers in namespace '{namespace}' to '{backup_path}'") + try: + issuerAPI = dynClient.resources.get(api_version="cert-manager.io/v1", kind="Issuer") + issuers = issuerAPI.get(namespace=namespace) + + for issuer in issuers.items: + issuer_name = issuer["metadata"]["name"] + issuer_file_path = f"{backup_path}/{issuer_name}.yaml" + filtered_issuer = filterResourceData(issuer.to_dict()) + if copyContentsToYamlFile(issuer_file_path, filtered_issuer): + display.v(f"Successfully backed up Issuer '{issuer_name}' to '{issuer_file_path}'") + else: + display.v(f"Failed to back up Issuer '{issuer_name}' to '{issuer_file_path}'") + return False + return True + + except NotFoundError: + display.v(f"No Issuers found in namespace {namespace}") + + return False + +def backupCertificatesInNamespace(dynClient: DynamicClient, namespace: str, backup_path: str) -> bool: + """ + Backup all Certificates in a namespace + """ + display.v(f"Backing up Certificates in namespace '{namespace}' to '{backup_path}'") + try: + certificateAPI = dynClient.resources.get(api_version="cert-manager.io/v1", kind="Certificate") + certificates = certificateAPI.get(namespace=namespace) + + for certificate in certificates.items: + certificate_name = certificate["metadata"]["name"] + certificate_file_path = f"{backup_path}/{certificate_name}.yaml" + filtered_certificate = filterResourceData(certificate.to_dict()) + if copyContentsToYamlFile(certificate_file_path, filtered_certificate): + display.v(f"Successfully backed up Certificate '{certificate_name}' to '{certificate_file_path}'") + else: + display.v(f"Failed to back up Certificate '{certificate_name}' to '{certificate_file_path}'") + return False + return True + + except NotFoundError: + display.v(f"No Certificates found in namespace {namespace}") + + return False + +def backupMongoCRContents(dynClient: DynamicClient, cr_data: dict, backup_path: str) -> bool: + """ + Backup MongoDB Community CR contents to a YAML file + """ + display.v(f"Backing up MongoDB Community CR contents to '{backup_path}/cr.yaml'") + cr_file_path = f"{backup_path}/cr.yaml" + filtered_cr = filterMongoCR(cr_data) + if copyContentsToYamlFile(cr_file_path, filtered_cr): + display.v(f"Successfully backed up MongoDB Community CR to '{cr_file_path}'") + return True + else: + display.v(f"Failed to back up MongoDB Community CR to '{cr_file_path}'") + return False + +def getMongoVersion(mongoCR: dict) -> str: + """ + Get MongoDB version from MongoDB Community CR + """ + if 'spec' in mongoCR: + if 'version' in mongoCR['spec']: + return mongoCR['spec']['version'] + return "" + +def createBackupDirectories(paths: list) -> bool: + """ + Create backup directories if they do not exist + """ + try: + for path in paths: + os.makedirs(path, exist_ok=True) + display.v(f"Created backup directory: {path}") + return True + except Exception as e: + display.v(f"Error creating backup directories: {e}") + return False +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Backup MongoDB instance resources (CR, secrets, configmaps, issuers, certificates)" + ibm.mas_devops.backup_mongo_cr: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + mongodb_instance_name = self._task.args.get('mongodb_instance_name') + mongodb_namespace = self._task.args.get('mongodb_namespace') + mongodb_backup_path = self._task.args.get('mongodb_backup_path') + + if mongodb_instance_name is None or mongodb_instance_name == "": + raise AnsibleError(f"Error: mongodb_instance_name argument was not provided") + if mongodb_namespace is None or mongodb_namespace == "": + raise AnsibleError(f"Error: mongodb_namespace argument was not provided") + if mongodb_backup_path is None or mongodb_backup_path == "": + raise AnsibleError(f"Error: mongodb_backup_path argument was not provided") + + display.v(f"Backing up MongoDB Community instance '{mongodb_instance_name}' in namespace '{mongodb_namespace}'") + + # Prepare backup resource path + mongodb_backup_resource_path = f"{mongodb_backup_path}/resources" + mongodb_backup_secrets_path = f"{mongodb_backup_resource_path}/secrets" + mongodb_backup_issuers_path = f"{mongodb_backup_resource_path}/issuers" + mongodb_backup_certificates_path = f"{mongodb_backup_resource_path}/certificates" + # Create backup directories + createBackupDirectories([mongodb_backup_resource_path, mongodb_backup_secrets_path, mongodb_backup_issuers_path, mongodb_backup_certificates_path]); + + # ======================================================= + # 1. Backup MongoDB Community CR + # ======================================================= + mongodb_cr = getMongoceCR(dynClient, mongodb_instance_name, mongodb_namespace) + if not mongodb_cr: + raise AnsibleError(f"Error: MongoDB Community instance '{mongodb_instance_name}' does not exist in namespace '{mongodb_namespace}'") + else: + display.v(f"MongoDB Community instance '{mongodb_instance_name}' exists in namespace '{mongodb_namespace}'") + + if not isMongoRunning(mongodb_cr): + raise AnsibleError(f"Error: MongoDB Community instance '{mongodb_instance_name}' is not in 'Running' state") + + is_cr_backup = backupMongoCRContents(dynClient, mongodb_cr, mongodb_backup_resource_path) + if not is_cr_backup: + raise AnsibleError(f"Error: Failed to back up MongoDB Community CR for instance '{mongodb_instance_name}'") + + display.v(f"Successfully backed up MongoDB Community CR to '{mongodb_backup_resource_path}/cr.yaml'") + + # =============================================================== + # 2. Backup MongoDB namespace Secrets defined in the MongoDB CR + # =============================================================== + display.v(f"Backing up MongoDB Secrets defined in the MongoDB CR from namespace '{mongodb_namespace}'") + # Get all relevant User Secret names from the MongoDB CR + user_secret_names = get_usersecrets_from_mongoce(mongodb_cr) + + # Get Prometheus and TLS Secret names from the MongoDB CR + prometheus_secret_names = get_prometheus_secretname_from_mongoce(mongodb_cr) + tls_secret_names = get_tlscertkey_secretname_from_mongoce(mongodb_cr) + + secret_names = user_secret_names + [prometheus_secret_names] + [tls_secret_names] + + display.v(f"All Secrets to backup: '{secret_names}'") + + # Backup each Secret + for secret_name in secret_names: + if not backupSecret(dynClient, mongodb_namespace, secret_name, f"{mongodb_backup_secrets_path}"): + raise AnsibleError(f"Error: Failed to back up Secret '{secret_name}'") + + display.v(f"Successfully backed up all Secrets in namespace '{mongodb_namespace}' to '{mongodb_backup_secrets_path}'") + + # ======================================================= + # 3. Backup Issuers in the MongoDB namespace + # ======================================================= + is_issuers_backup = backupIssuersInNamespace(dynClient, mongodb_namespace, f"{mongodb_backup_issuers_path}") + if not is_issuers_backup: + raise AnsibleError(f"Error: Failed to back up Issuers in namespace '{mongodb_namespace}'") + display.v(f"Successfully backed up Issuers in namespace '{mongodb_namespace}' to '{mongodb_backup_issuers_path}'") + + # ======================================================= + # Backup Certificates in the MongoDB namespace + # ======================================================= + is_certificates_backup = backupCertificatesInNamespace(dynClient, mongodb_namespace, f"{mongodb_backup_certificates_path}") + if not is_certificates_backup: + raise AnsibleError(f"Error: Failed to back up Certificates in namespace '{mongodb_namespace}'") + display.v(f"Successfully backed up Certificates in namespace '{mongodb_namespace}' to '{mongodb_backup_certificates_path}'") + + return dict( + message=f"Successfully backed up MongoDB Community instance '{mongodb_instance_name}' resources", + failed=False, + changed=False, + success=True, + ansible_facts={ + "mongodb_version": getMongoVersion(mongodb_cr) # this fact can be used in subsequent tasks + } + ) + + diff --git a/ibm/mas_devops/plugins/action/get_mongoce_info.py b/ibm/mas_devops/plugins/action/get_mongoce_info.py new file mode 100644 index 0000000000..9d46bd9f22 --- /dev/null +++ b/ibm/mas_devops/plugins/action/get_mongoce_info.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +import yaml +import base64 + +from mas.devops.ocp import getCR, getSecret + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +display = Display() + +def display_information(mongoDBCommunityCR : dict): + display.v(f"MongoCE instance name .......................... {mongoDBCommunityCR['metadata']['name']}") + display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") + display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") + +def get_mongoce_admin_secretname(mongoDBCommunityCR): + """ + short_description: Get admin secret name from MongoDBCommunity CR + """ + for user in mongoDBCommunityCR['spec']['users']: + if 'db' in user: + if user['db'] == 'admin' and 'passwordSecretRef' in user: + return user['passwordSecretRef']['name'] + return None + +def get_tlscertkey_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str: + if 'security' in mongoDBCommunityCR['spec']: + if 'tls' in mongoDBCommunityCR['spec']['security']: + if 'certificateKeySecretRef' in mongoDBCommunityCR['spec']['security']['tls']: + return mongoDBCommunityCR['spec']['security']['tls']['certificateKeySecretRef']['name'] + else: + return None + +def get_usersecrets_from_mongoce(mongoDBCommunityCR : dict) -> list: + user_secrets = [] + for user in mongoDBCommunityCR['spec'].get('users', []): + if 'passwordSecretRef' in user: + user_secrets.append(user['passwordSecretRef']['name']) + if 'scramCredentialsSecretName' in user: + user_secrets.append(f"{user['scramCredentialsSecretName']}-scram-credentials") + return user_secrets + +def isMongoRunning(mongoCR: dict) -> bool: + """ + Check if MongoDB Community instance is running + return True if running, else False + """ + display.v(f"Checking if MongoDB Community instance is in 'Running' state") + if 'status' in mongoCR: + if 'phase' in mongoCR['status']: + if mongoCR['status']['phase'] == 'Running': + display.v(f"MongoDB Community instance is in 'Running' state") + return True + display.v(f"MongoDB Community instance is not in 'Running' state") + return False + +def getMongoceCR(dynClient: DynamicClient, mongodb_instance_name: str, mongodb_namespace: str) -> dict: + """ + Check if MongoDB Community instance exists + return cr if exists, else return empty dict + """ + display.v(f"Checking if MongoDB Community instance '{mongodb_instance_name}' exists in namespace '{mongodb_namespace}'") + mongodbCR = getCR( + dynClient=dynClient, + cr_api_version="mongodbcommunity.mongodb.com/v1", + cr_kind="MongoDBCommunity", + cr_name=mongodb_instance_name, + namespace=mongodb_namespace + ) + if mongodbCR: + return mongodbCR.to_dict() + else: + return {} + +def getMongoVersion(mongoCR: dict) -> str: + """ + Get MongoDB version from MongoDB Community CR + """ + if 'spec' in mongoCR: + if 'version' in mongoCR['spec']: + return mongoCR['spec']['version'] + return "" + +def getMongoDBServiceName(mongoCR: dict) -> str: + """ + Get MongoDB Service name from MongoDB Community CR + """ + if 'spec' in mongoCR: + if 'statefulSet' in mongoCR['spec']: + if 'spec' in mongoCR['spec']['statefulSet']: + if 'serviceName' in mongoCR['spec']['statefulSet']['spec']: + return mongoCR['spec']['statefulSet']['spec']['serviceName'] + return "" + +def getPodNameFromLabels(dynClient: DynamicClient, namespace: str, label_selector: str) -> str: + """ + Get Pod name from labels + """ + display.v(f"Looking up Mongo Pod in namespace '{namespace}' with labels '{label_selector}'") + try: + podAPI = dynClient.resources.get(api_version="v1", kind="Pod") + pods = podAPI.get(namespace=namespace, label_selector=label_selector) + if pods.items: + pod_name = pods.items[0]["metadata"]["name"] + display.v(f"Found Pod '{pod_name}' in namespace '{namespace}' with labels '{label_selector}'") + return pod_name + else: + display.v(f"No Pods found in namespace '{namespace}' with labels '{label_selector}'") + except NotFoundError: + display.v(f"No Pods found in namespace '{namespace}' with labels '{label_selector}'") + return "" + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Retrieve info from MongoDB instance CR and resources" + ibm.mas_devops.get_mongoce_info: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + mongodb_instance_name = self._task.args.get('mongodb_instance_name') + mongodb_namespace = self._task.args.get('mongodb_namespace') + + if mongodb_instance_name is None: + raise AnsibleError(f"Error: mongodb_instance_name argument was not provided") + if mongodb_namespace is None: + raise AnsibleError(f"Error: mongodb_namespace argument was not provided") + + display.v(f"Retrieving MongoDB Community instance '{mongodb_instance_name}' in namespace '{mongodb_namespace}'") + + # 1. Check if MongoDB Community instance exists + mongodb_cr = getMongoceCR(dynClient, mongodb_instance_name, mongodb_namespace) + if not mongodb_cr: + raise AnsibleError(f"Error: MongoDB Community instance '{mongodb_instance_name}' does not exist in namespace '{mongodb_namespace}'") + else: + display.v(f"MongoDB Community instance '{mongodb_instance_name}' exists in namespace '{mongodb_namespace}'") + + # 2. Check if MongoDB Community instance is in 'Running' state + if not isMongoRunning(mongodb_cr): + raise AnsibleError(f"Error: MongoDB Community instance '{mongodb_instance_name}' is not in 'Running' state") + + display_information(mongodb_cr) + + # 3. Lookup mongoce Pod to retrieve mongo pod name + mongoce_pod_name = getPodNameFromLabels( + dynClient=dynClient, + namespace=mongodb_namespace, + label_selector="apps.kubernetes.io/pod-index=0" + ) + if not mongoce_pod_name: + raise AnsibleError(f"Error: Could not find Pod for MongoDB Community instance '{mongodb_instance_name}' in namespace '{mongodb_namespace}'") + + # 4. Retrieve MongoDB admin Secret data + mongodb_admin_secretname = get_mongoce_admin_secretname(mongodb_cr) + mongodb_admin_secret = getSecret( + dynClient=dynClient, + namespace=mongodb_namespace, + secret_name=mongodb_admin_secretname + ) + if not mongodb_admin_secret: + raise AnsibleError(f"Error: Could not find admin Secret '{mongodb_admin_secretname}' for MongoDB Community instance '{mongodb_instance_name}' in namespace '{mongodb_namespace}'") + + # 5. Retrieve mongodb service name + mongodb_service_name = getMongoDBServiceName(mongodb_cr) + if not mongodb_service_name: + raise AnsibleError(f"Error: Could not find MongoDB Service name for MongoDB Community instance '{mongodb_instance_name}' in namespace '{mongodb_namespace}'") + + # 6. Construct mongodb_host + mongodb_host = f"{mongoce_pod_name}.{mongodb_service_name}.{mongodb_namespace}.svc.cluster.local:27017" + + mongo_info = dict( + mongoce_pod_name=mongoce_pod_name, + mongodb_admin_user="admin", + mongodb_admin_password = base64.b64decode(mongodb_admin_secret['data']['password']), + mongodb_service_name=mongodb_service_name, + mongodb_host=mongodb_host, + mongodb_version=getMongoVersion(mongodb_cr) + ) + + return dict( + message=f"Successfully set facts from MongoDB Community instance '{mongodb_instance_name}' resources", + failed=False, + changed=False, + success=True, + **mongo_info + ) + + diff --git a/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py b/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py new file mode 100644 index 0000000000..6ed9597bc4 --- /dev/null +++ b/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +import yaml +import base64 +import os + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +display = Display() + +def display_information(mongoDBCommunityCR : dict): + display.v(f"MongoCE instance name .......................... {mongoDBCommunityCR['metadata']['name']}") + display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") + display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") + + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Retrieve and Set facts from MongoDB instance CR and resources" + ibm.mas_devops.get_mongodb_cr_to_restore: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + mongodb_resource_path = self._task.args.get('mongodb_resource_path') + mongodb_backup_version = self._task.args.get('mongodb_backup_version') + + if mongodb_resource_path is None or mongodb_resource_path == "": + raise AnsibleError(f"Error: mongodb_instance_name argument was not provided") + + if mongodb_backup_version is None or mongodb_backup_version == "": + raise AnsibleError(f"Error: mongodb_backup_version argument was not provided") + + # check if backup directory exists + display.v(f"Checking if MongoDB backup directory exists at path: {mongodb_resource_path}") + if not os.path.isdir(mongodb_resource_path): + raise AnsibleError(f"Directory {mongodb_resource_path} does NOT exist!") + display.v(f"MongoDB backup directory exists at path: {mongodb_resource_path}") + + # check if cr.yaml exists + mongodb_cr_file = f"{mongodb_resource_path}/cr.yaml" + display.v(f"Checking if MongoDB backup CR file exists at path: {mongodb_cr_file}") + if not os.path.isfile(mongodb_cr_file): + raise AnsibleError(f"MongoDB backup CR file does NOT exist at path: {mongodb_cr_file}") + display.v(f"MongoDB backup CR file exists at path: {mongodb_cr_file}") + + # read cr.yaml + with open(mongodb_cr_file, 'r') as cr_file: + mongodb_cr = yaml.safe_load(cr_file) + display.v("Successfully read MongoDB backup CR file") + + + return dict( + message=f"Successfully set mongodb CR as facts from backup at '{mongodb_resource_path}'", + failed=False, + changed=False, + success=True, + mongodb_cr= mongodb_cr + ) + + diff --git a/ibm/mas_devops/plugins/action/restore_mongoce_resources.py b/ibm/mas_devops/plugins/action/restore_mongoce_resources.py new file mode 100644 index 0000000000..c69b50d396 --- /dev/null +++ b/ibm/mas_devops/plugins/action/restore_mongoce_resources.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +import yaml +import os +from mas.devops.ocp import createNamespace, apply_resource + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +display = Display() + +def display_information(mongoDBCommunityCR : dict): + display.v(f"MongoCE instance name .......................... {mongoDBCommunityCR['metadata']['name']}") + display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") + display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Restore MongoDB instance resources (secrets, issuers, certificates)" + ibm.mas_devops.restore_mongoce_resources: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + mongodb_namespace = self._task.args.get('mongodb_namespace') + mongodb_backup_secrets_path = self._task.args.get('backup_secrets_path') + mongodb_backup_issuers_path = self._task.args.get('backup_issuers_path') + mongodb_backup_certs_path = self._task.args.get('backup_certificates_path') + + if mongodb_namespace is None: + raise AnsibleError(f"Error: mongodb_namespace argument was not provided") + if mongodb_backup_secrets_path is None: + raise AnsibleError(f"Error: mongodb_backup_secrets_path argument was not provided") + if mongodb_backup_issuers_path is None: + raise AnsibleError(f"Error: mongodb_backup_issuers_path argument was not provided") + if mongodb_backup_certs_path is None: + raise AnsibleError(f"Error: mongodb_backup_certs_path argument was not provided") + + # ======================================================= + # 1. Create MongoDB namespace if not exists + # ======================================================= + display.v(f"Creating MongoDB namespace '{mongodb_namespace}' if it does not already exist") + createNamespace(dynClient, mongodb_namespace) + + # ======================================================= + # 2. Restore MongoDB Secret resources from backup + # ======================================================= + display.v(f"Restoring MongoDB Secret resources from backup path '{mongodb_backup_secrets_path}'") + secret_files = os.listdir(mongodb_backup_secrets_path) + for secret_file in secret_files: + with open(os.path.join(mongodb_backup_secrets_path, secret_file), 'r') as f: + secret_yaml = f.read() + apply_resource(dynClient, secret_yaml, mongodb_namespace) + + # ======================================================= + # 3. Restore MongoDB Issuer resources from backup + # ======================================================= + display.v(f"Restoring MongoDB Issuer resources from backup path '{mongodb_backup_issuers_path}'") + issuer_files = os.listdir(mongodb_backup_issuers_path) + for issuer_file in issuer_files: + with open(os.path.join(mongodb_backup_issuers_path, issuer_file), 'r') as f: + issuer_yaml = f.read() + apply_resource(dynClient, issuer_yaml, mongodb_namespace) + + # ======================================================= + # 4. Restore MongoDB Certificate resources from backup + # ======================================================= + display.v(f"Restoring MongoDB Certificate resources from backup path '{mongodb_backup_certs_path}'") + cert_files = os.listdir(mongodb_backup_certs_path) + for cert_file in cert_files: + with open(os.path.join(mongodb_backup_certs_path, cert_file), 'r') as f: + cert_yaml = f.read() + apply_resource(dynClient, cert_yaml, mongodb_namespace) + + + return dict( + message=f"Successfully restored MongoDB instance's Secrets, Issuers and Certificates from backup paths.", + failed=False, + changed=False, + success=True, + restored=True + ) + + diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py new file mode 100644 index 0000000000..2d9d2e5fe9 --- /dev/null +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError + + +class ActionModule(ActionBase): + + REQUIRED = { + "mongodb": { + "backup": ["mongodb_instance_name", "mas_backup_dir", "mas_instance_id"], # mas_instance_id has a default value + "restore": ["mongodb_instance_name", "mas_backup_dir", "mongodb_backup_version"] + } + } + + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + component = self._task.args.get('component', None) + action = self._task.args.get('action', None) + + if component not in self.REQUIRED: + raise AnsibleError(f"Unknown component '{component}'. Allowed: {list(self.REQUIRED)}") + + if action not in self.REQUIRED[component]: + raise AnsibleError(f"Unknown action '{action}' for component '{component}'. Allowed: {list(self.REQUIRED[component])}") + + missing_args = [] + for req_arg in self.REQUIRED[component][action]: + r_arg = self._task.args.get(req_arg, None) + if r_arg is None or r_arg == '': + missing_args.append(req_arg) + + if len(missing_args) > 0: + raise AnsibleError(f"Missing required arguments for component '{component}' action '{action}': {missing_args}") + else: + return dict( + changed=False, + failed=False, + msg=f"All required arguments for component '{component}' action '{action}' are provided." + ) + + \ No newline at end of file diff --git a/ibm/mas_devops/plugins/action/verify_mongoce_version.py b/ibm/mas_devops/plugins/action/verify_mongoce_version.py new file mode 100644 index 0000000000..7f311cc88a --- /dev/null +++ b/ibm/mas_devops/plugins/action/verify_mongoce_version.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 + +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +from mas.devops.ocp import getStorageClass, getCR +from mas.devops.mas import getDefaultStorageClasses + + +# Disabling warnings will prevent InsecureRequestWarnings from dynClient +urllib3.disable_warnings() +display = Display() + +def getStorageClassFromCR(mongoCR: dict) -> str: + """ + Get Storage Class from MongoDB Community CR + spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName + """ + if mongoCR and 'spec' in mongoCR: + if 'statefulSet' in mongoCR['spec']: + if 'spec' in mongoCR['spec']['statefulSet']: + if 'volumeClaimTemplates' in mongoCR['spec']['statefulSet']['spec']: + if len(mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates']) > 0: + if 'spec' in mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates'][0]: + if 'storageClassName' in mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates'][0]['spec']: + return mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates'][0]['spec']['storageClassName'] + return "" + +def isMongoRunning(mongoCR: dict) -> bool: + """ + Check if MongoDB Community instance is running + return True if running, else False + """ + display.v(f"Checking if MongoDB Community instance is in 'Running' state") + if 'status' in mongoCR: + if 'phase' in mongoCR['status']: + if mongoCR['status']['phase'] == 'Running': + display.v(f"MongoDB Community instance is in 'Running' state") + return True + display.v(f"MongoDB Community instance is not in 'Running' state") + return False + +def getMongoceCR(dynClient: DynamicClient, mongodb_instance_name: str, mongodb_namespace: str) -> dict: + """ + Check if MongoDB Community instance exists + return cr if exists, else return empty dict + """ + display.v(f"Checking if MongoDB Community instance '{mongodb_instance_name}' exists in namespace '{mongodb_namespace}'") + mongodbCR = getCR( + dynClient=dynClient, + cr_api_version="mongodbcommunity.mongodb.com/v1", + cr_kind="MongoDBCommunity", + cr_name=mongodb_instance_name, + namespace=mongodb_namespace + ) + if mongodbCR: + return mongodbCR.to_dict() + else: + return {} + +def determineBestStorageClass(dynClient: DynamicClient, existing_storageclass: str, backup_storageclass: str) -> str: + """ + Determine best storage class, + 1. Picks existing storage class if exists, if not + 2. Picks storage class from backup if exists, if not + 3. Picks default rwo storage class. + """ + + best_sc = "" + # 1) Try existing storage class if provided + if existing_storageclass != "": + display.v(f"Checking existing storage class '{existing_storageclass}'") + sc = getStorageClass(dynClient, existing_storageclass) + if sc is not None: + best_sc = existing_storageclass + else: + display.v(f"Existing storage class '{existing_storageclass}' does not exist") + + # 2) Try backup storage class if existing not usable + if best_sc == "" and backup_storageclass != "": + display.v(f"Checking backup storage class '{backup_storageclass}'") + backup_sc = getStorageClass(dynClient, backup_storageclass) + if backup_sc is not None: + best_sc = backup_storageclass + else: + display.v(f"Backup storage class '{backup_storageclass}' does not exist") + + # 3) Fallback to default RWO storage class + if best_sc == "": + display.v("Falling back to default RWO storage class") + default_sc = getDefaultStorageClasses(dynClient) + if default_sc.provider is None: + raise AnsibleError("Error: Could not find default RWO storage class to use for MongoDB Community instance") + best_sc = default_sc.rwo + + display.v(f"Using storage class '{best_sc}' for MongoDB Community instance") + return best_sc + + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Verify Existing MongoDB version + ibm.mas_devops.verify_mongoce_version: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + mongodb_instance_name = self._task.args.get('mongodb_instance_name') + mongodb_namespace = self._task.args.get('mongodb_namespace') + backup_mongodb_storage_class = self._task.args.get('backup_mongodb_storage_class', None) + + if mongodb_instance_name is None: + raise AnsibleError(f"Error: mongodb_instance_name argument was not provided") + if mongodb_namespace is None: + raise AnsibleError(f"Error: mongodb_namespace argument was not provided") + + # Check for existing MongoDb install + mongodbCR = getMongoceCR( + dynClient=dynClient, + mongodb_instance_name=mongodb_instance_name, + mongodb_namespace=mongodb_namespace + ) + + # Determine best storage class to use + # skip storage class check if backup storage class is not provided + best_sc = "" + if backup_mongodb_storage_class is not None: + best_sc = determineBestStorageClass( + dynClient=dynClient, + existing_storageclass=getStorageClassFromCR(mongodbCR) if mongodbCR else "", + backup_storageclass=backup_mongodb_storage_class + ) + + if not mongodbCR: + display.v(f"MongoDB Community instance '{mongodb_instance_name}' does NOT exist in namespace '{mongodb_namespace}'") + return dict( + message=f"MongoDB Community instance '{mongodb_instance_name}' does NOT exist in namespace '{mongodb_namespace}'", + success=True, + failed=False, + exist=False, + running=False, + storage_class=best_sc + ) + elif isMongoRunning(mongodbCR): + display.v(f"MongoDB Community instance '{mongodb_instance_name}' is running version '{mongodbCR['spec']['version']}' in namespace '{mongodb_namespace}'") + return dict( + message=f"MongoDB Community instance '{mongodb_instance_name}' is running version '{mongodbCR['spec']['version']}' in namespace '{mongodb_namespace}'", + success=True, + failed=False, + exist=True, + running=True, + mongoce_version=mongodbCR['spec']['version'], + storage_class=getStorageClassFromCR(mongodbCR) + ) + else: + display.v(f"MongoDB Community instance '{mongodb_instance_name}' is NOT in 'Running' state in namespace '{mongodb_namespace}'") + return dict( + message=f"MongoDB Community instance '{mongodb_instance_name}' is NOT in 'Running' state in namespace '{mongodb_namespace}'", + success=True, + failed=False, + exist=True, + running=False, + storage_class=best_sc, + mongoce_version=mongodbCR['spec']['version'] + ) diff --git a/ibm/mas_devops/plugins/action/verify_storage_class.py b/ibm/mas_devops/plugins/action/verify_storage_class.py new file mode 100644 index 0000000000..ef2e88dfb9 --- /dev/null +++ b/ibm/mas_devops/plugins/action/verify_storage_class.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase + +from mas.devops.ocp import getStorageClass + + +# Disabling warnings will prevent InsecureRequestWarnings from dynClient +urllib3.disable_warnings() + + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Verify if Storage class exist + ibm.mas_devops.verify_storage_class: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + storageClass = self._task.args['storage_class'] + sc = getStorageClass(dynClient, storageClass) + + # We don't want to fail if we can't find the specific storage class, doing so will + # result in roles/playbooks failing in environments where none of the default + # storage classes are available. We use the success=false to track when we couldn't + # find a default storage class, which does not trigger Ansible treating the action as + # failed. + if sc is None: + return dict( + message=f"Failed to find {storageClass} storage class in cluster", + success=False, + failed=True, + name=storageClass + ) + + return dict( + message=f"Successfully found {storageClass} storage class in cluster", + success=True, + failed=False, + name=storageClass + ) diff --git a/ibm/mas_devops/plugins/action/wait_for_app_ready.py b/ibm/mas_devops/plugins/action/wait_for_app_ready.py new file mode 100644 index 0000000000..529d2a454d --- /dev/null +++ b/ibm/mas_devops/plugins/action/wait_for_app_ready.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display + +import urllib3 +import logging + +from mas.devops.mas import waitForAppReady + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + display = Display() + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + # Get task arguments + instanceId = self._task.args.get('instance_id', None) + applicationId = self._task.args.get('application_id', None) + workspaceId = self._task.args.get('workspace_id', None) + retries = self._task.args.get('retries', 50) + delay = self._task.args.get('delay', 60) + + if instanceId is None: + raise AnsibleError(f"Error: instance_id argument was not provided") + if applicationId is None: + raise AnsibleError(f"Error: application_id argument was not provided") + + resourceName = f"{instanceId}/{applicationId}" + if workspaceId is not None: + resourceName = f"{instanceId}/{applicationId}-{workspaceId}" + + isReady = waitForAppReady( + dynClient=dynClient, + instanceId=instanceId, + applicationId=applicationId, + workspaceId=workspaceId, + retries=retries, + delay=delay, + debugLogFunction=display.vv, + infoLogFunction=display.v + ) + + if isReady: + return dict( + message=f"Application {resourceName} is ready", + success=True, + changed=False + ) + else: + return dict( + message=f"Application {resourceName} is not ready", + success=False, + changed=False + ) diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index 1f3654739d..52181b585a 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -7,7 +7,7 @@ # ----------------------------------------------------------- import yaml import re - +import datetime def private_vlan(vlans): """ @@ -437,6 +437,25 @@ def get_ecr_repositories(image_mirror_output): repositories.append(repo_to_add) return repositories +def get_tlscert_configmapname_from_mongoce(mongoDBCommunityCR): + """ + filter: get_tlscert_configmapname_from_mongoce + author: Sanjay Prabhakar + version_added: 0.1 + short_description: Get the name of TLS Cert configmap from MongoDBCommunity CR + description: + - This filter returns the name of TLS Cert configmap from MongoDBCommunity CR + options: + mongoDBCommunityCR: + description: MongoDBCommunity CR definition + required: True + """ + if 'security' in mongoDBCommunityCR['spec']: + if 'tls' in mongoDBCommunityCR['spec']['security']: + if 'caConfigMapRef' in mongoDBCommunityCR['spec']['security']['tls']: + return mongoDBCommunityCR['spec']['security']['tls']['caConfigMapRef']['name'] + else: + return None class FilterModule(object): def filters(self): @@ -459,5 +478,6 @@ def filters(self): 'format_pre_version_without_buildid': format_pre_version_without_buildid, 'format_pre_version_with_buildid': format_pre_version_with_buildid, 'get_db2_instance_name': get_db2_instance_name, - 'get_ecr_repositories': get_ecr_repositories + 'get_ecr_repositories': get_ecr_repositories, + 'get_tlscert_configmapname_from_mongoce': get_tlscert_configmapname_from_mongoce } diff --git a/ibm/mas_devops/roles/aws_vpc/tasks/deprovision.yml b/ibm/mas_devops/roles/aws_vpc/tasks/deprovision.yml index 3ad01abf5b..ce69c0bbc7 100644 --- a/ibm/mas_devops/roles/aws_vpc/tasks/deprovision.yml +++ b/ibm/mas_devops/roles/aws_vpc/tasks/deprovision.yml @@ -31,14 +31,14 @@ - "VPC Resource {{ vpc_name }} doesn't exist in region {{ aws_region }}, Skipping deletion" - name: Delete VPC - when: vpc_id is defined and vpc_id != '' + when: vpc_id is defined and vpc_id != None and vpc_id != '' command: > aws ec2 delete-vpc \ --vpc-id '{{ vpc_id }}' register: delete_info - name: Debug ,Deleted VPC Info - when: vpc_id is defined and vpc_id != '' + when: vpc_id is defined and vpc_id != None and vpc_id != '' debug: msg: - "Deleted Vpc Info ............................ {{ delete_info }}" diff --git a/ibm/mas_devops/roles/cp4d/README.md b/ibm/mas_devops/roles/cp4d/README.md index fb11c15914..4dbbc8a847 100644 --- a/ibm/mas_devops/roles/cp4d/README.md +++ b/ibm/mas_devops/roles/cp4d/README.md @@ -18,6 +18,7 @@ Cloud Pak for Data will be configured as a [specialized installation](https://ww Currently supported Cloud Pak for Data release versions are: - 5.1.3 + - 5.2.0 !!! tip For more information about CPD versioning, see [IBM Cloud Pak for data Operator and operand versions 5.1.x](https://www.ibm.com/docs/en/software-hub/5.1.x?topic=planning-operator-operand-versions) @@ -33,6 +34,7 @@ Upgrade The role will automatically install or upgrade (if targeted to an existing CPD deployment) the corresponding Zen version associated to the chosen Cloud Pak for Data release, for example: - Cloud Pak for Data release version `5.1.3` installs Zen/Control Plane version `6.1.1` +- Cloud Pak for Data release version `5.2.0` installs Zen/Control Plane version `6.2.0` !!! tip For more information about IBM Cloud Pak for Data upgrade process, refer to the [Cloud Pak for Data official documentation](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.8.x?topic=upgrading). @@ -176,6 +178,7 @@ The CP4D Admin User password to call CP4D API to provision Discovery Instance. I Example Playbook ------------------------------------------------------------------------------- +### Install Cloud Pak for Data 5.1.3 ```yaml - hosts: localhost any_errors_fatal: true @@ -187,6 +190,18 @@ Example Playbook - ibm.mas_devops.cp4d ``` +### Install Cloud Pak for Data 5.2.0 +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + cpd_product_version: 5.2.0 + cpd_primary_storage_class: ibmc-file-gold-gid + cpd_metadata_storage_class: ibmc-block-gold + roles: + - ibm.mas_devops.cp4d +``` + License ------------------------------------------------------------------------------- EPL-2.0 diff --git a/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.2.0.yml b/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.2.0.yml new file mode 100644 index 0000000000..5aadaec53a --- /dev/null +++ b/ibm/mas_devops/roles/cp4d/templates/catalog_sources/5.2.0.yml @@ -0,0 +1,22 @@ +catsrc_info: + cpd_platform: + - catalog_name: cpd-platform + catalog_display_name: ibm-cp-datacore-6.2.0-108 + name: cpopen/ibm-cpd-platform-operator-catalog + registry: icr.io + tag: 6.2.0-108 + digest: sha256:3a431c554421a78e875a2e5b19bc5d0e15d909e9c43b0ffe7f7e76d9989d5638 + cpfs: + - catalog_name: opencloud-operators + catalog_display_name: ibm-cp-common-services-4.13.0 + name: cpopen/ibm-common-service-catalog + registry: icr.io + tag: 4.13.0 + digest: sha256:e45f8a6739bbeb194a86cbf3266668d77f15c8772025de3afd288c83abfcd67f + zen: + - catalog_name: ibm-zen-operator-catalog + catalog_display_name: ibm-zen-6.2.0-232 + name: cpopen/ibm-zen-operator-catalog + registry: icr.io + digest: sha256:51fded1a61917a357775e9bb3b41221292a21ec307762040b84cc9e5959cde45 + tag: 6.2.0-232 diff --git a/ibm/mas_devops/roles/cp4d/templates/config_maps/olm-utils-cm-5.2.0.yml.j2 b/ibm/mas_devops/roles/cp4d/templates/config_maps/olm-utils-cm-5.2.0.yml.j2 new file mode 100644 index 0000000000..67bb302d1a --- /dev/null +++ b/ibm/mas_devops/roles/cp4d/templates/config_maps/olm-utils-cm-5.2.0.yml.j2 @@ -0,0 +1,575 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: olm-utils-cm + namespace: "{{ cpd_instance_namespace }}" + labels: + app.kubernetes.io/name: olm-utils +data: + release_components_meta: |- + scheduler: + case_version: 1.50.0 + sub_channel: "v1.50" + csv_version: 1.50.0 + cr_version: 1.50.0 + + cpfs: + csv_version: 4.13.0 + case_version: 4.13.0 + sub_channel: "v4.13" + + ibm-common-service-operator: + cr_version: 4.13.0 + + ibm-odlm: + cr_version: 4.5.1 + + ibm-commonui-operator-app: + cr_version: 4.9.0 + + ibm-iam-operator: + cr_version: 4.12.0 + + ibm-namespace-scope-operator: + cr_version: 4.3.0 + + ibm-cert-manager: + csv_version: 4.2.14 + case_version: 4.2.14 + sub_channel: "v4.2" + + ibm_events_operator: + case_version: 5.1.2 + sub_channel: "v5.1" + csv_version: 5.1.2 + + streamsets: + case_version: 5.0.0 + csv_version: 6.2.0 + cr_version: 6.2.0 + sub_channel: "v6.2.0" + + ibm-licensing: + csv_version: 4.2.15 + case_version: 4.2.15 + sub_channel: "v4.2" + + cpd_platform: + case_version: 5.2.0 + csv_version: 6.2.0 + sub_channel: "v6.2" + cr_version: 5.2.0 #should be the same as release_version + + platform-config: + cr_version: 5.2.0 # this version should be matched to the helm chart appVersion + + ibm_swhcc: + case_version: 5.2.0 + csv_version: 5.2.0 + sub_channel: "v5.2" + cr_version: 5.2.0 + + zen: + case_version: 6.2.0 + sub_channel: "v6.2" + csv_version: 6.2.0 + cr_version: 6.2.0 + + analyticsengine: + case_version: 11.0.0 + sub_channel: "v8.0" + csv_version: 8.0.0 + cr_version: 5.2.0 + + cognos_analytics: + case_version: 28.0.0 + sub_channel: "v28.0" + cr_version: 28.0.0 + csv_version: 28.0.0 + + ccs: + case_version: 11.0.0 + sub_channel: "v11.0" + csv_version: 11.0.0 + cr_version: 11.0.0 + + dashboard: + case_version: 3.7.0 + sub_channel: "v3.7" + csv_version: 3.7.0 + cr_version: 5.2.0 + + datarefinery: + case_version: 11.0.0 + csv_version: 11.0.0 + sub_channel: "v11.0" + cr_version: 11.0.0 + + canvasbase: + case_version: 8.5.0 + sub_channel: "v8.5" + csv_version: 8.5.0 + cr_version: 8.5.0 + + spss: + case_version: 8.5.0 + sub_channel: "v8.5" + csv_version: 8.5.0 + cr_version: 8.5.0 + + syntheticdata: + licenses_urls: + WXAI: https://ibm.biz/BdnPE9 + case_version: 11.0.0 + sub_channel: "v11.0" + csv_version: 11.0.0 + cr_version: 11.0.0 + + datastage_ent: + case_version: 10.0.0 + csv_version: 8.0.0 + cr_version: 5.2.0 + sub_channel: "v8.0" + + datastage_ent_plus: + case_version: 10.0.0 + csv_version: 8.0.0 + cr_version: 5.2.0 + sub_channel: "v8.0" + + informix: + case_version: 9.0.0 + sub_channel: "v9.0" + csv_version: 9.0.0 + + informix_cp4d: + case_version: 9.0.0 + sub_channel: "v9.0" + csv_version: 9.0.0 + cr_version: 9.0.0 + + openscale: + case_version: 9.0.0 + sub_channel: "v8.0" + csv_version: 8.0.0 + cr_version: 5.2.0 + + ws: + case_version: 11.0.0 + sub_channel: "v11.0" + csv_version: 11.0.0 + cr_version: 11.0.0 + + ws_runtimes: + case_version: 11.0.0 + sub_channel: "v11.0" + csv_version: 11.0.0 + cr_version: 11.0.0 + + db2aaservice: + case_version: 5.2.0 + sub_channel: "v7.3" + csv_version: 7.3.0 + cr_version: 5.2.0 + + db2oltp: + licenses_urls: + DB2CE: https://ibm.biz/BdnPB7 + DB2SE: https://ibm.biz/BdnPdB + DB2AE: https://ibm.biz/BdnPBW + case_version: 5.2.0 + sub_channel: "v7.3" + csv_version: 7.3.0 + cr_version: 5.2.0 + + db2wh: + case_version: 5.2.0 + sub_channel: "v7.3" + csv_version: 7.3.0 + cr_version: 5.2.0 + + db2u: + case_version: 7.3.0 + sub_channel: "v7.3" + csv_version: 7.3.0 + + datagate: + licenses_urls: + DGWXD: https://ibm.biz/BdnGcZ + case_version: 10.0.0 + sub_channel: "v8.0" + csv_version: 8.0.0 + cr_version: 8.0.0 + + dods: + case_version: 11.0.0 + sub_channel: "v11.0" + csv_version: 11.0.0 + cr_version: 11.0.0 + + hee: + case_version: 5.2.0 + sub_channel: "v5.20" + csv_version: 5.2.0 + cr_version: 5.2.0 + + dv: + case_version: 7.0.0 + sub_channel: "v7.0" + csv_version: 7.0.0 + cr_version: 3.2.0 + + bigsql: + case_version: 13.0.0 + sub_channel: "v13.0" + csv_version: 13.0.0 + cr_version: 8.2.0 + + planning_analytics: + case_version: 5.2.0 + sub_channel: "v10.0" + csv_version: 5.2.0 + cr_version: 5.2.0 + + rstudio: + case_version: 11.0.0 + sub_channel: "v11.0" + csv_version: 11.0.0 + cr_version: 11.0.0 + + watson_assistant: + case_version: 5.7.0 + sub_channel: "v5.7" + csv_version: 5.7.0 + cr_version: "5.2.0" + + watson_discovery: + case_version: 10.0.0 + sub_channel: "v10.0" + csv_version: 10.0.0 + cr_version: 5.2.0 + + watson_speech: + case_version: 10.0.0 + sub_channel: "v10.0" + csv_version: 10.0.0 + cr_version: 5.2.0 + + wkc: + case_version: 5.2.0 + sub_channel: "v8.0" + csv_version: 2.2.0 + cr_version: 5.2.0 + + ikc_standard: + licenses_urls: + IKCS: https://ibm.biz/Bdmm4V + case_version: 5.2.0 + sub_channel: "v7.0" + csv_version: 5.2.0 + cr_version: 5.2.0 + + ikc_premium: + licenses_urls: + IKCP: https://ibm.biz/Bdmm4J + case_version: 5.2.0 + sub_channel: "v7.0" + csv_version: 5.2.0 + cr_version: 5.2.0 + + datalineage: + case_version: 5.2.0 + sub_channel: "v6.0" + csv_version: 5.2.0 + cr_version: 5.2.0 + + udp: + case_version: 5.2.0 + sub_channel: "v5.2" + csv_version: 5.2.0 + cr_version: 5.2.0 + + productmaster: + case_version: 8.0.0 + csv_version: 8.0.0 + sub_channel: "v8.0" + cr_version: 8.0.0 + + dataproduct: + case_version: 5.2.0 + sub_channel: "v7.0" + csv_version: 1.2.0 + cr_version: 5.2.0 + + semantic_automation: + licenses_urls: + IKCP: https://ibm.biz/Bdmm4J + case_version: 5.2.0 + sub_channel: "v7.0" + csv_version: 5.2.0 + cr_version: 5.2.0 + + productmaster_instance: + cr_version: 8.0.0 + + dp: + case_version: 10.0.0 + csv_version: 10.0.0 + sub_channel: "v10.0" + cr_version: 5.2.0 + + wml: + case_version: 11.0.0 + sub_channel: "v8.0" + csv_version: 8.0.0 + cr_version: 5.2.0 + + dmc: + case_version: 8.7.0 + csv_version: 5.7.0 + cr_version: 5.2.0 + sub_channel: "v5.7" + + openpages: + case_version: 9.0.0 + csv_version: 9.0.0 + cr_version: 9.5.0 + sub_channel: "v9.0" + + openpages_instance: + cr_version: 9.5.0 + + mantaflow: + case_version: 2.0.0 + csv_version: 2.0.0 + sub_channel: "v2.0" + cr_version: 42.12.0 + + match360: + case_version: 4.7.0 + csv_version: 4.7.0 + cr_version: 4.7.21 + sub_channel: "v4.7" + + opencontent_opensearch: + case_version: 1.1.2494 + csv_version: 1.1.2494 + cr_version: 1.1.2494 + sub_channel: "v1.1" + + opencontent_elasticsearch: + case_version: 1.1.2667 + csv_version: 1.1.2667 + cr_version: 1.1.2667 + sub_channel: "v1.1" + + opencontent_redis: + case_version: 1.6.11 + csv_version: 1.6.11 + sub_channel: "v1.6" + + ibm_redis_cp: + case_version: 1.2.8 + csv_version: 1.2.8 + sub_channel: "v1.2" + + opencontent_rabbitmq: + case_version: 1.0.50 + csv_version: 1.0.41 + sub_channel: "v1.0" + + opencontent_fdb: + case_version: 5.2.0 + csv_version: 5.2.0 + cr_version: 5.2.0 + sub_channel: "v5.2" + + fdb_k8s: + csv_version: 5.2.0 + sub_channel: "v5.2" + + opencontent_minio: + case_version: 1.0.23 + csv_version: 1.0.18 + + opencontent_etcd: + case_version: 2.0.54 + csv_version: 1.0.44 + + opencontent_auditwebhook: + case_version: 1.0.24 + csv_version: 0.3.1 + + watson_gateway: + case_version: 2.0.49 + csv_version: 1.0.46 + + data_governor: + case_version: 7.0.1 + csv_version: 7.0.1 + sub_channel: v7.0 + cr_version: 3.17.4 + + ws_pipelines: + case_version: 11.0.0 + sub_channel: "v11.0" + csv_version: 11.0.0 + cr_version: 5.2.0 + + postgresql: + case_version: 5.15.0 + sub_channel: "stable-v1.25" + csv_version: 1.25.1 + cr_version: 1.25.1 # this version should be matched to the helm chart appVersion + #operands_supported: 13.20, 14.17, 15.12, 16.8, and 17.4 + + edb_cp4d: + case_version: 5.20.0 + sub_channel: "v5.20" + csv_version: 5.20.0 + cr_version: 5.20.0 + #operands_supported: 13.20, 14.17, 15.12, 16.8, and 17.4 + + mongodb: + case_version: 5.20.0 + sub_channel: "stable" + csv_version: 1.31.0 + # MongoDB_Ops_Manager: "7.0.13, 8.0.3" + # Mongodb_server: "8.0.4-ent, 7.0.16-ent" + + mongodb_cp4d: + case_version: 5.20.0 + sub_channel: "v5.20" + csv_version: 5.20.0 + cr_version: 5.20.0 + + dpra: + case_version: 1.19.0 + sub_channel: "v1.19" + csv_version: 1.19.0 + cr_version: 1.19.0 + + factsheet: + case_version: 6.0.0 + csv_version: 6.0.0 + sub_channel: "v6.0" + cr_version: 5.2.0 + + replication: + licenses_urls: + IDRC: https://ibm.biz/BdnPcZ + IIDRC: https://ibm.biz/BdnPcY + IDRM: https://ibm.biz/BdnPcz + IIDRM: https://ibm.biz/BdnPcq + IDRZOS: https://ibm.biz/BdnPc2 + IIDRWXTO: https://ibm.biz/BdnPcf + IIDRWXAO: https://ibm.biz/BdnPcP + case_version: 5.2.0 + csv_version: 6.0.0 + cr_version: 5.2.0 + sub_channel: "v6.0" + + voice_gateway: + case_version: 1.9.0 + csv_version: 1.9.0 + sub_channel: v1.9 + cr_version: 1.9.0 + + watsonx_data: + licenses_urls: + WXD: https://ibm.biz/BdnKJk + case_version: 5.0.0 + sub_channel: "v5.0" + csv_version: 5.0.0 + cr_version: 2.2.0 + + watsonx_data_premium: + licenses_urls: + WXD: https://ibm.biz/BdnJr7 + case_version: 5.0.0 + sub_channel: "v5.0" + csv_version: 5.0.0 + cr_version: 2.2.0 + + wxd_query_optimizer: + component_dependencies: + - db2u + + watsonx_ai: + licenses_urls: + WXAI: https://ibm.biz/BdnPE9 + case_version: 11.0.0 + csv_version: 11.0.0 + sub_channel: v11.0 + cr_version: 11.0.0 + osai_min_version: 2.16.0 + + watsonx_ai_ifm: + licenses_urls: + WXAI: https://ibm.biz/BdnPE9 + case_version: 11.0.0 + csv_version: 11.0.0 + sub_channel: v11.0 + cr_version: 11.0.0 + + watsonx_bi_assistant: + case_version: 3.0.0 + csv_version: 3.0.0 + sub_channel: v3.0 + cr_version: 3.0.0 + + wca_ansible: + case_version: 3.0.0 + csv_version: 3.0.0 + sub_channel: v3.0 + cr_version: 5.2.0 + + wca: + case_version: 2.0.0 + csv_version: 2.0.0 + sub_channel: v2.0 + cr_version: 5.2.0 + + wca_z: + case_version: 3.0.0 + csv_version: 3.0.0 + sub_channel: v3.0 + cr_version: 5.2.0 + + wca_z_ce: + case_version: 5.2.0 + sub_channel: v2.6 + csv_version: 2.6.0 + cr_version: 2.6.0 + + wca_base: + case_version: 3.0.0 + csv_version: 3.0.0 + sub_channel: v3.0 + cr_version: 5.2.0 + + ibm_neo4j: + case_version: 1.2.0 + sub_channel: "v3.0" + csv_version: 1.2.0 + cr_version: 1.2.0 + + watsonx_governance: + case_version: 4.0.0 + sub_channel: "v4.0" + csv_version: 4.0.0 + cr_version: 2.2.0 + + + watsonx_orchestrate: + case_version: 6.0.0 + sub_channel: "v6.0" + csv_version: 6.0.0 + cr_version: "6.0.0" + + watsonx_dataintelligence: + case_version: 2.2.0 + sub_channel: "v2.20" + csv_version: 2.2.0 + cr_version: 2.2.0 diff --git a/ibm/mas_devops/roles/cp4d_service/README.md b/ibm/mas_devops/roles/cp4d_service/README.md index 0b8e457145..73babc0312 100644 --- a/ibm/mas_devops/roles/cp4d_service/README.md +++ b/ibm/mas_devops/roles/cp4d_service/README.md @@ -3,6 +3,7 @@ cp4d_service Install or upgrade a chosen CloudPak for Data service. Currently supported Cloud Pak for Data release versions supported are: - 5.1.3 + - 5.2.0 The role will automatically install the corresponding CPD service operator channel and custom resource version associated to the chosen Cloud Pak for Data release version. @@ -41,10 +42,15 @@ Subscriptions related to Watson Studio: - **ibm-cpd-datarefinery** - **ibm-cpd-ws-runtimes** +!!! note "Search Engine Dependency" + - **CPD 5.1.3**: Uses **Elasticsearch** operator (`ibm-elasticsearch-operator`) + - **CPD 5.2.0**: Uses **OpenSearch** operator (`ibm-opensearch-operator`) + Watson Studio is made up of many moving parts across multiple namespaces. In the **ibm-cpd-operators** namespace: +**For CPD 5.1.3:** ```bash oc -n ibm-cpd-operators get deployments NAME READY UP-TO-DATE AVAILABLE AGE @@ -55,6 +61,17 @@ ibm-cpd-ws-runtimes-operator 1/1 1 1 ibm-elasticsearch-operator-ibm-es-controller-manager 1/1 1 1 83m ``` +**For CPD 5.2.0:** +```bash +oc -n ibm-cpd-operators get deployments +NAME READY UP-TO-DATE AVAILABLE AGE +ibm-cpd-ccs-operator 1/1 1 1 83m +ibm-cpd-datarefinery-operator 1/1 1 1 83m +ibm-cpd-ws-operator 1/1 1 1 83m +ibm-cpd-ws-runtimes-operator 1/1 1 1 83m +ibm-opensearch-operator-controller-manager 1/1 1 1 83m +``` + In the **ibm-cpd** namespace: ```bash @@ -122,10 +139,15 @@ Subscriptions related to Watson Machine Learning: - **ibm-cpd-wml** - **ibm-cpd-ccs** +!!! note "Search Engine Dependency" + - **CPD 5.1.3**: Uses **Elasticsearch** operator (`ibm-elasticsearch-operator`) + - **CPD 5.2.0**: Uses **OpenSearch** operator (`ibm-opensearch-operator`) + Watson Machine Learning is made up of many moving parts across multiple namespaces. In the **ibm-cpd-operators** namespace: +**For CPD 5.1.3:** ```bash oc -n ibm-cpd-operators get deployments NAME READY UP-TO-DATE AVAILABLE AGE @@ -135,6 +157,16 @@ ibm-cpd-wml-operator 1/1 1 1 ibm-elasticsearch-operator-ibm-es-controller-manager 1/1 1 1 134m ``` +**For CPD 5.2.0:** +```bash +oc -n ibm-cpd-operators get deployments +NAME READY UP-TO-DATE AVAILABLE AGE +ibm-cpd-ccs-operator 1/1 1 1 134m +ibm-cpd-datarefinery-operator 1/1 1 1 134m +ibm-cpd-wml-operator 1/1 1 1 49m +ibm-opensearch-operator-controller-manager 1/1 1 1 134m +``` + In the **ibm-cpd** namespace: ```bash @@ -330,17 +362,30 @@ Local directory to save the generated resource definition. This can be used to Example Playbook ---------------- +### Install Watson Studio on CPD 5.1.3 ```yaml --- - hosts: localhost any_errors_fatal: true vars: - cpd_product_version: 5.0.0 + cpd_product_version: 5.1.3 cpd_service_storage_class: ibmc-file-gold-gid cpd_service_name: wsl roles: - ibm.mas_devops.cp4d_service +``` +### Install Watson Studio on CPD 5.2.0 +```yaml +--- +- hosts: localhost + any_errors_fatal: true + vars: + cpd_product_version: 5.2.0 + cpd_service_storage_class: ibmc-file-gold-gid + cpd_service_name: wsl + roles: + - ibm.mas_devops.cp4d_service ``` License diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml index 4abaade983..bf6ddeeba2 100644 --- a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-ccs.yml @@ -104,10 +104,15 @@ when: not is_ccs_already_patched -- include_tasks: "tasks/wait/wait-elasticsearch.yml" +- include_tasks: "tasks/wait/wait-opensearch.yml" when: - - not skip_ibm_entitlement_injection # eventually we hope to be able to skip patching the elastic search cr with image pull secret, but not for now + - not skip_ibm_entitlement_injection + - cpd_52_or_higher +- include_tasks: "tasks/wait/wait-elasticsearch.yml" + when: + - not skip_ibm_entitlement_injection + - cpd_51 # 5. Wait for CCS CR to be ready # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-opensearch.yml b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-opensearch.yml new file mode 100644 index 0000000000..d1dd14c158 --- /dev/null +++ b/ibm/mas_devops/roles/cp4d_service/tasks/wait/wait-opensearch.yml @@ -0,0 +1,79 @@ +--- +# 1. Wait for elasticsearch-master cluster custom resource to be created +# ----------------------------------------------------------------------------- +- name: "wait/ccs : Wait for the OpenSearch cluster custom resource to appear (60s delay)" + kubernetes.core.k8s_info: + api_version: opensearch.cloudpackopen.ibm.com/v1 + kind: Cluster + name: elasticsearch-master + namespace: "{{ cpd_instance_namespace }}" + register: opensearch_cr_lookup + retries: 30 # Up to 30 minutes + delay: 60 # Every 1 minute + until: + - opensearch_cr_lookup.resources is defined + - opensearch_cr_lookup.resources | length > 0 + +# 2. Check if opensearch cluster is already patched +# ----------------------------------------------------------------------------- +- set_fact: + is_opensearch_already_patched: "{{ opensearch_cr_lookup.resources[0].spec.imagePullSecret is defined and opensearch_cr_lookup.resources[0].spec.imagePullSecret != '' }}" + +- debug: + msg: + - "OpenSearch Already patched? ..................... {{ is_opensearch_already_patched }}" + +# 3. Only patch opensearch cluster with ibm-entitlement-key if is_opensearch_already_patched == False +# ----------------------------------------------------------------------------- +- block: + # Delete the snapshot repo job first as it has immutable fields + - name: "wait/ccs : Delete elasticsearch-master-snapshot-repo job so it can be recreated with correct imagePullSecret" + kubernetes.core.k8s: + state: absent + api_version: batch/v1 + kind: Job + name: elasticsearch-master-snapshot-repo + namespace: "{{ cpd_instance_namespace }}" + ignore_errors: true + + # Patch the cluster CR with imagePullSecret + - name: "wait/ccs : Patch the elasticsearch-master cluster custom resource to include right imagePullSecret" + kubernetes.core.k8s: + api_version: opensearch.cloudpackopen.ibm.com/v1 + kind: Cluster + name: elasticsearch-master + namespace: "{{ cpd_instance_namespace }}" + apply: yes + definition: + spec: + imagePullSecret: ibm-entitlement-key + + # Wait a bit for the operator to reconcile and recreate the job + - name: "wait/ccs : Wait for operator to reconcile (30s)" + pause: + seconds: 30 + + when: not is_opensearch_already_patched + +# 4. Wait for OpenSearch cluster to be ready +# ----------------------------------------------------------------------------- +- name: "wait/ccs : Wait for OpenSearch cluster to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: opensearch.cloudpackopen.ibm.com/v1 + kind: Cluster + name: elasticsearch-master + namespace: "{{ cpd_instance_namespace }}" + register: opensearch_status_lookup + retries: 60 # Up to 60 minutes + delay: 60 # Every 1 minute + until: + - opensearch_status_lookup.resources is defined + - opensearch_status_lookup.resources | length > 0 + - opensearch_status_lookup.resources[0].status is defined + - opensearch_status_lookup.resources[0].status.phase is defined + - opensearch_status_lookup.resources[0].status.phase == "Available" + +- name: "wait/ccs : Check that the OpenSearch cluster phase is 'Available'" + assert: + that: opensearch_status_lookup.resources[0].status.phase == "Available" + fail_msg: "OpenSearch cluster install failed (phase is not Available)" diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index e0b0f08340..3ba6b80441 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -419,116 +419,6 @@ This is only used when both `mas_config_dir` and `mas_instance_id` are set, and - Default: None -Role Variables - Backup and Restore ------------------------------------------------------------------------------------------------------------------ -### masbr_confirm_cluster -Set `true` or `false` to indicate the role whether to confirm the currently connected cluster before running the backup or restore job. - -- Optional -- Environment Variable: `MASBR_CONFIRM_CLUSTER` -- Default: `false` - -### masbr_copy_timeout_sec -Set the transfer files timeout in seconds. - -- Optional -- Environment Variable: `MASBR_COPY_TIMEOUT_SEC` -- Default: `43200` (12 hours) - -### masbr_job_timezone -Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for creating scheduled backup job. If not set a value for this variable, this role will use UTC time zone when creating a CronJob for running scheduled backup job. - -- Optional -- Environment Variable: `MASBR_JOB_TIMEZONE` -- Default: None - -### masbr_storage_local_folder -Set local path to save the backup files. - -- **Required** -- Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` -- Default: None - -### masbr_backup_type -Set `full` or `incr` to indicate the role to create a full backup or incremental backup. - -- Optional -- Environment Variable: `MASBR_BACKUP_TYPE` -- Default: `full` - -### masbr_backup_from_version -Set the full backup version to use in the incremental backup, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`). This variable is only valid when `MASBR_BACKUP_TYPE=incr`. If not set a value for this variable, this role will try to find the latest full backup version from the specified storage location. - -- Optional -- Environment Variable: `MASBR_BACKUP_FROM_VERSION` -- Default: None - -### masbr_backup_schedule -Set [Cron expression](ttps://en.wikipedia.org/wiki/Cron) to create a scheduled backup. If not set a value for this varialbe, this role will create an on-demand backup. - -- Optional -- Environment Variable: `MASBR_BACKUP_SCHEDULE` -- Default: None - -### masbr_restore_from_version -Set the backup version to use in the restore, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`) - -- **Required** only when `DB2_ACTION=restore` -- Environment Variable: `MASBR_RESTORE_FROM_VERSION` -- Default: None - - -Example Playbook ------------------------------------------------------------------------------------------------ - -### Install Db2 -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - ibm_entitlement_key: xxxxx - - # Configuration for the Db2 cluster - db2_instance_name: db2u-db01 - - db2_meta_storage_class: "ibmc-file-gold" - db2_data_storage_class: "ibmc-block-gold" - db2_backup_storage_class: "ibmc-file-gold" - db2_logs_storage_class: "ibmc-block-gold" - db2_temp_storage_class: "ibmc-block-gold" - - # Create the MAS JdbcCfg & Secret resource definitions - mas_instance_id: inst1 - mas_config_dir: /home/david/masconfig - roles: - - ibm.mas_devops.db2 -``` - -### Backup Db2 -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - db2_action: backup - db2_instance_name: db2u-db01 - masbr_storage_local_folder: /tmp/masbr - roles: - - ibm.mas_devops.db2 -``` - -### Restore Db2 -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - db2_action: restore - db2_instance_name: db2u-db01 - masbr_restore_from_version: 20240621021316 - masbr_storage_local_folder: /tmp/masbr - roles: - - ibm.mas_devops.db2 -``` - License ------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/tasks/after-backup-restore.yml b/ibm/mas_devops/roles/db2/tasks/after-backup-restore.yml deleted file mode 100644 index 60ea1112e3..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/after-backup-restore.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Clean up -# ------------------------------------------------------------------------- -- name: "Delete temporary folders" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - rm -f {{ masbr_pod_lock_file }}; - rm -rf {{ db2_pod_temp_folder }}; - rm -rf {{ db2_pvc_temp_folder }} - {{ exec_in_pod_end }} diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml deleted file mode 100644 index 72c6b14e64..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml +++ /dev/null @@ -1,163 +0,0 @@ ---- -# Update db2 database backup status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update db2 database backup status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - -- name: "Backup db2 database" - block: - # Prepare db2 database backup folder - # ------------------------------------------------------------------------- - - name: "Set fact: db2 database backup folder" - set_fact: - # We should use Db2 backup pvc to save the temporary backup files, the db2 pod - # ephemeral local storage has a limits up to 4Gi by default. - db2_backup_folder: "{{ db2_pvc_temp_folder }}/{{ masbr_job_data_type }}" - - - name: "Set fact: db2 database backup log" - set_fact: - db2_backup_log: "{{ db2_backup_folder }}/db2-backup.log" - - - name: "Create db2 database backup folder" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ db2_backup_folder }} - {{ exec_in_pod_end }} - - - name: "Debug: db2 database backup folder" - debug: - msg: "Db2 database backup folder ........ {{ db2_backup_folder }}" - - # Take a backup of db2 database - # https://www.ibm.com/docs/en/db2/11.5?topic=commands-backup-database - # ------------------------------------------------------------------------- - - name: "Take {{ 'Full' if masbr_backup_type == 'full' else 'Incremental' }} backup of db2 database" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ db2_backup_folder }}/db2backup && - db2 -v connect to {{ db2_dbname }} | tee -a {{ db2_backup_log }} && - db2 -v backup db {{ db2_dbname }} on all dbpartitionnums online - {{ "incremental" if masbr_backup_type == "incr" }} - to {{ db2_backup_folder }}/db2backup - compress UTIL_IMPACT_PRIORITY 50 include logs without prompting | tee -a {{ db2_backup_log }} - {{ exec_in_pod_end }} - register: _db2backup_output - - - name: "Debug: db2 backup output" - debug: - msg: "{{ _db2backup_output.stdout_lines }}" - - # Extract Db2 keystore master key - # https://www.ibm.com/docs/en/db2/11.5?topic=edr-restoring-encrypted-backup-image-different-system-local-keystore - # ------------------------------------------------------------------------- - - name: "Check master key label from keystore.p12" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - gsk8capicmd_64 -cert -list all -db {{ db2_keystore_folder }}/keystore.p12 -stashed - {{ exec_in_pod_end }} - register: _check_master_label_output - - - name: "Get master key label from keystore.p12" - vars: - regex: '\DB2(.*)' - when: item is regex('\DB2(.*)') - set_fact: - db2_master_key_label: "{{ item | regex_search(regex) }}" - with_items: "{{ _check_master_label_output.stdout_lines | list }}" - - - name: "Extract master key from keystore.p12" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - gsk8capicmd_64 -secretkey -extract -db {{ db2_keystore_folder }}/keystore.p12 -stashed - -label {{ db2_master_key_label }} -format ascii - -target {{ db2_backup_folder }}/db2backup/master_key_label.kdb - {{ exec_in_pod_end }} - register: _extract_master_key_output - - - name: "Copy keystore files to db2 backup folder" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - cp -rf {{ db2_keystore_folder }}/* {{ db2_backup_folder }}/db2backup && - ls -lA {{ db2_backup_folder }}/db2backup - {{ exec_in_pod_end }} - register: _copy_keystore_output - - - name: "Debug: files in db2 backup folder" - debug: - msg: "{{ _copy_keystore_output.stdout_lines }}" - - # Create tar.gz archives of database backup files - # ------------------------------------------------------------------------- - - name: "Create tar.gz archives of database backup files" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - tar -czf {{ db2_backup_folder }}/{{ masbr_job_name }}.tar.gz - -C {{ db2_backup_folder }}/db2backup . && - du -h {{ db2_backup_folder }}/* - {{ exec_in_pod_end }} - register: _du_files_output - - - name: "Debug: size of backup files" - debug: - msg: "{{ _du_files_output.stdout_lines }}" - - # Copy backup files from pod to specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup files from pod to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_pod_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_are_pvc_paths: true - masbr_cf_paths: - - src_file: "{{ db2_backup_folder }}/{{ masbr_job_name }}.tar.gz" - dest_folder: "{{ masbr_job_data_type }}" - - # Update database backup status: Completed - # ------------------------------------------------------------------------- - - name: "Update database backup status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update database backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update database backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy db2 backup log file from pod to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of db2 backup log" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - tar -czf {{ db2_backup_folder }}/db2-backup-log.tar.gz - -C {{ db2_backup_folder }} db2-backup.log - {{ exec_in_pod_end }} - - - name: "Copy db2 backup log file from pod to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_pod_files_to_storage.yml" - vars: - masbr_cf_job_type: "{{ masbr_job_type }}" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ db2_backup_folder }}/db2-backup-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index a915394c47..8428c99c0b 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -6,62 +6,3 @@ that: db2_instance_name is defined and db2_instance_name != "" fail_msg: "db2_instance_name is required" -# Set common backup job variables -# ----------------------------------------------------------------------------- -- name: "Set fact: common backup job variables" - set_fact: - masbr_job_component: - name: "db2" - instance: "{{ db2_instance_name }}" - namespace: "{{ db2_namespace }}" - masbr_job_data_list: - - seq: "1" - type: "database" - -# Before run tasks -# ------------------------------------------------------------------------- -- name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _ignore_masbr_backup_data: true - _job_type: "backup" - _component_before_task_path: "{{ role_path }}/tasks/before-backup-restore.yml" - -- name: "Run backup tasks" - block: - # Update backup job status: New - # ------------------------------------------------------------------------- - - name: "Update backup job status: New" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "1" - phase: "New" - - # Run backup tasks for each data type - # ------------------------------------------------------------------------- - - name: "Run backup tasks for each data type" - include_tasks: "{{ role_path }}/tasks/backup/backup-{{ job_data_item.type }}.yml" - vars: - masbr_job_data_seq: "{{ job_data_item.seq }}" - masbr_job_data_type: "{{ job_data_item.type }}" - loop: "{{ masbr_job_data_list }}" - loop_control: - loop_var: job_data_item - - rescue: - # Update backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update database backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_status: - phase: "Failed" - - always: - # After run tasks - # ------------------------------------------------------------------------- - - name: "After run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" - vars: - _component_after_task_path: "{{ role_path }}/tasks/after-backup-restore.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml b/ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml deleted file mode 100644 index 01061b37e0..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/before-backup-restore.yml +++ /dev/null @@ -1,147 +0,0 @@ ---- -# Get db2 version and status -# ----------------------------------------------------------------------------- -- name: "Get Db2uCluster" - kubernetes.core.k8s_info: - api_version: db2u.databases.ibm.com/v1 - kind: Db2uCluster - name: "{{ masbr_job_component.instance }}" - namespace: "{{ db2_namespace }}" - register: _db2ucluster_output - -- name: "Set fact: db2 version" - set_fact: - db2_version: "{{ _db2ucluster_output.resources[0].spec.version }}" - when: - - _db2ucluster_output is defined - - _db2ucluster_output.resources[0] is defined - - _db2ucluster_output.resources[0].spec.version is defined - -- name: "Fail if db2 does not exists" - assert: - that: db2_version is defined - fail_msg: "Db2 does not exists!" - -- name: "Set fact: db2 running status" - set_fact: - db2_running: true - when: - - _db2ucluster_output is defined - - _db2ucluster_output.resources[0] is defined - - _db2ucluster_output.resources[0].status is defined - - _db2ucluster_output.resources[0].status.state is defined - - _db2ucluster_output.resources[0].status.state == "Ready" - -- name: "Fail if db2 is not running" - assert: - that: db2_running is defined and db2_running - fail_msg: "Db2 is not running!" - -# Get db2 pod name -# ----------------------------------------------------------------------------- -- name: "Get db2 pod name" - kubernetes.core.k8s_info: - kind: Pod - namespace: "{{ db2_namespace }}" - label_selectors: - - type=engine - - app={{ masbr_job_component.instance }} - register: _db2_pod_output - failed_when: - - _db2_pod_output.resources is not defined - - _db2_pod_output.resources | length == 0 - -- name: "Set fact: db2 pod name" - set_fact: - db2_pod_name: "{{ _db2_pod_output.resources[0].metadata.name }}" - db2_container_name: db2u - -- name: "Set fact: exec command in db2 pod" - set_fact: - exec_in_pod_begin: >- - oc exec {{ db2_pod_name }} -c {{ db2_container_name }} -n {{ db2_namespace }} -- su -lc ' - exec_in_pod_end: "' {{ db2_jdbc_username }}" - -# Set db2 backup/restore variables -# ------------------------------------------------------------------------- -- name: "Set fact: db2 pod copy file variables" - set_fact: - masbr_cf_namespace: "{{ db2_namespace }}" - masbr_cf_pod_name: "{{ db2_pod_name }}" - masbr_cf_container_name: "{{ db2_container_name }}" - masbr_cf_pvc_name: "c-{{ db2_instance_name if masbr_job_type == 'restore' else db2_instance_name }}-backup" - masbr_cf_pvc_mount_path: "/mnt/backup" - masbr_cf_pvc_sub_path: "" - masbr_cf_are_pvc_paths: true - masbr_cf_affinity: false - -- name: "Set fact: temporary folders" - set_fact: - db2_pod_temp_folder: "{{ masbr_pod_temp_folder }}/{{ masbr_job_name }}" - db2_pvc_temp_folder: "{{ masbr_cf_pvc_mount_path }}/{{ masbr_job_name }}" - -- name: "Set fact: db2 keystore folder" - set_fact: - db2_keystore_folder: "/mnt/blumeta0/db2/keystore" - -# Output db2 information -# ----------------------------------------------------------------------------- -- name: "Debug: db2 information" - debug: - msg: - - "Db2 version ............................ {{ db2_version }}" - - "Db2 is running ......................... {{ db2_running }}" - - "Db2 pod name ........................... {{ db2_pod_name }}" - -# Check if an exiting job is running -# ------------------------------------------------------------------------- -- name: "Try to find job lock file in pod" - when: not masbr_allow_multi_jobs - changed_when: false - shell: > - {{ exec_in_pod_begin }} - [ -f {{ masbr_pod_lock_file }} ] && echo exist; exit 0 - {{ exec_in_pod_end }} - register: _get_lock_file_output - -- name: "Fail if found job lock file in pod" - when: not masbr_allow_multi_jobs - assert: - that: _get_lock_file_output.stdout != "exist" - fail_msg: "A backup/restore job is running now, please try to run job later!" - -- name: "Create job lock file in pod" - changed_when: true - shell: >- - {{ exec_in_pod_begin }} - mkdir -p {{ masbr_pod_lock_file | dirname }}; - touch {{ masbr_pod_lock_file }} - {{ exec_in_pod_end }} - -# Check storage usage -# ------------------------------------------------------------------------- -- name: "Get storage usage of pod temporary folder" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ db2_pod_temp_folder }}; - df -h {{ db2_pod_temp_folder }} - {{ exec_in_pod_end }} - register: _df_temp_output - -- name: "Debug: storage usage of pod temporary folder" - debug: - msg: "{{ _df_temp_output.stdout_lines }}" - -- name: "Get storage usage of pvc temporary folder" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ db2_pvc_temp_folder }}; - df -h {{ db2_pvc_temp_folder }} - {{ exec_in_pod_end }} - register: _df_pvc_output - -- name: "Debug: storage usage of pvc temporary folder" - debug: - msg: "{{ _df_pvc_output.stdout_lines }}" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml b/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml deleted file mode 100644 index 57fc684401..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/restore/copy-db2-backup-file.yml +++ /dev/null @@ -1,107 +0,0 @@ ---- -# Copy backup file from specified storage location -# ------------------------------------------------------------------------- -- name: "Copy backup file from specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_pod.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ _job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/{{ _job_name }}.tar.gz" - dest_folder: "{{ db2_restore_folder }}" - -# Extract the tar.gz file -# ------------------------------------------------------------------------- -- name: "Extract the tar.gz file" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ db2_restore_folder }}/{{ _job_name }} && - tar -xzf {{ db2_restore_folder }}/{{ _job_name }}.tar.gz - -C {{ db2_restore_folder }}/{{ _job_name }} && - ls {{ db2_restore_folder }}/{{ _job_name }} - {{ exec_in_pod_end }} - register: _extract_output - -- name: "Debug: list extracted files" - debug: - msg: - - "Extract output folder .............. {{ db2_restore_folder }}/{{ _job_name }}" - - "{{ _extract_output.stdout_lines }}" - -# Validate extracted files -# ------------------------------------------------------------------------- -- name: "Set fact: extracted file names" - set_fact: - db2_restore_filenames: "{{ _extract_output.stdout_lines }}" - -- name: "Set fact: Db2 keystore .p12 file" - vars: - regex: ".+?(?=.p12)" - when: item is regex(regex) - set_fact: - db2_keystore_p12_file: "{{ item }}" - loop: "{{ db2_restore_filenames }}" - -- name: "Fail if Db2 keystore .p12 file was not found" - assert: - that: - - db2_keystore_p12_file is defined - - db2_keystore_p12_file | length > 0 - fail_msg: "Db2 keystore .p12 file from source Db2 instance must be provided for the restore process to work." - -- name: "Set fact: Db2 keystore .sth file" - vars: - regex: ".+?(?=.sth)" - when: item is regex(regex) - set_fact: - db2_keystore_sth_file: "{{ item }}" - loop: "{{ db2_restore_filenames }}" - -- name: "Fail if Db2 keystore .sth file was not found" - assert: - that: - - db2_keystore_sth_file is defined - - db2_keystore_sth_file | length > 0 - fail_msg: "Db2 keystore .sth file from source Db2 instance must be provided for the restore process to work." - -- name: "Set fact: Db2 keystore .kdb file" - vars: - regex: ".+?(?=.kdb)" - when: item is regex(regex) - set_fact: - db2_keystore_kdb_file: "{{ item }}" - loop: "{{ db2_restore_filenames }}" - -- name: "Fail if Db2 keystore .kdb file was not found" - assert: - that: - - db2_keystore_kdb_file is defined - - db2_keystore_kdb_file | length > 0 - fail_msg: "Db2 keystore .kdb file from source Db2 instance must be provided for the restore process to work." - -- name: "Set fact: Db2 backup file timestamp" - vars: - regex: '\d+\d+\d+\d' - when_regex: "^{{ db2_dbname | upper }}.*" - when: item is regex(when_regex) - set_fact: - db2_backup_timestamp: "{{ item | regex_search(regex) }}" - loop: "{{ db2_restore_filenames }}" - -- name: "Fail if Db2 backup file timestamp was not found" - assert: - that: - - db2_backup_timestamp is defined - - db2_backup_timestamp != "" - fail_msg: >- - Db2 backup files were not found or it does not have the expected format - i.e '{{ db2_dbname | upper }}.0.db2inst1.DBPART000.202XXXXXXXXXXX.001' - -- name: "Debug: Db2 backup file timestamp" - debug: - msg: - - "Db2 keystore .p12 file ............. {{ db2_keystore_p12_file }}" - - "Db2 keystore .sth file ............. {{ db2_keystore_sth_file }}" - - "Db2 keystore .kdb file ............. {{ db2_keystore_kdb_file }}" - - "Db2 backup file timestamp .......... {{ db2_backup_timestamp }}" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/main.yml b/ibm/mas_devops/roles/db2/tasks/restore/main.yml index eb813e356d..8428c99c0b 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/main.yml @@ -1,66 +1,8 @@ --- -# Check db2 restore required variables +# Check db2 backup required variables # ----------------------------------------------------------------------------- - name: "Fail if db2_instance_name is not provided" assert: that: db2_instance_name is defined and db2_instance_name != "" fail_msg: "db2_instance_name is required" -# Set common restore variables -# ----------------------------------------------------------------------------- -- name: "Set fact: common restore variables" - set_fact: - masbr_job_component: - name: "db2" - instance: "{{ db2_instance_name }}" - namespace: "{{ db2_namespace }}" - masbr_job_data_list: - - seq: "1" - type: "database" - -# Before run tasks -# ------------------------------------------------------------------------- -- name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "restore" - _component_before_task_path: "{{ role_path }}/tasks/before-backup-restore.yml" - -- name: "Run restore tasks" - block: - # Update restore job status: New - # ------------------------------------------------------------------------- - - name: "Update restore job status: New" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "1" - phase: "New" - - # Run restore tasks for each data type - # ------------------------------------------------------------------------- - - name: "Run restore tasks for each data type" - include_tasks: "{{ role_path }}/tasks/restore/restore-{{ job_data_item.type }}.yml" - vars: - masbr_job_data_seq: "{{ job_data_item.seq }}" - masbr_job_data_type: "{{ job_data_item.type }}" - loop: "{{ masbr_job_data_list }}" - loop_control: - loop_var: job_data_item - - rescue: - # Update restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_status: - phase: "Failed" - - always: - # After run tasks - # ------------------------------------------------------------------------- - - name: "After run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" - vars: - _component_after_task_path: "{{ role_path }}/tasks/after-backup-restore.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml deleted file mode 100644 index 23e1dd324f..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml +++ /dev/null @@ -1,291 +0,0 @@ ---- -# Update db2 database restore status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update db2 database restore status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - -- name: "Restore db2 database" - block: - # Prepare db2 database restore folder - # ------------------------------------------------------------------------- - - name: "Set fact: db2 database restore variables" - set_fact: - # We should use Db2 backup pvc to save the temporary backup files, the db2 pod - # ephemeral local storage has a limits up to 4Gi by default. - db2_restore_folder: "{{ db2_pvc_temp_folder }}/{{ masbr_job_data_type }}" - - - name: "Set fact: db2 database restore log" - set_fact: - db2_restore_log: "{{ db2_restore_folder }}/db2-restore.log" - - - name: "Create db2 database restore folder" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ db2_restore_folder }} && - chmod a+w {{ db2_restore_folder }} - {{ exec_in_pod_end }} - - - name: "Debug: db2 database restore folder" - debug: - msg: "Db2 database restore folder ....... {{ db2_restore_folder }}" - - # Copy backup file from specified storage location to pod - # ------------------------------------------------------------------------- - - name: "Copy backup file from specified storage location to pod" - include_tasks: "{{ role_path }}/tasks/restore/copy-db2-backup-file.yml" - vars: - _job_name: "{{ masbr_restore_from }}" - - - name: "Set fact: Db2 backup file timestamp" - set_fact: - masbr_restore_from_timestamp: "{{ db2_backup_timestamp }}" - - # This is an incremental backup, we also need to copy the based on full backup file - # ------------------------------------------------------------------------- - - name: "This is an incremental backup, we also need to copy the based on full backup file" - when: masbr_restore_from_incr - block: - - name: "Copy based on full backup file from specified storage location to pod" - include_tasks: "{{ role_path }}/tasks/restore/copy-db2-backup-file.yml" - vars: - _job_name: "{{ masbr_restore_basedon }}" - - - name: "Set fact: Db2 backup file timestamp" - set_fact: - masbr_restore_basedon_timestamp: "{{ db2_backup_timestamp }}" - - # Add Db2 keystore master key - # https://www.ibm.com/docs/en/db2/11.5?topic=edr-restoring-encrypted-backup-image-different-system-local-keystore - # ------------------------------------------------------------------------- - - name: "Check master key label from source keystore.p12" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - gsk8capicmd_64 -cert -list all -db {{ db2_restore_folder }}/{{ masbr_restore_from }}/keystore.p12 -stashed - {{ exec_in_pod_end }} - register: _check_master_label_output - - - name: "Get master key label from source keystore.p12" - vars: - regex: '\DB2(.*)' - when: item is regex('\DB2(.*)') - set_fact: - db2_master_key_label: "{{ item | regex_search(regex) }}" - with_items: "{{ _check_master_label_output.stdout_lines | list }}" - - - name: "Add master key to target keystore.p12" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - gsk8capicmd_64 -secretkey -add -db {{ db2_keystore_folder }}/keystore.p12 -stashed - -label {{ db2_master_key_label }} -format ascii - -file {{ db2_restore_folder }}/{{ masbr_restore_from }}/master_key_label.kdb - {{ exec_in_pod_end }} - register: _add_master_key_output - failed_when: - - _add_master_key_output.rc != 0 - - ('CTGSK3005W' not in _add_master_key_output.stdout) - - # Deactivate Db2 in preparation for restore - # https://www.ibm.com/docs/en/db2/11.5?topic=r-restoring-db2-from-online-backup-using-commands - # ------------------------------------------------------------------------- - - name: "Deactivate Db2 in preparation for restore" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - echo "1. Temporarily disable the built-in HA" | tee -a {{ db2_restore_log }}; - sudo wvcli system disable -m "Disable HA before Db2 maintenance" | tee -a {{ db2_restore_log }}; - echo "2. Connect to the database" | tee -a {{ db2_restore_log }}; - db2 -v connect to {{ db2_dbname }} | tee -a {{ db2_restore_log }}; - echo "3. Disconnect all the applications that are connected to Db2" | tee -a {{ db2_restore_log }}; - db2 -v list applications | tee -a {{ db2_restore_log }}; - db2 -v force application all | tee -a {{ db2_restore_log }}; - echo "4. Terminate the database" | tee -a {{ db2_restore_log }}; - db2 -v terminate | tee -a {{ db2_restore_log }}; - echo "5. Stop the database" | tee -a {{ db2_restore_log }}; - db2stop force | tee -a {{ db2_restore_log }}; - echo "6. Ensure that all Db2 interprocess communications are cleaned for the instance" | tee -a {{ db2_restore_log }}; - ipclean -a | tee -a {{ db2_restore_log }}; - echo "7. Turn off all communications to the database by setting the value of the DB2COMM variable to null" | tee -a {{ db2_restore_log }}; - db2set -null DB2COMM | tee -a {{ db2_restore_log }}; - echo "8. Restart the database in restricted access mode" | tee -a {{ db2_restore_log }}; - db2start admin mode restricted access | tee -a {{ db2_restore_log }} - {{ exec_in_pod_end }} - register: _pre_restore_output - - - name: "Debug: deactivate Db2 in preparation for restore" - debug: - msg: "{{ _pre_restore_output.stdout_lines }}" - - - name: "Run Db2 restore commands" - block: - # Run Db2 full restore command - # https://www.ibm.com/docs/en/db2/11.5?topic=commands-restore-database - # ------------------------------------------------------------------------- - - name: "Restore Db2 from a full backup" - when: not masbr_restore_from_incr - changed_when: true - shell: > - {{ exec_in_pod_begin }} - echo "9. Restore Db2 from a full backup" | tee -a {{ db2_restore_log }}; - db2 -v restore db {{ db2_dbname }} - from {{ db2_restore_folder }}/{{ masbr_restore_from }} - taken at {{ masbr_restore_from_timestamp }} into {{ db2_dbname }} - logtarget {{ db2_restore_folder }}/{{ masbr_restore_from }} - replace existing without prompting | tee -a {{ db2_restore_log }} - {{ exec_in_pod_end }} - register: _run_full_restore_output - failed_when: - - _run_full_restore_output.rc != 0 - # SQL2581N: this Db2 error code means something went wrong in restore command - # - ('SQL2581N' in _run_full_restore_output.stdout) - - - name: "Debug: restore Db2 from a full backup" - when: not masbr_restore_from_incr - debug: - msg: "{{ _run_full_restore_output.stdout_lines }}" - - # Run Db2 incremental restore command - # https://www.ibm.com/docs/en/db2/11.5?topic=commands-restore-database - # ------------------------------------------------------------------------- - - name: "Restore Db2 from an incremental backup" - when: masbr_restore_from_incr - changed_when: true - shell: > - {{ exec_in_pod_begin }} - echo "9. Restore Db2 from an incremental backup" | tee -a {{ db2_restore_log }}; - db2ckrst -d BLUDB -t {{ masbr_restore_from_timestamp }} | tee -a {{ db2_restore_log }}; - db2 -v restore db {{ db2_dbname }} incremental - from {{ db2_restore_folder }}/{{ masbr_restore_from }} - taken at {{ masbr_restore_from_timestamp }} into {{ db2_dbname }} - logtarget {{ db2_restore_folder }}/{{ masbr_restore_from }} - replace existing without prompting | tee -a {{ db2_restore_log }}; - db2 -v restore db {{ db2_dbname }} incremental - from {{ db2_restore_folder }}/{{ masbr_restore_basedon }} - taken at {{ masbr_restore_basedon_timestamp }} into {{ db2_dbname }} - logtarget {{ db2_restore_folder }}/{{ masbr_restore_basedon }} - replace existing without prompting | tee -a {{ db2_restore_log }}; - db2 -v restore db {{ db2_dbname }} incremental - from {{ db2_restore_folder }}/{{ masbr_restore_from }} - taken at {{ masbr_restore_from_timestamp }} into {{ db2_dbname }} - logtarget {{ db2_restore_folder }}/{{ masbr_restore_from }} - replace existing without prompting | tee -a {{ db2_restore_log }} - {{ exec_in_pod_end }} - register: _run_incr_restore_output - failed_when: - - _run_incr_restore_output.rc != 0 - # SQL2581N: this Db2 error code means something went wrong in restore command - # - ('SQL2581N' in _run_incr_restore_output.stdout) - - - name: "Debug: run Db2 restore command" - when: masbr_restore_from_incr - debug: - msg: "{{ _run_incr_restore_output.stdout_lines }}" - - always: - # Run Db2 rollforward command regardless of whether Db2 restore success or not, - # otherwise the Db2 will be in pending status. - # https://www.ibm.com/docs/en/db2/11.5?topic=commands-rollforward-database - # ------------------------------------------------------------------------- - - name: "Check Db2 rollforward status" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - echo "10. Check Db2 rollforward status" | tee -a {{ db2_restore_log }}; - db2 -v rollforward db {{ db2_dbname }} query status | tee -a {{ db2_restore_log }} - {{ exec_in_pod_end }} - register: _query_rollforward_output - - - name: "Debug: check Db2 rollforward status" - debug: - msg: "{{ _query_rollforward_output.stdout_lines }}" - - - name: "Run Db2 rollforward command" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - echo "11. Run Db2 rollforward command" | tee -a {{ db2_restore_log }}; - db2 -v "rollforward db {{ db2_dbname }} to end of backup and complete - overflow log path ({{ db2_restore_folder }}/{{ masbr_restore_from }}) noretrieve" | tee -a {{ db2_restore_log }} - {{ exec_in_pod_end }} - register: _run_rollforward_output - failed_when: - - _run_rollforward_output.rc != 0 - # SQL1119N: this Db2 error code means something went wrong in rollforward command - # - ('SQL1119N' in _run_rollforward_output.stdout) - - - name: "Debug: run Db2 rollforward command" - debug: - msg: "{{ _run_rollforward_output.stdout_lines }}" - - # Active Db2 after successful rollforward - # https://www.ibm.com/docs/en/db2/11.5?topic=r-restoring-db2-from-online-backup-using-commands - # ------------------------------------------------------------------------- - - name: "Active Db2 after successful rollforward" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - echo "12. Stop the database" | tee -a {{ db2_restore_log }}; - db2stop force | tee -a {{ db2_restore_log }}; - echo "13. Ensure that all Db2 interprocess communications are cleaned for the instance" | tee -a {{ db2_restore_log }}; - ipclean -a | tee -a {{ db2_restore_log }}; - echo "14. Reinitialize the Db2 communication manager to accept database connections" | tee -a {{ db2_restore_log }}; - db2set DB2COMM=TCPIP,SSL | tee -a {{ db2_restore_log }}; - echo "15. Restart the database for normal operation" | tee -a {{ db2_restore_log }}; - db2start | tee -a {{ db2_restore_log }}; - echo "16. Activate the database" | tee -a {{ db2_restore_log }}; - db2 activate db {{ db2_dbname }} | tee -a {{ db2_restore_log }}; - echo "17. Re-enable the Wolverine high availability monitoring process" | tee -a {{ db2_restore_log }}; - wvcli system enable -m "Enable HA after Db2 maintenance" | tee -a {{ db2_restore_log }}; - echo "18. Connect to the database" | tee -a {{ db2_restore_log }}; - db2 connect to {{ db2_dbname }} | tee -a {{ db2_restore_log }} - {{ exec_in_pod_end }} - register: _post_restore_output - - - name: "Debug: active Db2 after successfull rollforward" - debug: - msg: "{{ _post_restore_output.stdout_lines }}" - - # Update database restore status: Completed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update database restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy db2 restore log file from pod to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of db2 restore log" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - tar -czf {{ db2_restore_folder }}/db2-restore-log.tar.gz - -C {{ db2_restore_folder }} db2-restore.log - {{ exec_in_pod_end }} - - - name: "Copy db2 restore log file from pod to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_pod_files_to_storage.yml" - vars: - masbr_cf_job_type: "restore" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ db2_restore_folder }}/db2-restore-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/db2/templates/tlsroute.yml.j2 b/ibm/mas_devops/roles/db2/templates/tlsroute.yml.j2 index c983fef69f..42bd9a0dac 100644 --- a/ibm/mas_devops/roles/db2/templates/tlsroute.yml.j2 +++ b/ibm/mas_devops/roles/db2/templates/tlsroute.yml.j2 @@ -21,5 +21,5 @@ spec: targetPort: ssl-server tls: termination: passthrough - nsecureEdgeTerminationPolicy: None + insecureEdgeTerminationPolicy: None wildcardPolicy: None diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/aws/uninstall.yml b/ibm/mas_devops/roles/kafka/tasks/provider/aws/uninstall.yml index f9fc3f9b21..8291315347 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/aws/uninstall.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/aws/uninstall.yml @@ -71,7 +71,7 @@ - name: "Delete AWS MSK Kafka cluster" shell: aws kafka delete-cluster --cluster-arn {{ kafka_arn }} --region {{ aws_region }} changed_when: false - when: kafka_arn != "" and kafka_state != 'Deleting' + when: kafka_arn != None and kafka_arn != "" and kafka_state != 'Deleting' - name: "Wait until AWS MSK Kafka cluster deleted" shell: aws kafka list-clusters --cluster-name-filter {{ kafka_cluster_name }} --region {{ aws_region }} @@ -144,6 +144,6 @@ sg_group_id: "{{sg_info.stdout | from_json | json_query('SecurityGroups[0].GroupId')}}" - name: Delete Security Group for AWS MSK Instance - when: sg_group_id is defined and sg_group_id != '' + when: sg_group_id is defined and sg_group_id != None and sg_group_id != '' shell: aws ec2 delete-security-group --group-id '{{ sg_group_id }}' register: sg_delete_info diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-security-group.yml b/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-security-group.yml index 46ed663807..5aa1991325 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-security-group.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-security-group.yml @@ -7,7 +7,7 @@ register: security_group_info - name: Set Fact , Security Group Id - when: security_group_info is defined and security_group_info != '' + when: security_group_info is defined and security_group_info != None and security_group_info != '' set_fact: sg_id: "{{security_group_info.stdout | from_json | json_query('SecurityGroups[0].GroupId')}}" @@ -24,7 +24,7 @@ when: not sg_id assert: that: - - sg_info is defined and sg_info != '' + - sg_info is defined and sg_info != None and sg_info != '' - sg_info.stdout - sg_info.stdout | from_json | json_query('GroupId') diff --git a/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-subnet.yml b/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-subnet.yml index 1c87e598a6..13c00d3e68 100644 --- a/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-subnet.yml +++ b/ibm/mas_devops/roles/kafka/tasks/provider/aws/utils/create-subnet.yml @@ -33,7 +33,7 @@ when: not subnet_id assert: that: - - create_subnet_info is defined and create_subnet_info != '' + - create_subnet_info is defined and create_subnet_info != None and create_subnet_info != '' - create_subnet_info.stdout - create_subnet_info.stdout | from_json | json_query('Subnet.SubnetId') diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index e8017d72ab..8f8748643a 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -117,6 +117,14 @@ docdb_master_password: "{{ lookup('env', 'DOCDB_MASTER_PASSWORD') }}" # Custom Labels custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string | ibm.mas_devops.string2dict() }}" +# MongoCE backup and restore vars +# ----------------------------------------------------------------------------- +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" +mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" +mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" +skip_instance_backup_restore: "{{ lookup('env', 'SKIP_INSTANCE_BACKUP_RESTORE') | default(false, true) }}" +mongodb_data_backup: "{{ lookup('env', 'MONGODB_DATA_BACKUP') | default(true, true) }}" + # Mongo upgrade flags # If identified that there's an existing Mongo that might lead to a v5 or v6 upgrade # the following flags must be set to confirm the upgrades otherwise the role will fail and not proceed with the upgrade. diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/install.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/install.yml index b00da75560..718a62e8e5 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/install.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/install.yml @@ -85,7 +85,7 @@ ('DBSubnetGroupNotFoundFault' not in subnet_group_exists_info.stderr ) - name: Set Fact, subnet group exists or not - when: subnet_group_exists_info is defined and subnet_group_exists_info != '' and subnet_group_exists_info.stdout != '' + when: subnet_group_exists_info is defined and subnet_group_exists_info != None and subnet_group_exists_info != '' and subnet_group_exists_info.stdout != '' set_fact: subnet_group_exists: "{{ subnet_group_exists_info.stdout | from_json | json_query('DBSubnetGroups[0].DBSubnetGroupName') != '' }}" @@ -120,7 +120,7 @@ register: security_group_info - name: Set Fact , Security Group Id - when: security_group_info is defined and security_group_info != '' + when: security_group_info is defined and security_group_info != None and security_group_info != '' set_fact: sg_id: "{{security_group_info.stdout | from_json | json_query('SecurityGroups[0].GroupId')}}" @@ -137,7 +137,7 @@ when: not sg_id assert: that: - - sg_info is defined and sg_info != '' + - sg_info is defined and sg_info != None and sg_info != '' - sg_info.stdout - sg_info.stdout | from_json | json_query('GroupId') @@ -217,7 +217,7 @@ when: not docdb_cluster_exists assert: that: - - docdb_cluster_create_info is defined and docdb_cluster_create_info != '' + - docdb_cluster_create_info is defined and docdb_cluster_create_info != None and docdb_cluster_create_info != '' - docdb_cluster_create_info.stdout - docdb_cluster_create_info.stdout | from_json | json_query('DBCluster.DBClusterIdentifier') == docdb_cluster_name diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/after-backup-restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/after-backup-restore.yml deleted file mode 100644 index 460fc3510c..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/after-backup-restore.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Clean up -# ------------------------------------------------------------------------- -- name: "Delete temporary folders" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - rm -f {{ masbr_pod_lock_file }}; - rm -rf {{ mongodb_pod_temp_folder }}; - rm -rf {{ mongodb_pvc_temp_folder }} - {{ exec_in_pod_end }} diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index 33111d295f..65acdd827e 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -1,340 +1,107 @@ --- -# Update database backup status: InProgress +# Backup Mongodb database Data using mondodump # ----------------------------------------------------------------------------- -- name: "Update database backup status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" -- name: "Backup mongodb databases" - block: - # Create mongodb role and user for backing up databases - # ------------------------------------------------------------------------- - - name: "Create mongodb role and user for backing up databases" - include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/create-role-user.yml" - - # Prepare mongodb database backup folder - # ------------------------------------------------------------------------- - - name: "Set fact: mongodb database backup folder" - set_fact: - # We should use mongodb pod ephemeral local storage to save the temporary files, - # the mongodb data pvc size is not big enough. - mongodb_backup_folder: "{{ mongodb_pod_temp_folder }}/{{ masbr_job_data_type }}" - - - name: "Set fact: mongodb database backup log" - set_fact: - mongodb_backup_log: "{{ mongodb_backup_folder }}/{{ masbr_job_name }}-backup.log" - - - name: "Create mongodb database backup folder" - changed_when: true - shell: > - mkdir -p {{ masbr_local_job_folder }}/{{ masbr_job_data_type }}; - {{ exec_in_pod_begin }} - mkdir -p {{ mongodb_backup_folder }} - {{ exec_in_pod_end }} - - # Set database name filter - # ------------------------------------------------------------------------- - - name: "Set fact: default database name filter" - when: mas_app_id is not defined or mas_app_id | length == 0 - set_fact: - # Backup all databases if not specified mas_app_id - mongodb_db_filter: "^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)" - - - name: "Set fact: database name filter for {{ mas_app_id }}" - when: mas_app_id is defined and mas_app_id | length > 0 - block: - - name: "Set fact: database name filters for each mas app" - set_fact: - mongodb_db_all_filters: - iot: "iot_{{ mas_instance_id }}_" - visualinspection: "mas-{{ mas_instance_id }}-(visualinspection|edgeman)" - optimizer: "mas_{{ mas_instance_id }}_optimizer" - - - name: "Set fact: always backup databases for core" - set_fact: - mongodb_db_app_filters: - ["mas_{{ mas_instance_id }}_(core|catalog|adoptionusage)"] - - - name: "Set fact: append database name filters for ({{ mas_app_id }})" - set_fact: - mongodb_db_app_filters: > - {{ mongodb_db_app_filters + [mongodb_db_all_filters[mas_app_id] | default('')] }} - - - name: "Set fact: database name filters for ({{ mas_app_id }})" - when: mongodb_db_app_filters is defined and mongodb_db_app_filters | length > 0 - set_fact: - mongodb_db_filter: "{{ mongodb_db_app_filters | select() | join('|') }}" - - - name: "Debug: database name filter" - debug: - msg: "{{ mongodb_db_filter }}" - - # Take a full backup of mongodb databases - # ------------------------------------------------------------------------- - - name: "Take a full backup of mongodb databases" - when: masbr_backup_type == "full" - block: - # Get all database name belong to specified mas instance - - name: "Get all database names belong to mas instance {{ mas_instance_id }}" - changed_when: false - shell: >- - {{ exec_in_pod_begin }} - {{ mongodb_shell }} --quiet --host={{ mongodb_primary_host }} - --username={{ mongodb_user }} --password={{ mongodb_password }} - --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin - --eval="JSON.stringify(db.adminCommand({ - listDatabases: 1, - nameOnly: true, - filter: { name: /{{ mongodb_db_filter }}/ } - }))" | tee -a {{ mongodb_backup_log }} - {{ exec_in_pod_end }} - register: _mongodb_db_names_output - no_log: true - - - name: "Set fact: database names" - set_fact: - mongodb_db_names: "{{ _mongodb_db_names_output.stdout | from_json | json_query('databases') }}" - - - name: "Debug: database names" - debug: - msg: "{{ mongodb_db_names }}" - - # Get the timestamp of current backup - - name: "Get the timestamp of current backup" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - date +%s - {{ exec_in_pod_end }} - register: _ts_output - - - name: "Set fact: timestamp of current backup" - set_fact: - mongodb_backup_ts: "{{ _ts_output.stdout }}" - - # Excluding sessions collection for Monitor database since it is not cleaned up automatically - # leading to very large backup sizes and long backup times. This is acceptable as - # sessions are transient and can be recreated. - # Refer DT425304 - MASCORE-5808 - - name: "Set fact for Monitor database" - set_fact: - monitor_db_name: "mas_{{ mas_instance_id }}_monitor" - exclude_monitor_session_collection: "--excludeCollection=sessions" - - # mongodump - - name: "Take full backup of mongodb databases" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mongodump --host={{ mongodb_primary_host }} - --username={{ mongodb_user }} --password={{ mongodb_password }} - --authenticationDatabase=admin --ssl --sslCAFile={{ mongodb_ca_file }} - --out={{ mongodb_backup_folder }}/mongodump - --db={{ item.name }} - {{ exclude_monitor_session_collection if (item.name == monitor_db_name) else '' }} - |& tee -a {{ mongodb_backup_log }} - {{ exec_in_pod_end }} - loop: "{{ mongodb_db_names }}" - register: _mongodump_output - no_log: true - - - name: "Debug: mongodump output" - debug: - msg: "{{ _mongodump_output | json_query('results[*].stdout_lines') }}" - - # Take an incremental backup of mongodb databases - # ------------------------------------------------------------------------- - - name: "Take an incremental backup of mongodb databases" - when: masbr_backup_type == "incr" - block: - # Get query file from specified backup job - - name: "Get query file from specified backup job" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_pod.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_backup_from }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/query.json" - dest_folder: "{{ mongodb_backup_folder }}/from" - - - name: "Get query file content" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - cat {{ mongodb_backup_folder }}/from/query.json - {{ exec_in_pod_end }} - register: _query_file_content_output - - - name: "Debug: query file content" - debug: - msg: "{{ _query_file_content_output.stdout }}" - - # Get the timestamp of current backup - - name: "Get the timestamp of current backup" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - date +%s - {{ exec_in_pod_end }} - register: _ts_output - - - name: "Set fact: timestamp of current backup" - set_fact: - mongodb_backup_ts: "{{ _ts_output.stdout }}" - - # mongodump - - name: "Take an incremental backup of mongodb databases" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mongodump --host={{ mongodb_primary_host }} - --username={{ mongodb_user }} --password={{ mongodb_password }} - --authenticationDatabase=admin --ssl --sslCAFile={{ mongodb_ca_file }} - --db=local --collection=oplog.rs --queryFile={{ mongodb_backup_folder }}/from/query.json - --out={{ mongodb_backup_folder }}/mongodump - |& tee -a {{ mongodb_backup_log }} - {{ exec_in_pod_end }} - register: _mongodump_output - no_log: true - - - name: "Debug: mongodump output" - debug: - msg: "{{ _mongodump_output.stdout_lines }}" - - # Create tar.gz archives of database backup files - # ------------------------------------------------------------------------- - - name: "Create tar.gz archives of database backup files" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - tar -czf {{ mongodb_backup_folder }}/{{ masbr_job_name }}.tar.gz - -C {{ mongodb_backup_folder }}/mongodump . && - du -h {{ mongodb_backup_folder }}/* - {{ exec_in_pod_end }} - register: _du_files_output - - - name: "Debug: size of backup files" - debug: - msg: "{{ _du_files_output.stdout_lines }}" - - # Copy backup files to specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup files from pod to specified storage location" - when: _mongodb_cf_in_server - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_pod_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_paths: - - src_file: "{{ mongodb_backup_folder }}/{{ masbr_job_name }}.tar.gz" - dest_folder: "{{ masbr_job_data_type }}" - - - name: "Download and copy backup files to specified storage location" - when: not _mongodb_cf_in_server - block: - - name: "Download backup files from pod to local" - changed_when: true - shell: > - oc cp --retries=50 -c {{ mongodb_container_name }} - {{ mongodb_namespace }}/{{ mongodb_pod_name }}:{{ mongodb_backup_folder }}/{{ masbr_job_name }}.tar.gz - {{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/{{ masbr_job_name }}.tar.gz - - - name: "Copy backup files from local to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/{{ masbr_job_name }}.tar.gz" - dest_folder: "{{ masbr_job_data_type }}" - - # Create query file used by the subsequent incremental backups - # ------------------------------------------------------------------------- - - name: "Create query file used by the subsequent incremental backups" - copy: - dest: "{{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/query.json" - content: "{{ mongodb_query | from_yaml | to_json }}" - mode: preserve - vars: - mongodb_query: "{{ lookup('template', 'templates/community/mongo-query.yml.j2') }}" - - - name: "Get query file content" - changed_when: false - shell: > - cat {{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/query.json - register: _cat_query_file_output - - - name: "Debug: query file content" - debug: - msg: "{{ _cat_query_file_output.stdout }}" - - - name: "Copy query file to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/query.json" - dest_folder: "{{ masbr_job_data_type }}" - - # Update database backup status: Completed - # ------------------------------------------------------------------------- - - name: "Update database backup status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update database backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update database backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy mongodb backup log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of mongodb backup log" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - tar -czf {{ mongodb_backup_folder }}/{{ masbr_job_name }}-backup-log.tar.gz - -C {{ mongodb_backup_folder }} {{ masbr_job_name }}-backup.log - {{ exec_in_pod_end }} - - - name: "Copy mongodb backup log file from pod to specified storage location" - when: _mongodb_cf_in_server - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_pod_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ mongodb_backup_folder }}/{{ masbr_job_name }}-backup-log.tar.gz" - dest_folder: "log" - - - name: "Download and copy backup log file to specified storage location" - when: not _mongodb_cf_in_server - block: - - name: "Download backup log file from pod to local" - changed_when: true - shell: > - oc cp --retries=50 -c {{ mongodb_container_name }} - {{ mongodb_namespace }}/{{ mongodb_pod_name }}:{{ mongodb_backup_folder }}/{{ masbr_job_name }}-backup-log.tar.gz - {{ masbr_local_job_folder }}/log/{{ masbr_job_name }}-backup-log.tar.gz - - - name: "Copy backup log file from local to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "log/{{ masbr_job_name }}-backup-log.tar.gz" - dest_folder: "log" +- name: "Retrieve mongodb info from cr and resources" + ibm.mas_devops.get_mongoce_info: + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_namespace: "{{ mongodb_namespace }}" + register: mongodb_info_result + +- name: "Set fact: mongodb info" + set_fact: + mongodb_backup_data_path: "{{ mongodb_backup_path }}/data" + mongodb_backup_data_filename: "mongodump-{{ mongodb_backup_version }}.tar.gz" + mongoce_pod_name: "{{ mongodb_info_result.mongoce_pod_name }}" + mongodb_version: "{{ mongodb_info_result.mongodb_version }}" + mongodb_service_name: "{{ mongodb_info_result.mongodb_service_name }}" + mongodb_host: "{{ mongodb_info_result.mongodb_host }}" + mongodb_admin_user: "{{ mongodb_info_result.mongodb_admin_user }}" + mongodb_admin_password: "{{ mongodb_info_result.mongodb_admin_password }}" + +- name: "Create {{ mongodb_backup_data_path }} directory for Mongodb database backup" + file: + path: "{{ mongodb_backup_data_path }}" + state: directory + mode: "0755" + +- name: "debug facts" + debug: + msg: + - "mongoce_pod_name .................................. {{ mongoce_pod_name }}" + - "mongodb_version ................................... {{ mongodb_version }}" + - "mongodb_service_name .............................. {{ mongodb_service_name }}" + - "mongodb_host ...................................... {{ mongodb_host }}" + - "mongodb_backup_version ............................ {{ mongodb_backup_version }}" + + +# Prepare shell scripts and transfer to mongo pod +# ----------------------------------------------------------------------------- +- block: + - name: Create create-role-user.sh script in local /tmp + ansible.builtin.template: + src: create-role-user.sh.j2 + dest: /tmp/create-role-user.sh + mode: '777' + + - name: Create database-backup.sh script in local /tmp + ansible.builtin.template: + src: database-backup.sh.j2 + dest: /tmp/database-backup.sh + mode: '777' + + - name: Copy the create-role-user.sh script into the mongodb pod + shell: "oc cp /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" + register: copy_result + retries: 2 + delay: 15 # seconds + until: copy_result.rc == 0 + + - name: Copy the database-backup.sh script into the mongodb pod + shell: "oc cp /tmp/database-backup.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-backup.sh -c mongod" + register: copy_result + retries: 2 + delay: 15 # seconds + until: copy_result.rc == 0 + +# The log file will also be available inside the pod /tmp/create-role-user.log +- name: Exec into mongo pod and run create-role-user.sh to setup backup role and user in Mongodb. + shell: | + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/create-role-user.sh + register: create_roleuser_output + ignore_errors: true + retries: 1 + delay: 10 # seconds + until: + - create_roleuser_output.rc == 0 + - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) + +- name: "Debug create-role-user logs" + debug: + msg: "{{ create_roleuser_output.stdout_lines }}" + +- name: Exec into mongo pod and run database-backup.sh to backup databases. + shell: | + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/database-backup.sh + register: database_backup_output + ignore_errors: true + retries: 1 + delay: 10 # seconds + until: + - database_backup_output.rc == 0 + - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) + +- name: "Debug database-backup logs" + debug: + msg: "{{ database_backup_output.stdout_lines }}" + +- name: "copy backup files from mongo pod to local backup directory" + shell: "oc cp --retries=50 -c mongod {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/masbr/{{ mongodb_backup_version }}/{{ mongodb_backup_data_filename }} {{ mongodb_backup_data_path }}/{{ mongodb_backup_data_filename }}" + register: copy_result + retries: 2 + delay: 10 # seconds + until: copy_result.rc == 0 + when: + - database_backup_output is defined + - database_backup_output.rc == 0 + - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml deleted file mode 100644 index ed74e87464..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/before-backup-restore.yml +++ /dev/null @@ -1,159 +0,0 @@ -# Set mongodb backup/restore variables -# ----------------------------------------------------------------------------- -- name: "Set fact: mongodb pod variables" - set_fact: - mongodb_pod_name: mas-mongo-ce-0 - mongodb_container_name: mongod - -- name: "Set fact: exec command in mongodb pod" - set_fact: - exec_in_pod_begin: >- - oc exec {{ mongodb_pod_name }} -c {{ mongodb_container_name }} -n {{ mongodb_namespace }} -- bash -c ' - exec_in_pod_end: "'" - -- name: "Set fact: copy file variables" - set_fact: - masbr_cf_namespace: "{{ mongodb_namespace }}" - masbr_cf_pod_name: "{{ mongodb_pod_name }}" - masbr_cf_container_name: "{{ mongodb_container_name }}" - masbr_cf_pvc_name: "data-volume-{{ mongodb_pod_name }}" - masbr_cf_pvc_mount_path: "/data" - masbr_cf_pvc_sub_path: "" - masbr_cf_are_pvc_paths: false - # The mongodb pvc access mode is 'ReadWriteOnce', we need to set affinity to schedule our copying file pod - # to the same node where mongodb pod located, so that the mongodb pvc can be mounted by multiple pods. - masbr_cf_affinity: true - -- name: "Set fact: temporary folders" - set_fact: - mongodb_pod_temp_folder: "{{ masbr_pod_temp_folder }}/{{ masbr_job_name }}" - mongodb_pvc_temp_folder: "{{ masbr_cf_pvc_mount_path }}/{{ masbr_job_name }}" - -# Get mongodb admin password -# ----------------------------------------------------------------------------- -- name: "Get mongodb admin password" - kubernetes.core.k8s_info: - kind: Secret - name: mas-mongo-ce-admin-password - namespace: "{{ mongodb_namespace }}" - register: _mongodb_password_output - no_log: true - -- name: "Set fact: mongodb admin password" - set_fact: - mongodb_password: "{{ _mongodb_password_output.resources[0].data.password | b64decode }}" - when: - - _mongodb_password_output is defined - - _mongodb_password_output.resources[0] is defined - - _mongodb_password_output.resources[0].data.password is defined - no_log: true - -# Get mongodb ca file location -# ----------------------------------------------------------------------------- -- name: "Get mongodb ca file" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - ls /var/lib/tls/ca/*.pem | head -n 1 - {{ exec_in_pod_end }} - register: _mongodb_ca_file_output - -- name: "Set fact: mongodb ca file" - set_fact: - mongodb_ca_file: "{{ _mongodb_ca_file_output.stdout }}" - when: - - _mongodb_ca_file_output.rc == 0 - - '"No such file or directory" not in _mongodb_ca_file_output.stdout' - -# Get mongodb primary host -# ----------------------------------------------------------------------------- -- name: "Get mongodb server information" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - {{ mongodb_shell }} --quiet --host={{ mongodb_pod_name }}.mas-mongo-ce-svc.{{ mongodb_namespace }}.svc.cluster.local:27017 - --username=admin --password={{ mongodb_password }} --authenticationDatabase=admin - --tls --tlsCAFile={{ mongodb_ca_file }} admin - --eval="print(JSON.stringify(db.runCommand({hello:1})))" - {{ exec_in_pod_end }} - register: _mongodb_info_output - no_log: true - -- name: "Set fact: mongodb primary host" - set_fact: - mongodb_primary_host: "{{ _mongodb_info_output.stdout_lines[-1] | from_json | json_query('primary') }}" - -# Output mongodb information -# ----------------------------------------------------------------------------- -- name: "Debug: mongodb information" - debug: - msg: - - "MongoDB version ............................ {{ mongodb_version }}" - - "MongoDB is running ......................... {{ mongodb_running }}" - - "MongoDB pod name ........................... {{ mongodb_pod_name }}" - - "MongoDB primary host ....................... {{ mongodb_primary_host }}" - - "MongoDB ca file ............................ {{ mongodb_ca_file }}" - -# Check if an exiting job is running -# ------------------------------------------------------------------------- -- name: "Try to find job lock file in pod" - when: not masbr_allow_multi_jobs - changed_when: false - shell: > - {{ exec_in_pod_begin }} - [ -f {{ masbr_pod_lock_file }} ] && echo exist; exit 0 - {{ exec_in_pod_end }} - register: _get_lock_file_output - -- name: "Fail if found job lock file in pod" - when: not masbr_allow_multi_jobs - assert: - that: _get_lock_file_output.stdout != "exist" - fail_msg: "A backup/restore job is running now, please try to run job later!" - -- name: "Create job lock file in pod" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ masbr_pod_lock_file | dirname }}; - touch {{ masbr_pod_lock_file }} - {{ exec_in_pod_end }} - register: _create_restore_lock_output - -# Check storage usage -# ------------------------------------------------------------------------- -- name: "Get storage usage of pod temporary folder" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ mongodb_pod_temp_folder }}; - df -h {{ mongodb_pod_temp_folder }} - {{ exec_in_pod_end }} - register: _df_temp_output - -- name: "Debug: storage usage of pod temporary folder" - debug: - msg: "{{ _df_temp_output.stdout_lines }}" - -- name: "Get storage usage of pvc temporary folder" - changed_when: false - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ mongodb_pvc_temp_folder }}; - df -h {{ mongodb_pvc_temp_folder }} - {{ exec_in_pod_end }} - register: _df_pvc_output - -- name: "Debug: storage usage of pvc temporary folder" - debug: - msg: "{{ _df_pvc_output.stdout_lines }}" - -# Workarounds -# ------------------------------------------------------------------------- -- name: "Set fact: how to copy backup files to specified storage location" - # When testing in the env using 'ibmc-block-gold' PVC, our copying file pod cannot mount - # the mongo data PVC even schedule it to the same node where mongo data pod located. - # Do not create Pod to copy mongo PVC data at this point, we will download the data to local first, - # then copy it to specified storage location. - set_fact: - _mongodb_cf_in_server: false diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.sh.j2 new file mode 100644 index 0000000000..85f93d3882 --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.sh.j2 @@ -0,0 +1,69 @@ +#!/bin/bash + + +MONGODB_BACKUP_ROLE="bradmin" +MONGODB_BACKUP_USER="bradmin" + +# Check if mongo is primary +tlsCAFile=$(ls /var/lib/tls/ca/*.pem) +echo "Using TLS CA file: $tlsCAFile" +primary_host="" + +is_primary=$(mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --host {{ mongodb_host }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --eval 'db.isMaster().ismaster') +cmdrc=$? +if [ $cmdrc -ne 0 ]; then + echo "Error checking if node is primary. Exiting." + exit 1 +fi + +if [ "$is_primary" != "true" ]; then + echo "This node is not primary. Get Primary from server status." + primary_host=$(mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --host {{ mongodb_host }} --authenticationDatabase=admin --tls --eval 'rs.isMaster().primary') + cmdrc=$? + if [ $cmdrc -ne 0 ]; then + echo "Error getting primary node. Exiting." + exit 1 + fi + echo "Primary host is: $primary_host" +fi + +if [ -n "$primary_host" ]; then + echo "Connecting to primary host: $primary_host to create user." + mongo_cmd="mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --host $primary_host" +else + echo "Creating user on current primary node." + mongo_cmd="mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --host {{ mongodb_host }}" +fi + +# Check if role already exists +role_exists=$($mongo_cmd --eval "db.getSiblingDB('admin').getRole('$MONGODB_BACKUP_ROLE');") +if [ -n "$role_exists" -a "$role_exists" != "null" ]; then + echo "Role $MONGODB_BACKUP_ROLE already exists. Skipping role creation." +else + echo "Creating role $MONGODB_BACKUP_ROLE." + # Create role user + $mongo_cmd --eval "db.getSiblingDB('admin').createRole({role: '$MONGODB_BACKUP_ROLE', privileges: [{ resource: { anyResource: true }, actions: [ 'anyAction' ] }],roles: []});" + cmdrc=$? + if [ $cmdrc -ne 0 ]; then + echo "Error creating backup role. Exiting." + exit 1 + fi +fi + +# Create backup user with the created role +user_exists=$($mongo_cmd --eval "db.getSiblingDB('admin').getUser('$MONGODB_BACKUP_USER');") +if [ -n "$user_exists" -a "$user_exists" != "null" ]; then + echo "User $MONGODB_BACKUP_USER already exists. Skipping user creation." +else + echo "Creating user $MONGODB_BACKUP_USER with role $MONGODB_BACKUP_ROLE." + $mongo_cmd --eval "db.getSiblingDB('admin').createUser({ user: '$MONGODB_BACKUP_USER', pwd: '{{ mongodb_admin_password }}', roles: [ { role: '$MONGODB_BACKUP_ROLE', db: 'admin' } ]});" + cmdrc=$? + if [ $cmdrc -ne 0 ]; then + echo "Error creating backup admin user. Exiting." + exit 1 + fi +fi + +echo "Role and user creation process completed." +echo "ROLEUSERstatus-SUCCESS" + diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml deleted file mode 100644 index 4a9d97c804..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -- name: "Set fact: mongodb role and user" - set_fact: - mongodb_role: sysadmin - mongodb_user: sysadmin - -# Create mongodb role -# ----------------------------------------------------------------------------- -- name: "Get mongodb role '{{ mongodb_role }}'" - changed_when: false - shell: > - oc exec {{ mongodb_pod_name }} -c {{ mongodb_container_name }} -n {{ mongodb_namespace }} -- bash -c - '{{ mongodb_shell }} --quiet --host={{ mongodb_primary_host }} --username=admin --password={{ mongodb_password }} - --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin - --eval="db.getRole( \"{{ mongodb_role }}\" )"' - register: _mongodb_get_role_output - no_log: true - -- name: "Debug: get mongodb role result" - debug: - msg: "Get mongodb role result ............... {{ _mongodb_get_role_output.stdout_lines }}" - -- name: "Create mongodb role '{{ mongodb_role }}'" - when: _mongodb_get_role_output.stdout_lines[-1] == "null" - block: - - name: "Create mongodb role '{{ mongodb_role }}'" - changed_when: true - shell: > - oc exec {{ mongodb_pod_name }} -c {{ mongodb_container_name }} -n {{ mongodb_namespace }} -- bash -c - '{{ mongodb_shell }} --quiet --host={{ mongodb_primary_host }} --username=admin --password={{ mongodb_password }} - --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin - --eval="db.createRole({ role: \"{{ mongodb_role }}\", roles: [], privileges: [{ resource: {anyResource: true}, actions: [\"anyAction\"] }]})"' - register: _mongodb_create_role_output - no_log: true - - - name: "Debug: create mongodb role result" - debug: - msg: "Create mongodb role result ........ {{ _mongodb_create_role_output.stdout_lines }}" - -# Create mongodb user -# ----------------------------------------------------------------------------- -- name: "Get mongodb user '{{ mongodb_user }}'" - changed_when: false - shell: > - oc exec {{ mongodb_pod_name }} -c {{ mongodb_container_name }} -n {{ mongodb_namespace }} -- bash -c - '{{ mongodb_shell }} --quiet --host={{ mongodb_primary_host }} --username=admin --password={{ mongodb_password }} - --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin - --eval="db.getUser( \"{{ mongodb_user }}\" )"' - register: _mongodb_get_user_output - no_log: true - -- name: "Create mongodb user '{{ mongodb_user }}'" - when: _mongodb_get_user_output.stdout_lines[-1] == "null" - block: - - name: "Create mongodb user '{{ mongodb_user }}'" - changed_when: true - shell: > - oc exec {{ mongodb_pod_name }} -c {{ mongodb_container_name }} -n {{ mongodb_namespace }} -- bash -c - '{{ mongodb_shell }} --quiet --host={{ mongodb_primary_host }} --username=admin --password={{ mongodb_password }} - --authenticationDatabase=admin --tls --tlsCAFile={{ mongodb_ca_file }} admin - --eval="db.createUser({ user: \"{{ mongodb_user }}\", pwd: \"{{ mongodb_password }}\", roles: [ \"{{ mongodb_role }}\" ]})"' - register: _mongodb_create_user_output - no_log: true - - - name: "Debug: create mongodb user result" - debug: - msg: "Create mongodb user result ........ {{ _mongodb_create_user_output.stdout_lines }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 new file mode 100644 index 0000000000..8b6e20ab55 --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 @@ -0,0 +1,81 @@ +#!/bin/bash + +MONGODB_BACKUP_USER="bradmin" + +TMP_BACKUP_DIR="/tmp/masbr/{{ mongodb_backup_version }}" + +ALL_DATABASES_FILTER="^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)" + +tlsCAFile=$(ls /var/lib/tls/ca/*.pem) +echo "Using TLS CA file: $tlsCAFile" +primary_host="" + +is_primary=$(mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --host {{ mongodb_host }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --eval 'db.isMaster().ismaster') +cmdrc=$? +if [ $cmdrc -ne 0 ]; then + echo "Error checking if node is primary. Exiting." + exit 1 +fi + +if [ "$is_primary" != "true" ]; then + echo "This node is not primary. Get Primary from server status." + primary_host=$(mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --host {{ mongodb_host }} --authenticationDatabase=admin --tls --eval 'rs.isMaster().primary') + cmdrc=$? + if [ $cmdrc -ne 0 ]; then + echo "Error getting primary node. Exiting." + exit 1 + fi + echo "Primary host is: $primary_host" +fi + +if [ -n "$primary_host" ]; then + echo "Connecting to primary host: $primary_host" + mongo_cmd="mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --host $primary_host" + mongodump_cmd="mongodump -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --sslCAFile $tlsCAFile --authenticationDatabase=admin --ssl --host $primary_host" +else + echo "Continuing on current primary node." + mongo_cmd="mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --host {{ mongodb_host }}" + mongodump_cmd="mongodump -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --sslCAFile $tlsCAFile --authenticationDatabase=admin --ssl --host {{ mongodb_host }}" +fi + +# Create temporary directory for backup + +mkdir -p $TMP_BACKUP_DIR +if [ $? -ne 0 ]; then + echo "Error creating temporary backup directory $TMP_BACKUP_DIR. Exiting." + exit 1 +fi + +# Get database names matching the filter +databases=$($mongo_cmd --eval "db.adminCommand('listDatabases').databases.map(db => db.name).filter(name => name.match(/$ALL_DATABASES_FILTER/)).join(',')") +cmdrc=$? +if [ $cmdrc -ne 0 ]; then + echo "Error retrieving database names. Exiting." + exit 1 +fi +echo "Databases to back up: $databases" +IFS=',' read -r -a db_array <<< "$databases" + +# Perform backup for each database +for db_name in "${db_array[@]}"; do + echo "Backing up database: $db_name" + $mongodump_cmd --db=$db_name --out=$TMP_BACKUP_DIR + cmdrc=$? + if [ $cmdrc -ne 0 ]; then + echo "Error backing up database $db_name. Exiting." + exit 1 + fi +done +echo "Backup completed successfully. Backup files are located in $TMP_BACKUP_DIR." + +# Create tar.gz archives of database backup files +tar -czf $TMP_BACKUP_DIR/{{ mongodb_backup_data_filename }} -C $TMP_BACKUP_DIR . +if [ $? -ne 0 ]; then + echo "Error creating tar.gz archive of backup files. Exiting." + exit 1 +fi + +echo "Created archive: $TMP_BACKUP_DIR/{{ mongodb_backup_data_filename }}" + +echo "DATABASEBACKUPstatus-SUCCESS" + diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 new file mode 100644 index 0000000000..1003efd775 --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 @@ -0,0 +1,73 @@ +#!/bin/bash + +MONGODB_BACKUP_USER="bradmin" + +BACKUP_DATA_FILENAME="{{ mongodb_backup_data_filename }}" +TMP_RESTORE_DIR="{{ mongodb_tmp_restore_dir }}" + + +# Extract backup data tar.gz file +echo "Extracting backup data file: $BACKUP_DATA_FILENAME" +tar -xzf $TMP_RESTORE_DIR/$BACKUP_DATA_FILENAME -C $TMP_RESTORE_DIR +cmdrc=$? +if [ $cmdrc -ne 0 ]; then + echo "Error extracting backup data file. Exiting." + exit 1 +fi + +# Remove the tar.gz file after extraction +rm -f $TMP_RESTORE_DIR/$BACKUP_DATA_FILENAME + +# Determine if current node is primary +tlsCAFile=$(ls /var/lib/tls/ca/*.pem) +echo "Using TLS CA file: $tlsCAFile" +primary_host="" + +is_primary=$(mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --host {{ mongodb_host }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --eval 'db.isMaster().ismaster') +cmdrc=$? +if [ $cmdrc -ne 0 ]; then + echo "Error checking if node is primary. Exiting." + exit 1 +fi + +if [ "$is_primary" != "true" ]; then + echo "This node is not primary. Get Primary from server status." + primary_host=$(mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --host {{ mongodb_host }} --authenticationDatabase=admin --tls --eval 'rs.isMaster().primary') + cmdrc=$? + if [ $cmdrc -ne 0 ]; then + echo "Error getting primary node. Exiting." + exit 1 + fi + echo "Primary host is: $primary_host" +fi + +# Exclude these collections because they should be populated by MAS installation +mongodb_ns_exclude_args=" --nsExclude=mas_{{ mas_instance_id }}_adoptionusage.* \ + --nsExclude=mas_{{ mas_instance_id }}_catalog.* \ + --nsExclude=mas_{{ mas_instance_id }}_core.OauthClient \ + --nsExclude=mas_{{ mas_instance_id }}_core.OauthToken \ + --nsExclude=mas_{{ mas_instance_id }}_core.bindings \ + --nsExclude=mas_{{ mas_instance_id }}_core.graphiteconfigtool.* \ + --nsExclude=mas_{{ mas_instance_id }}_core.workspaces" + +if [ -n "$primary_host" ]; then + echo "Connecting to primary host: $primary_host" + mongo_cmd="mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --host $primary_host" + mongorestore_cmd="mongorestore -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --sslCAFile $tlsCAFile --authenticationDatabase=admin --ssl --drop --dir=$TMP_RESTORE_DIR $mongodb_ns_exclude_args --host $primary_host" +else + echo "Continuing on current primary node." + mongo_cmd="mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --host {{ mongodb_host }}" + mongorestore_cmd="mongorestore -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --sslCAFile $tlsCAFile --authenticationDatabase=admin --ssl --drop --dir=$TMP_RESTORE_DIR $mongodb_ns_exclude_args --host {{ mongodb_host }}" +fi + +# Perform restore +echo "Restoring databases from $TMP_RESTORE_DIR" +$mongorestore_cmd +cmdrc=$? +if [ $cmdrc -ne 0 ]; then + echo "Error restoring databases. Exiting." + exit 1 +fi + +echo "DATABASERESTOREstatus-SUCCESS" + diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml index 34fb99117e..12b263335d 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml @@ -5,17 +5,17 @@ kubernetes.core.k8s_info: api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" - register: _mongodbcommunity_output + register: mongodbcommunity_output - name: "Set fact: mongodb version" set_fact: - mongodb_version: "{{ _mongodbcommunity_output.resources[0].spec.version }}" + mongodb_version: "{{ mongodbcommunity_output.resources[0].spec.version }}" when: - - _mongodbcommunity_output is defined - - _mongodbcommunity_output.resources[0] is defined - - _mongodbcommunity_output.resources[0].spec.version is defined + - mongodbcommunity_output is defined + - mongodbcommunity_output.resources[0] is defined + - mongodbcommunity_output.resources[0].spec.version is defined - name: "Fail if mongodb does not exists" assert: @@ -26,13 +26,20 @@ set_fact: mongodb_running: true when: - - _mongodbcommunity_output is defined - - _mongodbcommunity_output.resources[0] is defined - - _mongodbcommunity_output.resources[0].status is defined - - _mongodbcommunity_output.resources[0].status.phase is defined - - _mongodbcommunity_output.resources[0].status.phase == "Running" + - mongodbcommunity_output is defined + - mongodbcommunity_output.resources[0] is defined + - mongodbcommunity_output.resources[0].status is defined + - mongodbcommunity_output.resources[0].status.phase is defined + - mongodbcommunity_output.resources[0].status.phase == "Running" - name: "Fail if mongodb is not running" assert: that: mongodb_running is defined and mongodb_running fail_msg: "Mongodb is not running!" + +- name: "Set MongoDBCommunity Backup output" + set_fact: + mongodbcommunity_backup_output: "{{ mongodbcommunity_output.resources[0] }}" + + + diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml deleted file mode 100644 index 19f45bf41d..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-patch.yml +++ /dev/null @@ -1,76 +0,0 @@ ---- -- name: "Debug: cluster information" - debug: - msg: - - "Source cluster ................... {{ masbr_restore_from_yaml.source.domain }}" - - "Target cluster ................... {{ masbr_cluster_domain }}" - -# Because we will not restore the OauthClient collection, disable below patches for now. -- name: "Update domain in 'mas_{{ mas_instance_id }}_core.OauthClient' collection" - when: masbr_restore_to_diff_domain and false - block: - # Get Suite version - # ----------------------------------------------------------------------------- - - name: "Set fact: mas core namespace name" - set_fact: - mas_core_namespace: "mas-{{ mas_instance_id }}-core" - - - name: "Get Suite" - kubernetes.core.k8s_info: - api_version: core.mas.ibm.com/v1 - kind: Suite - name: "{{ mas_instance_id }}" - namespace: "{{ mas_core_namespace }}" - register: _suite_output - - - name: "Set fact: Suite version" - set_fact: - mas_core_version: "{{ _suite_output.resources[0].status.versions.reconciled }}" - when: - - _suite_output is defined - - (_suite_output.resources | length > 0) - - _suite_output.resources[0].status.versions.reconciled is defined - - - name: "Debug: Suite version" - debug: - msg: "Suite version ..................... {{ mas_core_version }}" - - - name: "This fix only for MAS 8.x" - when: mas_core_version is match("^8.") - block: - # Run oidcclientreg job - # --------------------------------------------------------------------- - - name: "Run oidcclientreg job" - changed_when: true - shell: > - oc get job {{ mas_instance_id }}-oidcclientreg -n {{ mas_core_namespace }} -o yaml - | yq 'del(.spec.selector)' | yq 'del(.spec.template.metadata.labels)' - | oc replace --force -f - - register: _run_job_output - - - name: "Debug: run oidcclientreg job" - debug: - msg: "{{ _run_job_output.stdout_lines }}" - - - name: "Wait for oidcclientreg job to be completed (10s delay)" - kubernetes.core.k8s_info: - api_version: batch/v1 - kind: Job - name: "{{ mas_instance_id }}-oidcclientreg" - namespace: "{{ mas_core_namespace }}" - register: _job_result - until: - - _job_result.resources is defined - - _job_result.resources | length > 0 - - _job_result.resources | json_query('[*].status.conditions[?type==`Complete`][].status') | select ('match','True') | list | length == 1 - retries: 30 - delay: 10 - - # Restart entitymgr-ws pod - # --------------------------------------------------------------------- - - name: "Restart entitymgr-ws pod" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/restart_and_reconsiled.yml" - vars: - _pod_namespace: "{{ mas_core_namespace }}" - _pod_keywords: "entitymgr-ws" - _container_name: "manager" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml deleted file mode 100644 index 581bc827e5..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database-perform.yml +++ /dev/null @@ -1,121 +0,0 @@ ---- -# Input parameters: -# _job_type -# _job_name - -# Copy backup file from specified storage location -# ------------------------------------------------------------------------- -- name: "Copy backup file from specified storage location to pod" - when: _mongodb_cf_in_server - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_pod.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ _job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/{{ _job_name }}.tar.gz" - dest_folder: "{{ mongodb_restore_folder }}" - -- name: "Download and copy backup files to pod" - when: not _mongodb_cf_in_server - block: - - name: "Download backup files from specified storage location to local" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_local.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ _job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/{{ _job_name }}.tar.gz" - dest_folder: "{{ masbr_local_job_folder }}/{{ masbr_job_data_type }}" - - - name: "Copy backup files from local to pod" - changed_when: true - shell: > - oc cp --retries=50 -c {{ mongodb_container_name }} - {{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/{{ _job_name }}.tar.gz - {{ mongodb_namespace }}/{{ mongodb_pod_name }}:{{ mongodb_restore_folder }} - -# Extract the tar.gz file -# ------------------------------------------------------------------------- -- name: "Extract the tar.gz file" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ mongodb_restore_folder }}/{{ _job_name }} && - tar -xzf {{ mongodb_restore_folder }}/{{ _job_name }}.tar.gz - -C {{ mongodb_restore_folder }}/{{ _job_name }} && - ls -lA {{ mongodb_restore_folder }}/{{ _job_name }} - {{ exec_in_pod_end }} - register: _extract_output - -- name: "Debug: list extracted files" - debug: - msg: - - "Extract output folder .............. {{ mongodb_restore_folder }}/{{ _job_name }}" - - "{{ _extract_output.stdout_lines }}" - -# Restore mongodb databases -# ------------------------------------------------------------------------- -- name: "Set fact: mongodb ns instance" - set_fact: - mongodb_ns_instance: "{{ mas_instance_id }}" - -- name: "Set fact: rename mongodb ns" - when: masbr_restore_to_diff_instance - set_fact: - mongodb_ns_rename: >- - --nsFrom="*_{{ masbr_restore_from_yaml.source.instance }}_*.*" --nsTo="*_{{ mas_instance_id }}_*.*" - mongodb_ns_instance: "{{ masbr_restore_from_yaml.source.instance }}" - -- name: "Set fact: mongodb ns filters (Only for restoring from Full backup)" - when: _job_type == "full" - set_fact: - # Exclude these collections because they should be populated by MAS installation - mongodb_ns_filter: >- - --nsExclude="mas_{{ mongodb_ns_instance }}_adoptionusage.*" - --nsExclude="mas_{{ mongodb_ns_instance }}_catalog.*" - --nsExclude="mas_{{ mongodb_ns_instance }}_core.OauthClient" - --nsExclude="mas_{{ mongodb_ns_instance }}_core.OauthToken" - --nsExclude="mas_{{ mongodb_ns_instance }}_core.bindings" - --nsExclude="mas_{{ mongodb_ns_instance }}_core.graphiteconfigtool.*" - --nsExclude="mas_{{ mongodb_ns_instance }}_core.workspaces" - -- name: "Debug: mongodb ns filters (Only for restoring from Full backup)" - when: _job_type == "full" - debug: - msg: "{{ mongodb_ns_filter }}" - -- name: "Restore mongodb databases" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mongorestore --host={{ mongodb_primary_host }} - --username={{ mongodb_user }} --password={{ mongodb_password }} - --authenticationDatabase=admin --ssl --sslCAFile={{ mongodb_ca_file }} - {{ '--oplogReplay' if _job_type == 'incr' }} - {{ mongodb_ns_filter if _job_type == 'full' }} - {{ mongodb_ns_rename if masbr_restore_to_diff_instance }} - --drop --dir={{ mongodb_restore_folder }}/{{ _job_name }} - |& tee -a {{ mongodb_restore_log }} - {{ exec_in_pod_end }} - register: _mongorestore_output - no_log: true - -- name: "Debug: mongorestore output" - debug: - msg: "{{ _mongorestore_output.stdout_lines }}" - -# Save restored database names -# ------------------------------------------------------------------------- -- name: "Get restored database names" - changed_when: false - when: _job_type == "full" - shell: > - {{ exec_in_pod_begin }} - ls {{ mongodb_restore_folder }}/{{ _job_name }} - {{ exec_in_pod_end }} - register: _ls_output - -- name: "Set fact: restored database names" - when: _job_type == "full" - set_fact: - mongodb_restored_db_names: "{{ _ls_output.stdout_lines }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index cfbdde90d9..4d7400d968 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -1,123 +1,123 @@ --- -# Update database restore status: InProgress + +- name: "Set fact: mongodb restore paths and filename" + set_fact: + mongodb_backup_data_filename: "mongodump-{{ mongodb_backup_version }}.tar.gz" + mongodb_tmp_restore_dir: "/tmp/masbr/{{ mongodb_backup_version }}" + +- name: "Check if backup data file exists" + stat: + path: "{{ mongodb_data_path }}/{{ mongodb_backup_data_filename }}" + register: backup_data_file_check + +- name: Assert backup data file exists + assert: + that: + - backup_data_file_check.stat.exists | default(False) + fail_msg: "Backup data file {{ mongodb_data_path }}/{{ mongodb_backup_data_filename }} does NOT exist!" + +- name: "Set facts: from mongodb cr and resources" + ibm.mas_devops.get_mongoce_info: + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_namespace: "{{ mongodb_namespace }}" + register: mongodb_info_result + +- name: "Assert mongodb_info_result" + assert: + that: + - mongodb_info_result.success + +- name: "Set fact: mongodb info" + set_fact: + mongoce_pod_name: "{{ mongodb_info_result.mongoce_pod_name }}" + mongodb_version: "{{ mongodb_info_result.mongodb_version }}" + mongodb_service_name: "{{ mongodb_info_result.mongodb_service_name }}" + mongodb_host: "{{ mongodb_info_result.mongodb_host }}" + mongodb_admin_user: "{{ mongodb_info_result.mongodb_admin_user }}" + mongodb_admin_password: "{{ mongodb_info_result.mongodb_admin_password }}" + +- name: "debug facts" + debug: + msg: + - "mongoce_pod_name .................................. {{ mongoce_pod_name }}" + - "mongodb_version ................................... {{ mongodb_version }}" + - "mongodb_service_name .............................. {{ mongodb_service_name }}" + - "mongodb_host ...................................... {{ mongodb_host }}" + - "mongodb_backup_version ............................ {{ mongodb_backup_version }}" + +# Prepare shell scripts for restore and transfer scripts to mongo pod # ----------------------------------------------------------------------------- -- name: "Update database restore status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - -- name: "Restore mongodb databases" - block: - # Create mongodb role and user for backing up databases - # ------------------------------------------------------------------------- - - name: "Create mongodb role and user for backing up databases" - include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/create-role-user.yml" - - # Prepare mongodb database restore folders - # ------------------------------------------------------------------------- - - name: "Set fact: mongodb database restore variables" - set_fact: - # We should use mongodb pod ephemeral local storage to save the temporary files, - # the mongodb data pvc size is not big enough. - mongodb_restore_folder: "{{ mongodb_pod_temp_folder }}/{{ masbr_job_data_type }}" - - - name: "Set fact: mongodb database restore log" - set_fact: - mongodb_restore_log: "{{ mongodb_restore_folder }}/{{ masbr_job_name }}-restore.log" - - - name: "Create mongodb database restore folder in pod" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - mkdir -p {{ mongodb_restore_folder }}; - chmod a+w {{ mongodb_restore_folder }} - {{ exec_in_pod_end }} - - - name: "Debug: mongodb database restore folder in pod" - debug: - msg: "Database restore folder ........... {{ mongodb_restore_folder }}" - - # This is an incremental backup, need to restore based on full backup first - # ------------------------------------------------------------------------- - - name: "Restore based on full backup" - when: masbr_restore_from_incr - block: - - name: "Copy based on full backup file from specified storage location to pod" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database-perform.yml" - vars: - _job_type: "full" - _job_name: "{{ masbr_restore_basedon }}" - - # Restore databases from the specified Full or Incremental backup - # ------------------------------------------------------------------------- - - name: "Restore databases from the specified {{ 'Incremental' if masbr_restore_from_incr else 'Full' }} backup" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database-perform.yml" - vars: - _job_type: "{{ 'incr' if masbr_restore_from_incr else 'full' }}" - _job_name: "{{ masbr_restore_from }}" - - # Do some post restoration tasks - # ------------------------------------------------------------------------- - - name: "Do some post restoration tasks " - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database-patch.yml" - - # Update database restore status: Completed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update database restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy mongodb restore log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of mongodb restore log" - changed_when: true - shell: > - {{ exec_in_pod_begin }} - tar -czf {{ mongodb_restore_folder }}/{{ masbr_job_name }}-restore-log.tar.gz - -C {{ mongodb_restore_folder }} {{ masbr_job_name }}-restore.log - {{ exec_in_pod_end }} - - - name: "Copy mongodb restore log file from pod to specified storage location" - when: _mongodb_cf_in_server - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_pod_files_to_storage.yml" - vars: - masbr_cf_job_type: "restore" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ mongodb_restore_folder }}/{{ masbr_job_name }}-restore-log.tar.gz" - dest_folder: "log" - - - name: "Download and copy restore log file to specified storage location" - when: not _mongodb_cf_in_server - block: - - name: "Download restore log file from pod to local" - changed_when: true - shell: > - oc cp --retries=50 -c {{ mongodb_container_name }} - {{ mongodb_namespace }}/{{ mongodb_pod_name }}:{{ mongodb_restore_folder }}/{{ masbr_job_name }}-restore-log.tar.gz - {{ masbr_local_job_folder }}/log/{{ masbr_job_name }}-restore-log.tar.gz - - - name: "Copy restore log file from local to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "restore" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "log/{{ masbr_job_name }}-restore-log.tar.gz" - dest_folder: "log" +- block: + - name: Create create-role-user.sh script in local /tmp + ansible.builtin.template: + src: create-role-user.sh.j2 + dest: /tmp/create-role-user.sh + mode: '777' + + - name: Create database-backup.sh script in local /tmp + ansible.builtin.template: + src: database-restore.sh.j2 + dest: /tmp/database-restore.sh + mode: '777' + + - name: Copy the create-role-user.sh script into the mongodb pod + shell: "oc cp /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" + register: copy_result + retries: 2 + delay: 15 # seconds + until: copy_result.rc == 0 + + - name: Copy the database-restore.sh script into the mongodb pod + shell: "oc cp /tmp/database-restore.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-restore.sh -c mongod" + register: copy_result + retries: 2 + delay: 15 # seconds + until: copy_result.rc == 0 + +# The log file will also be available inside the pod /tmp/create-role-user.log +- name: Exec into mongo pod and run create-role-user.sh to setup backup role and user in Mongodb. + shell: | + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/create-role-user.sh + register: create_roleuser_output + ignore_errors: true + retries: 1 + delay: 10 # seconds + until: + - create_roleuser_output.rc == 0 + - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) + +- name: "Debug create-role-user logs" + debug: + msg: "{{ create_roleuser_output.stdout_lines }}" + +# Create temporary restore directory +- name: "Create temporary restore directory in mongodb pod and clean up any existing files" + shell: | + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- mkdir -p {{ mongodb_tmp_restore_dir }} && rm -rf {{ mongodb_tmp_restore_dir }}/* + register: create_tmpdir_output + retries: 2 + delay: 15 # seconds + until: create_tmpdir_output.rc == 0 + +- name: "Copy backup data file into mongodb pod" + shell: | + oc cp {{ mongodb_data_path }}/{{ mongodb_backup_data_filename }} {{ mongodb_namespace }}/{{ mongoce_pod_name }}:{{ mongodb_tmp_restore_dir }}/{{ mongodb_backup_data_filename }} -c mongod + register: copy_backupdata_output + retries: 2 + delay: 15 # seconds + until: copy_backupdata_output.rc == 0 + +- name: "Exec into mongo pod and run database-restore.sh to restore the database from backup." + shell: | + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/database-restore.sh + register: database_restore_output + ignore_errors: true + retries: 1 + delay: 10 # seconds + until: + - database_restore_output.rc == 0 + - database_restore_output.stdout | regex_search('DATABASERESTOREstatus-SUCCESS', multiline=True) + +- name: "Debug database-restore logs" + debug: + msg: "{{ database_restore_output.stdout_lines }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index 605d26764e..fffc6f97c6 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -1,75 +1,58 @@ --- # Check mongodb backup variables # ----------------------------------------------------------------------------- -- name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" -# Get mongodb information -# ------------------------------------------------------------------------- -- name: "Get mongodb information" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/get-mongo-info.yml" +- name: "Fail if require variables for Mongodb backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_instance_name: "{{ mongodb_instance_name }}" + action: "{{ mongodb_action }}" + component: "mongodb" -# Set common backup job variables -# ----------------------------------------------------------------------------- -- name: "Set fact: common backup job variables" +- name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - masbr_job_component: - name: "mongodb" - instance: "{{ mas_instance_id }}" - app: "{{ mas_app_id }}" - namespace: "{{ mongodb_namespace }}" - provider: "{{ mongodb_provider }}" - version: "{{ mongodb_version }}" - masbr_job_data_list: - - seq: "1" - type: "database" + mongodb_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: mongodb_backup_version is not defined or mongodb_backup_version == "" -# Before run tasks -# ------------------------------------------------------------------------- -- name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _ignore_masbr_backup_data: true - _job_type: "backup" - _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/before-backup-restore.yml" +- name: "Set fact: Create Mongodb backup base directory path" + set_fact: + mongodb_backup_path: "{{ mas_backup_dir }}/{{ mongodb_action }}-{{ mongodb_backup_version }}-mongoce" -- name: "Perform backup" +- name: "Create {{ mongodb_backup_path }} directory for Mongodb backup" + file: + path: "{{ mongodb_backup_path }}" + state: directory + mode: "0755" + +# Backup Mongodb Kubernetes Resources +# ------------------------------------------------------------------------- +- name: "Start Mongo Instance backup process." block: - # Update backup job status: New - # ------------------------------------------------------------------------- - - name: "Update backup job status: New" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "1" - phase: "New" + - name: "Backup MongoDB Instance Kubernetes resources" + ibm.mas_devops.backup_mongo_instance: + mongodb_backup_path: "{{ mongodb_backup_path }}" + mongodb_namespace: "{{ mongodb_namespace }}" + mongodb_instance_name: "{{ mongodb_instance_name }}" + register: mongo_instance_backup_result - # Run backup tasks for each data type - # ------------------------------------------------------------------------- - - name: "Run backup tasks for each data type" - include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/backup-{{ job_data_item.type }}.yml" - vars: - masbr_job_data_seq: "{{ job_data_item.seq }}" - masbr_job_data_type: "{{ job_data_item.type }}" - loop: "{{ masbr_job_data_list }}" - loop_control: - loop_var: job_data_item + - name: "Assert Mongo Instance backup success" + assert: + that: + - mongo_instance_backup_result is defined + - mongo_instance_backup_result.success == true + fail_msg: "MongoDB Instance resources backup failed." + when: not skip_instance_backup_restore + +# Backup Mongodb database Data using mondodump +# ------------------------------------------------------------------------- - rescue: - # Update backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update database backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_status: - phase: "Failed" +- name: "Debug information - Mongodb database Data backup" + debug: + msg: + - "MongoCE namespace .......................... {{ mongodb_namespace }}" + - "MongoCE instance name ...................... {{ mongodb_instance_name }}" + - "{{ 'MongoCE version ............................ ' ~ mongodb_version if mongodb_version is defined else omit }}" - always: - # After run tasks - # ------------------------------------------------------------------------- - - name: "After run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" - vars: - _component_after_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/after-backup-restore.yml" +- name: "Start Database backup process." + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/backup-database.yml" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml index 6b14c39aac..14606bed8f 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml @@ -3,7 +3,7 @@ kubernetes.core.k8s_info: api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" register: existing_mongodb diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml index 2737278a1b..88cd591821 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml @@ -24,7 +24,7 @@ kubernetes.core.k8s_info: api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" register: existing_mongodb until: diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml index 55a4c5f870..646b11496e 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml @@ -146,6 +146,7 @@ wait_condition: status: "True" type: Ready + when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources - name: "Create a ca certificate in '{{ mongodb_namespace }}' namespace" kubernetes.core.k8s: @@ -155,6 +156,7 @@ wait_condition: status: "True" type: Ready + when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources - name: "Create a Issuer for server certificate in '{{ mongodb_namespace }}' namespace" kubernetes.core.k8s: @@ -164,6 +166,7 @@ wait_condition: status: "True" type: Ready + when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources - name: "Create a server certificate in '{{ mongodb_namespace }}' namespace" kubernetes.core.k8s: @@ -173,6 +176,7 @@ wait_condition: status: "True" type: Ready + when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources # Taking ca.crt value in a variable # Mongo needs a configmap having ca.crt value @@ -189,7 +193,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: ConfigMap - name: mas-mongo-ce-cert-map + name: "{{ mongodb_instance_name }}-cert-map" namespace: "{{ mongodb_namespace }}" register: ca_cfgmap_info @@ -210,7 +214,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: Secret - name: mas-mongo-ce-admin-password + name: "{{ mongodb_instance_name }}-admin-password" namespace: "{{ mongodb_namespace }}" register: admin_password_info @@ -228,7 +232,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: Secret - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" namespace: "{{ mongodb_namespace }}" register: metrics_endpoint_secret_info when: @@ -311,15 +315,15 @@ - debug: var: scale_down_mongo_deployment_output.stdout - - name: "community : install : Delete mas-mongo-ce statefulset so next time it recreates with right image" - shell: oc delete statefulset mas-mongo-ce -n {{ mongodb_namespace }} + - name: "community : install : Delete {{ mongodb_instance_name }} statefulset so next time it recreates with right image" + shell: oc delete statefulset {{ mongodb_instance_name }} -n {{ mongodb_namespace }} register: delete_mongo_ss_output - debug: var: delete_mongo_ss_output.stdout - - name: "community : install : Delete mas-mongo-ce-arb statefulset so next time it recreates with right image" - shell: oc delete statefulset mas-mongo-ce-arb -n {{ mongodb_namespace }} + - name: "community : install : Delete {{ mongodb_instance_name }}-arb statefulset so next time it recreates with right image" + shell: oc delete statefulset {{ mongodb_instance_name }}-arb -n {{ mongodb_namespace }} register: delete_mongo_ss_arb_output - debug: @@ -332,11 +336,11 @@ - debug: var: scale_up_mongo_deployment_output.stdout -- name: "community : install : Wait for mas-mongo-ce stateful set to be ready" +- name: "community : install : Wait for {{ mongodb_instance_name }} stateful set to be ready" kubernetes.core.k8s_info: api_version: apps/v1 kind: StatefulSet - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" vars: mongodb_replicas_check: "{{ existing_mongodb.resources[0].spec.members | default(mongodb_replicas|int) }}" @@ -349,12 +353,12 @@ - mongodb_statefulset.resources[0].status.readyReplicas is defined - mongodb_statefulset.resources[0].status.readyReplicas == (mongodb_replicas_check|int) -- name: "community : install : Wait for mas-mongo-ce-arb stateful set to be ready" +- name: "community : install : Wait for {{ mongodb_instance_name }}-arb stateful set to be ready" when: target_mongodb_version is version('4.4.0','>=') # this statefulset will only exist in Mongo v4.4+ kubernetes.core.k8s_info: api_version: apps/v1 kind: StatefulSet - name: mas-mongo-ce-arb + name: "{{ mongodb_instance_name }}-arb" namespace: "{{ mongodb_namespace }}" register: mongodb_arb_statefulset retries: 45 # Approx 90 minutes @@ -369,7 +373,7 @@ kubernetes.core.k8s_info: api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" register: mongodb_cr retries: 45 # Approx 45 minutes @@ -384,7 +388,7 @@ kubernetes.core.k8s_info: api_version: monitoring.coreos.com/v1 kind: ServiceMonitor - name: mas-mongo-ce-service-monitor + name: "{{ mongodb_instance_name }}-service-monitor" namespace: "{{ mongodb_namespace }}" register: mongoce_servicemonitor_info when: @@ -477,7 +481,7 @@ kind: Pod namespace: "{{ mongodb_namespace }}" label_selectors: - - app=mas-mongo-ce-svc + - "app={{ mongodb_instance_name }}-svc" register: mongo_pods_output # List all mongo replicas @@ -498,7 +502,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: Secret - name: mas-mongo-ce-admin-admin + name: "{{ mongodb_instance_name }}-admin-admin" namespace: "{{ mongodb_namespace }}" register: admin_password_lookup retries: 30 # 30 * 30 seconds = 30 minutes @@ -512,7 +516,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: ConfigMap - name: mas-mongo-ce-cert-map + name: "{{ mongodb_instance_name }}-cert-map" namespace: "{{ mongodb_namespace }}" register: mongodb_ca_lookup diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml index 5cb5b4be8f..ffabbbd1eb 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml @@ -1,77 +1,141 @@ --- # Check mongodb restore required variables # ----------------------------------------------------------------------------- -- name: "Set fact: " - set_fact: - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +- name: "Fail if require variables for Mongodb {{ mongodb_action }} are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_backup_version: "{{ mongodb_backup_version }}" + mongodb_instance_name: "{{ mongodb_instance_name }}" + action: "{{ mongodb_action }}" + component: "mongodb" -- name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" +- name: "Set fact: backup dir paths" + set_fact: + mongodb_backup_dir: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" + mongodb_resource_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/resources" + mongodb_data_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/data" -# Get mongodb information -# ------------------------------------------------------------------------- -- name: "Get mongodb information" - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/get-mongo-info.yml" +# Check for existing MongoDb install +# ----------------------------------------------------------------------------- +- name: "Check for existing MongoDB installation in namespace {{ mongodb_namespace }}" + ibm.mas_devops.verify_mongoce_version: + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_namespace: "{{ mongodb_namespace }}" + register: existing_mongo_info -# Set common restore job variables +# Prepare for restore # ----------------------------------------------------------------------------- -- name: "Set fact: common restore job variables" - set_fact: - masbr_job_component: - name: "mongodb" - instance: "{{ mas_instance_id }}" - namespace: "{{ mongodb_namespace }}" - provider: "{{ mongodb_provider }}" - version: "{{ mongodb_version }}" - masbr_job_data_list: - - seq: "1" - type: "database" +- name: "Prepare for MongoCE instance restore" + when: + - not skip_instance_backup_restore + - existing_mongo_info is defined + - not existing_mongo_info.running + block: -# Before run tasks -# ------------------------------------------------------------------------- -- name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "restore" - _component_before_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/before-backup-restore.yml" + # Get mongodb backup CR information + # ----------------------------------------------------------------------------- + - name: "Check and get mongodb backup cr.yml" + ibm.mas_devops.get_mongodb_cr_to_restore: + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_backup_version: "{{ mongodb_backup_version }}" + register: mongodb_backup_cr_result -- name: "Perform restore" - block: - # Update restore job status: New - # ------------------------------------------------------------------------- - - name: "Update restore job status: New" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "1" - phase: "New" + - name: "Set fact : cr.yml" + set_fact: + mongodb_cr: "{{ mongodb_backup_cr_result.mongodb_cr }}" + + - name: "Set fact: Retrieve information from backup CR" + set_fact: + mongodb_namespace: "{{ mongodb_cr.metadata.namespace }}" + mongodb_instance_name: "{{ mongodb_cr.metadata.name }}" + mongo_extras_version: "{{ mongodb_cr.spec.version }}" + target_mongodb_version: "{{ mongodb_cr.spec.version }}" + mongodb_security: "{{ mongodb_cr.spec.security }}" + mongodb_users: "{{ mongodb_cr.spec.users | default([]) }}" + mongodb_replicas: "{{ mongodb_cr.spec.members }}" + mongodb_cpu_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('500m') }}" + mongodb_memory_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('1Gi') }}" + mongodb_cpu_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('250m') }}" + mongodb_memory_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('512Mi') }}" + backup_mongodb_storage_class: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName }}" + mongodb_storage_capacity_data: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.resources.requests.storage }}" + mongodb_storage_capacity_logs: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[1].spec.resources.requests.storage }}" - # Run restore tasks for each data type - # ------------------------------------------------------------------------- - - name: "Run restore tasks for each data type" - include_tasks: "tasks/providers/{{ mongodb_provider }}/backup-restore/restore-{{ job_data_item.type }}.yml" - vars: - masbr_job_data_seq: "{{ job_data_item.seq }}" - masbr_job_data_type: "{{ job_data_item.type }}" - loop: "{{ masbr_job_data_list }}" - loop_control: - loop_var: job_data_item + - name: "Debug: Mongodb restore variables" + debug: + msg: + - "mongodb_namespace: {{ mongodb_namespace }}" + - "mongodb_instance_name: {{ mongodb_instance_name }}" + - "mongo_extras_version: {{ mongo_extras_version }}" + - "target_mongodb_version: {{ target_mongodb_version }}" + - "mongodb_security: {{ mongodb_security }}" + - "mongodb_users: {{ mongodb_users }}" + - "mongodb_replicas: {{ mongodb_replicas }}" + - "mongodb_cpu_limits: {{ mongodb_cpu_limits }}" + - "mongodb_memory_limits: {{ mongodb_memory_limits }}" + - "mongodb_cpu_requests: {{ mongodb_cpu_requests }}" + - "mongodb_memory_requests: {{ mongodb_memory_requests }}" + - "backup_mongodb_storage_class: {{ backup_mongodb_storage_class }}" + - "mongodb_storage_capacity_data: {{ mongodb_storage_capacity_data }}" + - "mongodb_storage_capacity_logs: {{ mongodb_storage_capacity_logs }}" - rescue: - # Update restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_status: - phase: "Failed" + # Prepare storage class + # ----------------------------------------------------------------------------- + # Set storage class name to use for mongodb restore. + # If existing mongo instance found, use its storage class. + # Else use storage class from backup CR. + # If that does not exist, use default storage class. + - name: "Determine storage class to use for MongoDB restore" + ibm.mas_devops.verify_mongoce_version: + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_namespace: "{{ mongodb_namespace }}" + backup_mongodb_storage_class: "{{ backup_mongodb_storage_class }}" + register: storage_class_check_result - always: - # After run tasks - # ------------------------------------------------------------------------- - - name: "After run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" - vars: - _component_after_task_path: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/after-backup-restore.yml" + - name: "Set fact: mongodb_storage_class to use for restore" + set_fact: + mongodb_storage_class: "{{ storage_class_check_result.storage_class }}" + when: + - storage_class_check_result is defined + - storage_class_check_result.storage_class is defined + + # Restore MongoDb instance's resources from backup + # We will restore the following resources from backup: + # - User, TLS Secrets defined in the MongoDb CR + # - Issuers in the MongoDb namespace + # - Certificates in the MongoDb namespace + # All the other resources will be created afresh during MongoDb installation + # ----------------------------------------------------------------------------- + + - name: "Restore MongoDB instance resources from backup" + ibm.mas_devops.restore_mongoce_resources: + mongodb_namespace: "{{ mongodb_namespace }}" + backup_secrets_path: "{{ mongodb_resource_path }}/secrets" + backup_issuers_path: "{{ mongodb_resource_path }}/issuers" + backup_certificates_path: "{{ mongodb_resource_path }}/certificates" + register: restore_mongoce_resources_result + + - name: "Assert: MongoDB resources restored successfully" + assert: + that: + - restore_mongoce_resources_result is defined + - restore_mongoce_resources_result.restored is defined + - restore_mongoce_resources_result.restored == True + fail_msg: "Failed to restore MongoDB instance resources from backup." + + # Install the selected MongoDb version for restore + # ----------------------------------------------------------------------------- + - name: "community : Setup mongo version {{ target_mongodb_version }} for restore" + include_tasks: tasks/providers/community/install-mongo.yml + when: + - existing_mongo_info is defined + - not existing_mongo_info.running + +# Restore the database data from backup +# ----------------------------------------------------------------------------- +- name: "Start Database restore process." + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database.yml" + when: + - existing_mongo_info is defined + - existing_mongo_info.running diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml index e71fcd4082..d412b42526 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml @@ -6,7 +6,7 @@ msg: - "MongoDb namespace ............ {{ mongodb_namespace }}" -# 2. Delete MongoDBCommunity (mas-mongo-ce) +# 2. Delete MongoDBCommunity CR # ----------------------------------------------------------------------------- - name: "community : uninstall : Delete the MongoDBCommunity CR" kubernetes.core.k8s: @@ -14,7 +14,7 @@ api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity namespace: "{{ mongodb_namespace }}" - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" # 3. Wait for StatefulSet to shut down # ----------------------------------------------------------------------------- @@ -37,7 +37,7 @@ api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity namespace: "{{ mongodb_namespace }}" - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" register: verify_mongo_delete - name: "uninstall : Verify the MongoDBCommunity CR was deleted" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 index 51b8a460e7..a08fc54ee6 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -19,7 +19,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: mas-mongo-ce-cert-map + name: "{{ mongodb_instance_name }}-cert-map" authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -28,12 +28,27 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" +{% if mongodb_users is defined %} + users: +{% for user in mongodb_users %} + - name: {{ user.name }} + db: {{ user.db }} + passwordSecretRef: + name: {{ user.passwordSecretRef.name }} + roles: +{% for role in user.roles %} + - name: {{ role.name }} + db: {{ role.db }} +{% endfor %} + scramCredentialsSecretName: "{{ user.scramCredentialsSecretName }}" +{% endfor %} +{% else %} users: - name: admin db: admin passwordSecretRef: - name: mas-mongo-ce-admin-password + name: "{{ mongodb_instance_name }}-admin-password" roles: - name: clusterAdmin db: admin @@ -43,14 +58,15 @@ spec: db: admin - name: readWriteAnyDatabase db: admin - scramCredentialsSecretName: mas-mongo-ce-scram + scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" +{% endif %} additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: mas-mongo-ce-svc + serviceName: "{{ mongodb_instance_name }}-svc" selector: {} template: spec: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 index e44b3468a6..2a9f190c90 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: mas-mongo-ce + name: "{{ mongodb_instance_name }} namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -18,7 +18,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: mas-mongo-ce-cert-map + name: "{{ mongodb_instance_name }}-cert-map" authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -27,12 +27,12 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" users: - name: admin db: admin passwordSecretRef: - name: mas-mongo-ce-admin-password + name: "{{ mongodb_instance_name }}-admin-password" roles: - name: clusterAdmin db: admin @@ -44,14 +44,14 @@ spec: db: admin - name: dbAdminAnyDatabase db: admin - scramCredentialsSecretName: mas-mongo-ce-scram + scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: mas-mongo-ce-svc + serviceName: "{{ mongodb_instance_name }}-svc" selector: {} template: spec: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 index 8cb97b18ea..c912aff927 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -19,7 +19,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: mas-mongo-ce-cert-map + name: "{{ mongodb_instance_name }}-cert-map" authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -28,12 +28,27 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" +{% if mongodb_users is defined %} + users: +{% for user in mongodb_users %} + - name: {{ user.name }} + db: {{ user.db }} + passwordSecretRef: + name: {{ user.passwordSecretRef.name }} + roles: +{% for role in user.roles %} + - name: {{ role.name }} + db: {{ role.db }} +{% endfor %} + scramCredentialsSecretName: "{{ user.scramCredentialsSecretName }}" +{% endfor %} +{% else %} users: - name: admin db: admin passwordSecretRef: - name: mas-mongo-ce-admin-password + name: "{{ mongodb_instance_name }}-admin-password" roles: - name: clusterAdmin db: admin @@ -45,14 +60,15 @@ spec: db: admin - name: dbAdminAnyDatabase db: admin - scramCredentialsSecretName: mas-mongo-ce-scram + scramCredentialsSecretName: ""{{ mongodb_instance_name }}-scram" +{% endif %} additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: mas-mongo-ce-svc + serviceName: "{{ mongodb_instance_name }}-svc" selector: {} template: spec: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 index 51b8a460e7..a08fc54ee6 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: mas-mongo-ce + name: "{{ mongodb_instance_name }}" namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -19,7 +19,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: mas-mongo-ce-cert-map + name: "{{ mongodb_instance_name }}-cert-map" authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -28,12 +28,27 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" +{% if mongodb_users is defined %} + users: +{% for user in mongodb_users %} + - name: {{ user.name }} + db: {{ user.db }} + passwordSecretRef: + name: {{ user.passwordSecretRef.name }} + roles: +{% for role in user.roles %} + - name: {{ role.name }} + db: {{ role.db }} +{% endfor %} + scramCredentialsSecretName: "{{ user.scramCredentialsSecretName }}" +{% endfor %} +{% else %} users: - name: admin db: admin passwordSecretRef: - name: mas-mongo-ce-admin-password + name: "{{ mongodb_instance_name }}-admin-password" roles: - name: clusterAdmin db: admin @@ -43,14 +58,15 @@ spec: db: admin - name: readWriteAnyDatabase db: admin - scramCredentialsSecretName: mas-mongo-ce-scram + scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" +{% endif %} additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: mas-mongo-ce-svc + serviceName: "{{ mongodb_instance_name }}-svc" selector: {} template: spec: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml b/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml index b6eece0373..8e96577723 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: - name: mas-mongo-ce-admin-password + name: "{{ mongodb_instance_name }}-admin-password" namespace: "{{ mongodb_namespace }}" type: Opaque stringData: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml b/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml index 9637715c32..e8174be116 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml @@ -13,7 +13,7 @@ spec: algorithm: ECDSA size: 256 dnsNames: - - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" + - "*.{{mongodb_instance_name}}-svc.{{mongodb_namespace}}.svc.cluster.local" - "127.0.0.1" - "localhost" issuerRef: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 index f5081d0825..212a04c75f 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" namespace: "{{ mongodb_namespace }}" type: Opaque stringData: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 index 8a7c795c2c..0238035588 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 @@ -1,4 +1,4 @@ {% for host in mongo_replicas %} -- host: "{{ host }}.mas-mongo-ce-svc.{{ mongodb_namespace }}.svc.cluster.local" +- host: "{{ host }}.{{ mongodb_instance_name }}-svc.{{ mongodb_namespace }}.svc.cluster.local" port: 27017 {% endfor %} diff --git a/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml b/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml index 3c6ca1e53f..9feaeddcd8 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml @@ -7,7 +7,7 @@ spec: duration: 8760h # 365d renewBefore: 360h # 15d dnsNames: - - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" + - "*.{{mongodb_instance_name}}-svc.{{mongodb_namespace}}.svc.cluster.local" - "127.0.0.1" - "localhost" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 index 3bf6ec62f8..f35f80d64f 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 @@ -2,17 +2,17 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: mas-mongo-ce-service-monitor + name: "{{ mongodb_instance_name }}-service-monitor" namespace: "{{ mongodb_namespace }}" spec: endpoints: - basicAuth: password: key: password - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" username: key: username - name: mas-mongo-ce-metrics-endpoint-secret + name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" # This port matches what we created in our MongoDB Service. port: prometheus @@ -33,4 +33,4 @@ spec: # Service labels to match selector: matchLabels: - app: mas-mongo-ce-svc \ No newline at end of file + app: "{{ mongodb_instance_name }}-svc" \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/tls.yml b/ibm/mas_devops/roles/mongodb/templates/community/tls.yml index 9e9b832dbf..030ef9f4f8 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/tls.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/tls.yml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: mas-mongo-ce-cert-map + name: "{{ mongodb_instance_name }}-cert-map" namespace: "{{ mongodb_namespace }}" type: Opaque data: diff --git a/ibm/mas_devops/roles/ocp_provision/tasks/providers/fyre/provision_fyre.yml b/ibm/mas_devops/roles/ocp_provision/tasks/providers/fyre/provision_fyre.yml index 56769868d4..9dbec3b747 100644 --- a/ibm/mas_devops/roles/ocp_provision/tasks/providers/fyre/provision_fyre.yml +++ b/ibm/mas_devops/roles/ocp_provision/tasks/providers/fyre/provision_fyre.yml @@ -87,8 +87,8 @@ timeout: 300 use_proxy: no register: _cluster_create - delay: 60 # Every 1 minute - retries: 30 + delay: 120 # Every 2 minutes + retries: 10 - name: "fyre : Debug cluster provision" debug: diff --git a/ibm/mas_devops/roles/ocs/tasks/upgrade/main.yml b/ibm/mas_devops/roles/ocs/tasks/upgrade/main.yml index 204b5ae9d9..f51357ba1d 100644 --- a/ibm/mas_devops/roles/ocs/tasks/upgrade/main.yml +++ b/ibm/mas_devops/roles/ocs/tasks/upgrade/main.yml @@ -163,7 +163,7 @@ - name: "Update the storage cluster version if needed" when: - - storageCluster_version is defined and storageCluster_version != '' + - storageCluster_version is defined and storageCluster_version != None and storageCluster_version != '' - storageCluster_version is not in ocp_version kubernetes.core.k8s_json_patch: api_version: v1 diff --git a/ibm/mas_devops/roles/suite_app_backup/README.md b/ibm/mas_devops/roles/suite_app_backup/README.md new file mode 100644 index 0000000000..9c729300cd --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_backup/README.md @@ -0,0 +1,98 @@ +Backup and Restore MAS Applications +=============================================================================== + +Overview +------------------------------------------------------------------------------- +This role supports backing up the data for below MAS applications: + +- `manage`: Manage namespace resources, persistent volume data (e.g. attachments) + + + +Supports creating on-demand full backups. + +!!! important + An application backup can only be restored to an instance with the same MAS instance ID. + + +Role Variables - General +------------------------------------------------------------------------------- +### mas_app_id +Defines the MAS application ID (`manage`, `iot`, `monitor`, `health`, `optimizer`, or `visualinspection`) for the backup or restore action. + +- **Required** +- Environment Variable: `MAS_APP_ID` +- Default: None + +### mas_instance_id +Defines the MAS instance ID for the backup or restore action. + +- **Required** +- Environment Variable: `MAS_INSTANCE_ID` +- Default: None + +### mas_workspace_id +Defines the MAS workspace ID for the backup or restore action. + +- **Required** +- Environment Variable: `MAS_WORKSPACE_ID` +- Default: None + + +Role Variables - Manage +------------------------------------------------------------------------------- +### masbr_manage_pvc_paths +Set the Manage PVC paths to use in backup and restore. The PVC path is in the format of `:/`. Multiple PVC paths are separated by commas (e.g. `manage-doclinks1-pvc:/mnt/doclinks1/attachments,manage-doclinks2-pvc:/mnt/doclinks2`). + +The `` and `` are defined in the `ManageWorkspace` CRD instance `spec.settings.deployment.persistentVolumes`: +``` +persistentVolumes: + - accessModes: + - ReadWriteMany + mountPath: /mnt/doclinks1 + pvcName: manage-doclinks1-pvc + size: '20' + storageClassName: ocs-storagecluster-cephfs + volumeName: '' + - accessModes: + - ReadWriteMany + mountPath: /mnt/doclinks2 + pvcName: manage-doclinks2-pvc + size: '20' + storageClassName: ocs-storagecluster-cephfs + volumeName: '' +``` + +If not set a value for this variable, this role will not backup and restore persistent valumne data for Manage. + + +- Optional +- Environment Variable: `MASBR_MANAGE_PVC_PATHS` +- Default: None + + +Example Playbook +------------------------------------------------------------------------------- + +### Backup +Backup Manage attachments, note that this does not include backup of any data in Db2, see the `backup` action in the [db2](db2.md) role. + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + masbr_action: backup + mas_instance_id: main + mas_workspace_id: ws1 + mas_app_id: manage + masbr_backup_data: pv + masbr_manage_pvc_paths: "manage-doclinks1-pvc:/mnt/doclinks1" + masbr_storage_local_folder: /tmp/masbr + roles: + - ibm.mas_devops.suite_app_backup_restore +``` + diff --git a/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml b/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml new file mode 100644 index 0000000000..77e18c291e --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml @@ -0,0 +1,3 @@ +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" +mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml new file mode 100644 index 0000000000..b3e958d33b --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml @@ -0,0 +1,18 @@ +--- +# Check mas app backup/restore required variables +# ----------------------------------------------------------------------------- +- name: "Fail if mas_instance_id is not provided" + assert: + that: mas_instance_id is defined and mas_instance_id != "" + fail_msg: "mas_instance_id is required" + +- name: "Fail if mas_workspace_id is not provided" + assert: + that: mas_workspace_id is defined and mas_workspace_id != "" + fail_msg: "mas_workspace_id is required" + +- name: "Fail if mas_app_id is not provided" + assert: + that: mas_app_id is defined and mas_app_id != "" + fail_msg: "mas_app_id is required" + diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml @@ -0,0 +1 @@ +--- diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/pv-info.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/pv-info.yml similarity index 100% rename from ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/pv-info.yml rename to ibm/mas_devops/roles/suite_app_backup/tasks/manage/pv-info.yml diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/restore-namespace.yml similarity index 100% rename from ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-namespace.yml rename to ibm/mas_devops/roles/suite_app_backup/tasks/manage/restore-namespace.yml diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/README.md b/ibm/mas_devops/roles/suite_app_backup_restore/README.md deleted file mode 100644 index f66461d6fc..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/README.md +++ /dev/null @@ -1,223 +0,0 @@ -Backup and Restore MAS Applications -=============================================================================== - -Overview -------------------------------------------------------------------------------- -This role supports backing up and restoring the data for below MAS applications: - -- `manage`: Manage namespace resources, persistent volume data (e.g. attachments) -- `iot`: IoT namespace resources -- `monitor`: Monitor namespace resources -- `health`: Health namespace resources, Watson Studio project asset -- `optimizer`: Optimizer namespace resources -- `visualinspection`: Visual Inspection namespace resources, persistent volume data (e.g. image datasets, models) - - -Supports creating on-demand or scheduled backup jobs for taking full or incremental backups, and optionally creating Kubernetes jobs for running the backup/restore process. - -!!! important - An application backup can only be restored to an instance with the same MAS instance ID. - - -Role Variables - General -------------------------------------------------------------------------------- -### masbr_action -Set `backup` or `restore` to indicate the role to create a backup or restore job. - -- **Required** -- Environment Variable: `MAS_BR_ACTION` -- Default: None - -### mas_app_id -Defines the MAS application ID (`manage`, `iot`, `monitor`, `health`, `optimizer`, or `visualinspection`) for the backup or restore action. - -- **Required** -- Environment Variable: `MAS_APP_ID` -- Default: None - -### mas_instance_id -Defines the MAS instance ID for the backup or restore action. - -- **Required** -- Environment Variable: `MAS_INSTANCE_ID` -- Default: None - -### mas_workspace_id -Defines the MAS workspace ID for the backup or restore action. - -- **Required** -- Environment Variable: `MAS_WORKSPACE_ID` -- Default: None - -### masbr_confirm_cluster -Set `true` or `false` to indicate the role whether to confirm the currently connected cluster before running the backup or restore job. - -- Optional -- Environment Variable: `MASBR_CONFIRM_CLUSTER` -- Default: `false` - -### masbr_copy_timeout_sec -Set the transfer files timeout in seconds. - -- Optional -- Environment Variable: `MASBR_COPY_TIMEOUT_SEC` -- Default: `43200` (12 hours) - -### masbr_job_timezone -Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for creating scheduled backup job. If not set a value for this variable, this role will use UTC time zone when creating a CronJob for running scheduled backup job. - -- Optional -- Environment Variable: `MASBR_JOB_TIMEZONE` -- Default: None - -### masbr_storage_local_folder -Set local path to save the backup files. - -- **Required** -- Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` -- Default: None - - -Role Variables - Backup -------------------------------------------------------------------------------- -### masbr_backup_type -Set `full` or `incr` to indicate the role to create a full backup or incremental backup. Only supports creating incremental backup for persistent volume data, this role will always create a full backup for other type of data regardless of whether this variable be set to `incr`. - -- Optional -- Environment Variable: `MASBR_BACKUP_TYPE` -- Default: `full` - -### masbr_backup_data -Set the types of data to be backed up, multiple data types are separated by commas (e.g. `namespace,pv`). If not set a value for this variable, this role will back up all types of data that supported by the specified MAS application. The data types supported by each MAS applications: - -| MAS App Name | MAS App ID | Data types | -| ----------------- | ------------------- | ------------------- | -| Manage | `manage` | `namespace`, `pv` | -| IoT | `iot` | `namespace` | -| Monitor | `monitor` | `namespace` | -| Health | `health` | `namespace`, `wsl` | -| Optimizer | `optimizer` | `namespace` | -| Visual Inspection | `visualinspection` | `namespace`, `pv` | - -- Optional -- Environment Variable: `MASBR_BACKUP_DATA` -- Default: None - -### masbr_backup_from_version -Set the full backup version to use in the incremental backup, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`). This variable is only valid when `MASBR_BACKUP_TYPE=incr`. If not set a value for this variable, this role will try to find the latest full backup version from the specified storage location. - -- Optional -- Environment Variable: `MASBR_BACKUP_FROM_VERSION` -- Default: None - -### masbr_backup_schedule -Set [Cron expression](https://en.wikipedia.org/wiki/Cron) to create a scheduled backup. If not set a value for this varialbe, this role will create an on-demand backup. - -- Optional -- Environment Variable: `MASBR_BACKUP_SCHEDULE` -- Default: None - - -Role Variables - Restore -------------------------------------------------------------------------------- -### masbr_restore_from_version -Set the backup version to use in the restore, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`) - -- **Required** only when `MAS_BR_ACTION=restore` -- Environment Variable: `MASBR_RESTORE_FROM_VERSION` -- Default: None - -### masbr_restore_data -Set the types of data to be restored, multiple data types are separated by commas (e.g. `namespace,pv`). If not set a value for this variable, this role will restore all types of data that supported by the specified MAS application. The data types supported by each MAS applications: - -| MAS App Name | MAS App ID | Data types | -| ----------------- | ------------------- | ------------------- | -| Manage | `manage` | `namespace`, `pv` | -| IoT | `iot` | `namespace` | -| Monitor | `monitor` | `namespace` | -| Health | `health` | `namespace`, `wsl` | -| Optimizer | `optimizer` | `namespace` | -| Visual Inspection | `visualinspection` | `namespace`, `pv` | - -- Optional -- Environment Variable: `MASBR_RESTORE_DATA` -- Default: None - - -Role Variables - Manage -------------------------------------------------------------------------------- -### masbr_manage_pvc_paths -Set the Manage PVC paths to use in backup and restore. The PVC path is in the format of `:/`. Multiple PVC paths are separated by commas (e.g. `manage-doclinks1-pvc:/mnt/doclinks1/attachments,manage-doclinks2-pvc:/mnt/doclinks2`). - -The `` and `` are defined in the `ManageWorkspace` CRD instance `spec.settings.deployment.persistentVolumes`: -``` -persistentVolumes: - - accessModes: - - ReadWriteMany - mountPath: /mnt/doclinks1 - pvcName: manage-doclinks1-pvc - size: '20' - storageClassName: ocs-storagecluster-cephfs - volumeName: '' - - accessModes: - - ReadWriteMany - mountPath: /mnt/doclinks2 - pvcName: manage-doclinks2-pvc - size: '20' - storageClassName: ocs-storagecluster-cephfs - volumeName: '' -``` - -If not set a value for this variable, this role will not backup and restore persistent valumne data for Manage. - - -- Optional -- Environment Variable: `MASBR_MANAGE_PVC_PATHS` -- Default: None - - -Example Playbook -------------------------------------------------------------------------------- - -### Backup -Backup Manage attachments, note that this does not include backup of any data in Db2, see the `backup` action in the [db2](db2.md) role. - -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - masbr_action: backup - mas_instance_id: main - mas_workspace_id: ws1 - mas_app_id: manage - masbr_backup_data: pv - masbr_manage_pvc_paths: "manage-doclinks1-pvc:/mnt/doclinks1" - masbr_storage_local_folder: /tmp/masbr - roles: - - ibm.mas_devops.suite_app_backup_restore -``` - -### Restore -Restore Manage attachments, note that this does not include restore of any data in Db2, see the `restore` action in the [db2](db2.md) role. - -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - masbr_action: restore - masbr_restore_from_version: 20240621021316 - mas_instance_id: main - mas_workspace_id: ws1 - mas_app_id: manage - masbr_backup_data: pv - masbr_manage_pvc_paths: "manage-doclinks1-pvc:/mnt/doclinks1" - masbr_storage_local_folder: /tmp/masbr - roles: - - ibm.mas_devops.suite_app_backup_restore -``` - - -License -------------------------------------------------------------------------------- - -EPL-2.0 diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/defaults/main.yml b/ibm/mas_devops/roles/suite_app_backup_restore/defaults/main.yml deleted file mode 100644 index 5618f24bc9..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/defaults/main.yml +++ /dev/null @@ -1,20 +0,0 @@ -mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" -mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" - -masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" - -# Manage PVC paths to backup/restore, format: ":/" separated by commas -# For example: "pvc-docs:/doclinks/attachments,pvc-maxlogs:/maxlogs" -masbr_manage_pvc_paths: "{{ lookup('env', 'MASBR_MANAGE_PVC_PATHS') | default('', true) }}" - -# Backup/Restore - Supported job types per app -# https://ibm-mas.github.io/ansible-devops/roles/suite_app_backup_restore/#masbr_backup_data -# https://ibm-mas.github.io/ansible-devops/roles/suite_app_backup_restore/#masbr_restore_data -supported_job_data_item_types: - health: ["namespace", "wsl"] - iot: ["namespace"] - manage: ["namespace", "pv"] - monitor: ["namespace"] - optimizer: ["namespace"] - visualinspection: ["namespace", pv] diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-namespace.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-namespace.yml deleted file mode 100644 index 8c867a9b89..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-namespace.yml +++ /dev/null @@ -1,113 +0,0 @@ ---- -# Update namespace resource backup status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update namespace resource backup status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -- name: "Backup namespace resources" - block: - # Prepare namespace resource backup folder - # ------------------------------------------------------------------------- - - name: "Set fact: namespace resource backup folder" - set_fact: - masbr_ns_backup_folder: "{{ masbr_local_job_folder }}/{{ masbr_job_data_type }}" - masbr_ns_backup_name: "{{ masbr_job_name }}-{{ masbr_job_data_type }}" - - - name: "Set fact: namespace resource backup log" - set_fact: - masbr_ns_backup_log: "{{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}.log" - - - name: "Create local backup folder for saving namespace resoruces" - changed_when: true - shell: > - mkdir -p {{ masbr_ns_backup_folder }} && - touch {{ masbr_ns_backup_log }} - - - # Run backup namespace resource script - # ------------------------------------------------------------------------- - - name: "Create backup namespace resource script" - template: - src: "{{ role_path }}/../../common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2" - dest: "{{ masbr_local_job_folder }}/backup-namespace-resources.sh" - mode: "777" - - - name: "Run backup namespace resource script" - changed_when: true - shell: > - {{ masbr_local_job_folder }}/backup-namespace-resources.sh - register: _script_output - - - name: "Debug: run backup namespace resource script" - debug: - msg: "{{ _script_output.stdout_lines }}" - - - # Create tar.gz archives of namespace resource backup files - # ------------------------------------------------------------------------- - - name: "Create tar.gz archives of namespace resource backup files" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}.tar.gz - -C {{ masbr_ns_backup_folder }} . && - ls -lA {{ masbr_ns_backup_folder }} - register: _list_files_output - - - name: "Debug: list of namespace resource backup files" - debug: - msg: "{{ _list_files_output.stdout_lines }}" - - - # Copy backup files to specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup files to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_ns_backup_name }}.tar.gz" - dest_folder: "{{ masbr_job_data_type }}" - - - # Update namespace resource backup status: Completed - # ------------------------------------------------------------------------- - - name: "Update namespace resource backup status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update namespace resource backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update namespace resource backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy namespace resource backup log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of namespace resource backup log" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}-log.tar.gz - -C {{ masbr_local_job_folder }} {{ masbr_ns_backup_name }}.log - - - name: "Copy namespace resource backup log file to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-pv.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-pv.yml deleted file mode 100644 index a8984b8002..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/backup-pv.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- -# Update pv data backup status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update pv data backup status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -# Get app pv information -# ----------------------------------------------------------------------------- -- name: "Set fact: mas_app_pv_list" - set_fact: - mas_app_pv_list: [] - -- name: "Get {{ mas_app_id }} pv information" - when: mas_app_id in ['manage', 'visualinspection'] - include_tasks: "tasks/{{ mas_app_id }}/pv-info.yml" - -- name: "Debug: {{ mas_app_id }} pv information" - debug: - msg: "{{ mas_app_pv_list }}" - - -# Not found pv need to be backed up, skip this task. -# (TODO: should set the status to 'Skip') -# ------------------------------------------------------------------------- -- name: "Update pv data backup status: Completed" - when: mas_app_pv_list | length == 0 - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - -- name: "Backup pv data" - when: mas_app_pv_list | length > 0 - block: - # Copy pv data to specified storage location - # ------------------------------------------------------------------------- - - name: "Set fact: copy file variables" - set_fact: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_namespace: "{{ mas_app_namespace }}" - masbr_cf_are_pvc_paths: true - - - name: "Set fact: incremental backup" - when: masbr_backup_type == "incr" - set_fact: - masbr_cf_from_job_name: "{{ masbr_backup_from }}" - - - name: "Copy pv data to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_pod_files_to_storage.yml" - vars: - masbr_cf_pvc_name: "{{ mas_app_pv_item.pvc_name }}" - masbr_cf_pvc_mount_path: "{{ mas_app_pv_item.mount_path }}" - masbr_cf_pvc_sub_path: "{{ mas_app_pv_item.sub_path | default('') }}" - masbr_cf_paths: "{{ mas_app_pv_item.backup_paths }}" - loop: "{{ mas_app_pv_list }}" - loop_control: - loop_var: mas_app_pv_item - - - # Update pv data backup status: Completed - # ------------------------------------------------------------------------- - - name: "Update pv data backup status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update pv data backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update pv data backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml deleted file mode 100644 index ae61fac2af..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/get-app-info.yml +++ /dev/null @@ -1,114 +0,0 @@ ---- -- name: "Load mas app information" - include_vars: "{{ role_path }}/../../common_vars/application_info.yml" - -- name: "Set fact: mas app information" - when: mas_app_id != "health" - set_fact: - mas_app_kind: "{{ app_info[mas_app_id].kind }}" - mas_ws_kind: "{{ app_info[mas_app_id].ws_kind }}" - mas_api_version: "{{ app_info[mas_app_id].api_version }}" - -- name: "Get health app information" - when: mas_app_id == "health" - include_tasks: "tasks/health/get-app-info.yml" - - -# Get app version and status -# ----------------------------------------------------------------------------- -- name: "Set fact: application CRD instance name" - set_fact: - mas_app_cr_name: "{{ mas_instance_id }}" - -- name: "Get {{ mas_app_kind }}/{{ mas_app_cr_name }}" - kubernetes.core.k8s_info: - api_version: "{{ mas_api_version }}" - kind: "{{ mas_app_kind }}" - name: "{{ mas_app_cr_name }}" - namespace: "{{ mas_app_namespace }}" - register: _app_output - -- name: "Set fact: {{ mas_app_kind }}/{{ mas_app_cr_name }} version" - set_fact: - mas_app_version: "{{ _app_output.resources[0].status.versions.reconciled }}" - when: - - _app_output is defined - - (_app_output.resources | length > 0) - - _app_output.resources[0].status.versions.reconciled is defined - -- name: "Fail if {{ mas_app_kind }}/{{ mas_app_cr_name }} does not exists" - assert: - that: mas_app_version is defined - fail_msg: "{{ mas_app_kind }}/{{ mas_app_cr_name }} does not exists!" - when: masbr_action is defined and masbr_action == "backup" - -- name: "Set fact: {{ mas_app_kind }}/{{ mas_app_cr_name }} status" - set_fact: - mas_app_ready: true - when: - - _app_output.resources is defined - - (_app_output.resources | length > 0) - - _app_output.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select ('match','True') | list | length == 1 - -# When performing restore, we shouldn't care about the status of app. -- name: "Fail if {{ mas_app_kind }}/{{ mas_app_cr_name }} is not ready" - when: masbr_action is defined and masbr_action == "backup" - assert: - that: mas_app_ready is defined and mas_app_ready - fail_msg: "{{ mas_app_kind }}/{{ mas_app_cr_name }} is not ready!" - - -# Get workspace version and status -# ----------------------------------------------------------------------------- -- name: "Set fact: workspace CRD instance name" - set_fact: - mas_ws_cr_name: "{{ mas_instance_id }}-{{ mas_workspace_id }}" - -- name: "Get {{ mas_ws_kind }}/{{ mas_ws_cr_name }}" - kubernetes.core.k8s_info: - api_version: "{{ mas_api_version }}" - kind: "{{ mas_ws_kind }}" - name: "{{ mas_ws_cr_name }}" - namespace: "{{ mas_app_namespace }}" - register: _ws_output - -- name: "Set fact: {{ mas_ws_kind }}/{{ mas_ws_cr_name }} version" - set_fact: - mas_ws_version: "{{ _ws_output.resources[0].status.versions.reconciled }}" - when: - - _ws_output is defined - - (_ws_output.resources | length > 0) - - _ws_output.resources[0].status.versions.reconciled is defined - -- name: "Fail if {{ mas_ws_kind }}/{{ mas_ws_cr_name }} does not exists" - assert: - that: mas_ws_version is defined - fail_msg: "{{ mas_ws_kind }}/{{ mas_ws_cr_name }} does not exists!" - when: masbr_action is defined and masbr_action == "backup" - -- name: "Set fact: {{ mas_ws_kind }}/{{ mas_ws_cr_name }} status" - set_fact: - mas_ws_ready: true - when: - - _ws_output.resources is defined - - (_ws_output.resources | length > 0) - - _ws_output.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select ('match','True') | list | length == 1 - -# When performing restore, we shouldn't care about the status of app. -- name: "Fail if {{ mas_ws_kind }}/{{ mas_ws_cr_name }} is not ready" - when: masbr_action is defined and masbr_action == "backup" - assert: - that: mas_ws_ready is defined and mas_ws_ready - fail_msg: "{{ mas_ws_kind }}/{{ mas_ws_cr_name }} is not ready!" - - -# Output app information -# ----------------------------------------------------------------------------- -- name: "Debug: {{ mas_app_id | capitalize }} information" - when: masbr_action is defined and masbr_action == "backup" - debug: - msg: - - "{{ mas_app_kind }}/{{ mas_app_cr_name }} version ............ {{ mas_app_version }}" - - "{{ mas_app_kind }}/{{ mas_app_cr_name }} is ready ........... {{ mas_app_ready | default(false, true) }}" - - "{{ mas_ws_kind }}/{{ mas_ws_cr_name }} version .......... {{ mas_ws_version }}" - - "{{ mas_ws_kind }}/{{ mas_ws_cr_name }} is ready ......... {{ mas_ws_ready | default(false, true) }}" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-vars.yml deleted file mode 100644 index 8a52ef7427..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-vars.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- name: "Set fact: default backup job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - - seq: "2" - type: "wsl" - -- name: "Set fact: health standalone namespace backup resources" - when: mas_health_standalone - set_fact: - masbr_ns_backup_resources: - - namespace: "{{ mas_app_namespace }}" - resources: - - kind: Subscription - name: ibm-mas-manage - - kind: OperatorGroup - name: ibm-health-operatorgroup - - kind: Secret - name: ibm-entitlement - # apps.mas.ibm.com - - kind: HealthApp - - kind: HealthWorkspace - -- name: "Set fact: health ext namespace backup resources" - when: not mas_health_standalone - set_fact: - masbr_ns_backup_resources: - - namespace: "{{ mas_app_namespace }}" - resources: - # apps.mas.ibm.com - - kind: HealthextWorkspace - - kind: HealthextAccelerator diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-wsl.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-wsl.yml deleted file mode 100644 index 3499cfe1b6..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/backup-wsl.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -# Update watson studio project backup status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update watson studio project backup status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -- name: "Backup watson studio project" - block: - # Prepare watson studio project backup folder - # ------------------------------------------------------------------------- - - name: "Set fact: watson studio project backup name" - set_fact: - masbr_wsl_backup_name: "{{ masbr_job_name }}-{{ masbr_job_data_type }}" - - - name: "Set fact: watson studio project backup log" - set_fact: - masbr_wsl_backup_log: "{{ masbr_local_job_folder }}/{{ masbr_wsl_backup_name }}.log" - - - name: "Create watson studio project backup folder" - changed_when: true - shell: > - touch {{ masbr_wsl_backup_log }} - - - # Get watson studio information - # ----------------------------------------------------------------------------- - - name: "Get watson studio information" - include_tasks: "tasks/health/get-wsl-info.yml" - vars: - _wsl_log: "{{ masbr_wsl_backup_log }}" - - - # Export watson studio project asset - # ----------------------------------------------------------------------------- - - name: "Export watson studio project asset" - changed_when: true - shell: >- - {{ cpd_cli_cmd }} config users set cpd-user --username={{ cpd_username }} --apikey={{ cpd_apikey }}; - {{ cpd_cli_cmd }} config profiles set cpd-profile --user=cpd-user --url={{ cpd_endpoint }}; - {{ cpd_cli_cmd }} asset export start --profile=cpd-profile --project-id={{ cpd_project_id }} - --name={{ masbr_job_name }} --assets='{"all_assets": true}' - --output-file={{ masbr_local_job_folder }}/{{ masbr_wsl_backup_name }}.tgz >> {{ masbr_wsl_backup_log }} - - - # Copy backup files to specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup files to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_wsl_backup_name }}.tgz" - dest_folder: "{{ masbr_job_data_type }}" - - - # Update watson studio project backup status: Completed - # ------------------------------------------------------------------------- - - name: "Update watson studio project backup status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update watson studio project backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update watson studio project backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy watson studio project backup log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of watson studio project backup log" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_wsl_backup_name }}-log.tar.gz - -C {{ masbr_local_job_folder }} {{ masbr_wsl_backup_name }}.log - - - name: "Copy watson studio project backup log file to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ masbr_local_job_folder }}/{{ masbr_wsl_backup_name }}-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-app-info.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-app-info.yml deleted file mode 100644 index 50daf8a436..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-app-info.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: "Determine health deployment types" - set_fact: - # Only support healthext deployment by now - mas_health_standalone: false - -- name: "Set fact: health standalone app information" - when: mas_health_standalone - set_fact: - mas_app_namespace: "mas-{{ mas_instance_id }}-health" - mas_app_kind: "HealthApp" - mas_ws_kind: "HealthWorkspace" - mas_api_version: "apps.mas.ibm.com/v1" - -- name: "Set fact: health ext app information" - when: not mas_health_standalone - set_fact: - mas_app_namespace: "mas-{{ mas_instance_id }}-manage" - mas_app_kind: "ManageApp" - mas_ws_kind: "HealthextWorkspace" - mas_api_version: "apps.mas.ibm.com/v1" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-wsl-info.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-wsl-info.yml deleted file mode 100644 index 6c4943ec3e..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/get-wsl-info.yml +++ /dev/null @@ -1,80 +0,0 @@ ---- -# Input parameters: -# _wsl_log - - -# Get watson studio information -# ----------------------------------------------------------------------------- -- name: "Set fact: watson studio secret" - set_fact: - wsl_secret_name: "{{ mas_instance_id }}-{{ mas_workspace_id }}-healthext-watsonstudio-secret" - -- name: "Get watson studio secret" - kubernetes.core.k8s_info: - kind: Secret - name: "{{ wsl_secret_name }}" - namespace: "{{ mas_app_namespace }}" - register: _wsl_secret_output - -- name: "Set fact: mongodb admin password" - set_fact: - cpd_endpoint: "{{ _wsl_secret_output.resources[0].data.endpoint | b64decode }}" - cpd_username: "{{ _wsl_secret_output.resources[0].data.username | b64decode }}" - cpd_password: "{{ _wsl_secret_output.resources[0].data.password | b64decode }}" - cpd_project_id: "{{ _wsl_secret_output.resources[0].data.projectid | b64decode }}" - when: - - _wsl_secret_output is defined - - _wsl_secret_output.resources is defined - - _wsl_secret_output.resources | length > 0 - -- name: "Debug: watson studio information" - debug: - msg: - - "CPD endpoint ........................... {{ cpd_endpoint }}" - - "CPD username ........................... {{ cpd_username }}" - - "CPD project ............................ {{ cpd_project_id }}" - - -# Generate a CPD API Key -# ----------------------------------------------------------------------------- -- name: "Generate CPD API Key" - changed_when: false - shell: >- - echo "Call {{ cpd_endpoint }}/icp4d-api/v1/authorize" >> {{ _wsl_log }}; - CPD_API_RESP=$(curl -k -X POST -H "Content-Type: application/json" - -d "{\"username\":\"{{ cpd_username }}\",\"password\":\"{{ cpd_password }}\"}" - "{{ cpd_endpoint }}/icp4d-api/v1/authorize"); - echo "${CPD_API_RESP}" >> {{ _wsl_log }}; - CPD_TOKEN=$(echo "${CPD_API_RESP}" | jq -r '.token'); - echo "Call {{ cpd_endpoint }}/usermgmt/v1/user/apiKey" >> {{ _wsl_log }}; - CPD_API_RESP=$(curl -k -X GET "{{ cpd_endpoint }}/usermgmt/v1/user/apiKey" -H "Accept: application/json" - -H "Authorization: Bearer ${CPD_TOKEN}"); - echo "${CPD_API_RESP}" >> {{ _wsl_log }}; - CPD_API_KEY=$(echo "${CPD_API_RESP}" | jq -r '.apiKey'); - echo "${CPD_API_KEY}" - register: _cpd_apikey_output - -- name: "Set fact: CPD API Key" - set_fact: - cpd_apikey: "{{ _cpd_apikey_output.stdout }}" - - -# Check if cpd-cli installed -# ----------------------------------------------------------------------------- -- name: "Get cpd-cli information" - changed_when: false - shell: > - cpd-cli version - register: _cpd_cli_output - -- name: "Download cpd-cli" - when: _cpd_cli_output.rc != 0 - changed_when: true - shell: >- - cd /tmp; - curl -L https://github.com/IBM/cpd-cli/releases/download/v13.1.5r1/cpd-cli-linux-EE-13.1.5.tgz -o cpd-cli-linux.tgz; - tar -xf cpd-cli-linux.tgz; - -- name: "Set fact: cpd-cli command" - set_fact: - cpd_cli_cmd: "{{ '/tmp/cpd-cli-linux-EE-13.1.5-242/cpd-cli' if _cpd_cli_output.rc != 0 else 'cpd-cli' }}" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-vars.yml deleted file mode 100644 index 9c7f51f5f3..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-vars.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: "Set fact: default restore job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - - seq: "2" - type: "wsl" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-wsl.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-wsl.yml deleted file mode 100644 index bb0bc6ca58..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/health/restore-wsl.yml +++ /dev/null @@ -1,127 +0,0 @@ ---- -# Update watson studio project restore status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update watson studio project restore status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -- name: "Restore watson studio project" - block: - # Prepare watson studio project restore folder - # ------------------------------------------------------------------------- - - name: "Set fact: watson studio project restore name" - set_fact: - masbr_wsl_restore_from_name: "{{ masbr_restore_from }}-{{ masbr_job_data_type }}" - masbr_wsl_restore_name: "{{ masbr_job_name }}-{{ masbr_job_data_type }}" - - - name: "Set fact: watson studio project restore folder" - set_fact: - masbr_wsl_restore_log: "{{ masbr_local_job_folder }}/{{ masbr_wsl_restore_name }}.log" - - - name: "Create watson studio project restore log" - changed_when: true - shell: > - touch {{ masbr_wsl_restore_log }} - - - # Copy backup file from specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup file from specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_local.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_restore_from }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/{{ masbr_wsl_restore_from_name }}.tgz" - dest_folder: "./" - - src_file: "{{ masbr_job_data_type }}/wsl-project-name.txt" - dest_folder: "./" - - - # Get watson studio information - # ----------------------------------------------------------------------------- - - name: "Get watson studio information" - include_tasks: "tasks/health/get-wsl-info.yml" - vars: - _wsl_log: "{{ masbr_wsl_restore_log }}" - - - # Import watson studio project asset - # ----------------------------------------------------------------------------- - - name: "Import watson studio project asset" - changed_when: true - shell: >- - {{ cpd_cli_cmd }} config users set cpd-user --username={{ cpd_username }} --apikey={{ cpd_apikey }}; - {{ cpd_cli_cmd }} config profiles set cpd-profile --user=cpd-user --url={{ cpd_endpoint }}; - echo "List projects" >> {{ masbr_wsl_restore_log }}; - {{ cpd_cli_cmd }} project list --profile=cpd-profile >> {{ masbr_wsl_restore_log }}; - WSL_PROJECT_NAME={{ mas_instance_id }}-{{ mas_workspace_id }}-healthext-{{ masbr_job_version }}; - echo "Create project ${WSL_PROJECT_NAME}" >> {{ masbr_wsl_restore_log }}; - CREATE_PROJECT_JSON=$({{ cpd_cli_cmd }} project create --profile=cpd-profile --name=${WSL_PROJECT_NAME} --output=json); - echo "${CREATE_PROJECT_JSON}" >> {{ masbr_wsl_restore_log }}; - WSL_PROJECT_LOC=$(echo "${CREATE_PROJECT_JSON}" | jq -r '.location'); - WSL_PROJECT_ID=$(echo "${WSL_PROJECT_LOC##*/}"); - echo "Import project asset" >> {{ masbr_wsl_restore_log }}; - {{ cpd_cli_cmd }} asset import start --profile=cpd-profile --project-id=${WSL_PROJECT_ID} - --import-file={{ masbr_local_job_folder }}/{{ masbr_wsl_restore_from_name }}.tgz >> {{ masbr_wsl_restore_log }}; - echo "${WSL_PROJECT_ID}" - register: _import_asset_output - - - name: "Set fact: new watson studio project id" - set_fact: - new_project_id: "{{ _import_asset_output.stdout }}" - - - name: "Debug: new watson studio project id" - debug: - msg: "New WS project id ................. {{ new_project_id }}" - - - # Update watson studio secret - # ----------------------------------------------------------------------------- - - name: "Update secret {{ wsl_secret_name }}" - changed_when: true - shell: >- - oc patch secret/{{ wsl_secret_name }} -n {{ mas_app_namespace }} - -p "{\"data\": {\"projectid\": \"{{ new_project_id | b64encode }}\"}}" - - - # Update watson studio project restore status: Completed - # ------------------------------------------------------------------------- - - name: "Update watson studio project restore status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update watson studio project restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update watson studio project restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy watson studio project restore log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of watson studio project restore log" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_wsl_restore_name }}-log.tar.gz - -C {{ masbr_local_job_folder }} {{ masbr_wsl_restore_name }}.log - - - name: "Copy watson studio project restore log file to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "restore" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ masbr_local_job_folder }}/{{ masbr_wsl_restore_name }}-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/backup-vars.yml deleted file mode 100644 index ebbe463c7b..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/backup-vars.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- name: "Set fact: default backup job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - -- name: "Set fact: namespace backup resources" - set_fact: - masbr_ns_backup_resources: - - namespace: "{{ mas_app_namespace }}" - resources: - - kind: Subscription - name: ibm-mas-iot - - kind: OperatorGroup - name: ibm-iot-operatorgroup - - kind: Secret - name: ibm-entitlement - - kind: Secret - name: "actions-credsenckey" - - kind: Secret - name: "auth-encryption-secret" - - kind: Secret - name: "provision-creds-enckey" - - kind: Secret - name: "auth-edc-user-sync-secret" - # apps.mas.ibm.com - - kind: IoT - - kind: IoTWorkspace diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-namespace.yml deleted file mode 100644 index c65917ad5b..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-namespace.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -# Apply secret yaml files -# ------------------------------------------------------------------------- -- name: "Set fact: namespace resources to be restored" - set_fact: - masbr_ns_restore_resources: - - "Secret-actions-credsenckey.yaml" - - "Secret-auth-encryption-secret.yaml" - - "Secret-provision-creds-enckey.yaml" - - "Secret-auth-edc-user-sync-secret.yaml" - -- name: "Replace mas instance in secret yaml files" - when: masbr_restore_to_diff_instance - changed_when: true - shell: > - yq -i 'with(.metadata; - .namespace="{{ mas_app_namespace }} | - .labels."app.kubernetes.io/instance"="{{ mas_app_namespace }}" | - .labels."mas.ibm.com/instanceId"="{{ mas_app_namespace }}" - )' {{ masbr_ns_restore_folder }}/{{ _ns_resource_file_name }} - loop: "{{ masbr_ns_restore_resources }}" - loop_control: - loop_var: _ns_resource_file_name - -- name: "Apply secret yaml files" - kubernetes.core.k8s: - apply: true - src: "{{ masbr_ns_restore_folder }}/{{ _ns_resource_file_name }}" - loop: "{{ masbr_ns_restore_resources }}" - loop_control: - loop_var: _ns_resource_file_name - - -# Restart pods -# ------------------------------------------------------------------------- -- name: "Delete pods in {{ mas_app_namespace }}" - changed_when: true - shell: >- - oc get pod -n {{ mas_app_namespace }} | grep "{{ _del_pod_name }}" | awk '{print $1}' | - xargs oc delete pod -n {{ mas_app_namespace }} - loop: - - "datapower-datapower" - - "auth-masuseragent" - loop_control: - loop_var: _del_pod_name - register: _del_pods_output - -- name: "Debug: delete pods in {{ mas_app_namespace }}" - debug: - msg: "{{ _del_pods_output | json_query('results[*].stdout_lines') }}" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-vars.yml deleted file mode 100644 index cf23734548..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/iot/restore-vars.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: "Set fact: default restore job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml deleted file mode 100644 index 6722ca867a..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/main.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -# Check mas app backup/restore required variables -# ----------------------------------------------------------------------------- -- name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - -- name: "Fail if mas_workspace_id is not provided" - assert: - that: mas_workspace_id is defined and mas_workspace_id != "" - fail_msg: "mas_workspace_id is required" - -- name: "Fail if mas_app_id is not provided" - assert: - that: mas_app_id is defined and mas_app_id != "" - fail_msg: "mas_app_id is required" - -- name: "Fail if masbr_action is not provided" - assert: - that: masbr_action is defined and masbr_action != "" - fail_msg: "masbr_action is required" - -- name: "Set fact: namespace name for {{ mas_app_id }}" - set_fact: - mas_app_namespace: "mas-{{ mas_instance_id }}-{{ mas_app_id }}" - - -# Get mas app information -# ----------------------------------------------------------------------------- -- name: "Get {{ mas_app_id }} information" - include_tasks: "tasks/get-app-info.yml" - - -# Set common job variables -# ----------------------------------------------------------------------------- -- name: "Set fact: common job variables" - set_fact: - masbr_job_component: - name: "{{ mas_app_id }}" - instance: "{{ mas_instance_id }}" - workspace: "{{ mas_workspace_id }}" - namespace: "{{ mas_app_namespace }}" - -- name: "Load mas app variables" - include_tasks: "tasks/{{ mas_app_id }}/{{ masbr_action }}-vars.yml" - - -# Before run tasks -# ----------------------------------------------------------------------------- -- name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - - -- name: "Run {{ masbr_action }} tasks" - block: - # Update job status: New - # ------------------------------------------------------------------------- - - name: "Update job status: New" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "1" - phase: "New" - - - # Run backup/restore tasks for each data type - # TODO: check and ignore unsupported data type - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks for each data type" - include_tasks: "{{ _include_tasks_folder }}/{{ masbr_action }}-{{ job_data_item.type }}.yml" - vars: - _include_tasks_folder: >- - {{ role_path }}/{{ 'tasks' if job_data_item.type in ['namespace', 'pv'] else 'tasks/' + mas_app_id }} - masbr_job_data_seq: "{{ job_data_item.seq }}" - masbr_job_data_type: "{{ job_data_item.type }}" - loop: "{{ masbr_job_data_list }}" - loop_control: - loop_var: job_data_item - when: job_data_item.type in supported_job_data_item_types[mas_app_id] - - rescue: - # Update job status: Failed - # ------------------------------------------------------------------------- - - name: "Update job status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_status: - phase: "Failed" - - always: - # After run tasks - # ------------------------------------------------------------------------- - - name: "After run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/backup-vars.yml deleted file mode 100644 index 700e185478..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/backup-vars.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: "Set fact: default backup job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - - seq: "2" - type: "pv" - -- name: "Set fact: namespace backup resources" - set_fact: - masbr_ns_backup_resources: - - namespace: "{{ mas_app_namespace }}" - resources: - - kind: Subscription - name: ibm-mas-manage - - kind: OperatorGroup - name: mas-{{ mas_instance_id }}-manage-operator-group - - kind: Secret - name: ibm-entitlement - - kind: Secret - name: "{{ mas_workspace_id }}-manage-encryptionsecret" - - kind: Secret - name: "{{ mas_workspace_id }}-manage-encryptionsecret-operator" - # apps.mas.ibm.com - - kind: ManageApp - - kind: ManageWorkspace diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-vars.yml deleted file mode 100644 index d55d2c8d27..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/manage/restore-vars.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: "Set fact: default restore job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - - seq: "2" - type: "pv" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/backup-vars.yml deleted file mode 100644 index 6e697ac281..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/backup-vars.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- name: "Set fact: default backup job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - -- name: "Set fact: namespace backup resources" - set_fact: - masbr_ns_backup_resources: - # monitor - - namespace: "{{ mas_app_namespace }}" - resources: - - kind: Subscription - name: ibm-mas-monitor - - kind: OperatorGroup - name: ibm-monitor-operatorgroup - - kind: Secret - name: ibm-entitlement - - kind: Secret - name: "{{ mas_instance_id }}-{{ mas_workspace_id }}-datadictionaryworkspace-workspace-binding" - - kind: Secret - name: "monitor-kitt" - # apps.mas.ibm.com - - kind: MonitorApp - - kind: MonitorWorkspace - # add - - namespace: "mas-{{ mas_instance_id }}-add" - resources: - - kind: Subscription - name: ibm-data-dictionary - - kind: OperatorGroup - name: ibm-dd-group - - kind: Secret - name: ibm-entitlement - - kind: Secret - name: "datadictionary-{{ mas_workspace_id }}" - - kind: Secret - name: "instance-admin" - # apps.mas.ibm.com - - kind: AssetDataDictionary - - kind: DataDictionaryWorkspace diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-namespace.yml deleted file mode 100644 index 5e245d6804..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-namespace.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -- name: "Set fact: instance and workspace id in the resource file name" - set_fact: - masbr_restore_from_instance: "{{ masbr_restore_from_yaml.component.instance }}" - masbr_restore_from_workspace: "{{ masbr_restore_from_yaml.component.workspace }}" - - -# Restore namespace resources for monitor -# ------------------------------------------------------------------------- -- name: "Replace mas instance in the resource files" - when: masbr_restore_to_diff_instance - changed_when: true - shell: > - yq -i 'with(.metadata; - .namespace="{{ mas_app_namespace }}" | - .name="{{ mas_instance_id }}-{{ mas_workspace_id }}-datadictionaryworkspace-workspace-binding" - )' {{ masbr_ns_restore_folder }}/Secret-{{ masbr_restore_from_instance }}-{{ masbr_restore_from_workspace }}-datadictionaryworkspace-workspace-binding.yaml; - - yq -i 'with(.metadata; - .namespace="{{ mas_app_namespace }}" - )' {{ masbr_ns_restore_folder }}/Secret-monitor-kitt.yaml; - -- name: "Apply secret yaml files" - kubernetes.core.k8s: - apply: true - src: "{{ masbr_ns_restore_folder }}/{{ _ns_resource_file_name }}" - loop: - - "Secret-{{ masbr_restore_from_instance }}-{{ masbr_restore_from_workspace }}-datadictionaryworkspace-workspace-binding.yaml" - - "Secret-monitor-kitt.yaml" - loop_control: - loop_var: _ns_resource_file_name - - -# Restore namespace resources for add -# ------------------------------------------------------------------------- -- name: "Replace mas instance in the resource files" - when: masbr_restore_to_diff_instance - changed_when: true - shell: > - yq -i 'with(.metadata; - .namespace="mas-{{ mas_instance_id }}-add" | - .name="datadictionary-{{ mas_workspace_id }}" - )' {{ masbr_ns_restore_folder }}/Secret-datadictionary-{{ masbr_restore_from_workspace }}.yaml; - - yq -i 'with(.metadata; - .namespace="mas-{{ mas_instance_id }}-add" - )' {{ masbr_ns_restore_folder }}/Secret-instance-admin.yaml; - -- name: "Apply secret yaml files" - kubernetes.core.k8s: - apply: true - src: "{{ masbr_ns_restore_folder }}/{{ _ns_resource_file_name }}" - loop: - - "Secret-datadictionary-{{ masbr_restore_from_workspace }}.yaml" - - "Secret-instance-admin.yaml" - loop_control: - loop_var: _ns_resource_file_name - - -# Restart pods -# ------------------------------------------------------------------------- -- name: "Delete pods in mas-{{ mas_instance_id }}-add" - changed_when: true - shell: >- - oc get pod -n mas-{{ mas_instance_id }}-add | grep "{{ _del_pod_name }}" | awk '{print $1}' | - xargs oc delete pod -n mas-{{ mas_instance_id }}-add - loop: - - "user-store" - - "series-store" - - "graph-store" - loop_control: - loop_var: _del_pod_name - register: _del_pods_output - -- name: "Debug: delete pods in {{ mas_app_namespace }}" - debug: - msg: "{{ _del_pods_output | json_query('results[*].stdout_lines') }}" - -- name: "Delete pods in {{ mas_app_namespace }}" - changed_when: true - shell: >- - oc get pod -n mas-{{ mas_instance_id }}-monitor | grep "{{ mas_instance_id }}" | awk '{print $1}' | - xargs oc delete pod -n mas-{{ mas_instance_id }}-monitor - register: _del_pods_output - -- name: "Debug: delete pods in {{ mas_app_namespace }}" - debug: - msg: "{{ _del_pods_output.stdout_lines }}" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-vars.yml deleted file mode 100644 index cf23734548..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/monitor/restore-vars.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: "Set fact: default restore job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/backup-vars.yml deleted file mode 100644 index dfa1711ab2..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/backup-vars.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: "Set fact: default backup job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - -- name: "Set fact: namespace backup resources" - set_fact: - masbr_ns_backup_resources: - - namespace: "{{ mas_app_namespace }}" - resources: - - kind: Subscription - name: ibm-mas-optimizer - - kind: OperatorGroup - name: mas-{{ mas_instance_id }}-optimizer-operator-group - # apps.mas.ibm.com - - kind: OptimizerApp - - kind: OptimizerWorkspace diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/restore-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/restore-vars.yml deleted file mode 100644 index cf23734548..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/optimizer/restore-vars.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: "Set fact: default restore job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml deleted file mode 100644 index 92931ceddf..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-namespace.yml +++ /dev/null @@ -1,109 +0,0 @@ ---- -# Update namespace resource restore status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update namespace resource restore status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -- name: "Restore namespace resources" - block: - # Prepare namespace resource restore folder - # ------------------------------------------------------------------------- - - name: "Set fact: namespace backup file name" - set_fact: - masbr_ns_restore_from_name: "{{ masbr_restore_from }}-{{ masbr_job_data_type }}" - - - name: "Set fact: namespace resource restore folder" - set_fact: - masbr_ns_restore_folder: "{{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/{{ masbr_ns_restore_from_name }}" - masbr_ns_restore_name: "{{ masbr_job_name }}-{{ masbr_job_data_type }}" - - - name: "Set fact: namespace resource restore log" - set_fact: - masbr_ns_restore_log: "{{ masbr_local_job_folder }}/{{ masbr_ns_restore_name }}.log" - - - name: "Create local restore folder for saving namespace resoruces" - changed_when: true - shell: > - mkdir -p {{ masbr_ns_restore_folder }} && - touch {{ masbr_ns_restore_log }} - - - name: "Debug: namespace resource restore folder" - debug: - msg: "Namespace resource restore folder ........ {{ masbr_ns_restore_folder }}" - - - # Copy backup file from specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup file from specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_local.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_restore_from }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/{{ masbr_ns_restore_from_name }}.tar.gz" - dest_folder: "{{ masbr_job_data_type }}" - - - # Extract the tar.gz file - # ------------------------------------------------------------------------- - - name: "Extract the tar.gz file" - changed_when: true - shell: > - tar -xzf {{ masbr_local_job_folder }}/{{ masbr_job_data_type }}/{{ masbr_ns_restore_from_name }}.tar.gz - -C {{ masbr_ns_restore_folder }} && - ls -lA {{ masbr_ns_restore_folder }} - register: _extract_output - - - name: "Debug: list extracted files" - debug: - msg: "{{ _extract_output.stdout_lines }}" - - - # Restore namespace resoruces - # ------------------------------------------------------------------------- - - name: "Restore namespace resoruces" - when: mas_app_id in ['manage', 'iot', 'monitor'] - include_tasks: "tasks/{{ mas_app_id }}/restore-namespace.yml" - - - # Update database restore status: Completed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update database restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy namespace resource restore log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of namespace resource restore log" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_ns_restore_name }}-log.tar.gz - -C {{ masbr_local_job_folder }} {{ masbr_ns_restore_name }}.log - - - name: "Copy namespace resource restore log file to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "restore" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ masbr_local_job_folder }}/{{ masbr_ns_restore_name }}-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml deleted file mode 100644 index c73928580a..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/restore-pv.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- -# Update pv data restore status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update pv data restore status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -# Get app pv information -# ----------------------------------------------------------------------------- -- name: "Set fact: mas_app_pv_list" - set_fact: - mas_app_pv_list: [] - -- name: "Get {{ mas_app_id }} pv information" - when: mas_app_id in ['manage', 'visualinspection'] - include_tasks: "tasks/{{ mas_app_id }}/pv-info.yml" - -- name: "Debug: {{ mas_app_id }} pv information" - debug: - msg: "{{ mas_app_pv_list }}" - - -# Not found pv need to be restored, skip this task. -# (TODO: should set the status to 'Skip') -# ------------------------------------------------------------------------- -- name: "Update pv data restore status: Completed" - when: mas_app_pv_list | length == 0 - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - -- name: "Restore pv data" - when: mas_app_pv_list | length > 0 - block: - # Copy pv data from specified storage location - # ------------------------------------------------------------------------- - - name: "Set fact: copy file variables" - set_fact: - masbr_cf_job_type: "backup" - masbr_cf_namespace: "{{ mas_app_namespace }}" - masbr_cf_job_name: "{{ masbr_restore_from }}" - masbr_cf_are_pvc_paths: true - - - name: "Set fact: restore from an incremental backup" - when: masbr_restore_from_incr - set_fact: - masbr_cf_from_job_name: "{{ masbr_restore_basedon }}" - - - name: "Copy pv data from specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_pod.yml" - vars: - masbr_cf_pvc_name: "{{ mas_app_pv_item.pvc_name }}" - masbr_cf_pvc_mount_path: "{{ mas_app_pv_item.mount_path }}" - masbr_cf_pvc_sub_path: "{{ mas_app_pv_item.sub_path | default('') }}" - masbr_cf_paths: "{{ mas_app_pv_item.restore_paths }}" - loop: "{{ mas_app_pv_list }}" - loop_control: - loop_var: mas_app_pv_item - - - # Update pv data restore status: Completed - # ------------------------------------------------------------------------- - - name: "Update pv data restore status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update pv data restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update pv data restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/backup-vars.yml deleted file mode 100644 index 307e9f0d6c..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/backup-vars.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: "Set fact: default backup job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - - seq: "2" - type: "pv" - -- name: "Set fact: namespace backup resources" - set_fact: - masbr_ns_backup_resources: - - namespace: "{{ mas_app_namespace }}" - resources: - - kind: Subscription - name: ibm-mas-visualinspection - - kind: OperatorGroup - name: mas-{{ mas_instance_id }}-visualinspection-operator-group - - kind: Secret - name: ibm-entitlement - # https://www.ibm.com/docs/en/mas-cd/maximo-vi/continuous-delivery?topic=managing-workload-scale-customization - - kind: ConfigMap - keywords: ^custom-.*-config$ - - kind: HorizontalPodAutoscaler - # apps.mas.ibm.com - - kind: VisualInspectionApp - - kind: VisualInspectionAppWorkspace diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/pv-info.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/pv-info.yml deleted file mode 100644 index 32f266ff76..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/pv-info.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -# Check ui pod information -# ------------------------------------------------------------------------- -- name: "Get ui pod information" - kubernetes.core.k8s_info: - kind: Pod - namespace: "{{ mas_app_namespace }}" - label_selectors: - - app.kubernetes.io/name=ui - register: _ui_pod_output - failed_when: - - _ui_pod_output.resources is not defined - - _ui_pod_output.resources | length == 0 - -- name: "Set fact: copy pvc file variables" - set_fact: - masbr_cf_pod_name: "{{ _ui_pod_output.resources[0].metadata.name }}" - masbr_cf_container_name: "{{ _ui_pod_output.resources[0].spec.containers[0].name }}" - masbr_cf_affinity: false - -- name: "Debug: ui pod information" - debug: - msg: - - "ui pod name ................... {{ masbr_cf_pod_name }}" - - "ui container name ............. {{ masbr_cf_container_name }}" - - -# Set pv information variables -# ------------------------------------------------------------------------- -- name: "Set fact: mas_app_pv_list" - set_fact: - mas_app_pv_list: - - mount_path: "/opt/powerai-vision/data" - sub_path: "data" - pvc_name: "{{ mas_instance_id }}-data-pvc" - backup_paths: - - src_folder: "/opt/powerai-vision/data" - dest_folder: "pv" - restore_paths: - - src_folder: "pv" - dest_folder: "/opt/powerai-vision/data" diff --git a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/restore-vars.yml b/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/restore-vars.yml deleted file mode 100644 index d55d2c8d27..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup_restore/tasks/visualinspection/restore-vars.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -- name: "Set fact: default restore job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - - seq: "2" - type: "pv" diff --git a/ibm/mas_devops/roles/suite_app_config/tasks/main.yml b/ibm/mas_devops/roles/suite_app_config/tasks/main.yml index c37dca1875..528e2f8bec 100644 --- a/ibm/mas_devops/roles/suite_app_config/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_config/tasks/main.yml @@ -35,7 +35,7 @@ when: mas_app_id is in ['manage'] # applications which have something to process before configuration include_tasks: "tasks/{{ mas_app_id }}/pre-config/main.yml" -# Resolve the WSL project ID that will be used by Predict and HP Utilities +# Resolve the WSL project ID that will be used by Predict - name: Lookup Watson Studio Project ID when: mas_app_id is in ['predict'] include_tasks: "tasks/determine-watson-studio-id.yml" @@ -115,42 +115,37 @@ # # If the application is not in ready state we can fail fast rather than waiting # for the workspace to be Ready -- because it never will! -- name: "Lookup application information" - kubernetes.core.k8s_info: - api_version: "{{ mas_app_api_version }}" - name: "{{ mas_instance_id }}" - namespace: "{{ mas_app_namespace }}" - kind: "{{ mas_app_kind }}" - register: app_cr_result - retries: 15 # Number of retries before failing - delay: 20 # Wait 20 seconds between retries - until: - - app_cr_result.resources is defined - - app_cr_result.resources | length > 0 - - app_cr_result.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select('match', 'True') | list | length == 1 -- name: "Check that the application is ready to configure a workspace" - assert: - that: - - app_cr_result.resources is defined - - app_cr_result.resources | length > 0 - - app_cr_result.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select ('match','True') | list | length == 1 - fail_msg: "Workspace {{ mas_workspace_id }} created but application {{ mas_app_id }} is not ready" +- name: "Verify application is ready" + ibm.mas_devops.wait_for_app_ready: + instance_id: "{{ mas_instance_id }}" + application_id: "{{ mas_app_id }}" + retries: 10 + delay: 30 # 8. Wait for application workspace to be ready # ----------------------------------------------------------------------------- -- ansible.builtin.include_role: - name: ibm.mas_devops.suite_app_verify +- name: "Wait for application workspace to be ready ({{ mas_app_cfg_delay }}s delay)" + ibm.mas_devops.wait_for_app_ready: + instance_id: "{{ mas_instance_id }}" + application_id: "{{ mas_app_id }}" + workspace_id: "{{ mas_workspace_id }}" + retries: "{{ mas_app_cfg_retries }}" + delay: "{{ mas_app_cfg_delay }}" # 9. Run Application Specific Post-configuration # ----------------------------------------------------------------------------- - # Post-config specific application settings after workspace CR be successfully deployed -- name: "Run Manage specific post-configuration" - when: mas_app_id is in ['manage'] # applications which have something to process after configuration +- name: "Run application-specific post-configuration" + when: mas_app_id is in ['manage'] include_tasks: "tasks/{{ mas_app_id }}/post-config/main.yml" -- ansible.builtin.include_role: - name: ibm.mas_devops.suite_app_verify - when: mas_app_id is in ['manage'] # applications which have something to process after configuration +- name: "Wait for application workspace to be ready after post-configuration" + when: mas_app_id is in ['manage'] + ibm.mas_devops.wait_for_app_ready: + instance_id: "{{ mas_instance_id }}" + application_id: "{{ mas_app_id }}" + workspace_id: "{{ mas_workspace_id }}" + retries: "{{ mas_app_cfg_retries }}" + delay: "{{ mas_app_cfg_delay }}" diff --git a/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml b/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml index dc26e069db..e3b9f69c82 100644 --- a/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml +++ b/ibm/mas_devops/roles/suite_app_config/tasks/manage/post-config/main.yml @@ -21,7 +21,7 @@ name: ibm.mas_devops.suite_manage_imagestitching_config when: mas_app_id is in ['manage'] -- name: "Run Manage post-configuration: Set up attachments - Get DB2 Instancd Name" +- name: "Run Manage post-configuration: Set up attachments - Get DB2 Instance Name" set_fact: db2_instance_name: "{{ mas_appws_bindings_jdbc | ibm.mas_devops.get_db2_instance_name(mas_instance_id, mas_workspace_id, 'manage') }}" when: diff --git a/ibm/mas_devops/roles/suite_app_install/tasks/main.yml b/ibm/mas_devops/roles/suite_app_install/tasks/main.yml index 95f2813c50..6b8a176418 100644 --- a/ibm/mas_devops/roles/suite_app_install/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_install/tasks/main.yml @@ -97,25 +97,11 @@ # 8. Wait for application to be ready # ----------------------------------------------------------------------------- - name: "Wait for application to be ready ({{ mas_app_install_delay }}s delay)" - kubernetes.core.k8s_info: - api_version: "{{ mas_app_api_version }}" - name: "{{ mas_instance_id }}" - namespace: "{{ mas_app_namespace }}" - kind: "{{ mas_app_kind }}" - wait: yes # changed true to yes - wait_condition: - status: "True" - type: Ready - wait_sleep: 30 - wait_timeout: 120 # before we give up and fall back into the retry loop - register: app_cr_result - retries: "{{ mas_app_install_retries }}" - delay: "{{ mas_app_install_delay }}" - until: - - app_cr_result.resources is defined - - app_cr_result.resources is not none - - app_cr_result.resources | length > 0 - - app_cr_result.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select ('match','True') | list | length == 1 + ibm.mas_devops.wait_for_app_ready: + instance_id: "{{ mas_instance_id }}" + application_id: "{{ mas_app_id }}" + retries: "{{ mas_app_install_retries }}" + delay: "{{ mas_app_install_delay }}" # 9. Apply final security context constraints diff --git a/ibm/mas_devops/roles/suite_app_verify/README.md b/ibm/mas_devops/roles/suite_app_verify/README.md deleted file mode 100644 index 20b5a71c44..0000000000 --- a/ibm/mas_devops/roles/suite_app_verify/README.md +++ /dev/null @@ -1,35 +0,0 @@ -suite_app_verify -============ - -Verify if a MAS application is ready to use. This is done by verifying if Workspace CR is in READY state. If CR is not in READY state, it repeats verification every minute for ten minutes (you can override this rule and add more time if needed, check Role Variables section for more details) - -Role Variables --------------- - - - -Example Playbook ----------------- - -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - mas_instance_id: masinst1 - mas_workspace_id: masdev - mas_app_ws_apiversion: apps.mas.ibm.com/v1 - mas_app_ws_kind: ManageWorkspace - mas_app_namespace: mas-masinst1-manage - roles: - - ibm.mas_devops.suite_app_verify -``` - -License -------- - -EPL-2.0 diff --git a/ibm/mas_devops/roles/suite_app_verify/defaults/main.yml b/ibm/mas_devops/roles/suite_app_verify/defaults/main.yml deleted file mode 100644 index c481298da0..0000000000 --- a/ibm/mas_devops/roles/suite_app_verify/defaults/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - -mas_app_ws_apiversion: "{{ lookup('env', 'MAS_APP_WS_APIVERSION') }}" -mas_app_ws_kind: "{{ lookup('env', 'MAS_APP_WS_KIND') }}" -mas_app_namespace: "{{ lookup('env', 'MAS_APP_NAMESPACE') }}" - -# By default, verify every minute up to ten minutes -mas_app_cfg_retries: "{{ lookup('env', 'MAS_APP_CFG_RETRIES') | default(10, True) }}" -mas_app_cfg_delay: "{{ lookup('env', 'MAS_APP_CFG_DELAY') | default(60, True) }}" diff --git a/ibm/mas_devops/roles/suite_app_verify/meta/main.yml b/ibm/mas_devops/roles/suite_app_verify/meta/main.yml deleted file mode 100644 index 4f6ac25323..0000000000 --- a/ibm/mas_devops/roles/suite_app_verify/meta/main.yml +++ /dev/null @@ -1,22 +0,0 @@ -galaxy_info: - author: Alexandre Quinteiro (@alequint) - description: Verify an installation of IBM Maximo Application Suite application in the target RHOCP cluster - company: IBM - - license: EPL-2.0 - - min_ansible_version: 2.10 - - platforms: - - name: GenericLinux - versions: - - all - - galaxy_tags: - - ibm - - mas - - devops - - rhocp - -dependencies: - - role: ibm.mas_devops.ansible_version_check diff --git a/ibm/mas_devops/roles/suite_app_verify/tasks/main.yml b/ibm/mas_devops/roles/suite_app_verify/tasks/main.yml deleted file mode 100644 index 9ad0a45a34..0000000000 --- a/ibm/mas_devops/roles/suite_app_verify/tasks/main.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -# 1. Check for undefined properties that do not have a default -# ----------------------------------------------------------------------------- -- name: "Assert that mas_instance_id is defined" - assert: - that: - - mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - -# 2. Provide debug information -# ----------------------------------------------------------------------------- -- name: "Debug information" - debug: - msg: - - "MAS Instance ID ......................... {{ mas_instance_id }}" - - "MAS Workspace ID ........................ {{ mas_workspace_id }}" - - "Application Workspace API Version ....... {{ mas_app_ws_apiversion }}" - - "Application Workspace Kind .............. {{ mas_app_ws_kind }}" - - "Application Namespace ................... {{ mas_app_namespace }}" - - "Retries ................................. {{ mas_app_cfg_retries }}" - - "Delay ................................... {{ mas_app_cfg_delay }}" - -# 3. Wait for application workspace to be ready -# ----------------------------------------------------------------------------- -- name: "Wait for application workspace to be ready ({{ mas_app_cfg_delay }}s delay)" - kubernetes.core.k8s_info: - api_version: "{{ mas_app_ws_apiversion }}" - kind: "{{ mas_app_ws_kind }}" - name: "{{ mas_instance_id }}-{{ mas_workspace_id }}" - namespace: "{{ mas_app_namespace }}" - wait: yes - wait_condition: - status: "True" - type: Ready - wait_sleep: 30 - wait_timeout: 120 # before we give up and fall back into the retry loop - register: app_ws_cr_result - retries: "{{ mas_app_cfg_retries }}" - delay: "{{ mas_app_cfg_delay }}" - until: - - app_ws_cr_result.resources is defined - - app_ws_cr_result.resources | length > 0 - - app_ws_cr_result.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select ('match','True') | list | length == 1 diff --git a/ibm/mas_devops/roles/suite_backup/README.md b/ibm/mas_devops/roles/suite_backup/README.md new file mode 100644 index 0000000000..5f22afaafa --- /dev/null +++ b/ibm/mas_devops/roles/suite_backup/README.md @@ -0,0 +1,13 @@ +Backup and Restore MAS Core +=============================================================================== + +Overview +------------------------------------------------------------------------------- +This role supports backing up MAS Core namespace resources; supports creating on-demand full backups. + +!!! important + Backup can only be restored to an instance with the same MAS instance ID. + + +Role Variables +------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/suite_backup/defaults/main.yml b/ibm/mas_devops/roles/suite_backup/defaults/main.yml new file mode 100644 index 0000000000..83247e808c --- /dev/null +++ b/ibm/mas_devops/roles/suite_backup/defaults/main.yml @@ -0,0 +1,2 @@ +--- +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" diff --git a/ibm/mas_devops/roles/suite_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_backup/tasks/main.yml new file mode 100644 index 0000000000..be78b506f1 --- /dev/null +++ b/ibm/mas_devops/roles/suite_backup/tasks/main.yml @@ -0,0 +1,12 @@ +--- +# Check mas core backup/restore required variables +# ----------------------------------------------------------------------------- +- name: "Fail if mas_instance_id is not provided" + assert: + that: mas_instance_id is defined and mas_instance_id != "" + fail_msg: "mas_instance_id is required" + +- name: "Set fact: mas core namespace name" + set_fact: + mas_core_namespace: "mas-{{ mas_instance_id }}-core" + diff --git a/ibm/mas_devops/roles/suite_backup_restore/README.md b/ibm/mas_devops/roles/suite_backup_restore/README.md deleted file mode 100644 index 865b401715..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/README.md +++ /dev/null @@ -1,113 +0,0 @@ -Backup and Restore MAS Core -=============================================================================== - -Overview -------------------------------------------------------------------------------- -This role supports backing up and restoring MAS Core namespace resources; supports creating on-demand or scheduled backup jobs for taking full or incremental backups, and optionally creating Kubernetes jobs for running the backup/restore process. - -!!! important - A backup can only be restored to an instance with the same MAS instance ID. - - -Role Variables - General -------------------------------------------------------------------------------- -### masbr_action -Set `backup` or `restore` to indicate the role to create a backup or restore job. - -- **Required** -- Environment Variable: `MAS_BR_ACTION` -- Default: None - -### mas_instance_id -Defines the MAS instance ID for the backup or restore action. - -- **Required** -- Environment Variable: `MAS_INSTANCE_ID` -- Default: None - -### masbr_confirm_cluster -Set `true` or `false` to indicate the role whether to confirm the currently connected cluster before running the backup or restore job. - -- Optional -- Environment Variable: `MASBR_CONFIRM_CLUSTER` -- Default: `false` - -### masbr_copy_timeout_sec -Set the transfer files timeout in seconds. - -- Optional -- Environment Variable: `MASBR_COPY_TIMEOUT_SEC` -- Default: `43200` (12 hours) - -### masbr_job_timezone -Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for creating scheduled backup job. If not set a value for this variable, this role will use UTC time zone when creating a CronJob for running scheduled backup job. - -- Optional -- Environment Variable: `MASBR_JOB_TIMEZONE` -- Default: None - -### masbr_storage_local_folder -Set local path to save the backup files. - -- **Required** -- Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` -- Default: None - - -Role Variables - Backup -------------------------------------------------------------------------------- -### masbr_backup_schedule -Set [Cron expression](https://en.wikipedia.org/wiki/Cron) to create a scheduled backup. If not set a value for this varialbe, this role will create an on-demand backup. - -- Optional -- Environment Variable: `MASBR_BACKUP_SCHEDULE` -- Default: None - - -Role Variables - Restore -------------------------------------------------------------------------------- -### masbr_restore_from_version -Set the backup version to use in the restore, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`) - -- **Required** only when `MAS_BR_ACTION=restore` -- Environment Variable: `MASBR_RESTORE_FROM_VERSION` -- Default: None - - -Example Playbook -------------------------------------------------------------------------------- - -### Backup -Backup MAS Core namespace resources, note that this does not include backup of any data in MongoDb, see the `backup` action in the[mongodb](mongodb.md) role. - -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - masbr_action: backup - mas_instance_id: main - masbr_storage_local_folder: /tmp/masbr - roles: - - ibm.mas_devops.suite_backup_restore -``` - -### Restore -Restore MAS Core namespace resources, note that this does not include backup of any data in MongoDb, see the `restore` action in the [mongodb](mongodb.md) role. - -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - masbr_action: restore - masbr_restore_from_version: 20240621021316 - mas_instance_id: main - masbr_storage_local_folder: /tmp/masbr - roles: - - ibm.mas_devops.suite_backup_restore -``` - - -License -------------------------------------------------------------------------------- - -EPL-2.0 diff --git a/ibm/mas_devops/roles/suite_backup_restore/defaults/main.yml b/ibm/mas_devops/roles/suite_backup_restore/defaults/main.yml deleted file mode 100644 index e778df58fa..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/defaults/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -masbr_action: "{{ lookup('env', 'MASBR_ACTION') }}" -mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - -# Backup/Restore - Supported job types -supported_job_data_item_types: ["namespace"] diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/backup-namespace.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/backup-namespace.yml deleted file mode 100644 index 8c867a9b89..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/backup-namespace.yml +++ /dev/null @@ -1,113 +0,0 @@ ---- -# Update namespace resource backup status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update namespace resource backup status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -- name: "Backup namespace resources" - block: - # Prepare namespace resource backup folder - # ------------------------------------------------------------------------- - - name: "Set fact: namespace resource backup folder" - set_fact: - masbr_ns_backup_folder: "{{ masbr_local_job_folder }}/{{ masbr_job_data_type }}" - masbr_ns_backup_name: "{{ masbr_job_name }}-{{ masbr_job_data_type }}" - - - name: "Set fact: namespace resource backup log" - set_fact: - masbr_ns_backup_log: "{{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}.log" - - - name: "Create local backup folder for saving namespace resoruces" - changed_when: true - shell: > - mkdir -p {{ masbr_ns_backup_folder }} && - touch {{ masbr_ns_backup_log }} - - - # Run backup namespace resource script - # ------------------------------------------------------------------------- - - name: "Create backup namespace resource script" - template: - src: "{{ role_path }}/../../common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2" - dest: "{{ masbr_local_job_folder }}/backup-namespace-resources.sh" - mode: "777" - - - name: "Run backup namespace resource script" - changed_when: true - shell: > - {{ masbr_local_job_folder }}/backup-namespace-resources.sh - register: _script_output - - - name: "Debug: run backup namespace resource script" - debug: - msg: "{{ _script_output.stdout_lines }}" - - - # Create tar.gz archives of namespace resource backup files - # ------------------------------------------------------------------------- - - name: "Create tar.gz archives of namespace resource backup files" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}.tar.gz - -C {{ masbr_ns_backup_folder }} . && - ls -lA {{ masbr_ns_backup_folder }} - register: _list_files_output - - - name: "Debug: list of namespace resource backup files" - debug: - msg: "{{ _list_files_output.stdout_lines }}" - - - # Copy backup files to specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup files to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name }}" - masbr_cf_paths: - - src_file: "{{ masbr_ns_backup_name }}.tar.gz" - dest_folder: "{{ masbr_job_data_type }}" - - - # Update namespace resource backup status: Completed - # ------------------------------------------------------------------------- - - name: "Update namespace resource backup status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update namespace resource backup status: Failed - # ------------------------------------------------------------------------- - - name: "Update namespace resource backup status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy namespace resource backup log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of namespace resource backup log" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}-log.tar.gz - -C {{ masbr_local_job_folder }} {{ masbr_ns_backup_name }}.log - - - name: "Copy namespace resource backup log file to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ masbr_local_job_folder }}/{{ masbr_ns_backup_name }}-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/backup-vars.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/backup-vars.yml deleted file mode 100644 index 8c427d75b7..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/backup-vars.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- name: "Set fact: default backup job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" - -- name: "Set fact: namespace backup resources" - set_fact: - masbr_ns_backup_resources: - - namespace: "{{ mas_core_namespace }}" - resources: - - kind: Subscription - name: ibm-mas-operator - - kind: OperatorGroup - name: ibm-mas-operator-group - - kind: Secret - name: ibm-entitlement - - kind: Secret - name: "{{ mas_instance_id }}-credentials-superuser" - # addons.mas.ibm.com - - kind: MVIEdge - - kind: ReplicaDB - # config.mas.ibm.com - - kind: BasCfg - - kind: IDPCfg - - kind: JdbcCfg - - kind: KafkaCfg - - kind: MongoCfg - - kind: ObjectStorageCfg - - kind: PushNotificationCfg - - kind: ScimCfg - - kind: SlsCfg - - kind: SmtpCfg - - kind: WatsonStudioCfg - # core.mas.ibm.com - - kind: Suite - - kind: Workspace - # internal.mas.ibm.com - - kind: CoreIDP diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/get-suite-info.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/get-suite-info.yml deleted file mode 100644 index c07d9b9369..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/get-suite-info.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -# Get Suite version and status -# ----------------------------------------------------------------------------- -- name: "Get Suite" - kubernetes.core.k8s_info: - api_version: core.mas.ibm.com/v1 - kind: Suite - name: "{{ mas_instance_id }}" - namespace: "{{ mas_core_namespace }}" - register: _suite_output - -- name: "Set fact: Suite version" - set_fact: - mas_core_version: "{{ _suite_output.resources[0].status.versions.reconciled }}" - when: - - _suite_output is defined - - (_suite_output.resources | length > 0) - - _suite_output.resources[0].status.versions.reconciled is defined - -- name: "Fail if Suite does not exists" - assert: - that: mas_core_version is defined - fail_msg: "Suite does not exists!" - -- name: "Set fact: Suite status" - set_fact: - mas_core_ready: true - when: - - _suite_output.resources is defined - - (_suite_output.resources | length > 0) - - _suite_output.resources | json_query('[*].status.conditions[?type==`Ready`][].status') | select ('match','True') | list | length == 1 - -- name: "Fail if Suite is not ready" - assert: - that: mas_core_ready is defined and mas_core_ready - fail_msg: "Suite is not ready!" - - -# Output Suite information -# ----------------------------------------------------------------------------- -- name: "Debug: Suite information" - debug: - msg: - - "Suite version .......................... {{ mas_core_version }}" - - "Suite is ready ......................... {{ mas_core_ready }}" diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml deleted file mode 100644 index d22d41f048..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/main.yml +++ /dev/null @@ -1,78 +0,0 @@ ---- -# Check mas core backup/restore required variables -# ----------------------------------------------------------------------------- -- name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - -- name: "Fail if masbr_action is not provided" - assert: - that: masbr_action is defined and masbr_action != "" - fail_msg: "masbr_action is required" - -- name: "Set fact: mas core namespace name" - set_fact: - mas_core_namespace: "mas-{{ mas_instance_id }}-core" - - -# Set common job variables -# ----------------------------------------------------------------------------- -- name: "Set fact: common job variables" - set_fact: - masbr_job_component: - name: "core" - instance: "{{ mas_instance_id }}" - namespace: "{{ mas_core_namespace }}" - -- name: "Load mas core variables" - include_tasks: "tasks/{{ masbr_action }}-vars.yml" - - -# Before run tasks -# ------------------------------------------------------------------------- -- name: "Before run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/before_run_tasks.yml" - vars: - _job_type: "{{ masbr_action }}" - _component_before_task_path: "{{ role_path }}/tasks/get-suite-info.yml" - - -- name: "Run {{ masbr_action }} tasks" - block: - # Update job status: New - # ------------------------------------------------------------------------- - - name: "Update job status: New" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "1" - phase: "New" - - - # Run backup/restore tasks for each data type - # ------------------------------------------------------------------------- - - name: "Run {{ masbr_action }} tasks for each data type" - include_tasks: "{{ role_path }}/tasks/{{ masbr_action }}-{{ job_data_item.type }}.yml" - vars: - masbr_job_data_seq: "{{ job_data_item.seq }}" - masbr_job_data_type: "{{ job_data_item.type }}" - loop: "{{ masbr_job_data_list }}" - loop_control: - loop_var: job_data_item - when: job_data_item.type in supported_job_data_item_types - - rescue: - # Update job status: Failed - # ------------------------------------------------------------------------- - - name: "Update job status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_status: - phase: "Failed" - - always: - # After run tasks - # ------------------------------------------------------------------------- - - name: "After run tasks" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/after_run_tasks.yml" diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml deleted file mode 100644 index bffd88a435..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-namespace.yml +++ /dev/null @@ -1,120 +0,0 @@ ---- -# Update namespace resource restore status: InProgress -# ----------------------------------------------------------------------------- -- name: "Update namespace resource restore status: InProgress" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "InProgress" - - -- name: "Restore namespace resources" - block: - # Prepare namespace resource restore folder - # ------------------------------------------------------------------------- - - name: "Set fact: namespace resource restore folder" - set_fact: - masbr_ns_restore_folder: "{{ masbr_local_job_folder }}/{{ masbr_job_data_type }}" - masbr_ns_restore_name: "{{ masbr_job_name }}-{{ masbr_job_data_type }}" - masbr_ns_restore_from_name: "{{ masbr_restore_from }}-{{ masbr_job_data_type }}" - - - name: "Set fact: namespace resource restore log" - set_fact: - masbr_ns_restore_log: "{{ masbr_local_job_folder }}/{{ masbr_ns_restore_name }}.log" - - - name: "Create local restore folder for saving namespace resoruces" - changed_when: true - shell: > - mkdir -p {{ masbr_ns_restore_folder }} && - touch {{ masbr_ns_restore_log }} - - - name: "Debug: namespace resource restore folder" - debug: - msg: "Namespace resource restore folder ........ {{ masbr_ns_restore_folder }}" - - - # Copy backup file from specified storage location - # ------------------------------------------------------------------------- - - name: "Copy backup file from specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_storage_files_to_local.yml" - vars: - masbr_cf_job_type: "backup" - masbr_cf_job_name: "{{ masbr_restore_from }}" - masbr_cf_paths: - - src_file: "{{ masbr_job_data_type }}/{{ masbr_ns_restore_from_name }}.tar.gz" - dest_folder: "{{ masbr_job_data_type }}" - - - # Extract the tar.gz file - # ------------------------------------------------------------------------- - - name: "Extract the tar.gz file" - changed_when: true - shell: > - mkdir -p {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }} && - tar -xzf {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }}.tar.gz - -C {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }} && - ls -lA {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }} - register: _extract_output - - - name: "Debug: list extracted files" - debug: - msg: - - "Extract output folder .............. {{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }}" - - "{{ _extract_output.stdout_lines }}" - - - # Restore namespace resoruces - # ------------------------------------------------------------------------- - # Loop through the folder - - name: "Get the list of files from restore directory" - find: - paths: "{{ masbr_ns_restore_folder }}/{{ masbr_ns_restore_from_name }}" - patterns: '*.yml,*.yaml' - recurse: no - register: find_result - - - name: "Apply configs" - kubernetes.core.k8s: - state: present - template: "{{ item.path }}" - with_items: "{{ find_result.files }}" - when: find_result is defined - - - # Update database restore status: Completed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Completed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Completed" - - rescue: - # Update database restore status: Failed - # ------------------------------------------------------------------------- - - name: "Update database restore status: Failed" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/update_job_status.yml" - vars: - _job_data_list: - - seq: "{{ masbr_job_data_seq }}" - phase: "Failed" - - always: - # Copy namespace resource restore log file to specified storage location - # ------------------------------------------------------------------------- - - name: "Create a tar.gz archive of namespace resource restore log" - changed_when: true - shell: > - tar -czf {{ masbr_local_job_folder }}/{{ masbr_ns_restore_name }}-log.tar.gz - -C {{ masbr_local_job_folder }} {{ masbr_ns_restore_name }}.log - - - name: "Copy namespace resource restore log file to specified storage location" - include_tasks: "{{ role_path }}/../../common_tasks/backup_restore/copy_local_files_to_storage.yml" - vars: - masbr_cf_job_type: "restore" - masbr_cf_job_name: "{{ masbr_job_name_final }}" - masbr_cf_paths: - - src_file: "{{ masbr_local_job_folder }}/{{ masbr_ns_restore_name }}-log.tar.gz" - dest_folder: "log" diff --git a/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-vars.yml b/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-vars.yml deleted file mode 100644 index cf23734548..0000000000 --- a/ibm/mas_devops/roles/suite_backup_restore/tasks/restore-vars.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: "Set fact: default restore job data list" - set_fact: - masbr_job_data_list: - - seq: "1" - type: "namespace" diff --git a/mkdocs.yml b/mkdocs.yml index a7299f615c..a511b397a4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,7 +25,6 @@ nav: - "Add Real Estate and Facilities": playbooks/mas-facilities.md - "Update": playbooks/mas-update.md - "Upgrade": playbooks/mas-upgrade.md - - "Backup & Restore": playbooks/backup-restore.md - "Roles: OCP Mgmt": - "ocp_cluster_monitoring": roles/ocp_cluster_monitoring.md - "ocp_config": roles/ocp_config.md @@ -79,7 +78,6 @@ nav: - "suite_app_uninstall": roles/suite_app_uninstall.md - "suite_app_upgrade": roles/suite_app_upgrade.md - "suite_app_rollback": roles/suite_app_rollback.md - - "suite_app_backup_restore": roles/suite_app_backup_restore.md - "suite_certs": roles/suite_certs.md - "suite_config": roles/suite_config.md - "suite_db2_setup_for_manage": roles/suite_db2_setup_for_manage.md @@ -98,7 +96,6 @@ nav: - "suite_upgrade": roles/suite_upgrade.md - "suite_rollback": roles/suite_rollback.md - "suite_verify": roles/suite_verify.md - - "suite_backup_restore": roles/suite_backup_restore.md - "Roles: Utilities": - "ansible_version_check": roles/ansible_version_check.md - "entitlement_key_rotation": roles/entitlement_key_rotation.md From a203a88b200718801c27853953e1c4a19f2913ec Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 15 Dec 2025 11:13:28 +0000 Subject: [PATCH 02/61] [patch] update mongo backup-restore playbook and documentation (#2030) Co-authored-by: Sanjay Prabhakar --- docs/playbooks/backup-restore.md | 148 ++++++++++++++++++ ibm/mas_devops/playbooks/br_mongodb.yml | 35 +++++ .../action/get_mongodb_cr_to_restore.py | 2 +- .../action/verify_backup_restore_vars.py | 2 +- ibm/mas_devops/roles/mongodb/README.md | 70 +++------ .../roles/mongodb/defaults/main.yml | 2 +- .../backup-restore/backup-cluster.yml | 19 +++ .../restore-cluster-from-backup.yml | 100 ++++++++++++ .../backup-restore/restore-database.yml | 4 +- .../tasks/providers/community/backup.yml | 24 +-- .../tasks/providers/community/restore.yml | 125 +++------------ 11 files changed, 349 insertions(+), 182 deletions(-) create mode 100644 docs/playbooks/backup-restore.md create mode 100644 ibm/mas_devops/playbooks/br_mongodb.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-cluster.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-cluster-from-backup.yml diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md new file mode 100644 index 0000000000..3bd3fefbf6 --- /dev/null +++ b/docs/playbooks/backup-restore.md @@ -0,0 +1,148 @@ +Backup and Restore +=============================================================================== + +Overview +------------------------------------------------------------------------------- +MAS Devops Collection includes playbooks for backing up and restoring of the following MAS components and their dependencies: + +- [MongoDB](#backuprestore-for-mongodb) +- [Db2](#backuprestore-for-db2) +- [MAS Core](#backuprestore-for-mas-core) +- [Manage](#backuprestore-for-manage) + +# MongoDB Community Edition Backup and Restore Playbook +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_mongodb` will invoke the role [mongodb](../roles/mongodb.md) to backup/restore the MongoDB cluster instance and databases. + +### Overview + +This playbook supports backup and restore operations for a MongoDB Community Edition deployment used by a MAS instance. +If you are using other MongoDB venders, such as IBM Cloud Databases for MongoDB, Amazon DocumentDB or MongoDB Atlas Database, please refer to the corresponding vender's documentation for more information about their provided backup/restore service. + +The process is split into instance-level and database-level operations to provide flexibility depending on the target environment and recovery scenario. + +## Backup Workflow + +The backup operation consists of **two independent phases**. + +### 1. MongoDB Instance Backup (Optional) + +This phase captures the **Kubernetes resources** required to recreate a MongoDB Community Edition cluster, including: + +- MongoDBCommunity custom resource +- Configuration parameters +- User credentials and authentication settings +- TLS and secret references (if applicable) + +This backup enables reinstallation of a MongoDB instance while **preserving configuration and users**. + +#### Skipping Instance Backup + +If instance-level backup is not required, it can be skipped by setting the following environment variable: + +```bash +export BR_SKIP_INSTANCE=true +``` + +2. MongoDB Database Backup + +This phase performs a backup of the MongoDB databases associated with the specified MAS instance ID. + +- Only databases belonging to the MAS instance are included +- The backup is independent of the MongoDB installation method +- Can be used for restore into a new or existing MongoDB instance + +### Restore Workflow + +The restore operation also consists of two phases, with behavior determined by the state of the target environment. + +### 1. MongoDB Instance Restore (Conditional) + +- If no MongoDB instance is currently running in the target namespace: + - The playbook restores the MongoDB Community Edition instance using the previously backed-up Kubernetes resources + - Configuration, users, and credentials are recreated + - Once the instance is ready, database restore begins +- If a MongoDB instance already exists: + - Instance restore is skipped automatically + - The playbook proceeds directly to database restore + + +### 2. MongoDB Database Restore + +- Restores the MongoDB databases from the backup +- Databases are restored into the target MongoDB instance +- Supports restoring into: + - A newly recreated MongoDB instance + - An existing, running MongoDB instance + + +| Scenario | Instance Restore | Database Restore | +| -------------------------------------- | ---------------- | ---------------- | +| Backup with `BR_SKIP_INSTANCE=true` | Skipped | Executed | +| Restore with no MongoDB instance | Executed | Executed | +| Restore with existing MongoDB instance | Skipped | Executed | + +### Notes for Operators +- Ensure the correct MAS instance ID is provided before running the playbook +- Verify namespace access and required permissions for Kubernetes resources and secrets +- Instance restore is idempotent and safely skipped when not applicable + +| Variable Name | Required | Default | Description | +| ----------------------- | -------- | --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `MASBR_ACTION` | Yes | N/A | Specifies whether the playbook should perform a `backup` or a `restore`. | +| `MASBR_BACKUP_VERSION` | Conditional | `YYMMDD-HHMMSS` | Identifies the backup version. For **backup**, this value is optional and defaults to a timestamp in the format YYMMDD-HHMMSS if not provided. For **restore**, this value is mandatory and must match an existing backup version. | +| `MAS_INSTANCE_ID` | Yes | N/A | Identifies the MAS instance whose MongoDB databases should be backed up or restored. To back up multiple MAS instances that share the same MongoDB CE instance, run the playbook multiple times with different values. | +| `MONGODB_NAMESPACE` | No | `mongoce` | Namespace where MongoDB Community Edition is installed. Set this if MongoDB CE is deployed in a custom namespace. | +| `MONGODB_INSTANCE_NAME` | No | `mas-mongo-ce` | Name of the MongoDB Community Edition instance. For backup, this value is used to locate the instance. For restore, the value is taken from the backup data. | +| `BR_SKIP_INSTANCE` | No | `false` | Skips MongoDB instance backup or restore. Set to `true` to back up or restore **databases only**. | + + +### Examples +```bash +# Backup of MongoDB CE cluster instance and database for the dev1 instance +export MASBR_ACTION=backup +export MAS_BACKUP_DIR=/tmp/backup +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_mongodb + + +# Backup of ONLY MongoDB database for the dev1 instance +export MASBR_ACTION=backup +export MAS_BACKUP_DIR=/tmp/backup +export MAS_INSTANCE_ID=dev +export BR_SKIP_INSTANCE=true +ansible-playbook ibm.mas_devops.br_mongodb + + +# Restore MongoDB cluster instance and database +export MASBR_ACTION=restore +export MAS_BACKUP_DIR=/tmp/backup +export MASBR_BACKUP_VERSION=251212-101010 +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_mongodb + +# Restore ONLY MongoDB database +export MASBR_ACTION=restore +export MAS_BACKUP_DIR=/tmp/backup +export MASBR_BACKUP_VERSION=251212-101010 +export MAS_INSTANCE_ID=dev +export BR_SKIP_INSTANCE=true +ansible-playbook ibm.mas_devops.br_mongodb +``` + + + +Backup/Restore for Db2 +------------------------------------------------------------------------------- + +Coming soon... + +Backup/Restore for MAS Core +------------------------------------------------------------------------------- +Coming soon... + + +Backup/Restore for Manage +------------------------------------------------------------------------------- +Coming soon... + diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml new file mode 100644 index 0000000000..fe20acdc86 --- /dev/null +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -0,0 +1,35 @@ +--- +- hosts: localhost + any_errors_fatal: true + vars: + # Define the target for backup/restore + mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups + mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" + mongodb_namespace: "{{ lookup('env', 'MONGODB_NAMESPACE') | default('mongoce', true) }}" + masbr_action: "{{ lookup('env', 'MASBR_ACTION') | default('backup', true) }}" # backup or restore + mongodb_provider: "community" # only community is supported currently + mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Optional + masbr_backup_version: "{{ lookup('env', 'MASBR_BACKUP_VERSION') | default('None', true)}}" # Required for restore action + br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" # set to True to skip mongodb instance backup/restore + + pre_tasks: + - name: "Fail if masbr_action is not set to backup|restore" + assert: + that: masbr_action in ["backup", "restore"] + fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" + + - name: "Fail if require variables for Mongodb backup/restore are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_backup_version: "{{ masbr_backup_version }}" + action: "{{ masbr_action }}" + component: "mongodb" + + roles: + - role: ibm.mas_devops.mongodb + vars: + mongodb_action: "{{ masbr_action }}" + mongodb_backup_version: "{{ masbr_backup_version }}" diff --git a/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py b/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py index 6ed9597bc4..732de926d8 100644 --- a/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py +++ b/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py @@ -40,7 +40,7 @@ def run(self, tmp=None, task_vars=None): mongodb_backup_version = self._task.args.get('mongodb_backup_version') if mongodb_resource_path is None or mongodb_resource_path == "": - raise AnsibleError(f"Error: mongodb_instance_name argument was not provided") + raise AnsibleError(f"Error: mongodb_resource_path argument was not provided") if mongodb_backup_version is None or mongodb_backup_version == "": raise AnsibleError(f"Error: mongodb_backup_version argument was not provided") diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index 2d9d2e5fe9..0eb5bd5865 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -8,7 +8,7 @@ class ActionModule(ActionBase): REQUIRED = { "mongodb": { - "backup": ["mongodb_instance_name", "mas_backup_dir", "mas_instance_id"], # mas_instance_id has a default value + "backup": ["mongodb_instance_name", "mas_backup_dir", "mas_instance_id"], # mongodb_instance_name has a default value "restore": ["mongodb_instance_name", "mas_backup_dir", "mongodb_backup_version"] } } diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index 0c683fe238..7be9d96700 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -178,61 +178,27 @@ Set this to `true` to confirm you want to upgrade your existing Mongo instance f - Environment Variable: `MONGODB_V8_UPGRADE` - Default Value: `false` -### masbr_confirm_cluster -Set `true` or `false` to indicate the role whether to confirm the currently connected cluster before running the backup or restore job. +### mas_backup_dir +Parent directory to store backups. Each component will create its own directory in this parent directory. -- Optional -- Environment Variable: `MASBR_CONFIRM_CLUSTER` -- Default: `false` - -### masbr_copy_timeout_sec -Set the transfer files timeout in seconds. - -- Optional -- Environment Variable: `MASBR_COPY_TIMEOUT_SEC` -- Default: `43200` (12 hours) - -### masbr_job_timezone -Set the [time zone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for creating scheduled backup job. If not set a value for this variable, this role will use UTC time zone when creating a CronJob for running scheduled backup job. - -- Optional -- Environment Variable: `MASBR_JOB_TIMEZONE` -- Default: None - -### masbr_storage_local_folder -Set local path to save the backup files. - -- **Required** -- Environment Variable: `MASBR_STORAGE_LOCAL_FOLDER` +- **Required** only when `MONGODB_ACTION=backup or restore` +- Environment Variable: `MAS_BACKUP_DIR` - Default: None -### masbr_backup_type -Set `full` or `incr` to indicate the role to create a full backup or incremental backup. +### mongodb_backup_version +When `MONGODB_ACTION=backup`, Set version to override the default `YYMMDD-HHMMSS` timestamp version. +When `MONGODB_ACTION=restore`, Set version to use in the restore. Backup with this version will be used to restore. -- Optional -- Environment Variable: `MASBR_BACKUP_TYPE` -- Default: `full` +- **Required** when `MONGODB_ACTION=restore`, default: None +- **Optional when `MONGODB_ACTION=backup`, default: `YYMMDD-HHMMSS` timestamp. +- Environment Variable: `MONGODB_BACKUP_VERSION` -### masbr_backup_from_version -Set the full backup version to use in the incremental backup, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`). This variable is only valid when `MASBR_BACKUP_TYPE=incr`. If not set a value for this variable, this role will try to find the latest full backup version from the specified storage location. +### br_skip_instance +Set this to `true` to skip the instance level or kubernetes resources backup/restore i.e., skips instance/kubernetes resources and processes only database backup/restore. - Optional -- Environment Variable: `MASBR_BACKUP_FROM_VERSION` -- Default: None - -### masbr_backup_schedule -Set [Cron expression](ttps://en.wikipedia.org/wiki/Cron) to create a scheduled backup. If not set a value for this varialbe, this role will create an on-demand backup. - -- Optional -- Environment Variable: `MASBR_BACKUP_SCHEDULE` -- Default: None - -### masbr_restore_from_version -Set the backup version to use in the restore, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`) - -- **Required** only when `MONGODB_ACTION=restore` -- Environment Variable: `MASBR_RESTORE_FROM_VERSION` -- Default: None +- Environment Variable: `BR_SKIP_INSTANCE` +- Default Value: `false` Role Variables - IBM Cloud @@ -538,9 +504,9 @@ Example Playbooks - hosts: localhost any_errors_fatal: true vars: - mongodb_action: backup mas_instance_id: masinst1 - masbr_storage_local_folder: /tmp/masbr + mas_backup_dir: /tmp/masbr + mongodb_action: backup roles: - ibm.mas_devops.mongodb ``` @@ -552,8 +518,8 @@ Example Playbooks vars: mongodb_action: restore mas_instance_id: masinst1 - masbr_restore_from_version: 20240621021316 - masbr_storage_local_folder: /tmp/masbr + mongodb_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr roles: - ibm.mas_devops.mongodb ``` diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index 8f8748643a..f6753ea99a 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -122,7 +122,7 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" -skip_instance_backup_restore: "{{ lookup('env', 'SKIP_INSTANCE_BACKUP_RESTORE') | default(false, true) }}" +br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" mongodb_data_backup: "{{ lookup('env', 'MONGODB_DATA_BACKUP') | default(true, true) }}" # Mongo upgrade flags diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-cluster.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-cluster.yml new file mode 100644 index 0000000000..ec8ffcf3e3 --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-cluster.yml @@ -0,0 +1,19 @@ +--- +# Backup Mongodb Community edition cluster Instance +# ----------------------------------------------------------------------------- + +# Backup Mongodb instance Resources +# ------------------------------------------------------------------------- +- name: "Start Mongo Instance backup process." + ibm.mas_devops.backup_mongo_instance: + mongodb_backup_path: "{{ mongodb_backup_path }}" + mongodb_namespace: "{{ mongodb_namespace }}" + mongodb_instance_name: "{{ mongodb_instance_name }}" + register: mongo_instance_backup_result + +- name: "Assert Mongo Instance backup success" + assert: + that: + - mongo_instance_backup_result is defined + - mongo_instance_backup_result.success == true + fail_msg: "MongoDB Instance resources backup failed." diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-cluster-from-backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-cluster-from-backup.yml new file mode 100644 index 0000000000..7ac60cce2b --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-cluster-from-backup.yml @@ -0,0 +1,100 @@ +--- + +# Prepare for restore +# ----------------------------------------------------------------------------- + +# Get mongodb backup CR information +# ----------------------------------------------------------------------------- +- name: "Check and get mongodb backup cr.yml" + ibm.mas_devops.get_mongodb_cr_to_restore: + mongodb_resource_path: "{{ mongodb_resource_path }}" + mongodb_backup_version: "{{ mongodb_backup_version }}" + register: mongodb_backup_cr_result + +- name: "Set fact : cr.yml" + set_fact: + mongodb_cr: "{{ mongodb_backup_cr_result.mongodb_cr }}" + +- name: "Set fact: Retrieve information from backup CR" + set_fact: + mongodb_namespace: "{{ mongodb_cr.metadata.namespace }}" + mongodb_instance_name: "{{ mongodb_cr.metadata.name }}" + mongo_extras_version: "{{ mongodb_cr.spec.version }}" + target_mongodb_version: "{{ mongodb_cr.spec.version }}" + mongodb_security: "{{ mongodb_cr.spec.security }}" + mongodb_users: "{{ mongodb_cr.spec.users | default([]) }}" + mongodb_replicas: "{{ mongodb_cr.spec.members }}" + mongodb_cpu_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('500m') }}" + mongodb_memory_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('1Gi') }}" + mongodb_cpu_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('250m') }}" + mongodb_memory_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('512Mi') }}" + backup_mongodb_storage_class: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName }}" + mongodb_storage_capacity_data: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.resources.requests.storage }}" + mongodb_storage_capacity_logs: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[1].spec.resources.requests.storage }}" + +- name: "Debug: Mongodb restore variables" + debug: + msg: + - "mongodb_namespace: {{ mongodb_namespace }}" + - "mongodb_instance_name: {{ mongodb_instance_name }}" + - "mongo_extras_version: {{ mongo_extras_version }}" + - "target_mongodb_version: {{ target_mongodb_version }}" + - "mongodb_security: {{ mongodb_security }}" + - "mongodb_users: {{ mongodb_users }}" + - "mongodb_replicas: {{ mongodb_replicas }}" + - "mongodb_cpu_limits: {{ mongodb_cpu_limits }}" + - "mongodb_memory_limits: {{ mongodb_memory_limits }}" + - "mongodb_cpu_requests: {{ mongodb_cpu_requests }}" + - "mongodb_memory_requests: {{ mongodb_memory_requests }}" + - "backup_mongodb_storage_class: {{ backup_mongodb_storage_class }}" + - "mongodb_storage_capacity_data: {{ mongodb_storage_capacity_data }}" + - "mongodb_storage_capacity_logs: {{ mongodb_storage_capacity_logs }}" + +# Prepare storage class +# ----------------------------------------------------------------------------- +# Set storage class name to use for mongodb restore. +# If existing mongo instance found, use its storage class. +# Else use storage class from backup CR. +# If that does not exist, use default storage class. +- name: "Determine storage class to use for MongoDB restore" + ibm.mas_devops.verify_mongoce_version: + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_namespace: "{{ mongodb_namespace }}" + backup_mongodb_storage_class: "{{ backup_mongodb_storage_class }}" + register: storage_class_check_result + +- name: "Set fact: mongodb_storage_class to use for restore" + set_fact: + mongodb_storage_class: "{{ storage_class_check_result.storage_class }}" + when: + - storage_class_check_result is defined + - storage_class_check_result.storage_class is defined + +# Restore MongoDb instance's resources from backup +# We will restore the following resources from backup: +# - User, TLS Secrets defined in the MongoDb CR +# - Issuers in the MongoDb namespace +# - Certificates in the MongoDb namespace +# All the other resources will be created afresh during MongoDb installation +# ----------------------------------------------------------------------------- + +- name: "Restore MongoDB instance resources from backup" + ibm.mas_devops.restore_mongoce_resources: + mongodb_namespace: "{{ mongodb_namespace }}" + backup_secrets_path: "{{ mongodb_resource_path }}/secrets" + backup_issuers_path: "{{ mongodb_resource_path }}/issuers" + backup_certificates_path: "{{ mongodb_resource_path }}/certificates" + register: restore_mongoce_resources_result + +- name: "Assert: MongoDB resources restored successfully" + assert: + that: + - restore_mongoce_resources_result is defined + - restore_mongoce_resources_result.restored is defined + - restore_mongoce_resources_result.restored == True + fail_msg: "Failed to restore MongoDB instance resources from backup." + +# Install the selected MongoDb version for restore +# ----------------------------------------------------------------------------- +- name: "community : Setup mongo version {{ target_mongodb_version }} for restore" + include_tasks: tasks/providers/community/install-mongo.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 4d7400d968..781e663236 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -54,7 +54,7 @@ dest: /tmp/create-role-user.sh mode: '777' - - name: Create database-backup.sh script in local /tmp + - name: Create database-restore.sh script in local /tmp ansible.builtin.template: src: database-restore.sh.j2 dest: /tmp/database-restore.sh @@ -75,7 +75,7 @@ until: copy_result.rc == 0 # The log file will also be available inside the pod /tmp/create-role-user.log -- name: Exec into mongo pod and run create-role-user.sh to setup backup role and user in Mongodb. +- name: Exec into mongo pod and run create-role-user.sh to setup restore role and user in Mongodb. shell: | oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/create-role-user.sh register: create_roleuser_output diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index fffc6f97c6..e2b331e5c1 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -13,7 +13,7 @@ - name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: mongodb_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" - when: mongodb_backup_version is not defined or mongodb_backup_version == "" + when: mongodb_backup_version is not defined or mongodb_backup_version == "" or mongodb_backup_version == "None" - name: "Set fact: Create Mongodb backup base directory path" set_fact: @@ -25,24 +25,12 @@ state: directory mode: "0755" -# Backup Mongodb Kubernetes Resources +# Backup Mongodb Cluster Instance Kubernetes Resources # ------------------------------------------------------------------------- -- name: "Start Mongo Instance backup process." - block: - - name: "Backup MongoDB Instance Kubernetes resources" - ibm.mas_devops.backup_mongo_instance: - mongodb_backup_path: "{{ mongodb_backup_path }}" - mongodb_namespace: "{{ mongodb_namespace }}" - mongodb_instance_name: "{{ mongodb_instance_name }}" - register: mongo_instance_backup_result - - - name: "Assert Mongo Instance backup success" - assert: - that: - - mongo_instance_backup_result is defined - - mongo_instance_backup_result.success == true - fail_msg: "MongoDB Instance resources backup failed." - when: not skip_instance_backup_restore + +- name: "Start MongoCE Instance backup process." + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/backup-cluster.yml" + when: not br_skip_instance | bool # Backup Mongodb database Data using mondodump # ------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml index ffabbbd1eb..8a99894733 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml @@ -24,118 +24,29 @@ mongodb_namespace: "{{ mongodb_namespace }}" register: existing_mongo_info -# Prepare for restore +# Start MongoCE Cluster instance restore +# When the existing mongo instance is not running or not present # ----------------------------------------------------------------------------- -- name: "Prepare for MongoCE instance restore" +- name: "Start MongoCE cluster instance restore" when: - - not skip_instance_backup_restore + - not br_skip_instance - existing_mongo_info is defined - not existing_mongo_info.running - block: + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-cluster-from-backup.yml" - # Get mongodb backup CR information - # ----------------------------------------------------------------------------- - - name: "Check and get mongodb backup cr.yml" - ibm.mas_devops.get_mongodb_cr_to_restore: - mas_backup_dir: "{{ mas_backup_dir }}" - mongodb_backup_version: "{{ mongodb_backup_version }}" - register: mongodb_backup_cr_result - - - name: "Set fact : cr.yml" - set_fact: - mongodb_cr: "{{ mongodb_backup_cr_result.mongodb_cr }}" - - - name: "Set fact: Retrieve information from backup CR" - set_fact: - mongodb_namespace: "{{ mongodb_cr.metadata.namespace }}" - mongodb_instance_name: "{{ mongodb_cr.metadata.name }}" - mongo_extras_version: "{{ mongodb_cr.spec.version }}" - target_mongodb_version: "{{ mongodb_cr.spec.version }}" - mongodb_security: "{{ mongodb_cr.spec.security }}" - mongodb_users: "{{ mongodb_cr.spec.users | default([]) }}" - mongodb_replicas: "{{ mongodb_cr.spec.members }}" - mongodb_cpu_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('500m') }}" - mongodb_memory_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('1Gi') }}" - mongodb_cpu_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('250m') }}" - mongodb_memory_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('512Mi') }}" - backup_mongodb_storage_class: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName }}" - mongodb_storage_capacity_data: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.resources.requests.storage }}" - mongodb_storage_capacity_logs: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[1].spec.resources.requests.storage }}" - - - name: "Debug: Mongodb restore variables" - debug: - msg: - - "mongodb_namespace: {{ mongodb_namespace }}" - - "mongodb_instance_name: {{ mongodb_instance_name }}" - - "mongo_extras_version: {{ mongo_extras_version }}" - - "target_mongodb_version: {{ target_mongodb_version }}" - - "mongodb_security: {{ mongodb_security }}" - - "mongodb_users: {{ mongodb_users }}" - - "mongodb_replicas: {{ mongodb_replicas }}" - - "mongodb_cpu_limits: {{ mongodb_cpu_limits }}" - - "mongodb_memory_limits: {{ mongodb_memory_limits }}" - - "mongodb_cpu_requests: {{ mongodb_cpu_requests }}" - - "mongodb_memory_requests: {{ mongodb_memory_requests }}" - - "backup_mongodb_storage_class: {{ backup_mongodb_storage_class }}" - - "mongodb_storage_capacity_data: {{ mongodb_storage_capacity_data }}" - - "mongodb_storage_capacity_logs: {{ mongodb_storage_capacity_logs }}" - - # Prepare storage class - # ----------------------------------------------------------------------------- - # Set storage class name to use for mongodb restore. - # If existing mongo instance found, use its storage class. - # Else use storage class from backup CR. - # If that does not exist, use default storage class. - - name: "Determine storage class to use for MongoDB restore" - ibm.mas_devops.verify_mongoce_version: - mongodb_instance_name: "{{ mongodb_instance_name }}" - mongodb_namespace: "{{ mongodb_namespace }}" - backup_mongodb_storage_class: "{{ backup_mongodb_storage_class }}" - register: storage_class_check_result - - - name: "Set fact: mongodb_storage_class to use for restore" - set_fact: - mongodb_storage_class: "{{ storage_class_check_result.storage_class }}" - when: - - storage_class_check_result is defined - - storage_class_check_result.storage_class is defined - - # Restore MongoDb instance's resources from backup - # We will restore the following resources from backup: - # - User, TLS Secrets defined in the MongoDb CR - # - Issuers in the MongoDb namespace - # - Certificates in the MongoDb namespace - # All the other resources will be created afresh during MongoDb installation - # ----------------------------------------------------------------------------- - - - name: "Restore MongoDB instance resources from backup" - ibm.mas_devops.restore_mongoce_resources: - mongodb_namespace: "{{ mongodb_namespace }}" - backup_secrets_path: "{{ mongodb_resource_path }}/secrets" - backup_issuers_path: "{{ mongodb_resource_path }}/issuers" - backup_certificates_path: "{{ mongodb_resource_path }}/certificates" - register: restore_mongoce_resources_result - - - name: "Assert: MongoDB resources restored successfully" - assert: - that: - - restore_mongoce_resources_result is defined - - restore_mongoce_resources_result.restored is defined - - restore_mongoce_resources_result.restored == True - fail_msg: "Failed to restore MongoDB instance resources from backup." - - # Install the selected MongoDb version for restore - # ----------------------------------------------------------------------------- - - name: "community : Setup mongo version {{ target_mongodb_version }} for restore" - include_tasks: tasks/providers/community/install-mongo.yml - when: - - existing_mongo_info is defined - - not existing_mongo_info.running - -# Restore the database data from backup +# Recheck for existing MongoDb state +# Just make sure the restore-cluster-from-backup task has created the instance and is running # ----------------------------------------------------------------------------- -- name: "Start Database restore process." - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database.yml" +- name: "Recheck for existing MongoDB installation in namespace {{ mongodb_namespace }}" + ibm.mas_devops.verify_mongoce_version: + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_namespace: "{{ mongodb_namespace }}" + register: existing_mongo_info_2 when: + - not br_skip_instance - existing_mongo_info is defined - - existing_mongo_info.running + - not existing_mongo_info.running + +- name: "Start Database restore process." + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database.yml" + when: (existing_mongo_info is defined and existing_mongo_info.running) or (existing_mongo_info_2 is defined and existing_mongo_info_2.running) From b86828ec63a813f9687b2c21eb7d55af8676b85c Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 22 Dec 2025 18:59:31 +0000 Subject: [PATCH 03/61] [minor] add Full online DB2 backup capability (#2039) Co-authored-by: Sanjay Prabhakar --- .github/workflows/ansible-publish.yml | 98 -------- .github/workflows/ansible.yml | 60 ++++- build/bin/.functions.sh | 2 +- build/ee/execution-environment.yml | 4 +- ibm/mas_devops/meta/runtime.yml | 1 + .../plugins/action/backup_db2_instance.py | 221 ++++++++++++++++++ .../plugins/action/backup_mongo_instance.py | 5 +- .../plugins/action/get_db2u_pod_name.py | 71 ++++++ .../action/update_global_pull_secret.py | 57 +++++ .../action/verify_backup_restore_vars.py | 5 + .../plugins/module_utils/__init__.py | 0 .../plugins/module_utils/backuprestore.py | 120 ++++++++++ ibm/mas_devops/roles/db2/defaults/main.yml | 13 ++ .../db2/files/backup/download_bkp_images.sh | 95 ++++++++ .../files/backup/prepare_backup_scripts.sh | 33 +++ .../db2/tasks/backup/backup-database.yml | 200 ++++++++++++++++ .../db2/tasks/backup/backup-instance.yml | 16 ++ .../roles/db2/tasks/backup/main.yml | 39 +++- .../roles/db2/tasks/install/main.yml | 85 ++++--- .../db2/templates/backup/db2_backup.sh.j2 | 115 +++++++++ .../backup/setup_cos_storage_access.sh.j2 | 8 + .../roles/mongodb/defaults/main.yml | 3 +- .../backup-restore/backup-database.yml | 9 + .../backup-restore/restore-database.yml | 27 ++- .../ocp_idms/tasks/update-pull-secret-dev.yml | 56 +---- .../ocp_idms/tasks/update-pull-secret.yml | 56 +---- .../roles/suite_install/defaults/main.yml | 2 +- .../roles/suite_verify/defaults/main.yml | 2 +- 28 files changed, 1137 insertions(+), 266 deletions(-) delete mode 100644 .github/workflows/ansible-publish.yml create mode 100644 ibm/mas_devops/plugins/action/backup_db2_instance.py create mode 100644 ibm/mas_devops/plugins/action/get_db2u_pod_name.py create mode 100644 ibm/mas_devops/plugins/action/update_global_pull_secret.py create mode 100644 ibm/mas_devops/plugins/module_utils/__init__.py create mode 100644 ibm/mas_devops/plugins/module_utils/backuprestore.py create mode 100755 ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh create mode 100644 ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh create mode 100644 ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml create mode 100644 ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 create mode 100644 ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 diff --git a/.github/workflows/ansible-publish.yml b/.github/workflows/ansible-publish.yml deleted file mode 100644 index 6916461f37..0000000000 --- a/.github/workflows/ansible-publish.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Publish Ansible Collection -on: - release: - types: [ published ] -jobs: - ansible-publish: - runs-on: ubuntu-latest - steps: - - name: Install Python v3.12 - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Checkout - uses: actions/checkout@v2.3.1 - - - name: Initialise the build system - run: | - chmod u+x $GITHUB_WORKSPACE/build/bin/*.sh - $GITHUB_WORKSPACE/build/bin/initbuild.sh - source $GITHUB_WORKSPACE/build/bin/.functions.sh - python -m pip install -q ansible==13.0.0 yamllint - - # Note: Use "--format standard" to ensure filenames and line numbers are in the output - # https://gitanswer.com/yamllint-missing-filenames-and-line-numbers-in-github-action-output-python-794550803 - - name: Validate that the ansible collection lints successfully - run: | - yamllint --format standard -c $GITHUB_WORKSPACE/yamllint.yaml $GITHUB_WORKSPACE/ibm/mas_devops - - - name: Build the Ansible collection - run: | - $GITHUB_WORKSPACE/build/bin/build-collection.sh - - - name: Upload Ansible Collection - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: ${{ github.workspace }}/ibm/mas_devops/ibm-mas_devops-${{ env.VERSION }}.tar.gz - asset_name: ibm-mas_devops-${{ env.VERSION }}.tar.gz - tag: ${{ github.ref }} - overwrite: true - - # Publish - - name: Publish Collection - run: | - ansible-galaxy collection publish ${{ github.workspace }}/ibm/mas_devops/ibm-mas_devops-${{ env.VERSION }}.tar.gz --token=${{ secrets.ANSIBLE_GALAXY_TOKEN }} - - # Execution environment - - name: Build the Ansible execution environment - run: | - podman login --username "${{ secrets.REDHAT_USERNAME }}" --password "${{ secrets.REDHAT_PASSWORD }}" registry.redhat.io - $GITHUB_WORKSPACE/build/bin/build-execution-environment.sh - - - name: Push the Ansible execution environment container image (master) - id: docker-push-master - if: github.ref == 'refs/heads/master' - run: | - podman tag ibmmas/ansible-devops-ee:master quay.io/ibmmas/ansible-devops-ee:master - podman images - podman login --username "${{ secrets.QUAYIO_USERNAME }}" --password "${{ secrets.QUAYIO_PASSWORD }}" quay.io - podman push quay.io/ibmmas/ansible-devops-ee:master - - - name: Push the Ansible execution environment container image (latest) - id: docker-push-latest - if: github.event_name == 'release' - run: | - podman tag ibmmas/ansible-devops-ee quay.io/ibmmas/ansible-devops-ee:latest - podman tag ibmmas/ansible-devops-ee quay.io/ibmmas/ansible-devops-ee:${{ env.DOCKER_TAG }} - podman images - podman login --username "${{ secrets.QUAYIO_USERNAME }}" --password "${{ secrets.QUAYIO_PASSWORD }}" quay.io - podman push quay.io/ibmmas/ansible-devops-ee:latest - podman push quay.io/ibmmas/ansible-devops-ee:${{ env.DOCKER_TAG }} - - - name: Trigger ibm-mas/cli rebuild on Ansible Collection release - run: | - curl -XPOST https://api.github.com/repos/ibm-mas/cli/actions/workflows/build-cli.yml/dispatches \ - -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" \ - -u ${{ secrets.ACTIONS_KEY }} \ - --data '{"ref": "master"}' - - uses: actions/checkout@v4 - - - name: Perform dependency check - uses: dependency-check/Dependency-Check_Action@main - id: owasp-depcheck - with: - project: 'ansible-devops' - path: '.' - format: 'HTML' - args: > - --failOnCVSS 7 - --enableRetired - - - name: Upload dependency check results - uses: actions/upload-artifact@v4 - with: - name: OWASP dependency check report - path: ${{github.workspace}}/reports - retention-days: 90 diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index f776b15c9e..9266c6987e 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -1,12 +1,19 @@ name: Build Ansible Collection + on: push: - branches: - - '**' - tags-ignore: - - '**' + branches: [ "**" ] + tags-ignore: [ "**" ] + release: + types: [ published ] + +# Ensure only one build at a time for any branch, cancelling any in-progress builds +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - ansible-build: + build-ansible: name: Build Ansible Collection runs-on: ubuntu-latest if: ${{ !contains(github.event.head_commit.message, '[doc]') }} @@ -39,7 +46,7 @@ jobs: run: | $GITHUB_WORKSPACE/build/bin/build-collection.sh - - name: Upload Ansible Collection to Github Actions + - name: Upload Ansible Collection to Github Actions Build Output uses: actions/upload-artifact@v4 with: name: ibm-mas_devops-${{ env.VERSION }}.tar.gz @@ -51,14 +58,42 @@ jobs: ARTIFACTORY_GENERIC_RELEASE_URL: ${{ secrets.ARTIFACTORY_GENERIC_RELEASE_URL }} ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }} run: | - echo "Copying $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops-${{ env.VERSION }}.tar.gz to $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops.tar.gz..." + echo "Copying $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops-${{ env.VERSION }}.tar.gz to $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops.tar.gz ..." cp $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops-${{ env.VERSION }}.tar.gz $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops.tar.gz $GITHUB_WORKSPACE/build/bin/artifactory-release.sh $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops.tar.gz - # Execution environment + - name: Publish Collection + if: ${{ github.event_name == 'release' }} + run: | + ansible-galaxy collection publish ${{ github.workspace }}/ibm/mas_devops/ibm-mas_devops-${{ env.VERSION }}.tar.gz --token=${{ secrets.ANSIBLE_GALAXY_TOKEN }} + + + build-ee: + name: Build Ansible Execution Environment + runs-on: ubuntu-latest + if: ${{ !contains(github.event.head_commit.message, '[doc]') }} && ${{ !contains(github.event.head_commit.message, '[skip ee]') }} + steps: + - name: Checkout + uses: actions/checkout@v2.3.1 + # Without this option, we don't get the tag information + with: + fetch-depth: 0 + + - name: Initialise the build system + run: | + chmod u+x $GITHUB_WORKSPACE/build/bin/*.sh + $GITHUB_WORKSPACE/build/bin/initbuild.sh + source $GITHUB_WORKSPACE/build/bin/.functions.sh + python -m pip install -q ansible==13.0.0 yamllint + + - name: Build the Ansible collection + run: | + $GITHUB_WORKSPACE/build/bin/build-collection.sh + - name: Build the Ansible execution environment if: env.VERSION != '' run: | + cp $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops-${{ env.VERSION }}.tar.gz $GITHUB_WORKSPACE/ibm/mas_devops/ibm-mas_devops.tar.gz podman login --username "${{ secrets.REDHAT_USERNAME }}" --password "${{ secrets.REDHAT_PASSWORD }}" registry.redhat.io $GITHUB_WORKSPACE/build/bin/build-execution-environment.sh @@ -72,11 +107,16 @@ jobs: podman login --username "${{ secrets.QUAYIO_USERNAME }}" --password "${{ secrets.QUAYIO_PASSWORD }}" quay.io podman push quay.io/ibmmas/ansible-devops-ee:${{ env.DOCKER_TAG }} + rebuild-cli: + name: Rebuild ibmmas/cli:master + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' + needs: build-ansible + steps: - name: Trigger ibm-mas/cli rebuild on Ansible Collection master build - if: github.ref == 'refs/heads/master' + if: ${{ !contains(github.event.head_commit.message, '[doc]') }} run: | curl -XPOST https://api.github.com/repos/ibm-mas/cli/actions/workflows/build-cli.yml/dispatches \ -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" \ -u ${{ secrets.ACTIONS_KEY }} \ --data '{"ref": "master"}' - - uses: actions/checkout@v4 diff --git a/build/bin/.functions.sh b/build/bin/.functions.sh index 6f19b01a7f..618e27bcf7 100644 --- a/build/bin/.functions.sh +++ b/build/bin/.functions.sh @@ -86,5 +86,5 @@ function artifactory_upload() { sha1Value="${sha1Value:0:40}" echo "Uploading $1 to $2" - curl -H "Authorization:Bearer $ARTIFACTORY_TOKEN" -H "X-Checksum-Md5: $md5Value" -H "X-Checksum-Sha1: $sha1Value" -T $1 $2 || exit 1 + curl --fail -H "Authorization:Bearer $ARTIFACTORY_TOKEN" -H "X-Checksum-Md5: $md5Value" -H "X-Checksum-Sha1: $sha1Value" -T $1 $2 || exit 1 } diff --git a/build/ee/execution-environment.yml b/build/ee/execution-environment.yml index 20769ab98f..6a16cf59c7 100644 --- a/build/ee/execution-environment.yml +++ b/build/ee/execution-environment.yml @@ -11,7 +11,9 @@ dependencies: system: bindep.txt images: base_image: - name: registry.redhat.io/ansible-automation-platform-24/ee-supported-rhel9:latest + # Important: do not use "latest" - on multiple occasions it has broken our builds now + # New version of ee-supported-rhel9 image must be tested before being introduced here + name: registry.redhat.io/ansible-automation-platform-24/ee-supported-rhel9:1.0.0-1107 # Custom package manager path for the RHEL based images options: package_manager_path: /usr/bin/microdnf diff --git a/ibm/mas_devops/meta/runtime.yml b/ibm/mas_devops/meta/runtime.yml index 94bea8cb7a..be19d92986 100644 --- a/ibm/mas_devops/meta/runtime.yml +++ b/ibm/mas_devops/meta/runtime.yml @@ -20,3 +20,4 @@ action_groups: - verify_mongoce_version - wait_for_app_ready - wait_for_conditions + - update_global_pull_secret diff --git a/ibm/mas_devops/plugins/action/backup_db2_instance.py b/ibm/mas_devops/plugins/action/backup_db2_instance.py new file mode 100644 index 0000000000..7517819054 --- /dev/null +++ b/ibm/mas_devops/plugins/action/backup_db2_instance.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +import logging +import urllib3 + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import createBackupDirectories, backupSecret, getSubscription, getDb2uInstance, getDb2VersionFromCR, isDb2uReady + +import yaml +import os +import base64 + +from mas.devops.ocp import getCR, getSecret + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +display = Display() + + +def filterDb2CR(crData: dict) -> dict: + """ + Filter out unnecessary fields from a Custom Resource + """ + metadata_fields_to_remove = [ + 'annotations', + 'creationTimestamp', + 'generation', + 'resourceVersion', + 'selfLink', + 'uid', + 'managedFields' + ] + filteredCR = crData.copy() + if 'metadata' in filteredCR: + for field in metadata_fields_to_remove: + if field in filteredCR['metadata']: + del filteredCR['metadata'][field] + # remove status field + if 'status' in filteredCR: + del filteredCR['status'] + + return filteredCR + +def backupDB2uCR(db2u_cr: dict, backup_path: str) -> bool: + """ + Backup Db2uCluster/Db2uInstance CR to specified backup path + """ + try: + filteredCR = filterDb2CR(db2u_cr) + backup_file_path = os.path.join(backup_path, "cr.yaml") + with open(backup_file_path, 'w') as backup_file: + yaml.dump(filteredCR, backup_file) + display.v(f"Backed up Db2u CR to {backup_file_path}") + return True + except Exception as e: + display.v(f"Error backing up Db2u CR: {e}") + return False + +def getImagePullSecretFromCR(db2uCR: dict) -> list: + """ + Extract image pull secret name from Db2uCluster/Db2uInstance CR + """ + try: + if 'spec' in db2uCR and 'account' in db2uCR['spec'] and 'imagePullSecrets' in db2uCR['spec']['account']: + image_pull_secrets = db2uCR['spec']['account']['imagePullSecrets'] + if isinstance(image_pull_secrets, list) and len(image_pull_secrets) > 0: + return image_pull_secrets + return None + except Exception as e: + display.v(f"Error extracting image pull secret from CR: {e}") + return None + +def processUserCredentialsSecret(dynClient: DynamicClient, mas_instance_id: str, db2_instance_name: str, backup_path: str) -> bool: + """ + Process user credentials secret for Db2u instance + Check user credentials from jdbc credentials secret in Core namespace + If the user is not the default db2inst1 user, backup the jdbc credentials secret as well + this will be used during restore to set the correct user + """ + jdbc_core_secret_name = f"jdbc-{db2_instance_name}-credentials".lower() + jdbc_core_secret = getSecret(dynClient, namespace=f"mas-{mas_instance_id}-core".lower(), secret_name=jdbc_core_secret_name) + if jdbc_core_secret: + if 'data' in jdbc_core_secret and 'username' in jdbc_core_secret['data']: + username_encoded = jdbc_core_secret['data']['username'] + username = base64.b64decode(username_encoded).decode('utf-8') + if username.lower() != 'db2inst1': + # Backup the jdbc credentials secret + backup_status = backupSecret(dynClient=dynClient, namespace=f"mas-{mas_instance_id}-core".lower(), secret_name=jdbc_core_secret_name, backup_path=backup_path) + if backup_status: + display.v(f"Backed up JDBC credentials secret '{jdbc_core_secret_name}' for Db2u instance '{db2_instance_name}'") + return True + else: + display.v(f"Warning: Failed to backup JDBC credentials secret '{jdbc_core_secret_name}' for Db2u instance '{db2_instance_name}'") + return False + else: + display.v(f"JDBC credentials secret '{jdbc_core_secret_name}' uses default admin user. No additional user backup needed.") + return False + +class ActionModule(ActionBase): + + """ + Usage Example + ------------- + tasks: + - name: "Backup Db2u instance resources (CR, secrets, configmaps, issuers, certificates)" + ibm.mas_devops.backup_db2_instance: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + db2_instance_name = self._task.args.get('db2_instance_name', None) + db2_namespace = self._task.args.get('db2_namespace', None) + db2_backup_path = self._task.args.get('db2_backup_path', None) + mas_instance_id = self._task.args.get('mas_instance_id', None) + db2_dbname = self._task.args.get('db2_dbname', None) + + if not db2_instance_name or db2_instance_name == '': + raise AnsibleError("db2_instance_name is a required parameter and cannot be empty") + if not db2_namespace or db2_namespace == '': + raise AnsibleError("db2_namespace is a required parameter and cannot be empty") + if not db2_backup_path or db2_backup_path == '': + raise AnsibleError("db2_backup_path is a required parameter and cannot be empty") + if not mas_instance_id or mas_instance_id == '': + raise AnsibleError("mas_instance_id is a required parameter and cannot be empty") + if not db2_dbname or db2_dbname == '': + raise AnsibleError("db2_dbname is a required parameter and cannot be empty") + + # Prepare backup resource path + db2_backup_resource_path = f"{db2_backup_path}/resources" + db2_backup_secrets_path = f"{db2_backup_resource_path}/secrets" + #db2_backup_issuers_path = f"{db2_backup_resource_path}/issuers" + + # Create backup directory if it does not exist + createBackupDirectories([db2_backup_path, db2_backup_resource_path, db2_backup_secrets_path]) + + # Get Db2uCluster or Db2uInstance CR + db2u_cr = getDb2uInstance(DynamicClient=dynClient, db2_instance_name=db2_instance_name, db2_namespace=db2_namespace) + if not db2u_cr: + raise AnsibleError(f"Db2uCluster or Db2uInstance CR '{db2_instance_name}' not found in namespace '{db2_namespace}'") + + # Check if db2u instance is in a ready state + if not isDb2uReady(db2u_cr): + raise AnsibleError(f"Db2u instance '{db2_instance_name}' is not in a ready state. Cannot proceed with backup.") + + # Backup Db2uCluster/Db2uInstance CR + backup_cr_status = backupDB2uCR(db2u_cr=db2u_cr, backup_path=db2_backup_resource_path) + if not backup_cr_status: + raise AnsibleError(f"Failed to backup Db2u CR for instance '{db2_instance_name}'") + + # Backup Db2u Secrets + # Backup image pull secret if specified in CR + image_pull_secrets = getImagePullSecretFromCR(db2u_cr) + if image_pull_secrets: + for secret in image_pull_secrets: + backupSecretStatus = backupSecret(dynClient=dynClient, namespace=db2_namespace, secret_name=secret, backup_path=db2_backup_secrets_path) + if not backupSecretStatus: + display.v(f"Warning: Failed to backup image pull secret '{secret}' for Db2u instance '{db2_instance_name}'") + else: + display.v(f"Backed up image pull secret '{secret}' for Db2u instance '{db2_instance_name}'") + + # Backup Db2u instance secret + db2_instance_secret_name = f"c-{db2_instance_name}-instancepassword" + backupSecretStatus = backupSecret(dynClient=dynClient, namespace=db2_namespace, secret_name=db2_instance_secret_name, backup_path=db2_backup_secrets_path) + if not backupSecretStatus: + display.v(f"Warning: Failed to backup Db2u instance secret '{db2_instance_secret_name}' for Db2u instance '{db2_instance_name}'") + else: + display.v(f"Backed up Db2u instance secret '{db2_instance_secret_name}' for Db2u instance '{db2_instance_name}'") + + # Check user credentials from jdbc credentials secret in Core namespace + # If the user is not the default admin user, backup the jdbc credentials secret as well + # this will be used during restore to set the correct user + # If secret does not exist or uses default user, no action is taken. + # Restore process will use default instance user in that case + processUserCredentialsSecret(dynClient=dynClient, mas_instance_id=mas_instance_id, db2_instance_name=db2_instance_name, backup_path=db2_backup_secrets_path) + + # Get Channel details from Subscription + # Package name hardcoded to db2u-operator + subscription = getSubscription(dynClient=dynClient, namespace=db2_namespace, package_name="db2u-operator") + channel_name = None + if subscription: + if 'spec' in subscription and 'channel' in subscription['spec']: + channel_name = subscription['spec']['channel'] + else: + display.v(f"Warning: Subscription 'db2u-operator' not found in namespace '{db2_namespace}'. Cannot retrieve channel information.") + raise AnsibleError(f"Failed to retrieve Subscription for Db2u operator in namespace '{db2_namespace}'") + + # write db2-info.yaml file + # using str() to ensure no yaml serialization issues with non-string types + # Ansible wraps variables with metadata + db2_info = { + 'db2_instance_name': str(db2_instance_name), + 'db2_namespace': str(db2_namespace), + 'db2_dbname': str(db2_dbname), + 'mas_instance_id': str(mas_instance_id), + 'db2_version': getDb2VersionFromCR(db2uCR=db2u_cr), + 'db2_channel': channel_name + } + + db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-info.yaml") + with open(db2_info_file_path, 'w') as info_file: + yaml.dump(db2_info, info_file) + display.v(f"Wrote Db2 instance info to {db2_info_file_path}") + + return dict( + message=f"Successfully backed up Standalone Db2 Universal Operator instance '{db2_instance_name}' resources", + failed=False, + changed=False, + success=True, + ) \ No newline at end of file diff --git a/ibm/mas_devops/plugins/action/backup_mongo_instance.py b/ibm/mas_devops/plugins/action/backup_mongo_instance.py index d1974e5bb9..c3efbba98b 100644 --- a/ibm/mas_devops/plugins/action/backup_mongo_instance.py +++ b/ibm/mas_devops/plugins/action/backup_mongo_instance.py @@ -264,7 +264,7 @@ class ActionModule(ActionBase): ------------- tasks: - name: "Backup MongoDB instance resources (CR, secrets, configmaps, issuers, certificates)" - ibm.mas_devops.backup_mongo_cr: + ibm.mas_devops.backup_mongo_instance: """ def run(self, tmp=None, task_vars=None): super(ActionModule, self).run(tmp, task_vars) @@ -357,9 +357,6 @@ def run(self, tmp=None, task_vars=None): failed=False, changed=False, success=True, - ansible_facts={ - "mongodb_version": getMongoVersion(mongodb_cr) # this fact can be used in subsequent tasks - } ) diff --git a/ibm/mas_devops/plugins/action/get_db2u_pod_name.py b/ibm/mas_devops/plugins/action/get_db2u_pod_name.py new file mode 100644 index 0000000000..9b7b71784e --- /dev/null +++ b/ibm/mas_devops/plugins/action/get_db2u_pod_name.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import getDb2uInstance, getDb2VersionFromCR, isDb2uReady + + +# Disabling warnings will prevent InsecureRequestWarnings from dynClient +urllib3.disable_warnings() +display = Display() + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Get Db2u pod name + ibm.mas_devops.get_db2u_pod_name: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + db2_instance_name = self._task.args.get('db2_instance_name') + db2_namespace = self._task.args.get('db2_namespace') + + if db2_instance_name is None: + raise AnsibleError(f"Error: db2_instance_name argument was not provided") + if db2_namespace is None: + raise AnsibleError(f"Error: db2_namespace argument was not provided") + + # First, check if the Db2u Instance is present and healthy state. + db2u_cr = getDb2uInstance(dynClient, db2_instance_name, db2_namespace) + if not db2u_cr: + raise AnsibleError(f"Error: Db2u instance '{db2_instance_name}' not found in namespace '{db2_namespace}'") + if not isDb2uReady(db2u_cr): + raise AnsibleError(f"Error: Db2u instance '{db2_instance_name}' is not in 'Ready' state") + + # Next, get the Pod name for the Db2u instance + label_selector = f"app={db2_instance_name},type=engine" + display.v(f"Looking up Db2u Pod in namespace '{db2_namespace}' with labels '{label_selector}'") + try: + podAPI = dynClient.resources.get(api_version="v1", kind="Pod") + pods = podAPI.get(namespace=db2_namespace, label_selector=label_selector) + if pods.items: + pod_name = pods.items[0]["metadata"]["name"] + display.v(f"Found Pod '{pod_name}' in namespace '{db2_namespace}' with labels '{label_selector}'") + return dict( + failed=False, + success=True, + pod_name=pod_name, + db2version=getDb2VersionFromCR(db2u_cr), + msg="Db2u Pod found" + ) + else: + display.v(f"No Pods found in namespace '{db2_namespace}' with labels '{label_selector}'") + except NotFoundError: + display.v(f"No Pods found in namespace '{db2_namespace}' with labels '{label_selector}'") + return dict(failed=True, success=False, pod_name="", db2version="", msg="Db2u Pod not found") + diff --git a/ibm/mas_devops/plugins/action/update_global_pull_secret.py b/ibm/mas_devops/plugins/action/update_global_pull_secret.py new file mode 100644 index 0000000000..9f89de599f --- /dev/null +++ b/ibm/mas_devops/plugins/action/update_global_pull_secret.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase + +from mas.devops.ocp import updateGlobalPullSecret + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +class ActionModule(ActionBase): + """ + Update the global pull secret in openshift-config namespace with registry credentials. + + Usage Example + ------------- + tasks: + - name: "Update Global Pull Secret" + ibm.mas_devops.update_global_pull_secret: + registry_url: "{{ registry_private_url }}" + username: "{{ registry_username }}" + password: "{{ registry_password }}" + register: result + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + registryUrl = self._task.args.get('registry_url', None) + username = self._task.args.get('username', None) + password = self._task.args.get('password', None) + + if registryUrl is None: + raise AnsibleError(f"Error: registry_url argument was not provided") + if username is None: + raise AnsibleError(f"Error: username argument was not provided") + if password is None: + raise AnsibleError(f"Error: password argument was not provided") + + # Initialize DynamicClient and update the global pull secret + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + result = updateGlobalPullSecret(dynClient, registryUrl, username, password) + + return dict( + message=f"Successfully updated global pull secret with credentials for {registryUrl}", + success=True, + failed=False, + changed=result.get('changed', True), + name=result.get('name'), + namespace=result.get('namespace'), + registry=result.get('registry') + ) \ No newline at end of file diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index 0eb5bd5865..94c155185e 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -10,6 +10,11 @@ class ActionModule(ActionBase): "mongodb": { "backup": ["mongodb_instance_name", "mas_backup_dir", "mas_instance_id"], # mongodb_instance_name has a default value "restore": ["mongodb_instance_name", "mas_backup_dir", "mongodb_backup_version"] + }, + "db2": { + "backup": ["db2_instance_name", "mas_backup_dir", "mas_instance_id", "mas_app_id"], + "restore": ["db2_instance_name", "mas_backup_dir", "db2_backup_version", "mas_app_id"], + "s3_setup": ["backup_vendor","backup_s3_name", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] } } diff --git a/ibm/mas_devops/plugins/module_utils/__init__.py b/ibm/mas_devops/plugins/module_utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ibm/mas_devops/plugins/module_utils/backuprestore.py b/ibm/mas_devops/plugins/module_utils/backuprestore.py new file mode 100644 index 0000000000..eede7f0b47 --- /dev/null +++ b/ibm/mas_devops/plugins/module_utils/backuprestore.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +import os +import yaml +from mas.devops.ocp import getSecret +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +from mas.devops.ocp import getCR, getSecret + + +def createBackupDirectories(paths: list) -> bool: + """ + Create backup directories if they do not exist + """ + try: + for path in paths: + os.makedirs(path, exist_ok=True) + return True + except Exception as e: + return False + +def copyContentsToYamlFile(file_path: str, content: dict) -> bool: + """ + Write dictionary content to a YAML file + """ + try: + with open(file_path, 'w') as yaml_file: + yaml.dump(content, yaml_file, default_flow_style=False) + return True + except Exception as e: + return False + +def filterResourceData(data: dict) -> dict: + """ + filter metadata from Resource data and create minimal dict + """ + metadata_fields_to_remove = [ + 'annotations', + 'creationTimestamp', + 'generation', + 'resourceVersion', + 'selfLink', + 'uid', + 'managedFields' + ] + filteredCopy = data.copy() + if 'metadata' in filteredCopy: + for field in metadata_fields_to_remove: + if field in filteredCopy['metadata']: + del filteredCopy['metadata'][field] + + if 'status' in filteredCopy: + del filteredCopy['status'] + + return filteredCopy + +def backupSecret(dynClient: DynamicClient, namespace: str, secret_name: str, backup_path: str) -> bool: + """ + Backup a Secret to a YAML file + """ + secret = getSecret(dynClient, namespace, secret_name) + if secret: + secret_file_path = f"{backup_path}/{secret_name}.yaml" + filtered_secret = filterResourceData(secret) + if copyContentsToYamlFile(secret_file_path, filtered_secret): + return True + else: + return False + else: + return False + +def getSubscription(dynClient: DynamicClient, namespace: str, package_name: str): + """ + Retrieve Subscription resource by package name in the specified namespace + """ + try: + subscription_resource = dynClient.resources.get(api_version="operators.coreos.com/v1alpha1", kind="Subscription") + subscription = subscription_resource.get(namespace=namespace, name=package_name) + return subscription.to_dict() + except NotFoundError: + return None + except Exception as e: + return None + +def getDb2VersionFromCR(db2uCR: dict) -> str: + """ + Extract Db2 version from Db2uCluster/Db2uInstance CR + """ + try: + if 'spec' in db2uCR and 'version' in db2uCR['spec']: + return db2uCR['spec']['version'] + return None + except Exception as e: + return None + +def getDb2uInstance(DynamicClient, db2_instance_name: str, db2_namespace: str): + """ + Retrieve Db2uCluster CR instance + """ + + db2uCR = getCR(DynamicClient, cr_api_version="db2u.databases.ibm.com/v1", cr_kind="Db2uCluster", cr_name=db2_instance_name, namespace=db2_namespace) + if db2uCR: + return db2uCR.to_dict() + else: + # Db2uCluster CR not found, try Db2uInstance + db2uCR = getCR(DynamicClient, cr_api_version="db2u.databases.ibm.com/v1", cr_kind="Db2uInstance", cr_name=db2_instance_name, namespace=db2_namespace) + if db2uCR: + return db2uCR.to_dict() + return None + +def isDb2uReady(db2uCR: dict) -> bool: + """ + Check if Db2uCluster/Db2uInstance is in ready state + """ + if 'status' in db2uCR: + status = db2uCR['status'] + if 'state' in status and status['state'] == 'Ready': + return True + return False \ No newline at end of file diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index 72ae6ec606..e272b54115 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -123,3 +123,16 @@ ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" # Custom Labels # ----------------------------------------------------------------------------- custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string | ibm.mas_devops.string2dict() }}" + +# Backup and Restore variables +# ----------------------------------------------------------------------------- +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" +db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" +br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" +backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('s3', true) }}" # Supported values are s3 and disk +backup_s3_name: "{{ lookup('env', 'BACKUP_S3_NAME') | default('S3DB2COS', true) }}" +backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" +backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" +backup_s3_access_key: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY') }}" +backup_s3_secret_key: "{{ lookup('env', 'BACKUP_S3_SECRET_KEY') }}" +backup_type: full diff --git a/ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh b/ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh new file mode 100755 index 0000000000..483fe819b8 --- /dev/null +++ b/ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh @@ -0,0 +1,95 @@ +#!/bin/bash +set -o pipefail + +# ********************************************************** +# Download DB2 backup from IBMCOS +# +# Operation: DOWNLOAD_BACKUP +# +# Parameters: +# $1 stgEndpoint - IBMCOS storage endpoint +# $2 access_key_id - IBMCOS access_key_id +# $3 secret_access_key - IBMCOS secret_access_key +# $$ ibm_cos_bucket - IBMCOS bucket +# $5 ibm_cos_path - IBMCOS ibm_cos_path +# $6 flex_db_name - Original migrated database name +# +#********************************************************** +#DOWNLOAD_BACKUP +function downloadDBBackup() +{ + stgEndpoint=$1 + access_key_id=$2 + secret_access_key=$3 + ibm_cos_bucket=$4 + ibm_cos_path=$5 + flex_db_name=$6 + download_path=$7 + file_name_to_be_downloaded=$8 + db2_version=$9 + cos_bucket_alias=${10} + DOWNLOAD_BACK_LOG=~/bin/.DownloadBackupImagesLOG.out + rm -f ${DOWNLOAD_BACK_LOG} + + echo "[INFO] DOWNLOAD_BACKUP" >> ${DOWNLOAD_BACK_LOG} + + echo "[DEBUG] DOWNLOAD_BACKUP stgEndpoint: $stgEndpoint" >> ${DOWNLOAD_BACK_LOG} + + #echo "[DEBUG] DOWNLOAD_BACKUP access_key_id; $access_key_id" + #echo "[DEBUG] DOWNLOAD_BACKUP secret_access_key: $secret_access_key" + echo "[DEBUG] DOWNLOAD_BACKUP access_key_id; XXXXXXXXXX" >> ${DOWNLOAD_BACK_LOG} + echo "[DEBUG] DOWNLOAD_BACKUP secret_access_key: XXXXXXXXXX" >> ${DOWNLOAD_BACK_LOG} + + echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_bucket: $ibm_cos_bucket" >> ${DOWNLOAD_BACK_LOG} + echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_path: $ibm_cos_path" >> ${DOWNLOAD_BACK_LOG} + echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_bucket: $ibm_cos_bucket" >> ${DOWNLOAD_BACK_LOG} + echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_path: $ibm_cos_path" >> ${DOWNLOAD_BACK_LOG} + echo "[DEBUG] DOWNLOAD_BACKUP flex_db_name: $flex_db_name" >> ${DOWNLOAD_BACK_LOG} + echo "[DEBUG] DOWNLOAD_BACKUP download_path: $download_path" >> ${DOWNLOAD_BACK_LOG} + echo "[DEBUG] DOWNLOAD_BACKUP file_name_to_be_downloaded: $file_name_to_be_downloaded" >> ${DOWNLOAD_BACK_LOG} + if [ $db2_version == 'v11.5.7.0' ]; then + db2RemStgManager S3 get server=${stgEndpoint} auth1=${access_key_id} auth2=${secret_access_key} container=${ibm_cos_bucket} source=${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded 2>&1 | tee -a ${DOWNLOAD_BACK_LOG} + else + db2RemStgManager ALIAS GET source=DB2REMOTE://${cos_bucket_alias}/${ibm_cos_bucket}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded 2>&1 | tee -a ${DOWNLOAD_BACK_LOG} + fi + rc=$? + if [ $rc -ne 0 ]; then + if [ $db2_version == 'v11.5.7.0' ]; then + echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager s3 get ${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} + else + echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager ALIAS GET source=DB2REMOTE://${cos_bucket_alias}/${ibm_cos_bucket}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} + fi + echo "status=fail" >> ${DOWNLOAD_BACK_LOG} + exit 1 + fi + + checkDownload=$(ls -1 $download_path/$file_name_to_be_downloaded) + echo "[DEBUG] checkDownload= $checkDownload" >> ${DOWNLOAD_BACK_LOG} + + if [[ $checkDownload =~ "$download_path/$file_name_to_be_downloaded" ]]; then + echo "[DEBUG] Backup image $file_name_to_be_downloaded donwloaded successfully" >> ${DOWNLOAD_BACK_LOG} + else + if [ $db2_version == 'v11.5.7.0' ]; then + echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager s3 get ${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} + else + echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager ALIAS GET source=DB2REMOTE://${cos_bucket_alias}/${ibm_cos_bucket}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} + fi + echo "status=fail" >> ${DOWNLOAD_BACK_LOG} + exit 1 + fi + + echo "status=success" | tee -a ${DOWNLOAD_BACK_LOG} + sleep 5 + exit 0 + +} + +load_source_props=${11} +if [ "${load_source_props}" = true ]; then + echo "Load from source props was true" + source /mnt/backup/bin/.PROPS + downloadDBBackup $SERVER $PARM1 $PARM2 $CONTAINER $5 $6 $7 $8 $9 ${10} +else + echo "Load from source props was False" + downloadDBBackup $1 $2 $3 $4 $5 $6 $7 $8 $9 ${10} +fi \ No newline at end of file diff --git a/ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh b/ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh new file mode 100644 index 0000000000..6132d8dfb5 --- /dev/null +++ b/ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Finding the Instance owner +INSTOWNER=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | awk -F ',' '{print $4}' ` + +# Finding Instnace owner Group +GRPID=`cat /etc/passwd | grep ${INSTOWNER} | cut -d: -f4` +INSTGROUP=`cat /etc/group | grep ${GRPID} | cut -d: -f1` + +# Find the home directory +INSTHOME=` cat /etc/passwd | grep ${INSTOWNER} | cut -d: -f6` + +# Resolving INSTOWNER's executables path (sqllib): +DBPATH=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${INSTOWNER}" | awk -F ',' '{print $5}' ` + +# Source the db2profile for the root user to be able to issue several db2 commands locally: +SOURCEPATH="$DBPATH/db2profile" +. $SOURCEPATH + +cd /tmp/db2-scripts/ + +echo -e "\nCopying the files to bin directory under Instance Home . . . " +cp -rp db2_backup.sh ${INSTHOME}/bin/ + +if [ -f setup_cos_storage_access.sh ]; then + echo -e "\nCopying setup_cos_storage_access.sh to bin directory under Instance Home . . . " + cp -rp setup_cos_storage_access.sh ${INSTHOME}/bin/ +fi + +chown -R ${INSTOWNER}:${INSTGROUP} ${INSTHOME}/bin + +echo -e "\nINSTHOME=${INSTHOME}\n" +echo -e "PrepareSuccess: Backup scripts have been copied to Instance Home bin directory." diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml new file mode 100644 index 0000000000..4715e19d8d --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml @@ -0,0 +1,200 @@ +--- +# Check if backup vendor is s3, then check s3 related variables +# ----------------------------------------------------------------------------- +- name: "Verify DB2 S3 backup variables" + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "s3_setup" + backup_vendor: "{{ backup_vendor }}" + backup_s3_name: "{{ backup_s3_name }}" + backup_s3_endpoint: "{{ backup_s3_endpoint }}" + backup_s3_bucket: "{{ backup_s3_bucket }}" + backup_s3_access_key: "{{ backup_s3_access_key }}" + backup_s3_secret_key: "{{ backup_s3_secret_key }}" + when: backup_vendor == "s3" + +- name: Set fact db2_backup_data_path + set_fact: + db2_backup_data_path: "{{ db2_backup_path }}/data" + +- name: "Create {{ db2_backup_path }}/data directory" + file: + path: "{{ db2_backup_data_path }}" + state: directory + mode: "0755" + +- name: "Set fact backup_path for the backup script when backup vendor is s3" + set_fact: + full_backup_path: "DB2REMOTE://{{ backup_s3_name }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }}" + when: backup_vendor == "s3" + +- name: "Set fact backup_path for the backup script when backup vendor is disk" + set_fact: + full_backup_path: "/mnt/backup/{{ db2_backup_version }}" + when: backup_vendor == "disk" + +# Check if db2 instance is running and get the pod name +# ----------------------------------------------------------------------------- +- name: "Check DB2 is running and get DB2 pod name" + ibm.mas_devops.get_db2u_pod_name: + db2_instance_name: "{{ db2_instance_name }}" + db2_namespace: "{{ db2_namespace }}" + register: db2_pod_name_result + +- name: Assert DB2 pod name is found + assert: + that: + - db2_pod_name_result is defined + - db2_pod_name_result.success + - db2_pod_name_result.pod_name != "" + fail_msg: "DB2 pod name could not be found. Ensure the DB2 instance is running." + +- name: "Set fact db2_pod_name" + set_fact: + db2_pod_name: "{{ db2_pod_name_result.pod_name }}" + +- name: "Create /tmp/db2-scripts directory on localhost" + ansible.builtin.file: + path: "/tmp/db2-scripts" + state: directory + mode: "0755" + +- name: "Copy prepare_backup_scripts.sh to /tmp" + ansible.builtin.copy: + src: "{{ role_path }}/files/backup/prepare_backup_scripts.sh" + dest: "/tmp/db2-scripts/prepare_backup_scripts.sh" + mode: "0755" + +- name: "Template the DB2 S3 storage access setup script" + ansible.builtin.template: + src: "backup/setup_cos_storage_access.sh.j2" + dest: "/tmp/db2-scripts/setup_cos_storage_access.sh" + mode: "0755" + when: backup_vendor == "s3" + +- name: "Template the DB2 backup script" + ansible.builtin.template: + src: "backup/db2_backup.sh.j2" + dest: "/tmp/db2-scripts/db2_backup.sh" + mode: "0755" + +- name: Zip the DB2 backup scripts + ansible.builtin.archive: + path: /tmp/db2-scripts + dest: /tmp/db2-scripts.zip + format: zip + mode: "0755" + +# Create /tmp/db2-scripts directory in DB2 pod and copy scripts into the pod +# ----------------------------------------------------------------------------- +- name: create /tmp/db2-scripts directory in DB2 pod + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'mkdir -p /tmp/db2-scripts/' db2inst1" + register: create_dir_result + retries: 2 + delay: 15 # seconds + until: create_dir_result.rc == 0 + +- name: Copy /tmp/db2-scripts.zip to DB2 pod /tmp/db2-scripts.zip + shell: "oc cp /tmp/db2-scripts.zip {{ db2_namespace }}/{{ db2_pod_name }}:/tmp/db2-scripts.zip -c db2u" + register: copy_scripts_result + retries: 2 + delay: 15 # seconds + until: copy_scripts_result.rc == 0 + +- name: Unzip DB2 backup scripts in DB2 pod + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'cd /tmp; unzip -o db2-scripts.zip; chmod -R +x /tmp/db2-scripts' db2inst1" + register: unzip_result + retries: 2 + delay: 15 # seconds + until: unzip_result.rc == 0 + +# Execute prepare_backup_scripts.sh in DB2 pod +# this script will unzip the backup scripts, and sets the right permissions to the scripts. +# ----------------------------------------------------------------------------- +- name: "Execute prepare_backup_scripts.sh in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '/tmp/db2-scripts/prepare_backup_scripts.sh | tee /tmp/prepare_backup_scripts.log' db2inst1" + register: prepare_scripts_result + retries: 2 + delay: 15 # seconds + until: + - prepare_scripts_result.rc == 0 + - prepare_scripts_result.stdout | regex_search('PrepareSuccess', multiline=True) + +- name: "Debug prepare_backup_scripts.sh output" + ansible.builtin.debug: + msg: "{{ prepare_scripts_result.stdout_lines }}" + +- name: Get INSTHOME from prepare_backup_scripts.log + ansible.builtin.set_fact: + db2_insthome: "{{ (prepare_scripts_result.stdout | regex_search('INSTHOME=(.*)', '\\1'))[0] }}" + when: + - prepare_scripts_result is defined + - prepare_scripts_result.rc == 0 + - prepare_scripts_result.stdout is defined + +# Execute setup_cos_storage_access.sh in DB2 pod if backup vendor is s3 +# ----------------------------------------------------------------------------- +- name: "Execute setup_cos_storage_access.sh in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/setup_cos_storage_access.sh | tee /tmp/setup_cos_storage_access.log' db2inst1" + register: setup_s3_result + retries: 2 + delay: 15 # seconds + until: setup_s3_result.rc == 0 + when: backup_vendor == "s3" + +- name: "Debug setup_cos_storage_access.sh output" + ansible.builtin.debug: + msg: "{{ setup_s3_result.stdout_lines }}" + when: backup_vendor == "s3" + +# Execute db2_backup.sh in DB2 pod +# ----------------------------------------------------------------------------- +- name: "Execute db2_backup.sh in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/db2_backup.sh {{ db2_backup_path }} | tee /tmp/db2_backup.log' db2inst1" + register: db2_backup_result + retries: 2 + delay: 15 # seconds + until: + - db2_backup_result.rc == 0 + - db2_backup_result.stdout | regex_search('BACKUP COMPLETED SUCCESSFULLY', multiline=True) + +- name: Get backup file timestamp from db2_backup.log + ansible.builtin.set_fact: + db2_backup_timestamp: "{{ (db2_backup_result.stdout | regex_search('BACKUP_FILE_TIMESTAMP=(.*)', '\\1'))[0] }}" + when: + - db2_backup_result is defined + - db2_backup_result.rc == 0 + - db2_backup_result.stdout is defined + +# If vendor is disk, tar the backup files and rsync to backup_data_path +# ----------------------------------------------------------------------------- +- name: "Move backup files to {{ db2_backup_data_path }} when backup vendor is disk" + shell: "oc rsync -n {{ db2_namespace }} {{ db2_pod_name }}:{{ full_backup_path }}/db2_{{ db2_dbname }}_backup_{{ db2_backup_version }}.tar.gz {{ db2_backup_data_path }}/" + register: rsync_result + retries: 2 + delay: 15 # seconds + until: rsync_result.rc == 0 + when: backup_vendor == "disk" + +- name: "Debug rsync output" + ansible.builtin.debug: + msg: "{{ rsync_result.stdout_lines }}" + when: backup_vendor == "disk" + + +# Create yaml file with db2 backup details, add local_backup_path if vendor is disk +# ----------------------------------------------------------------------------- +- name: "Create yaml file with db2 backup details" + copy: + dest: "{{ db2_backup_data_path }}/db2-backup-info.yml" + content: | + source_db2_backup_version: "{{ db2_backup_version }}" + source_db2_backup_timestamp: "{{ db2_backup_timestamp }}" + source_db2_instance_name: "{{ db2_instance_name }}" + source_db2_instance_version: "{{ db2_pod_name_result.db2version }}" + backup_vendor: "{{ backup_vendor }}" + vendor_backup_path: "{{ full_backup_path }}" + {% if backup_vendor == 'disk' %} + local_backup_path: "{{ db2_backup_data_path }}/db2_{{ db2_dbname }}_backup_{{ db2_backup_version }}.tar.gz" + {% endif %} + diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml new file mode 100644 index 0000000000..191896e83c --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml @@ -0,0 +1,16 @@ +--- +- name: "Start Db2 Instance backup process." + ibm.mas_devops.backup_db2_instance: + db2_backup_path: "{{ db2_backup_path }}" + db2_namespace: "{{ db2_namespace }}" + db2_instance_name: "{{ db2_instance_name }}" + mas_instance_id: "{{ mas_instance_id }}" + db2_dbname: "{{ db2_dbname }}" + register: db2_instance_backup_result + +- name: "Assert Db2 Instance backup success" + assert: + that: + - db2_instance_backup_result is defined + - db2_instance_backup_result.success == true + fail_msg: "Db2 Instance resources backup failed." diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 8428c99c0b..9550762ded 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -1,8 +1,39 @@ --- # Check db2 backup required variables # ----------------------------------------------------------------------------- -- name: "Fail if db2_instance_name is not provided" - assert: - that: db2_instance_name is defined and db2_instance_name != "" - fail_msg: "db2_instance_name is required" +- name: Verify DB2 backup variables + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "{{ db2_action }}" + db2_instance_name: "{{ db2_instance_name }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mas_instance_id: "{{ mas_instance_id }}" + mas_app_id: "{{ mas_application_id }}" +# Set DB2 backup version if not provided +# ----------------------------------------------------------------------------- +- name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + db2_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: db2_backup_version is not defined or db2_backup_version == "" or db2_backup_version == "None" + +- name: "Set fact: DB2 backup base directory path" + set_fact: + db2_backup_path: "{{ mas_backup_dir }}/{{ db2_action }}-{{ db2_backup_version }}-db2u" + +- name: "Create {{ db2_backup_path }} directory for Db2 backup" + file: + path: "{{ db2_backup_path }}" + state: directory + mode: "0755" + +# Backup Db2 Universal operator Instance Kubernetes Resources +# ------------------------------------------------------------------------- +- name: "Start Db2 Universal operator Instance backup process." + include_tasks: "{{ role_path }}/tasks/backup/backup-instance.yml" + when: not br_skip_instance | bool + +# Backup Db2 database Data using Db2Backup CR +# ------------------------------------------------------------------------- +- name: "Start Database backup process." + include_tasks: "{{ role_path }}/tasks/backup/backup-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/install/main.yml b/ibm/mas_devops/roles/db2/tasks/install/main.yml index 37ba9a99b0..eb04619a7d 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/main.yml @@ -128,44 +128,57 @@ template: "templates/db2u_namespace.yaml" register: _db2_namespace_result -- name: Set 'ibm-registry' secret content - no_log: true - vars: - entitledAuthStr: "{{ registry_user }}:{{ ibm_entitlement_key }}" - entitledAuth: "{{ entitledAuthStr | b64encode }}" - content: - - '{"auths":{"{{ registry }}/cp/cpd":{"username":"{{ registry_user }}","password":"{{ ibm_entitlement_key }}","email":"{{ registry_user }}","auth":"{{ entitledAuth }}"}' - - "}" - - "}" - set_fact: - new_secret: "{{ content | join('') }}" +- name: Check if ibm-registry secret exists + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: ibm-registry + namespace: "{{ db2_namespace }}" + register: ibm_registry_secret_info -# Note: We use "new_secret | to_json " because older versions of Ansible create -# invalid json representations containing single quotes -# However, in newer versions of Ansible to_json returns an object, which can't be passed to b64encode -- name: "Generate docker secret (old Ansible versions)" - when: ansible_version.full is version_compare(2.20, '<') - set_fact: - dockerconfigjsonB64: "{{ new_secret | to_json | b64encode }}" +- name: "Create ibm-registry secret if not present" + when: + - ibm-registry_secret_info.resources is defined + - ibm_registry_secret_info.resources | length == 0 + block: + - name: Set 'ibm-registry' secret content + no_log: true + vars: + entitledAuthStr: "{{ registry_user }}:{{ ibm_entitlement_key }}" + entitledAuth: "{{ entitledAuthStr | b64encode }}" + content: + - '{"auths":{"{{ registry }}/cp/cpd":{"username":"{{ registry_user }}","password":"{{ ibm_entitlement_key }}","email":"{{ registry_user }}","auth":"{{ entitledAuth }}"}' + - "}" + - "}" + set_fact: + new_secret: "{{ content | join('') }}" -- name: "Generate docker secret (new Ansible versions)" - when: ansible_version.full is version_compare(2.20, '>=') - set_fact: - dockerconfigjsonB64: "{{ new_secret | b64encode }}" + # Note: We use "new_secret | to_json " because older versions of Ansible create + # invalid json representations containing single quotes + # However, in newer versions of Ansible to_json returns an object, which can't be passed to b64encode + - name: "Generate docker secret (old Ansible versions)" + when: ansible_version.full is version_compare(2.20, '<') + set_fact: + dockerconfigjsonB64: "{{ new_secret | to_json | b64encode }}" -- name: "Generate 'ibm-registry' secret" - no_log: true - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson - metadata: - name: ibm-registry - namespace: "{{ db2_namespace }}" - data: - .dockerconfigjson: "{{ dockerconfigjsonB64 }}" - register: secretUpdateResult + - name: "Generate docker secret (new Ansible versions)" + when: ansible_version.full is version_compare(2.20, '>=') + set_fact: + dockerconfigjsonB64: "{{ new_secret | b64encode }}" + + - name: "Generate 'ibm-registry' secret" + no_log: true + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: kubernetes.io/dockerconfigjson + metadata: + name: ibm-registry + namespace: "{{ db2_namespace }}" + data: + .dockerconfigjson: "{{ dockerconfigjsonB64 }}" + register: secretUpdateResult - name: "Delete old db2 subscription, operand request and csv from {{ ibm_common_services_namespace }}" include_tasks: "tasks/delete_db2_operand_request.yml" @@ -227,7 +240,7 @@ # 10. Get information from the db2u-release ConfigMap # ----------------------------------------------------------------------------- # if db2_version is not set, then we define it based on the latest version supported by the db2u-license-keys secret -# Starting with s11.5.8.0-cn3, the 's' prefix is removed in db2u-license-keys, we are recommeded to use db2u-release configmap. +# Starting with s11.5.8.0-cn3, the 's' prefix is removed in db2u-license-keys, it is recommended to use db2u-release configmap. - block: - name: "Wait until the db2u-release configmap is available" no_log: true diff --git a/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 new file mode 100644 index 0000000000..48f9d31ed8 --- /dev/null +++ b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 @@ -0,0 +1,115 @@ +#!/bin/bash + +DATABASE={{ db2_dbname }} +BACKUP_TYPE={{ backup_type }} +VENDOR={{ backup_vendor }} +### for S3, BACKUP_PATH=DB2REMOTE://{{ backup_s3_name }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }} +BACKUP_PATH={{ full_backup_path }} + +# Finding the db2 Instance owner +INSTOWNER=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | awk -F ',' '{print $4}' ` + +instance=`whoami` +BACKUP_BASE=/mnt/backup + +# Find the home directory +instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` +BACK_LOG=$instance_home/bin/{{ db2_backup_version }}_BackupLOG.out + +if [ ! -f "$instance_home/sqllib/db2profile" ] +then + echo "ERROR - $instance_home/sqllib/db2profile not found" + EXIT_STATUS=1 +else + . $instance_home/sqllib/db2profile +fi + +### Check for the existance of /home/ctginst1/sqllib/db2dump/libdb2compr.so...if it exists, delete it +COMPRESS_LOC=$instance_home/sqllib/db2dump/libdb2compr.so +if [[ -f ${COMPRESS_LOC} ]] +then + rm ${COMPRESS_LOC} +fi + +### Check to see if the database is Running +ps -ef | grep db2sys | grep -v grep > /dev/null 2>&1 +if [ $? -eq 1 ]; then + echo "Database is not active " + echo "Database Not Active,BACKUP Not Run" > $instance_home/bin/LASTbkupRUN + exit +fi + +### Create Backup Directory if it does not exist when vendor is disk +if [ ${VENDOR} = 'disk' ] ; then + if [ ! -d "${BACKUP_PATH}" ] ; then + mkdir -p ${BACKUP_PATH} + if [ $? -ne 0 ] ; then + echo "ERROR: Could not create backup directory ${BACKUP_PATH}" + exit 1 + fi + fi +fi + +DATETIME=`date +%Y-%m-%d_%H%M%S`; +echo "BACKUP Start time : ${DATETIME}" +echo " " +echo " " + +db2 -v archive log for db $DATABASE | tee -a $BACK_LOG +sleep 30 + + +if [ ${BACKUP_TYPE} = 'full' ] ; then + db2 -v backup db $DATABASE online to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 include logs without prompting | tee -a $BACK_LOG +else + db2 -v backup db $DATABASE online INCREMENTAL DELTA to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 include logs without prompting | tee -a $BACK_LOG +fi +grep -Fq "Backup successful." $BACK_LOG +if [ $? = 0 ]; then + backup_timestamp=`grep timestamp $BACK_LOG | cut -d: -f2` +else + echo " " + echo "********** BACKUP FAILED ************" + echo " " + exit 1 +fi + +######## Copy keystore to COS or Disk location based on VENDOR ######### +if [ ${VENDOR} = 's3' ] ; then + echo "Copying Keystore to COS location" + set -x + SOURCE1=/mnt/blumeta0/db2/keystore/keystore.p12 + SOURCE2=/mnt/blumeta0/db2/keystore/keystore.sth + TARGET1=KEYSTORE/keystore.p12 + TARGET2=KEYSTORE/keystore.sth + + db2RemStgManager ALIAS PUT source=${SOURCE1} target=${BACKUP_PATH}/${TARGET1} + db2RemStgManager ALIAS PUT source=${SOURCE2} target=${BACKUP_PATH}/${TARGET2} + echo "Copying Keystore to COS location - Completed" +else + echo "Copying Keystore to local disk location" + cp /mnt/blumeta0/db2/keystore/keystore.p12 ${BACKUP_PATH}/keystore.p12 + cp /mnt/blumeta0/db2/keystore/keystore.sth ${BACKUP_PATH}/keystore.sth + echo "Copying Keystore to local disk location - Completed" +fi + +DATETIME=`date +%Y-%m-%d_%H%M%S`; +echo "BACKUP End time : ${DATETIME}" >> $BACK_LOG +echo "BACKUP End time : ${DATETIME}" +echo " " + + +###### If vendor is disk, tar the backup files +if [ ${VENDOR} = 'disk' ] ; then + echo "Starting to tar the backup files..." + tar -czf ${BACKUP_PATH}/db2_${DATABASE}_backup_{{ db2_backup_version }}.tar.gz -C ${BACKUP_PATH} . + echo "Tar backup files completed. Showing disk usage:" + df -h ${BACKUP_PATH}/* +else + echo "Backup vendor is not disk. Skipping tar of backup files." +fi + + +echo "BACKUP_FILE_TIMESTAMP=${backup_timestamp}" +echo " " +echo "********** BACKUP COMPLETED SUCCESSFULLY ************" \ No newline at end of file diff --git a/ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 new file mode 100644 index 0000000000..10200eb03e --- /dev/null +++ b/ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 @@ -0,0 +1,8 @@ +#!/bin/bash + +if db2 list storage access | grep {{ backup_s3_name }}; then + echo "{{ backup_s3_name }} is available already." +else + echo "{{ backup_s3_name }} is not available. Creating" + db2 catalog storage access alias {{ backup_s3_name }} VENDOR S3 server {{ backup_s3_endpoint }} user {{ backup_s3_access_key }} password {{ backup_s3_secret_key }} container {{ backup_s3_bucket }} +fi \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index f6753ea99a..75338a7f18 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -6,6 +6,7 @@ mongodb_provider: "{{ lookup('env','MONGODB_PROVIDER') | default('community', Tr mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('install', True) }}" +mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" # Backup mongodb databases for specified MAS application. Backup all mongodb databases if not specify this value. mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" @@ -121,9 +122,7 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # ----------------------------------------------------------------------------- mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" -mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" -mongodb_data_backup: "{{ lookup('env', 'MONGODB_DATA_BACKUP') | default(true, true) }}" # Mongo upgrade flags # If identified that there's an existing Mongo that might lead to a v5 or v6 upgrade diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index 65acdd827e..42f250a80e 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -105,3 +105,12 @@ - database_backup_output is defined - database_backup_output.rc == 0 - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) + +- name: "Create yaml file with mongodb backup details" + copy: + dest: "{{ mongodb_backup_data_path }}/mongodb-info.yml" + content: | + source_mongodb_backup_version: "{{ mongodb_backup_version }}" + source_mongodb_backup_data_filename: "{{ mongodb_backup_data_filename }}" + source_mongodb_version: "{{ mongodb_version }}" + diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 781e663236..77c6095823 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -5,16 +5,25 @@ mongodb_backup_data_filename: "mongodump-{{ mongodb_backup_version }}.tar.gz" mongodb_tmp_restore_dir: "/tmp/masbr/{{ mongodb_backup_version }}" -- name: "Check if backup data file exists" +- name: "Check if backup data files exists" stat: - path: "{{ mongodb_data_path }}/{{ mongodb_backup_data_filename }}" - register: backup_data_file_check + path: "{{ item }}" + register: backup_data_files + loop: + - "{{ mongodb_data_path }}/{{ mongodb_backup_data_filename }}" + - "{{ mongodb_data_path }}/mongodb-info.yaml" -- name: Assert backup data file exists +- name: "Assert backup data files exist" assert: that: - - backup_data_file_check.stat.exists | default(False) - fail_msg: "Backup data file {{ mongodb_data_path }}/{{ mongodb_backup_data_filename }} does NOT exist!" + - backup_data_files.results[0].stat.exists + - backup_data_files.results[1].stat.exists + fail_msg: "Required backup data files are missing in {{ mongodb_data_path }}. Ensure that the backup process completed successfully." + +- name: include vars from mongodb-info.yaml + include_vars: + file: "{{ mongodb_data_path }}/mongodb-info.yaml" + name: mongodb_backup_info - name: "Set facts: from mongodb cr and resources" ibm.mas_devops.get_mongoce_info: @@ -36,6 +45,12 @@ mongodb_admin_user: "{{ mongodb_info_result.mongodb_admin_user }}" mongodb_admin_password: "{{ mongodb_info_result.mongodb_admin_password }}" +- name: "Assert if target mongoce version match backup mongoce version" + assert: + that: + - mongodb_version == mongodb_backup_version + fail_msg: "Target MongoCE version {{ mongodb_version }} does NOT match backup MongoCE version {{ mongodb_backup_info.source_mongodb_version }}. Restore cannot proceed." + - name: "debug facts" debug: msg: diff --git a/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret-dev.yml b/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret-dev.yml index de9277bbc1..4bdd8254ab 100644 --- a/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret-dev.yml +++ b/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret-dev.yml @@ -2,57 +2,11 @@ # 1. Update default cluster image pull secret # ============================================================================= -# 1.1 Generate the new secret content -- name: "update-pull-secret-dev : Set new secret content" - vars: - artifactoryAuthB64: "{{ artifactory_auth | b64encode }}" - content: - - "{\"auths\":{\"{{ fvt_image_registry }}\":{\"username\":\"{{ artifactory_username }}\",\"password\":\"{{ artifactory_token }}\",\"email\":\"{{ artifactory_username }}\",\"auth\":\"{{ artifactoryAuthB64 }}\"}" - - "}" - - "}" - set_fact: - new_secret_dev: "{{ content | join('') }}" - no_log: true - -# 1.2 Find the existing secret, and we are going to modify it rather than replace -- name: "update-pull-secret-dev : Retrieve existing pull-secret content" - kubernetes.core.k8s_info: - api: v1 - kind: Secret - name: pull-secret - namespace: openshift-config - register: pullsecret - no_log: true - -- name: "update-pull-secret-dev : Get the original cred secrets" - set_fact: - original_secret: "{{ item.data }}" - with_items: "{{ pullsecret.resources }}" - no_log: true - -- name: "update-pull-secret-dev : Get the dockerconfigjson info" - set_fact: - secret_string: '{{ original_secret[".dockerconfigjson"] | b64decode | from_json }}' - no_log: true - -# 1.3 Append our new credentials to the secret -- name: "update-pull-secret-dev : Combine new secret content" - set_fact: - new_secret_string: '{{ secret_string | combine( new_secret_dev, recursive=True) }}' - no_log: true - -# 1.4. Overwrite the secret -- name: "update-pull-secret-dev : Update new pull-secret" - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson - metadata: - name: pull-secret - namespace: openshift-config - data: - .dockerconfigjson: "{{ new_secret_string | to_json | b64encode }}" +- name: "update-pull-secret-dev : Update global pull secret" + ibm.mas_devops.update_global_pull_secret: + registry_url: "{{ fvt_image_registry }}" + username: "{{ artifactory_username }}" + password: "{{ artifactory_token }}" register: secretUpdateResult no_log: true diff --git a/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret.yml b/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret.yml index d40a158399..c74a3a42fc 100644 --- a/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret.yml +++ b/ibm/mas_devops/roles/ocp_idms/tasks/update-pull-secret.yml @@ -2,57 +2,11 @@ # 1. Update default cluster image pull secret # ============================================================================= -# 1.1 Generate the new secret content -- name: "update-pull-secret : Set new secret content" - vars: - registryAuthB64: "{{ registry_auth | b64encode }}" - content: - - "{\"auths\":{\"{{ registry_private_url }}\":{\"username\":\"{{ registry_username }}\",\"password\":\"{{ registry_password }}\",\"email\":\"{{ registry_username }}\",\"auth\":\"{{ registryAuthB64 }}\"}" - - "}" - - "}" - set_fact: - new_secret: "{{ content | join('') }}" - no_log: true - -# 1.2 Find the existing secret, and we are going to modify it rather than replace -- name: "update-pull-secret : Retrieve existing pull-secret content" - kubernetes.core.k8s_info: - api: v1 - kind: Secret - name: pull-secret - namespace: openshift-config - register: pullsecret - no_log: true - -- name: "update-pull-secret : Get the original cred secrets" - set_fact: - original_secret: "{{ item.data }}" - with_items: "{{ pullsecret.resources }}" - no_log: true - -- name: "update-pull-secret : Get the dockerconfigjson info" - set_fact: - secret_string: '{{ original_secret[".dockerconfigjson"] | b64decode | from_json }}' - no_log: true - -# 1.3 Append our new credentials to the secret -- name: "update-pull-secret : Combine new secret content" - set_fact: - new_secret_string: '{{ secret_string | combine( new_secret, recursive=True) }}' - no_log: true - -# 1.4. Overwrite the secret -- name: "update-pull-secret : Update new pull-secret" - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson - metadata: - name: pull-secret - namespace: openshift-config - data: - .dockerconfigjson: "{{ new_secret_string | to_json | b64encode }}" +- name: "update-pull-secret : Update global pull secret" + ibm.mas_devops.update_global_pull_secret: + registry_url: "{{ registry_private_url }}" + username: "{{ registry_username }}" + password: "{{ registry_password }}" register: secretUpdateResult no_log: true diff --git a/ibm/mas_devops/roles/suite_install/defaults/main.yml b/ibm/mas_devops/roles/suite_install/defaults/main.yml index b4040c5790..4989c52fba 100644 --- a/ibm/mas_devops/roles/suite_install/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_install/defaults/main.yml @@ -45,7 +45,7 @@ mas_trust_default_cas: "{{ lookup('env', 'MAS_TRUST_DEFAULT_CAS') }}" # Network Routing # ----------------------------------------------------------------------------- -mas_routing_mode: "{{ lookup('env', 'MAS_ROUTING_MODE') | default('path', true) }}" +mas_routing_mode: "{{ lookup('env', 'MAS_ROUTING_MODE') | default('subdomain', true) }}" # Source container registry # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/suite_verify/defaults/main.yml b/ibm/mas_devops/roles/suite_verify/defaults/main.yml index e75f44d0f1..a0b5318bce 100644 --- a/ibm/mas_devops/roles/suite_verify/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_verify/defaults/main.yml @@ -1,3 +1,3 @@ --- mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" -mas_hide_superuser_credentials: "{{ lookup('env', 'MAS_HIDE_SUPERUSER_CREDENTIALS') | default('True', True) }}" +mas_hide_superuser_credentials: "{{ (lookup('env', 'MAS_HIDE_SUPERUSER_CREDENTIALS') | default('True', True)) | bool }}" From aa4b7edc695eb6896b3d319b93b6f310e3886541 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 22 Dec 2025 19:18:24 +0000 Subject: [PATCH 04/61] [patch] added debug of config summary --- .../roles/db2/tasks/backup/backup-database.yml | 15 +++++++++++++++ .../roles/db2/tasks/backup/backup-instance.yml | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml index 4715e19d8d..fed39a20cb 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml @@ -1,4 +1,19 @@ --- +- name: Debug – configuration summary + ansible.builtin.debug: + msg: | + ================== BACKUP DB2 DATABASE CONFIGURATION SUMMARY ================== + MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }} + DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }} + NAMESPACE : {{ db2_namespace | default('UNDEFINED') }} + DB NAME : {{ db2_dbname | default('UNDEFINED') }} + BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }} + VENDOR : {{ backup_vendor | default('UNDEFINED') }} + S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }} + S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }} + BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }} + =========================================================== + # Check if backup vendor is s3, then check s3 related variables # ----------------------------------------------------------------------------- - name: "Verify DB2 S3 backup variables" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml index 191896e83c..6a690c347c 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml @@ -1,4 +1,16 @@ --- +- name: Debug – configuration summary + ansible.builtin.debug: + msg: | + ================== BACKUP DB2 INSTANCE CONFIGURATION SUMMARY ================== + MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }} + DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }} + NAMESPACE : {{ db2_namespace | default('UNDEFINED') }} + DB NAME : {{ db2_dbname | default('UNDEFINED') }} + BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }} + BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }} + =========================================================== + - name: "Start Db2 Instance backup process." ibm.mas_devops.backup_db2_instance: db2_backup_path: "{{ db2_backup_path }}" From 958980b62bf65159bc00ed959a13f8a7e1bdfc04 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 10:13:17 +0000 Subject: [PATCH 05/61] [minor] db2 backup and restore (#2051) Co-authored-by: Sanjay Prabhakar Co-authored-by: mnivedithaa --- ibm/mas_devops/playbooks/br_db2.yml | 49 ++ .../plugins/action/backup_db2_instance.py | 26 +- .../plugins/action/backup_mongo_instance.py | 17 +- .../plugins/action/get_db2u_pod_name.py | 2 +- .../plugins/action/restore_db2_resources.py | 203 ++++++++ .../action/verify_backup_restore_vars.py | 7 +- .../plugins/module_utils/backuprestore.py | 6 +- ibm/mas_devops/roles/db2/README.md | 311 ++++++++++++ ibm/mas_devops/roles/db2/defaults/main.yml | 9 +- .../db2/files/backup/download_bkp_images.sh | 95 ---- .../files/backup/prepare_backup_scripts.sh | 18 +- .../db2/tasks/backup/backup-database.yml | 145 +++--- .../db2/tasks/backup/backup-instance.yml | 33 +- .../roles/db2/tasks/backup/main.yml | 2 +- .../roles/db2/tasks/install/main.yml | 9 +- .../roles/db2/tasks/restore/main.yml | 13 +- .../db2/tasks/restore/restore-database.yml | 40 ++ .../tasks/restore/restore-db-from-disk.yml | 199 ++++++++ .../db2/tasks/restore/restore-db-from-s3.yml | 142 ++++++ .../db2/tasks/restore/restore-instance.yml | 87 ++++ .../db2/templates/backup/db2_backup.sh.j2 | 321 +++++++++++-- .../templates/backup/db2_restore_disk.sh.j2 | 332 +++++++++++++ .../db2/templates/backup/db2_restore_s3.sh.j2 | 453 ++++++++++++++++++ .../templates/backup/download_from_cos.sh.j2 | 21 + .../backup/setup_cos_storage_access.sh.j2 | 8 +- .../mongodb/tasks/providers/aws/uninstall.yml | 2 +- 26 files changed, 2295 insertions(+), 255 deletions(-) create mode 100644 ibm/mas_devops/playbooks/br_db2.yml create mode 100644 ibm/mas_devops/plugins/action/restore_db2_resources.py delete mode 100755 ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-disk.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-s3.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml create mode 100644 ibm/mas_devops/roles/db2/templates/backup/db2_restore_disk.sh.j2 create mode 100644 ibm/mas_devops/roles/db2/templates/backup/db2_restore_s3.sh.j2 create mode 100644 ibm/mas_devops/roles/db2/templates/backup/download_from_cos.sh.j2 diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml new file mode 100644 index 0000000000..500f71f075 --- /dev/null +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -0,0 +1,49 @@ +--- +- hosts: localhost + any_errors_fatal: true + vars: + # Define the target for backup/restore + mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups + db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" + db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" + db2_action: "{{ lookup('env', 'MASBR_ACTION') }}" # backup or restore + db2_backup_version: "{{ lookup('env', 'MASBR_BACKUP_VERSION') | default('None', true)}}" # Required for restore action + br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false to enable DB2 instance backup/restore + backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline and required for backup + backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) }}" + backup_s3_alias: "{{ lookup('env', 'BACKUP_S3_ALIAS') | default('S3DB2COS', true) }}" # Required for backup_vendor=s3 + backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" # Required for backup_vendor=s3 + backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" # Required for backup_vendor=s3 + backup_s3_access_key: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY') }}" # Required for backup_vendor=s3 + backup_s3_secret_key: "{{ lookup('env', 'BACKUP_S3_SECRET_KEY') }}" # Required for backup_vendor=s3 + + + pre_tasks: + - name: "Fail if MASBR_ACTION is not set to backup|restore" + assert: + that: db2_action in ["backup", "restore"] + fail_msg: "MASBR_ACTION is required and must be set to 'backup' or 'restore'" + + - name: "Fail if BACKUP_VENDOR is not set to s3|disk" + assert: + that: backup_vendor in ["s3", "disk"] + fail_msg: "BACKUP_VENDOR is required and must be set to 's3' or 'disk'" + + - name: "Fail if DB2_BACKUP_TYPE is not set to online|offline" + assert: + that: backup_type in ["online", "offline"] + fail_msg: "DB2_BACKUP_TYPE is required and must be set to 'online' or 'offline'" + + - name: "Fail if S3 variables are not set when backup_vendor is s3" + assert: + that: + - backup_s3_endpoint is defined and backup_s3_endpoint != "" + - backup_s3_bucket is defined and backup_s3_bucket != "" + - backup_s3_access_key is defined and backup_s3_access_key != "" + - backup_s3_secret_key is defined and backup_s3_secret_key != "" + fail_msg: "BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY and BACKUP_S3_SECRET_KEY are required when BACKUP_VENDOR is s3" + when: backup_vendor == "s3" + + roles: + - role: ibm.mas_devops.db2 diff --git a/ibm/mas_devops/plugins/action/backup_db2_instance.py b/ibm/mas_devops/plugins/action/backup_db2_instance.py index 7517819054..fc4208a0cd 100644 --- a/ibm/mas_devops/plugins/action/backup_db2_instance.py +++ b/ibm/mas_devops/plugins/action/backup_db2_instance.py @@ -80,7 +80,7 @@ def processUserCredentialsSecret(dynClient: DynamicClient, mas_instance_id: str, """ Process user credentials secret for Db2u instance Check user credentials from jdbc credentials secret in Core namespace - If the user is not the default db2inst1 user, backup the jdbc credentials secret as well + If the user is not the default db2inst1 user, Store the username and password this will be used during restore to set the correct user """ jdbc_core_secret_name = f"jdbc-{db2_instance_name}-credentials".lower() @@ -90,14 +90,15 @@ def processUserCredentialsSecret(dynClient: DynamicClient, mas_instance_id: str, username_encoded = jdbc_core_secret['data']['username'] username = base64.b64decode(username_encoded).decode('utf-8') if username.lower() != 'db2inst1': - # Backup the jdbc credentials secret - backup_status = backupSecret(dynClient=dynClient, namespace=f"mas-{mas_instance_id}-core".lower(), secret_name=jdbc_core_secret_name, backup_path=backup_path) - if backup_status: - display.v(f"Backed up JDBC credentials secret '{jdbc_core_secret_name}' for Db2u instance '{db2_instance_name}'") - return True - else: - display.v(f"Warning: Failed to backup JDBC credentials secret '{jdbc_core_secret_name}' for Db2u instance '{db2_instance_name}'") - return False + # create a yaml file with LDAP user details + ldap_user_info = dict( + db2_ldap_username=username_encoded, + db2_ldap_password=jdbc_core_secret['data']['password'] + ) + ldap_info_file_path = os.path.join(backup_path, "ldapuser-NOT_SECRET.yaml") + with open(ldap_info_file_path, 'w') as ldap_info_file: + yaml.dump(ldap_user_info, ldap_info_file) + display.v(f"Wrote LDAP user info to {ldap_info_file_path}") else: display.v(f"JDBC credentials secret '{jdbc_core_secret_name}' uses default admin user. No additional user backup needed.") return False @@ -146,7 +147,7 @@ def run(self, tmp=None, task_vars=None): createBackupDirectories([db2_backup_path, db2_backup_resource_path, db2_backup_secrets_path]) # Get Db2uCluster or Db2uInstance CR - db2u_cr = getDb2uInstance(DynamicClient=dynClient, db2_instance_name=db2_instance_name, db2_namespace=db2_namespace) + db2u_cr = getDb2uInstance(dynClient=dynClient, db2_instance_name=db2_instance_name, db2_namespace=db2_namespace) if not db2u_cr: raise AnsibleError(f"Db2uCluster or Db2uInstance CR '{db2_instance_name}' not found in namespace '{db2_namespace}'") @@ -205,10 +206,11 @@ def run(self, tmp=None, task_vars=None): 'db2_dbname': str(db2_dbname), 'mas_instance_id': str(mas_instance_id), 'db2_version': getDb2VersionFromCR(db2uCR=db2u_cr), - 'db2_channel': channel_name + 'db2_channel': channel_name, + 'status': 'SUCCESS' } - db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-info.yaml") + db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-backup-info.yaml") with open(db2_info_file_path, 'w') as info_file: yaml.dump(db2_info, info_file) display.v(f"Wrote Db2 instance info to {db2_info_file_path}") diff --git a/ibm/mas_devops/plugins/action/backup_mongo_instance.py b/ibm/mas_devops/plugins/action/backup_mongo_instance.py index c3efbba98b..ac86139d67 100644 --- a/ibm/mas_devops/plugins/action/backup_mongo_instance.py +++ b/ibm/mas_devops/plugins/action/backup_mongo_instance.py @@ -25,13 +25,12 @@ def display_information(mongoDBCommunityCR : dict): display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") -def get_tlscertkey_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str: +def get_tlscertkey_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str | None: if 'security' in mongoDBCommunityCR['spec']: if 'tls' in mongoDBCommunityCR['spec']['security']: if 'certificateKeySecretRef' in mongoDBCommunityCR['spec']['security']['tls']: return mongoDBCommunityCR['spec']['security']['tls']['certificateKeySecretRef']['name'] - else: - return None + return None def get_usersecrets_from_mongoce(mongoDBCommunityCR : dict) -> list: user_secrets = [] @@ -42,12 +41,11 @@ def get_usersecrets_from_mongoce(mongoDBCommunityCR : dict) -> list: user_secrets.append(f"{user['scramCredentialsSecretName']}-scram-credentials") return user_secrets -def get_prometheus_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str: +def get_prometheus_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str | None: if 'prometheus' in mongoDBCommunityCR['spec']: if 'passwordSecretRef' in mongoDBCommunityCR['spec']['prometheus']: return mongoDBCommunityCR['spec']['prometheus']['passwordSecretRef']['name'] - else: - return None + return None def getMongoVersionFromCR(mongoCR: dict) -> str: """ @@ -325,7 +323,12 @@ def run(self, tmp=None, task_vars=None): prometheus_secret_names = get_prometheus_secretname_from_mongoce(mongodb_cr) tls_secret_names = get_tlscertkey_secretname_from_mongoce(mongodb_cr) - secret_names = user_secret_names + [prometheus_secret_names] + [tls_secret_names] + # Build secret_names list, filtering out None values + secret_names = user_secret_names.copy() + if prometheus_secret_names is not None: + secret_names.append(prometheus_secret_names) + if tls_secret_names is not None: + secret_names.append(tls_secret_names) display.v(f"All Secrets to backup: '{secret_names}'") diff --git a/ibm/mas_devops/plugins/action/get_db2u_pod_name.py b/ibm/mas_devops/plugins/action/get_db2u_pod_name.py index 9b7b71784e..2a8d7de444 100644 --- a/ibm/mas_devops/plugins/action/get_db2u_pod_name.py +++ b/ibm/mas_devops/plugins/action/get_db2u_pod_name.py @@ -60,7 +60,7 @@ def run(self, tmp=None, task_vars=None): failed=False, success=True, pod_name=pod_name, - db2version=getDb2VersionFromCR(db2u_cr), + db2_version=getDb2VersionFromCR(db2u_cr), msg="Db2u Pod found" ) else: diff --git a/ibm/mas_devops/plugins/action/restore_db2_resources.py b/ibm/mas_devops/plugins/action/restore_db2_resources.py new file mode 100644 index 0000000000..7dc0732c6f --- /dev/null +++ b/ibm/mas_devops/plugins/action/restore_db2_resources.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +import logging +import urllib3 + +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic.exceptions import NotFoundError + +from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import createBackupDirectories, backupSecret, getSubscription, getDb2uInstance, getDb2VersionFromCR, isDb2uReady + +import yaml +import os +import base64 + +from mas.devops.ocp import createNamespace, apply_resource + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +display = Display() + +def checkBackupDirectoryExists(db2_backup_path: str, db2_backup_version: str): + + if not os.path.exists(db2_backup_path): + raise AnsibleError(f"DB2 Backup path {db2_backup_path} does not exist.") + + db2_resources_path = os.path.join(db2_backup_path, "resources") + + if not os.path.exists(db2_resources_path): + raise AnsibleError(f"Db2 instance resources backup path {db2_resources_path} does not exist.") + + cr_path = os.path.join(db2_resources_path, "cr.yaml") + if not os.path.exists(cr_path): + raise AnsibleError(f"Db2 instance CR backup file {cr_path} does not exist.") + + db2_secrets_path = os.path.join(db2_resources_path, "secrets") + if not os.path.exists(db2_secrets_path): + raise AnsibleError(f"Db2 instance resources secrets backup path {db2_secrets_path} does not exist.") + + return True + + +class ActionModule(ActionBase): + + """ + Usage Example + ------------- + tasks: + - name: "Restore Db2u instance resources (CR, secrets, configmaps, issuers, certificates)" + ibm.mas_devops.restore_db2_instance: + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + mas_backup_dir = self._task.args.get('mas_backup_dir', None) + db2_backup_version = self._task.args.get('db2_backup_version', None) + + if not mas_backup_dir or mas_backup_dir == '': + raise AnsibleError("mas_backup_dir is a required parameter and cannot be empty") + if not db2_backup_version or db2_backup_version == '': + raise AnsibleError("db2_backup_version is a required parameter and cannot be empty") + + # Check if backup directory exists + db2_backup_path = os.path.join(mas_backup_dir, f"backup-{db2_backup_version}-db2u") + checkBackupDirectoryExists(db2_backup_path, db2_backup_version) + display.v(f"Db2 backup path {db2_backup_path} exists. Proceeding with restore...") + + db2_backup_resource_path = os.path.join(db2_backup_path, "resources") + # Get DB2 instance name from backed up CR + cr_path = os.path.join(db2_backup_resource_path, "cr.yaml") + + # read cr.yaml + with open(cr_path, 'r') as cr_file: + backup_db2u_cr = yaml.safe_load(cr_file) + display.v("Successfully read DB2 backup CR file") + + db2_instance_name = backup_db2u_cr['metadata']['name'] + db2_namespace = backup_db2u_cr['metadata']['namespace'] + + # Check if Db2u instance is already present + db2u_instance_cr = getDb2uInstance(dynClient, db2_instance_name, db2_namespace) + if db2u_instance_cr is not None: + display.v(f"Db2u instance {db2_instance_name} already exists in namespace {db2_namespace}. Checking if it is ready...") + if isDb2uReady(db2u_instance_cr): + display.v(f"Db2u instance {db2_instance_name} is already in Ready state. Checking version...") + if getDb2VersionFromCR(db2u_instance_cr) == getDb2VersionFromCR(backup_db2u_cr): + display.v(f"Db2u instance {db2_instance_name} version matches the backup version. Skipping DB2 instance restore...") + return dict( + changed=False, + failed=False, + proceed=False, + success=True, + msg=f"Db2u instance {db2_instance_name} is already in Ready state. Skipping DB2 instance restore..." + ) + else: + display.v(f"Db2u instance {db2_instance_name} version does not match the backup version. Abandoning restore... Check the instance manually.") + return dict( + changed=False, + failed=True, + proceed=False, + success=False, + msg=f"Db2u instance {db2_instance_name} version does not match the backup version. Abandoning restore... Check the instance manually." + ) + else: + display.v(f"Db2u instance {db2_instance_name} is not in Ready state. Abandoning restore... Check the instance status manually.") + return dict( + changed=False, + failed=True, + proceed=False, + success=False, + msg=f"Db2u instance {db2_instance_name} is not in Ready state. Abandoning restore... Check the instance status manually." + ) + else: + display.v(f"Db2u instance {db2_instance_name} does not exist in namespace {db2_namespace}. Proceeding with B2 instance restore...") + + # ======================================================= + # 1. Create DB2 namespace if not exists + # ======================================================= + display.v(f"Creating Db2 namespace '{db2_namespace}' if it does not already exist") + createNamespace(dynClient, db2_namespace) + + # ======================================================= + # 2. Restore Db2 Secret resources from backup + # ======================================================= + display.v(f"Restoring Db2 Secret resources from backup path '{db2_backup_resource_path}/secrets'") + db2_secrets_path = os.path.join(db2_backup_resource_path, "secrets") + secret_files = os.listdir(db2_secrets_path) + for secret_file in secret_files: + # Some info files will be kept in secrets folder with NOT_SECRET in the filename + if "NOT_SECRET" in secret_file: + continue + display.v(f"Restoring Db2 Secret resource from backup file '{secret_file}'") + with open(os.path.join(db2_secrets_path, secret_file), 'r') as f: + secret_yaml = f.read() + apply_resource(dynClient, secret_yaml, db2_namespace) + + # ======================================================= + # 3. Gather info from backup files to recreate new Db2u instance + # ======================================================= + + db2_info = {} + db2_info['db2_instance_name'] = db2_instance_name + db2_info['db2_namespace'] = db2_namespace + db2_info['db2_type'] = backup_db2u_cr['spec'].get('environment', {}).get('dbType', 'db2wh') + # Get DB name from backup CR + db2_info['db2_database_name'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('name', 'BLUDB') + + # Get Db2 configs from backup CR + db2_info['db2_database_db_config'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('dbConfig', {}) + db2_info['db2_instance_dbm_config'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('dbmConfig', {}) + db2_info['db2_instance_registry'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('registry', {}) + + # Get dftTableOrg + db2_info['db2_table_org'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('settings', {}).get('dftTableOrg', 'ROW') + + # Get Pod config + db2_pod_config = backup_db2u_cr['spec'].get('podConfig', {}).get('db2u', {}).get('resource', {}).get('db2u', {}) + db2_info['db2_cpu_requests'] = db2_pod_config.get('requests', {}).get('cpu', None) + db2_info['db2_memory_requests'] = db2_pod_config.get('requests', {}).get('memory', None) + db2_info['db2_cpu_limits'] = db2_pod_config.get('limits', {}).get('cpu', None) + db2_info['db2_memory_limits'] = db2_pod_config.get('limits', {}).get('memory', None) + + # Get LDAP user info if present + ldap_info_file_path = os.path.join(db2_backup_resource_path, "ldapuser-NOT_SECRET.yaml") + if os.path.exists(ldap_info_file_path): + with open(ldap_info_file_path, 'r') as ldap_info_file: + ldap_info = yaml.safe_load(ldap_info_file) + db2_info['db2_ldap_username'] = ldap_info['db2_ldap_username'] + db2_info['db2_ldap_password'] = ldap_info['db2_ldap_password'] + display.v(f"Successfully read LDAP user info from {ldap_info_file_path}") + + # Get DB2 backup info file + db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-backup-info.yaml") + if os.path.exists(db2_info_file_path): + with open(db2_info_file_path, 'r') as info_file: + backup_db2_info = yaml.safe_load(info_file) + db2_info['db2_version'] = backup_db2_info['db2_version'] + db2_info['db2_channel'] = backup_db2_info['db2_channel'] + display.v(f"Successfully read Db2 backup info from {db2_info_file_path}") + else: + db2_info['db2_version'] = getDb2VersionFromCR(backup_db2u_cr) + if "s11.5.9.0" in db2_info['db2_version']: + db2_info['db2_channel'] = "v110509.0" + else: + display.v("Warning: Could not find Db2 backup info file. Using default channel for the version from CR.") + + return dict( + message=f"Successfully restored Db2 instance's Secrets from backup paths.", + failed=False, + changed=False, + success=True, + proceed=True, + **db2_info + ) diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index 94c155185e..a2f093d0e7 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -12,9 +12,10 @@ class ActionModule(ActionBase): "restore": ["mongodb_instance_name", "mas_backup_dir", "mongodb_backup_version"] }, "db2": { - "backup": ["db2_instance_name", "mas_backup_dir", "mas_instance_id", "mas_app_id"], - "restore": ["db2_instance_name", "mas_backup_dir", "db2_backup_version", "mas_app_id"], - "s3_setup": ["backup_vendor","backup_s3_name", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] + "backup": ["db2_instance_name", "db2_namespace", "mas_backup_dir", "mas_instance_id"], + "restore_instance": ["mas_backup_dir", "db2_backup_version"], + "restore_database": ["mas_backup_dir", "db2_backup_version", "db2_instance_name", "backup_vendor"], + "s3_setup": ["backup_vendor","backup_s3_alias", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] } } diff --git a/ibm/mas_devops/plugins/module_utils/backuprestore.py b/ibm/mas_devops/plugins/module_utils/backuprestore.py index eede7f0b47..dea1ab15ac 100644 --- a/ibm/mas_devops/plugins/module_utils/backuprestore.py +++ b/ibm/mas_devops/plugins/module_utils/backuprestore.py @@ -94,17 +94,17 @@ def getDb2VersionFromCR(db2uCR: dict) -> str: except Exception as e: return None -def getDb2uInstance(DynamicClient, db2_instance_name: str, db2_namespace: str): +def getDb2uInstance(dynClient: DynamicClient, db2_instance_name: str, db2_namespace: str): """ Retrieve Db2uCluster CR instance """ - db2uCR = getCR(DynamicClient, cr_api_version="db2u.databases.ibm.com/v1", cr_kind="Db2uCluster", cr_name=db2_instance_name, namespace=db2_namespace) + db2uCR = getCR(dynClient, cr_api_version="db2u.databases.ibm.com/v1", cr_kind="Db2uCluster", cr_name=db2_instance_name, namespace=db2_namespace) if db2uCR: return db2uCR.to_dict() else: # Db2uCluster CR not found, try Db2uInstance - db2uCR = getCR(DynamicClient, cr_api_version="db2u.databases.ibm.com/v1", cr_kind="Db2uInstance", cr_name=db2_instance_name, namespace=db2_namespace) + db2uCR = getCR(dynClient, cr_api_version="db2u.databases.ibm.com/v1", cr_kind="Db2uInstance", cr_name=db2_instance_name, namespace=db2_namespace) if db2uCR: return db2uCR.to_dict() return None diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index 3ba6b80441..11a25e15c7 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -419,6 +419,317 @@ This is only used when both `mas_config_dir` and `mas_instance_id` are set, and - Default: None +Role Variables - Backup and Restore +------------------------------------------------------------------------------- + +### db2_action +When set to `backup` or `restore`, the role will perform backup or restore operations on the DB2 instance and database. + +- Optional +- Environment Variable: `DB2_ACTION` +- Default: `install` +- Supported values: `install`, `upgrade`, `backup`, `restore` + +### mas_backup_dir +The local directory path where backup files will be stored (for backup) or read from (for restore). + +- **Required** for backup and restore operations +- Environment Variable: `MAS_BACKUP_DIR` +- Default: None +- Example: `/tmp/mas_backups` + +### db2_backup_version +Version identifier for the backup. For backup operations, if not provided, it defaults to `YYMMDD-HHMMSS` format. For restore operations, this is required to identify which backup to restore. + +- Optional for backup (auto-generated if not provided) +- **Required** for restore +- Environment Variable: `DB2_BACKUP_VERSION` +- Default: Auto-generated timestamp in `YYMMDD-HHMMSS` format for backup operations + +### br_skip_instance +Set to `false` to back up or restore the DB2 instance Kubernetes resources. + +- Optional +- Environment Variable: `BR_SKIP_INSTANCE` +- Default: `true` (only backup/restore the database data) + +### backup_vendor +Specifies where the database backup will be stored or retrieved from. + +- Optional +- Environment Variable: `BACKUP_VENDOR` +- Default: `disk` +- Supported values: `s3`, `disk` + +### backup_type +Type of database backup to perform. + +- Optional +- Environment Variable: `DB2_BACKUP_TYPE` +- Default: `online` +- Supported values: `online`, `offline` + +### backup_s3_alias +Alias name for the S3 storage configuration in DB2. + +- **Required** when `backup_vendor=s3` +- Environment Variable: `BACKUP_S3_ALIAS` +- Default: `S3DB2COS` + +### backup_s3_endpoint +S3-compatible storage endpoint URL. + +- **Required** when `backup_vendor=s3` +- Environment Variable: `BACKUP_S3_ENDPOINT` +- Default: None +- Example: `https://s3.us-east.cloud-object-storage.appdomain.cloud` + +### backup_s3_bucket +S3 bucket name where backups will be stored. + +- **Required** when `backup_vendor=s3` +- Environment Variable: `BACKUP_S3_BUCKET` +- Default: None + +### backup_s3_access_key +S3 access key for authentication. + +- **Required** when `backup_vendor=s3` +- Environment Variable: `BACKUP_S3_ACCESS_KEY` +- Default: None + +### backup_s3_secret_key +S3 secret key for authentication. + +- **Required** when `backup_vendor=s3` +- Environment Variable: `BACKUP_S3_SECRET_KEY` +- Default: None + + +Backup and Restore Operations +------------------------------------------------------------------------------- + +### Overview +The DB2 role supports comprehensive backup and restore operations for both DB2 instance configuration and database data. The backup process creates a snapshot of necessary resources that can be used to restore the DB2 instance to a previous state. + +### Backup Components +The backup operation consists of two main components: + +1. **Instance Backup**: Backs up Kubernetes resources including: + - DB2UCluster custom resource + - Secrets (instance password, LDAP credentials if configured) + - ConfigMaps + - Other DB2 instance metadata + +2. **Database Backup**: Backs up the actual database data using DB2's native backup utilities + +### Backup Process + +#### Prerequisites +- DB2 instance must be running and healthy +- Sufficient storage space in the backup location +- For S3 backups: Valid S3 credentials and accessible S3 bucket +- For disk backups: Sufficient local disk space + +#### Backup Workflow +1. **Validation**: Verifies all required variables are set +2. **Instance Backup** (when `BR_SKIP_INSTANCE=false`): + - Exports DB2UCluster CR and related Kubernetes resources + - Saves instance configuration and secrets +3. **Database Backup**: + - Prepares backup scripts and copies them to the DB2 pod + - For S3: Configures S3 storage access in DB2 + - Executes DB2 backup command (online or offline) + - For disk backups: Copies backup files to local storage + - Creates `db2-backup-info.yaml` with backup metadata + +#### Backup Storage Locations +- **Disk**: `/backup--db2u/data/` +- **S3**: `s3:///backups-db2//` + +### Restore Process + +#### Prerequisites +- Valid backup files exist in the specified location +- For instance restore: DB2 operator must be installed +- For database restore: DB2 instance must be running with matching version +- For S3 restores: Valid S3 credentials and accessible S3 bucket + +#### Restore Workflow +1. **Validation**: Verifies backup version and required variables +2. **Instance Restore** (when `BR_SKIP_INSTANCE=false`): + - Checks if DB2 instance exists and is healthy + - If not present or unhealthy: Creates new instance from backup configuration + - Restores secrets and configuration +3. **Database Restore**: + - For disk: Validates DB2 version matches backup version + - Prepares restore scripts + - For S3: Configures S3 storage access + - For disk: Copies backup archive to DB2 pod + - Executes DB2 restore command + - Validates restore completion + +### Backup Metadata +Each backup creates a `db2-backup-info.yaml` file containing: +```yaml +source_db2_backup_version: "241225-143022" +source_db2_backup_timestamp: "20241225143022" +source_db2_instance_name: "db2u-manage" +source_db2_instance_version: "11.5.8.0-cn7" +database: "BLUDB" +backup_vendor: "s3" +vendor_backup_path: "DB2REMOTE://S3DB2COS/my-bucket/backups-db2/241225-143022" +status: "SUCCESS" +``` + +### Example: Database-Only Backup (Skip Instance) to S3 +```bash +export DB2_ACTION=backup +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/tmp/mas_backups +export DB2_INSTANCE_NAME=db2u-manage +export DB2_NAMESPACE=db2u +export BR_SKIP_INSTANCE=true +export BACKUP_VENDOR=s3 +export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud +export BACKUP_S3_BUCKET=mas-db2-backups +export BACKUP_S3_ACCESS_KEY= +export BACKUP_S3_SECRET_KEY= + +ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +``` + +### Example: Database-Only Backup (Skip Instance) to Disk +```bash +export DB2_ACTION=backup +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/tmp/mas_backups +export DB2_INSTANCE_NAME=db2u-manage +export DB2_NAMESPACE=db2u +export BR_SKIP_INSTANCE=true +export BACKUP_VENDOR=disk + +ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +``` + +### Example: Backup both Instance and Database to S3 (only database backup is uploaded to S3) +```bash +export DB2_ACTION=backup +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/tmp/mas_backups +export DB2_INSTANCE_NAME=db2u-manage +export DB2_NAMESPACE=db2u +export BR_SKIP_INSTANCE=false +export BACKUP_VENDOR=s3 +export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud +export BACKUP_S3_BUCKET=mas-db2-backups +export BACKUP_S3_ACCESS_KEY= +export BACKUP_S3_SECRET_KEY= +export DB2_BACKUP_TYPE=online + +ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +``` + +### Example: Backup both Instance and Database to Disk +```bash +export DB2_ACTION=backup +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/tmp/mas_backups +export DB2_INSTANCE_NAME=db2u-manage +export BR_SKIP_INSTANCE=false +export DB2_NAMESPACE=db2u +export BACKUP_VENDOR=disk +export DB2_BACKUP_TYPE=online + +ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +``` + +### Example: Restore Database from S3 +```bash +export DB2_ACTION=restore +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/tmp/mas_backups +export DB2_INSTANCE_NAME=db2u-manage +export DB2_NAMESPACE=db2u +export DB2_BACKUP_VERSION=241225-143022 +export BR_SKIP_INSTANCE=true +export BACKUP_VENDOR=s3 +export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud +export BACKUP_S3_BUCKET=mas-db2-backups +export BACKUP_S3_ACCESS_KEY= +export BACKUP_S3_SECRET_KEY= + +ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +``` + +### Example: Restore Database from Disk +```bash +export DB2_ACTION=restore +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/tmp/mas_backups +export DB2_INSTANCE_NAME=db2u-manage +export BR_SKIP_INSTANCE=true +export DB2_NAMESPACE=db2u +export DB2_BACKUP_VERSION=241225-143022 +export BACKUP_VENDOR=disk + +ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +``` + +### Example: Restore Instance and database from Disk +```bash +export DB2_ACTION=restore +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/tmp/mas_backups +export DB2_INSTANCE_NAME=db2u-manage +export BR_SKIP_INSTANCE=false +export DB2_NAMESPACE=db2u +export DB2_BACKUP_VERSION=241225-143022 +export BACKUP_VENDOR=disk + +ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +``` + +### Backup Types + +#### Online Backup +- Database remains available during backup +- Recommended for production environments +- Requires archive logging to be enabled +- Set `DB2_BACKUP_TYPE=online` + +#### Offline Backup +- Database is taken offline during backup +- Faster than online backup +- Requires downtime +- Set `DB2_BACKUP_TYPE=offline` + +### Important Notes + +1. **Version Compatibility**: The DB2 version of the restore target must match the backup source version +2. **Storage Requirements**: Ensure sufficient storage space for backups (typically 1-2x database size) +3. **S3 Credentials**: Keep S3 credentials secure and use appropriate IAM policies +4. **Backup Retention**: Implement a backup retention policy to manage storage costs +5. **Testing**: Regularly test restore procedures to ensure backup integrity +6. **Archive Logging**: For online backups, ensure DB2 archive logging is properly configured +7. **Network Connectivity**: For S3 backups, ensure DB2 pod has network access to S3 endpoint + +### Troubleshooting + +#### Backup Failures +- Check DB2 pod logs: `oc logs -n -c db2u` +- Review backup script logs in the pod: `/tmp/db2_backup.log` +- Verify S3 credentials and connectivity (for S3 backups) +- Ensure sufficient storage space + +#### Restore Failures +- Verify backup version exists and is complete +- Check DB2 version compatibility +- Review restore script logs: `/tmp/db2_restore_disk.log` or `/tmp/db2_restore_s3.log` +- Ensure DB2 instance is running and healthy before database restore +- For S3 restores, verify S3 connectivity and credentials + + License ------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index e272b54115..4291de6097 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -128,11 +128,12 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # ----------------------------------------------------------------------------- mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" -br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" -backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('s3', true) }}" # Supported values are s3 and disk -backup_s3_name: "{{ lookup('env', 'BACKUP_S3_NAME') | default('S3DB2COS', true) }}" +br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set flag to false, to backup db2 instance +backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) }}" # Supported values are s3 and disk +backup_s3_alias: "{{ lookup('env', 'BACKUP_S3_ALIAS') | default('S3DB2COS', true) }}" backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" backup_s3_access_key: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY') }}" backup_s3_secret_key: "{{ lookup('env', 'BACKUP_S3_SECRET_KEY') }}" -backup_type: full +backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline + diff --git a/ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh b/ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh deleted file mode 100755 index 483fe819b8..0000000000 --- a/ibm/mas_devops/roles/db2/files/backup/download_bkp_images.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -set -o pipefail - -# ********************************************************** -# Download DB2 backup from IBMCOS -# -# Operation: DOWNLOAD_BACKUP -# -# Parameters: -# $1 stgEndpoint - IBMCOS storage endpoint -# $2 access_key_id - IBMCOS access_key_id -# $3 secret_access_key - IBMCOS secret_access_key -# $$ ibm_cos_bucket - IBMCOS bucket -# $5 ibm_cos_path - IBMCOS ibm_cos_path -# $6 flex_db_name - Original migrated database name -# -#********************************************************** -#DOWNLOAD_BACKUP -function downloadDBBackup() -{ - stgEndpoint=$1 - access_key_id=$2 - secret_access_key=$3 - ibm_cos_bucket=$4 - ibm_cos_path=$5 - flex_db_name=$6 - download_path=$7 - file_name_to_be_downloaded=$8 - db2_version=$9 - cos_bucket_alias=${10} - DOWNLOAD_BACK_LOG=~/bin/.DownloadBackupImagesLOG.out - rm -f ${DOWNLOAD_BACK_LOG} - - echo "[INFO] DOWNLOAD_BACKUP" >> ${DOWNLOAD_BACK_LOG} - - echo "[DEBUG] DOWNLOAD_BACKUP stgEndpoint: $stgEndpoint" >> ${DOWNLOAD_BACK_LOG} - - #echo "[DEBUG] DOWNLOAD_BACKUP access_key_id; $access_key_id" - #echo "[DEBUG] DOWNLOAD_BACKUP secret_access_key: $secret_access_key" - echo "[DEBUG] DOWNLOAD_BACKUP access_key_id; XXXXXXXXXX" >> ${DOWNLOAD_BACK_LOG} - echo "[DEBUG] DOWNLOAD_BACKUP secret_access_key: XXXXXXXXXX" >> ${DOWNLOAD_BACK_LOG} - - echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_bucket: $ibm_cos_bucket" >> ${DOWNLOAD_BACK_LOG} - echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_path: $ibm_cos_path" >> ${DOWNLOAD_BACK_LOG} - echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_bucket: $ibm_cos_bucket" >> ${DOWNLOAD_BACK_LOG} - echo "[DEBUG] DOWNLOAD_BACKUP ibm_cos_path: $ibm_cos_path" >> ${DOWNLOAD_BACK_LOG} - echo "[DEBUG] DOWNLOAD_BACKUP flex_db_name: $flex_db_name" >> ${DOWNLOAD_BACK_LOG} - echo "[DEBUG] DOWNLOAD_BACKUP download_path: $download_path" >> ${DOWNLOAD_BACK_LOG} - echo "[DEBUG] DOWNLOAD_BACKUP file_name_to_be_downloaded: $file_name_to_be_downloaded" >> ${DOWNLOAD_BACK_LOG} - if [ $db2_version == 'v11.5.7.0' ]; then - db2RemStgManager S3 get server=${stgEndpoint} auth1=${access_key_id} auth2=${secret_access_key} container=${ibm_cos_bucket} source=${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded 2>&1 | tee -a ${DOWNLOAD_BACK_LOG} - else - db2RemStgManager ALIAS GET source=DB2REMOTE://${cos_bucket_alias}/${ibm_cos_bucket}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded 2>&1 | tee -a ${DOWNLOAD_BACK_LOG} - fi - rc=$? - if [ $rc -ne 0 ]; then - if [ $db2_version == 'v11.5.7.0' ]; then - echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager s3 get ${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} - else - echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager ALIAS GET source=DB2REMOTE://${cos_bucket_alias}/${ibm_cos_bucket}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} - fi - echo "status=fail" >> ${DOWNLOAD_BACK_LOG} - exit 1 - fi - - checkDownload=$(ls -1 $download_path/$file_name_to_be_downloaded) - echo "[DEBUG] checkDownload= $checkDownload" >> ${DOWNLOAD_BACK_LOG} - - if [[ $checkDownload =~ "$download_path/$file_name_to_be_downloaded" ]]; then - echo "[DEBUG] Backup image $file_name_to_be_downloaded donwloaded successfully" >> ${DOWNLOAD_BACK_LOG} - else - if [ $db2_version == 'v11.5.7.0' ]; then - echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager s3 get ${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} - else - echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager ALIAS GET source=DB2REMOTE://${cos_bucket_alias}/${ibm_cos_bucket}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} - fi - echo "status=fail" >> ${DOWNLOAD_BACK_LOG} - exit 1 - fi - - echo "status=success" | tee -a ${DOWNLOAD_BACK_LOG} - sleep 5 - exit 0 - -} - -load_source_props=${11} -if [ "${load_source_props}" = true ]; then - echo "Load from source props was true" - source /mnt/backup/bin/.PROPS - downloadDBBackup $SERVER $PARM1 $PARM2 $CONTAINER $5 $6 $7 $8 $9 ${10} -else - echo "Load from source props was False" - downloadDBBackup $1 $2 $3 $4 $5 $6 $7 $8 $9 ${10} -fi \ No newline at end of file diff --git a/ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh b/ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh index 6132d8dfb5..3855288c47 100644 --- a/ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh +++ b/ibm/mas_devops/roles/db2/files/backup/prepare_backup_scripts.sh @@ -17,16 +17,32 @@ DBPATH=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "$ SOURCEPATH="$DBPATH/db2profile" . $SOURCEPATH +mkdir -p ${INSTHOME}/bin/ + cd /tmp/db2-scripts/ echo -e "\nCopying the files to bin directory under Instance Home . . . " -cp -rp db2_backup.sh ${INSTHOME}/bin/ + +if [ -f db2_backup.sh ]; then + echo -e "\nCopying db2_backup.sh to bin directory under Instance Home . . . " + cp -rp db2_backup.sh ${INSTHOME}/bin/ +fi if [ -f setup_cos_storage_access.sh ]; then echo -e "\nCopying setup_cos_storage_access.sh to bin directory under Instance Home . . . " cp -rp setup_cos_storage_access.sh ${INSTHOME}/bin/ fi +if [ -f db2_restore_disk.sh ]; then + echo -e "\nCopying db2_restore_disk.sh to bin directory under Instance Home . . . " + cp -rp db2_restore_disk.sh ${INSTHOME}/bin/ +fi + +if [ -f db2_restore_s3.sh ]; then + echo -e "\nCopying db2_restore_s3.sh to bin directory under Instance Home . . . " + cp -rp db2_restore_s3.sh ${INSTHOME}/bin/ +fi + chown -R ${INSTOWNER}:${INSTGROUP} ${INSTHOME}/bin echo -e "\nINSTHOME=${INSTHOME}\n" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml index fed39a20cb..ba9b2425d0 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml @@ -1,18 +1,18 @@ --- - name: Debug – configuration summary ansible.builtin.debug: - msg: | - ================== BACKUP DB2 DATABASE CONFIGURATION SUMMARY ================== - MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }} - DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }} - NAMESPACE : {{ db2_namespace | default('UNDEFINED') }} - DB NAME : {{ db2_dbname | default('UNDEFINED') }} - BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }} - VENDOR : {{ backup_vendor | default('UNDEFINED') }} - S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }} - S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }} - BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }} - =========================================================== + msg: + - "================== BACKUP DB2 DATABASE CONFIGURATION SUMMARY ==================" + - "MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }}" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" + - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" + - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" + - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" + - "================================================================================" # Check if backup vendor is s3, then check s3 related variables # ----------------------------------------------------------------------------- @@ -21,7 +21,7 @@ component: "db2" action: "s3_setup" backup_vendor: "{{ backup_vendor }}" - backup_s3_name: "{{ backup_s3_name }}" + backup_s3_alias: "{{ backup_s3_alias }}" backup_s3_endpoint: "{{ backup_s3_endpoint }}" backup_s3_bucket: "{{ backup_s3_bucket }}" backup_s3_access_key: "{{ backup_s3_access_key }}" @@ -40,12 +40,12 @@ - name: "Set fact backup_path for the backup script when backup vendor is s3" set_fact: - full_backup_path: "DB2REMOTE://{{ backup_s3_name }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }}" + full_backup_path: "DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }}" when: backup_vendor == "s3" - name: "Set fact backup_path for the backup script when backup vendor is disk" set_fact: - full_backup_path: "/mnt/backup/{{ db2_backup_version }}" + full_backup_path: "/mnt/backup/{{ db2_backup_version }}" when: backup_vendor == "disk" # Check if db2 instance is running and get the pod name @@ -68,7 +68,12 @@ set_fact: db2_pod_name: "{{ db2_pod_name_result.pod_name }}" -- name: "Create /tmp/db2-scripts directory on localhost" +- name: "Remove any existing files in /tmp/db2-scripts directory on localhost" + ansible.builtin.file: + path: "/tmp/db2-scripts" + state: absent + +- name: "Recreate /tmp/db2-scripts directory on localhost" ansible.builtin.file: path: "/tmp/db2-scripts" state: directory @@ -103,7 +108,7 @@ # Create /tmp/db2-scripts directory in DB2 pod and copy scripts into the pod # ----------------------------------------------------------------------------- - name: create /tmp/db2-scripts directory in DB2 pod - shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'mkdir -p /tmp/db2-scripts/' db2inst1" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'rm -rf /tmp/db2-scripts && mkdir -p /tmp/db2-scripts/' db2inst1" register: create_dir_result retries: 2 delay: 15 # seconds @@ -152,9 +157,13 @@ - name: "Execute setup_cos_storage_access.sh in DB2 pod" shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/setup_cos_storage_access.sh | tee /tmp/setup_cos_storage_access.log' db2inst1" register: setup_s3_result - retries: 2 - delay: 15 # seconds - until: setup_s3_result.rc == 0 + when: backup_vendor == "s3" + +- name: "Assert S3 storage access setup completed successfully" + assert: + that: + - setup_s3_result is defined + - setup_s3_result.rc == 0 when: backup_vendor == "s3" - name: "Debug setup_cos_storage_access.sh output" @@ -164,52 +173,80 @@ # Execute db2_backup.sh in DB2 pod # ----------------------------------------------------------------------------- -- name: "Execute db2_backup.sh in DB2 pod" - shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/db2_backup.sh {{ db2_backup_path }} | tee /tmp/db2_backup.log' db2inst1" - register: db2_backup_result - retries: 2 - delay: 15 # seconds - until: - - db2_backup_result.rc == 0 - - db2_backup_result.stdout | regex_search('BACKUP COMPLETED SUCCESSFULLY', multiline=True) - -- name: Get backup file timestamp from db2_backup.log - ansible.builtin.set_fact: - db2_backup_timestamp: "{{ (db2_backup_result.stdout | regex_search('BACKUP_FILE_TIMESTAMP=(.*)', '\\1'))[0] }}" - when: - - db2_backup_result is defined - - db2_backup_result.rc == 0 - - db2_backup_result.stdout is defined - -# If vendor is disk, tar the backup files and rsync to backup_data_path -# ----------------------------------------------------------------------------- -- name: "Move backup files to {{ db2_backup_data_path }} when backup vendor is disk" - shell: "oc rsync -n {{ db2_namespace }} {{ db2_pod_name }}:{{ full_backup_path }}/db2_{{ db2_dbname }}_backup_{{ db2_backup_version }}.tar.gz {{ db2_backup_data_path }}/" - register: rsync_result - retries: 2 - delay: 15 # seconds - until: rsync_result.rc == 0 - when: backup_vendor == "disk" - -- name: "Debug rsync output" - ansible.builtin.debug: - msg: "{{ rsync_result.stdout_lines }}" - when: backup_vendor == "disk" +- name: "Executing DB2 backup.." + block: + - name: "Execute db2_backup.sh in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/db2_backup.sh | tee /tmp/db2_backup.log' db2inst1" + register: db2_backup_result + + - name: "Debug db2_backup.sh output" + ansible.builtin.debug: + msg: "{{ db2_backup_result.stdout_lines }}" + + - name: "Assert DB2 backup completed successfully" + assert: + that: + - db2_backup_result is defined + - db2_backup_result.rc == 0 + - db2_backup_result.stdout | regex_search('BACKUP COMPLETED SUCCESSFULLY', multiline=True) + fail_msg: "DB2 backup did not complete successfully. Check /tmp/db2_backup.log in {{ db2_pod_name }} pod for details." + + - name: Get backup file timestamp from db2_backup.log + ansible.builtin.set_fact: + db2_backup_timestamp: "{{ (db2_backup_result.stdout | regex_search('BACKUP_FILE_TIMESTAMP=(.*)', '\\1'))[0] }}" + when: + - db2_backup_result is defined + - db2_backup_result.rc == 0 + - db2_backup_result.stdout is defined + + # If vendor is disk, tar the backup files and cp to backup_data_path + # ----------------------------------------------------------------------------- + - name: "Move backup files to {{ db2_backup_data_path }} when backup vendor is disk.. This will take a while..." + shell: "oc cp --retries=50 -c db2u {{ db2_namespace }}/{{ db2_pod_name }}:{{ full_backup_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz {{ db2_backup_data_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" + register: rsync_result + when: backup_vendor == "disk" + + - name: "Debug rsync output" + ansible.builtin.debug: + msg: "{{ rsync_result.stdout_lines }}" + when: backup_vendor == "disk" + + always: + - name: "Clean up backup directory in the pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'rm -rf {{ full_backup_path }}' db2inst1" + when: backup_vendor == "disk" # Create yaml file with db2 backup details, add local_backup_path if vendor is disk # ----------------------------------------------------------------------------- - name: "Create yaml file with db2 backup details" copy: - dest: "{{ db2_backup_data_path }}/db2-backup-info.yml" + dest: "{{ db2_backup_data_path }}/db2-backup-info.yaml" content: | source_db2_backup_version: "{{ db2_backup_version }}" source_db2_backup_timestamp: "{{ db2_backup_timestamp }}" source_db2_instance_name: "{{ db2_instance_name }}" - source_db2_instance_version: "{{ db2_pod_name_result.db2version }}" + source_db2_instance_version: "{{ db2_pod_name_result.db2_version }}" + database: "{{ db2_dbname }}" backup_vendor: "{{ backup_vendor }}" vendor_backup_path: "{{ full_backup_path }}" {% if backup_vendor == 'disk' %} - local_backup_path: "{{ db2_backup_data_path }}/db2_{{ db2_dbname }}_backup_{{ db2_backup_version }}.tar.gz" + local_backup_path: "{{ db2_backup_data_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" {% endif %} + status: "SUCCESS" + +- name: Database backup summary + ansible.builtin.debug: + msg: + - "================== BACKUP DB2 DATABASE CONFIGURATION SUMMARY ==================" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" + - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" + - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" + - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" + - "STATUS : SUCCESS" + - "================================================================================" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml index 6a690c347c..940136c5d8 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml @@ -1,15 +1,15 @@ --- -- name: Debug – configuration summary +- name: DB2 Instance backup configuration summary ansible.builtin.debug: - msg: | - ================== BACKUP DB2 INSTANCE CONFIGURATION SUMMARY ================== - MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }} - DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }} - NAMESPACE : {{ db2_namespace | default('UNDEFINED') }} - DB NAME : {{ db2_dbname | default('UNDEFINED') }} - BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }} - BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }} - =========================================================== + msg: + - "================== BACKUP DB2 INSTANCE CONFIGURATION SUMMARY ==================" + - "MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }}" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" + - "================================================================================" - name: "Start Db2 Instance backup process." ibm.mas_devops.backup_db2_instance: @@ -26,3 +26,16 @@ - db2_instance_backup_result is defined - db2_instance_backup_result.success == true fail_msg: "Db2 Instance resources backup failed." + +- name: DB2 Instance backup summary + ansible.builtin.debug: + msg: + - "================== BACKUP DB2 INSTANCE CONFIGURATION SUMMARY ==================" + - "MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }}" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" + - "STATUS : SUCCESS" + - "================================================================================" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 9550762ded..943779b10c 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -8,7 +8,7 @@ db2_instance_name: "{{ db2_instance_name }}" mas_backup_dir: "{{ mas_backup_dir }}" mas_instance_id: "{{ mas_instance_id }}" - mas_app_id: "{{ mas_application_id }}" + db2_namespace: "{{ db2_namespace }}" # Set DB2 backup version if not provided # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/tasks/install/main.yml b/ibm/mas_devops/roles/db2/tasks/install/main.yml index eb04619a7d..f1c00fead8 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/main.yml @@ -138,7 +138,7 @@ - name: "Create ibm-registry secret if not present" when: - - ibm-registry_secret_info.resources is defined + - ibm_registry_secret_info.resources is defined - ibm_registry_secret_info.resources | length == 0 block: - name: Set 'ibm-registry' secret content @@ -293,7 +293,12 @@ msg: - "DB2 Channel ........................... {{ db2_channel }}" - "DB2 Version (determined) .............. {{ db2_version }}" - - "Available versions .................... {{ db2_releases_content.databases.db2u.keys() | list }}" + +- name: Debug available DB2 versions + ansible.builtin.debug: + msg: + - "Available DB2 Versions ................ {{ db2_releases_content.databases.db2u.keys() | list }}" + when: db2_releases_content is defined # 11. Fail if required parameters are not set # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/tasks/restore/main.yml b/ibm/mas_devops/roles/db2/tasks/restore/main.yml index 8428c99c0b..ca0a978557 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/main.yml @@ -1,8 +1,9 @@ --- -# Check db2 backup required variables -# ----------------------------------------------------------------------------- -- name: "Fail if db2_instance_name is not provided" - assert: - that: db2_instance_name is defined and db2_instance_name != "" - fail_msg: "db2_instance_name is required" +# Restore Db2 Universal operator Instance Kubernetes Resources +# ------------------------------------------------------------------------- +- name: "Start Db2 Universal operator Instance restore process." + include_tasks: "{{ role_path }}/tasks/restore/restore-instance.yml" + when: not br_skip_instance | bool +- name: "Start Database restore process." + include_tasks: "{{ role_path }}/tasks/restore/restore-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml new file mode 100644 index 0000000000..4ff7679848 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml @@ -0,0 +1,40 @@ +--- +# Check db2 Restore required variables +# ----------------------------------------------------------------------------- +- name: Verify DB2 restore variables + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "{{ db2_action }}_database" + db2_backup_version: "{{ db2_backup_version }}" + mas_backup_dir: "{{ mas_backup_dir }}" + db2_instance_name: "{{ db2_instance_name }}" + backup_vendor: "{{ backup_vendor }}" + +# Check s3 variables when backup vendor is s3 +# ----------------------------------------------------------------------------- +- name: "Verify DB2 S3 variables" + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "s3_setup" + backup_vendor: "{{ backup_vendor }}" + backup_s3_alias: "{{ backup_s3_alias }}" + backup_s3_endpoint: "{{ backup_s3_endpoint }}" + backup_s3_bucket: "{{ backup_s3_bucket }}" + backup_s3_access_key: "{{ backup_s3_access_key }}" + backup_s3_secret_key: "{{ backup_s3_secret_key }}" + when: backup_vendor == "s3" + +- name: Debug – "Start database restore process" + ansible.builtin.debug: + msg: + - "================== STARTING RESTORE DB2 DATABASE ==============================" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" + - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" + - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" + - "BACKUP DIR : {{ mas_backup_dir | default('UNDEFINED') }}" + - "================================================================================" + +- name: "Run restore-db-from-disk.yml when backup vendor is disk" + include_tasks: "{{ role_path }}/tasks/restore/restore-db-from-{{ backup_vendor }}.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-disk.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-disk.yml new file mode 100644 index 0000000000..5c6a0597aa --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-disk.yml @@ -0,0 +1,199 @@ + +- name: "Set fact backup_path for the backup script when backup vendor is disk" + set_fact: + pod_full_backup_path: "/mnt/backup/{{ db2_backup_version }}" + local_full_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u/data" + +# Check backup files in local backup directory if vendor is disk +# ----------------------------------------------------------------------------- +# check db2-backup-info.yaml file exists in local backup directory +- name: "Check db2-backup-info.yaml file exists in local backup directory" + stat: + path: "{{ local_full_backup_path }}/db2-backup-info.yaml" + register: db2_backup_info_file_stat + +- name: "Assert db2-backup-info.yaml file exists" + assert: + that: + - db2_backup_info_file_stat.stat.exists == true + fail_msg: "db2-backup-info.yaml file does not exist in {{ mas_backup_dir }}/data. Cannot proceed with restore." + +- name: include vars from db2-backup-info.yaml + include_vars: + file: "{{ local_full_backup_path }}/db2-backup-info.yaml" + name: db2_backup_info + +# check tar.gz file exists in local backup directory +- name: Check for tar.gz file in backup directory + stat: + path: "{{ local_full_backup_path }}/db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz" + register: db2_backup_tar_stat + +- name: "Fail if db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz file not found" + assert: + that: + - db2_backup_tar_stat.stat.exists == true + fail_msg: "No db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz file found in {{ local_full_backup_path }}. Cannot proceed with restore." + +- name: "Set fact backup_archive" + set_fact: + backup_archive_filename: "db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz" + +# Check if Db2 instance is running and version match with backup version +# ----------------------------------------------------------------------------- +- name: "Check DB2 is running and get DB2 pod name" + ibm.mas_devops.get_db2u_pod_name: + db2_instance_name: "{{ db2_instance_name }}" + db2_namespace: "{{ db2_namespace }}" + register: db2_pod_name_result + +- name: Assert DB2 is running and pod name is found + assert: + that: + - db2_pod_name_result is defined + - db2_pod_name_result.success + - db2_pod_name_result.pod_name != "" + - db2_pod_name_result.db2_version != "" + fail_msg: "DB2 Instance {{ db2_instance_name }} is not running in namespace {{ db2_namespace }}. Ensure the DB2 instance is running." + +- name: "Assert DB2 version matches backup version" + assert: + that: + - db2_pod_name_result.db2_version == db2_backup_info.source_db2_instance_version + fail_msg: "DB2 version {{ db2_pod_name_result.db2_version }} does not match backup version {{ db2_backup_info.source_db2_instance_version }}. Cannot proceed with restore." + +- name: "Set fact db2_pod_name" + set_fact: + db2_pod_name: "{{ db2_pod_name_result.pod_name }}" + +# Copy backup files to the pod +# ----------------------------------------------------------------------------- +- name: "Remove any existing files in /tmp/db2-scripts directory on localhost" + ansible.builtin.file: + path: "/tmp/db2-scripts" + state: absent + +- name: "Recreate /tmp/db2-scripts directory on localhost" + ansible.builtin.file: + path: "/tmp/db2-scripts" + state: directory + mode: "0755" + +- name: "Copy prepare_backup_scripts.sh to /tmp" + ansible.builtin.copy: + src: "{{ role_path }}/files/backup/prepare_backup_scripts.sh" + dest: "/tmp/db2-scripts/prepare_backup_scripts.sh" + mode: "0755" + +- name: create template restore script in /tmp/db2-scripts directory on localhost + ansible.builtin.template: + src: "backup/db2_restore_disk.sh.j2" + dest: "/tmp/db2-scripts/db2_restore_disk.sh" + mode: "0755" + vars: + db2_dbname: "{{ db2_backup_info.database }}" + db2_backup_version: "{{ db2_backup_info.source_db2_backup_version }}" + db2_backup_timestamp: "{{ db2_backup_info.source_db2_backup_timestamp | trim }}" + backup_type: "online" + +- name: Zip the DB2 backup scripts + ansible.builtin.archive: + path: /tmp/db2-scripts + dest: /tmp/db2-scripts.zip + format: zip + mode: "0755" + +# Create /tmp/db2-scripts directory in DB2 pod and copy scripts into the pod +# ----------------------------------------------------------------------------- +- name: create /tmp/db2-scripts directory in DB2 pod + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'mkdir -p /tmp/db2-scripts/ && rm -rf /tmp/db2-scripts/*' db2inst1" + register: create_dir_result + retries: 2 + delay: 15 # seconds + until: create_dir_result.rc == 0 + +- name: Copy /tmp/db2-scripts.zip to DB2 pod /tmp/db2-scripts.zip + shell: "oc cp /tmp/db2-scripts.zip {{ db2_namespace }}/{{ db2_pod_name }}:/tmp/db2-scripts.zip -c db2u" + register: copy_scripts_result + retries: 2 + delay: 15 # seconds + until: copy_scripts_result.rc == 0 + +- name: Unzip DB2 backup scripts in DB2 pod + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'cd /tmp; unzip -o db2-scripts.zip; chmod -R +x /tmp/db2-scripts' db2inst1" + register: unzip_result + retries: 2 + delay: 15 # seconds + until: unzip_result.rc == 0 + +# Execute prepare_backup_scripts.sh in DB2 pod +# this script will unzip the backup scripts, and sets the right permissions to the scripts. +# ----------------------------------------------------------------------------- +- name: "Execute prepare_backup_scripts.sh in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '/tmp/db2-scripts/prepare_backup_scripts.sh | tee /tmp/prepare_backup_scripts.log' db2inst1" + register: prepare_scripts_result + retries: 2 + delay: 15 # seconds + until: + - prepare_scripts_result.rc == 0 + - prepare_scripts_result.stdout | regex_search('PrepareSuccess', multiline=True) + +- name: "Debug prepare_backup_scripts.sh output" + ansible.builtin.debug: + msg: "{{ prepare_scripts_result.stdout_lines }}" + +- name: Get INSTHOME from prepare_backup_scripts.log + ansible.builtin.set_fact: + db2_insthome: "{{ (prepare_scripts_result.stdout | regex_search('INSTHOME=(.*)', '\\1'))[0] }}" + when: + - prepare_scripts_result is defined + - prepare_scripts_result.rc == 0 + - prepare_scripts_result.stdout is defined + +# Copy backup archive to DB2 pod +# ----------------------------------------------------------------------------- +- name: "Create {{ pod_full_backup_path }} directory in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'rm -rf {{ pod_full_backup_path }} && mkdir -p {{ pod_full_backup_path }} && chmod a+w {{ pod_full_backup_path}}' db2inst1" + register: create_dir_result + failed_when: create_dir_result.rc != 0 + +- name: "Copy backup archive to DB2 pod. This will take a while..." + shell: "oc cp --retries=50 {{ local_full_backup_path }}/{{ backup_archive_filename }} {{ db2_namespace }}/{{ db2_pod_name }}:{{ pod_full_backup_path }}/{{ backup_archive_filename }} -c db2u" + register: copy_backup_result + failed_when: copy_backup_result.rc != 0 + +- name: "Copying backup archive to DB2 pod result" + debug: + msg: + - "Backup archive {{ backup_archive_filename }} copied to DB2 pod {{ db2_pod_name }} in path {{ pod_full_backup_path }}/{{ backup_archive_filename }}" + +- name: "Extract backup files" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'tar -xzf {{ pod_full_backup_path }}/{{ backup_archive_filename }} -C {{ pod_full_backup_path }} && ls {{ pod_full_backup_path }}' db2inst1" + register: extract_backup_result + failed_when: extract_backup_result.rc != 0 + +- name: "Extract backup files result" + debug: + msg: + - "------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------" + - "Backup archive {{ backup_archive_filename }} extracted in DB2 pod {{ db2_pod_name }} in path {{ pod_full_backup_path }}" + - "------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------" + - "{{ extract_backup_result.stdout_lines }}" + +# Execute db2_restore_disk.sh in DB2 pod if backup vendor is disk +# ----------------------------------------------------------------------------- +- name: "Execute db2_restore_disk.sh in DB2 pod, Check logs in /tmp/db2_restore_disk.log in DB2 pod {{ db2_pod_name }}" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/db2_restore_disk.sh | tee /tmp/db2_restore_disk.log' db2inst1" + register: restore_db_result + failed_when: restore_db_result.rc != 0 + +- name: "Debug db2_restore_disk.sh output" + ansible.builtin.debug: + msg: "{{ restore_db_result.stdout_lines }}" + +- name: "Assert DB2 restore was successful" + assert: + that: + - restore_db_result.stdout is defined + - restore_db_result.stdout | regex_search('RestoreSuccess', multiline=True) + fail_msg: "DB2 restore failed. Check /tmp/db2_restore_disk.log in DB2 pod {{ db2_pod_name }} for more details." diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-s3.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-s3.yml new file mode 100644 index 0000000000..88b903fa23 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-s3.yml @@ -0,0 +1,142 @@ + +- name: "Set fact backup_path for the backup script when backup vendor is disk" + set_fact: + pod_full_backup_path: "/mnt/backup/{{ db2_backup_version }}" + s3_full_backup_path: "DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }}" + +# Check if Db2 instance is running and version match with backup version +# ----------------------------------------------------------------------------- +- name: "Check DB2 is running and get DB2 pod name" + ibm.mas_devops.get_db2u_pod_name: + db2_instance_name: "{{ db2_instance_name }}" + db2_namespace: "{{ db2_namespace }}" + register: db2_pod_name_result + +- name: Assert DB2 is running and pod name is found + assert: + that: + - db2_pod_name_result is defined + - db2_pod_name_result.success + - db2_pod_name_result.pod_name != "" + - db2_pod_name_result.db2_version != "" + fail_msg: "DB2 Instance {{ db2_instance_name }} is not running in namespace {{ db2_namespace }}. Ensure the DB2 instance is running." + +- name: "Set fact db2_pod_name" + set_fact: + db2_pod_name: "{{ db2_pod_name_result.pod_name }}" + +# Copy Restore files to the pod +# ----------------------------------------------------------------------------- +- name: "Remove any existing files in /tmp/db2-scripts directory on localhost" + ansible.builtin.file: + path: "/tmp/db2-scripts" + state: absent + +- name: "Recreate /tmp/db2-scripts directory on localhost" + ansible.builtin.file: + path: "/tmp/db2-scripts" + state: directory + mode: "0755" + +- name: "Copy prepare_backup_scripts.sh to /tmp" + ansible.builtin.copy: + src: "{{ role_path }}/files/backup/prepare_backup_scripts.sh" + dest: "/tmp/db2-scripts/prepare_backup_scripts.sh" + mode: "0755" + +- name: "Template the DB2 S3 storage access setup script" + ansible.builtin.template: + src: "backup/setup_cos_storage_access.sh.j2" + dest: "/tmp/db2-scripts/setup_cos_storage_access.sh" + mode: "0755" + +- name: create template restore script in /tmp/db2-scripts directory on localhost + ansible.builtin.template: + src: "backup/db2_restore_s3.sh.j2" + dest: "/tmp/db2-scripts/db2_restore_s3.sh" + mode: "0755" + +- name: Zip the DB2 backup scripts + ansible.builtin.archive: + path: /tmp/db2-scripts + dest: /tmp/db2-scripts.zip + format: zip + mode: "0755" + +# Create /tmp/db2-scripts directory in DB2 pod and copy scripts into the pod +# ----------------------------------------------------------------------------- +- name: create /tmp/db2-scripts directory in DB2 pod + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'mkdir -p /tmp/db2-scripts/ && rm -rf /tmp/db2-scripts/*' db2inst1" + register: create_dir_result + retries: 2 + delay: 15 # seconds + until: create_dir_result.rc == 0 + +- name: Copy /tmp/db2-scripts.zip to DB2 pod /tmp/db2-scripts.zip + shell: "oc cp /tmp/db2-scripts.zip {{ db2_namespace }}/{{ db2_pod_name }}:/tmp/db2-scripts.zip -c db2u" + register: copy_scripts_result + retries: 2 + delay: 15 # seconds + until: copy_scripts_result.rc == 0 + +- name: Unzip DB2 backup scripts in DB2 pod + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'cd /tmp; unzip -o db2-scripts.zip; chmod -R +x /tmp/db2-scripts' db2inst1" + register: unzip_result + retries: 2 + delay: 15 # seconds + until: unzip_result.rc == 0 + +# Execute prepare_backup_scripts.sh in DB2 pod +# this script will unzip the backup scripts, and sets the right permissions to the scripts. +# ----------------------------------------------------------------------------- +- name: "Execute prepare_backup_scripts.sh in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '/tmp/db2-scripts/prepare_backup_scripts.sh | tee /tmp/prepare_backup_scripts.log' db2inst1" + register: prepare_scripts_result + retries: 2 + delay: 15 # seconds + until: + - prepare_scripts_result.rc == 0 + - prepare_scripts_result.stdout | regex_search('PrepareSuccess', multiline=True) + +- name: "Debug prepare_backup_scripts.sh output" + ansible.builtin.debug: + msg: "{{ prepare_scripts_result.stdout_lines }}" + +- name: Get INSTHOME from prepare_backup_scripts.log + ansible.builtin.set_fact: + db2_insthome: "{{ (prepare_scripts_result.stdout | regex_search('INSTHOME=(.*)', '\\1'))[0] }}" + when: + - prepare_scripts_result is defined + - prepare_scripts_result.rc == 0 + - prepare_scripts_result.stdout is defined + +# Execute setup_cos_storage_access.sh in DB2 pod if backup vendor is s3 +# ----------------------------------------------------------------------------- +- name: "Execute setup_cos_storage_access.sh in DB2 pod" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/setup_cos_storage_access.sh | tee /tmp/setup_cos_storage_access.log' db2inst1" + register: setup_s3_result + retries: 2 + delay: 15 # seconds + until: setup_s3_result.rc == 0 + +- name: "Debug setup_cos_storage_access.sh output" + ansible.builtin.debug: + msg: "{{ setup_s3_result.stdout_lines }}" + +# Execute db2_restore_s3.sh in DB2 pod if backup vendor is s3 +# ----------------------------------------------------------------------------- +- name: "Execute db2_restore_s3.sh in DB2 pod, Check logs in /tmp/db2_restore_s3.log in DB2 pod {{ db2_pod_name }}" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/db2_restore_s3.sh | tee /tmp/db2_restore_s3.log' db2inst1" + register: restore_db_result + failed_when: restore_db_result.rc != 0 + +- name: "Debug db2_restore_s3.sh output" + ansible.builtin.debug: + msg: "{{ restore_db_result.stdout_lines }}" + +- name: "Assert DB2 restore was successful" + assert: + that: + - restore_db_result.stdout is defined + - restore_db_result.stdout | regex_search('RestoreSuccess', multiline=True) + fail_msg: "DB2 restore failed. Check /tmp/db2_restore_s3.log in DB2 pod {{ db2_pod_name }} for more details." diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml new file mode 100644 index 0000000000..df06115f58 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml @@ -0,0 +1,87 @@ +--- +# Check db2 Restore required variables +# ----------------------------------------------------------------------------- +- name: Verify DB2 restore variables + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "{{ db2_action }}_instance" + db2_backup_version: "{{ db2_backup_version }}" + mas_backup_dir: "{{ mas_backup_dir }}" + +# Restore Db2 Universal operator Instance Kubernetes Resources +# ------------------------------------------------------------------------- +- name: "Start Db2 Universal operator Instance restore process." + ibm.mas_devops.restore_db2_resources: + db2_backup_version: "{{ db2_backup_version }}" + mas_backup_dir: "{{ mas_backup_dir }}" + register: restore_db2_instance_result + +- name: "Assert Restore Db2 Universal operator Instance Kubernetes Resources result" + assert: + that: + - restore_db2_instance_result is defined + - restore_db2_instance_result.success == true + fail_msg: "Restoring Db2 Universal operator Instance Kubernetes Resources failed." + +- name: "Check if DB2 instance exist and healthy. If not, proceed with restore." + ansible.builtin.debug: + msg: + - "================== FINISHED RESTORE DB2 INSTANCE ==============================" + - "{{ restore_db2_instance_result.msg }}" + - "===============================================================================" + when: + - restore_db2_instance_result.proceed == False + +# Create Db2 Instance using info from backup CR if required +# ------------------------------------------------------------------------- +- name: "Create Db2 Instance if not present" + block: + - name: "Set fact from Db2 info" + set_fact: + db2_database_db_config: "{{ restore_db2_instance_result.db2_database_db_config }}" + db2_instance_dbm_config: "{{ restore_db2_instance_result.db2_instance_dbm_config }}" + db2_instance_registry: "{{ restore_db2_instance_result.db2_instance_registry }}" + db2_channel: "{{ restore_db2_instance_result.db2_channel }}" + db2_version: "{{ restore_db2_instance_result.db2_version }}" + db2_instance_name: "{{ restore_db2_instance_result.db2_instance_name }}" + db2_namespace: "{{ restore_db2_instance_result.db2_namespace }}" + db2_type: "{{ restore_db2_instance_result.db2_type }}" + db2_dbname: "{{ restore_db2_instance_result.db2_database_name }}" + db2_table_org: "{{ restore_db2_instance_result.db2_table_org }}" + db2_cpu_requests: "{{ restore_db2_instance_result.db2_cpu_requests }}" + db2_memory_requests: "{{ restore_db2_instance_result.db2_memory_requests }}" + db2_cpu_limits: "{{ restore_db2_instance_result.db2_cpu_limits }}" + db2_memory_limits: "{{ restore_db2_instance_result.db2_memory_limits }}" + + - name: "Set fact LDAP credentials if exist." + set_fact: + db2_ldap_username: "{{ restore_db2_instance_result.db2_ldap_username }}" + db2_ldap_password: "{{ restore_db2_instance_result.db2_ldap_password }}" + when: restore_db2_instance_result.db2_ldap_username is defined and restore_db2_instance_result.db2_ldap_password is defined + + - name: DB2 Instance restore configuration summary + ansible.builtin.debug: + msg: + - "================== STARTING RESTORE DB2 INSTANCE ===============================" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "BACKUP DIR : {{ mas_backup_dir | default('UNDEFINED') }}" + - "================================================================================" + + - name: "Creating Db2 Universal operator Instance" + include_tasks: "{{ role_path }}/tasks/install/main.yml" + + - name: DB2 Instance restore summary + ansible.builtin.debug: + msg: + - "================== FINISHED RESTORE DB2 INSTANCE ===============================" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "BACKUP DIR : {{ mas_backup_dir | default('UNDEFINED') }}" + - "================================================================================" + + when: restore_db2_instance_result.proceed == true diff --git a/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 index 48f9d31ed8..b24c6b833a 100644 --- a/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 +++ b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 @@ -1,9 +1,130 @@ #!/bin/bash + +function stopDBConnections() +{ + databaseName=${1} + + echo "" + echo "STEP 1: Temporarily disable the built-in HA" + if [ -f /etc/wolverine/config.json ]; then + wvcli system disable -m "Disable HA before Db2 maintenance" + echo "INFO: Wolverine HA disabled" + else + echo "INFO: Wolverine HA not enabled" + fi + + echo "" + echo "STEP 2: Connect to the database" + connect_status=$(db2 -v connect to ${databaseName}) + rc=$? + if [ $rc -ne 0 ]; then + if echo "$connect_status" | grep -q "SQL1032N"; then + echo "INFO: Database ${databaseName} is not currently active. Proceeding with restore" + elif echo "$connect_status" | grep -q "SQL1013N"; then + echo "INFO: Database ${databaseName} does not exist. Proceeding with restore" + elif echo "$connect_status" | grep -q "SQL30081N"; then + echo "ERROR: Network error while connecting to database ${databaseName}" + exit 1 + elif echo "$connect_status" | grep -q "SQL1119N"; then + echo "ERROR: $connect_status" + exit 1 + else + echo "INFO: $connect_status. Proceeding with restore" + fi + fi + + echo "" + echo "STEP 3: Disconnect all applications connected to Db2" + db2 -v force application all + echo "INFO: Waiting for connections to close (30 seconds)..." + sleep 30 + + echo "" + echo "STEP 4: Terminate the database" + db2 -v terminate + + echo "" + echo "STEP 5: Stop the database" + db2stop force + + echo "" + echo "STEP 6: Clean Db2 interprocess communications" + ipclean -a + + echo "" + echo "STEP 7: Disable database communications" + db2set -null DB2COMM + + echo "" + echo "STEP 8: Restart database in restricted access mode" + db2start admin mode restricted access + echo "INFO: Waiting for database to start in restricted access mode (60 seconds)..." + sleep 60 +} + +function startDB() +{ + databaseName=${1} + + echo "" + echo "STEP 10: Halt the restricted access mode" + db2stop force + + echo "" + echo "STEP 11: Clean Db2 interprocess communications" + ipclean -a + + echo "" + echo "STEP 12: Reinitialize the Db2 communication manager to accept database connections" + db2set DB2COMM=TCPIP,SSL + + echo "" + echo "STEP 13: Restart database for normal operation" + db2start + echo "INFO: Waiting for database to start (60 seconds)..." + sleep 60 + + echo "" + echo "STEP 14: Activate the database" + db2 activate db ${databaseName} + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to activate database ${databaseName}. Return code: $rc" + else + echo "INFO: Database activated successfully" + fi + + echo "" + echo "STEP 15: Re-enable Wolverine high availability" + sudo wvcli system enable -m "Enable HA after Db2 maintenance" + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to enable HA. Return code: $rc" + else + echo "INFO: Wolverine HA re-enabled successfully" + fi + + echo "" + echo "STEP 16: Show HA monitoring status" + sudo wvcli system status + sudo wvcli system devices + + echo "" + echo "STEP 17: Connect to the database" + db2 -v connect to ${databaseName} + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to connect to database ${databaseName}. Return code: $rc" + else + echo "INFO: Successfully connected to database ${databaseName}" + fi +} + DATABASE={{ db2_dbname }} BACKUP_TYPE={{ backup_type }} VENDOR={{ backup_vendor }} -### for S3, BACKUP_PATH=DB2REMOTE://{{ backup_s3_name }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }} +### for S3, BACKUP_PATH=DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }} BACKUP_PATH={{ full_backup_path }} # Finding the db2 Instance owner @@ -14,11 +135,10 @@ BACKUP_BASE=/mnt/backup # Find the home directory instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` -BACK_LOG=$instance_home/bin/{{ db2_backup_version }}_BackupLOG.out if [ ! -f "$instance_home/sqllib/db2profile" ] then - echo "ERROR - $instance_home/sqllib/db2profile not found" + echo "ERROR: $instance_home/sqllib/db2profile not found" EXIT_STATUS=1 else . $instance_home/sqllib/db2profile @@ -31,10 +151,10 @@ then rm ${COMPRESS_LOC} fi -### Check to see if the database is Running +### Check to see if the database is Running ps -ef | grep db2sys | grep -v grep > /dev/null 2>&1 if [ $? -eq 1 ]; then - echo "Database is not active " + echo "ERROR: Database is not active" echo "Database Not Active,BACKUP Not Run" > $instance_home/bin/LASTbkupRUN exit fi @@ -42,74 +162,173 @@ fi ### Create Backup Directory if it does not exist when vendor is disk if [ ${VENDOR} = 'disk' ] ; then if [ ! -d "${BACKUP_PATH}" ] ; then + echo "INFO: Creating backup directory: ${BACKUP_PATH}" mkdir -p ${BACKUP_PATH} if [ $? -ne 0 ] ; then echo "ERROR: Could not create backup directory ${BACKUP_PATH}" exit 1 fi + echo "INFO: Backup directory created successfully" + fi +fi + +### Check LOGARCHMETH1 is not OFF to proceed if ONLINE backup +if [ ${BACKUP_TYPE} = 'online' ] ; then + echo "INFO: Checking if LOGARCHMETH1 is ON to proceed with ONLINE backup..." + logarchmeth1_cmd=$(db2 get db cfg for ${DATABASE} | grep LOGARCHMETH1 | awk -F'= ' '{print $2}') + + if [ ${logarchmeth1_cmd} = 'OFF' ]; then + echo "ERROR: LOGARCHMETH1 is OFF. Cannot proceed with online backup, Please choose offline backup option. Exiting..." + exit 1 fi + echo "INFO: LOGARCHMETH1 is ON: $logarchmeth1_cmd" fi +# ============================================================================ +# Start Backup Process +# ============================================================================ DATETIME=`date +%Y-%m-%d_%H%M%S`; -echo "BACKUP Start time : ${DATETIME}" -echo " " -echo " " - -db2 -v archive log for db $DATABASE | tee -a $BACK_LOG -sleep 30 - - -if [ ${BACKUP_TYPE} = 'full' ] ; then - db2 -v backup db $DATABASE online to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 include logs without prompting | tee -a $BACK_LOG -else - db2 -v backup db $DATABASE online INCREMENTAL DELTA to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 include logs without prompting | tee -a $BACK_LOG -fi -grep -Fq "Backup successful." $BACK_LOG -if [ $? = 0 ]; then - backup_timestamp=`grep timestamp $BACK_LOG | cut -d: -f2` +echo "" +echo "========================================================================" +echo " DB2 BACKUP PROCESS STARTED" +echo "========================================================================" +echo " Database : ${DATABASE}" +echo " Backup Type : ${BACKUP_TYPE}" +echo " Backup Vendor : ${VENDOR}" +echo " Backup Path : ${BACKUP_PATH}" +echo " Start Time : ${DATETIME}" +echo "========================================================================" +echo "" + + +# ============================================================================ +# Execute Backup Command +# ============================================================================ +if [ ${BACKUP_TYPE} = 'online' ] ; then + echo "INFO: Performing full online backup of database ${DATABASE}..." + db2 -v archive log for db $DATABASE + sleep 30 + echo "INFO: Command: db2 -v backup db ${DATABASE} on all dbpartitionnums online to ${BACKUP_PATH} compress UTIL_IMPACT_PRIORITY 50 include logs without prompting" + BACKUP_CMD=$(db2 -v backup db $DATABASE on all dbpartitionnums online to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 include logs without prompting) +elif [ ${BACKUP_TYPE} = 'offline' ] ; then + echo "INFO: Performing full offline backup of database ${DATABASE}..." + + stopDBConnections $DATABASE + + echo "" + echo "STEP 9: Backup database in offline mode" + BACKUP_CMD=$(db2 -v backup db $DATABASE on all dbpartitionnums to $BACKUP_PATH compress UTIL_IMPACT_PRIORITY 50 without prompting) + + startDB $DATABASE + +else + echo "ERROR: Invalid backup type: ${BACKUP_TYPE}" + exit 1 +fi + +if echo "$BACKUP_CMD" | grep -q "Backup successful."; then + echo "$BACKUP_CMD" + backup_timestamp=`echo "$BACKUP_CMD" | grep timestamp | cut -d: -f2` + echo "" + echo "INFO: Backup operation completed successfully" else - echo " " - echo "********** BACKUP FAILED ************" - echo " " + echo "" + echo "========================================================================" + echo " BACKUP FAILED" + echo "========================================================================" + echo "ERROR: Backup operation did not complete successfully" >&2 + echo "ERROR: Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" >&2 + echo "========================================================================" exit 1 fi -######## Copy keystore to COS or Disk location based on VENDOR ######### +# ============================================================================ +# Copy Keystore Files +# ============================================================================ + +# Get KEYSTORE_LOCATION from dbm cfg +KEYSTORE_LOC=$(db2 get dbm cfg | grep KEYSTORE_LOCATION | awk -F'= ' '{print $2}') + +# Get .sth file by replacing the .p12 to .sth +STH_LOC=$(echo $KEYSTORE_LOC | sed 's/\.p12$/.sth/') + if [ ${VENDOR} = 's3' ] ; then - echo "Copying Keystore to COS location" - set -x - SOURCE1=/mnt/blumeta0/db2/keystore/keystore.p12 - SOURCE2=/mnt/blumeta0/db2/keystore/keystore.sth - TARGET1=KEYSTORE/keystore.p12 - TARGET2=KEYSTORE/keystore.sth - - db2RemStgManager ALIAS PUT source=${SOURCE1} target=${BACKUP_PATH}/${TARGET1} - db2RemStgManager ALIAS PUT source=${SOURCE2} target=${BACKUP_PATH}/${TARGET2} - echo "Copying Keystore to COS location - Completed" + echo "" + echo "INFO: Copying keystore files to COS location..." + set -x + TARGET1=keystore.p12 + TARGET2=keystore.sth + + db2RemStgManager ALIAS PUT source=${KEYSTORE_LOC} target=${BACKUP_PATH}/${TARGET1} + db2RemStgManager ALIAS PUT source=${STH_LOC} target=${BACKUP_PATH}/${TARGET2} + set +x + echo "INFO: Keystore files copied to COS location successfully" else - echo "Copying Keystore to local disk location" - cp /mnt/blumeta0/db2/keystore/keystore.p12 ${BACKUP_PATH}/keystore.p12 - cp /mnt/blumeta0/db2/keystore/keystore.sth ${BACKUP_PATH}/keystore.sth - echo "Copying Keystore to local disk location - Completed" + echo "" + echo "INFO: Copying keystore files to local disk location..." + cp ${KEYSTORE_LOC} ${BACKUP_PATH}/keystore.p12 + cp ${STH_LOC} ${BACKUP_PATH}/keystore.sth + echo "INFO: Keystore files copied to local disk location successfully" fi -DATETIME=`date +%Y-%m-%d_%H%M%S`; -echo "BACKUP End time : ${DATETIME}" >> $BACK_LOG -echo "BACKUP End time : ${DATETIME}" -echo " " +# ============================================================================ +# Create Backup Information File +# ============================================================================ +echo "" +echo "INFO: Creating db2-backup-info.yaml file..." +YAML_FILE=/tmp/db2-backup-info.yaml +cat <> ${YAML_FILE} +source_db2_backup_version: {{ db2_backup_version }} +database: ${DATABASE} +backup_type: ${BACKUP_TYPE} +backup_vendor: ${VENDOR} +vendor_backup_path: ${BACKUP_PATH} +backup_timestamp: ${backup_timestamp} +source_db2_instance_name: {{ db2_instance_name }} +source_db2_instance_version: {{ db2_version }} +source_db2_instance_channel: {{ db2_channel }} +status: SUCCESS +EOT +echo "INFO: db2-backup-info.yaml file created at ${YAML_FILE}" +if [ ${VENDOR} = 's3' ] ; then + echo "INFO: Uploading db2-backup-info.yaml file to COS location..." + db2RemStgManager ALIAS PUT source=${YAML_FILE} target=${BACKUP_PATH}/db2-backup-info.yaml + echo "INFO: db2-backup-info.yaml file uploaded successfully" +fi -###### If vendor is disk, tar the backup files +# ============================================================================ +# Archive Backup Files (Disk Vendor Only) +# ============================================================================ if [ ${VENDOR} = 'disk' ] ; then - echo "Starting to tar the backup files..." - tar -czf ${BACKUP_PATH}/db2_${DATABASE}_backup_{{ db2_backup_version }}.tar.gz -C ${BACKUP_PATH} . - echo "Tar backup files completed. Showing disk usage:" + cp ${YAML_FILE} ${BACKUP_PATH}/db2-backup-info.yaml + echo "" + echo "INFO: Creating tar archive of backup files..." + tar -czf ${BACKUP_PATH}/db2-${DATABASE}-backup-{{ db2_backup_version }}.tar.gz -C ${BACKUP_PATH} . + echo "INFO: Tar archive created successfully" + echo "" + echo "Disk Usage:" df -h ${BACKUP_PATH}/* else - echo "Backup vendor is not disk. Skipping tar of backup files." + echo "" + echo "INFO: Backup vendor is not disk. Skipping tar archive creation." fi - echo "BACKUP_FILE_TIMESTAMP=${backup_timestamp}" -echo " " -echo "********** BACKUP COMPLETED SUCCESSFULLY ************" \ No newline at end of file + +# ============================================================================ +# Backup Completion Summary +# ============================================================================ +DATETIME=`date +%Y-%m-%d_%H%M%S`; +echo "" +echo "========================================================================" +echo " DB2 BACKUP COMPLETED SUCCESSFULLY" +echo "========================================================================" +echo " Database : ${DATABASE}" +echo " Backup Type : ${BACKUP_TYPE}" +echo " Backup Vendor : ${VENDOR}" +echo " Backup Path : ${BACKUP_PATH}" +echo " End Time : ${DATETIME}" +echo " Backup Timestamp: ${backup_timestamp}" +echo "========================================================================" +echo "" diff --git a/ibm/mas_devops/roles/db2/templates/backup/db2_restore_disk.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/db2_restore_disk.sh.j2 new file mode 100644 index 0000000000..4451c130aa --- /dev/null +++ b/ibm/mas_devops/roles/db2/templates/backup/db2_restore_disk.sh.j2 @@ -0,0 +1,332 @@ +#!/bin/bash + +DATABASE={{ db2_dbname }} +BACKUP_TYPE={{ backup_type }} +DB2_BACKUP_VERSION={{ db2_backup_version }} +DB2_BACKUP_TIMESTAMP={{ db2_backup_timestamp }} + +# Finding the db2 Instance owner +INSTOWNER=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | awk -F ',' '{print $4}' ` + +instance=`whoami` +BACKUP_BASE=/mnt/backup +# Find the home directory +instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` + +if [ ! -f "$instance_home/sqllib/db2profile" ] +then + echo "ERROR: $instance_home/sqllib/db2profile not found" + EXIT_STATUS=1 +else + . $instance_home/sqllib/db2profile +fi + +### Verify the backup files + +### Check if keystore files are present + +POD_BACKUP_PATH="{{ pod_full_backup_path }}" + +if [[ ! -f "${POD_BACKUP_PATH}/keystore.p12" ]]; then + echo "ERROR: Db2 keystore.p12 file is missing in the backup location: ${POD_BACKUP_PATH}" + exit 1 +fi + +if [[ ! -f "${POD_BACKUP_PATH}/keystore.sth" ]]; then + echo "ERROR: Db2 keystore.sth file is missing in the backup location: ${POD_BACKUP_PATH}" + exit 1 +fi + +### Check if backup files are present +ls ${POD_BACKUP_PATH}/${DATABASE}*.DBPART000.* 2>/dev/null +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Db2 backup files are not found in the backup location: ${POD_BACKUP_PATH}" + exit 1 +fi + +# from the backup check command, check if the logs are included in the backup files. +BACKUP_FILES=${POD_BACKUP_PATH}/${DATABASE}*.DBPART000.* +backup_contain_logs=true +for FILE in $BACKUP_FILES; do + echo "INFO: Verifying downloaded backup $FILE" + bkpchk_cmd=$(db2ckbkp -h ${FILE}) + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to verify backup file ${FILE}. Return code: $rc" + exit 1 + else + echo "INFO: Successfully verified: $(basename "$FILE")" + echo "INFO: Check if the backup contains logs" + logs_value=$(echo "$bkpchk_cmd" | grep 'Includes Logs' | awk -F '-- ' '{print $2}' | awk '{print $1}') + # 0 when the backup doesn't contain logs + if [ ${logs_value} = '0' ]; then + echo "INFO: Backup doesn't contain logs" + backup_contain_logs=false + fi + fi +done +echo "INFO: Db2 backup files verified successfully." + +echo "INFO: Checking if LOGARCHMETH1 is ON." +logarchmeth1_enabled=true +logarchmeth1_cmd=$(db2 get db cfg for ${DATABASE} | grep LOGARCHMETH1 | awk -F'= ' '{print $2}') +if [ ${logarchmeth1_cmd} = 'OFF' ]; then + echo "INFO: LOGARCHMETH1 is OFF." + logarchmeth1_enabled=false +else + echo "INFO: LOGARCHMETH1 is ON." +fi + +echo "INFO: Db2 keystore files verified successfully." + +# Extract Db2 keystore master key +# https://www.ibm.com/docs/en/db2/11.5?topic=edr-restoring-encrypted-backup-image-different-system-local-keystore + +MASTER_KEY_LABEL=`gsk8capicmd_64 -cert -list all -db ${POD_BACKUP_PATH}/keystore.p12 -stashed 2>&1 | grep -i $INSTOWNER_$DATABASE | awk '/^#/ { print $2 }'` +if [ -z "$MASTER_KEY_LABEL" ]; then + echo "ERROR: MASTER_KEY_LABEL is empty. Cannot proceed with secret key extraction." + exit 1 +else + ### Check if MASTER_KEY_LABEL exists in the source keystore + SOURCE_KEYSTORE_LOC=$(db2 get dbm cfg | grep KEYSTORE_LOCATION | awk -F'= ' '{print $2}') + SOURCE_LABELS=$(gsk8capicmd_64 -cert -list all -db ${SOURCE_KEYSTORE_LOC} -stashed 2>&1 | awk '/^#/ { print $2 }') + if ! echo "$SOURCE_LABELS" | grep -q "^${MASTER_KEY_LABEL}$"; then + echo "INFO: MASTER_KEY_LABEL '${MASTER_KEY_LABEL}' not found in source keystore." + gsk8capicmd_64 -secretkey -extract -db ${POD_BACKUP_PATH}/keystore.p12 -stashed -label ${MASTER_KEY_LABEL} -format ascii -target ${POD_BACKUP_PATH}/master_key_label.kdb + echo "INFO: MASTER_KEY_LABEL is not empty - Secret key extraction Completed." + ## Add Master key to target keystore + result=$(gsk8capicmd_64 -secretkey -add -db ${SOURCE_KEYSTORE_LOC} -stashed -label ${MASTER_KEY_LABEL} -file ${POD_BACKUP_PATH}/master_key_label.kdb 2>&1) + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Error adding master key to target keystore. Return code: $rc" + exit 1 + fi + + if echo "$result" | grep -q "CTGSK3005W"; then + echo "ERROR: CTGSK3005W warning detected. Exiting" + echo "$result" + exit 1 + fi + else + echo "INFO: MASTER_KEY_LABEL '${MASTER_KEY_LABEL}' already exists in target keystore. Skipping addition." + fi +fi + +echo "INFO: Db2 keystore master key restored successfully." + +# Deactivate Db2 in preparation for restore +# https://www.ibm.com/docs/en/db2/11.5?topic=r-restoring-db2-from-online-backup-using-commands + +dbRestoreSuccess=true +dbRollbackSuccess=true + +echo "" +echo "========================================================================" +echo " PREPARING DATABASE FOR RESTORE" +echo "========================================================================" +echo "" + +echo "STEP 1: Temporarily disable the built-in HA" +if [ -f /etc/wolverine/config.json ]; then + wvcli system disable -m "Disable HA before Db2 maintenance" +else + echo "INFO: Wolverine HA not enabled" +fi + +echo "" +echo "STEP 2: Connect to the database" +connect_status=$(db2 -v connect to ${DATABASE}) +rc=$? +if [ $rc -ne 0 ]; then + if echo "$connect_status" | grep -q "SQL1032N"; then + echo "INFO: Database ${DATABASE} is not currently active. Proceeding with restore." + elif echo "$connect_status" | grep -q "SQL1013N"; then + echo "INFO: Database ${DATABASE} does not exist. Proceeding with restore." + elif echo "$connect_status" | grep -q "SQL30081N"; then + echo "ERROR: Network error while connecting to database ${DATABASE}" + exit 1 + elif echo "$connect_status" | grep -q "SQL1119N"; then + echo "ERROR: $connect_status" + exit 1 + else + echo "INFO: $connect_status. Proceeding with restore." + fi +fi + +echo "" +echo "STEP 3: Disconnect all the applications that are connected to Db2" +db2 -v force application all + +echo "" +echo "INFO: Waiting for connections to close, sleep for 30 seconds" +sleep 30 + +echo "" +echo "STEP 4: Terminate the database" +db2 -v terminate + +echo "" +echo "STEP 5: Stop the database" +db2stop force + +echo "" +echo "STEP 6: Ensure that all Db2 interprocess communications are cleaned for the instance" +ipclean -a + +echo "" +echo "STEP 7: Turn off all communications to the database by setting the value of the DB2COMM variable to null" +db2set -null DB2COMM + +echo "" +echo "STEP 8: Restart the database in restricted access mode" +db2start admin mode restricted access + +echo "" +echo "INFO: Waiting for db to start" +sleep 60 + +echo "" +echo "========================================================================" +echo " EXECUTING DATABASE RESTORE" +echo "========================================================================" +echo "" + +echo "STEP 9: Restore Db2 from a full backup" +if [[ "$backup_contain_logs" == "true" && "logarchmeth1_enabled" == "true" ]]; then + restore_cmd=$(db2 -v restore db ${DATABASE} from ${POD_BACKUP_PATH} taken at ${DB2_BACKUP_TIMESTAMP} into ${DATABASE} logtarget ${POD_BACKUP_PATH} replace existing without prompting) +else + echo "INFO: Restoring database ${DATABASE} from backup timestamp ${DB2_BACKUP_TIMESTAMP} WITHOUT logs..." + restore_cmd=$(db2 -v restore db ${DATABASE} from ${POD_BACKUP_PATH} taken at ${DB2_BACKUP_TIMESTAMP} into ${DATABASE} replace existing without prompting) +fi +rc=$? +if [ $rc -ne 0 ]; then + echo "$restore_cmd" + if echo "$restore_cmd" | grep -q "Restore is successful"; then + echo "INFO: Database ${DATABASE} restored successfully from backup." + else + echo "ERROR: Error restoring database ${DATABASE} from backup. Return code: $rc" + dbRestoreSuccess=false + fi +fi + +echo "" +echo "STEP 10: Check Db2 rollforward status" +if [[ "$backup_contain_logs" == "true" && "logarchmeth1_enabled" == "true" ]]; then + db2 -v rollforward db ${DATABASE} query status + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Error querying rollforward status for database ${DATABASE}. Return code: $rc" + fi +else + echo "INFO: Skipping rollforward status check as either backup does not contain logs or logarchmeth1 is OFF." +fi + +echo "" +echo "STEP 11: Run Db2 rollforward command" +if [[ "$backup_contain_logs" == "true" && "logarchmeth1_enabled" == "true" ]]; then + db2 -v rollforward db ${DATABASE} to end of backup and complete overflow log path "(${POD_BACKUP_PATH})" noretrieve + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Error during rollforward for database ${DATABASE}. Return code: $rc" + dbRollbackSuccess=false + fi +else + echo "INFO: Skipping rollforward as either backup does not contain logs or logarchmeth1 is OFF." +fi + +echo "" +echo "========================================================================" +echo " RESTARTING DATABASE FOR NORMAL OPERATION" +echo "========================================================================" +echo "" + +echo "STEP 12: Stop the database" +db2stop force +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Error stopping database ${DATABASE}. Return code: $rc" +fi + +echo "" +echo "STEP 13: Ensure that all Db2 interprocess communications are cleaned for the instance" +ipclean -a +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Error cleaning Db2 interprocess communications. Return code: $rc" +fi + +echo "" +echo "STEP 14: Reinitialize the Db2 communication manager to accept database connections" +db2set DB2COMM=TCPIP,SSL +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Error reinitializing Db2 communication manager. Return code: $rc" +fi + +echo "" +echo "STEP 15: Restart the database for normal operation" +db2start +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Error starting database ${DATABASE}. Return code: $rc" +fi + +echo "" +echo "INFO: waiting for db to start, sleep 60 seconds" +sleep 60 + +echo "" +echo "STEP 16: Activate the database" +db2 activate db ${DATABASE} +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Error activating database ${DATABASE}. Return code: $rc" +fi + +echo "" +echo "STEP 17: Re-enable the Wolverine high availability monitoring process" +sudo wvcli system enable -m "Enable HA after Db2 maintenance" +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Error enabling HA. Return code: $rc" +fi + +echo "" +echo "STEP 18: Connect to the database" +db2 -v connect to ${DATABASE} +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Error connecting to database ${DATABASE}. Return code: $rc" +fi + +# Cleanup the backup files from local path +echo "INFO: Cleaning up local backup files at: ${POD_BACKUP_PATH}" +rm -rf ${POD_BACKUP_PATH} + +echo "" +echo "========================================================================" +echo " RESTORE COMPLETION STATUS" +echo "========================================================================" + +if [ "$dbRestoreSuccess" = false ]; then + echo " Status: FAILED-RestoreFailed" + echo " Reason: Database restore encountered errors" + echo "========================================================================" + exit 1 +fi + +if [ "$dbRollbackSuccess" = false ]; then + echo " Status: FAILED-RestoreFailed" + echo " Reason: Database rollforward encountered errors" + echo "========================================================================" + exit 1 +fi + +echo " Database : ${DATABASE}" +echo " Backup Timestamp: ${DB2_BACKUP_TIMESTAMP}" +echo " POD Path : ${POD_BACKUP_PATH}" +echo " Status : SUCCESS-RestoreSuccess" +echo "========================================================================" +echo "" +exit 0 + diff --git a/ibm/mas_devops/roles/db2/templates/backup/db2_restore_s3.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/db2_restore_s3.sh.j2 new file mode 100644 index 0000000000..a96f0043ad --- /dev/null +++ b/ibm/mas_devops/roles/db2/templates/backup/db2_restore_s3.sh.j2 @@ -0,0 +1,453 @@ +#!/bin/bash +DB2_BACKUP_VERSION={{ db2_backup_version }} + +# Finding the db2 Instance owner +INSTOWNER=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | awk -F ',' '{print $4}' ` + +instance=`whoami` +BACKUP_BASE=/mnt/backup + +# Find the home directory +instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` + +if [ ! -f "$instance_home/sqllib/db2profile" ] +then + echo "ERROR: $instance_home/sqllib/db2profile not found" + EXIT_STATUS=1 +else + . $instance_home/sqllib/db2profile +fi + +REMOTE_PATH="{{ s3_full_backup_path }}" + +# ============================================================================ +# Verify Backup Files in S3 +# ============================================================================ +echo "" +echo "========================================================================" +echo " DB2 RESTORE FROM S3 BACKUP" +echo "========================================================================" +echo " Remote Path: ${REMOTE_PATH}" +echo "========================================================================" +echo "" + +echo "INFO: Verifying backup files in S3 path: ${REMOTE_PATH}" + +LIST_OUTPUT=$(db2RemStgManager ALIAS LIST source=${REMOTE_PATH} 2>&1) +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to list files in S3 path ${REMOTE_PATH}. Return code: $rc" + exit 1 +fi + +echo "$LIST_OUTPUT" + +TOTAL_FILES=$(echo "$LIST_OUTPUT" | awk -F= '/Total number of files found/ { gsub(/[^0-9]/,"",$2); print $2 }') +if [ "$TOTAL_FILES" -eq 0 ]; then + echo "ERROR: No files found in S3 path: ${REMOTE_PATH}" + exit 1 +fi + +FILES=$(echo "$LIST_OUTPUT" | awk '/^[0-9]/ { print $2 }') + +FOUND_P12=false +FOUND_STH=false +#FOUND_INFO=false +FOUND_BACKUP=false +BACKUP_FILE="" + +# ============================================================================ +# Validate Required Backup Files +# ============================================================================ +echo "" +echo "INFO: Validating required backup files..." + +for FILE in $FILES; do + if [[ "$FILE" == *.p12 ]]; then + FOUND_P12=true + elif [[ "$FILE" == *.sth ]]; then + FOUND_STH=true +# elif [[ "$FILE" == *db2-backup-info.yaml ]]; then +# FOUND_INFO=true + elif [[ "$FILE" == *.DBPART000.* ]]; then + FOUND_BACKUP=true + BACKUP_FILE=$(basename "$FILE") + fi +done + +if [[ "$FOUND_P12" == false ]]; then + echo "ERROR: Db2 keystore.p12 file is missing in the backup location: ${REMOTE_PATH}" + exit 1 +fi + +if [[ "$FOUND_STH" == false ]]; then + echo "ERROR: Db2 keystore.sth file is missing in the backup location: ${REMOTE_PATH}" + exit 1 +fi + +#if [[ "$FOUND_INFO" == false ]]; then +# echo "ERROR: Db2 db2-backup-info.yaml file is missing in the backup location: ${REMOTE_PATH}" +# exit 1 +#fi + +if [[ "$FOUND_BACKUP" == false ]]; then + echo "ERROR: Db2 backup files are missing in the backup location: ${REMOTE_PATH}" + exit 1 +fi + +echo "INFO: All required backup files verified successfully" + +# ============================================================================ +# Download Backup Files from S3 +# ============================================================================ +POD_BACKUP_PATH="/mnt/backup/{{ db2_backup_version }}" +rm -rf ${POD_BACKUP_PATH} +mkdir -p ${POD_BACKUP_PATH} + +echo "" +echo "INFO: Downloading keystore files from S3 to local path: ${POD_BACKUP_PATH}" +db2RemStgManager ALIAS GET source=${REMOTE_PATH}/keystore.p12 target=${POD_BACKUP_PATH}/keystore.p12 +db2RemStgManager ALIAS GET source=${REMOTE_PATH}/keystore.sth target=${POD_BACKUP_PATH}/keystore.sth +#db2RemStgManager ALIAS GET source=${REMOTE_PATH}/db2-backup-info.yaml target=${POD_BACKUP_PATH}/db2-backup-info.yaml +echo "INFO: Keystore files downloaded successfully" + +echo "" +echo "INFO: Downloading backup image files from S3..." + +# from the backup check command, check if the logs are included in the backup files. +backup_contain_logs=true +for FILE in $FILES; do + if [[ "$FILE" == *.DBPART000.* ]]; then + echo "INFO: Downloading backup file: $(basename "$FILE")" + db2RemStgManager ALIAS GET source=${REMOTE_PATH}/$(basename "$FILE") target=${POD_BACKUP_PATH}/$(basename "$FILE") + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to download backup file ${FILE}. Return code: $rc" + exit 1 + else + echo "INFO: Successfully downloaded: $(basename "$FILE")" + fi + + echo "INFO: Verifying downloaded backup file" + bkpchk_cmd=$(db2ckbkp -h ${POD_BACKUP_PATH}/$(basename "$FILE")) + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to verify backup file ${FILE}. Return code: $rc" + exit 1 + else + echo "INFO: Successfully verified: $(basename "$FILE")" + # Check if the backup contains logs + echo "INFO: Check if the backup contains logs" + logs_value=$(echo "$bkpchk_cmd" | grep 'Includes Logs' | awk -F '-- ' '{print $2}' | awk '{print $1}') + # 0 when the backup doesn't contain logs + if [ ${logs_value} = '0' ]; then + echo "INFO: Backup doesn't contain logs" + backup_contain_logs=false + fi + fi + fi +done +echo "INFO: All backup files downloaded successfully" + +# ============================================================================ +# Extract Database Information and Restore Keystore Master Key +# ============================================================================ +# Reference: https://www.ibm.com/docs/en/db2/11.5?topic=edr-restoring-encrypted-backup-image-different-system-local-keystore + +echo "" +echo "INFO: Extracting database information from backup file..." +# Example backup file name: DBNAME.0.db2inst1.DBPART000.20231015120000.001 +DATABASE=$(echo "$BACKUP_FILE" | awk -F. '{print $1}') +echo "INFO: Database name: ${DATABASE}" + +DB2_BACKUP_TIMESTAMP=$(echo "$BACKUP_FILE" | awk -F. '{print $5}') +echo "INFO: Backup timestamp: ${DB2_BACKUP_TIMESTAMP}" + +echo "INFO: Checking if LOGARCHMETH1 is ON." +logarchmeth1_enabled=true +logarchmeth1_cmd=$(db2 get db cfg for ${DATABASE} | grep LOGARCHMETH1 | awk -F'= ' '{print $2}') +if [ ${logarchmeth1_cmd} = 'OFF' ]; then + echo "INFO: LOGARCHMETH1 is OFF." + logarchmeth1_enabled=false +else + echo "INFO: LOGARCHMETH1 is ON." +fi + + +echo "" +echo "INFO: Extracting keystore master key..." +MASTER_KEY_LABEL=`gsk8capicmd_64 -cert -list all -db ${POD_BACKUP_PATH}/keystore.p12 -stashed 2>&1 | grep -i $INSTOWNER_$DATABASE | awk '/^#/ { print $2 }'` +if [ -z "$MASTER_KEY_LABEL" ]; then + echo "ERROR: MASTER_KEY_LABEL is empty. Cannot proceed with secret key extraction" + exit 1 +else + echo "INFO: Master key label: ${MASTER_KEY_LABEL}" + + # Check if MASTER_KEY_LABEL exists in the source keystore + SOURCE_KEYSTORE_LOC=$(db2 get dbm cfg | grep KEYSTORE_LOCATION | awk -F'= ' '{print $2}') + SOURCE_LABELS=$(gsk8capicmd_64 -cert -list all -db ${SOURCE_KEYSTORE_LOC} -stashed 2>&1 | awk '/^#/ { print $2 }') + if ! echo "$SOURCE_LABELS" | grep -q "^${MASTER_KEY_LABEL}$"; then + echo "INFO: Master key label '${MASTER_KEY_LABEL}' not found in target keystore" + echo "INFO: Extracting secret key from backup keystore..." + gsk8capicmd_64 -secretkey -extract -db ${POD_BACKUP_PATH}/keystore.p12 -stashed -label ${MASTER_KEY_LABEL} -format ascii -target ${POD_BACKUP_PATH}/master_key_label.kdb + echo "INFO: Secret key extraction completed" + + echo "INFO: Adding master key to target keystore..." + result=$(gsk8capicmd_64 -secretkey -add -db ${SOURCE_KEYSTORE_LOC} -stashed -label ${MASTER_KEY_LABEL} -file ${POD_BACKUP_PATH}/master_key_label.kdb 2>&1) + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to add master key to target keystore. Return code: $rc" + exit 1 + fi + + if echo "$result" | grep -q "CTGSK3005W"; then + echo "ERROR: CTGSK3005W warning detected" + echo "$result" + exit 1 + fi + echo "INFO: Master key added to target keystore successfully" + else + echo "INFO: Master key label '${MASTER_KEY_LABEL}' already exists in target keystore. Skipping addition" + fi +fi + +echo "INFO: Keystore master key restored successfully" + +# ============================================================================ +# Prepare Database for Restore +# ============================================================================ +# Reference: https://www.ibm.com/docs/en/db2/11.5?topic=r-restoring-db2-from-online-backup-using-commands + +dbRestoreSuccess=true +dbRollbackSuccess=true + +echo "" +echo "========================================================================" +echo " PREPARING DATABASE FOR RESTORE" +echo "========================================================================" +echo "" + +echo "STEP 1: Temporarily disable the built-in HA" +if [ -f /etc/wolverine/config.json ]; then + wvcli system disable -m "Disable HA before Db2 maintenance" + echo "INFO: Wolverine HA disabled" +else + echo "INFO: Wolverine HA not enabled" +fi + +echo "" +echo "STEP 2: Connect to the database" +connect_status=$(db2 -v connect to ${DATABASE}) +rc=$? +if [ $rc -ne 0 ]; then + if echo "$connect_status" | grep -q "SQL1032N"; then + echo "INFO: Database ${DATABASE} is not currently active. Proceeding with restore" + elif echo "$connect_status" | grep -q "SQL1013N"; then + echo "INFO: Database ${DATABASE} does not exist. Proceeding with restore" + elif echo "$connect_status" | grep -q "SQL30081N"; then + echo "ERROR: Network error while connecting to database ${DATABASE}" + exit 1 + elif echo "$connect_status" | grep -q "SQL1119N"; then + echo "ERROR: $connect_status" + exit 1 + else + echo "INFO: $connect_status. Proceeding with restore" + fi +fi + +echo "" +echo "STEP 3: Disconnect all applications connected to Db2" +db2 -v force application all +echo "INFO: Waiting for connections to close (30 seconds)..." +sleep 30 + +echo "" +echo "STEP 4: Terminate the database" +db2 -v terminate + +echo "" +echo "STEP 5: Stop the database" +db2stop force + +echo "" +echo "STEP 6: Clean Db2 interprocess communications" +ipclean -a + +echo "" +echo "STEP 7: Disable database communications" +db2set -null DB2COMM + +echo "" +echo "STEP 8: Restart database in restricted access mode" +db2start admin mode restricted access +echo "INFO: Waiting for database to start (60 seconds)..." +sleep 60 + +# ============================================================================ +# Execute Database Restore +# ============================================================================ +echo "" +echo "========================================================================" +echo " EXECUTING DATABASE RESTORE" +echo "========================================================================" +echo "" + +echo "STEP 9: Restore Db2 from full backup" + +if [[ "$backup_contain_logs" == "true" && "logarchmeth1_enabled" == "true" ]]; then + echo "INFO: Restoring database ${DATABASE} from backup timestamp ${DB2_BACKUP_TIMESTAMP}..." + restore_cmd=$(db2 -v restore db ${DATABASE} from ${POD_BACKUP_PATH} taken at ${DB2_BACKUP_TIMESTAMP} into ${DATABASE} logtarget ${POD_BACKUP_PATH} replace existing without prompting) +else + echo "INFO: Restoring database ${DATABASE} from backup timestamp ${DB2_BACKUP_TIMESTAMP} WITHOUT logs..." + restore_cmd=$(db2 -v restore db ${DATABASE} from ${POD_BACKUP_PATH} taken at ${DB2_BACKUP_TIMESTAMP} into ${DATABASE} replace existing without prompting) +fi +rc=$? +if [ $rc -ne 0 ]; then + echo "$restore_cmd" + if echo "$restore_cmd" | grep -q "Restore is successful"; then + echo "INFO: Database ${DATABASE} restored successfully from backup" + else + echo "ERROR: Failed to restore database ${DATABASE} from backup. Return code: $rc" + dbRestoreSuccess=false + fi +else + echo "INFO: Database restore completed successfully" +fi + +echo "" +if [[ "$backup_contain_logs" == "true" && "logarchmeth1_enabled" == "true" ]]; then + echo "STEP 10: Check Db2 rollforward status" + db2 -v rollforward db ${DATABASE} query status + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Failed to query rollforward status for database ${DATABASE}. Return code: $rc" + fi +else + echo "STEP 10: Check Db2 rollforward status" + echo "INFO: Skipping rollforward status as either backup does not contain logs or logarchmeth1 is OFF." +fi + +echo "" +echo "STEP 11: Execute Db2 rollforward" +if [[ "$backup_contain_logs" == "true" && "logarchmeth1_enabled" == "true" ]]; then + echo "INFO: Rolling forward database ${DATABASE}..." + db2 -v rollforward db ${DATABASE} to end of backup and complete overflow log path "(${POD_BACKUP_PATH})" noretrieve + rc=$? + if [ $rc -ne 0 ]; then + echo "ERROR: Rollforward failed for database ${DATABASE}. Return code: $rc" + dbRollbackSuccess=false + else + echo "INFO: Rollforward completed successfully" + fi +else + echo "INFO: Skipping rollforward as either backup does not contain logs or logarchmeth1 is OFF." +fi +echo "" + +# ============================================================================ +# Restart Database for Normal Operation +# ============================================================================ +echo "" +echo "========================================================================" +echo " RESTARTING DATABASE FOR NORMAL OPERATION" +echo "========================================================================" +echo "" + +echo "STEP 12: Stop the database" +db2stop force +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to stop database ${DATABASE}. Return code: $rc" +fi + +echo "" +echo "STEP 13: Clean Db2 interprocess communications" +ipclean -a +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to clean Db2 interprocess communications. Return code: $rc" +fi + +echo "" +echo "STEP 14: Reinitialize Db2 communication manager" +db2set DB2COMM=TCPIP,SSL +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to reinitialize Db2 communication manager. Return code: $rc" +fi + +echo "" +echo "STEP 15: Restart database for normal operation" +db2start +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to start database ${DATABASE}. Return code: $rc" +fi +echo "INFO: Waiting for database to start (60 seconds)..." +sleep 60 + +echo "" +echo "STEP 16: Activate the database" +db2 activate db ${DATABASE} +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to activate database ${DATABASE}. Return code: $rc" +else + echo "INFO: Database activated successfully" +fi + +echo "" +echo "STEP 17: Re-enable Wolverine high availability" +sudo wvcli system enable -m "Enable HA after Db2 maintenance" +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to enable HA. Return code: $rc" +else + echo "INFO: Wolverine HA re-enabled successfully" +fi + +echo "" +echo "STEP 18: Connect to the database" +db2 -v connect to ${DATABASE} +rc=$? +if [ $rc -ne 0 ]; then + echo "ERROR: Failed to connect to database ${DATABASE}. Return code: $rc" +else + echo "INFO: Successfully connected to database ${DATABASE}" +fi + +# ============================================================================ +# Cleanup and Final Status +# ============================================================================ +echo "" +echo "INFO: Cleaning up local backup files at: ${POD_BACKUP_PATH}" +rm -rf ${POD_BACKUP_PATH} +echo "INFO: Cleanup completed" + +echo "" +echo "========================================================================" +echo " RESTORE COMPLETION STATUS" +echo "========================================================================" + +if [ "$dbRestoreSuccess" = false ]; then + echo " Status: FAILED-RestoreFailed" + echo " Reason: Database restore encountered errors" + echo "========================================================================" + exit 1 +fi + +if [ "$dbRollbackSuccess" = false ]; then + echo " Status: FAILED-RestoreFailed" + echo " Reason: Database rollforward encountered errors" + echo "========================================================================" + exit 1 +fi + +echo " Database : ${DATABASE}" +echo " Backup Timestamp: ${DB2_BACKUP_TIMESTAMP}" +echo " Remote Path : ${REMOTE_PATH}" +echo " Status : SUCCESS-RestoreSuccess" +echo "========================================================================" +echo "" +exit 0 + diff --git a/ibm/mas_devops/roles/db2/templates/backup/download_from_cos.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/download_from_cos.sh.j2 new file mode 100644 index 0000000000..7e692ea9e6 --- /dev/null +++ b/ibm/mas_devops/roles/db2/templates/backup/download_from_cos.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash + +### COS_SOURCE_PATH=DB2REMOTE://backup_s3_alias/backup_s3_bucket/backups-db2/db2_backup_version +COS_SOURCE_PATH={{ cos_source_path }} +DB2_BACKUP_VERSION={{ db2_backup_version }} +TARGET_PATH=/mnt/backup/backups-db2/$DB2_BACKUP_VERSION + +instance=`whoami` + +# Find the home directory +instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` +DOWNLOAD_LOG=$instance_home/bin/download_LOG.out + +db2RemStgManager ALIAS GET source=${COS_SOURCE_PATH}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded 2>&1 | tee -a ${DOWNLOAD_LOG} + +rc=$? +if [ $rc -ne 0 ]; then + echo "FAILED_DOWNLOAD_BACKUP db2RemStgManager ALIAS GET source=${COS_SOURCE_PATH}/${ibm_cos_path} target=$download_path/$file_name_to_be_downloaded rc=$rc" >> ${DOWNLOAD_BACK_LOG} + echo "status=fail" >> ${DOWNLOAD_BACK_LOG} + exit 1 +fi \ No newline at end of file diff --git a/ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 index 10200eb03e..000ca14a93 100644 --- a/ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 +++ b/ibm/mas_devops/roles/db2/templates/backup/setup_cos_storage_access.sh.j2 @@ -1,8 +1,8 @@ #!/bin/bash -if db2 list storage access | grep {{ backup_s3_name }}; then - echo "{{ backup_s3_name }} is available already." +if db2 list storage access | grep {{ backup_s3_alias }}; then + echo "{{ backup_s3_alias }} is available already." else - echo "{{ backup_s3_name }} is not available. Creating" - db2 catalog storage access alias {{ backup_s3_name }} VENDOR S3 server {{ backup_s3_endpoint }} user {{ backup_s3_access_key }} password {{ backup_s3_secret_key }} container {{ backup_s3_bucket }} + echo "{{ backup_s3_alias }} is not available. Creating" + db2 catalog storage access alias {{ backup_s3_alias }} VENDOR S3 server {{ backup_s3_endpoint }} user {{ backup_s3_access_key }} password {{ backup_s3_secret_key }} container {{ backup_s3_bucket }} fi \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/uninstall.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/uninstall.yml index 7812423b57..c5bf07237b 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/aws/uninstall.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/aws/uninstall.yml @@ -108,7 +108,7 @@ sg_group_id: "{{sg_info.stdout | from_json | json_query('SecurityGroups[0].GroupId')}}" - name: Delete Security Group for DocDb Instance - when: sg_group_id is defined and sg_group_id != '' + when: sg_group_id is defined and sg_group_id != '' and sg_group_id != None command: > aws ec2 delete-security-group \ --group-id '{{ sg_group_id }}' From c29f0fa7c1754036169d21dd38d7760354360dec Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 11:16:55 +0000 Subject: [PATCH 06/61] [patch] ignore sessions mongo collection in monitor mongo database --- .../community/backup-restore/database-backup.sh.j2 | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 index 8b6e20ab55..cbd44ffb9f 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 @@ -57,9 +57,20 @@ echo "Databases to back up: $databases" IFS=',' read -r -a db_array <<< "$databases" # Perform backup for each database + +# Excluding sessions collection for Monitor database since it is not cleaned up automatically +# leading to very large backup sizes and long backup times. This is acceptable as +# sessions are transient and can be recreated. +# Refer DT425304 - MASCORE-5808 + for db_name in "${db_array[@]}"; do echo "Backing up database: $db_name" - $mongodump_cmd --db=$db_name --out=$TMP_BACKUP_DIR + # ignore sessions collection for monitor database + if [[ "$db_name" == *_monitor ]]; then + $mongodump_cmd --db=$db_name --excludeCollection=sessions --out=$TMP_BACKUP_DIR + else + $mongodump_cmd --db=$db_name --out=$TMP_BACKUP_DIR + fi cmdrc=$? if [ $cmdrc -ne 0 ]; then echo "Error backing up database $db_name. Exiting." From aa33406ebe1c17841e36e35d176c7cedda659fc1 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 16:42:31 +0000 Subject: [PATCH 07/61] [patch] mongo b&r tweaks (#2052) Co-authored-by: Sanjay Prabhakar --- docs/playbooks/backup-restore.md | 2 +- ibm/mas_devops/playbooks/br_db2.yml | 25 +- ibm/mas_devops/playbooks/br_mongodb.yml | 19 +- .../db2/tasks/backup/backup-database.yml | 9 +- .../db2/templates/backup/db2_backup.sh.j2 | 10 +- ibm/mas_devops/roles/mongodb/README.md | 340 ++++++++++++++++++ .../roles/mongodb/defaults/main.yml | 2 +- .../backup-restore/backup-database.yml | 127 ++++--- .../backup-restore/database-backup.sh.j2 | 9 +- .../backup-restore/database-restore.sh.j2 | 10 +- .../backup-restore/restore-database.yml | 44 ++- 11 files changed, 470 insertions(+), 127 deletions(-) diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 3bd3fefbf6..89cd75f872 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -94,7 +94,7 @@ The restore operation also consists of two phases, with behavior determined by t | `MAS_INSTANCE_ID` | Yes | N/A | Identifies the MAS instance whose MongoDB databases should be backed up or restored. To back up multiple MAS instances that share the same MongoDB CE instance, run the playbook multiple times with different values. | | `MONGODB_NAMESPACE` | No | `mongoce` | Namespace where MongoDB Community Edition is installed. Set this if MongoDB CE is deployed in a custom namespace. | | `MONGODB_INSTANCE_NAME` | No | `mas-mongo-ce` | Name of the MongoDB Community Edition instance. For backup, this value is used to locate the instance. For restore, the value is taken from the backup data. | -| `BR_SKIP_INSTANCE` | No | `false` | Skips MongoDB instance backup or restore. Set to `true` to back up or restore **databases only**. | +| `BR_SKIP_INSTANCE` | No | `true` | Skips MongoDB instance backup or restore. Set to `false` to back up or restore instances. | ### Examples diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index 500f71f075..ed97768248 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -7,8 +7,8 @@ mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - db2_action: "{{ lookup('env', 'MASBR_ACTION') }}" # backup or restore - db2_backup_version: "{{ lookup('env', 'MASBR_BACKUP_VERSION') | default('None', true)}}" # Required for restore action + db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or restore + db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" # Required for restore action br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false to enable DB2 instance backup/restore backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline and required for backup backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) }}" @@ -20,10 +20,10 @@ pre_tasks: - - name: "Fail if MASBR_ACTION is not set to backup|restore" + - name: "Fail if DB2_ACTION is not set to backup|restore" assert: that: db2_action in ["backup", "restore"] - fail_msg: "MASBR_ACTION is required and must be set to 'backup' or 'restore'" + fail_msg: "DB2_ACTION is required and must be set to 'backup' or 'restore'" - name: "Fail if BACKUP_VENDOR is not set to s3|disk" assert: @@ -36,14 +36,15 @@ fail_msg: "DB2_BACKUP_TYPE is required and must be set to 'online' or 'offline'" - name: "Fail if S3 variables are not set when backup_vendor is s3" - assert: - that: - - backup_s3_endpoint is defined and backup_s3_endpoint != "" - - backup_s3_bucket is defined and backup_s3_bucket != "" - - backup_s3_access_key is defined and backup_s3_access_key != "" - - backup_s3_secret_key is defined and backup_s3_secret_key != "" - fail_msg: "BACKUP_S3_ENDPOINT, BACKUP_S3_BUCKET, BACKUP_S3_ACCESS_KEY and BACKUP_S3_SECRET_KEY are required when BACKUP_VENDOR is s3" - when: backup_vendor == "s3" + ibm.mas_devops.verify_backup_restore_vars: + backup_vendor: "{{ backup_vendor }}" + backup_s3_alias: "{{ backup_s3_alias }}" + backup_s3_endpoint: "{{ backup_s3_endpoint }}" + backup_s3_bucket: "{{ backup_s3_bucket }}" + backup_s3_access_key: "{{ backup_s3_access_key }}" + backup_s3_secret_key: "{{ backup_s3_secret_key }}" + action: "s3_setup" + component: "db2" roles: - role: ibm.mas_devops.db2 diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index fe20acdc86..e6397cc774 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -7,29 +7,26 @@ mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" mongodb_namespace: "{{ lookup('env', 'MONGODB_NAMESPACE') | default('mongoce', true) }}" - masbr_action: "{{ lookup('env', 'MASBR_ACTION') | default('backup', true) }}" # backup or restore + mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('backup', true) }}" # backup or restore mongodb_provider: "community" # only community is supported currently mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Optional - masbr_backup_version: "{{ lookup('env', 'MASBR_BACKUP_VERSION') | default('None', true)}}" # Required for restore action - br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" # set to True to skip mongodb instance backup/restore + mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') | default('None', true)}}" # Required for restore action + br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false for mongodb instance backup/restore pre_tasks: - - name: "Fail if masbr_action is not set to backup|restore" + - name: "Fail if mongodb_action is not set to backup|restore" assert: - that: masbr_action in ["backup", "restore"] - fail_msg: "masbr_action is required and must be set to 'backup' or 'restore'" + that: mongodb_action in ["backup", "restore"] + fail_msg: "mongodb_action is required and must be set to 'backup' or 'restore'" - name: "Fail if require variables for Mongodb backup/restore are not provided" ibm.mas_devops.verify_backup_restore_vars: mas_instance_id: "{{ mas_instance_id }}" mas_backup_dir: "{{ mas_backup_dir }}" mongodb_instance_name: "{{ mongodb_instance_name }}" - mongodb_backup_version: "{{ masbr_backup_version }}" - action: "{{ masbr_action }}" + mongodb_backup_version: "{{ mongodb_backup_version }}" + action: "{{ mongodb_action }}" component: "mongodb" roles: - role: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ masbr_action }}" - mongodb_backup_version: "{{ masbr_backup_version }}" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml index ba9b2425d0..9d1893ab9e 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml @@ -45,7 +45,8 @@ - name: "Set fact backup_path for the backup script when backup vendor is disk" set_fact: - full_backup_path: "/mnt/backup/{{ db2_backup_version }}" + full_backup_path: "/mnt/backup/{{ db2_backup_version }}/data" + base_backup_path: "/mnt/backup/{{ db2_backup_version }}" when: backup_vendor == "disk" # Check if db2 instance is running and get the pod name @@ -175,7 +176,7 @@ # ----------------------------------------------------------------------------- - name: "Executing DB2 backup.." block: - - name: "Execute db2_backup.sh in DB2 pod" + - name: "Execute db2_backup.sh in DB2 pod, check logs in /tmp/db2_backup.log in pod" shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc '{{ db2_insthome }}/bin/db2_backup.sh | tee /tmp/db2_backup.log' db2inst1" register: db2_backup_result @@ -202,7 +203,7 @@ # If vendor is disk, tar the backup files and cp to backup_data_path # ----------------------------------------------------------------------------- - name: "Move backup files to {{ db2_backup_data_path }} when backup vendor is disk.. This will take a while..." - shell: "oc cp --retries=50 -c db2u {{ db2_namespace }}/{{ db2_pod_name }}:{{ full_backup_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz {{ db2_backup_data_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" + shell: "oc cp --retries=50 -c db2u {{ db2_namespace }}/{{ db2_pod_name }}:{{ base_backup_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz {{ db2_backup_data_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" register: rsync_result when: backup_vendor == "disk" @@ -213,7 +214,7 @@ always: - name: "Clean up backup directory in the pod" - shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'rm -rf {{ full_backup_path }}' db2inst1" + shell: "oc exec -n {{ db2_namespace }} {{ db2_pod_name }} -c db2u -- su -lc 'rm -rf {{ base_backup_path }}' db2inst1" when: backup_vendor == "disk" diff --git a/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 index b24c6b833a..13e69479b4 100644 --- a/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 +++ b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 @@ -125,13 +125,17 @@ DATABASE={{ db2_dbname }} BACKUP_TYPE={{ backup_type }} VENDOR={{ backup_vendor }} ### for S3, BACKUP_PATH=DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }} +### for disk, BACKUP_PATH=/mnt/backup/{{ db2_backup_version }}/data BACKUP_PATH={{ full_backup_path }} +DB2_BACKUP_VERSION={{ db2_backup_version }} + +### only for disk, DISK_BACKUP_BASE=/mnt/backup/{{ db2_backup_version }} where tar is stored +DISK_BACKUP_BASE={{ base_backup_path }} # Finding the db2 Instance owner INSTOWNER=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | awk -F ',' '{print $4}' ` instance=`whoami` -BACKUP_BASE=/mnt/backup # Find the home directory instance_home=`/usr/local/bin/db2greg -dump | grep -ae "I," | grep -v "/das," | grep "${instance}" | awk -F ',' '{print $5}'| cut -d/ -f 1,2,3,4,5` @@ -304,11 +308,11 @@ if [ ${VENDOR} = 'disk' ] ; then cp ${YAML_FILE} ${BACKUP_PATH}/db2-backup-info.yaml echo "" echo "INFO: Creating tar archive of backup files..." - tar -czf ${BACKUP_PATH}/db2-${DATABASE}-backup-{{ db2_backup_version }}.tar.gz -C ${BACKUP_PATH} . + tar -czf ${DISK_BACKUP_BASE}/db2-${DATABASE}-backup-${DB2_BACKUP_VERSION}.tar.gz -C ${BACKUP_PATH} . echo "INFO: Tar archive created successfully" echo "" echo "Disk Usage:" - df -h ${BACKUP_PATH}/* + du -h ${BACKUP_PATH}/* else echo "" echo "INFO: Backup vendor is not disk. Skipping tar archive creation." diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index 3eccd5c389..f68d9ea892 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -178,6 +178,9 @@ Set this to `true` to confirm you want to upgrade your existing Mongo instance f - Environment Variable: `MONGODB_V8_UPGRADE` - Default Value: `false` +Role Variables - Backup and Restore +------------------------------------------------------------------------------- + ### mas_backup_dir Parent directory to store backups. Each component will create its own directory in this parent directory. @@ -200,6 +203,343 @@ Set this to `true` to skip the instance level or kubernetes resources backup/res - Environment Variable: `BR_SKIP_INSTANCE` - Default Value: `false` +MongoDB Community Edition Backup and Restore +------------------------------------------------------------------------------- + +### Overview + +The MongoDB role supports comprehensive backup and restore operations for MongoDB Community Edition deployments. The process is designed to handle both cluster-level Kubernetes resources and database-level data, providing flexibility for different recovery scenarios. + +### Backup and Restore Architecture + +The backup and restore operations are split into two independent phases: + +#### 1. Instance-Level Backup/Restore (Optional) +Captures and restores Kubernetes resources required to recreate the MongoDB cluster: +- MongoDBCommunity custom resource (CR) +- Configuration parameters (replicas, storage, CPU/memory settings) +- User credentials and authentication settings +- TLS certificates and secrets +- Issuers and certificate resources + +#### 2. Database-Level Backup/Restore (Always Executed) +Performs backup and restore of MongoDB databases: +- Uses `mongodump` and `mongorestore` utilities +- Backs up only databases matching the MAS instance ID pattern +- Excludes transient collections (e.g., Monitor sessions) +- Excludes system collections that should be regenerated by MAS + +### Backup Process + +#### Prerequisites +- MongoDB Community Edition instance must be running +- Sufficient storage space in the backup directory +- OpenShift CLI (`oc`) access to the cluster +- Appropriate permissions to access MongoDB namespace and resources + +#### Backup Workflow + +**Phase 1: Instance Backup** (if `BR_SKIP_INSTANCE=false`) +1. Retrieves the MongoDBCommunity CR and related resources +2. Backs up secrets (admin credentials, TLS certificates) +3. Backs up issuers and certificates +4. Saves all resources to `/backup--mongoce/resources/` + +**Phase 2: Database Backup** (always executed) +1. Connects to the MongoDB primary node +2. Creates a backup role and user (`bradmin`) with full privileges +3. Identifies databases matching pattern: `^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)` +4. Executes `mongodump` for each database +5. Excludes `sessions` collection from Monitor databases +6. Creates compressed archive: `mongodump-.tar.gz` +7. Saves backup metadata in `mongodb-info.yml` +8. Cleans up temporary files from MongoDB pod + +#### Backup Directory Structure +``` +/ +└── backup--mongoce/ + ├── resources/ + │ ├── cr.yml # MongoDBCommunity CR + │ ├── secrets/ # User and TLS secrets + │ ├── issuers/ # Certificate issuers + │ └── certificates/ # TLS certificates + └── data/ + ├── mongodump-.tar.gz # Database backup archive + └── mongodb-info.yml # Backup metadata +``` + +#### Backup Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `MONGODB_ACTION` | Yes | `install` | Must be set to `backup` | +| `MAS_INSTANCE_ID` | Yes | None | MAS instance ID whose databases to backup | +| `MAS_BACKUP_DIR` | Yes | None | Parent directory for backup storage | +| `MONGODB_NAMESPACE` | No | `mongoce` | Namespace where MongoDB is installed | +| `MONGODB_INSTANCE_NAME` | No | `mas-mongo-ce` | Name of the MongoDB instance | +| `MONGODB_BACKUP_VERSION` | No | `YYMMDD-HHMMSS` | Custom version identifier for the backup | +| `BR_SKIP_INSTANCE` | No | `false` | Skip instance-level backup if `true` | + +#### Backup Example +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mongodb_action: backup + mas_instance_id: masinst1 + mas_backup_dir: /mnt/backups + mongodb_namespace: mongoce + mongodb_instance_name: mas-mongo-ce + # Optional: mongodb_backup_version: "20260109-120000" + # Optional: br_skip_instance: false + roles: + - ibm.mas_devops.mongodb +``` + +### Restore Process + +#### Prerequisites +- Valid backup directory with complete backup data +- OpenShift CLI (`oc`) access to the cluster +- Appropriate permissions to create/modify resources +- Target namespace must exist +- Storage class from backup must be available (or default storage class) + +#### Restore Workflow + +**Phase 1: Instance Restore** (conditional) +- **If MongoDB instance does NOT exist or is NOT running:** + 1. Reads MongoDBCommunity CR from backup + 2. Determines appropriate storage class to use + 3. Restores secrets (admin credentials, TLS certificates) + 4. Restores issuers and certificates + 5. Installs MongoDB operator and creates instance + 6. Waits for MongoDB cluster to become ready + +- **If MongoDB instance already exists and is running:** + - Instance restore is automatically skipped + - Proceeds directly to database restore + +**Phase 2: Database Restore** (always executed if instance is running) +1. Validates backup data files exist +2. Verifies MongoDB version compatibility +3. Creates restore role and user +4. Copies backup archive to MongoDB pod +5. Extracts backup data +6. Executes `mongorestore` with exclusions: + - `mas__adoptionusage.*` + - `mas__catalog.*` + - `mas__core.OauthClient` + - `mas__core.OauthToken` + - `mas__core.bindings` + - `mas__core.graphiteconfigtool.*` + - `mas__core.workspaces` +7. Cleans up temporary files + +#### Restore Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `MONGODB_ACTION` | Yes | `install` | Must be set to `restore` | +| `MAS_INSTANCE_ID` | Yes | None | MAS instance ID to restore | +| `MAS_BACKUP_DIR` | Yes | None | Parent directory containing backups | +| `MONGODB_BACKUP_VERSION` | Yes | None | Version identifier of backup to restore | +| `MONGODB_NAMESPACE` | No | `mongoce` | Target namespace for restore | +| `MONGODB_INSTANCE_NAME` | No | `mas-mongo-ce` | Name for the MongoDB instance | +| `BR_SKIP_INSTANCE` | No | `false` | Skip instance-level restore if `true` | + +#### Restore Example +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mongodb_action: restore + mas_instance_id: masinst1 + mas_backup_dir: /mnt/backups + mongodb_backup_version: "260109-120000" + mongodb_namespace: mongoce + mongodb_instance_name: mas-mongo-ce + # Optional: br_skip_instance: false + roles: + - ibm.mas_devops.mongodb +``` + +### Backup and Restore Scenarios + +#### Scenario 1: Full Backup and Restore +Complete backup of instance and databases, restore to new environment. +```bash +# Backup +export MONGODB_ACTION=backup +export MAS_INSTANCE_ID=prod1 +export MAS_BACKUP_DIR=/backups +ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role + +# Restore (to new cluster) +export MONGODB_ACTION=restore +export MAS_INSTANCE_ID=prod1 +export MAS_BACKUP_DIR=/backups +export MONGODB_BACKUP_VERSION=260109-120000 +ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role +``` + +#### Scenario 2: Database-Only Backup and Restore +Backup only databases, restore to existing MongoDB instance. +```bash +# Backup (skip instance) +export MONGODB_ACTION=backup +export MAS_INSTANCE_ID=prod1 +export MAS_BACKUP_DIR=/backups +export BR_SKIP_INSTANCE=true +ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role + +# Restore (to existing instance) +export MONGODB_ACTION=restore +export MAS_INSTANCE_ID=prod1 +export MAS_BACKUP_DIR=/backups +export MONGODB_BACKUP_VERSION=260109-120000 +export BR_SKIP_INSTANCE=true +ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role +``` + +#### Scenario 3: Disaster Recovery +Full restore including instance recreation. +```bash +export MONGODB_ACTION=restore +export MAS_INSTANCE_ID=prod1 +export MAS_BACKUP_DIR=/backups +export MONGODB_BACKUP_VERSION=260109-120000 +export MONGODB_NAMESPACE=mongoce +ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role +``` + +### Important Considerations + +#### Version Compatibility +- Target MongoDB version must match the backup MongoDB version +- The restore process validates version compatibility before proceeding +- Cross-version restores are not supported + +#### Storage Class Handling +The restore process determines storage class in the following order: +1. If existing MongoDB instance found: uses its storage class +2. If no instance exists: uses storage class from backup CR +3. If backup storage class doesn't exist: uses cluster default storage class + +#### Database Filtering +- Only databases matching the MAS instance ID pattern are backed up +- Pattern: `^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)` +- Examples: `mas_prod1_core`, `iot-prod1-data` + +#### Excluded Collections +The following collections are excluded from restore as they should be regenerated: +- Adoption usage data +- Catalog data +- OAuth tokens and clients +- Core bindings and workspaces +- Graphite configuration + +#### Security Considerations +- Backup files contain sensitive data including credentials +- Secure the backup directory with appropriate permissions +- Use encryption for backup storage and transmission +- Rotate backup credentials regularly + +### Troubleshooting + +#### Backup Failures + +**Issue: Cannot connect to MongoDB primary** +``` +Error checking if node is primary. Exiting. +``` +**Solution:** Verify MongoDB cluster is healthy and all pods are running. + +**Issue: Insufficient disk space** +``` +Error creating tar.gz archive of backup files. +``` +**Solution:** Ensure adequate space in backup directory and MongoDB pod. + +**Issue: Permission denied copying files** +``` +Error copying backup files from mongo pod +``` +**Solution:** Verify OpenShift permissions and pod access. + +#### Restore Failures + +**Issue: Version mismatch** +``` +Target MongoCE version X does NOT match backup MongoCE version Y +``` +**Solution:** Ensure target MongoDB version matches backup version. + +**Issue: Backup files not found** +``` +Required backup data files are missing +``` +**Solution:** Verify backup directory path and version are correct. + +**Issue: Storage class not available** +``` +Storage class from backup does not exist +``` +**Solution:** Create the storage class or let restore use default. + +### Best Practices + +1. **Regular Backups**: Schedule automated backups using cron or OpenShift CronJobs +2. **Backup Retention**: Implement a retention policy to manage backup storage +3. **Test Restores**: Regularly test restore procedures in non-production environments +4. **Monitor Backup Size**: Track backup sizes to anticipate storage requirements +5. **Document Procedures**: Maintain runbooks for backup and restore operations +6. **Secure Backups**: Encrypt backups at rest and in transit +7. **Verify Backups**: Validate backup integrity after completion +8. **Multiple Instances**: For shared MongoDB, backup each MAS instance separately + +### Backup Automation Example + +```yaml +# OpenShift CronJob for automated MongoDB backups +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mongodb-backup + namespace: mongoce +spec: + schedule: "0 2 * * *" # Daily at 2 AM + jobTemplate: + spec: + template: + spec: + serviceAccountName: mongodb-backup-sa + containers: + - name: backup + image: quay.io/ibmmas/ansible-devops:latest + env: + - name: MONGODB_ACTION + value: "backup" + - name: MAS_INSTANCE_ID + value: "prod1" + - name: MAS_BACKUP_DIR + value: "/backups" + - name: BR_SKIP_INSTANCE + value: "false" + volumeMounts: + - name: backup-storage + mountPath: /backups + command: + - ansible-playbook + - ibm.mas_devops.run_role + volumes: + - name: backup-storage + persistentVolumeClaim: + claimName: mongodb-backup-pvc + restartPolicy: OnFailure +``` + Role Variables - IBM Cloud ------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index ba2c3c34eb..29e2fb4b01 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -122,7 +122,7 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # ----------------------------------------------------------------------------- mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" -br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" +br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # Mongo upgrade flags # If identified that there's an existing Mongo that might lead to a v5 or v6 upgrade diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index 42f250a80e..c72739d837 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -37,80 +37,77 @@ # Prepare shell scripts and transfer to mongo pod # ----------------------------------------------------------------------------- -- block: - - name: Create create-role-user.sh script in local /tmp - ansible.builtin.template: - src: create-role-user.sh.j2 - dest: /tmp/create-role-user.sh - mode: '777' - - - name: Create database-backup.sh script in local /tmp - ansible.builtin.template: - src: database-backup.sh.j2 - dest: /tmp/database-backup.sh - mode: '777' - - - name: Copy the create-role-user.sh script into the mongodb pod - shell: "oc cp /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" - register: copy_result - retries: 2 - delay: 15 # seconds - until: copy_result.rc == 0 - - - name: Copy the database-backup.sh script into the mongodb pod - shell: "oc cp /tmp/database-backup.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-backup.sh -c mongod" - register: copy_result - retries: 2 - delay: 15 # seconds - until: copy_result.rc == 0 +- name: Create create-role-user.sh script in local /tmp + ansible.builtin.template: + src: create-role-user.sh.j2 + dest: /tmp/create-role-user.sh + mode: '777' + +- name: Create database-backup.sh script in local /tmp + ansible.builtin.template: + src: database-backup.sh.j2 + dest: /tmp/database-backup.sh + mode: '777' + +- name: Copy the create-role-user.sh script into the mongodb pod + shell: "oc cp --retries=50 /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" + register: copy_result + +- name: Copy the database-backup.sh script into the mongodb pod + shell: "oc cp --retries=50 /tmp/database-backup.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-backup.sh -c mongod" + register: copy_result # The log file will also be available inside the pod /tmp/create-role-user.log - name: Exec into mongo pod and run create-role-user.sh to setup backup role and user in Mongodb. shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/create-role-user.sh + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash /tmp/create-role-user.sh register: create_roleuser_output - ignore_errors: true - retries: 1 - delay: 10 # seconds - until: - - create_roleuser_output.rc == 0 - - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) + +- name: "Assert Create role user" + assert: + that: + - create_roleuser_output.rc == 0 + - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) + fail_msg: "Failed to create role and user for restore" - name: "Debug create-role-user logs" debug: msg: "{{ create_roleuser_output.stdout_lines }}" -- name: Exec into mongo pod and run database-backup.sh to backup databases. - shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/database-backup.sh - register: database_backup_output - ignore_errors: true - retries: 1 - delay: 10 # seconds - until: - - database_backup_output.rc == 0 - - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) - -- name: "Debug database-backup logs" - debug: - msg: "{{ database_backup_output.stdout_lines }}" +- name: "Starting Mongo database backup" + block: + - name: "Running backup script in pod. Check logs in /tmp/database-backup.log in pod" + shell: | + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash /tmp/database-backup.sh | tee /tmp/database-backup.log + register: database_backup_output + + - name: "Assert Database backup" + assert: + that: + - database_backup_output.rc == 0 + - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) + fail_msg: "Failed to backup databases" + + - name: "Debug database-backup logs" + debug: + msg: "{{ database_backup_output.stdout_lines }}" + + - name: "copy backup files from mongo pod to local backup directory" + shell: "oc cp --retries=50 -c mongod {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/masbr/{{ mongodb_backup_version }}/{{ mongodb_backup_data_filename }} {{ mongodb_backup_data_path }}/{{ mongodb_backup_data_filename }}" + register: copy_result + + - name: "Create yaml file with mongodb backup details" + copy: + dest: "{{ mongodb_backup_data_path }}/mongodb-info.yml" + content: | + source_mongodb_backup_version: "{{ mongodb_backup_version }}" + source_mongodb_backup_data_filename: "{{ mongodb_backup_data_filename }}" + source_mongodb_version: "{{ mongodb_version }}" + + always: + - name: Cleanup backup files from mongo pod + shell: | + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- rm -rf /tmp/masbr/{{ mongodb_backup_version }} + register: database_backup_cleanup_output -- name: "copy backup files from mongo pod to local backup directory" - shell: "oc cp --retries=50 -c mongod {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/masbr/{{ mongodb_backup_version }}/{{ mongodb_backup_data_filename }} {{ mongodb_backup_data_path }}/{{ mongodb_backup_data_filename }}" - register: copy_result - retries: 2 - delay: 10 # seconds - until: copy_result.rc == 0 - when: - - database_backup_output is defined - - database_backup_output.rc == 0 - - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) - -- name: "Create yaml file with mongodb backup details" - copy: - dest: "{{ mongodb_backup_data_path }}/mongodb-info.yml" - content: | - source_mongodb_backup_version: "{{ mongodb_backup_version }}" - source_mongodb_backup_data_filename: "{{ mongodb_backup_data_filename }}" - source_mongodb_version: "{{ mongodb_version }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 index cbd44ffb9f..a745968a76 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 @@ -3,6 +3,7 @@ MONGODB_BACKUP_USER="bradmin" TMP_BACKUP_DIR="/tmp/masbr/{{ mongodb_backup_version }}" +TMP_BACKUP_MONGODUMP_DIR="${TMP_BACKUP_DIR}/mongodump" ALL_DATABASES_FILTER="^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)" @@ -67,9 +68,9 @@ for db_name in "${db_array[@]}"; do echo "Backing up database: $db_name" # ignore sessions collection for monitor database if [[ "$db_name" == *_monitor ]]; then - $mongodump_cmd --db=$db_name --excludeCollection=sessions --out=$TMP_BACKUP_DIR + $mongodump_cmd --db=$db_name --excludeCollection=sessions --out=$TMP_BACKUP_MONGODUMP_DIR else - $mongodump_cmd --db=$db_name --out=$TMP_BACKUP_DIR + $mongodump_cmd --db=$db_name --out=$TMP_BACKUP_MONGODUMP_DIR fi cmdrc=$? if [ $cmdrc -ne 0 ]; then @@ -77,10 +78,10 @@ for db_name in "${db_array[@]}"; do exit 1 fi done -echo "Backup completed successfully. Backup files are located in $TMP_BACKUP_DIR." +echo "Backup completed successfully. Backup files are located in $TMP_BACKUP_MONGODUMP_DIR." # Create tar.gz archives of database backup files -tar -czf $TMP_BACKUP_DIR/{{ mongodb_backup_data_filename }} -C $TMP_BACKUP_DIR . +tar -czf $TMP_BACKUP_DIR/{{ mongodb_backup_data_filename }} -C $TMP_BACKUP_MONGODUMP_DIR . if [ $? -ne 0 ]; then echo "Error creating tar.gz archive of backup files. Exiting." exit 1 diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 index 1003efd775..1ee950d44d 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 @@ -63,8 +63,14 @@ fi # Perform restore echo "Restoring databases from $TMP_RESTORE_DIR" $mongorestore_cmd -cmdrc=$? -if [ $cmdrc -ne 0 ]; then +restore_rc=$? + +# Clean up temporary dir +echo "Cleaning up temporary dir" +rm -rf $TMP_RESTORE_DIR + + +if [ $restore_rc -ne 0 ]; then echo "Error restoring databases. Exiting." exit 1 fi diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 77c6095823..c90da7dfe7 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -76,14 +76,14 @@ mode: '777' - name: Copy the create-role-user.sh script into the mongodb pod - shell: "oc cp /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" + shell: "oc cp --retries=50 /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" register: copy_result retries: 2 delay: 15 # seconds until: copy_result.rc == 0 - name: Copy the database-restore.sh script into the mongodb pod - shell: "oc cp /tmp/database-restore.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-restore.sh -c mongod" + shell: "oc cp --retries=50 /tmp/database-restore.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-restore.sh -c mongod" register: copy_result retries: 2 delay: 15 # seconds @@ -94,44 +94,40 @@ shell: | oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/create-role-user.sh register: create_roleuser_output - ignore_errors: true - retries: 1 - delay: 10 # seconds - until: - - create_roleuser_output.rc == 0 - - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) - name: "Debug create-role-user logs" debug: msg: "{{ create_roleuser_output.stdout_lines }}" +- name: "Assert Create role user" + assert: + that: + - create_roleuser_output.rc == 0 + - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) + fail_msg: "Failed to create role and user for restore" + # Create temporary restore directory - name: "Create temporary restore directory in mongodb pod and clean up any existing files" shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- mkdir -p {{ mongodb_tmp_restore_dir }} && rm -rf {{ mongodb_tmp_restore_dir }}/* + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- rm -rf {{ mongodb_tmp_restore_dir }} && mkdir -p {{ mongodb_tmp_restore_dir }} register: create_tmpdir_output - retries: 2 - delay: 15 # seconds - until: create_tmpdir_output.rc == 0 - name: "Copy backup data file into mongodb pod" shell: | - oc cp {{ mongodb_data_path }}/{{ mongodb_backup_data_filename }} {{ mongodb_namespace }}/{{ mongoce_pod_name }}:{{ mongodb_tmp_restore_dir }}/{{ mongodb_backup_data_filename }} -c mongod + oc cp --retries=50 {{ mongodb_data_path }}/{{ mongodb_backup_data_filename }} {{ mongodb_namespace }}/{{ mongoce_pod_name }}:{{ mongodb_tmp_restore_dir }}/{{ mongodb_backup_data_filename }} -c mongod register: copy_backupdata_output - retries: 2 - delay: 15 # seconds - until: copy_backupdata_output.rc == 0 -- name: "Exec into mongo pod and run database-restore.sh to restore the database from backup." +- name: "Running restore script in pod, check logs in /tmp/database-restore.log in pod" shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -- bash /tmp/database-restore.sh + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash /tmp/database-restore.sh | tee /tmp/database-restore.log register: database_restore_output - ignore_errors: true - retries: 1 - delay: 10 # seconds - until: - - database_restore_output.rc == 0 - - database_restore_output.stdout | regex_search('DATABASERESTOREstatus-SUCCESS', multiline=True) + +- name: Assert database restore + assert: + that: + - database_restore_output.rc == 0 + - database_restore_output.stdout | regex_search('DATABASERESTOREstatus-SUCCESS', multiline=True) + fail_msg: "Failed to restore database from backup" - name: "Debug database-restore logs" debug: From df46365153c6d73536dcd326c27f2cf7f916eefc Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 17:05:24 +0000 Subject: [PATCH 08/61] [patch] fix mongodb-info-yaml --- ibm/mas_devops/roles/mongodb/README.md | 4 ++-- .../providers/community/backup-restore/backup-database.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index f68d9ea892..f182139165 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -252,7 +252,7 @@ Performs backup and restore of MongoDB databases: 4. Executes `mongodump` for each database 5. Excludes `sessions` collection from Monitor databases 6. Creates compressed archive: `mongodump-.tar.gz` -7. Saves backup metadata in `mongodb-info.yml` +7. Saves backup metadata in `mongodb-info.yaml` 8. Cleans up temporary files from MongoDB pod #### Backup Directory Structure @@ -266,7 +266,7 @@ Performs backup and restore of MongoDB databases: │ └── certificates/ # TLS certificates └── data/ ├── mongodump-.tar.gz # Database backup archive - └── mongodb-info.yml # Backup metadata + └── mongodb-info.yaml # Backup metadata ``` #### Backup Variables diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index c72739d837..cdb7ba2807 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -98,7 +98,7 @@ - name: "Create yaml file with mongodb backup details" copy: - dest: "{{ mongodb_backup_data_path }}/mongodb-info.yml" + dest: "{{ mongodb_backup_data_path }}/mongodb-info.yaml" content: | source_mongodb_backup_version: "{{ mongodb_backup_version }}" source_mongodb_backup_data_filename: "{{ mongodb_backup_data_filename }}" From 4508ac2f2046344b3996e65120a5d25e59adbcfd Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 17:47:04 +0000 Subject: [PATCH 09/61] [patch] fix mongodb restore version check --- .../providers/community/backup-restore/restore-database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index c90da7dfe7..287a3269ca 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -48,7 +48,7 @@ - name: "Assert if target mongoce version match backup mongoce version" assert: that: - - mongodb_version == mongodb_backup_version + - mongodb_version == mongodb_backup_info.source_mongodb_version fail_msg: "Target MongoCE version {{ mongodb_version }} does NOT match backup MongoCE version {{ mongodb_backup_info.source_mongodb_version }}. Restore cannot proceed." - name: "debug facts" From bede5502dcd6077e52028996cb6f34a8a3444868 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 18:14:15 +0000 Subject: [PATCH 10/61] [patch] fix temp dir command --- .../providers/community/backup-restore/restore-database.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 287a3269ca..2861b9fb9e 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -108,8 +108,7 @@ # Create temporary restore directory - name: "Create temporary restore directory in mongodb pod and clean up any existing files" - shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- rm -rf {{ mongodb_tmp_restore_dir }} && mkdir -p {{ mongodb_tmp_restore_dir }} + shell: oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash -c 'rm -rf {{ mongodb_tmp_restore_dir }}; mkdir -p {{ mongodb_tmp_restore_dir }}' register: create_tmpdir_output - name: "Copy backup data file into mongodb pod" From 52128e45730934aa7a3eddb4aa092bd09494bc04 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 18:18:29 +0000 Subject: [PATCH 11/61] [patch] create temp mongo backup dir --- .../providers/community/backup-restore/database-backup.sh.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 index a745968a76..e6224ec856 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 @@ -41,9 +41,9 @@ fi # Create temporary directory for backup -mkdir -p $TMP_BACKUP_DIR +mkdir -p $TMP_BACKUP_MONGODUMP_DIR if [ $? -ne 0 ]; then - echo "Error creating temporary backup directory $TMP_BACKUP_DIR. Exiting." + echo "Error creating temporary backup directory $TMP_BACKUP_MONGODUMP_DIR. Exiting." exit 1 fi From a3293134504fb807b59f48df75a6db439856fc78 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 9 Jan 2026 18:33:29 +0000 Subject: [PATCH 12/61] [patch] tee logs --- .../providers/community/backup-restore/backup-database.yml | 2 +- .../providers/community/backup-restore/database-backup.sh.j2 | 4 +++- .../providers/community/backup-restore/restore-database.yml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index cdb7ba2807..664fcdc8f2 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -78,7 +78,7 @@ block: - name: "Running backup script in pod. Check logs in /tmp/database-backup.log in pod" shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash /tmp/database-backup.sh | tee /tmp/database-backup.log + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash -c '/tmp/database-backup.sh | tee /tmp/database-backup.log' register: database_backup_output - name: "Assert Database backup" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 index e6224ec856..9879920f1b 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 @@ -39,8 +39,10 @@ else mongodump_cmd="mongodump -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --sslCAFile $tlsCAFile --authenticationDatabase=admin --ssl --host {{ mongodb_host }}" fi -# Create temporary directory for backup +# cleanup base backup dir +rm -rf $TMP_BACKUP_DIR +# Create temporary directory for backup mkdir -p $TMP_BACKUP_MONGODUMP_DIR if [ $? -ne 0 ]; then echo "Error creating temporary backup directory $TMP_BACKUP_MONGODUMP_DIR. Exiting." diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 2861b9fb9e..0116599252 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -118,7 +118,7 @@ - name: "Running restore script in pod, check logs in /tmp/database-restore.log in pod" shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash /tmp/database-restore.sh | tee /tmp/database-restore.log + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash -c '/tmp/database-restore.sh | tee /tmp/database-restore.log' register: database_restore_output - name: Assert database restore From de66ab1408e5f25d92de90a352ae806333b5bb4c Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 12 Jan 2026 12:33:51 +0000 Subject: [PATCH 13/61] [patch] fix space --- .../providers/community/backup-restore/backup-database.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index 664fcdc8f2..967d19bf34 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -78,7 +78,7 @@ block: - name: "Running backup script in pod. Check logs in /tmp/database-backup.log in pod" shell: | - oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash -c '/tmp/database-backup.sh | tee /tmp/database-backup.log' + oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- bash -c '/tmp/database-backup.sh | tee /tmp/database-backup.log' register: database_backup_output - name: "Assert Database backup" From fb8b05291d7e1740cb5751194068394a005d92d2 Mon Sep 17 00:00:00 2001 From: Andrew Whitfield Date: Wed, 14 Jan 2026 14:44:42 +0000 Subject: [PATCH 14/61] [minor] Add the suite_backup logic (#2055) --- .../plugins/action/backup_resource.py | 187 ++++++++++++++++++ .../plugins/action/get_mas_cluster_issuer.py | 67 +++++++ ibm/mas_devops/roles/suite_backup/README.md | 24 ++- .../roles/suite_backup/defaults/main.yml | 3 + .../roles/suite_backup/tasks/main.yml | 172 +++++++++++++++- 5 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 ibm/mas_devops/plugins/action/backup_resource.py create mode 100644 ibm/mas_devops/plugins/action/get_mas_cluster_issuer.py diff --git a/ibm/mas_devops/plugins/action/backup_resource.py b/ibm/mas_devops/plugins/action/backup_resource.py new file mode 100644 index 0000000000..88c6b754cc --- /dev/null +++ b/ibm/mas_devops/plugins/action/backup_resource.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 + +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes import client, config +from kubernetes.dynamic import DynamicClient + +from mas.devops.backup import createBackupDirectories, backupResources + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +display = Display() + + +class ActionModule(ActionBase): + """ + Backup MAS Suite resources based on a list of resource definitions + + Usage Example + ------------- + tasks: + - name: "Backup MAS Suite resources" + ibm.mas_devops.backup_resource: + backup_resources: + - namespace: "mas-inst1-core" + resources: + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: ibm-mas-operator + - kind: Suite + api_version: core.mas.ibm.com/v1 + - kind: Workspace + api_version: core.mas.ibm.com/v1 + backup_path: "/backup/mas-suite" + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + # Load kubernetes configuration + try: + if host and api_key: + # Use provided host and api_key + configuration = client.Configuration() + configuration.host = host + configuration.api_key = {'authorization': f'Bearer {api_key}'} + configuration.verify_ssl = False + api_client = client.ApiClient(configuration) + else: + # Load from kubeconfig + config.load_kube_config() + api_client = client.ApiClient() + except Exception as e: + raise AnsibleError(f"Failed to initialize Kubernetes client: {e}") + + dynClient = DynamicClient(api_client) + + backup_resources = self._task.args.get('backup_resources') + backup_path = self._task.args.get('backup_path') + + if backup_resources is None or not isinstance(backup_resources, list): + raise AnsibleError(f"Error: backup_resources argument must be provided as a list") + if backup_path is None or backup_path == "": + raise AnsibleError(f"Error: backup_path argument was not provided") + + display.v(f"Starting backup of MAS Suite resources to '{backup_path}'") + + total_backed_up = 0 + total_failed = 0 + total_not_found = 0 + all_discovered_secrets = set() + secrets_by_namespace = {} # Track secrets per namespace + failed_resources = [] # Track failed resources with details + + # Process each namespace and its resources + for namespace_config in backup_resources: + namespace = namespace_config.get('namespace') + resources = namespace_config.get('resources', []) + + # Namespace can be blank for cluster-scoped resources + if namespace: + display.v(f"Processing namespace: {namespace}") + else: + display.v(f"Processing cluster-scoped resources") + + # Process each resource in the namespace (or cluster-scoped) + for resource_def in resources: + kind = resource_def.get('kind') + api_version = resource_def.get('api_version') + name = resource_def.get('name') + labels = resource_def.get('labels', []) + + if not kind: + display.v(f"Warning: Skipping resource without kind defined") + continue + + if not api_version: + raise AnsibleError(f"Error: api_version is required for resource kind '{kind}'") + + # Backup resources (either specific or all of that kind) + # Pass namespace, name, and labels as named optional arguments + backed_up, not_found, failed, discovered_secrets = backupResources( + dynClient, kind, api_version, backup_path, + namespace=namespace, name=name, labels=labels + ) + total_backed_up += backed_up + total_not_found += not_found + total_failed += failed + + # Track failed resources with details + if failed > 0: + scope = namespace if namespace else 'cluster-scoped' + resource_desc = f"{kind}/{name}" if name else f"{kind} (all)" + if labels: + resource_desc += f" with labels {labels}" + failed_resources.append({ + 'scope': scope, + 'kind': kind, + 'name': name if name else 'all', + 'api_version': api_version, + 'description': resource_desc + }) + + # Track discovered secrets by namespace (only if namespace is provided) + if discovered_secrets and namespace: + if namespace not in secrets_by_namespace: + secrets_by_namespace[namespace] = set() + secrets_by_namespace[namespace].update(discovered_secrets) + all_discovered_secrets.update(discovered_secrets) + + display.v(f"Backup complete for named resources: {total_backed_up} resources backed up, {total_not_found} not found, {total_failed} failed") + + # Now backup all discovered secrets per namespace + for ns, secret_names in secrets_by_namespace.items(): + if secret_names: + display.v(f"Backing up {len(secret_names)} discovered secret(s) in namespace '{ns}': {', '.join(sorted(secret_names))}") + + for secret_name in sorted(secret_names): + display.v(f"Backing up discovered secret: {secret_name}") + backed_up, not_found, failed, _ = backupResources( + dynClient, 'Secret', 'v1', backup_path, + namespace=ns, name=secret_name + ) + total_backed_up += backed_up + total_not_found += not_found + total_failed += failed + + # Track failed secret backups + if failed > 0: + failed_resources.append({ + 'scope': ns, + 'kind': 'Secret', + 'name': secret_name, + 'api_version': 'v1', + 'description': f"Secret/{secret_name} (auto-discovered)" + }) + + if not_found: + display.v(f"Warning: Referenced secret '{secret_name}' not found in namespace '{ns}'") + + display.v(f"Backup complete for all: {total_backed_up} resources backed up, {total_not_found} not found, {total_failed} failed") + + # Determine if the backup was successful + has_failures = total_failed > 0 + + return dict( + message=f"Backed up {total_backed_up} MAS Suite resources" + (f" with {total_failed} failures" if has_failures else ""), + failed=has_failures, + changed=False, + success=not has_failures, + backed_up_count=total_backed_up, + not_found_count=total_not_found, + failed_count=total_failed, + discovered_secrets_count=len(all_discovered_secrets), + discovered_secrets=sorted(list(all_discovered_secrets)), + failed_resources=failed_resources + ) + +# Made with Bob diff --git a/ibm/mas_devops/plugins/action/get_mas_cluster_issuer.py b/ibm/mas_devops/plugins/action/get_mas_cluster_issuer.py new file mode 100644 index 0000000000..1c0b0458e8 --- /dev/null +++ b/ibm/mas_devops/plugins/action/get_mas_cluster_issuer.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError + +from mas.devops.mas import getMasPublicClusterIssuer + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + +class ActionModule(ActionBase): + """ + Retrieve the Public Cluster Issuer for a MAS instance. + + This action plugin queries the Suite custom resource and retrieves the + certificate issuer name from spec.certificateIssuer.name. If not specified, + it returns the default issuer name. + + Usage Example + ------------- + tasks: + - name: "Get MAS Cluster Issuer" + ibm.mas_devops.get_mas_cluster_issuer: + instance_id: "{{ mas_instance_id }}" + register: cluster_issuer + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + instanceId = self._task.args.get('instance_id', None) + + if instanceId is None: + raise AnsibleError(f"Error: instance_id argument was not provided") + if not isinstance(instanceId, str): + raise AnsibleError(f"Error: instance_id argument is not a string") + + # Get the cluster issuer name + issuerName = getMasPublicClusterIssuer(dynClient, instanceId) + + if issuerName is None: + return dict( + message=f"Failed to retrieve cluster issuer for MAS instance '{instanceId}'", + success=False, + failed=True, + changed=False, + instance_id=instanceId + ) + + return dict( + message=f"Successfully retrieved cluster issuer for MAS instance '{instanceId}'", + success=True, + failed=False, + changed=False, + instance_id=instanceId, + issuer_name=issuerName + ) + diff --git a/ibm/mas_devops/roles/suite_backup/README.md b/ibm/mas_devops/roles/suite_backup/README.md index 5f22afaafa..3791bc45db 100644 --- a/ibm/mas_devops/roles/suite_backup/README.md +++ b/ibm/mas_devops/roles/suite_backup/README.md @@ -1,4 +1,4 @@ -Backup and Restore MAS Core +Backup MAS Core =============================================================================== Overview @@ -11,3 +11,25 @@ This role supports backing up MAS Core namespace resources; supports creating on Role Variables ------------------------------------------------------------------------------- + +### mas_instance_id +The instance ID of the Maximo Application Suite installation to backup. + +- **Required** +- Environment Variable: `MAS_INSTANCE_ID` +- Default Value: None + +### mas_backup_dir +The local directory path where backup files will be stored (for backup). + +- **Required** +- Environment Variable: `MAS_BACKUP_DIR` +- Default: None +- Example: `/tmp/mas_backups` + +### suite_backup_version +Set version to override the default `YYMMDD-HHMMSS` timestamp version used in the name of the backup file. + +- **Optional** +- Default: `YYMMDD-HHMMSS` timestamp. +- Environment Variable: `SUITE_BACKUP_VERSION` diff --git a/ibm/mas_devops/roles/suite_backup/defaults/main.yml b/ibm/mas_devops/roles/suite_backup/defaults/main.yml index 83247e808c..1483756d8b 100644 --- a/ibm/mas_devops/roles/suite_backup/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_backup/defaults/main.yml @@ -1,2 +1,5 @@ --- mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" +suite_backup_version: "{{ lookup('env', 'SUITE_BACKUP_VERSION') }}" diff --git a/ibm/mas_devops/roles/suite_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_backup/tasks/main.yml index be78b506f1..0473c9fdf0 100644 --- a/ibm/mas_devops/roles/suite_backup/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_backup/tasks/main.yml @@ -1,12 +1,182 @@ --- -# Check mas core backup/restore required variables +# 1. Check mas core backup required variables # ----------------------------------------------------------------------------- - name: "Fail if mas_instance_id is not provided" assert: that: mas_instance_id is defined and mas_instance_id != "" fail_msg: "mas_instance_id is required" +- name: "Fail if mas_backup_dir is not provided" + assert: + that: mas_backup_dir is defined and mas_backup_dir != "" + fail_msg: "mas_backup_dir is required" + +- name: "Check if SUITE_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + suite_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: suite_backup_version is not defined or suite_backup_version == "" or suite_backup_version == "None" + - name: "Set fact: mas core namespace name" set_fact: mas_core_namespace: "mas-{{ mas_instance_id }}-core" +- name: "Set fact: mas suite backup base directory path" + set_fact: + suite_backup_path: "{{ mas_backup_dir }}/backup-{{ suite_backup_version }}-suite" + +# 2. Determine version of cert-manager in use on the cluster +# ----------------------------------------------------------------------------- +- name: Detect Certificate Manager installation + include_tasks: "{{ role_path }}/../../common_tasks/detect_cert_manager.yml" + +# 3. Determine if a public clusterissuer is being used +# ----------------------------------------------------------------------------- +- name: Detect MAS Public Cluster Issuer + ibm.mas_devops.get_mas_cluster_issuer: + instance_id: "{{ mas_instance_id }}" + register: mas_cluster_issuer + +- name: "Display detected MAS cluster issuer" + debug: + msg: "Detected MAS cluster issuer: {{ mas_cluster_issuer.issuer_name }}" + when: mas_cluster_issuer.success + +- name: "Fail if MAS cluster issuer detection failed" + fail: + msg: "Failed to detect MAS cluster issuer" + when: mas_cluster_issuer.failed + +# 4. Set the backup_resources we want to backup. Note that if the resource doesn't +# exist, the backup will still succeed. If a resource contains `secretName` key +# then the secret will be backed up as well. +# ----------------------------------------------------------------------------- +- name: "Set fact: mas suite backup resources" + set_fact: + suite_backup_resources: + - resources: + - kind: ClusterIssuer + api_version: cert-manager.io/v1 + name: "{{ mas_cluster_issuer.issuer_name }}" + - kind: ClusterIssuer + api_version: cert-manager.io/v1 + name: "mas-{{ mas_instance_id }}-core-internal-issuer" + - kind: ClusterIssuer + api_version: cert-manager.io/v1 + name: "mas-{{ mas_instance_id }}-ca" + - namespace: "{{ cert_manager_cluster_resource_namespace }}" + resources: + - kind: Issuer + api_version: cert-manager.io/v1 + name: "mas-{{ mas_instance_id }}-core-internal-ca-issuer" + - kind: Issuer + api_version: cert-manager.io/v1 + name: "mas-{{ mas_instance_id }}-core-public-ca-issuer" + - kind: Certificate + api_version: cert-manager.io/v1 + name: "{{ mas_instance_id }}-cert-internal-ca" + - kind: Certificate + api_version: cert-manager.io/v1 + name: "{{ mas_instance_id }}-cert-public-ca" + - namespace: "{{ mas_core_namespace }}" + resources: + # subscription + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: ibm-mas-operator + - kind: OperatorGroup + api_version: operators.coreos.com/v1 + name: ibm-mas-operator-group + # secrets + - kind: Secret + api_version: v1 + name: ibm-entitlement + - kind: Secret + api_version: v1 + name: "{{ mas_instance_id }}-credentials-superuser" + # public cert in case of manual managed certs + - kind: Secret + api_version: v1 + name: "{{ mas_instance_id }}-cert-public" + # certificates + - kind: Certificate + api_version: cert-manager.io/v1 + labels: + - "mas.ibm.com/instanceId={{ mas_instance_id }}" + # addons.mas.ibm.com + - kind: MVIEdge + api_version: addons.mas.ibm.com/v1 + - kind: ReplicaDB + api_version: addons.mas.ibm.com/v1 + - kind: GenericAddon + api_version: addons.mas.ibm.com/v1 + # config.mas.ibm.com + - kind: AppCfg + api_version: config.mas.ibm.com/v1 + - kind: BasCfg + api_version: config.mas.ibm.com/v1 + - kind: IDPCfg + api_version: config.mas.ibm.com/v1 + - kind: JdbcCfg + api_version: config.mas.ibm.com/v1 + - kind: KafkaCfg + api_version: config.mas.ibm.com/v1 + - kind: MongoCfg + api_version: config.mas.ibm.com/v1 + - kind: ObjectStorageCfg + api_version: config.mas.ibm.com/v1 + - kind: PushNotificationCfg + api_version: config.mas.ibm.com/v1 + - kind: ScimCfg + api_version: config.mas.ibm.com/v1 + - kind: SlsCfg + api_version: config.mas.ibm.com/v1 + - kind: SmtpCfg + api_version: config.mas.ibm.com/v1 + - kind: WatsonStudioCfg + api_version: config.mas.ibm.com/v1 + # core.mas.ibm.com + - kind: Suite + api_version: core.mas.ibm.com/v1 + - kind: Workspace + api_version: core.mas.ibm.com/v1 + # internal.mas.ibm.com + - kind: CoreIDP + api_version: internal.mas.ibm.com/v1 + +# 5. Call the backup_resources plugin to execute the backup to the path provided +# ----------------------------------------------------------------------------- +- name: "Backup MAS Suite resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ suite_backup_resources }}" + backup_path: "{{ suite_backup_path }}" + register: backup_result + +# 6. Show the results +# ----------------------------------------------------------------------------- +- name: "Display backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ backup_result.backed_up_count }}" + - "Total resources failed: {{ backup_result.failed_count }}" + - "Resources not found: {{ backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" + - "Backup location: {{ suite_backup_path }}" + +# 7. Fail task if any errors occurred. +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ backup_result.failed_resources | to_nice_yaml }}" + when: backup_result.failed_count > 0 + +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ backup_result.failed_count }} resource(s): + {% for resource in backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: backup_result.failed_count > 0 From a543a45c152d5b11de3f0dfc703f37715b1eb65a Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Wed, 14 Jan 2026 14:58:11 +0000 Subject: [PATCH 15/61] [major] refactored mongodb and DB2 Backup and Restore (#2056) Co-authored-by: Sanjay Prabhakar --- ibm/mas_devops/playbooks/br_db2.yml | 4 +- ibm/mas_devops/playbooks/br_mongodb.yml | 21 +- .../plugins/action/backup_db2_instance.py | 47 +- .../plugins/action/backup_mongo_instance.py | 1 + .../plugins/action/get_db2u_pod_name.py | 6 +- .../plugins/action/restore_db2_resources.py | 258 ++++--- .../action/verify_backup_restore_vars.py | 2 +- .../plugins/action/verify_mongoce_version.py | 75 +- .../plugins/module_utils/backuprestore.py | 1 + ibm/mas_devops/roles/db2/README.md | 652 ++++++++++++------ ibm/mas_devops/roles/db2/defaults/main.yml | 1 + .../db2/tasks/install/get-backup-info.yml | 53 ++ .../roles/db2/tasks/install/install.yml | 467 +++++++++++++ .../roles/db2/tasks/install/main.yml | 480 +------------ .../roles/db2/tasks/restore/main.yml | 9 - .../db2/tasks/restore/restore-instance.yml | 87 --- .../roles/db2/tasks/restore_database/main.yml | 4 + .../restore-database.yml | 4 +- .../restore-db-from-disk.yml | 4 +- .../restore-db-from-s3.yml | 4 +- ibm/mas_devops/roles/mongodb/README.md | 503 +++++--------- .../roles/mongodb/defaults/main.yml | 2 + ibm/mas_devops/roles/mongodb/tasks/main.yml | 7 + .../backup-restore/backup-database.yml | 4 +- ...backup-cluster.yml => backup-instance.yml} | 0 ...er-from-backup.yml => get-backup-info.yml} | 52 +- .../backup-restore/get-mongo-info.yml | 45 -- .../backup-restore/restore-database.yml | 8 +- .../tasks/providers/community/backup.yml | 2 +- .../tasks/providers/community/install.yml | 29 +- .../tasks/providers/community/restore.yml | 52 -- .../providers/community/restore_database.yml | 28 + .../backup-restore/create-role-user.sh.j2 | 0 .../backup-restore/database-backup.sh.j2 | 0 .../backup-restore/database-restore.sh.j2 | 0 35 files changed, 1407 insertions(+), 1505 deletions(-) create mode 100644 ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/install/install.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/restore/main.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore_database/main.yml rename ibm/mas_devops/roles/db2/tasks/{restore => restore_database}/restore-database.yml (93%) rename ibm/mas_devops/roles/db2/tasks/{restore => restore_database}/restore-db-from-disk.yml (97%) rename ibm/mas_devops/roles/db2/tasks/{restore => restore_database}/restore-db-from-s3.yml (96%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/{backup-cluster.yml => backup-instance.yml} (100%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/{restore-cluster-from-backup.yml => get-backup-info.yml} (71%) delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_database.yml rename ibm/mas_devops/roles/mongodb/{tasks/providers => templates}/community/backup-restore/create-role-user.sh.j2 (100%) rename ibm/mas_devops/roles/mongodb/{tasks/providers => templates}/community/backup-restore/database-backup.sh.j2 (100%) rename ibm/mas_devops/roles/mongodb/{tasks/providers => templates}/community/backup-restore/database-restore.sh.j2 (100%) diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index ed97768248..1601056d4a 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -7,7 +7,7 @@ mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or restore + db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or restore_database or install db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" # Required for restore action br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false to enable DB2 instance backup/restore backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline and required for backup @@ -22,7 +22,7 @@ pre_tasks: - name: "Fail if DB2_ACTION is not set to backup|restore" assert: - that: db2_action in ["backup", "restore"] + that: db2_action in ["backup", "restore_database", "install"] fail_msg: "DB2_ACTION is required and must be set to 'backup' or 'restore'" - name: "Fail if BACKUP_VENDOR is not set to s3|disk" diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index e6397cc774..e1d61d0ab1 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -4,29 +4,20 @@ vars: # Define the target for backup/restore mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('backup', true) }}" # backup or restore_database or install mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups + mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" # Required for install from backup or restore_database action mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" mongodb_namespace: "{{ lookup('env', 'MONGODB_NAMESPACE') | default('mongoce', true) }}" - mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('backup', true) }}" # backup or restore mongodb_provider: "community" # only community is supported currently + br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false for mongodb instance backup mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Optional - mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') | default('None', true)}}" # Required for restore action - br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false for mongodb instance backup/restore pre_tasks: - - name: "Fail if mongodb_action is not set to backup|restore" + - name: "Fail if mongodb_action is not set to backup|install|restore_database" assert: - that: mongodb_action in ["backup", "restore"] - fail_msg: "mongodb_action is required and must be set to 'backup' or 'restore'" - - - name: "Fail if require variables for Mongodb backup/restore are not provided" - ibm.mas_devops.verify_backup_restore_vars: - mas_instance_id: "{{ mas_instance_id }}" - mas_backup_dir: "{{ mas_backup_dir }}" - mongodb_instance_name: "{{ mongodb_instance_name }}" - mongodb_backup_version: "{{ mongodb_backup_version }}" - action: "{{ mongodb_action }}" - component: "mongodb" + that: mongodb_action in ["backup", "restore_database", "install"] + fail_msg: "mongodb_action is required and must be set to 'backup' or 'restore_database' or 'install'" roles: - role: ibm.mas_devops.mongodb diff --git a/ibm/mas_devops/plugins/action/backup_db2_instance.py b/ibm/mas_devops/plugins/action/backup_db2_instance.py index fc4208a0cd..9bf739473a 100644 --- a/ibm/mas_devops/plugins/action/backup_db2_instance.py +++ b/ibm/mas_devops/plugins/action/backup_db2_instance.py @@ -8,8 +8,9 @@ from ansible.utils.display import Display from kubernetes.dynamic import DynamicClient from kubernetes.dynamic.exceptions import NotFoundError +from mas.devops.backup import backupResources, createBackupDirectories -from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import createBackupDirectories, backupSecret, getSubscription, getDb2uInstance, getDb2VersionFromCR, isDb2uReady +from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import backupSecret, getSubscription, getDb2uInstance, getDb2VersionFromCR, isDb2uReady import yaml import os @@ -99,9 +100,22 @@ def processUserCredentialsSecret(dynClient: DynamicClient, mas_instance_id: str, with open(ldap_info_file_path, 'w') as ldap_info_file: yaml.dump(ldap_user_info, ldap_info_file) display.v(f"Wrote LDAP user info to {ldap_info_file_path}") + return True else: display.v(f"JDBC credentials secret '{jdbc_core_secret_name}' uses default admin user. No additional user backup needed.") - return False + return False + +def backupCertsResources(dynClient: DynamicClient, namespace: str, kind: str, api_version: str, backup_path: str, name: str, all_discovered_secrets=None): + backed_up, not_found, failed, discovered_secrets = backupResources(dynClient=dynClient, namespace=namespace, kind=kind, name=name, api_version=api_version, backup_path=backup_path) + if backed_up > 0: + if discovered_secrets and all_discovered_secrets!=None: + display.v(f"Discovered secret(s) '{discovered_secrets}' in {kind}") + all_discovered_secrets.update(discovered_secrets) + display.v(f"Backed up resource '{kind}' '{name}'") + elif not_found > 0: + display.v(f"Resource '{kind}' '{name}' is not found.") + else: + raise AnsibleError("Failed to backup Resource '{kind}' '{name}'") class ActionModule(ActionBase): @@ -141,10 +155,11 @@ def run(self, tmp=None, task_vars=None): # Prepare backup resource path db2_backup_resource_path = f"{db2_backup_path}/resources" db2_backup_secrets_path = f"{db2_backup_resource_path}/secrets" - #db2_backup_issuers_path = f"{db2_backup_resource_path}/issuers" + db2_backup_issuers_path = f"{db2_backup_resource_path}/issuers" + db2_backup_certs_path = f"{db2_backup_resource_path}/certificates" # Create backup directory if it does not exist - createBackupDirectories([db2_backup_path, db2_backup_resource_path, db2_backup_secrets_path]) + createBackupDirectories([db2_backup_path, db2_backup_resource_path, db2_backup_secrets_path, db2_backup_issuers_path, db2_backup_certs_path]) # Get Db2uCluster or Db2uInstance CR db2u_cr = getDb2uInstance(dynClient=dynClient, db2_instance_name=db2_instance_name, db2_namespace=db2_namespace) @@ -184,7 +199,29 @@ def run(self, tmp=None, task_vars=None): # this will be used during restore to set the correct user # If secret does not exist or uses default user, no action is taken. # Restore process will use default instance user in that case - processUserCredentialsSecret(dynClient=dynClient, mas_instance_id=mas_instance_id, db2_instance_name=db2_instance_name, backup_path=db2_backup_secrets_path) + if mas_instance_id: + processUserCredentialsSecret(dynClient=dynClient, mas_instance_id=mas_instance_id, db2_instance_name=db2_instance_name, backup_path=db2_backup_secrets_path) + + # Backup following Issuers and Certificates + # these are hardcoded as these are hardcoded in the templates. + ca_issuer = "db2u-ca-issuer" + ca_certificate = "db2u-ca-certificate" + server_issuer = "db2u-issuer" + server_certificate = f"db2u-certificate-{db2_instance_name}" + + all_discovered_secrets = set() + backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Issuer", name=ca_issuer, api_version="cert-manager.io/v1", backup_path=db2_backup_issuers_path, all_discovered_secrets=all_discovered_secrets) + backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Certificate", name=ca_certificate, api_version="cert-manager.io/v1", backup_path=db2_backup_certs_path, all_discovered_secrets=all_discovered_secrets) + backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Issuer", name=server_issuer, api_version="cert-manager.io/v1", backup_path=db2_backup_issuers_path, all_discovered_secrets=all_discovered_secrets) + backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Certificate", name=server_certificate, api_version="cert-manager.io/v1", backup_path=db2_backup_certs_path, all_discovered_secrets=all_discovered_secrets) + + if len(all_discovered_secrets)>0: + # Backup discovered secrets from Issuers/certs + display.v(f"Backing up the discovered secrets from Issuers/certs - {all_discovered_secrets}") + for secret in all_discovered_secrets: + backed_up, not_found, failed, discovered_secrets = backupResources(dynClient=dynClient, namespace=db2_namespace, kind="Secret", name=secret, api_version="v1", backup_path=db2_backup_secrets_path) + if failed > 0: + display.v(f"Failed to backup secret '{secret}'") # Get Channel details from Subscription # Package name hardcoded to db2u-operator diff --git a/ibm/mas_devops/plugins/action/backup_mongo_instance.py b/ibm/mas_devops/plugins/action/backup_mongo_instance.py index ac86139d67..2589d86492 100644 --- a/ibm/mas_devops/plugins/action/backup_mongo_instance.py +++ b/ibm/mas_devops/plugins/action/backup_mongo_instance.py @@ -107,6 +107,7 @@ def filterResourceData(data: dict) -> dict: 'generation', 'resourceVersion', 'selfLink', + 'ownerReferences' 'uid', 'managedFields' ] diff --git a/ibm/mas_devops/plugins/action/get_db2u_pod_name.py b/ibm/mas_devops/plugins/action/get_db2u_pod_name.py index 2a8d7de444..d47e281c8b 100644 --- a/ibm/mas_devops/plugins/action/get_db2u_pod_name.py +++ b/ibm/mas_devops/plugins/action/get_db2u_pod_name.py @@ -55,7 +55,7 @@ def run(self, tmp=None, task_vars=None): pods = podAPI.get(namespace=db2_namespace, label_selector=label_selector) if pods.items: pod_name = pods.items[0]["metadata"]["name"] - display.v(f"Found Pod '{pod_name}' in namespace '{db2_namespace}' with labels '{label_selector}'") + display.v(f"- Found Pod '{pod_name}' in namespace '{db2_namespace}' with labels '{label_selector}'") return dict( failed=False, success=True, @@ -64,8 +64,8 @@ def run(self, tmp=None, task_vars=None): msg="Db2u Pod found" ) else: - display.v(f"No Pods found in namespace '{db2_namespace}' with labels '{label_selector}'") + display.v(f"- No Pods found in namespace '{db2_namespace}' with labels '{label_selector}'") except NotFoundError: - display.v(f"No Pods found in namespace '{db2_namespace}' with labels '{label_selector}'") + display.v(f"- No Pods found in namespace '{db2_namespace}' with labels '{label_selector}'") return dict(failed=True, success=False, pod_name="", db2version="", msg="Db2u Pod not found") diff --git a/ibm/mas_devops/plugins/action/restore_db2_resources.py b/ibm/mas_devops/plugins/action/restore_db2_resources.py index 7dc0732c6f..8ea4c1069e 100644 --- a/ibm/mas_devops/plugins/action/restore_db2_resources.py +++ b/ibm/mas_devops/plugins/action/restore_db2_resources.py @@ -6,14 +6,11 @@ from ansible.plugins.action import ActionBase from ansible.errors import AnsibleError from ansible.utils.display import Display -from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic.exceptions import NotFoundError -from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import createBackupDirectories, backupSecret, getSubscription, getDb2uInstance, getDb2VersionFromCR, isDb2uReady +from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import getDb2VersionFromCR import yaml import os -import base64 from mas.devops.ocp import createNamespace, apply_resource @@ -69,135 +66,136 @@ def run(self, tmp=None, task_vars=None): if not db2_backup_version or db2_backup_version == '': raise AnsibleError("db2_backup_version is a required parameter and cannot be empty") - # Check if backup directory exists - db2_backup_path = os.path.join(mas_backup_dir, f"backup-{db2_backup_version}-db2u") - checkBackupDirectoryExists(db2_backup_path, db2_backup_version) - display.v(f"Db2 backup path {db2_backup_path} exists. Proceeding with restore...") + try: - db2_backup_resource_path = os.path.join(db2_backup_path, "resources") - # Get DB2 instance name from backed up CR - cr_path = os.path.join(db2_backup_resource_path, "cr.yaml") + # Check if backup directory exists + db2_backup_path = os.path.join(mas_backup_dir, f"backup-{db2_backup_version}-db2u") + checkBackupDirectoryExists(db2_backup_path, db2_backup_version) + display.v(f"- Db2 backup path {db2_backup_path} exists. Proceeding with restore...") + + db2_backup_resource_path = os.path.join(db2_backup_path, "resources") + # Get DB2 instance name from backed up CR + cr_path = os.path.join(db2_backup_resource_path, "cr.yaml") - # read cr.yaml - with open(cr_path, 'r') as cr_file: - backup_db2u_cr = yaml.safe_load(cr_file) - display.v("Successfully read DB2 backup CR file") - - db2_instance_name = backup_db2u_cr['metadata']['name'] - db2_namespace = backup_db2u_cr['metadata']['namespace'] - - # Check if Db2u instance is already present - db2u_instance_cr = getDb2uInstance(dynClient, db2_instance_name, db2_namespace) - if db2u_instance_cr is not None: - display.v(f"Db2u instance {db2_instance_name} already exists in namespace {db2_namespace}. Checking if it is ready...") - if isDb2uReady(db2u_instance_cr): - display.v(f"Db2u instance {db2_instance_name} is already in Ready state. Checking version...") - if getDb2VersionFromCR(db2u_instance_cr) == getDb2VersionFromCR(backup_db2u_cr): - display.v(f"Db2u instance {db2_instance_name} version matches the backup version. Skipping DB2 instance restore...") - return dict( - changed=False, - failed=False, - proceed=False, - success=True, - msg=f"Db2u instance {db2_instance_name} is already in Ready state. Skipping DB2 instance restore..." - ) - else: - display.v(f"Db2u instance {db2_instance_name} version does not match the backup version. Abandoning restore... Check the instance manually.") - return dict( - changed=False, - failed=True, - proceed=False, - success=False, - msg=f"Db2u instance {db2_instance_name} version does not match the backup version. Abandoning restore... Check the instance manually." - ) - else: - display.v(f"Db2u instance {db2_instance_name} is not in Ready state. Abandoning restore... Check the instance status manually.") - return dict( - changed=False, - failed=True, - proceed=False, - success=False, - msg=f"Db2u instance {db2_instance_name} is not in Ready state. Abandoning restore... Check the instance status manually." - ) - else: - display.v(f"Db2u instance {db2_instance_name} does not exist in namespace {db2_namespace}. Proceeding with B2 instance restore...") - - # ======================================================= - # 1. Create DB2 namespace if not exists - # ======================================================= - display.v(f"Creating Db2 namespace '{db2_namespace}' if it does not already exist") - createNamespace(dynClient, db2_namespace) - - # ======================================================= - # 2. Restore Db2 Secret resources from backup - # ======================================================= - display.v(f"Restoring Db2 Secret resources from backup path '{db2_backup_resource_path}/secrets'") - db2_secrets_path = os.path.join(db2_backup_resource_path, "secrets") - secret_files = os.listdir(db2_secrets_path) - for secret_file in secret_files: - # Some info files will be kept in secrets folder with NOT_SECRET in the filename - if "NOT_SECRET" in secret_file: - continue - display.v(f"Restoring Db2 Secret resource from backup file '{secret_file}'") - with open(os.path.join(db2_secrets_path, secret_file), 'r') as f: - secret_yaml = f.read() - apply_resource(dynClient, secret_yaml, db2_namespace) + # read cr.yaml + with open(cr_path, 'r') as cr_file: + backup_db2u_cr = yaml.safe_load(cr_file) + display.v("- Successfully read DB2 backup CR file") + + db2_instance_name = backup_db2u_cr['metadata']['name'] + db2_namespace = backup_db2u_cr['metadata']['namespace'] + + # ======================================================= + # 1. Create DB2 namespace if not exists + # ======================================================= + display.v(f"- Creating Db2 namespace '{db2_namespace}' if it does not already exist") + createNamespace(dynClient, db2_namespace) + + # ======================================================= + # 2. Restore Db2 Secret resources from backup + # ======================================================= + db2_secrets_path = os.path.join(db2_backup_resource_path, "secrets") + display.v(f"- Restoring Db2 Secret resources from backup path '{db2_secrets_path}'") + secret_files = os.listdir(db2_secrets_path) + for secret_file in secret_files: + # Some info files will be kept in secrets folder with NOT_SECRET in the filename + if "NOT_SECRET" in secret_file: + continue + display.v(f"- Restoring Db2 Secret resource from backup file '{secret_file}'") + with open(os.path.join(db2_secrets_path, secret_file), 'r') as f: + secret_yaml = f.read() + apply_resource(dynClient, secret_yaml, db2_namespace) + + # ======================================================= + # 3. Restore DB2 Issuer resources from backup + # ======================================================= + db2_issuers_path = os.path.join(db2_backup_resource_path, "issuers") + display.v(f"- Restoring DB2 Issuer resources from backup path '{db2_issuers_path}'") + issuer_files = os.listdir(db2_issuers_path) + for issuer_file in issuer_files: + with open(os.path.join(db2_issuers_path, issuer_file), 'r') as f: + issuer_yaml = f.read() + apply_resource(dynClient, issuer_yaml, db2_namespace) - # ======================================================= - # 3. Gather info from backup files to recreate new Db2u instance - # ======================================================= - - db2_info = {} - db2_info['db2_instance_name'] = db2_instance_name - db2_info['db2_namespace'] = db2_namespace - db2_info['db2_type'] = backup_db2u_cr['spec'].get('environment', {}).get('dbType', 'db2wh') - # Get DB name from backup CR - db2_info['db2_database_name'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('name', 'BLUDB') - - # Get Db2 configs from backup CR - db2_info['db2_database_db_config'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('dbConfig', {}) - db2_info['db2_instance_dbm_config'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('dbmConfig', {}) - db2_info['db2_instance_registry'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('registry', {}) - - # Get dftTableOrg - db2_info['db2_table_org'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('settings', {}).get('dftTableOrg', 'ROW') - - # Get Pod config - db2_pod_config = backup_db2u_cr['spec'].get('podConfig', {}).get('db2u', {}).get('resource', {}).get('db2u', {}) - db2_info['db2_cpu_requests'] = db2_pod_config.get('requests', {}).get('cpu', None) - db2_info['db2_memory_requests'] = db2_pod_config.get('requests', {}).get('memory', None) - db2_info['db2_cpu_limits'] = db2_pod_config.get('limits', {}).get('cpu', None) - db2_info['db2_memory_limits'] = db2_pod_config.get('limits', {}).get('memory', None) - - # Get LDAP user info if present - ldap_info_file_path = os.path.join(db2_backup_resource_path, "ldapuser-NOT_SECRET.yaml") - if os.path.exists(ldap_info_file_path): - with open(ldap_info_file_path, 'r') as ldap_info_file: - ldap_info = yaml.safe_load(ldap_info_file) - db2_info['db2_ldap_username'] = ldap_info['db2_ldap_username'] - db2_info['db2_ldap_password'] = ldap_info['db2_ldap_password'] - display.v(f"Successfully read LDAP user info from {ldap_info_file_path}") + # ======================================================= + # 4. Restore DB2 Certificate resources from backup + # ======================================================= + db2_certs_path = os.path.join(db2_backup_resource_path, "certificates") + display.v(f"- Restoring DB2 Certificate resources from backup path '{db2_certs_path}'") + cert_files = os.listdir(db2_certs_path) + for cert_file in cert_files: + with open(os.path.join(db2_certs_path, cert_file), 'r') as f: + cert_yaml = f.read() + apply_resource(dynClient, cert_yaml, db2_namespace) - # Get DB2 backup info file - db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-backup-info.yaml") - if os.path.exists(db2_info_file_path): - with open(db2_info_file_path, 'r') as info_file: - backup_db2_info = yaml.safe_load(info_file) - db2_info['db2_version'] = backup_db2_info['db2_version'] - db2_info['db2_channel'] = backup_db2_info['db2_channel'] - display.v(f"Successfully read Db2 backup info from {db2_info_file_path}") - else: - db2_info['db2_version'] = getDb2VersionFromCR(backup_db2u_cr) - if "s11.5.9.0" in db2_info['db2_version']: - db2_info['db2_channel'] = "v110509.0" + # ======================================================= + # 5. Gather info from backup files to recreate new Db2u instance + # ======================================================= + + db2_info = {} + db2_info['db2_instance_name'] = db2_instance_name + db2_info['db2_namespace'] = db2_namespace + db2_info['db2_type'] = backup_db2u_cr['spec'].get('environment', {}).get('dbType', 'db2wh') + # Get DB name from backup CR + db2_info['db2_database_name'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('name', 'BLUDB') + + # Get Db2 configs from backup CR + db2_info['db2_database_db_config'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('dbConfig', {}) + db2_info['db2_instance_dbm_config'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('dbmConfig', {}) + db2_info['db2_instance_registry'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('registry', {}) + + # Get dftTableOrg + db2_info['db2_table_org'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('settings', {}).get('dftTableOrg', 'ROW') + + # Get Pod config + db2_pod_config = backup_db2u_cr['spec'].get('podConfig', {}).get('db2u', {}).get('resource', {}).get('db2u', {}) + db2_info['db2_cpu_requests'] = db2_pod_config.get('requests', {}).get('cpu', None) + db2_info['db2_memory_requests'] = db2_pod_config.get('requests', {}).get('memory', None) + db2_info['db2_cpu_limits'] = db2_pod_config.get('limits', {}).get('cpu', None) + db2_info['db2_memory_limits'] = db2_pod_config.get('limits', {}).get('memory', None) + + # Get timezone if present + db2_info['db2_timezone'] = backup_db2u_cr['spec'].get('advOpts', {}).get('timezone', '') + + # Get number of db2 pods + db2_info['db2_num_pods'] = backup_db2u_cr['spec']['size'] + + # Get LDAP user info if present + ldap_info_file_path = os.path.join(db2_backup_resource_path, "ldapuser-NOT_SECRET.yaml") + if os.path.exists(ldap_info_file_path): + with open(ldap_info_file_path, 'r') as ldap_info_file: + ldap_info = yaml.safe_load(ldap_info_file) + db2_info['db2_ldap_username'] = ldap_info['db2_ldap_username'] + db2_info['db2_ldap_password'] = ldap_info['db2_ldap_password'] + display.v(f"- Successfully read LDAP user info from {ldap_info_file_path}") + + # Get DB2 backup info file + db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-backup-info.yaml") + if os.path.exists(db2_info_file_path): + with open(db2_info_file_path, 'r') as info_file: + backup_db2_info = yaml.safe_load(info_file) + db2_info['db2_version'] = backup_db2_info['db2_version'] + db2_info['db2_channel'] = backup_db2_info['db2_channel'] + display.v(f"- Successfully read Db2 backup info from {db2_info_file_path}") else: - display.v("Warning: Could not find Db2 backup info file. Using default channel for the version from CR.") + db2_info['db2_version'] = getDb2VersionFromCR(backup_db2u_cr) + if "s11.5.9.0" in db2_info['db2_version']: + db2_info['db2_channel'] = "v110509.0" + else: + display.v("- Warning: Could not find Db2 backup info file. Using default channel for the version from CR.") - return dict( - message=f"Successfully restored Db2 instance's Secrets from backup paths.", - failed=False, - changed=False, - success=True, - proceed=True, - **db2_info - ) + return dict( + message=f"Successfully restored Db2 instance's resources from backup paths.", + failed=False, + changed=False, + success=True, + **db2_info + ) + except Exception as e: + display.v(f"- Failed to restore Db2 instance. Exception occurred: {e}") + return dict( + message=f"Exception occurred while restoring DB2 instance's resources from backup paths. {e}", + failed=True, + changed=False, + success=False + ) diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index a2f093d0e7..fb5be19866 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -9,7 +9,7 @@ class ActionModule(ActionBase): REQUIRED = { "mongodb": { "backup": ["mongodb_instance_name", "mas_backup_dir", "mas_instance_id"], # mongodb_instance_name has a default value - "restore": ["mongodb_instance_name", "mas_backup_dir", "mongodb_backup_version"] + "restore": ["mas_backup_dir", "mongodb_backup_version"] }, "db2": { "backup": ["db2_instance_name", "db2_namespace", "mas_backup_dir", "mas_instance_id"], diff --git a/ibm/mas_devops/plugins/action/verify_mongoce_version.py b/ibm/mas_devops/plugins/action/verify_mongoce_version.py index 7f311cc88a..913ced1662 100644 --- a/ibm/mas_devops/plugins/action/verify_mongoce_version.py +++ b/ibm/mas_devops/plugins/action/verify_mongoce_version.py @@ -8,29 +8,12 @@ from kubernetes.dynamic import DynamicClient from kubernetes.dynamic.exceptions import NotFoundError -from mas.devops.ocp import getStorageClass, getCR -from mas.devops.mas import getDefaultStorageClasses - +from mas.devops.ocp import getCR # Disabling warnings will prevent InsecureRequestWarnings from dynClient urllib3.disable_warnings() display = Display() -def getStorageClassFromCR(mongoCR: dict) -> str: - """ - Get Storage Class from MongoDB Community CR - spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName - """ - if mongoCR and 'spec' in mongoCR: - if 'statefulSet' in mongoCR['spec']: - if 'spec' in mongoCR['spec']['statefulSet']: - if 'volumeClaimTemplates' in mongoCR['spec']['statefulSet']['spec']: - if len(mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates']) > 0: - if 'spec' in mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates'][0]: - if 'storageClassName' in mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates'][0]['spec']: - return mongoCR['spec']['statefulSet']['spec']['volumeClaimTemplates'][0]['spec']['storageClassName'] - return "" - def isMongoRunning(mongoCR: dict) -> bool: """ Check if MongoDB Community instance is running @@ -63,44 +46,6 @@ def getMongoceCR(dynClient: DynamicClient, mongodb_instance_name: str, mongodb_n else: return {} -def determineBestStorageClass(dynClient: DynamicClient, existing_storageclass: str, backup_storageclass: str) -> str: - """ - Determine best storage class, - 1. Picks existing storage class if exists, if not - 2. Picks storage class from backup if exists, if not - 3. Picks default rwo storage class. - """ - - best_sc = "" - # 1) Try existing storage class if provided - if existing_storageclass != "": - display.v(f"Checking existing storage class '{existing_storageclass}'") - sc = getStorageClass(dynClient, existing_storageclass) - if sc is not None: - best_sc = existing_storageclass - else: - display.v(f"Existing storage class '{existing_storageclass}' does not exist") - - # 2) Try backup storage class if existing not usable - if best_sc == "" and backup_storageclass != "": - display.v(f"Checking backup storage class '{backup_storageclass}'") - backup_sc = getStorageClass(dynClient, backup_storageclass) - if backup_sc is not None: - best_sc = backup_storageclass - else: - display.v(f"Backup storage class '{backup_storageclass}' does not exist") - - # 3) Fallback to default RWO storage class - if best_sc == "": - display.v("Falling back to default RWO storage class") - default_sc = getDefaultStorageClasses(dynClient) - if default_sc.provider is None: - raise AnsibleError("Error: Could not find default RWO storage class to use for MongoDB Community instance") - best_sc = default_sc.rwo - - display.v(f"Using storage class '{best_sc}' for MongoDB Community instance") - return best_sc - class ActionModule(ActionBase): """ @@ -121,7 +66,6 @@ def run(self, tmp=None, task_vars=None): mongodb_instance_name = self._task.args.get('mongodb_instance_name') mongodb_namespace = self._task.args.get('mongodb_namespace') - backup_mongodb_storage_class = self._task.args.get('backup_mongodb_storage_class', None) if mongodb_instance_name is None: raise AnsibleError(f"Error: mongodb_instance_name argument was not provided") @@ -135,16 +79,6 @@ def run(self, tmp=None, task_vars=None): mongodb_namespace=mongodb_namespace ) - # Determine best storage class to use - # skip storage class check if backup storage class is not provided - best_sc = "" - if backup_mongodb_storage_class is not None: - best_sc = determineBestStorageClass( - dynClient=dynClient, - existing_storageclass=getStorageClassFromCR(mongodbCR) if mongodbCR else "", - backup_storageclass=backup_mongodb_storage_class - ) - if not mongodbCR: display.v(f"MongoDB Community instance '{mongodb_instance_name}' does NOT exist in namespace '{mongodb_namespace}'") return dict( @@ -152,8 +86,7 @@ def run(self, tmp=None, task_vars=None): success=True, failed=False, exist=False, - running=False, - storage_class=best_sc + running=False ) elif isMongoRunning(mongodbCR): display.v(f"MongoDB Community instance '{mongodb_instance_name}' is running version '{mongodbCR['spec']['version']}' in namespace '{mongodb_namespace}'") @@ -163,8 +96,7 @@ def run(self, tmp=None, task_vars=None): failed=False, exist=True, running=True, - mongoce_version=mongodbCR['spec']['version'], - storage_class=getStorageClassFromCR(mongodbCR) + mongoce_version=mongodbCR['spec']['version'] ) else: display.v(f"MongoDB Community instance '{mongodb_instance_name}' is NOT in 'Running' state in namespace '{mongodb_namespace}'") @@ -174,6 +106,5 @@ def run(self, tmp=None, task_vars=None): failed=False, exist=True, running=False, - storage_class=best_sc, mongoce_version=mongodbCR['spec']['version'] ) diff --git a/ibm/mas_devops/plugins/module_utils/backuprestore.py b/ibm/mas_devops/plugins/module_utils/backuprestore.py index dea1ab15ac..c84c40509e 100644 --- a/ibm/mas_devops/plugins/module_utils/backuprestore.py +++ b/ibm/mas_devops/plugins/module_utils/backuprestore.py @@ -42,6 +42,7 @@ def filterResourceData(data: dict) -> dict: 'resourceVersion', 'selfLink', 'uid', + 'ownerReferences' 'managedFields' ] filteredCopy = data.copy() diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index 11a25e15c7..2944c009a8 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -418,20 +418,11 @@ This is only used when both `mas_config_dir` and `mas_instance_id` are set, and - Environment Variable: `'MAS_APP_ID` - Default: None - Role Variables - Backup and Restore ------------------------------------------------------------------------------- -### db2_action -When set to `backup` or `restore`, the role will perform backup or restore operations on the DB2 instance and database. - -- Optional -- Environment Variable: `DB2_ACTION` -- Default: `install` -- Supported values: `install`, `upgrade`, `backup`, `restore` - ### mas_backup_dir -The local directory path where backup files will be stored (for backup) or read from (for restore). +Local directory path where backups will be stored or restored from. - **Required** for backup and restore operations - Environment Variable: `MAS_BACKUP_DIR` @@ -439,47 +430,44 @@ The local directory path where backup files will be stored (for backup) or read - Example: `/tmp/mas_backups` ### db2_backup_version -Version identifier for the backup. For backup operations, if not provided, it defaults to `YYMMDD-HHMMSS` format. For restore operations, this is required to identify which backup to restore. +The backup version timestamp to restore from. This is automatically generated during backup in the format `YYMMDD-HHMMSS`. -- Optional for backup (auto-generated if not provided) -- **Required** for restore +- **Required** for restore operations - Environment Variable: `DB2_BACKUP_VERSION` -- Default: Auto-generated timestamp in `YYMMDD-HHMMSS` format for backup operations +- Default: Auto-generated for backup operations +- Example: `251212-021316` -### br_skip_instance -Set to `false` to back up or restore the DB2 instance Kubernetes resources. +### backup_type +Type of backup to perform. Online backups keep the database available during backup, while offline backups require database downtime but are faster. +If your DB2 instance has got circular logging enabled i.e `LOGARCHMETH1: OFF or/and LOGARCHMETH2: OFF`, you can only use `offline` backup type. +If your DB2 instance has got circular logging disabled, you can use either `online` or `offline` backup type. +If you are unsure, you can use default `online` backup type. - Optional -- Environment Variable: `BR_SKIP_INSTANCE` -- Default: `true` (only backup/restore the database data) +- Environment Variable: `DB2_BACKUP_TYPE` +- Default: `online` +- Supported values: `online`, `offline` ### backup_vendor -Specifies where the database backup will be stored or retrieved from. +Storage vendor for backup files. Use `disk` for local storage or `s3` for S3-compatible object storage. +*Note* : Only database backup is stored in S3, instance backup is always stored in local disk. - Optional - Environment Variable: `BACKUP_VENDOR` - Default: `disk` -- Supported values: `s3`, `disk` +- Supported values: `disk`, `s3` -### backup_type -Type of database backup to perform. +### br_skip_instance +When set to `false`, includes Db2 instance resources (secrets, certificates, custom resources) in the backup. - Optional -- Environment Variable: `DB2_BACKUP_TYPE` -- Default: `online` -- Supported values: `online`, `offline` - -### backup_s3_alias -Alias name for the S3 storage configuration in DB2. - -- **Required** when `backup_vendor=s3` -- Environment Variable: `BACKUP_S3_ALIAS` -- Default: `S3DB2COS` +- Environment Variable: `BR_SKIP_INSTANCE` +- Default: `true` ### backup_s3_endpoint -S3-compatible storage endpoint URL. +S3 endpoint URL for S3-compatible object storage. -- **Required** when `backup_vendor=s3` +- **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_ENDPOINT` - Default: None - Example: `https://s3.us-east.cloud-object-storage.appdomain.cloud` @@ -487,248 +475,466 @@ S3-compatible storage endpoint URL. ### backup_s3_bucket S3 bucket name where backups will be stored. -- **Required** when `backup_vendor=s3` +- **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_BUCKET` - Default: None +- Example: `mas-db2-backups` ### backup_s3_access_key S3 access key for authentication. -- **Required** when `backup_vendor=s3` +- **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_ACCESS_KEY` - Default: None ### backup_s3_secret_key S3 secret key for authentication. -- **Required** when `backup_vendor=s3` +- **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_SECRET_KEY` - Default: None +### backup_s3_alias +S3 alias name used in Db2 configuration. -Backup and Restore Operations -------------------------------------------------------------------------------- - -### Overview -The DB2 role supports comprehensive backup and restore operations for both DB2 instance configuration and database data. The backup process creates a snapshot of necessary resources that can be used to restore the DB2 instance to a previous state. +- Optional +- Environment Variable: `BACKUP_S3_ALIAS` +- Default: `S3DB2COS` -### Backup Components -The backup operation consists of two main components: -1. **Instance Backup**: Backs up Kubernetes resources including: - - DB2UCluster custom resource - - Secrets (instance password, LDAP credentials if configured) - - ConfigMaps - - Other DB2 instance metadata +Example Usage - Backup and Restore +------------------------------------------------------------------------------- -2. **Database Backup**: Backs up the actual database data using DB2's native backup utilities +### Backup Db2 Database to Disk +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: masinst1 + mas_backup_dir: /tmp/masbr + db2_action: backup + db2_instance_name: db2u-manage + db2_namespace: db2u + backup_type: online + backup_vendor: disk + roles: + - ibm.mas_devops.db2 +``` -### Backup Process +### Backup Db2 Database to S3 +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: masinst1 + mas_backup_dir: /tmp/masbr + db2_action: backup + db2_instance_name: db2u-manage + backup_type: online + backup_vendor: s3 + backup_s3_endpoint: https://s3.us-east.cloud-object-storage.appdomain.cloud + backup_s3_bucket: mas-db2-backups # your bucket name + backup_s3_access_key: "{{ lookup('env', 'S3_ACCESS_KEY') }}" + backup_s3_secret_key: "{{ lookup('env', 'S3_SECRET_KEY') }}" + roles: + - ibm.mas_devops.db2 +``` -#### Prerequisites -- DB2 instance must be running and healthy -- Sufficient storage space in the backup location -- For S3 backups: Valid S3 credentials and accessible S3 bucket -- For disk backups: Sufficient local disk space - -#### Backup Workflow -1. **Validation**: Verifies all required variables are set -2. **Instance Backup** (when `BR_SKIP_INSTANCE=false`): - - Exports DB2UCluster CR and related Kubernetes resources - - Saves instance configuration and secrets -3. **Database Backup**: - - Prepares backup scripts and copies them to the DB2 pod - - For S3: Configures S3 storage access in DB2 - - Executes DB2 backup command (online or offline) - - For disk backups: Copies backup files to local storage - - Creates `db2-backup-info.yaml` with backup metadata - -#### Backup Storage Locations -- **Disk**: `/backup--db2u/data/` -- **S3**: `s3:///backups-db2//` - -### Restore Process - -#### Prerequisites -- Valid backup files exist in the specified location -- For instance restore: DB2 operator must be installed -- For database restore: DB2 instance must be running with matching version -- For S3 restores: Valid S3 credentials and accessible S3 bucket - -#### Restore Workflow -1. **Validation**: Verifies backup version and required variables -2. **Instance Restore** (when `BR_SKIP_INSTANCE=false`): - - Checks if DB2 instance exists and is healthy - - If not present or unhealthy: Creates new instance from backup configuration - - Restores secrets and configuration -3. **Database Restore**: - - For disk: Validates DB2 version matches backup version - - Prepares restore scripts - - For S3: Configures S3 storage access - - For disk: Copies backup archive to DB2 pod - - Executes DB2 restore command - - Validates restore completion - -### Backup Metadata -Each backup creates a `db2-backup-info.yaml` file containing: +### Backup with Instance Resources and Database to disk ```yaml -source_db2_backup_version: "241225-143022" -source_db2_backup_timestamp: "20241225143022" -source_db2_instance_name: "db2u-manage" -source_db2_instance_version: "11.5.8.0-cn7" -database: "BLUDB" -backup_vendor: "s3" -vendor_backup_path: "DB2REMOTE://S3DB2COS/my-bucket/backups-db2/241225-143022" -status: "SUCCESS" +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: masinst1 + mas_backup_dir: /tmp/masbr + db2_action: backup + db2_instance_name: db2u-manage + br_skip_instance: false # Include instance resources + backup_vendor: disk + roles: + - ibm.mas_devops.db2 ``` -### Example: Database-Only Backup (Skip Instance) to S3 -```bash -export DB2_ACTION=backup -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/tmp/mas_backups -export DB2_INSTANCE_NAME=db2u-manage -export DB2_NAMESPACE=db2u -export BR_SKIP_INSTANCE=true -export BACKUP_VENDOR=s3 -export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud -export BACKUP_S3_BUCKET=mas-db2-backups -export BACKUP_S3_ACCESS_KEY= -export BACKUP_S3_SECRET_KEY= - -ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role -``` - -### Example: Database-Only Backup (Skip Instance) to Disk -```bash -export DB2_ACTION=backup -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/tmp/mas_backups -export DB2_INSTANCE_NAME=db2u-manage -export DB2_NAMESPACE=db2u -export BR_SKIP_INSTANCE=true -export BACKUP_VENDOR=disk +### Restore Db2 Database from Disk +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + db2_action: restore_database + mas_instance_id: masinst1 + db2_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + db2_instance_name: db2u-manage + db2_namespace: db2u + backup_vendor: disk + roles: + - ibm.mas_devops.db2 +``` -ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +### Restore Db2 Database from S3 +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + db2_action: restore_database + mas_instance_id: masinst1 + db2_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + db2_instance_name: db2u-manage + backup_vendor: s3 + backup_s3_endpoint: https://s3.us-east.cloud-object-storage.appdomain.cloud + backup_s3_bucket: mas-db2-backups # your bucket name + backup_s3_access_key: "{{ lookup('env', 'S3_ACCESS_KEY') }}" + backup_s3_secret_key: "{{ lookup('env', 'S3_SECRET_KEY') }}" + roles: + - ibm.mas_devops.db2 ``` -### Example: Backup both Instance and Database to S3 (only database backup is uploaded to S3) -```bash -export DB2_ACTION=backup -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/tmp/mas_backups -export DB2_INSTANCE_NAME=db2u-manage -export DB2_NAMESPACE=db2u -export BR_SKIP_INSTANCE=false -export BACKUP_VENDOR=s3 -export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud -export BACKUP_S3_BUCKET=mas-db2-backups -export BACKUP_S3_ACCESS_KEY= -export BACKUP_S3_SECRET_KEY= -export DB2_BACKUP_TYPE=online - -ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role -``` - -### Example: Backup both Instance and Database to Disk -```bash -export DB2_ACTION=backup -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/tmp/mas_backups -export DB2_INSTANCE_NAME=db2u-manage -export BR_SKIP_INSTANCE=false -export DB2_NAMESPACE=db2u -export BACKUP_VENDOR=disk -export DB2_BACKUP_TYPE=online +### Install Db2 from Backup (Instance + Database) +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + db2_action: install + mas_instance_id: masinst1 + db2_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + backup_vendor: disk + roles: + - ibm.mas_devops.db2 +``` -ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +### Install Db2 from Backup (Instance + Database(S3)) +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + db2_action: install + mas_instance_id: masinst1 + db2_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + backup_vendor: s3 + backup_s3_endpoint: https://s3.us-east.cloud-object-storage.appdomain.cloud + backup_s3_bucket: mas-db2-backups # your bucket name + backup_s3_access_key: "{{ lookup('env', 'S3_ACCESS_KEY') }}" + backup_s3_secret_key: "{{ lookup('env', 'S3_SECRET_KEY') }}" + roles: + - ibm.mas_devops.db2 ``` -### Example: Restore Database from S3 -```bash -export DB2_ACTION=restore -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/tmp/mas_backups -export DB2_INSTANCE_NAME=db2u-manage -export DB2_NAMESPACE=db2u -export DB2_BACKUP_VERSION=241225-143022 -export BR_SKIP_INSTANCE=true -export BACKUP_VENDOR=s3 -export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud -export BACKUP_S3_BUCKET=mas-db2-backups -export BACKUP_S3_ACCESS_KEY= -export BACKUP_S3_SECRET_KEY= - -ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role -``` - -### Example: Restore Database from Disk -```bash -export DB2_ACTION=restore -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/tmp/mas_backups -export DB2_INSTANCE_NAME=db2u-manage -export BR_SKIP_INSTANCE=true -export DB2_NAMESPACE=db2u -export DB2_BACKUP_VERSION=241225-143022 -export BACKUP_VENDOR=disk +Backup and Restore Details +------------------------------------------------------------------------------- -ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +### Backup Process +1. Validates Db2 instance is running +2. Prepares backup scripts and copies them to the Db2 pod +3. Configures S3 storage access (if using S3) +4. Executes Db2 backup command (online or offline) +5. Compresses and transfers backup files (for disk storage) +6. Creates metadata file (`db2-backup-info.yaml`) with backup details + +### Database Restore Process +1. Validates backup files and Db2 version compatibility +2. Prepares restore scripts and copies them to the Db2 pod +3. Configures S3 storage access (if restoring from S3) +4. Copies backup files to the Db2 pod (for disk restores) +5. Executes Db2 restore command +6. Verifies restore completion + +### Install From Backup Process +1. Validates backup files +2. Creates namespace and copies resources to namespace +3. Gets Db2 instance details from backup metadata +4. Installs Db2 instance using the backup details +5. Waits for Db2 instance to be ready +6. Performs post deployment actions like restoring instance password +7. Performs Database restore process as mentioned above + +### Backup Directory Structure (Disk) +``` +/tmp/masbr/ +└── backup--db2u/ + ├── data/ + │ ├── db2-BLUDB-backup-.tar.gz + │ └── db2-backup-info.yaml + └── resources/ + ├── cr.yaml + ├── secrets/ + ├── certificates/ + └── issuers/ ``` -### Example: Restore Instance and database from Disk -```bash -export DB2_ACTION=restore -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/tmp/mas_backups -export DB2_INSTANCE_NAME=db2u-manage -export BR_SKIP_INSTANCE=false -export DB2_NAMESPACE=db2u -export DB2_BACKUP_VERSION=241225-143022 -export BACKUP_VENDOR=disk - -ROLE_NAME=db2 ansible-playbook ibm.mas_devops.run_role +### Database backup Metadata (db2-backup-info.yaml) +```yaml +source_db2_backup_version: "251212-021316" +source_db2_backup_timestamp: "20251212021316" +source_db2_instance_name: "db2u-manage" +source_db2_instance_version: "11.5.8.0-cn7" +database: "BLUDB" +backup_vendor: "disk" +vendor_backup_path: "/mnt/backup/251212-021316/data" +local_backup_path: "/tmp/masbr/backup-251212-021316-db2u/data/db2-BLUDB-backup-251212-021316.tar.gz" +status: "SUCCESS" ``` -### Backup Types +### Important Considerations + +**Version Compatibility:** +- The restore operation requires the target Db2 version to match the backup version +- Always verify version compatibility before attempting a restore -#### Online Backup -- Database remains available during backup -- Recommended for production environments -- Requires archive logging to be enabled -- Set `DB2_BACKUP_TYPE=online` +**Backup Types:** +If your DB2 instance has got circular logging enabled i.e `LOGARCHMETH1: OFF or/and LOGARCHMETH2: OFF`, you can only use `offline` backup type. +If your DB2 instance has got circular logging disabled, you can use either `online` or `offline` backup type. +If you are unsure, you can use default `online` backup type. +- **Online Backup**: Database remains available during backup (recommended for production) +- **Offline Backup**: Database is taken offline during backup (faster but requires downtime) -#### Offline Backup -- Database is taken offline during backup -- Faster than online backup -- Requires downtime -- Set `DB2_BACKUP_TYPE=offline` +**Storage Options:** +- **Disk Storage**: Backups stored locally and copied to backup directory +- **S3 Storage**: Backups stored directly to S3-compatible object storage (no local storage required) -### Important Notes +**Security:** +- Backup files contain sensitive data and credentials +- Secure the backup directory with appropriate permissions +- Consider encrypting backup files for long-term storage -1. **Version Compatibility**: The DB2 version of the restore target must match the backup source version -2. **Storage Requirements**: Ensure sufficient storage space for backups (typically 1-2x database size) -3. **S3 Credentials**: Keep S3 credentials secure and use appropriate IAM policies -4. **Backup Retention**: Implement a backup retention policy to manage storage costs -5. **Testing**: Regularly test restore procedures to ensure backup integrity -6. **Archive Logging**: For online backups, ensure DB2 archive logging is properly configured -7. **Network Connectivity**: For S3 backups, ensure DB2 pod has network access to S3 endpoint +**Performance:** +- Online backups may impact Db2 performance during execution +- Schedule backups during low-usage periods +- Monitor Db2 resource utilization during backup -### Troubleshooting + +Backup and Restore Troubleshooting +------------------------------------------------------------------------------- + +### Common Issues and Solutions #### Backup Failures + - Check DB2 pod logs: `oc logs -n -c db2u` - Review backup script logs in the pod: `/tmp/db2_backup.log` - Verify S3 credentials and connectivity (for S3 backups) -- Ensure sufficient storage space +- Ensure sufficient storage space in the backup PVC(/mnt/backup) + +**Issue: Backup fails with "insufficient storage space"** +``` +Error: SQL2062N An error occurred while accessing media "backup_path" +``` +**Solution:** +- Check available disk space on the Db2 pod: `oc exec -n -- df -h` +- Verify backup storage PVC has sufficient capacity +- For S3 backups, ensure bucket has adequate space and proper permissions +- Consider using compression or incremental backups to reduce storage requirements + +**Issue: Backup fails with "database is in use"** +``` +Error: SQL1035N The database is currently in use. SQLSTATE=57019 +``` +**Solution:** +- For offline backups, ensure all applications are disconnected +- Use online backup instead: `export DB2_BACKUP_TYPE=online` +- Check active connections: `oc exec -n -- su - db2inst1 -c "db2 list applications"` +- Force disconnect if necessary: `db2 force applications all` + +**Issue: S3 backup fails with authentication errors** +``` +Error: Unable to authenticate with S3 endpoint +``` +**Solution:** +- Verify S3 credentials are correct: `BACKUP_S3_ACCESS_KEY` and `BACKUP_S3_SECRET_KEY` +- Test S3 connectivity from the Db2 pod +- Ensure S3 endpoint URL is correct and accessible +- Check firewall rules and network policies allow S3 access +- Verify bucket exists and credentials have write permissions + +**Issue: Backup script execution timeout** +``` +Error: Backup operation timed out after 3600 seconds +``` +**Solution:** +- Large databases may require extended timeout periods +- Monitor backup progress: `oc logs -n -f` +- Check Db2 performance and resource utilization +- Consider scheduling backups during low-usage periods +- For very large databases, use incremental backups #### Restore Failures + - Verify backup version exists and is complete - Check DB2 version compatibility - Review restore script logs: `/tmp/db2_restore_disk.log` or `/tmp/db2_restore_s3.log` - Ensure DB2 instance is running and healthy before database restore - For S3 restores, verify S3 connectivity and credentials +**Issue: Restore fails with version mismatch** +``` +Error: DB2 version mismatch. Backup version: 11.5.8.0, Target version: 11.5.9.0 +``` +**Solution:** +- Ensure target Db2 version matches backup version +- Check backup metadata: `cat /data/db2-backup-info.yaml` +- Install matching Db2 version: `export DB2_VERSION=11.5.8.0-cn7` +- Alternatively, upgrade backup to target version (requires manual intervention) + +**Issue: Restore fails with "database already exists"** +``` +Error: SQL1005N The database alias "BLUDB" already exists +``` +**Solution:** +- Drop existing database before restore: + ```bash + oc exec -n -- su - db2inst1 -c "db2 drop database BLUDB" + ``` +- Or use a different database name during restore +- Verify database state: `db2 list database directory` + +**Issue: Restore fails with corrupted backup files** +``` +Error: SQL2025N The database cannot be restored from backup image +``` +**Solution:** +- Verify backup file integrity: + ```bash + tar -tzf .tar.gz > /dev/null + ``` +- Check backup metadata for status: `status: SUCCESS` +- Re-run backup if corruption detected +- For S3 restores, verify file was completely uploaded +- Check storage system for hardware errors + +**Issue: Restore fails with insufficient permissions** +``` +Error: SQL0551N User does not have required authorization +``` +**Solution:** +- Verify db2inst1 user has proper permissions +- Check pod security context and service account +- Ensure restore scripts have execute permissions +- Review OpenShift security policies (SCC) + +**Issue: S3 restore fails to download backup files** +``` +Error: Failed to download backup from S3 bucket +``` +**Solution:** +- Verify S3 credentials have read permissions +- Check S3 bucket name and path are correct +- Test S3 connectivity: `aws s3 ls s3:///` +- Ensure network policies allow outbound S3 access +- Verify backup files exist in S3 bucket + +#### Performance Issues + +**Issue: Backup taking too long** +**Solution:** +- Use online backups to avoid database downtime +- Schedule backups during low-usage periods +- Increase Db2 pod resources (CPU/memory) +- Use compression to reduce backup size +- Consider incremental backups for large databases +- Check network bandwidth for S3 backups + +**Issue: Restore taking too long** +**Solution:** +- Ensure adequate resources allocated to Db2 pod +- Monitor pod resource utilization during restore +- Check storage performance (IOPS, throughput) +- For S3 restores, verify network bandwidth +- Consider using faster storage classes + +#### Validation and Verification + +**Issue: How to verify backup completed successfully** +**Solution:** +1. Check backup metadata file: + ```bash + cat /data/db2-backup-info.yaml + ``` + Verify `status: SUCCESS` + +2. Verify backup file exists and has reasonable size: + ```bash + ls -lh /data/db2-*.tar.gz + ``` + +3. For S3 backups, verify files in bucket: + ```bash + aws s3 ls s3://// + ``` + +4. Check Db2 backup history: + ```bash + oc exec -n -- su - db2inst1 -c "db2 list history backup all for BLUDB" + ``` + +**Issue: How to verify restore completed successfully** +**Solution:** +1. Check database is online: + ```bash + oc exec -n -- su - db2inst1 -c "db2 connect to BLUDB" + ``` + +2. Verify table counts and data integrity: + ```bash + oc exec -n -- su - db2inst1 -c "db2 'select count(*) from '" + ``` + +3. Check database configuration: + ```bash + oc exec -n -- su - db2inst1 -c "db2 get db cfg for BLUDB" + ``` + +4. Review restore logs for errors: + ```bash + oc logs -n | grep -i error + ``` + +### Diagnostic Commands + +**Check Db2 pod status:** +```bash +oc get pods -n | grep db2 +oc describe pod -n +``` + +**Check Db2 instance status:** +```bash +oc exec -n -- su - db2inst1 -c "db2pd -" +``` + +**Check database status:** +```bash +oc exec -n -- su - db2inst1 -c "db2 list active databases" +``` + +**Check backup storage:** +```bash +oc get pvc -n +oc exec -n -- df -h /mnt/backup +``` + +**View Db2 diagnostic logs:** +```bash +oc exec -n -- tail -f /var/log/db2u.log +oc exec -n -- cat /database/config/db2inst1/sqllib/db2dump/db2diag.log +``` + +**Check S3 configuration (if using S3):** +```bash +oc exec -n -- su - db2inst1 -c "db2 list storage access" +``` + +### Getting Help + +If you encounter issues not covered in this troubleshooting guide: + +1. **Check Db2 logs**: Review `/var/log/db2u.log` and `db2diag.log` for detailed error messages +2. **Review backup metadata**: Check `db2-backup-info.yaml` for backup details and status +3. **Verify prerequisites**: Ensure all required variables are set correctly +4. **Test connectivity**: Verify network access to storage (S3 or PVC) +5. **Check resources**: Ensure adequate CPU, memory, and storage are available +6. **Open an issue**: Report problems at the project repository with logs and configuration details License ------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index 4291de6097..f2a2511dc8 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -1,6 +1,7 @@ --- # db2 action # --------------------------------------------------------------------------- +# Supported actions: install, uninstall, backup, restore_database db2_action: "{{ lookup('env', 'DB2_ACTION') | default('install', true) }}" # Configure Db2 instance diff --git a/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml b/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml new file mode 100644 index 0000000000..4922900806 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml @@ -0,0 +1,53 @@ +--- +# Check db2 Restore required variables +# ----------------------------------------------------------------------------- +- name: Verify DB2 restore variables + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "restore_instance" + db2_backup_version: "{{ db2_backup_version }}" + mas_backup_dir: "{{ mas_backup_dir }}" + +# Restore Db2 Universal operator Instance Kubernetes Resources +# ------------------------------------------------------------------------- +- name: "Start Db2 Universal operator Instance restore process." + ibm.mas_devops.restore_db2_resources: + db2_backup_version: "{{ db2_backup_version }}" + mas_backup_dir: "{{ mas_backup_dir }}" + register: restore_db2_instance_result + +- name: Assert if restoring DB2 instance resources succeeeded + assert: + that: + - restore_db2_instance_result is defined + - restore_db2_instance_result.success is defined + - restore_db2_instance_result.success == true + fail_msg: "Failed to restore DB2 instance resources" + +# Create Db2 Instance using info from backup CR if required +# ------------------------------------------------------------------------- +- name: "Set fact from DB2 backup" + set_fact: + db2_database_db_config: "{{ restore_db2_instance_result.db2_database_db_config }}" + db2_instance_dbm_config: "{{ restore_db2_instance_result.db2_instance_dbm_config }}" + db2_instance_registry: "{{ restore_db2_instance_result.db2_instance_registry }}" + db2_channel: "{{ restore_db2_instance_result.db2_channel }}" + db2_version: "{{ restore_db2_instance_result.db2_version }}" + db2_instance_name: "{{ restore_db2_instance_result.db2_instance_name }}" + db2_namespace: "{{ restore_db2_instance_result.db2_namespace }}" + db2_type: "{{ restore_db2_instance_result.db2_type }}" + db2_dbname: "{{ restore_db2_instance_result.db2_database_name }}" + db2_table_org: "{{ restore_db2_instance_result.db2_table_org }}" + db2_cpu_requests: "{{ restore_db2_instance_result.db2_cpu_requests }}" + db2_memory_requests: "{{ restore_db2_instance_result.db2_memory_requests }}" + db2_cpu_limits: "{{ restore_db2_instance_result.db2_cpu_limits }}" + db2_memory_limits: "{{ restore_db2_instance_result.db2_memory_limits }}" + db2_timezone: "{{ restore_db2_instance_result.db2_timezone }}" + db2_num_pods: "{{ restore_db2_instance_result.db2_num_pods }}" + +- name: "Set fact LDAP credentials if exist." + no_log: true + set_fact: + db2_ldap_username: "{{ restore_db2_instance_result.db2_ldap_username }}" + db2_ldap_password: "{{ restore_db2_instance_result.db2_ldap_password }}" + when: restore_db2_instance_result.db2_ldap_username is defined and restore_db2_instance_result.db2_ldap_password is defined diff --git a/ibm/mas_devops/roles/db2/tasks/install/install.yml b/ibm/mas_devops/roles/db2/tasks/install/install.yml new file mode 100644 index 0000000000..0be39b86e9 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/install/install.yml @@ -0,0 +1,467 @@ +--- +- name: "Fail if required db2_dbname is over 8 characters" + assert: + that: + - db2_dbname is defined and db2_dbname != "" + - db2_dbname | length <= 8 + fail_msg: "Property value of db2_dbname is set to '{{ db2_dbname }}' and is greater than 8 character long." + +# 2. Load default storage classes (if not provided by the user) +# ----------------------------------------------------------------------------- +- include_tasks: tasks/install/determine-storage-classes.yml + +# 3. Setup the norootsquash daemonsets for db2u pods to work with NFS backed storage +# ----------------------------------------------------------------------------- +- include_tasks: tasks/install/setup_norootsquash.yml + when: + - db2_meta_storage_class is defined + - db2_data_storage_class is defined + - '"ibmc-file" in db2_data_storage_class or "ibmc-file" in db2_meta_storage_class or "ibmc-vpc-file" in db2_data_storage_class or "ibmc-vpc-file" in db2_meta_storage_class' + +# 4. Provide debug information to the user +# ----------------------------------------------------------------------------- +- name: "Debug - MAS" + debug: + msg: + - "MAS Instance ID ........................ {{ mas_instance_id }}" + - "MAS Config directory ................... {{ mas_config_dir }}" + - "MAS Instance ID ........................ {{ mas_instance_id | default('', true) }}" + - "MAS Workspace ID ....................... {{ mas_workspace_id | default('', true) }}" + - "MAS Application ID ..................... {{ mas_application_id | default('', true) }}" + - "MAS Config Directory ................... {{ mas_config_dir | default('', true) }}" + - "MAS Config Scope ....................... {{ mas_config_scope | default('', true) }}" + +- name: "Debug - Affinity & Tolerations" + debug: + msg: + - "Affinity key ........................... {{ db2_affinity_key | default('', true) }}" + - "Affinity value ......................... {{ db2_affinity_value | default('', true) }}" + + - "Toleration key ......................... {{ db2_tolerate_key | default('', true) }}" + - "Toleration value ....................... {{ db2_tolerate_value | default('', true) }}" + - "Toleration effect ...................... {{ db2_tolerate_effect | default('', true) }}" + +- name: "Debug - Db2 Instance" + debug: + msg: + - "Namespace .............................. {{ db2_namespace }}" + - "Db2 Instance ........................... {{ db2_instance_name }}" + +- name: "Debug - Database Settings" + debug: + msg: + - "Database Name .......................... {{ db2_dbname }}" + - "4K Device Support ...................... {{ db2_4k_device_support }}" + - "Table Organization ..................... {{ db2_table_org }}" + - "TLS Version ............................ {{ tls_version }}" + - "Workload ............................... {{ db2_workload }}" + +- name: "Debug - Resources" + debug: + msg: + - "CPU Request ............................ {{ db2_cpu_requests }}" + - "CPU Limit .............................. {{ db2_cpu_limits }}" + - "Memory Request ......................... {{ db2_memory_requests }}" + - "Memory Limit ........................... {{ db2_memory_limits }}" + +- name: "Debug - Storage" + debug: + msg: + - "Meta ................................... {{ db2_meta_storage_class }} - {{ db2_meta_storage_size }} @ {{ db2_meta_storage_accessmode }}" + - "Data ................................... {{ db2_data_storage_class }} - {{ db2_data_storage_size }} @ {{ db2_data_storage_accessmode }}" + - "Backup ................................. {{ db2_backup_storage_class }} - {{ db2_backup_storage_size }} @ {{ db2_backup_storage_accessmode }}" + - "Logs ................................... {{ db2_logs_storage_class }} - {{ db2_logs_storage_size }} @ {{ db2_logs_storage_accessmode }}" + - "Temp ................................... {{ db2_temp_storage_class }} - {{ db2_temp_storage_size }} @ {{ db2_temp_storage_accessmode }}" + +# Lookup db2 operator group +- name: "Check if operator group is present in {{ db2_namespace }} namespace already" + kubernetes.core.k8s_info: + namespace: "{{ db2_namespace }}" + kind: OperatorGroup + register: db2_og_info + +# Look up the default channel for the db2u-operator package manifest +- name: Lookup db2u-operator packagemanifest + kubernetes.core.k8s_info: + api_version: packages.operators.coreos.com/v1 + kind: PackageManifest + name: db2u-operator + namespace: "{{ ibm_common_services_namespace }}" + register: db2u_manifest_info + until: db2u_manifest_info.resources[0].status.defaultChannel is defined + retries: 60 # Approximately 30 minutes before we give up + delay: 30 # seconds + when: db2_channel is not defined or db2_channel == "" or db2_version is not defined or db2_version == "" + +- name: Set db2u-operator channel + ansible.builtin.set_fact: + db2_channel: "{{ db2u_manifest_info.resources[0].status.defaultChannel }}" + when: db2_channel is not defined or db2_channel == "" + +# 5. Fail if required parameters are not set +# ----------------------------------------------------------------------------- +- name: "Verify db2_channel is set" + assert: + that: + - db2_channel is defined and db2_channel != "" + fail_msg: "Unable to determine db2_channel from catalog" + +- name: Debug DB2 upgrade channel + ansible.builtin.debug: + msg: + - "Db2 Channel ............................ {{ db2_channel }}" + +# 6. Install a Db2u Operator +# ----------------------------------------------------------------------------- +- name: "Create db2u Namespace" + kubernetes.core.k8s: + apply: yes + template: "templates/db2u_namespace.yaml" + register: _db2_namespace_result + +- name: Check if ibm-registry secret exists + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: ibm-registry + namespace: "{{ db2_namespace }}" + register: ibm_registry_secret_info + +- name: "Create ibm-registry secret if not present" + when: + - ibm_registry_secret_info.resources is defined + - ibm_registry_secret_info.resources | length == 0 + block: + - name: Set 'ibm-registry' secret content + no_log: true + vars: + entitledAuthStr: "{{ registry_user }}:{{ ibm_entitlement_key }}" + entitledAuth: "{{ entitledAuthStr | b64encode }}" + content: + - '{"auths":{"{{ registry }}/cp/cpd":{"username":"{{ registry_user }}","password":"{{ ibm_entitlement_key }}","email":"{{ registry_user }}","auth":"{{ entitledAuth }}"}' + - "}" + - "}" + set_fact: + new_secret: "{{ content | join('') }}" + + # Note: We use "new_secret | to_json " because older versions of Ansible create + # invalid json representations containing single quotes + # However, in newer versions of Ansible to_json returns an object, which can't be passed to b64encode + - name: "Generate docker secret (old Ansible versions)" + when: ansible_version.full is version_compare(2.20, '<') + set_fact: + dockerconfigjsonB64: "{{ new_secret | to_json | b64encode }}" + + - name: "Generate docker secret (new Ansible versions)" + when: ansible_version.full is version_compare(2.20, '>=') + set_fact: + dockerconfigjsonB64: "{{ new_secret | b64encode }}" + + - name: "Generate 'ibm-registry' secret" + no_log: true + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: kubernetes.io/dockerconfigjson + metadata: + name: ibm-registry + namespace: "{{ db2_namespace }}" + data: + .dockerconfigjson: "{{ dockerconfigjsonB64 }}" + register: secretUpdateResult + +- name: "Delete old db2 subscription, operand request and csv from {{ ibm_common_services_namespace }}" + include_tasks: "tasks/delete_db2_operand_request.yml" + +- name: "Create Db2 Universal Operator Subscription in {{ db2_namespace }} namespace" + ibm.mas_devops.apply_subscription: + namespace: "{{ db2_namespace }}" + package_name: db2u-operator + package_channel: "{{ db2_channel }}" + register: subscription + +# 7. Get the cluster subdomain to be used for the certificate and route creation +# ----------------------------------------------------------------------------- +- name: "Get cluster subdomain" + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + kind: Ingress + name: cluster + register: _cluster_subdomain + +# 8. Create self-signed certificate for Db2u SSL +# ----------------------------------------------------------------------------- +- name: "Create internal CA certificate issuer" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/ca_issuer.yml.j2" + register: createCaIssuer + +- name: "Create and wait for CA certificate" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/ca_certificate.yml.j2" + wait: yes + wait_timeout: 600 #10 minutes + wait_condition: + type: Ready + status: True + register: createCaCert + +- name: "Create certificate issuer" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/issuer.yml.j2" + register: createIssuer + +- name: "Create db2u certificate" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/certificate.yml.j2" + register: createCertificate + +# 9. Wait until the Db2uCluster CRD is available +# ----------------------------------------------------------------------------- +- name: "Wait until the Db2uCluster CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: "db2uclusters.db2u.databases.ibm.com" + +# 10. Get information from the db2u-release ConfigMap +# ----------------------------------------------------------------------------- +# if db2_version is not set, then we define it based on the latest version supported by the db2u-license-keys secret +# Starting with s11.5.8.0-cn3, the 's' prefix is removed in db2u-license-keys, it is recommended to use db2u-release configmap. +- block: + - name: "Wait until the db2u-release configmap is available" + no_log: true + kubernetes.core.k8s_info: + api_version: v1 + name: db2u-release + namespace: "{{ db2_namespace }}" + kind: ConfigMap + register: db2_release_info + retries: 20 # ~approx 10 minutes before we give up waiting for the CRD to be created + delay: 30 # seconds + until: + - db2_release_info.resources is defined + - db2_release_info.resources | length > 0 + - db2_release_info.resources[0].data is defined + - db2_release_info.resources[0].data | length > 0 + + - name: Set db2u-release configmap content + no_log: true + set_fact: + db2_releases_content: "{{ db2_release_info.resources[0].data.json | from_json }}" + + - name: Extract major version from channel + set_fact: + db2_major_version: "{{ db2_channel | regex_replace('^v(\\d{2})(\\d+).*', '\\1') }}" + + - name: Debug extracted version + ansible.builtin.debug: + msg: + - "DB2 Channel ........................... {{ db2_channel }}" + - "Extracted Major Version ............... {{ db2_major_version }}" + - "Filter Pattern ........................ ^s{{ db2_major_version }}" + + - name: Determine DB2 version based on channel + set_fact: + db2_version: >- + {{ + (db2_releases_content.databases.db2u.keys() + | select('match', '^s' + db2_major_version) + | sort) + | last + }} + when: + - (db2_releases_content.databases.db2u.keys() | select('match', '^s' + db2_major_version) | list | length) > 0 + + when: db2_version is not defined or db2_version == "" + +- name: Debug DB2 channel and determined version + ansible.builtin.debug: + msg: + - "DB2 Channel ........................... {{ db2_channel }}" + - "DB2 Version (determined) .............. {{ db2_version }}" + +- name: Debug available DB2 versions + ansible.builtin.debug: + msg: + - "Available DB2 Versions ................ {{ db2_releases_content.databases.db2u.keys() | list }}" + when: db2_releases_content is defined + +# 11. Fail if required parameters are not set +# ----------------------------------------------------------------------------- +- name: "Verify db2_channel and db2_version set" + assert: + that: + - db2_channel is defined and db2_channel != "" + - db2_version is defined and db2_version != "" + fail_msg: "Unable to determine db2_channel and/or db2_version" + +- name: Debug DB2 upgrade channel and version + ansible.builtin.debug: + msg: + - "Db2 Channel ............................ {{ db2_channel }}" + - "Db2 Version ............................ {{ db2_version }}" + +# 12. Lookup db2 instance to see if it exists already +# ----------------------------------------------------------------------------- +- name: "See if db2u instance already exists" + kubernetes.core.k8s_info: + api_version: db2u.databases.ibm.com/v1 + name: "{{ db2_instance_name | lower }}" + namespace: "{{db2_namespace}}" + kind: Db2uCluster + register: initial_db2_cluster_lookup + +# 13. Create a Db2 instance +# ----------------------------------------------------------------------------- +- name: "Create db2 instance" + kubernetes.core.k8s: + apply: yes + template: "templates/db2ucluster.yml.j2" + +- name: "Set db2 instance timezone" + include_tasks: "tasks/install/setup_timezone.yml" + when: + - db2_timezone is defined + - db2_timezone != "" + +# 14. Wait for the cluster to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for db2u instance to be ready (5m delay)" + kubernetes.core.k8s_info: + api_version: db2u.databases.ibm.com/v1 + name: "{{ db2_instance_name | lower }}" + namespace: "{{db2_namespace}}" + kind: Db2uCluster + register: db2_cluster_lookup + until: + - db2_cluster_lookup.resources is defined + - db2_cluster_lookup.resources | length == 1 + - db2_cluster_lookup.resources[0].status is defined + - db2_cluster_lookup.resources[0].status.state is defined + - db2_cluster_lookup.resources[0].status.state == "Ready" + retries: 24 # Approximately 2 hours before we give up + delay: 300 # 5 minutes + +# 15. Configure a public route for Db2 +# ----------------------------------------------------------------------------- +- name: Lookup db2u Engn Service + kubernetes.core.k8s_info: + api_version: v1 + kind: Service + name: "c-{{db2_instance_name | lower}}-db2u-engn-svc" + namespace: "{{db2_namespace}}" + register: _db2_instance_engn_svc + until: + - _db2_instance_engn_svc.resources[0] is defined + retries: 15 # approx 5 minutes before we give up + delay: 20 # seconds + +- name: Lookup db2u TLS certificates + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "db2u-certificate-{{db2_instance_name}}" + namespace: "{{db2_namespace}}" + register: _db2_instance_certificates + +- name: Set Db2u certificates as Facts + set_fact: + db2_ca_pem: "{{ _db2_instance_certificates.resources[0].data['ca.crt'] | b64decode }}" + db2_tls_crt: "{{ _db2_instance_certificates.resources[0].data['tls.crt'] | b64decode }}" + db2_tls_key: "{{ _db2_instance_certificates.resources[0].data['tls.key'] | b64decode }}" + when: + - _db2_instance_certificates is defined + - (_db2_instance_certificates.resources | length > 0) + +- name: Set Db2u TLS port + set_fact: + db2_tls_serviceport: "{{item.targetPort}}" + when: "item.name == 'ssl-server'" + loop: "{{_db2_instance_engn_svc.resources[0].spec.ports}}" + +- name: "Create dedicated route: db2u-{{ db2_instance_name }}-tls-route" + kubernetes.core.k8s: + apply: yes + template: "templates/tlsroute.yml.j2" + +- name: Lookup existing db2u-tls-route + kubernetes.core.k8s_info: + api_version: v1 + kind: Route + name: "db2u-tls-route" + namespace: "{{db2_namespace}}" + register: _db2_tls_route + +# delete existing db2u-tls-route if that exists and matches with the same host/location from route created by previous step. +# that way we ensure that clean up should only happen if there's a conflicting route present. +- name: Clean up db2u-tls-route if needed + vars: + expected_host: "{{db2_instance_name | lower }}-{{db2_namespace}}.{{_cluster_subdomain.resources[0].spec.domain}}" + when: + - _db2_tls_route.resources | length > 0 + - _db2_tls_route.resources[0].spec.host == expected_host + kubernetes.core.k8s: + api_version: v1 + kind: Route + name: "db2u-tls-route" + namespace: "{{db2_namespace}}" + state: absent + + +## 16. create an LDAP user if db2_ldap_username specified +# ----------------------------------------------------------------------------- +- name: Create LDAP user if username and password is provided + include_tasks: tasks/install/create_ldap_user.yml + when: + - db2_ldap_username is defined + - db2_ldap_username != "" + - db2_ldap_password is defined + - db2_ldap_password != "" + - db2_rotate_password == false + +- debug: + msg: + - "{{db2_ldap_username}}" + - "{{db2_rotate_password}}" + +# 17. Rotate db2 ldap password +# ----------------------------------------------------------------------------- +- name: Rotate Db2 LDAP password if db2_rotate_password is True and username is provided + include_tasks: tasks/install/rotate_ldap_user_password.yml + when: + - db2_ldap_username is defined + - db2_ldap_username != "" + - db2_rotate_password == true + +# 18. Wait for the statefulset to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for Db2 Stateful set to be ready" + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: StatefulSet + name: "c-{{ db2_instance_name | lower }}-db2u" + namespace: "{{ db2_namespace }}" + register: db2_sts + until: + - db2_sts.resources is defined + - db2_sts.resources | length > 0 + - db2_sts.resources[0].status is defined + - db2_sts.resources[0].status.replicas is defined + - db2_sts.resources[0].status.readyReplicas is defined + - db2_sts.resources[0].status.readyReplicas == db2_sts.resources[0].status.replicas + retries: 20 # approx 10 minutes before we give up + delay: 30 # seconds + +# 19. Generate a JdbcCfg for MAS configuration +# ----------------------------------------------------------------------------- +- include_tasks: tasks/install/suite_jdbccfg.yml + when: + - mas_instance_id is defined + - mas_instance_id != "" + - mas_config_dir is defined + - mas_config_dir != "" diff --git a/ibm/mas_devops/roles/db2/tasks/install/main.yml b/ibm/mas_devops/roles/db2/tasks/install/main.yml index f1c00fead8..14bcb60364 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/main.yml @@ -1,5 +1,14 @@ --- -# 1. Fail if required parameters are not set +# Detect if this is an install-from-backup scenario +- name: "Check for backup restore variables" + set_fact: + install_from_backup: "{{ (db2_backup_version is defined and db2_backup_version != '' and db2_backup_version != 'None') and (mas_backup_dir is defined and mas_backup_dir != '') }}" + +- include_tasks: tasks/install/get-backup-info.yml + when: + - install_from_backup | bool + +# Fail if required parameters are not set # ----------------------------------------------------------------------------- - name: "Fail if required properties have not been provided" assert: @@ -7,470 +16,13 @@ - db2_instance_name is defined and db2_instance_name != "" - ibm_entitlement_key is defined and ibm_entitlement_key != "" fail_msg: "One or more required properties have not been set" + when: not install_from_backup | bool -- name: "Fail if required db2_dbname is over 8 characters" - assert: - that: - - db2_dbname is defined and db2_dbname != "" - - db2_dbname | length <= 8 - fail_msg: "Property value of db2_dbname is set to '{{ db2_dbname }}' and is greater than 8 character long." - -# 2. Load default storage classes (if not provided by the user) -# ----------------------------------------------------------------------------- -- include_tasks: tasks/install/determine-storage-classes.yml - -# 3. Setup the norootsquash daemonsets for db2u pods to work with NFS backed storage -# ----------------------------------------------------------------------------- -- include_tasks: tasks/install/setup_norootsquash.yml - when: - - db2_meta_storage_class is defined - - db2_data_storage_class is defined - - '"ibmc-file" in db2_data_storage_class or "ibmc-file" in db2_meta_storage_class or "ibmc-vpc-file" in db2_data_storage_class or "ibmc-vpc-file" in db2_meta_storage_class' - -# 4. Provide debug information to the user -# ----------------------------------------------------------------------------- -- name: "Debug - MAS" - debug: - msg: - - "MAS Instance ID ........................ {{ mas_instance_id }}" - - "MAS Config directory ................... {{ mas_config_dir }}" - - "MAS Instance ID ........................ {{ mas_instance_id | default('', true) }}" - - "MAS Workspace ID ....................... {{ mas_workspace_id | default('', true) }}" - - "MAS Application ID ..................... {{ mas_application_id | default('', true) }}" - - "MAS Config Directory ................... {{ mas_config_dir | default('', true) }}" - - "MAS Config Scope ....................... {{ mas_config_scope | default('', true) }}" - -- name: "Debug - Affinity & Tolerations" - debug: - msg: - - "Affinity key ........................... {{ db2_affinity_key | default('', true) }}" - - "Affinity value ......................... {{ db2_affinity_value | default('', true) }}" - - - "Toleration key ......................... {{ db2_tolerate_key | default('', true) }}" - - "Toleration value ....................... {{ db2_tolerate_value | default('', true) }}" - - "Toleration effect ...................... {{ db2_tolerate_effect | default('', true) }}" - -- name: "Debug - Db2 Instance" - debug: - msg: - - "Namespace .............................. {{ db2_namespace }}" - - "Db2 Instance ........................... {{ db2_instance_name }}" - -- name: "Debug - Database Settings" - debug: - msg: - - "Database Name .......................... {{ db2_dbname }}" - - "4K Device Support ...................... {{ db2_4k_device_support }}" - - "Table Organization ..................... {{ db2_table_org }}" - - "TLS Version ............................ {{ tls_version }}" - - "Workload ............................... {{ db2_workload }}" - -- name: "Debug - Resources" - debug: - msg: - - "CPU Request ............................ {{ db2_cpu_requests }}" - - "CPU Limit .............................. {{ db2_cpu_limits }}" - - "Memory Request ......................... {{ db2_memory_requests }}" - - "Memory Limit ........................... {{ db2_memory_limits }}" - -- name: "Debug - Storage" - debug: - msg: - - "Meta ................................... {{ db2_meta_storage_class }} - {{ db2_meta_storage_size }} @ {{ db2_meta_storage_accessmode }}" - - "Data ................................... {{ db2_data_storage_class }} - {{ db2_data_storage_size }} @ {{ db2_data_storage_accessmode }}" - - "Backup ................................. {{ db2_backup_storage_class }} - {{ db2_backup_storage_size }} @ {{ db2_backup_storage_accessmode }}" - - "Logs ................................... {{ db2_logs_storage_class }} - {{ db2_logs_storage_size }} @ {{ db2_logs_storage_accessmode }}" - - "Temp ................................... {{ db2_temp_storage_class }} - {{ db2_temp_storage_size }} @ {{ db2_temp_storage_accessmode }}" - -# Lookup db2 operator group -- name: "Check if operator group is present in {{ db2_namespace }} namespace already" - kubernetes.core.k8s_info: - namespace: "{{ db2_namespace }}" - kind: OperatorGroup - register: db2_og_info - -# Look up the default channel for the db2u-operator package manifest -- name: Lookup db2u-operator packagemanifest - kubernetes.core.k8s_info: - api_version: packages.operators.coreos.com/v1 - kind: PackageManifest - name: db2u-operator - namespace: "{{ ibm_common_services_namespace }}" - register: db2u_manifest_info - until: db2u_manifest_info.resources[0].status.defaultChannel is defined - retries: 60 # Approximately 30 minutes before we give up - delay: 30 # seconds - when: db2_channel is not defined or db2_channel == "" or db2_version is not defined or db2_version == "" - -- name: Set db2u-operator channel - ansible.builtin.set_fact: - db2_channel: "{{ db2u_manifest_info.resources[0].status.defaultChannel }}" - when: db2_channel is not defined or db2_channel == "" - -# 5. Fail if required parameters are not set -# ----------------------------------------------------------------------------- -- name: "Verify db2_channel is set" - assert: - that: - - db2_channel is defined and db2_channel != "" - fail_msg: "Unable to determine db2_channel from catalog" - -- name: Debug DB2 upgrade channel - ansible.builtin.debug: - msg: - - "Db2 Channel ............................ {{ db2_channel }}" - -# 6. Install a Db2u Operator -# ----------------------------------------------------------------------------- -- name: "Create db2u Namespace" - kubernetes.core.k8s: - apply: yes - template: "templates/db2u_namespace.yaml" - register: _db2_namespace_result - -- name: Check if ibm-registry secret exists - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: ibm-registry - namespace: "{{ db2_namespace }}" - register: ibm_registry_secret_info - -- name: "Create ibm-registry secret if not present" - when: - - ibm_registry_secret_info.resources is defined - - ibm_registry_secret_info.resources | length == 0 - block: - - name: Set 'ibm-registry' secret content - no_log: true - vars: - entitledAuthStr: "{{ registry_user }}:{{ ibm_entitlement_key }}" - entitledAuth: "{{ entitledAuthStr | b64encode }}" - content: - - '{"auths":{"{{ registry }}/cp/cpd":{"username":"{{ registry_user }}","password":"{{ ibm_entitlement_key }}","email":"{{ registry_user }}","auth":"{{ entitledAuth }}"}' - - "}" - - "}" - set_fact: - new_secret: "{{ content | join('') }}" - - # Note: We use "new_secret | to_json " because older versions of Ansible create - # invalid json representations containing single quotes - # However, in newer versions of Ansible to_json returns an object, which can't be passed to b64encode - - name: "Generate docker secret (old Ansible versions)" - when: ansible_version.full is version_compare(2.20, '<') - set_fact: - dockerconfigjsonB64: "{{ new_secret | to_json | b64encode }}" - - - name: "Generate docker secret (new Ansible versions)" - when: ansible_version.full is version_compare(2.20, '>=') - set_fact: - dockerconfigjsonB64: "{{ new_secret | b64encode }}" - - - name: "Generate 'ibm-registry' secret" - no_log: true - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson - metadata: - name: ibm-registry - namespace: "{{ db2_namespace }}" - data: - .dockerconfigjson: "{{ dockerconfigjsonB64 }}" - register: secretUpdateResult - -- name: "Delete old db2 subscription, operand request and csv from {{ ibm_common_services_namespace }}" - include_tasks: "tasks/delete_db2_operand_request.yml" - -- name: "Create Db2 Universal Operator Subscription in {{ db2_namespace }} namespace" - ibm.mas_devops.apply_subscription: - namespace: "{{ db2_namespace }}" - package_name: db2u-operator - package_channel: "{{ db2_channel }}" - register: subscription - -# 7. Get the cluster subdomain to be used for the certificate and route creation -# ----------------------------------------------------------------------------- -- name: "Get cluster subdomain" - kubernetes.core.k8s_info: - api_version: config.openshift.io/v1 - kind: Ingress - name: cluster - register: _cluster_subdomain - -# 8. Create self-signed certificate for Db2u SSL -# ----------------------------------------------------------------------------- -- name: "Create internal CA certificate issuer" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/ca_issuer.yml.j2" - register: createCaIssuer - -- name: "Create and wait for CA certificate" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/ca_certificate.yml.j2" - wait: yes - wait_timeout: 600 #10 minutes - wait_condition: - type: Ready - status: True - register: createCaCert - -- name: "Create certificate issuer" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/issuer.yml.j2" - register: createIssuer - -- name: "Create db2u certificate" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/certificate.yml.j2" - register: createCertificate - -# 9. Wait until the Db2uCluster CRD is available +# Start the installation task # ----------------------------------------------------------------------------- -- name: "Wait until the Db2uCluster CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: "db2uclusters.db2u.databases.ibm.com" +- include_tasks: tasks/install/install.yml -# 10. Get information from the db2u-release ConfigMap +# Start database restore task # ----------------------------------------------------------------------------- -# if db2_version is not set, then we define it based on the latest version supported by the db2u-license-keys secret -# Starting with s11.5.8.0-cn3, the 's' prefix is removed in db2u-license-keys, it is recommended to use db2u-release configmap. -- block: - - name: "Wait until the db2u-release configmap is available" - no_log: true - kubernetes.core.k8s_info: - api_version: v1 - name: db2u-release - namespace: "{{ db2_namespace }}" - kind: ConfigMap - register: db2_release_info - retries: 20 # ~approx 10 minutes before we give up waiting for the CRD to be created - delay: 30 # seconds - until: - - db2_release_info.resources is defined - - db2_release_info.resources | length > 0 - - db2_release_info.resources[0].data is defined - - db2_release_info.resources[0].data | length > 0 - - - name: Set db2u-release configmap content - no_log: true - set_fact: - db2_releases_content: "{{ db2_release_info.resources[0].data.json | from_json }}" - - - name: Extract major version from channel - set_fact: - db2_major_version: "{{ db2_channel | regex_replace('^v(\\d{2})(\\d+).*', '\\1') }}" - - - name: Debug extracted version - ansible.builtin.debug: - msg: - - "DB2 Channel ........................... {{ db2_channel }}" - - "Extracted Major Version ............... {{ db2_major_version }}" - - "Filter Pattern ........................ ^s{{ db2_major_version }}" - - - name: Determine DB2 version based on channel - set_fact: - db2_version: >- - {{ - (db2_releases_content.databases.db2u.keys() - | select('match', '^s' + db2_major_version) - | sort) - | last - }} - when: - - (db2_releases_content.databases.db2u.keys() | select('match', '^s' + db2_major_version) | list | length) > 0 - - when: db2_version is not defined or db2_version == "" - -- name: Debug DB2 channel and determined version - ansible.builtin.debug: - msg: - - "DB2 Channel ........................... {{ db2_channel }}" - - "DB2 Version (determined) .............. {{ db2_version }}" - -- name: Debug available DB2 versions - ansible.builtin.debug: - msg: - - "Available DB2 Versions ................ {{ db2_releases_content.databases.db2u.keys() | list }}" - when: db2_releases_content is defined - -# 11. Fail if required parameters are not set -# ----------------------------------------------------------------------------- -- name: "Verify db2_channel and db2_version set" - assert: - that: - - db2_channel is defined and db2_channel != "" - - db2_version is defined and db2_version != "" - fail_msg: "Unable to determine db2_channel and/or db2_version" - -- name: Debug DB2 upgrade channel and version - ansible.builtin.debug: - msg: - - "Db2 Channel ............................ {{ db2_channel }}" - - "Db2 Version ............................ {{ db2_version }}" - -# 12. Lookup db2 instance to see if it exists already -# ----------------------------------------------------------------------------- -- name: "See if db2u instance already exists" - kubernetes.core.k8s_info: - api_version: db2u.databases.ibm.com/v1 - name: "{{ db2_instance_name | lower }}" - namespace: "{{db2_namespace}}" - kind: Db2uCluster - register: initial_db2_cluster_lookup - -# 13. Create a Db2 instance -# ----------------------------------------------------------------------------- -- name: "Create db2 instance" - kubernetes.core.k8s: - apply: yes - template: "templates/db2ucluster.yml.j2" - -- name: "Set db2 instance timezone" - include_tasks: "tasks/install/setup_timezone.yml" - when: - - db2_timezone is defined - - db2_timezone != "" - -# 14. Wait for the cluster to be ready -# ----------------------------------------------------------------------------- -- name: "Wait for db2u instance to be ready (5m delay)" - kubernetes.core.k8s_info: - api_version: db2u.databases.ibm.com/v1 - name: "{{ db2_instance_name | lower }}" - namespace: "{{db2_namespace}}" - kind: Db2uCluster - register: db2_cluster_lookup - until: - - db2_cluster_lookup.resources is defined - - db2_cluster_lookup.resources | length == 1 - - db2_cluster_lookup.resources[0].status is defined - - db2_cluster_lookup.resources[0].status.state is defined - - db2_cluster_lookup.resources[0].status.state == "Ready" - retries: 24 # Approximately 2 hours before we give up - delay: 300 # 5 minutes - -# 15. Configure a public route for Db2 -# ----------------------------------------------------------------------------- -- name: Lookup db2u Engn Service - kubernetes.core.k8s_info: - api_version: v1 - kind: Service - name: "c-{{db2_instance_name | lower}}-db2u-engn-svc" - namespace: "{{db2_namespace}}" - register: _db2_instance_engn_svc - until: - - _db2_instance_engn_svc.resources[0] is defined - retries: 15 # approx 5 minutes before we give up - delay: 20 # seconds - -- name: Lookup db2u TLS certificates - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: "db2u-certificate-{{db2_instance_name}}" - namespace: "{{db2_namespace}}" - register: _db2_instance_certificates - -- name: Set Db2u certificates as Facts - set_fact: - db2_ca_pem: "{{ _db2_instance_certificates.resources[0].data['ca.crt'] | b64decode }}" - db2_tls_crt: "{{ _db2_instance_certificates.resources[0].data['tls.crt'] | b64decode }}" - db2_tls_key: "{{ _db2_instance_certificates.resources[0].data['tls.key'] | b64decode }}" - when: - - _db2_instance_certificates is defined - - (_db2_instance_certificates.resources | length > 0) - -- name: Set Db2u TLS port - set_fact: - db2_tls_serviceport: "{{item.targetPort}}" - when: "item.name == 'ssl-server'" - loop: "{{_db2_instance_engn_svc.resources[0].spec.ports}}" - -- name: "Create dedicated route: db2u-{{ db2_instance_name }}-tls-route" - kubernetes.core.k8s: - apply: yes - template: "templates/tlsroute.yml.j2" - -- name: Lookup existing db2u-tls-route - kubernetes.core.k8s_info: - api_version: v1 - kind: Route - name: "db2u-tls-route" - namespace: "{{db2_namespace}}" - register: _db2_tls_route - -# delete existing db2u-tls-route if that exists and matches with the same host/location from route created by previous step. -# that way we ensure that clean up should only happen if there's a conflicting route present. -- name: Clean up db2u-tls-route if needed - vars: - expected_host: "{{db2_instance_name | lower }}-{{db2_namespace}}.{{_cluster_subdomain.resources[0].spec.domain}}" - when: - - _db2_tls_route.resources | length > 0 - - _db2_tls_route.resources[0].spec.host == expected_host - kubernetes.core.k8s: - api_version: v1 - kind: Route - name: "db2u-tls-route" - namespace: "{{db2_namespace}}" - state: absent - - -## 16. create an LDAP user if db2_ldap_username specified -# ----------------------------------------------------------------------------- -- name: Create LDAP user if username and password is provided - include_tasks: tasks/install/create_ldap_user.yml - when: - - db2_ldap_username is defined - - db2_ldap_username != "" - - db2_ldap_password is defined - - db2_ldap_password != "" - - db2_rotate_password == false - -- debug: - msg: - - "{{db2_ldap_username}}" - - "{{db2_rotate_password}}" - -# 17. Rotate db2 ldap password -# ----------------------------------------------------------------------------- -- name: Rotate Db2 LDAP password if db2_rotate_password is True and username is provided - include_tasks: tasks/install/rotate_ldap_user_password.yml - when: - - db2_ldap_username is defined - - db2_ldap_username != "" - - db2_rotate_password == true - -# 18. Wait for the statefulset to be ready -# ----------------------------------------------------------------------------- -- name: "Wait for Db2 Stateful set to be ready" - kubernetes.core.k8s_info: - api_version: apps/v1 - kind: StatefulSet - name: "c-{{ db2_instance_name | lower }}-db2u" - namespace: "{{ db2_namespace }}" - register: db2_sts - until: - - db2_sts.resources is defined - - db2_sts.resources | length > 0 - - db2_sts.resources[0].status is defined - - db2_sts.resources[0].status.replicas is defined - - db2_sts.resources[0].status.readyReplicas is defined - - db2_sts.resources[0].status.readyReplicas == db2_sts.resources[0].status.replicas - retries: 20 # approx 10 minutes before we give up - delay: 30 # seconds - -# 19. Generate a JdbcCfg for MAS configuration -# ----------------------------------------------------------------------------- -- include_tasks: tasks/install/suite_jdbccfg.yml - when: - - mas_instance_id is defined - - mas_instance_id != "" - - mas_config_dir is defined - - mas_config_dir != "" +- include_tasks: tasks/restore_database/restore-database.yml + when: install_from_backup | bool diff --git a/ibm/mas_devops/roles/db2/tasks/restore/main.yml b/ibm/mas_devops/roles/db2/tasks/restore/main.yml deleted file mode 100644 index ca0a978557..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/restore/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -# Restore Db2 Universal operator Instance Kubernetes Resources -# ------------------------------------------------------------------------- -- name: "Start Db2 Universal operator Instance restore process." - include_tasks: "{{ role_path }}/tasks/restore/restore-instance.yml" - when: not br_skip_instance | bool - -- name: "Start Database restore process." - include_tasks: "{{ role_path }}/tasks/restore/restore-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml deleted file mode 100644 index df06115f58..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml +++ /dev/null @@ -1,87 +0,0 @@ ---- -# Check db2 Restore required variables -# ----------------------------------------------------------------------------- -- name: Verify DB2 restore variables - ibm.mas_devops.verify_backup_restore_vars: - component: "db2" - action: "{{ db2_action }}_instance" - db2_backup_version: "{{ db2_backup_version }}" - mas_backup_dir: "{{ mas_backup_dir }}" - -# Restore Db2 Universal operator Instance Kubernetes Resources -# ------------------------------------------------------------------------- -- name: "Start Db2 Universal operator Instance restore process." - ibm.mas_devops.restore_db2_resources: - db2_backup_version: "{{ db2_backup_version }}" - mas_backup_dir: "{{ mas_backup_dir }}" - register: restore_db2_instance_result - -- name: "Assert Restore Db2 Universal operator Instance Kubernetes Resources result" - assert: - that: - - restore_db2_instance_result is defined - - restore_db2_instance_result.success == true - fail_msg: "Restoring Db2 Universal operator Instance Kubernetes Resources failed." - -- name: "Check if DB2 instance exist and healthy. If not, proceed with restore." - ansible.builtin.debug: - msg: - - "================== FINISHED RESTORE DB2 INSTANCE ==============================" - - "{{ restore_db2_instance_result.msg }}" - - "===============================================================================" - when: - - restore_db2_instance_result.proceed == False - -# Create Db2 Instance using info from backup CR if required -# ------------------------------------------------------------------------- -- name: "Create Db2 Instance if not present" - block: - - name: "Set fact from Db2 info" - set_fact: - db2_database_db_config: "{{ restore_db2_instance_result.db2_database_db_config }}" - db2_instance_dbm_config: "{{ restore_db2_instance_result.db2_instance_dbm_config }}" - db2_instance_registry: "{{ restore_db2_instance_result.db2_instance_registry }}" - db2_channel: "{{ restore_db2_instance_result.db2_channel }}" - db2_version: "{{ restore_db2_instance_result.db2_version }}" - db2_instance_name: "{{ restore_db2_instance_result.db2_instance_name }}" - db2_namespace: "{{ restore_db2_instance_result.db2_namespace }}" - db2_type: "{{ restore_db2_instance_result.db2_type }}" - db2_dbname: "{{ restore_db2_instance_result.db2_database_name }}" - db2_table_org: "{{ restore_db2_instance_result.db2_table_org }}" - db2_cpu_requests: "{{ restore_db2_instance_result.db2_cpu_requests }}" - db2_memory_requests: "{{ restore_db2_instance_result.db2_memory_requests }}" - db2_cpu_limits: "{{ restore_db2_instance_result.db2_cpu_limits }}" - db2_memory_limits: "{{ restore_db2_instance_result.db2_memory_limits }}" - - - name: "Set fact LDAP credentials if exist." - set_fact: - db2_ldap_username: "{{ restore_db2_instance_result.db2_ldap_username }}" - db2_ldap_password: "{{ restore_db2_instance_result.db2_ldap_password }}" - when: restore_db2_instance_result.db2_ldap_username is defined and restore_db2_instance_result.db2_ldap_password is defined - - - name: DB2 Instance restore configuration summary - ansible.builtin.debug: - msg: - - "================== STARTING RESTORE DB2 INSTANCE ===============================" - - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" - - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" - - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" - - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" - - "BACKUP DIR : {{ mas_backup_dir | default('UNDEFINED') }}" - - "================================================================================" - - - name: "Creating Db2 Universal operator Instance" - include_tasks: "{{ role_path }}/tasks/install/main.yml" - - - name: DB2 Instance restore summary - ansible.builtin.debug: - msg: - - "================== FINISHED RESTORE DB2 INSTANCE ===============================" - - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" - - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" - - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" - - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" - - "BACKUP DIR : {{ mas_backup_dir | default('UNDEFINED') }}" - - "================================================================================" - - when: restore_db2_instance_result.proceed == true diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/main.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/main.yml new file mode 100644 index 0000000000..1862e5e2e0 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/main.yml @@ -0,0 +1,4 @@ +--- +# Restore Db2 Universal operator database +- name: "Start Database restore process." + include_tasks: "{{ role_path }}/tasks/restore_database/restore-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml similarity index 93% rename from ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml rename to ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml index 4ff7679848..d8f23aedee 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml @@ -4,7 +4,7 @@ - name: Verify DB2 restore variables ibm.mas_devops.verify_backup_restore_vars: component: "db2" - action: "{{ db2_action }}_database" + action: "restore_database" db2_backup_version: "{{ db2_backup_version }}" mas_backup_dir: "{{ mas_backup_dir }}" db2_instance_name: "{{ db2_instance_name }}" @@ -37,4 +37,4 @@ - "================================================================================" - name: "Run restore-db-from-disk.yml when backup vendor is disk" - include_tasks: "{{ role_path }}/tasks/restore/restore-db-from-{{ backup_vendor }}.yml" + include_tasks: "{{ role_path }}/tasks/restore_database/restore-db-from-{{ backup_vendor }}.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-disk.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml similarity index 97% rename from ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-disk.yml rename to ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml index 5c6a0597aa..c3fa09343e 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-disk.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml @@ -43,7 +43,7 @@ # ----------------------------------------------------------------------------- - name: "Check DB2 is running and get DB2 pod name" ibm.mas_devops.get_db2u_pod_name: - db2_instance_name: "{{ db2_instance_name }}" + db2_instance_name: "{{ db2_instance_name | lower}}" db2_namespace: "{{ db2_namespace }}" register: db2_pod_name_result @@ -54,7 +54,7 @@ - db2_pod_name_result.success - db2_pod_name_result.pod_name != "" - db2_pod_name_result.db2_version != "" - fail_msg: "DB2 Instance {{ db2_instance_name }} is not running in namespace {{ db2_namespace }}. Ensure the DB2 instance is running." + fail_msg: "DB2 Instance {{ db2_instance_name | lower }} is not running in namespace {{ db2_namespace }}. Ensure the DB2 instance is running." - name: "Assert DB2 version matches backup version" assert: diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-s3.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml similarity index 96% rename from ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-s3.yml rename to ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml index 88b903fa23..bc3ccd4973 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-db-from-s3.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml @@ -8,7 +8,7 @@ # ----------------------------------------------------------------------------- - name: "Check DB2 is running and get DB2 pod name" ibm.mas_devops.get_db2u_pod_name: - db2_instance_name: "{{ db2_instance_name }}" + db2_instance_name: "{{ db2_instance_name | lower}}" db2_namespace: "{{ db2_namespace }}" register: db2_pod_name_result @@ -19,7 +19,7 @@ - db2_pod_name_result.success - db2_pod_name_result.pod_name != "" - db2_pod_name_result.db2_version != "" - fail_msg: "DB2 Instance {{ db2_instance_name }} is not running in namespace {{ db2_namespace }}. Ensure the DB2 instance is running." + fail_msg: "DB2 Instance {{ db2_instance_name | lower}} is not running in namespace {{ db2_namespace }}. Ensure the DB2 instance is running." - name: "Set fact db2_pod_name" set_fact: diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index f182139165..8e40ee2952 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -178,369 +178,219 @@ Set this to `true` to confirm you want to upgrade your existing Mongo instance f - Environment Variable: `MONGODB_V8_UPGRADE` - Default Value: `false` -Role Variables - Backup and Restore +Role Variables - Backup and Restore (CE Operator) ------------------------------------------------------------------------------- +### mongodb_action +For backup and restore operations, set `mongodb_action` to one of the following: +- `backup`: Create a backup of MongoDB databases and optionally instance resources +- `restore_database`: Restore MongoDB databases from a backup to an existing instance +- `install`: Deploy a new MongoDB instance and restore data from backup + +- Environment Variable: `MONGODB_ACTION` +- Default Value: `install` + ### mas_backup_dir -Parent directory to store backups. Each component will create its own directory in this parent directory. +**Required for backup/restore operations**. Local directory path where backup files will be stored or read from. -- **Required** only when `MONGODB_ACTION=backup or restore` - Environment Variable: `MAS_BACKUP_DIR` -- Default: None +- Default Value: None +- Example: `/tmp/masbr` ### mongodb_backup_version -When `MONGODB_ACTION=backup`, Set version to override the default `YYMMDD-HHMMSS` timestamp version. -When `MONGODB_ACTION=restore`, Set version to use in the restore. Backup with this version will be used to restore. +**Required for restore operations**. The backup version timestamp to restore from. This is automatically generated during backup in the format `YYMMDD-HHMMSS`. -- **Required** when `MONGODB_ACTION=restore`, default: None -- **Optional when `MONGODB_ACTION=backup`, default: `YYMMDD-HHMMSS` timestamp. - Environment Variable: `MONGODB_BACKUP_VERSION` +- Default Value: YYMMDD-HHMMSS +- Example: `251212-021316` ### br_skip_instance -Set this to `true` to skip the instance level or kubernetes resources backup/restore i.e., skips instance/kubernetes resources and processes only database backup/restore. +Controls whether to backup MongoDB instance resources (secrets, certificates, issuers) along with database data. Set to `false` to include instance resources in the backup. -- Optional - Environment Variable: `BR_SKIP_INSTANCE` -- Default Value: `false` - -MongoDB Community Edition Backup and Restore -------------------------------------------------------------------------------- +- Default Value: `true` -### Overview +### mongodb_instance_name +The name of the MongoDB instance to backup. -The MongoDB role supports comprehensive backup and restore operations for MongoDB Community Edition deployments. The process is designed to handle both cluster-level Kubernetes resources and database-level data, providing flexibility for different recovery scenarios. +- Environment Variable: `MONGODB_INSTANCE_NAME` +- Default Value: `mas-mongo-ce` -### Backup and Restore Architecture +### mas_app_id +Optional. Specific MAS application ID for targeted backup/restore operations. -The backup and restore operations are split into two independent phases: +- Environment Variable: `MAS_APP_ID` +- Default Value: None -#### 1. Instance-Level Backup/Restore (Optional) -Captures and restores Kubernetes resources required to recreate the MongoDB cluster: -- MongoDBCommunity custom resource (CR) -- Configuration parameters (replicas, storage, CPU/memory settings) -- User credentials and authentication settings -- TLS certificates and secrets -- Issuers and certificate resources -#### 2. Database-Level Backup/Restore (Always Executed) -Performs backup and restore of MongoDB databases: -- Uses `mongodump` and `mongorestore` utilities -- Backs up only databases matching the MAS instance ID pattern -- Excludes transient collections (e.g., Monitor sessions) -- Excludes system collections that should be regenerated by MAS +Backup and Restore Operations (authored by IBM Bob) +------------------------------------------------------------------------------- ### Backup Process -#### Prerequisites -- MongoDB Community Edition instance must be running -- Sufficient storage space in the backup directory -- OpenShift CLI (`oc`) access to the cluster -- Appropriate permissions to access MongoDB namespace and resources - -#### Backup Workflow - -**Phase 1: Instance Backup** (if `BR_SKIP_INSTANCE=false`) -1. Retrieves the MongoDBCommunity CR and related resources -2. Backs up secrets (admin credentials, TLS certificates) -3. Backs up issuers and certificates -4. Saves all resources to `/backup--mongoce/resources/` - -**Phase 2: Database Backup** (always executed) -1. Connects to the MongoDB primary node -2. Creates a backup role and user (`bradmin`) with full privileges -3. Identifies databases matching pattern: `^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)` -4. Executes `mongodump` for each database -5. Excludes `sessions` collection from Monitor databases -6. Creates compressed archive: `mongodump-.tar.gz` -7. Saves backup metadata in `mongodb-info.yaml` -8. Cleans up temporary files from MongoDB pod - -#### Backup Directory Structure +The MongoDB backup operation creates a backup of your MongoDB instance and databases associated with your MAS instance: + +1. **Database Backup**: Uses `mongodump` to export databases with filter `"^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)"` to match databases like `mas--` or `iot--`. +2. **Instance Resources** (optional): Backs up Kubernetes resources including: + - MongoDB Custom Resource (CR) + - Secrets (TLS certificates, credentials) + - Certificate resources + - Issuer resources + +**Backup Directory Structure:** ``` -/ -└── backup--mongoce/ - ├── resources/ - │ ├── cr.yml # MongoDBCommunity CR - │ ├── secrets/ # User and TLS secrets - │ ├── issuers/ # Certificate issuers - │ └── certificates/ # TLS certificates - └── data/ - ├── mongodump-.tar.gz # Database backup archive - └── mongodb-info.yaml # Backup metadata +/tmp/masbr/ +└── backup--mongoce/ + ├── data/ + │ ├── mongodump-.tar.gz + │ └── mongodb-info.yaml + └── resources/ + ├── cr.yml + ├── secrets/ + ├── certificates/ + └── issuers/ ``` -#### Backup Variables +### Restore Process + +**Database Restore (`restore_database` action):** +- Restores database data to an existing MongoDB instance +- Requires MongoDB version to match the backup version +- Uses `mongorestore` to import required databases + +**Install from Backup (`install` action):** +- Validates backup files +- Creates namespace and restores resources to namespace +- Gets MongoDbCE instance details from backup data +- Installs MongoDb with configuration from backup +- Waits for instance to be ready +- Restores database data after instance is ready + +### Important Considerations -| Variable | Required | Default | Description | -|----------|----------|---------|-------------| -| `MONGODB_ACTION` | Yes | `install` | Must be set to `backup` | -| `MAS_INSTANCE_ID` | Yes | None | MAS instance ID whose databases to backup | -| `MAS_BACKUP_DIR` | Yes | None | Parent directory for backup storage | -| `MONGODB_NAMESPACE` | No | `mongoce` | Namespace where MongoDB is installed | -| `MONGODB_INSTANCE_NAME` | No | `mas-mongo-ce` | Name of the MongoDB instance | -| `MONGODB_BACKUP_VERSION` | No | `YYMMDD-HHMMSS` | Custom version identifier for the backup | -| `BR_SKIP_INSTANCE` | No | `false` | Skip instance-level backup if `true` | +**Version Compatibility:** +- Target MongoDB version must match the backup version +- Version upgrades should be performed separately, not during restore -#### Backup Example +**Storage Requirements:** +- Ensure sufficient storage in the backup directory +- Plan for at least 2x the database size for backup storage + +**Security:** +- Backup files contain sensitive data and credentials +- Secure backup directory with appropriate permissions +- Consider encrypting backups for long-term storage + +**Performance:** +- Backup operations may impact MongoDB performance +- Schedule backups during low-usage periods +- Monitor resource utilization during backup/restore + +### Backup and Restore Best Practices + +1. **Regular Backups**: Schedule automated backups at regular intervals +2. **Test Restores**: Periodically test restore procedures in non-production environments +3. **Monitor Operations**: Implement monitoring and alerting for backup failures +4. **Backup Validation**: Verify backup integrity after completion +5. **Retention Policy**: Implement and document backup retention policies +6. **Disaster Recovery**: Include MongoDB backup/restore in your DR plan + +### Backup and Restore Issues + +**Backup Failures:** + +- **Permission Errors**: Verify backup role and user creation succeeded. Check MongoDB pod logs with `oc logs -n mongoce -c mongod` +- **Disk Space Issues**: Ensure sufficient space in backup directory. Clean up old backups if needed like `/tmp/`. Check with `df -h`. +- **Pod Access Issues**: Verify pod is running and accessible. Check network connectivity to the cluster + +**Restore Failures:** + +- **Version Mismatch**: Ensure target MongoDB version matches backup version. Deploy correct version before restoring +- **Authentication Errors**: Verify admin credentials are correct. Check MongoDB health status +- **Missing Backup Files**: Verify backup directory path and version. Ensure backup completed successfully +- **Data Inconsistency**: Verify backup integrity. Check restore logs for errors. Consider re-running restore + +**General Issues:** + +- **Pods Not Starting After Restore**: Check pod events with `oc describe pod -n mongoce `. Verify PVCs are bound. Check resource limits and node capacity +- **Connection Issues**: Verify network policies and service configurations. Check certificate validity + +Example Playbooks +------------------------------------------------------------------------------- + +### Install (CE Operator) ```yaml - hosts: localhost any_errors_fatal: true vars: - mongodb_action: backup + mongodb_storage_class: ibmc-block-gold mas_instance_id: masinst1 - mas_backup_dir: /mnt/backups - mongodb_namespace: mongoce - mongodb_instance_name: mas-mongo-ce - # Optional: mongodb_backup_version: "20260109-120000" - # Optional: br_skip_instance: false + mas_config_dir: ~/masconfig roles: - ibm.mas_devops.mongodb ``` -### Restore Process - -#### Prerequisites -- Valid backup directory with complete backup data -- OpenShift CLI (`oc`) access to the cluster -- Appropriate permissions to create/modify resources -- Target namespace must exist -- Storage class from backup must be available (or default storage class) - -#### Restore Workflow - -**Phase 1: Instance Restore** (conditional) -- **If MongoDB instance does NOT exist or is NOT running:** - 1. Reads MongoDBCommunity CR from backup - 2. Determines appropriate storage class to use - 3. Restores secrets (admin credentials, TLS certificates) - 4. Restores issuers and certificates - 5. Installs MongoDB operator and creates instance - 6. Waits for MongoDB cluster to become ready - -- **If MongoDB instance already exists and is running:** - - Instance restore is automatically skipped - - Proceeds directly to database restore - -**Phase 2: Database Restore** (always executed if instance is running) -1. Validates backup data files exist -2. Verifies MongoDB version compatibility -3. Creates restore role and user -4. Copies backup archive to MongoDB pod -5. Extracts backup data -6. Executes `mongorestore` with exclusions: - - `mas__adoptionusage.*` - - `mas__catalog.*` - - `mas__core.OauthClient` - - `mas__core.OauthToken` - - `mas__core.bindings` - - `mas__core.graphiteconfigtool.*` - - `mas__core.workspaces` -7. Cleans up temporary files - -#### Restore Variables - -| Variable | Required | Default | Description | -|----------|----------|---------|-------------| -| `MONGODB_ACTION` | Yes | `install` | Must be set to `restore` | -| `MAS_INSTANCE_ID` | Yes | None | MAS instance ID to restore | -| `MAS_BACKUP_DIR` | Yes | None | Parent directory containing backups | -| `MONGODB_BACKUP_VERSION` | Yes | None | Version identifier of backup to restore | -| `MONGODB_NAMESPACE` | No | `mongoce` | Target namespace for restore | -| `MONGODB_INSTANCE_NAME` | No | `mas-mongo-ce` | Name for the MongoDB instance | -| `BR_SKIP_INSTANCE` | No | `false` | Skip instance-level restore if `true` | - -#### Restore Example +### Backup (CE Operator) ```yaml - hosts: localhost any_errors_fatal: true vars: - mongodb_action: restore mas_instance_id: masinst1 - mas_backup_dir: /mnt/backups - mongodb_backup_version: "260109-120000" - mongodb_namespace: mongoce - mongodb_instance_name: mas-mongo-ce - # Optional: br_skip_instance: false + mas_backup_dir: /tmp/masbr + mongodb_action: backup roles: - ibm.mas_devops.mongodb ``` -### Backup and Restore Scenarios - -#### Scenario 1: Full Backup and Restore -Complete backup of instance and databases, restore to new environment. -```bash -# Backup -export MONGODB_ACTION=backup -export MAS_INSTANCE_ID=prod1 -export MAS_BACKUP_DIR=/backups -ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role - -# Restore (to new cluster) -export MONGODB_ACTION=restore -export MAS_INSTANCE_ID=prod1 -export MAS_BACKUP_DIR=/backups -export MONGODB_BACKUP_VERSION=260109-120000 -ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role -``` +### Backup with Instance Resources (CE Operator) +Create a complete backup including both database data and instance resources (secrets, certificates, issuers). -#### Scenario 2: Database-Only Backup and Restore -Backup only databases, restore to existing MongoDB instance. -```bash -# Backup (skip instance) -export MONGODB_ACTION=backup -export MAS_INSTANCE_ID=prod1 -export MAS_BACKUP_DIR=/backups -export BR_SKIP_INSTANCE=true -ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role - -# Restore (to existing instance) -export MONGODB_ACTION=restore -export MAS_INSTANCE_ID=prod1 -export MAS_BACKUP_DIR=/backups -export MONGODB_BACKUP_VERSION=260109-120000 -export BR_SKIP_INSTANCE=true -ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role -``` - -#### Scenario 3: Disaster Recovery -Full restore including instance recreation. -```bash -export MONGODB_ACTION=restore -export MAS_INSTANCE_ID=prod1 -export MAS_BACKUP_DIR=/backups -export MONGODB_BACKUP_VERSION=260109-120000 -export MONGODB_NAMESPACE=mongoce -ROLE_NAME=mongodb ansible-playbook ibm.mas_devops.run_role -``` - -### Important Considerations - -#### Version Compatibility -- Target MongoDB version must match the backup MongoDB version -- The restore process validates version compatibility before proceeding -- Cross-version restores are not supported - -#### Storage Class Handling -The restore process determines storage class in the following order: -1. If existing MongoDB instance found: uses its storage class -2. If no instance exists: uses storage class from backup CR -3. If backup storage class doesn't exist: uses cluster default storage class - -#### Database Filtering -- Only databases matching the MAS instance ID pattern are backed up -- Pattern: `^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)` -- Examples: `mas_prod1_core`, `iot-prod1-data` - -#### Excluded Collections -The following collections are excluded from restore as they should be regenerated: -- Adoption usage data -- Catalog data -- OAuth tokens and clients -- Core bindings and workspaces -- Graphite configuration - -#### Security Considerations -- Backup files contain sensitive data including credentials -- Secure the backup directory with appropriate permissions -- Use encryption for backup storage and transmission -- Rotate backup credentials regularly - -### Troubleshooting - -#### Backup Failures - -**Issue: Cannot connect to MongoDB primary** -``` -Error checking if node is primary. Exiting. -``` -**Solution:** Verify MongoDB cluster is healthy and all pods are running. - -**Issue: Insufficient disk space** -``` -Error creating tar.gz archive of backup files. -``` -**Solution:** Ensure adequate space in backup directory and MongoDB pod. - -**Issue: Permission denied copying files** -``` -Error copying backup files from mongo pod -``` -**Solution:** Verify OpenShift permissions and pod access. - -#### Restore Failures - -**Issue: Version mismatch** -``` -Target MongoCE version X does NOT match backup MongoCE version Y +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: masinst1 + mas_backup_dir: /tmp/masbr + mongodb_action: backup + br_skip_instance: false + roles: + - ibm.mas_devops.mongodb ``` -**Solution:** Ensure target MongoDB version matches backup version. -**Issue: Backup files not found** -``` -Required backup data files are missing -``` -**Solution:** Verify backup directory path and version are correct. +### Restore Database (CE Operator) +Restore MongoDB databases from a backup to an existing MongoDB instance. -**Issue: Storage class not available** -``` -Storage class from backup does not exist +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mongodb_action: restore_database + mas_instance_id: masinst1 + mongodb_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + mongodb_namespace: mongoce + mongodb_instance_name: mas-mongo-ce + roles: + - ibm.mas_devops.mongodb ``` -**Solution:** Create the storage class or let restore use default. - -### Best Practices - -1. **Regular Backups**: Schedule automated backups using cron or OpenShift CronJobs -2. **Backup Retention**: Implement a retention policy to manage backup storage -3. **Test Restores**: Regularly test restore procedures in non-production environments -4. **Monitor Backup Size**: Track backup sizes to anticipate storage requirements -5. **Document Procedures**: Maintain runbooks for backup and restore operations -6. **Secure Backups**: Encrypt backups at rest and in transit -7. **Verify Backups**: Validate backup integrity after completion -8. **Multiple Instances**: For shared MongoDB, backup each MAS instance separately -### Backup Automation Example +### Install from Backup (CE Operator) +Deploy a new MongoDB instance using configuration from a backup and restore data from a backup. This is useful for disaster recovery or migrating MongoDB to a new cluster. ```yaml -# OpenShift CronJob for automated MongoDB backups -apiVersion: batch/v1 -kind: CronJob -metadata: - name: mongodb-backup - namespace: mongoce -spec: - schedule: "0 2 * * *" # Daily at 2 AM - jobTemplate: - spec: - template: - spec: - serviceAccountName: mongodb-backup-sa - containers: - - name: backup - image: quay.io/ibmmas/ansible-devops:latest - env: - - name: MONGODB_ACTION - value: "backup" - - name: MAS_INSTANCE_ID - value: "prod1" - - name: MAS_BACKUP_DIR - value: "/backups" - - name: BR_SKIP_INSTANCE - value: "false" - volumeMounts: - - name: backup-storage - mountPath: /backups - command: - - ansible-playbook - - ibm.mas_devops.run_role - volumes: - - name: backup-storage - persistentVolumeClaim: - claimName: mongodb-backup-pvc - restartPolicy: OnFailure +- hosts: localhost + any_errors_fatal: true + vars: + mongodb_action: install + mas_instance_id: masinst1 + mongodb_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + roles: + - ibm.mas_devops.mongodb ``` + Role Variables - IBM Cloud ------------------------------------------------------------------------------- ### ibm_mongo_name @@ -822,48 +672,9 @@ Mongo Certificates, please refer to the below example playbook section for detai - Environment Variable: `CERTIFICATES` - Default Value: None - - Example Playbooks ------------------------------------------------------------------------------- -### Install (CE Operator) -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - mongodb_storage_class: ibmc-block-gold - mas_instance_id: masinst1 - mas_config_dir: ~/masconfig - roles: - - ibm.mas_devops.mongodb -``` - -### Backup (CE Operator) -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - mas_instance_id: masinst1 - mas_backup_dir: /tmp/masbr - mongodb_action: backup - roles: - - ibm.mas_devops.mongodb -``` - -### Restore (CE Operator) -```yaml -- hosts: localhost - any_errors_fatal: true - vars: - mongodb_action: restore - mas_instance_id: masinst1 - mongodb_backup_version: 251212-021316 - mas_backup_dir: /tmp/masbr - roles: - - ibm.mas_devops.mongodb -``` - ### Install (IBM Cloud) ```yaml - hosts: localhost diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index 29e2fb4b01..c15a37f071 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -5,6 +5,8 @@ mongodb_provider: "{{ lookup('env','MONGODB_PROVIDER') | default('community', Tr # When these are defined we will generate a MAS MongoCfg template mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" + +# Supported actions: install, uninstall, backup, restore, restore_database mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('install', True) }}" mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/main.yml b/ibm/mas_devops/roles/mongodb/tasks/main.yml index 72e7f79484..889fa4c543 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/main.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/main.yml @@ -6,6 +6,13 @@ that: mongodb_provider is defined and mongodb_provider != "" fail_msg: "mongodb_provider property is required" +- name: "Fail if mongodb_action is not provided or invalid" + assert: + that: + - mongodb_action is defined + - mongodb_action in ["install", "uninstall", "backup", "restore_database"] + fail_msg: "mongodb_action property is required and must be one of 'install', 'uninstall', 'backup', 'restore_database'" + # 2. Run the install / uninstall for specified provider # ----------------------------------------------------------------------------- - include_tasks: "tasks/providers/{{ mongodb_provider }}/{{ mongodb_action }}.yml" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index 967d19bf34..84523316c9 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -39,13 +39,13 @@ # ----------------------------------------------------------------------------- - name: Create create-role-user.sh script in local /tmp ansible.builtin.template: - src: create-role-user.sh.j2 + src: community/backup-restore/create-role-user.sh.j2 dest: /tmp/create-role-user.sh mode: '777' - name: Create database-backup.sh script in local /tmp ansible.builtin.template: - src: database-backup.sh.j2 + src: community/backup-restore/database-backup.sh.j2 dest: /tmp/database-backup.sh mode: '777' diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-cluster.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-cluster.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-cluster-from-backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-backup-info.yml similarity index 71% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-cluster-from-backup.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-backup-info.yml index 7ac60cce2b..27be65f306 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-cluster-from-backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-backup-info.yml @@ -1,10 +1,20 @@ --- - -# Prepare for restore +# Check mongodb restore required variables # ----------------------------------------------------------------------------- +- name: "Fail if require variables for Mongodb {{ mongodb_action }} are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_backup_version: "{{ mongodb_backup_version }}" + action: "restore" + component: "mongodb" + +- name: "Set fact: backup dir paths" + set_fact: + mongodb_backup_dir: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" + mongodb_resource_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/resources" + mongodb_data_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/data" -# Get mongodb backup CR information -# ----------------------------------------------------------------------------- - name: "Check and get mongodb backup cr.yml" ibm.mas_devops.get_mongodb_cr_to_restore: mongodb_resource_path: "{{ mongodb_resource_path }}" @@ -15,10 +25,13 @@ set_fact: mongodb_cr: "{{ mongodb_backup_cr_result.mongodb_cr }}" -- name: "Set fact: Retrieve information from backup CR" +- name: "Set fact: Retrieve namespace and instance from backup CR" set_fact: mongodb_namespace: "{{ mongodb_cr.metadata.namespace }}" mongodb_instance_name: "{{ mongodb_cr.metadata.name }}" + +- name: "Set fact: Retrieve information from backup CR" + set_fact: mongo_extras_version: "{{ mongodb_cr.spec.version }}" target_mongodb_version: "{{ mongodb_cr.spec.version }}" mongodb_security: "{{ mongodb_cr.spec.security }}" @@ -28,15 +41,14 @@ mongodb_memory_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('1Gi') }}" mongodb_cpu_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('250m') }}" mongodb_memory_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('512Mi') }}" - backup_mongodb_storage_class: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName }}" mongodb_storage_capacity_data: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.resources.requests.storage }}" mongodb_storage_capacity_logs: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[1].spec.resources.requests.storage }}" - name: "Debug: Mongodb restore variables" debug: msg: - - "mongodb_namespace: {{ mongodb_namespace }}" - "mongodb_instance_name: {{ mongodb_instance_name }}" + - "mongodb_namespace: {{ mongodb_namespace }}" - "mongo_extras_version: {{ mongo_extras_version }}" - "target_mongodb_version: {{ target_mongodb_version }}" - "mongodb_security: {{ mongodb_security }}" @@ -46,30 +58,9 @@ - "mongodb_memory_limits: {{ mongodb_memory_limits }}" - "mongodb_cpu_requests: {{ mongodb_cpu_requests }}" - "mongodb_memory_requests: {{ mongodb_memory_requests }}" - - "backup_mongodb_storage_class: {{ backup_mongodb_storage_class }}" - "mongodb_storage_capacity_data: {{ mongodb_storage_capacity_data }}" - "mongodb_storage_capacity_logs: {{ mongodb_storage_capacity_logs }}" -# Prepare storage class -# ----------------------------------------------------------------------------- -# Set storage class name to use for mongodb restore. -# If existing mongo instance found, use its storage class. -# Else use storage class from backup CR. -# If that does not exist, use default storage class. -- name: "Determine storage class to use for MongoDB restore" - ibm.mas_devops.verify_mongoce_version: - mongodb_instance_name: "{{ mongodb_instance_name }}" - mongodb_namespace: "{{ mongodb_namespace }}" - backup_mongodb_storage_class: "{{ backup_mongodb_storage_class }}" - register: storage_class_check_result - -- name: "Set fact: mongodb_storage_class to use for restore" - set_fact: - mongodb_storage_class: "{{ storage_class_check_result.storage_class }}" - when: - - storage_class_check_result is defined - - storage_class_check_result.storage_class is defined - # Restore MongoDb instance's resources from backup # We will restore the following resources from backup: # - User, TLS Secrets defined in the MongoDb CR @@ -93,8 +84,3 @@ - restore_mongoce_resources_result.restored is defined - restore_mongoce_resources_result.restored == True fail_msg: "Failed to restore MongoDB instance resources from backup." - -# Install the selected MongoDb version for restore -# ----------------------------------------------------------------------------- -- name: "community : Setup mongo version {{ target_mongodb_version }} for restore" - include_tasks: tasks/providers/community/install-mongo.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml deleted file mode 100644 index 12b263335d..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-mongo-info.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -# Get mongodb version and status -# ----------------------------------------------------------------------------- -- name: "Get MongoDBCommunity" - kubernetes.core.k8s_info: - api_version: mongodbcommunity.mongodb.com/v1 - kind: MongoDBCommunity - name: "{{ mongodb_instance_name }}" - namespace: "{{ mongodb_namespace }}" - register: mongodbcommunity_output - -- name: "Set fact: mongodb version" - set_fact: - mongodb_version: "{{ mongodbcommunity_output.resources[0].spec.version }}" - when: - - mongodbcommunity_output is defined - - mongodbcommunity_output.resources[0] is defined - - mongodbcommunity_output.resources[0].spec.version is defined - -- name: "Fail if mongodb does not exists" - assert: - that: mongodb_version is defined - fail_msg: "Mongodb does not exists!" - -- name: "Set fact: mongodb running status" - set_fact: - mongodb_running: true - when: - - mongodbcommunity_output is defined - - mongodbcommunity_output.resources[0] is defined - - mongodbcommunity_output.resources[0].status is defined - - mongodbcommunity_output.resources[0].status.phase is defined - - mongodbcommunity_output.resources[0].status.phase == "Running" - -- name: "Fail if mongodb is not running" - assert: - that: mongodb_running is defined and mongodb_running - fail_msg: "Mongodb is not running!" - -- name: "Set MongoDBCommunity Backup output" - set_fact: - mongodbcommunity_backup_output: "{{ mongodbcommunity_output.resources[0] }}" - - - diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 0116599252..d0cc551bc6 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -48,8 +48,8 @@ - name: "Assert if target mongoce version match backup mongoce version" assert: that: - - mongodb_version == mongodb_backup_info.source_mongodb_version - fail_msg: "Target MongoCE version {{ mongodb_version }} does NOT match backup MongoCE version {{ mongodb_backup_info.source_mongodb_version }}. Restore cannot proceed." + - mongodb_version.split('.')[:2] == mongodb_backup_info.source_mongodb_version.split('.')[:2] + fail_msg: "MongoDB major.minor version mismatch.. Target MongoCE version {{ mongodb_version }} does NOT match backup MongoCE version {{ mongodb_backup_info.source_mongodb_version }}. Restore cannot proceed." - name: "debug facts" debug: @@ -65,13 +65,13 @@ - block: - name: Create create-role-user.sh script in local /tmp ansible.builtin.template: - src: create-role-user.sh.j2 + src: community/backup-restore/create-role-user.sh.j2 dest: /tmp/create-role-user.sh mode: '777' - name: Create database-restore.sh script in local /tmp ansible.builtin.template: - src: database-restore.sh.j2 + src: community/backup-restore/database-restore.sh.j2 dest: /tmp/database-restore.sh mode: '777' diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index e2b331e5c1..3f7d000303 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -29,7 +29,7 @@ # ------------------------------------------------------------------------- - name: "Start MongoCE Instance backup process." - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/backup-cluster.yml" + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/backup-instance.yml" when: not br_skip_instance | bool # Backup Mongodb database Data using mondodump diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml index ccc4bda8ed..89da497d40 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml @@ -1,10 +1,21 @@ --- -# 1. Check for existing MongoDb install +# Detect if this is an install-from-backup scenario +- name: "community : install : Check for backup restore variables" + set_fact: + install_from_backup: "{{ (mongodb_backup_version is defined and mongodb_backup_version != '' and mongodb_backup_version != 'None') and (mas_backup_dir is defined and mas_backup_dir != '') }}" + +# Restore MongoDB instance from backup if needed +- name: "community : install : Restore MongoDB instance from backup" + when: + - install_from_backup | bool + include_tasks: tasks/providers/community/backup-restore/get-backup-info.yml + +# Check for existing MongoDb install # ----------------------------------------------------------------------------- - name: "community : install : Lookup existing mongo install" include_tasks: tasks/providers/community/check-mongo-exists.yml -# 2. Load default storage classes (if not provided by the user and not an update) +# Load default storage classes (if not provided by the user and not an update) # ----------------------------------------------------------------------------- - name: Use chosen (or default) storage class when: existing_mongo_storage_class is not defined @@ -40,7 +51,7 @@ mongodb_storage_capacity_data: "{{ existing_mongodb_storage_capacity_data }}" mongodb_storage_capacity_logs: "{{ existing_mongodb_storage_capacity_logs }}" -# 3. Perform upgrade from 4.2 to 4.4 if necessary +# Perform upgrade from 4.2 to 4.4 if necessary # ----------------------------------------------------------------------------- # We will ALWAYS upgrade from 4.2 to 4.4, no matter what. v4.4 is the minimum version everyone should be running at - name: "community : install : Upgrade from v4.2 to v4.4" @@ -51,12 +62,20 @@ - existing_mongo_minor_version is defined - existing_mongo_minor_version == '4.2' -# 4. Check for existing MongoDb install (again) +# Check for existing MongoDb install (again) # ----------------------------------------------------------------------------- - name: "community : install : Lookup existing mongo install (again)" include_tasks: tasks/providers/community/check-mongo-exists.yml -# 5. Install the selected MongoDb version +# Install MongoDB # ----------------------------------------------------------------------------- - name: "community : install : Install/upgrade to chosen mongo version" include_tasks: tasks/providers/community/install-mongo.yml + +# Restore MongoDB database from backup +# ----------------------------------------------------------------------------- +- name: "community : install : Restore MongoDB database from backup" + include_tasks: tasks/providers/community/backup-restore/restore-database.yml + when: + - install_from_backup is defined + - install_from_backup | bool diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml deleted file mode 100644 index 8a99894733..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -# Check mongodb restore required variables -# ----------------------------------------------------------------------------- -- name: "Fail if require variables for Mongodb {{ mongodb_action }} are not provided" - ibm.mas_devops.verify_backup_restore_vars: - mas_instance_id: "{{ mas_instance_id }}" - mas_backup_dir: "{{ mas_backup_dir }}" - mongodb_backup_version: "{{ mongodb_backup_version }}" - mongodb_instance_name: "{{ mongodb_instance_name }}" - action: "{{ mongodb_action }}" - component: "mongodb" - -- name: "Set fact: backup dir paths" - set_fact: - mongodb_backup_dir: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" - mongodb_resource_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/resources" - mongodb_data_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/data" - -# Check for existing MongoDb install -# ----------------------------------------------------------------------------- -- name: "Check for existing MongoDB installation in namespace {{ mongodb_namespace }}" - ibm.mas_devops.verify_mongoce_version: - mongodb_instance_name: "{{ mongodb_instance_name }}" - mongodb_namespace: "{{ mongodb_namespace }}" - register: existing_mongo_info - -# Start MongoCE Cluster instance restore -# When the existing mongo instance is not running or not present -# ----------------------------------------------------------------------------- -- name: "Start MongoCE cluster instance restore" - when: - - not br_skip_instance - - existing_mongo_info is defined - - not existing_mongo_info.running - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-cluster-from-backup.yml" - -# Recheck for existing MongoDb state -# Just make sure the restore-cluster-from-backup task has created the instance and is running -# ----------------------------------------------------------------------------- -- name: "Recheck for existing MongoDB installation in namespace {{ mongodb_namespace }}" - ibm.mas_devops.verify_mongoce_version: - mongodb_instance_name: "{{ mongodb_instance_name }}" - mongodb_namespace: "{{ mongodb_namespace }}" - register: existing_mongo_info_2 - when: - - not br_skip_instance - - existing_mongo_info is defined - - not existing_mongo_info.running - -- name: "Start Database restore process." - include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database.yml" - when: (existing_mongo_info is defined and existing_mongo_info.running) or (existing_mongo_info_2 is defined and existing_mongo_info_2.running) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_database.yml new file mode 100644 index 0000000000..19eba5e2e0 --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_database.yml @@ -0,0 +1,28 @@ +--- +# Check mongodb restore required variables +# ----------------------------------------------------------------------------- +- name: "Fail if required variables for Mongodb database restore are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_backup_version: "{{ mongodb_backup_version }}" + mongodb_instance_name: "{{ mongodb_instance_name }}" + action: "restore" + component: "mongodb" + +- name: "Set fact: backup dir paths" + set_fact: + mongodb_backup_dir: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" + mongodb_data_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/data" + +# Check for existing MongoDb install +# ----------------------------------------------------------------------------- +- name: "Check for existing MongoDB installation in namespace {{ mongodb_namespace }}" + ibm.mas_devops.verify_mongoce_version: + mongodb_instance_name: "{{ mongodb_instance_name }}" + mongodb_namespace: "{{ mongodb_namespace }}" + register: existing_mongo_info + +- name: "Start Database restore process." + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/restore-database.yml" + when: existing_mongo_info is defined and existing_mongo_info.running diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.sh.j2 b/ibm/mas_devops/roles/mongodb/templates/community/backup-restore/create-role-user.sh.j2 similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/create-role-user.sh.j2 rename to ibm/mas_devops/roles/mongodb/templates/community/backup-restore/create-role-user.sh.j2 diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 b/ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-backup.sh.j2 similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-backup.sh.j2 rename to ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-backup.sh.j2 diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 b/ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-restore.sh.j2 similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/database-restore.sh.j2 rename to ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-restore.sh.j2 From 3d7dd6012681567db83d2b8e8a8162ac174304f2 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 15 Jan 2026 10:42:21 +0000 Subject: [PATCH 16/61] [patch] use backup_resource plugin for mongo & db2 backup (#2058) --- .../plugins/action/backup_db2_instance.py | 260 ------------- .../plugins/action/backup_mongo_instance.py | 366 ------------------ .../plugins/action/backup_resource.py | 2 +- .../plugins/action/get_mongoce_info.py | 17 - .../db2/tasks/backup/backup-instance.yml | 119 ++++-- .../backup-restore/backup-instance.yml | 71 +++- 6 files changed, 141 insertions(+), 694 deletions(-) delete mode 100644 ibm/mas_devops/plugins/action/backup_db2_instance.py delete mode 100644 ibm/mas_devops/plugins/action/backup_mongo_instance.py diff --git a/ibm/mas_devops/plugins/action/backup_db2_instance.py b/ibm/mas_devops/plugins/action/backup_db2_instance.py deleted file mode 100644 index 9bf739473a..0000000000 --- a/ibm/mas_devops/plugins/action/backup_db2_instance.py +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env python3 -import logging -import urllib3 - -from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client -from ansible.plugins.action import ActionBase -from ansible.errors import AnsibleError -from ansible.utils.display import Display -from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic.exceptions import NotFoundError -from mas.devops.backup import backupResources, createBackupDirectories - -from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import backupSecret, getSubscription, getDb2uInstance, getDb2VersionFromCR, isDb2uReady - -import yaml -import os -import base64 - -from mas.devops.ocp import getCR, getSecret - -urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - -display = Display() - - -def filterDb2CR(crData: dict) -> dict: - """ - Filter out unnecessary fields from a Custom Resource - """ - metadata_fields_to_remove = [ - 'annotations', - 'creationTimestamp', - 'generation', - 'resourceVersion', - 'selfLink', - 'uid', - 'managedFields' - ] - filteredCR = crData.copy() - if 'metadata' in filteredCR: - for field in metadata_fields_to_remove: - if field in filteredCR['metadata']: - del filteredCR['metadata'][field] - # remove status field - if 'status' in filteredCR: - del filteredCR['status'] - - return filteredCR - -def backupDB2uCR(db2u_cr: dict, backup_path: str) -> bool: - """ - Backup Db2uCluster/Db2uInstance CR to specified backup path - """ - try: - filteredCR = filterDb2CR(db2u_cr) - backup_file_path = os.path.join(backup_path, "cr.yaml") - with open(backup_file_path, 'w') as backup_file: - yaml.dump(filteredCR, backup_file) - display.v(f"Backed up Db2u CR to {backup_file_path}") - return True - except Exception as e: - display.v(f"Error backing up Db2u CR: {e}") - return False - -def getImagePullSecretFromCR(db2uCR: dict) -> list: - """ - Extract image pull secret name from Db2uCluster/Db2uInstance CR - """ - try: - if 'spec' in db2uCR and 'account' in db2uCR['spec'] and 'imagePullSecrets' in db2uCR['spec']['account']: - image_pull_secrets = db2uCR['spec']['account']['imagePullSecrets'] - if isinstance(image_pull_secrets, list) and len(image_pull_secrets) > 0: - return image_pull_secrets - return None - except Exception as e: - display.v(f"Error extracting image pull secret from CR: {e}") - return None - -def processUserCredentialsSecret(dynClient: DynamicClient, mas_instance_id: str, db2_instance_name: str, backup_path: str) -> bool: - """ - Process user credentials secret for Db2u instance - Check user credentials from jdbc credentials secret in Core namespace - If the user is not the default db2inst1 user, Store the username and password - this will be used during restore to set the correct user - """ - jdbc_core_secret_name = f"jdbc-{db2_instance_name}-credentials".lower() - jdbc_core_secret = getSecret(dynClient, namespace=f"mas-{mas_instance_id}-core".lower(), secret_name=jdbc_core_secret_name) - if jdbc_core_secret: - if 'data' in jdbc_core_secret and 'username' in jdbc_core_secret['data']: - username_encoded = jdbc_core_secret['data']['username'] - username = base64.b64decode(username_encoded).decode('utf-8') - if username.lower() != 'db2inst1': - # create a yaml file with LDAP user details - ldap_user_info = dict( - db2_ldap_username=username_encoded, - db2_ldap_password=jdbc_core_secret['data']['password'] - ) - ldap_info_file_path = os.path.join(backup_path, "ldapuser-NOT_SECRET.yaml") - with open(ldap_info_file_path, 'w') as ldap_info_file: - yaml.dump(ldap_user_info, ldap_info_file) - display.v(f"Wrote LDAP user info to {ldap_info_file_path}") - return True - else: - display.v(f"JDBC credentials secret '{jdbc_core_secret_name}' uses default admin user. No additional user backup needed.") - return False - -def backupCertsResources(dynClient: DynamicClient, namespace: str, kind: str, api_version: str, backup_path: str, name: str, all_discovered_secrets=None): - backed_up, not_found, failed, discovered_secrets = backupResources(dynClient=dynClient, namespace=namespace, kind=kind, name=name, api_version=api_version, backup_path=backup_path) - if backed_up > 0: - if discovered_secrets and all_discovered_secrets!=None: - display.v(f"Discovered secret(s) '{discovered_secrets}' in {kind}") - all_discovered_secrets.update(discovered_secrets) - display.v(f"Backed up resource '{kind}' '{name}'") - elif not_found > 0: - display.v(f"Resource '{kind}' '{name}' is not found.") - else: - raise AnsibleError("Failed to backup Resource '{kind}' '{name}'") - -class ActionModule(ActionBase): - - """ - Usage Example - ------------- - tasks: - - name: "Backup Db2u instance resources (CR, secrets, configmaps, issuers, certificates)" - ibm.mas_devops.backup_db2_instance: - """ - def run(self, tmp=None, task_vars=None): - super(ActionModule, self).run(tmp, task_vars) - - # Initialize DynamicClient and grab the task args - host = self._task.args.get('host', None) - api_key = self._task.args.get('api_key', None) - - dynClient = get_api_client(api_key=api_key, host=host) - - db2_instance_name = self._task.args.get('db2_instance_name', None) - db2_namespace = self._task.args.get('db2_namespace', None) - db2_backup_path = self._task.args.get('db2_backup_path', None) - mas_instance_id = self._task.args.get('mas_instance_id', None) - db2_dbname = self._task.args.get('db2_dbname', None) - - if not db2_instance_name or db2_instance_name == '': - raise AnsibleError("db2_instance_name is a required parameter and cannot be empty") - if not db2_namespace or db2_namespace == '': - raise AnsibleError("db2_namespace is a required parameter and cannot be empty") - if not db2_backup_path or db2_backup_path == '': - raise AnsibleError("db2_backup_path is a required parameter and cannot be empty") - if not mas_instance_id or mas_instance_id == '': - raise AnsibleError("mas_instance_id is a required parameter and cannot be empty") - if not db2_dbname or db2_dbname == '': - raise AnsibleError("db2_dbname is a required parameter and cannot be empty") - - # Prepare backup resource path - db2_backup_resource_path = f"{db2_backup_path}/resources" - db2_backup_secrets_path = f"{db2_backup_resource_path}/secrets" - db2_backup_issuers_path = f"{db2_backup_resource_path}/issuers" - db2_backup_certs_path = f"{db2_backup_resource_path}/certificates" - - # Create backup directory if it does not exist - createBackupDirectories([db2_backup_path, db2_backup_resource_path, db2_backup_secrets_path, db2_backup_issuers_path, db2_backup_certs_path]) - - # Get Db2uCluster or Db2uInstance CR - db2u_cr = getDb2uInstance(dynClient=dynClient, db2_instance_name=db2_instance_name, db2_namespace=db2_namespace) - if not db2u_cr: - raise AnsibleError(f"Db2uCluster or Db2uInstance CR '{db2_instance_name}' not found in namespace '{db2_namespace}'") - - # Check if db2u instance is in a ready state - if not isDb2uReady(db2u_cr): - raise AnsibleError(f"Db2u instance '{db2_instance_name}' is not in a ready state. Cannot proceed with backup.") - - # Backup Db2uCluster/Db2uInstance CR - backup_cr_status = backupDB2uCR(db2u_cr=db2u_cr, backup_path=db2_backup_resource_path) - if not backup_cr_status: - raise AnsibleError(f"Failed to backup Db2u CR for instance '{db2_instance_name}'") - - # Backup Db2u Secrets - # Backup image pull secret if specified in CR - image_pull_secrets = getImagePullSecretFromCR(db2u_cr) - if image_pull_secrets: - for secret in image_pull_secrets: - backupSecretStatus = backupSecret(dynClient=dynClient, namespace=db2_namespace, secret_name=secret, backup_path=db2_backup_secrets_path) - if not backupSecretStatus: - display.v(f"Warning: Failed to backup image pull secret '{secret}' for Db2u instance '{db2_instance_name}'") - else: - display.v(f"Backed up image pull secret '{secret}' for Db2u instance '{db2_instance_name}'") - - # Backup Db2u instance secret - db2_instance_secret_name = f"c-{db2_instance_name}-instancepassword" - backupSecretStatus = backupSecret(dynClient=dynClient, namespace=db2_namespace, secret_name=db2_instance_secret_name, backup_path=db2_backup_secrets_path) - if not backupSecretStatus: - display.v(f"Warning: Failed to backup Db2u instance secret '{db2_instance_secret_name}' for Db2u instance '{db2_instance_name}'") - else: - display.v(f"Backed up Db2u instance secret '{db2_instance_secret_name}' for Db2u instance '{db2_instance_name}'") - - # Check user credentials from jdbc credentials secret in Core namespace - # If the user is not the default admin user, backup the jdbc credentials secret as well - # this will be used during restore to set the correct user - # If secret does not exist or uses default user, no action is taken. - # Restore process will use default instance user in that case - if mas_instance_id: - processUserCredentialsSecret(dynClient=dynClient, mas_instance_id=mas_instance_id, db2_instance_name=db2_instance_name, backup_path=db2_backup_secrets_path) - - # Backup following Issuers and Certificates - # these are hardcoded as these are hardcoded in the templates. - ca_issuer = "db2u-ca-issuer" - ca_certificate = "db2u-ca-certificate" - server_issuer = "db2u-issuer" - server_certificate = f"db2u-certificate-{db2_instance_name}" - - all_discovered_secrets = set() - backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Issuer", name=ca_issuer, api_version="cert-manager.io/v1", backup_path=db2_backup_issuers_path, all_discovered_secrets=all_discovered_secrets) - backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Certificate", name=ca_certificate, api_version="cert-manager.io/v1", backup_path=db2_backup_certs_path, all_discovered_secrets=all_discovered_secrets) - backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Issuer", name=server_issuer, api_version="cert-manager.io/v1", backup_path=db2_backup_issuers_path, all_discovered_secrets=all_discovered_secrets) - backupCertsResources(dynClient=dynClient, namespace=db2_namespace, kind="Certificate", name=server_certificate, api_version="cert-manager.io/v1", backup_path=db2_backup_certs_path, all_discovered_secrets=all_discovered_secrets) - - if len(all_discovered_secrets)>0: - # Backup discovered secrets from Issuers/certs - display.v(f"Backing up the discovered secrets from Issuers/certs - {all_discovered_secrets}") - for secret in all_discovered_secrets: - backed_up, not_found, failed, discovered_secrets = backupResources(dynClient=dynClient, namespace=db2_namespace, kind="Secret", name=secret, api_version="v1", backup_path=db2_backup_secrets_path) - if failed > 0: - display.v(f"Failed to backup secret '{secret}'") - - # Get Channel details from Subscription - # Package name hardcoded to db2u-operator - subscription = getSubscription(dynClient=dynClient, namespace=db2_namespace, package_name="db2u-operator") - channel_name = None - if subscription: - if 'spec' in subscription and 'channel' in subscription['spec']: - channel_name = subscription['spec']['channel'] - else: - display.v(f"Warning: Subscription 'db2u-operator' not found in namespace '{db2_namespace}'. Cannot retrieve channel information.") - raise AnsibleError(f"Failed to retrieve Subscription for Db2u operator in namespace '{db2_namespace}'") - - # write db2-info.yaml file - # using str() to ensure no yaml serialization issues with non-string types - # Ansible wraps variables with metadata - db2_info = { - 'db2_instance_name': str(db2_instance_name), - 'db2_namespace': str(db2_namespace), - 'db2_dbname': str(db2_dbname), - 'mas_instance_id': str(mas_instance_id), - 'db2_version': getDb2VersionFromCR(db2uCR=db2u_cr), - 'db2_channel': channel_name, - 'status': 'SUCCESS' - } - - db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-backup-info.yaml") - with open(db2_info_file_path, 'w') as info_file: - yaml.dump(db2_info, info_file) - display.v(f"Wrote Db2 instance info to {db2_info_file_path}") - - return dict( - message=f"Successfully backed up Standalone Db2 Universal Operator instance '{db2_instance_name}' resources", - failed=False, - changed=False, - success=True, - ) \ No newline at end of file diff --git a/ibm/mas_devops/plugins/action/backup_mongo_instance.py b/ibm/mas_devops/plugins/action/backup_mongo_instance.py deleted file mode 100644 index 2589d86492..0000000000 --- a/ibm/mas_devops/plugins/action/backup_mongo_instance.py +++ /dev/null @@ -1,366 +0,0 @@ -#!/usr/bin/env python3 - -import logging -import urllib3 - -from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client -from ansible.plugins.action import ActionBase -from ansible.errors import AnsibleError -from ansible.utils.display import Display -from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic.exceptions import NotFoundError - -import yaml -import os - -from mas.devops.ocp import getCR, getSecret - -urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - -display = Display() - -def display_information(mongoDBCommunityCR : dict): - display.v(f"MongoCE instance name .......................... {mongoDBCommunityCR['metadata']['name']}") - display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") - display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") - -def get_tlscertkey_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str | None: - if 'security' in mongoDBCommunityCR['spec']: - if 'tls' in mongoDBCommunityCR['spec']['security']: - if 'certificateKeySecretRef' in mongoDBCommunityCR['spec']['security']['tls']: - return mongoDBCommunityCR['spec']['security']['tls']['certificateKeySecretRef']['name'] - return None - -def get_usersecrets_from_mongoce(mongoDBCommunityCR : dict) -> list: - user_secrets = [] - for user in mongoDBCommunityCR['spec'].get('users', []): - if 'passwordSecretRef' in user: - user_secrets.append(user['passwordSecretRef']['name']) - if 'scramCredentialsSecretName' in user: - user_secrets.append(f"{user['scramCredentialsSecretName']}-scram-credentials") - return user_secrets - -def get_prometheus_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str | None: - if 'prometheus' in mongoDBCommunityCR['spec']: - if 'passwordSecretRef' in mongoDBCommunityCR['spec']['prometheus']: - return mongoDBCommunityCR['spec']['prometheus']['passwordSecretRef']['name'] - return None - -def getMongoVersionFromCR(mongoCR: dict) -> str: - """ - Get MongoDB version from MongoDB Community CR - """ - if 'spec' in mongoCR: - if 'version' in mongoCR['spec']: - return mongoCR['spec']['version'] - return "" - -def copyContentsToYamlFile(file_path: str, content: dict) -> bool: - """ - Write dictionary content to a YAML file - """ - try: - with open(file_path, 'w') as yaml_file: - yaml.dump(content, yaml_file, default_flow_style=False) - return True - except Exception as e: - display.v(f"Error writing to YAML file {file_path}: {e}") - return False - -def filterMongoCR(crData: dict) -> dict: - """ - Filter out unnecessary fields from a Custom Resource - """ - metadata_fields_to_remove = [ - 'annotations', - 'creationTimestamp', - 'generation', - 'resourceVersion', - 'selfLink', - 'uid', - 'managedFields' - ] - filteredCR = crData.copy() - if 'metadata' in filteredCR: - for field in metadata_fields_to_remove: - if field in filteredCR['metadata']: - del filteredCR['metadata'][field] - # remove status field - if 'status' in filteredCR: - del filteredCR['status'] - - # remove replicaSetHorizons field from spec if exists - if 'spec' in filteredCR: - if 'replicaSetHorizons' in filteredCR['spec']: - del filteredCR['spec']['replicaSetHorizons'] - - return filteredCR - -def filterResourceData(data: dict) -> dict: - """ - filter metadata from Resource data and create minimal dict - """ - metadata_fields_to_remove = [ - 'annotations', - 'creationTimestamp', - 'generation', - 'resourceVersion', - 'selfLink', - 'ownerReferences' - 'uid', - 'managedFields' - ] - filteredCopy = data.copy() - if 'metadata' in filteredCopy: - for field in metadata_fields_to_remove: - if field in filteredCopy['metadata']: - del filteredCopy['metadata'][field] - - if 'status' in filteredCopy: - del filteredCopy['status'] - - return filteredCopy - -def backupSecret(dynClient: DynamicClient, namespace: str, secret_name: str, backup_path: str) -> bool: - """ - Backup a Secret to a YAML file - """ - secret = getSecret(dynClient, namespace, secret_name) - if secret: - secret_file_path = f"{backup_path}/{secret_name}.yaml" - filtered_secret = filterResourceData(secret) - if copyContentsToYamlFile(secret_file_path, filtered_secret): - display.v(f"Successfully backed up Secret '{secret_name}' to '{secret_file_path}'") - return True - else: - display.v(f"Failed to back up Secret '{secret_name}' to '{secret_file_path}'") - return False - else: - display.v(f"Secret '{secret_name}' not found in namespace '{namespace}', skipping backup") - return False - -def isMongoRunning(mongoCR: dict) -> bool: - """ - Check if MongoDB Community instance is running - return True if running, else False - """ - display.v(f"Checking if MongoDB Community instance is in 'Running' state") - if 'status' in mongoCR: - if 'phase' in mongoCR['status']: - if mongoCR['status']['phase'] == 'Running': - display.v(f"MongoDB Community instance is in 'Running' state") - return True - display.v(f"MongoDB Community instance is not in 'Running' state") - return False - -def getMongoceCR(dynClient: DynamicClient, mongodb_instance_name: str, mongodb_namespace: str) -> dict: - """ - Check if MongoDB Community instance exists - return cr if exists, else return empty dict - """ - display.v(f"Checking if MongoDB Community instance '{mongodb_instance_name}' exists in namespace '{mongodb_namespace}'") - mongodbCR = getCR( - dynClient=dynClient, - cr_api_version="mongodbcommunity.mongodb.com/v1", - cr_kind="MongoDBCommunity", - cr_name=mongodb_instance_name, - namespace=mongodb_namespace - ) - if mongodbCR: - return mongodbCR.to_dict() - else: - return {} - -def backupIssuersInNamespace(dynClient: DynamicClient, namespace: str, backup_path: str) -> bool: - """ - Backup all Issuers in a namespace - """ - display.v(f"Backing up Issuers in namespace '{namespace}' to '{backup_path}'") - try: - issuerAPI = dynClient.resources.get(api_version="cert-manager.io/v1", kind="Issuer") - issuers = issuerAPI.get(namespace=namespace) - - for issuer in issuers.items: - issuer_name = issuer["metadata"]["name"] - issuer_file_path = f"{backup_path}/{issuer_name}.yaml" - filtered_issuer = filterResourceData(issuer.to_dict()) - if copyContentsToYamlFile(issuer_file_path, filtered_issuer): - display.v(f"Successfully backed up Issuer '{issuer_name}' to '{issuer_file_path}'") - else: - display.v(f"Failed to back up Issuer '{issuer_name}' to '{issuer_file_path}'") - return False - return True - - except NotFoundError: - display.v(f"No Issuers found in namespace {namespace}") - - return False - -def backupCertificatesInNamespace(dynClient: DynamicClient, namespace: str, backup_path: str) -> bool: - """ - Backup all Certificates in a namespace - """ - display.v(f"Backing up Certificates in namespace '{namespace}' to '{backup_path}'") - try: - certificateAPI = dynClient.resources.get(api_version="cert-manager.io/v1", kind="Certificate") - certificates = certificateAPI.get(namespace=namespace) - - for certificate in certificates.items: - certificate_name = certificate["metadata"]["name"] - certificate_file_path = f"{backup_path}/{certificate_name}.yaml" - filtered_certificate = filterResourceData(certificate.to_dict()) - if copyContentsToYamlFile(certificate_file_path, filtered_certificate): - display.v(f"Successfully backed up Certificate '{certificate_name}' to '{certificate_file_path}'") - else: - display.v(f"Failed to back up Certificate '{certificate_name}' to '{certificate_file_path}'") - return False - return True - - except NotFoundError: - display.v(f"No Certificates found in namespace {namespace}") - - return False - -def backupMongoCRContents(dynClient: DynamicClient, cr_data: dict, backup_path: str) -> bool: - """ - Backup MongoDB Community CR contents to a YAML file - """ - display.v(f"Backing up MongoDB Community CR contents to '{backup_path}/cr.yaml'") - cr_file_path = f"{backup_path}/cr.yaml" - filtered_cr = filterMongoCR(cr_data) - if copyContentsToYamlFile(cr_file_path, filtered_cr): - display.v(f"Successfully backed up MongoDB Community CR to '{cr_file_path}'") - return True - else: - display.v(f"Failed to back up MongoDB Community CR to '{cr_file_path}'") - return False - -def getMongoVersion(mongoCR: dict) -> str: - """ - Get MongoDB version from MongoDB Community CR - """ - if 'spec' in mongoCR: - if 'version' in mongoCR['spec']: - return mongoCR['spec']['version'] - return "" - -def createBackupDirectories(paths: list) -> bool: - """ - Create backup directories if they do not exist - """ - try: - for path in paths: - os.makedirs(path, exist_ok=True) - display.v(f"Created backup directory: {path}") - return True - except Exception as e: - display.v(f"Error creating backup directories: {e}") - return False -class ActionModule(ActionBase): - """ - Usage Example - ------------- - tasks: - - name: "Backup MongoDB instance resources (CR, secrets, configmaps, issuers, certificates)" - ibm.mas_devops.backup_mongo_instance: - """ - def run(self, tmp=None, task_vars=None): - super(ActionModule, self).run(tmp, task_vars) - - # Initialize DynamicClient and grab the task args - host = self._task.args.get('host', None) - api_key = self._task.args.get('api_key', None) - - dynClient = get_api_client(api_key=api_key, host=host) - - mongodb_instance_name = self._task.args.get('mongodb_instance_name') - mongodb_namespace = self._task.args.get('mongodb_namespace') - mongodb_backup_path = self._task.args.get('mongodb_backup_path') - - if mongodb_instance_name is None or mongodb_instance_name == "": - raise AnsibleError(f"Error: mongodb_instance_name argument was not provided") - if mongodb_namespace is None or mongodb_namespace == "": - raise AnsibleError(f"Error: mongodb_namespace argument was not provided") - if mongodb_backup_path is None or mongodb_backup_path == "": - raise AnsibleError(f"Error: mongodb_backup_path argument was not provided") - - display.v(f"Backing up MongoDB Community instance '{mongodb_instance_name}' in namespace '{mongodb_namespace}'") - - # Prepare backup resource path - mongodb_backup_resource_path = f"{mongodb_backup_path}/resources" - mongodb_backup_secrets_path = f"{mongodb_backup_resource_path}/secrets" - mongodb_backup_issuers_path = f"{mongodb_backup_resource_path}/issuers" - mongodb_backup_certificates_path = f"{mongodb_backup_resource_path}/certificates" - # Create backup directories - createBackupDirectories([mongodb_backup_resource_path, mongodb_backup_secrets_path, mongodb_backup_issuers_path, mongodb_backup_certificates_path]); - - # ======================================================= - # 1. Backup MongoDB Community CR - # ======================================================= - mongodb_cr = getMongoceCR(dynClient, mongodb_instance_name, mongodb_namespace) - if not mongodb_cr: - raise AnsibleError(f"Error: MongoDB Community instance '{mongodb_instance_name}' does not exist in namespace '{mongodb_namespace}'") - else: - display.v(f"MongoDB Community instance '{mongodb_instance_name}' exists in namespace '{mongodb_namespace}'") - - if not isMongoRunning(mongodb_cr): - raise AnsibleError(f"Error: MongoDB Community instance '{mongodb_instance_name}' is not in 'Running' state") - - is_cr_backup = backupMongoCRContents(dynClient, mongodb_cr, mongodb_backup_resource_path) - if not is_cr_backup: - raise AnsibleError(f"Error: Failed to back up MongoDB Community CR for instance '{mongodb_instance_name}'") - - display.v(f"Successfully backed up MongoDB Community CR to '{mongodb_backup_resource_path}/cr.yaml'") - - # =============================================================== - # 2. Backup MongoDB namespace Secrets defined in the MongoDB CR - # =============================================================== - display.v(f"Backing up MongoDB Secrets defined in the MongoDB CR from namespace '{mongodb_namespace}'") - # Get all relevant User Secret names from the MongoDB CR - user_secret_names = get_usersecrets_from_mongoce(mongodb_cr) - - # Get Prometheus and TLS Secret names from the MongoDB CR - prometheus_secret_names = get_prometheus_secretname_from_mongoce(mongodb_cr) - tls_secret_names = get_tlscertkey_secretname_from_mongoce(mongodb_cr) - - # Build secret_names list, filtering out None values - secret_names = user_secret_names.copy() - if prometheus_secret_names is not None: - secret_names.append(prometheus_secret_names) - if tls_secret_names is not None: - secret_names.append(tls_secret_names) - - display.v(f"All Secrets to backup: '{secret_names}'") - - # Backup each Secret - for secret_name in secret_names: - if not backupSecret(dynClient, mongodb_namespace, secret_name, f"{mongodb_backup_secrets_path}"): - raise AnsibleError(f"Error: Failed to back up Secret '{secret_name}'") - - display.v(f"Successfully backed up all Secrets in namespace '{mongodb_namespace}' to '{mongodb_backup_secrets_path}'") - - # ======================================================= - # 3. Backup Issuers in the MongoDB namespace - # ======================================================= - is_issuers_backup = backupIssuersInNamespace(dynClient, mongodb_namespace, f"{mongodb_backup_issuers_path}") - if not is_issuers_backup: - raise AnsibleError(f"Error: Failed to back up Issuers in namespace '{mongodb_namespace}'") - display.v(f"Successfully backed up Issuers in namespace '{mongodb_namespace}' to '{mongodb_backup_issuers_path}'") - - # ======================================================= - # Backup Certificates in the MongoDB namespace - # ======================================================= - is_certificates_backup = backupCertificatesInNamespace(dynClient, mongodb_namespace, f"{mongodb_backup_certificates_path}") - if not is_certificates_backup: - raise AnsibleError(f"Error: Failed to back up Certificates in namespace '{mongodb_namespace}'") - display.v(f"Successfully backed up Certificates in namespace '{mongodb_namespace}' to '{mongodb_backup_certificates_path}'") - - return dict( - message=f"Successfully backed up MongoDB Community instance '{mongodb_instance_name}' resources", - failed=False, - changed=False, - success=True, - ) - - diff --git a/ibm/mas_devops/plugins/action/backup_resource.py b/ibm/mas_devops/plugins/action/backup_resource.py index 88c6b754cc..36cf8ce1ba 100644 --- a/ibm/mas_devops/plugins/action/backup_resource.py +++ b/ibm/mas_devops/plugins/action/backup_resource.py @@ -19,7 +19,7 @@ class ActionModule(ActionBase): """ - Backup MAS Suite resources based on a list of resource definitions + Backup Kubernetes resources based on a list of resource definitions Usage Example ------------- diff --git a/ibm/mas_devops/plugins/action/get_mongoce_info.py b/ibm/mas_devops/plugins/action/get_mongoce_info.py index 9d46bd9f22..9817ce19a9 100644 --- a/ibm/mas_devops/plugins/action/get_mongoce_info.py +++ b/ibm/mas_devops/plugins/action/get_mongoce_info.py @@ -35,23 +35,6 @@ def get_mongoce_admin_secretname(mongoDBCommunityCR): return user['passwordSecretRef']['name'] return None -def get_tlscertkey_secretname_from_mongoce(mongoDBCommunityCR : dict) -> str: - if 'security' in mongoDBCommunityCR['spec']: - if 'tls' in mongoDBCommunityCR['spec']['security']: - if 'certificateKeySecretRef' in mongoDBCommunityCR['spec']['security']['tls']: - return mongoDBCommunityCR['spec']['security']['tls']['certificateKeySecretRef']['name'] - else: - return None - -def get_usersecrets_from_mongoce(mongoDBCommunityCR : dict) -> list: - user_secrets = [] - for user in mongoDBCommunityCR['spec'].get('users', []): - if 'passwordSecretRef' in user: - user_secrets.append(user['passwordSecretRef']['name']) - if 'scramCredentialsSecretName' in user: - user_secrets.append(f"{user['scramCredentialsSecretName']}-scram-credentials") - return user_secrets - def isMongoRunning(mongoCR: dict) -> bool: """ Check if MongoDB Community instance is running diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml index 940136c5d8..7c6d778ab9 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml @@ -1,41 +1,88 @@ --- -- name: DB2 Instance backup configuration summary - ansible.builtin.debug: - msg: - - "================== BACKUP DB2 INSTANCE CONFIGURATION SUMMARY ==================" - - "MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }}" - - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" - - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" - - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" - - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" - - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" - - "================================================================================" +- name: "Set fact: DB2 universal operator backup resources" + set_fact: + db2_backup_resources: + - namespace: "{{ db2_namespace }}" + resources: + # db2u.databases.ibm.com + - kind: Db2uCluster + api_version: db2u.databases.ibm.com/v1 + name: "{{ db2_instance_name }}" + # support for db2uinstance migration + - kind: Db2uInstance + api_version: db2u.databases.ibm.com/v1 + name: "{{ db2_instance_name }}" + # subscription + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: db2u-operator + # operatorgroup + - kind: OperatorGroup + api_version: operators.coreos.com/v1 + # secrets + - kind: Secret + api_version: v1 + name: ibm-registry + - kind: Secret + api_version: v1 + name: "c-{{ db2_instance_name }}-instancepassword" + # Issuers + - kind: Issuer + api_version: cert-manager.io/v1 + name: db2u-issuer + - kind: Issuer + api_version: cert-manager.io/v1 + name: db2u-ca-issuer + # Certificates + - kind: Certificate + api_version: cert-manager.io/v1 + name: "db2u-certificate-{{ db2_instance_name }}" + - kind: Certificate + api_version: cert-manager.io/v1 + name: db2u-ca-certificate + - namespace: "mas-{{ mas_instance_id }}-core" + resources: + # secrets + - kind: Secret + api_version: v1 + name: "jdbc-{{ db2_instance_name }}-credentials" -- name: "Start Db2 Instance backup process." - ibm.mas_devops.backup_db2_instance: - db2_backup_path: "{{ db2_backup_path }}" - db2_namespace: "{{ db2_namespace }}" - db2_instance_name: "{{ db2_instance_name }}" - mas_instance_id: "{{ mas_instance_id }}" - db2_dbname: "{{ db2_dbname }}" - register: db2_instance_backup_result +# Call the backup_resources plugin to execute the backup to the path provided +# ----------------------------------------------------------------------------- +- name: "Backup DB2 resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ db2_backup_resources }}" + backup_path: "{{ db2_backup_path }}" + register: backup_result -- name: "Assert Db2 Instance backup success" - assert: - that: - - db2_instance_backup_result is defined - - db2_instance_backup_result.success == true - fail_msg: "Db2 Instance resources backup failed." +# Show the results +# ----------------------------------------------------------------------------- +- name: "Display backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ backup_result.backed_up_count }}" + - "Total resources failed: {{ backup_result.failed_count }}" + - "Resources not found: {{ backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" + - "Backup location: {{ db2_backup_path }}" -- name: DB2 Instance backup summary - ansible.builtin.debug: +# Fail task if any errors occurred. +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: msg: - - "================== BACKUP DB2 INSTANCE CONFIGURATION SUMMARY ==================" - - "MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }}" - - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" - - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" - - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" - - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" - - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" - - "STATUS : SUCCESS" - - "================================================================================" + - "Failed resources:" + - "{{ backup_result.failed_resources | to_nice_yaml }}" + when: backup_result.failed_count > 0 + +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ backup_result.failed_count }} resource(s): + {% for resource in backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: backup_result.failed_count > 0 + + diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml index ec8ffcf3e3..79e60d849c 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml @@ -1,19 +1,62 @@ --- # Backup Mongodb Community edition cluster Instance # ----------------------------------------------------------------------------- +- name: "Set fact: MongoDB CE backup resources" + set_fact: + mongoce_backup_resources: + - namespace: "{{ mongodb_namespace }}" + resources: + # mongodbcommunity.mongodb.com + - kind: MongoDBCommunity + api_version: mongodbcommunity.mongodb.com/v1 + name: "{{ mongodb_instance_name }}" + # Issuers + - kind: Issuer + api_version: cert-manager.io/v1 + # Certificates + - kind: Certificate + api_version: cert-manager.io/v1 + # secrets + - kind: Secret + api_version: v1 + name: "{{ mongodb_instance_name }}-scram-scram-credentials" + +# Call the backup_resources plugin to execute the backup to the path provided +# ----------------------------------------------------------------------------- +- name: "Backup MongoCE resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ mongoce_backup_resources }}" + backup_path: "{{ mongodb_backup_path }}" + register: backup_result + +# Show the results +# ----------------------------------------------------------------------------- +- name: "Display backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ backup_result.backed_up_count }}" + - "Total resources failed: {{ backup_result.failed_count }}" + - "Resources not found: {{ backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" + - "Backup location: {{ mongodb_backup_path }}" + +# Fail task if any errors occurred. +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ backup_result.failed_resources | to_nice_yaml }}" + when: backup_result.failed_count > 0 + +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ backup_result.failed_count }} resource(s): + {% for resource in backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: backup_result.failed_count > 0 -# Backup Mongodb instance Resources -# ------------------------------------------------------------------------- -- name: "Start Mongo Instance backup process." - ibm.mas_devops.backup_mongo_instance: - mongodb_backup_path: "{{ mongodb_backup_path }}" - mongodb_namespace: "{{ mongodb_namespace }}" - mongodb_instance_name: "{{ mongodb_instance_name }}" - register: mongo_instance_backup_result -- name: "Assert Mongo Instance backup success" - assert: - that: - - mongo_instance_backup_result is defined - - mongo_instance_backup_result.success == true - fail_msg: "MongoDB Instance resources backup failed." From bc7d6e82dd808e410749733fe55f321178905c95 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 15 Jan 2026 14:09:33 +0000 Subject: [PATCH 17/61] [patch] mongodb backup more items (#2060) --- .../backup-restore/backup-instance.yml | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml index 79e60d849c..3243220a40 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml @@ -6,20 +6,59 @@ mongoce_backup_resources: - namespace: "{{ mongodb_namespace }}" resources: + # CRD - mongodbcommunity.mongodbcommunity.mongodb.com + - kind: CustomResourceDefinition + api_version: apiextensions.k8s.io/v1 + name: mongodbcommunity.mongodbcommunity.mongodb.com # mongodbcommunity.mongodb.com - kind: MongoDBCommunity api_version: mongodbcommunity.mongodb.com/v1 name: "{{ mongodb_instance_name }}" + # Role + - kind: Role + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-kubernetes-operator + - kind: Role + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-database + # RoleBinding + - kind: RoleBinding + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-kubernetes-operator + - kind: RoleBinding + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-database + # ServiceAccount + - kind: ServiceAccount + api_version: v1 + name: mongodb-kubernetes-operator + - kind: ServiceAccount + api_version: v1 + name: mongodb-database # Issuers - kind: Issuer api_version: cert-manager.io/v1 # Certificates - kind: Certificate api_version: cert-manager.io/v1 + # Deployment + - kind: Deployment + api_version: apps/v1 + name: mongodb-kubernetes-operator # secrets - kind: Secret api_version: v1 name: "{{ mongodb_instance_name }}-scram-scram-credentials" + - kind: Secret + api_version: v1 + name: "{{ mongodb_instance_name }}-admin-password" + # Configmap + - kind: ConfigMap + api_version: v1 + name: "{{ mongodb_instance_name }}-cert-map" + # grafanaDashboard + - kind: GrafanaDashboard + api_version: grafana.integreatly.org/v1beta1 # Call the backup_resources plugin to execute the backup to the path provided # ----------------------------------------------------------------------------- @@ -58,5 +97,3 @@ - {{ resource.description }} in {{ resource.scope }} {% endfor %} when: backup_result.failed_count > 0 - - From 92bf5a67062b4b7c261e4074f71c4f8105b64b4c Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 15 Jan 2026 16:35:12 +0000 Subject: [PATCH 18/61] [patch] restore instance password by passing it in CR. (#2062) Co-authored-by: Sanjay Prabhakar --- .../plugins/action/restore_db2_resources.py | 44 +++++++++++++------ .../db2/tasks/backup/backup-database.yml | 4 +- .../db2/tasks/install/get-backup-info.yml | 6 +++ .../roles/db2/templates/db2ucluster.yml.j2 | 3 ++ 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/ibm/mas_devops/plugins/action/restore_db2_resources.py b/ibm/mas_devops/plugins/action/restore_db2_resources.py index 8ea4c1069e..d31cb03109 100644 --- a/ibm/mas_devops/plugins/action/restore_db2_resources.py +++ b/ibm/mas_devops/plugins/action/restore_db2_resources.py @@ -11,6 +11,7 @@ import yaml import os +import base64 from mas.devops.ocp import createNamespace, apply_resource @@ -19,7 +20,7 @@ display = Display() -def checkBackupDirectoryExists(db2_backup_path: str, db2_backup_version: str): +def checkBackupDirectoryExists(db2_backup_path: str): if not os.path.exists(db2_backup_path): raise AnsibleError(f"DB2 Backup path {db2_backup_path} does not exist.") @@ -70,7 +71,7 @@ def run(self, tmp=None, task_vars=None): # Check if backup directory exists db2_backup_path = os.path.join(mas_backup_dir, f"backup-{db2_backup_version}-db2u") - checkBackupDirectoryExists(db2_backup_path, db2_backup_version) + checkBackupDirectoryExists(db2_backup_path) display.v(f"- Db2 backup path {db2_backup_path} exists. Proceeding with restore...") db2_backup_resource_path = os.path.join(db2_backup_path, "resources") @@ -82,8 +83,12 @@ def run(self, tmp=None, task_vars=None): backup_db2u_cr = yaml.safe_load(cr_file) display.v("- Successfully read DB2 backup CR file") + db2_info = {} + db2_instance_name = backup_db2u_cr['metadata']['name'] db2_namespace = backup_db2u_cr['metadata']['namespace'] + db2_info['db2_instance_name'] = db2_instance_name + db2_info['db2_namespace'] = db2_namespace # ======================================================= # 1. Create DB2 namespace if not exists @@ -106,6 +111,29 @@ def run(self, tmp=None, task_vars=None): secret_yaml = f.read() apply_resource(dynClient, secret_yaml, db2_namespace) + # Get instance password to use for post-install restoration + inst_pwd_filepath = os.path.join(db2_secrets_path, f"c-{db2_instance_name}-instancepassword.yaml") + if os.path.exists(inst_pwd_filepath): + with open(inst_pwd_filepath, 'r') as instpwd_file: + instpwd_data = yaml.safe_load(instpwd_file) + if 'data' in instpwd_data and 'password' in instpwd_data['data']: + pwd_encoded = instpwd_data['data']['password'] # base64 encoded + db2_info['db2_instance_password'] = base64.b64decode(pwd_encoded).decode('utf-8') + else: + display.v(f"Unable to find instance password in {inst_pwd_filepath}") + else: + display.v(f"Unable to find instance password file {inst_pwd_filepath}") + + # Get LDAP user info if present + ldap_info_file_path = os.path.join(db2_secrets_path, "ldapuser-NOT_SECRET.yaml") + if os.path.exists(ldap_info_file_path): + with open(ldap_info_file_path, 'r') as ldap_info_file: + ldap_info = yaml.safe_load(ldap_info_file) + db2_info['db2_ldap_username'] = base64.b64decode(ldap_info['db2_ldap_username']).decode('utf-8') + db2_info['db2_ldap_password'] = base64.b64decode(ldap_info['db2_ldap_password']).decode('utf-8') + display.v(f"- Successfully read LDAP user info from {ldap_info_file_path}") + + # ======================================================= # 3. Restore DB2 Issuer resources from backup # ======================================================= @@ -132,9 +160,6 @@ def run(self, tmp=None, task_vars=None): # 5. Gather info from backup files to recreate new Db2u instance # ======================================================= - db2_info = {} - db2_info['db2_instance_name'] = db2_instance_name - db2_info['db2_namespace'] = db2_namespace db2_info['db2_type'] = backup_db2u_cr['spec'].get('environment', {}).get('dbType', 'db2wh') # Get DB name from backup CR db2_info['db2_database_name'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('name', 'BLUDB') @@ -160,15 +185,6 @@ def run(self, tmp=None, task_vars=None): # Get number of db2 pods db2_info['db2_num_pods'] = backup_db2u_cr['spec']['size'] - # Get LDAP user info if present - ldap_info_file_path = os.path.join(db2_backup_resource_path, "ldapuser-NOT_SECRET.yaml") - if os.path.exists(ldap_info_file_path): - with open(ldap_info_file_path, 'r') as ldap_info_file: - ldap_info = yaml.safe_load(ldap_info_file) - db2_info['db2_ldap_username'] = ldap_info['db2_ldap_username'] - db2_info['db2_ldap_password'] = ldap_info['db2_ldap_password'] - display.v(f"- Successfully read LDAP user info from {ldap_info_file_path}") - # Get DB2 backup info file db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-backup-info.yaml") if os.path.exists(db2_info_file_path): diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml index 9d1893ab9e..3f21625a9f 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml @@ -53,7 +53,7 @@ # ----------------------------------------------------------------------------- - name: "Check DB2 is running and get DB2 pod name" ibm.mas_devops.get_db2u_pod_name: - db2_instance_name: "{{ db2_instance_name }}" + db2_instance_name: "{{ db2_instance_name | lower }}" db2_namespace: "{{ db2_namespace }}" register: db2_pod_name_result @@ -226,7 +226,7 @@ content: | source_db2_backup_version: "{{ db2_backup_version }}" source_db2_backup_timestamp: "{{ db2_backup_timestamp }}" - source_db2_instance_name: "{{ db2_instance_name }}" + source_db2_instance_name: "{{ db2_instance_name | lower }}" source_db2_instance_version: "{{ db2_pod_name_result.db2_version }}" database: "{{ db2_dbname }}" backup_vendor: "{{ backup_vendor }}" diff --git a/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml b/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml index 4922900806..83760797b2 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml @@ -45,6 +45,12 @@ db2_timezone: "{{ restore_db2_instance_result.db2_timezone }}" db2_num_pods: "{{ restore_db2_instance_result.db2_num_pods }}" +- name: Set fact Instance password + no_log: true + set_fact: + db2_instance_password: "{{ restore_db2_instance_result.db2_instance_password }}" + when: restore_db2_instance_result.db2_instance_password is defined + - name: "Set fact LDAP credentials if exist." no_log: true set_fact: diff --git a/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 b/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 index 563c846582..16e88a463e 100644 --- a/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 +++ b/ibm/mas_devops/roles/db2/templates/db2ucluster.yml.j2 @@ -60,6 +60,9 @@ spec: secretName: "db2u-certificate-{{db2_instance_name}}" certLabel: "CN=db2u" instance: +{% if db2_instance_password is defined and db2_instance_password != "" %} + password: "{{ db2_instance_password }}" +{% endif %} {% if db2_instance_registry is defined and db2_instance_registry != "" %} registry: {{ db2_instance_registry }} From 07b18ee051baf27f8c757a2965d739842bb5db0c Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 16 Jan 2026 13:05:12 +0000 Subject: [PATCH 19/61] [patch] backup sls mongo database (#2066) --- .../templates/community/backup-restore/database-backup.sh.j2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-backup.sh.j2 b/ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-backup.sh.j2 index 9879920f1b..ca479bf4a7 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-backup.sh.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/backup-restore/database-backup.sh.j2 @@ -5,10 +5,9 @@ MONGODB_BACKUP_USER="bradmin" TMP_BACKUP_DIR="/tmp/masbr/{{ mongodb_backup_version }}" TMP_BACKUP_MONGODUMP_DIR="${TMP_BACKUP_DIR}/mongodump" -ALL_DATABASES_FILTER="^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)" +ALL_DATABASES_FILTER="^(mas|iot|sls|ibm-sls)(_|-)({{ mas_instance_id }}|sls)(_|-)(?!.*monitor$)" tlsCAFile=$(ls /var/lib/tls/ca/*.pem) -echo "Using TLS CA file: $tlsCAFile" primary_host="" is_primary=$(mongosh --quiet -u {{ mongodb_admin_user }} -p {{ mongodb_admin_password }} --host {{ mongodb_host }} --tlsCAFile $tlsCAFile --authenticationDatabase=admin --tls --eval 'db.isMaster().ismaster') From 036f0fed70def9fdbde132ddfe53bb0e7bf12eb1 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 20 Jan 2026 11:00:31 +0000 Subject: [PATCH 20/61] [patch] backup for catalog, cert-manage, grafana and sls (#2069) --- .../action/save_sls_registration_info.py | 52 +++++++++ .../action/verify_backup_restore_vars.py | 12 ++ .../roles/cert_manager/defaults/main.yml | 4 + .../tasks/provider/redhat/backup.yml | 69 ++++++++++++ ibm/mas_devops/roles/grafana/README.md | 2 +- .../roles/grafana/defaults/main.yml | 4 + .../roles/grafana/tasks/backup/main.yml | 99 +++++++++++++++++ ibm/mas_devops/roles/grafana/tasks/main.yml | 5 +- .../roles/ibm_catalogs/defaults/main.yml | 4 + .../roles/ibm_catalogs/tasks/backup/main.yml | 68 ++++++++++++ ibm/mas_devops/roles/sls/defaults/main.yml | 4 + .../roles/sls/tasks/backup/main.yml | 103 ++++++++++++++++++ ibm/mas_devops/roles/sls/tasks/main.yml | 2 +- 13 files changed, 425 insertions(+), 3 deletions(-) create mode 100644 ibm/mas_devops/plugins/action/save_sls_registration_info.py create mode 100644 ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml create mode 100644 ibm/mas_devops/roles/grafana/tasks/backup/main.yml create mode 100644 ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml create mode 100644 ibm/mas_devops/roles/sls/tasks/backup/main.yml diff --git a/ibm/mas_devops/plugins/action/save_sls_registration_info.py b/ibm/mas_devops/plugins/action/save_sls_registration_info.py new file mode 100644 index 0000000000..63cf044b48 --- /dev/null +++ b/ibm/mas_devops/plugins/action/save_sls_registration_info.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import logging +import yaml +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +import os + +from mas.devops.sls import getSLSRegistrationDetails + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + slsNamespace = self._task.args.get('namespace', None) + slsName = self._task.args.get('name', None) + backupPath = self._task.args.get('sls_backup_path', None) + + if slsNamespace is None: + raise AnsibleError(f"Error: slsNamespace argument was not provided") + if slsName is None: + raise AnsibleError(f"Error: slsName argument was not provided") + if backupPath is None: + raise AnsibleError(f"Error: sls_backup_dir argument was not provided") + + # Initialize DynamicClient, ensure the namespace exists, and create/update the entitlement secret + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + + registrationDetails = getSLSRegistrationDetails(name=slsName, namespace=slsNamespace, dynClient=dynClient) + + if registrationDetails: + with open(os.path.join(backupPath, 'sls-registration.yaml'), 'w') as outfile: + yaml.dump(registrationDetails, outfile, default_flow_style=False) + return dict( + msg=f"Successfully stored SLS registration details to {os.path.join(backupPath, 'sls-registration.yml')}", + failed=False, + success=True + ) + else: + return dict( + msg=f"Couldn't get registration details from CR status of SLS {slsName} in namespace {slsNamespace}.", + failed=True, + success=False + ) diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index fb5be19866..7bad7d847a 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -7,6 +7,18 @@ class ActionModule(ActionBase): REQUIRED = { + "catalog": { + "backup": ["mas_backup_dir"] + }, + "certmanager": { + "backup": ["mas_backup_dir"] + }, + "grafana": { + "backup": ["mas_backup_dir"] + }, + "sls": { + "backup": ["mas_backup_dir", "sls_namespace", "sls_instance_name"] + }, "mongodb": { "backup": ["mongodb_instance_name", "mas_backup_dir", "mas_instance_id"], # mongodb_instance_name has a default value "restore": ["mas_backup_dir", "mongodb_backup_version"] diff --git a/ibm/mas_devops/roles/cert_manager/defaults/main.yml b/ibm/mas_devops/roles/cert_manager/defaults/main.yml index 826b47d211..af7c0fca42 100644 --- a/ibm/mas_devops/roles/cert_manager/defaults/main.yml +++ b/ibm/mas_devops/roles/cert_manager/defaults/main.yml @@ -5,3 +5,7 @@ cert_manager_action: "{{ lookup('env', 'CERT_MANAGER_ACTION') | default('install cert_manager_operator_namespace: "cert-manager-operator" cert_manager_namespace: "cert-manager" cert_manager_channel: "{{ lookup('env', 'REDHAT_CERT_MANAGER_CHANNEL') | default('stable-v1', true) }}" + +# Backup and restore variables +certmanager_backup_version: "{{ lookup('env', 'CERTMANAGER_BACKUP_VERSION') }}" +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml new file mode 100644 index 0000000000..f4babbcb38 --- /dev/null +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml @@ -0,0 +1,69 @@ +--- +- name: "Fail if require variables for Redhat cert-manager backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_backup_dir: "{{ mas_backup_dir }}" + action: "backup" + component: "certmanager" + +- name: "Check if CERTMANAGER_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + certmanager_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: certmanager_backup_version is not defined or certmanager_backup_version == "" or certmanager_backup_version == "None" + +- name: "Set fact: cert-manager backup base directory path" + set_fact: + certmgr_backup_path: "{{ mas_backup_dir }}/backup-{{ certmanager_backup_version }}-certmanager" + +- name: "Set fact: cert-manager backup resources" + set_fact: + certmanager_backup_resources: + - namespace: "{{ cert_manager_operator_namespace }}" + resources: + # certmanager source + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: openshift-cert-manager-operator + # operator group + - kind: OperatorGroup + api_version: operators.coreos.com/v1 + name: operatorgroup + + +# Call the backup_resources plugin to execute the backup to the path provided +# ----------------------------------------------------------------------------- +- name: "Backup cert-manager resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ certmanager_backup_resources }}" + backup_path: "{{ certmgr_backup_path }}" + register: backup_result + +# Show the results +# ----------------------------------------------------------------------------- +- name: "Display backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ backup_result.backed_up_count }}" + - "Total resources failed: {{ backup_result.failed_count }}" + - "Resources not found: {{ backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" + - "Backup location: {{ certmgr_backup_path }}" + +# Fail task if any errors occurred. +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ backup_result.failed_resources | to_nice_yaml }}" + when: backup_result.failed_count > 0 + +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ backup_result.failed_count }} resource(s): + {% for resource in backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: backup_result.failed_count > 0 + diff --git a/ibm/mas_devops/roles/grafana/README.md b/ibm/mas_devops/roles/grafana/README.md index 31ba48a3e0..366dac9651 100644 --- a/ibm/mas_devops/roles/grafana/README.md +++ b/ibm/mas_devops/roles/grafana/README.md @@ -162,7 +162,7 @@ Example Playbook - hosts: localhost vars: grafana_instance_storage_class: "ibmc-file-gold-gid" - grafana_instance_storage_class: "15Gi" + grafana_instance_storage_size: "15Gi" roles: - ibm.mas_devops.grafana ``` diff --git a/ibm/mas_devops/roles/grafana/defaults/main.yml b/ibm/mas_devops/roles/grafana/defaults/main.yml index 3a12f9d38c..a19f16c31d 100644 --- a/ibm/mas_devops/roles/grafana/defaults/main.yml +++ b/ibm/mas_devops/roles/grafana/defaults/main.yml @@ -11,3 +11,7 @@ grafana_v5_namespace: "{{ lookup('env', 'GRAFANA_V5_NAMESPACE') | default('grafa # Settings to set grafana to define a specific storage class and size grafana_instance_storage_class: "{{ lookup('env', 'GRAFANA_INSTANCE_STORAGE_CLASS') }}" grafana_instance_storage_size: "{{ lookup('env', 'GRAFANA_INSTANCE_STORAGE_SIZE') | default('10Gi', true) }}" + +# Backup and restore variables +grafana_backup_version: "{{ lookup('env', 'GRAFANA_BACKUP_VERSION') }}" +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" diff --git a/ibm/mas_devops/roles/grafana/tasks/backup/main.yml b/ibm/mas_devops/roles/grafana/tasks/backup/main.yml new file mode 100644 index 0000000000..820d586e1a --- /dev/null +++ b/ibm/mas_devops/roles/grafana/tasks/backup/main.yml @@ -0,0 +1,99 @@ +--- +- name: "Fail if require variables for Grafana backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_backup_dir: "{{ mas_backup_dir }}" + action: "backup" + component: "grafana" + +- name: "Check if GRAFANA_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + grafana_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: grafana_backup_version is not defined or grafana_backup_version == "" or grafana_backup_version == "None" + +- name: "Set fact: Grafana backup base directory path" + set_fact: + grafana_backup_path: "{{ mas_backup_dir }}/backup-{{ grafana_backup_version }}-grafana" + +- name: "Set fact: cert-manager backup resources" + set_fact: + grafana_backup_resources: + - namespace: "{{ grafana_v5_namespace }}" + resources: + # Clusterrole + - kind: ClusterRole + api_version: rbac.authorization.k8s.io/v1 + name: grafana-operator + - kind: ClusterRole + api_version: rbac.authorization.k8s.io/v1 + name: prometheus-role + # Clusterrolebinding + - kind: ClusterRoleBinding + api_version: rbac.authorization.k8s.io/v1 + name: prometheus-rolebinding + - kind: ClusterRoleBinding + api_version: rbac.authorization.k8s.io/v1 + name: grafana-operator + # Secret + - kind: Secret + api_version: v1 + name: prometheus-serviceaccount-token + # Service account + - kind: ServiceAccount + api_version: v1 + name: prometheus-serviceaccount + # Subscription + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: grafana-operator + # Grafana CR + - kind: Grafana + api_version: grafana.integreatly.org/v1beta1 + name: mas-grafana + # Dashboard + - kind: GrafanaDashboard + api_version: grafana.integreatly.org/v1beta1 + - namespace: openshift-monitoring + resources: + # Configmap + - kind: ConfigMap + api_version: v1 + name: cluster-monitoring-config + +# Call the backup_resources plugin to execute the backup to the path provided +# ----------------------------------------------------------------------------- +- name: "Backup Grafana resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ grafana_backup_resources }}" + backup_path: "{{ grafana_backup_path }}" + register: backup_result + +# Show the results +# ----------------------------------------------------------------------------- +- name: "Display backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ backup_result.backed_up_count }}" + - "Total resources failed: {{ backup_result.failed_count }}" + - "Resources not found: {{ backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" + - "Backup location: {{ grafana_backup_path }}" + +# Fail task if any errors occurred. +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ backup_result.failed_resources | to_nice_yaml }}" + when: backup_result.failed_count > 0 + +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ backup_result.failed_count }} resource(s): + {% for resource in backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: backup_result.failed_count > 0 + diff --git a/ibm/mas_devops/roles/grafana/tasks/main.yml b/ibm/mas_devops/roles/grafana/tasks/main.yml index cc807a184c..cd5b6faee6 100644 --- a/ibm/mas_devops/roles/grafana/tasks/main.yml +++ b/ibm/mas_devops/roles/grafana/tasks/main.yml @@ -6,6 +6,8 @@ - "Grafana major version................... {{ grafana_major_version }}" - "Grafana v4 namespace ................... {{ grafana_v4_namespace }}" - "Grafana v5 namespace ................... {{ grafana_v5_namespace }}" + when: + - grafana_action in ['install', 'uninstall', 'update'] # 1. Perform the selected action # ----------------------------------------------------------------------------- @@ -13,5 +15,6 @@ # - install # - uninstall # - update +# - backup - include_tasks: "{{ grafana_action }}/main.yml" - when: grafana_action in ['install', 'uninstall', 'update'] + when: grafana_action in ['install', 'uninstall', 'update', 'backup'] diff --git a/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml b/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml index 1514354933..5afe63ec22 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml @@ -10,3 +10,7 @@ artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" # mas_catalog_digest is needed for development airgap. This environment variable should be set before running the code mas_catalog_digest: "{{ lookup('env', 'MAS_CATALOG_DIGEST') }}" mas_catalog_version: "{{ lookup('env', 'MAS_CATALOG_VERSION') | default ('@@MAS_LATEST_CATALOG@@', True) }}" + +# Backup and restore variables +catalog_backup_version: "{{ lookup('env', 'CATALOG_BACKUP_VERSION') }}" +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml new file mode 100644 index 0000000000..df369eeebc --- /dev/null +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml @@ -0,0 +1,68 @@ +--- +- name: "Fail if require variables for IBM operator catalog backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_backup_dir: "{{ mas_backup_dir }}" + action: "backup" + component: "catalog" + +- name: "Check if CATALOG_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + catalog_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: catalog_backup_version is not defined or catalog_backup_version == "" or catalog_backup_version == "None" + +- name: "Set fact: Catalog backup base directory path" + set_fact: + catalog_backup_path: "{{ mas_backup_dir }}/backup-{{ catalog_backup_version }}-catalog" + +- name: "Set fact: IBM operator catalog backup resources" + set_fact: + catalog_backup_resources: + - namespace: "openshift-marketplace" + resources: + # Catalog source + - kind: CatalogSource + api_version: operators.coreos.com/v1alpha1 + name: ibm-operator-catalog + # Secret (in dev envs) + - kind: Secret + api_version: v1 + name: wiotp-docker-local + +# Call the backup_resources plugin to execute the backup to the path provided +# ----------------------------------------------------------------------------- +- name: "Backup catalogs resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ catalog_backup_resources }}" + backup_path: "{{ catalog_backup_path }}" + register: backup_result + +# Show the results +# ----------------------------------------------------------------------------- +- name: "Display backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ backup_result.backed_up_count }}" + - "Total resources failed: {{ backup_result.failed_count }}" + - "Resources not found: {{ backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" + - "Backup location: {{ catalog_backup_path }}" + +# Fail task if any errors occurred. +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ backup_result.failed_resources | to_nice_yaml }}" + when: backup_result.failed_count > 0 + +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ backup_result.failed_count }} resource(s): + {% for resource in backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: backup_result.failed_count > 0 + diff --git a/ibm/mas_devops/roles/sls/defaults/main.yml b/ibm/mas_devops/roles/sls/defaults/main.yml index 6ffdc3e97c..7b30689e50 100644 --- a/ibm/mas_devops/roles/sls/defaults/main.yml +++ b/ibm/mas_devops/roles/sls/defaults/main.yml @@ -62,3 +62,7 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # mas_pod_templates_dir: path to directory containing podTemplates configuration # ----------------------------------------------------------------------------- mas_pod_templates_dir: "{{ lookup('env', 'MAS_POD_TEMPLATES_DIR') | default('', true) }}" + +# Backup and restore variables +sls_backup_version: "{{ lookup('env', 'SLS_BACKUP_VERSION') }}" +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" diff --git a/ibm/mas_devops/roles/sls/tasks/backup/main.yml b/ibm/mas_devops/roles/sls/tasks/backup/main.yml new file mode 100644 index 0000000000..f961ac05e0 --- /dev/null +++ b/ibm/mas_devops/roles/sls/tasks/backup/main.yml @@ -0,0 +1,103 @@ +--- +- name: "Fail if require variables for SLS backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_backup_dir: "{{ mas_backup_dir }}" + sls_namespace: "{{ sls_namespace }}" + sls_instance_name: "{{ sls_instance_name }}" + action: "backup" + component: "sls" + +- name: "Check if SLS_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + sls_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: sls_backup_version is not defined or sls_backup_version == "" or sls_backup_version == "None" + +- name: "Set fact: cert-manager backup base directory path" + set_fact: + sls_backup_path: "{{ mas_backup_dir }}/backup-{{ sls_backup_version }}-sls" + +- name: "Set fact: cert-manager backup resources" + set_fact: + sls_backup_resources: + - namespace: "{{ sls_namespace }}" + resources: + # secret + - kind: Secret + api_version: v1 + name: ibm-entitlement + - kind: Secret + api_version: v1 + name: ibm-sls-mongo-credentials + - kind: Secret + api_version: v1 + name: "{{ sls_instance_name }}-bootstrap" + - kind: Secret + api_version: v1 + name: "ibm-sls-{{ sls_instance_name }}-entitlement" + # configmap + - kind: ConfigMap + api_version: v1 + name: "{{ sls_instance_name }}-suite-registration" + # subscription + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: ibm-sls + # operator group + - kind: OperatorGroup + api_version: operators.coreos.com/v1 + name: operatorgroup + # Licenseservice CR + - kind: LicenseService + api_version: sls.ibm.com/v1 + name: "{{ sls_instance_name }}" + +# Call the backup_resources plugin to execute the backup to the path provided +# ----------------------------------------------------------------------------- +- name: "Backup sls resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ sls_backup_resources }}" + backup_path: "{{ sls_backup_path }}" + register: backup_result + +# Show the results +# ----------------------------------------------------------------------------- +- name: "Display backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ backup_result.backed_up_count }}" + - "Total resources failed: {{ backup_result.failed_count }}" + - "Resources not found: {{ backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" + - "Backup location: {{ sls_backup_path }}" + +# Fail task if any errors occurred. +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ backup_result.failed_resources | to_nice_yaml }}" + when: backup_result.failed_count > 0 + +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ backup_result.failed_count }} resource(s): + {% for resource in backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: backup_result.failed_count > 0 + +- name: "Store SLS registration details" + ibm.mas_devops.save_sls_registration_info: + namespace: "{{ sls_namespace }}" + name: "{{ sls_instance_name }}" + sls_backup_path: "{{ sls_backup_path }}" + register: sls_registration_result + +- name: "Fail if storing SLS registration details has failed." + fail: + msg: "{{ sls_registration_result.msg }}" + when: sls_registration_result.failed + diff --git a/ibm/mas_devops/roles/sls/tasks/main.yml b/ibm/mas_devops/roles/sls/tasks/main.yml index 7cf575437c..3d2f5b4b20 100644 --- a/ibm/mas_devops/roles/sls/tasks/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/main.yml @@ -7,7 +7,7 @@ - name: Run the specified action ansible.builtin.include_tasks: "tasks/{{ sls_action }}/main.yml" when: - - sls_action in ["install", "uninstall"] + - sls_action in ["install", "uninstall", "backup"] - sls_url is not defined or sls_url == "" # TODO: We should take a bigger look at how the "only generate a cfg" mode works From 1c140b1e815329a5f15831b6724513df5364c010 Mon Sep 17 00:00:00 2001 From: Andrew Whitfield Date: Tue, 20 Jan 2026 11:29:49 +0000 Subject: [PATCH 21/61] Add suite_restore using new restore function. (#2068) --- .gitignore | 1 + .../plugins/action/restore_resource.py | 354 ++++++++++++++++++ ibm/mas_devops/roles/mongodb/README.md | 179 +++++++-- .../roles/mongodb/defaults/main.yml | 2 +- ibm/mas_devops/roles/mongodb/tasks/main.yml | 4 +- .../backup-restore/backup-database.yml | 6 +- .../backup-restore/backup-instance.yml | 7 + .../backup-restore/get-backup-info.yml | 86 ----- .../tasks/providers/community/install.yml | 29 +- .../tasks/providers/community/restore.yml | 9 + .../providers/community/restore_instance.yml | 256 +++++++++++++ .../roles/suite_backup/tasks/main.yml | 10 +- ibm/mas_devops/roles/suite_restore/README.md | 62 +++ .../roles/suite_restore/defaults/main.yml | 10 + .../roles/suite_restore/tasks/main.yml | 294 +++++++++++++++ 15 files changed, 1165 insertions(+), 144 deletions(-) create mode 100644 ibm/mas_devops/plugins/action/restore_resource.py delete mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-backup-info.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml create mode 100644 ibm/mas_devops/roles/suite_restore/README.md create mode 100644 ibm/mas_devops/roles/suite_restore/defaults/main.yml create mode 100644 ibm/mas_devops/roles/suite_restore/tasks/main.yml diff --git a/.gitignore b/.gitignore index 5ab7a9c78b..817124f181 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ cpd-cli-workspace/* /node_modules package-lock.json package.json +test*.sh # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/ibm/mas_devops/plugins/action/restore_resource.py b/ibm/mas_devops/plugins/action/restore_resource.py new file mode 100644 index 0000000000..24f9cdbadc --- /dev/null +++ b/ibm/mas_devops/plugins/action/restore_resource.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 + +import logging +import os +import yaml +import urllib3 + +from ansible.plugins.action import ActionBase +from ansible.errors import AnsibleError +from ansible.utils.display import Display +from kubernetes import client, config +from kubernetes.dynamic import DynamicClient + +from mas.devops.restore import loadYamlFile, restoreResource + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient + +display = Display() + + +# Custom logging handler to forward Python logs to Ansible display +class AnsibleDisplayHandler(logging.Handler): + """Custom logging handler that forwards log messages to Ansible's display system""" + + def emit(self, record): + try: + msg = self.format(record) + if record.levelno >= logging.ERROR: + display.error(msg) + elif record.levelno >= logging.WARNING: + display.warning(msg) + else: + display.vvv(msg) # Use vvv for info/debug messages (visible with -vvv) + except Exception: + self.handleError(record) + + +# Configure logging to use both console and Ansible display +def setup_logging(): + """Setup logging to output to both console and Ansible display""" + # Create formatter + formatter = logging.Formatter('%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + + # Setup root logger + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + + # Add Ansible display handler + ansible_handler = AnsibleDisplayHandler() + ansible_handler.setFormatter(formatter) + root_logger.addHandler(ansible_handler) + + # Also add console handler for direct execution/debugging + console_handler = logging.StreamHandler() + console_handler.setFormatter(formatter) + root_logger.addHandler(console_handler) + + +# Initialize logging +setup_logging() + + +def apply_overrides(resource_data: dict, override_values: dict, resource_kind: str) -> dict: + """ + Apply override values to a resource based on key paths, filtered by resource kind. + + Args: + resource_data: The resource dictionary to modify + override_values: Dictionary mapping resource kinds to lists of override dictionaries + resource_kind: The kind of the current resource (e.g., 'Suite', 'Secret', 'ConfigMap') + + Returns: + dict: Modified resource data + + Example: + override_values = { + 'Suite': [ + {'spec.domain': 'mydomain.com'}, + {'spec.clusterIssuer.name': 'bob'} + ], + 'Secret': [ + {'data.value': 'newvalue'} + ] + } + """ + if not override_values or resource_kind not in override_values: + return resource_data + + kind_overrides = override_values[resource_kind] + if not kind_overrides: + return resource_data + + for override_dict in kind_overrides: + for key_path, new_value in override_dict.items(): + # Skip if value is NO_OVERRIDE (use backup value) + if new_value == "NO_OVERRIDE": + display.vvv(f"Skipping override for {resource_kind}: {key_path} (NO_OVERRIDE)") + continue + + # Split the key path by dots to navigate nested structure + keys = key_path.split('.') + + # Navigate to the parent of the target key + current = resource_data + for i, key in enumerate(keys[:-1]): + if key not in current: + # Create missing intermediate dictionaries + current[key] = {} + elif not isinstance(current[key], dict): + # Can't navigate further if intermediate value is not a dict + display.warning(f"Cannot apply override '{key_path}': '{'.'.join(keys[:i+1])}' is not a dictionary") + break + current = current[key] + else: + # Set the final value + final_key = keys[-1] + old_value = current.get(final_key, '') + current[final_key] = new_value + display.vvv(f"Applied override for {resource_kind}: {key_path}: {old_value} -> {new_value}") + + return resource_data + + +class ActionModule(ActionBase): + """ + Restore Kubernetes resources from a backup archive directory. + Automatically discovers and restores all resources found in the backup. + + - If a resource doesn't exist, it will be created + - If a resource exists and replace_resource=True, it will be updated (replaced) + - If a resource exists and replace_resource=False, it will be skipped + + Usage Example + ------------- + tasks: + - name: "Restore and replace specific MAS Suite resources" + ibm.mas_devops.restore_resource: + backup_path: "/backup/backup-250115-120000-suite" + resource_kinds: + - Secret + - ConfigMap + replace_resource: true + + - name: "Restore all resources (skip existing)" + ibm.mas_devops.restore_resource: + backup_path: "/backup/backup-250115-120000-suite" + replace_resource: false + + - name: "Restore resources with overrides" + ibm.mas_devops.restore_resource: + backup_path: "/backup/backup-250115-120000-suite" + resource_kinds: + - Suite + - Secret + - ConfigMap + replace_resource: true + override_values: + Suite: + - spec.domain: mydomain.com + - spec.clusterIssuer.name: bob + Secret: + - data.value: newvalue + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Initialize DynamicClient and grab the task args + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + # Load kubernetes configuration + try: + if host and api_key: + # Use provided host and api_key + configuration = client.Configuration() + configuration.host = host + configuration.api_key = {'authorization': f'Bearer {api_key}'} + configuration.verify_ssl = False + api_client = client.ApiClient(configuration) + else: + # Load from kubeconfig + config.load_kube_config() + api_client = client.ApiClient() + except Exception as e: + raise AnsibleError(f"Failed to initialize Kubernetes client: {e}") + + dynClient = DynamicClient(api_client) + + backup_path = self._task.args.get('backup_path') + replace_resource = self._task.args.get('replace_resource', True) + resource_kinds = self._task.args.get('resource_kinds', None) + override_values = self._task.args.get('override_values', None) + + if backup_path is None or backup_path == "": + raise AnsibleError(f"Error: backup_path argument was not provided") + + # Check if backup path exists + if not os.path.exists(backup_path): + raise AnsibleError(f"Error: backup_path does not exist: {backup_path}") + + # Check if resources directory exists + resources_path = os.path.join(backup_path, 'resources') + if not os.path.exists(resources_path): + raise AnsibleError(f"Error: resources directory not found in backup: {resources_path}") + + display.v(f"Starting restore of MAS Suite resources from '{backup_path}'") + display.v(f"Replace existing resources: {'enabled' if replace_resource else 'disabled'}") + if override_values: + override_kinds = ', '.join(override_values.keys()) + display.v(f"Override values will be applied for resource kinds: {override_kinds}") + + total_created = 0 + total_updated = 0 + total_skipped = 0 + total_failed = 0 + failed_resources = [] # Track failed resources with details + + # Discover all resource types in the backup + try: + resource_dirs = [d for d in os.listdir(resources_path) + if os.path.isdir(os.path.join(resources_path, d))] + except Exception as e: + raise AnsibleError(f"Error listing resource directories in {resources_path}: {e}") + + if not resource_dirs: + display.warning(f"No resource directories found in {resources_path}") + return dict( + message="No resources found to restore", + failed=False, + changed=False, + success=True, + created_count=0, + updated_count=0, + skipped_count=0, + failed_count=0, + failed_resources=[] + ) + + display.v(f"Found {len(resource_dirs)} resource type(s) in backup: {', '.join(sorted(resource_dirs))}") + + # Filter resource directories if specific kinds requested + if resource_kinds: + # Convert resource_kinds to lowercase directory names (add 's' suffix) + requested_dirs = set() + for kind in resource_kinds: + # Handle both singular and plural forms + dir_name = kind.lower() + if not dir_name.endswith('s'): + dir_name = dir_name + 's' + requested_dirs.add(dir_name) + + # Filter to only requested directories + resource_dirs = [d for d in resource_dirs if d in requested_dirs] + + if not resource_dirs: + display.warning(f"None of the requested resource kinds found in backup") + return dict( + message="No requested resources found to restore", + failed=False, + changed=False, + success=True, + created_count=0, + updated_count=0, + skipped_count=0, + failed_count=0, + failed_resources=[] + ) + + display.v(f"Restoring {len(resource_dirs)} requested resource type(s): {', '.join(sorted(resource_dirs))}") + + # Process each resource directory + for resource_dir in sorted(resource_dirs): + resource_dir_path = os.path.join(resources_path, resource_dir) + + # Get all YAML files in this directory + try: + yaml_files = [f for f in os.listdir(resource_dir_path) if f.endswith('.yaml')] + except Exception as e: + display.warning(f"Error listing files in {resource_dir_path}: {e}") + continue + + if not yaml_files: + display.v(f"No YAML files found in {resource_dir}/") + continue + + display.v(f"Restoring {len(yaml_files)} resource(s) from {resource_dir}/") + + # Process each YAML file + for yaml_file in sorted(yaml_files): + yaml_file_path = os.path.join(resource_dir_path, yaml_file) + + # Load the resource data + resource_data = loadYamlFile(yaml_file_path) + if not resource_data: + display.warning(f"Failed to load {yaml_file_path}") + total_failed += 1 + failed_resources.append({ + 'kind': 'Unknown', + 'name': yaml_file.replace('.yaml', ''), + 'description': f"Unknown/{yaml_file.replace('.yaml', '')}", + 'error': 'Failed to load YAML file' + }) + continue + + # Apply overrides if provided + if override_values: + resource_kind = resource_data.get('kind', 'Unknown') + resource_data = apply_overrides(resource_data, override_values, resource_kind) + + # Restore the resource + success, resource_name, status_msg = restoreResource( + dynClient, resource_data, namespace=None, replace_resource=replace_resource + ) + + if success: + if status_msg == "updated": + # Resource was updated + total_updated += 1 + elif status_msg == "skipped": + # Resource was skipped + total_skipped += 1 + else: + # Resource was created + total_created += 1 + else: + total_failed += 1 + kind = resource_data.get('kind', 'Unknown') + failed_resources.append({ + 'kind': kind, + 'name': resource_name, + 'description': f"{kind}/{resource_name}", + 'error': status_msg + }) + + display.v(f"Progress: {total_created} created, {total_updated} updated, {total_skipped} skipped, {total_failed} failed") + + display.v(f"Restore complete: {total_created} resources created, {total_updated} updated, {total_skipped} skipped, {total_failed} failed") + + # Determine if the restore was successful + has_failures = total_failed > 0 + + return dict( + message=f"Restored {total_created + total_updated} MAS Suite resources ({total_created} created, {total_updated} updated, {total_skipped} skipped)" + (f" with {total_failed} failures" if has_failures else ""), + failed=has_failures, + changed=(total_created + total_updated) > 0, + success=not has_failures, + created_count=total_created, + updated_count=total_updated, + skipped_count=total_skipped, + failed_count=total_failed, + failed_resources=failed_resources + ) + +# Made with Bob \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index 3e30f88905..95bb6ae81f 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -121,7 +121,7 @@ Specifies which operation to perform on the MongoDB instance. - Use `create-mongo-service-credentials` to generate service credentials (ibm only) **Valid values** (provider-specific): -- **community**: `install`, `uninstall`, `backup`, `restore` +- **community**: `install`, `uninstall`, `backup`, `restore`, `restore_database` - **aws**: `install`, `uninstall`, `docdb_secret_rotate`, `destroy-data` - **ibm**: `install`, `uninstall`, `backup`, `restore`, `create-mongo-service-credentials` @@ -550,18 +550,23 @@ Confirmation flag to upgrade MongoDB from version 7 to version 8. - Environment Variable: `MONGODB_V8_UPGRADE` - Default Value: `false` -Role Variables - Backup and Restore (CE Operator) +Role Variables - Backup and Restore (CE Operator) - (authored by IBM Bob) ------------------------------------------------------------------------------- ### mongodb_action For backup and restore operations, set `mongodb_action` to one of the following: - `backup`: Create a backup of MongoDB databases and optionally instance resources -- `restore_database`: Restore MongoDB databases from a backup to an existing instance -- `install`: Deploy a new MongoDB instance and restore data from backup +- `restore`: Restore both MongoDB instance resources and databases from a backup (full restore) +- `restore_database`: Restore only MongoDB databases from a backup to an existing instance (database-only restore) - Environment Variable: `MONGODB_ACTION` - Default Value: `install` +**Action Details:** +- **backup**: Creates a complete backup including database data and optionally Kubernetes resources (secrets, certificates, CRs) +- **restore**: Performs a full restore by recreating the MongoDB instance from backup resources and then restoring database data +- **restore_database**: Restores only the database data to an already running MongoDB instance without touching instance resources + ### mas_backup_dir **Required for backup/restore operations**. Local directory path where backup files will be stored or read from. @@ -577,10 +582,10 @@ For backup and restore operations, set `mongodb_action` to one of the following: - Example: `251212-021316` ### br_skip_instance -Controls whether to backup MongoDB instance resources (secrets, certificates, issuers) along with database data. Set to `false` to include instance resources in the backup. +Controls whether to backup MongoDB instance resources (secrets, certificates, issuers) along with database data. Set to `true` to skip instance resources in the backup. - Environment Variable: `BR_SKIP_INSTANCE` -- Default Value: `true` +- Default Value: `false` ### mongodb_instance_name The name of the MongoDB instance to backup. @@ -595,19 +600,33 @@ Optional. Specific MAS application ID for targeted backup/restore operations. - Default Value: None -Backup and Restore Operations (authored by IBM Bob) +Backup and Restore Operations - (authored by IBM Bob) ------------------------------------------------------------------------------- +This section provides comprehensive information about MongoDB backup and restore operations for the Community Edition (CE) operator. + +### Action Comparison + +| Action | Purpose | Instance Resources | Database Data | Prerequisites | Use Case | +|--------|---------|-------------------|---------------|---------------|----------| +| `backup` | Create backup | Optional (controlled by `br_skip_instance`) | Yes | Running MongoDB instance | Regular backups, disaster recovery preparation | +| `restore` | Full restore | Yes (recreates instance) | Yes | Backup with instance resources | Disaster recovery, cluster migration, complete restoration | +| `restore_database` | Database-only restore | No (preserves existing) | Yes | Running MongoDB instance with matching version | Data recovery, selective restore, testing | + ### Backup Process The MongoDB backup operation creates a backup of your MongoDB instance and databases associated with your MAS instance: -1. **Database Backup**: Uses `mongodump` to export databases with filter `"^(mas|iot)(_|-){{ mas_instance_id }}(_|-)(?!.*monitor$)"` to match databases like `mas--` or `iot--`. -2. **Instance Resources** (optional): Backs up Kubernetes resources including: - - MongoDB Custom Resource (CR) +1. **Database Backup**: Uses `mongodump` to export databases with filter `"^(mas|iot|sls|ibm-sls)(_|-)({{ mas_instance_id }}|sls)(_|-)(?!.*monitor$)"` to match databases like `mas--` or `iot--` or `ibm-sls_sls_licensing` or `sls-_sls_licensing`. +2. **Instance Resources** : Backs up Kubernetes resources including: + - CustomResourceDefinition (CRD) + - MongoDBCommunity Custom Resource (CR) - Secrets (TLS certificates, credentials) - Certificate resources - Issuer resources + - Mongodb operator deployment + - service monitoring + - grafana dashboards **Backup Directory Structure:** ``` @@ -617,46 +636,102 @@ The MongoDB backup operation creates a backup of your MongoDB instance and datab │ ├── mongodump-.tar.gz │ └── mongodb-info.yaml └── resources/ - ├── cr.yml + ├── mongodbcommunitys/ ├── secrets/ ├── certificates/ └── issuers/ + └── {kind}s/ ``` ### Restore Process -**Database Restore (`restore_database` action):** -- Restores database data to an existing MongoDB instance -- Requires MongoDB version to match the backup version -- Uses `mongorestore` to import required databases - -**Install from Backup (`install` action):** -- Validates backup files -- Creates namespace and restores resources to namespace -- Gets MongoDbCE instance details from backup data -- Installs MongoDb with configuration from backup -- Waits for instance to be ready -- Restores database data after instance is ready +The MongoDB role supports two types of restore operations: + +#### 1. Full Restore (`restore` action) +Performs a complete restoration of both the MongoDB instance and its databases: + +**Steps:** +1. Validates backup files and required variables +2. Restores the namespace (Project) +3. Restores Secrets and ConfigMaps +4. Restores CustomResourceDefinitions (CRDs) +5. Restores RBAC resources (ServiceAccount, Role, RoleBinding) +6. Configures anyuid permissions for MongoDB service accounts +7. Restores the MongoDB Operator Deployment +8. Waits for MongoDB operator to be ready +9. Restores Certificate Manager resources (Issuer, Certificate) +10. Restores the MongoDBCommunity Custom Resource +11. Waits for MongoDB StatefulSets to be ready +12. Restores ServiceMonitor and GrafanaDashboard resources +13. Restores database data using `mongorestore` + +**When to use:** +- Disaster recovery scenarios +- Migrating MongoDB to a new cluster +- Recreating a deleted MongoDB instance +- Setting up a new environment from backup + +**Requirements:** +- `mas_instance_id`: MAS instance identifier +- `mas_backup_dir`: Directory containing the backup +- `mongodb_backup_version`: Timestamp of the backup to restore +- `br_skip_instance`: Set to `false` to restore instance resources + +#### 2. Database-Only Restore (`restore_database` action) +Restores only the database data to an existing MongoDB instance: + +**Steps:** +1. Validates backup files and required variables +2. Verifies existing MongoDB installation is running +3. Checks MongoDB version compatibility +4. Restores database data using `mongorestore` + +**When to use:** +- Restoring data to an existing MongoDB instance +- Recovering from data corruption without recreating the instance +- Selective database restoration +- Testing data restoration without affecting instance configuration + +**Requirements:** +- `mas_instance_id`: MAS instance identifier +- `mas_backup_dir`: Directory containing the backup +- `mongodb_backup_version`: Timestamp of the backup to restore +- `mongodb_namespace`: Namespace where MongoDB is running +- `mongodb_instance_name`: Name of the existing MongoDB instance +- MongoDB instance must already be running and accessible ### Important Considerations **Version Compatibility:** -- Target MongoDB version must match the backup version +- Target MongoDB version must match the backup version exactly - Version upgrades should be performed separately, not during restore +- The restore process validates version compatibility before proceeding +- For `restore_database` action, the existing MongoDB instance must be running the same version as the backup **Storage Requirements:** - Ensure sufficient storage in the backup directory - Plan for at least 2x the database size for backup storage +- Backup directory structure: `/tmp/masbr/backup--mongoce/` +- Monitor disk space during backup operations **Security:** - Backup files contain sensitive data and credentials -- Secure backup directory with appropriate permissions +- Secure backup directory with appropriate permissions (chmod 700 recommended) - Consider encrypting backups for long-term storage +- Backup includes TLS certificates and admin credentials +- Restrict access to backup files to authorized personnel only **Performance:** - Backup operations may impact MongoDB performance - Schedule backups during low-usage periods - Monitor resource utilization during backup/restore +- Large databases may take significant time to backup/restore +- Network bandwidth may affect backup/restore speed + +**Restore Action Differences:** +- **`restore` action**: Recreates the entire MongoDB instance from scratch, including all Kubernetes resources +- **`restore_database` action**: Only restores database data to an existing instance, preserving current configuration +- Use `br_skip_instance` variable to control whether instance resources are included in backup/restore ### Backup and Restore Best Practices @@ -730,7 +805,14 @@ Create a complete backup including both database data and instance resources (se ``` ### Restore Database (CE Operator) -Restore MongoDB databases from a backup to an existing MongoDB instance. +Restore MongoDB databases from a backup to an existing MongoDB instance without recreating the instance resources. + +**Use Case**: This action is ideal when you need to restore database data to an already running MongoDB instance, such as recovering from data corruption or restoring specific databases. + +**Prerequisites**: +- MongoDB instance must already be installed and running +- MongoDB version must match the backup version +- Backup files must be available in the specified backup directory ```yaml - hosts: localhost @@ -746,6 +828,51 @@ Restore MongoDB databases from a backup to an existing MongoDB instance. - ibm.mas_devops.mongodb ``` +**What this does**: +1. Validates that the backup exists at the specified location +2. Verifies the MongoDB instance is running in the specified namespace +3. Checks version compatibility between backup and running instance +4. Restores database data using `mongorestore` command +5. Preserves all existing instance configuration and resources + +**Note**: This action does NOT restore instance resources (secrets, certificates, CRs). Use the `restore` action for full instance restoration. + +### Full Restore (CE Operator) +Perform a complete restoration of MongoDB instance including all Kubernetes resources and database data from a backup. + +**Use Case**: This action is ideal for disaster recovery scenarios where you need to recreate the entire MongoDB instance from scratch, including all configuration, secrets, certificates, and data. + +**Prerequisites**: +- Backup files must be available in the specified backup directory +- Backup must include instance resources (created with `br_skip_instance: false`) +- Target cluster must have cert-manager installed +- Sufficient storage and resources available + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mongodb_action: restore + mas_instance_id: masinst1 + mongodb_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + br_skip_instance: false + roles: + - ibm.mas_devops.mongodb +``` + +**What this does**: +1. Validates backup files and required variables +2. Restores namespace, secrets, and ConfigMaps +3. Restores CRDs and RBAC resources +4. Restores MongoDB Operator deployment +5. Restores Certificate Manager resources +6. Restores MongoDBCommunity Custom Resource +7. Waits for MongoDB instance to be fully operational +8. Restores database data using `mongorestore` + +**Note**: This is a complete restoration that recreates the MongoDB instance from scratch. Use `restore_database` action if you only need to restore data to an existing instance. + ### Install from Backup (CE Operator) Deploy a new MongoDB instance using configuration from a backup and restore data from a backup. This is useful for disaster recovery or migrating MongoDB to a new cluster. diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index c15a37f071..ddc1d20ffa 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -124,7 +124,7 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # ----------------------------------------------------------------------------- mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" -br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" +br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" # Mongo upgrade flags # If identified that there's an existing Mongo that might lead to a v5 or v6 upgrade diff --git a/ibm/mas_devops/roles/mongodb/tasks/main.yml b/ibm/mas_devops/roles/mongodb/tasks/main.yml index 889fa4c543..1d7cc18132 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/main.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/main.yml @@ -10,8 +10,8 @@ assert: that: - mongodb_action is defined - - mongodb_action in ["install", "uninstall", "backup", "restore_database"] - fail_msg: "mongodb_action property is required and must be one of 'install', 'uninstall', 'backup', 'restore_database'" + - mongodb_action in ["install", "uninstall", "backup", "restore_database", "restore"] + fail_msg: "mongodb_action property is required and must be one of 'install', 'uninstall', 'backup', 'restore_database', 'restore'" # 2. Run the install / uninstall for specified provider # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index 84523316c9..ac49f665d5 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -67,8 +67,8 @@ assert: that: - create_roleuser_output.rc == 0 - - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) - fail_msg: "Failed to create role and user for restore" + - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) is not none + fail_msg: "Failed to create role and user for backup" - name: "Debug create-role-user logs" debug: @@ -85,7 +85,7 @@ assert: that: - database_backup_output.rc == 0 - - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) + - database_backup_output.stdout | regex_search('DATABASEBACKUPstatus-SUCCESS', multiline=True) is not none fail_msg: "Failed to backup databases" - name: "Debug database-backup logs" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml index 3243220a40..2db41832bf 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml @@ -6,6 +6,9 @@ mongoce_backup_resources: - namespace: "{{ mongodb_namespace }}" resources: + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ mongodb_namespace }}" # CRD - mongodbcommunity.mongodbcommunity.mongodb.com - kind: CustomResourceDefinition api_version: apiextensions.k8s.io/v1 @@ -59,6 +62,10 @@ # grafanaDashboard - kind: GrafanaDashboard api_version: grafana.integreatly.org/v1beta1 + # Service Monitor + - kind: ServiceMonitor + api_version: monitoring.coreos.com/v1 + name: "{{ mongodb_instance_name }}-service-monitor" # Call the backup_resources plugin to execute the backup to the path provided # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-backup-info.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-backup-info.yml deleted file mode 100644 index 27be65f306..0000000000 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/get-backup-info.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- -# Check mongodb restore required variables -# ----------------------------------------------------------------------------- -- name: "Fail if require variables for Mongodb {{ mongodb_action }} are not provided" - ibm.mas_devops.verify_backup_restore_vars: - mas_instance_id: "{{ mas_instance_id }}" - mas_backup_dir: "{{ mas_backup_dir }}" - mongodb_backup_version: "{{ mongodb_backup_version }}" - action: "restore" - component: "mongodb" - -- name: "Set fact: backup dir paths" - set_fact: - mongodb_backup_dir: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" - mongodb_resource_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/resources" - mongodb_data_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/data" - -- name: "Check and get mongodb backup cr.yml" - ibm.mas_devops.get_mongodb_cr_to_restore: - mongodb_resource_path: "{{ mongodb_resource_path }}" - mongodb_backup_version: "{{ mongodb_backup_version }}" - register: mongodb_backup_cr_result - -- name: "Set fact : cr.yml" - set_fact: - mongodb_cr: "{{ mongodb_backup_cr_result.mongodb_cr }}" - -- name: "Set fact: Retrieve namespace and instance from backup CR" - set_fact: - mongodb_namespace: "{{ mongodb_cr.metadata.namespace }}" - mongodb_instance_name: "{{ mongodb_cr.metadata.name }}" - -- name: "Set fact: Retrieve information from backup CR" - set_fact: - mongo_extras_version: "{{ mongodb_cr.spec.version }}" - target_mongodb_version: "{{ mongodb_cr.spec.version }}" - mongodb_security: "{{ mongodb_cr.spec.security }}" - mongodb_users: "{{ mongodb_cr.spec.users | default([]) }}" - mongodb_replicas: "{{ mongodb_cr.spec.members }}" - mongodb_cpu_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('500m') }}" - mongodb_memory_limits: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.limits.cpu | default('1Gi') }}" - mongodb_cpu_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('250m') }}" - mongodb_memory_requests: "{{ mongodb_cr.spec.statefulSet.spec.template.spec.containers[0].resources.requests.cpu | default('512Mi') }}" - mongodb_storage_capacity_data: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[0].spec.resources.requests.storage }}" - mongodb_storage_capacity_logs: "{{ mongodb_cr.spec.statefulSet.spec.volumeClaimTemplates[1].spec.resources.requests.storage }}" - -- name: "Debug: Mongodb restore variables" - debug: - msg: - - "mongodb_instance_name: {{ mongodb_instance_name }}" - - "mongodb_namespace: {{ mongodb_namespace }}" - - "mongo_extras_version: {{ mongo_extras_version }}" - - "target_mongodb_version: {{ target_mongodb_version }}" - - "mongodb_security: {{ mongodb_security }}" - - "mongodb_users: {{ mongodb_users }}" - - "mongodb_replicas: {{ mongodb_replicas }}" - - "mongodb_cpu_limits: {{ mongodb_cpu_limits }}" - - "mongodb_memory_limits: {{ mongodb_memory_limits }}" - - "mongodb_cpu_requests: {{ mongodb_cpu_requests }}" - - "mongodb_memory_requests: {{ mongodb_memory_requests }}" - - "mongodb_storage_capacity_data: {{ mongodb_storage_capacity_data }}" - - "mongodb_storage_capacity_logs: {{ mongodb_storage_capacity_logs }}" - -# Restore MongoDb instance's resources from backup -# We will restore the following resources from backup: -# - User, TLS Secrets defined in the MongoDb CR -# - Issuers in the MongoDb namespace -# - Certificates in the MongoDb namespace -# All the other resources will be created afresh during MongoDb installation -# ----------------------------------------------------------------------------- - -- name: "Restore MongoDB instance resources from backup" - ibm.mas_devops.restore_mongoce_resources: - mongodb_namespace: "{{ mongodb_namespace }}" - backup_secrets_path: "{{ mongodb_resource_path }}/secrets" - backup_issuers_path: "{{ mongodb_resource_path }}/issuers" - backup_certificates_path: "{{ mongodb_resource_path }}/certificates" - register: restore_mongoce_resources_result - -- name: "Assert: MongoDB resources restored successfully" - assert: - that: - - restore_mongoce_resources_result is defined - - restore_mongoce_resources_result.restored is defined - - restore_mongoce_resources_result.restored == True - fail_msg: "Failed to restore MongoDB instance resources from backup." diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml index 89da497d40..ccc4bda8ed 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install.yml @@ -1,21 +1,10 @@ --- -# Detect if this is an install-from-backup scenario -- name: "community : install : Check for backup restore variables" - set_fact: - install_from_backup: "{{ (mongodb_backup_version is defined and mongodb_backup_version != '' and mongodb_backup_version != 'None') and (mas_backup_dir is defined and mas_backup_dir != '') }}" - -# Restore MongoDB instance from backup if needed -- name: "community : install : Restore MongoDB instance from backup" - when: - - install_from_backup | bool - include_tasks: tasks/providers/community/backup-restore/get-backup-info.yml - -# Check for existing MongoDb install +# 1. Check for existing MongoDb install # ----------------------------------------------------------------------------- - name: "community : install : Lookup existing mongo install" include_tasks: tasks/providers/community/check-mongo-exists.yml -# Load default storage classes (if not provided by the user and not an update) +# 2. Load default storage classes (if not provided by the user and not an update) # ----------------------------------------------------------------------------- - name: Use chosen (or default) storage class when: existing_mongo_storage_class is not defined @@ -51,7 +40,7 @@ mongodb_storage_capacity_data: "{{ existing_mongodb_storage_capacity_data }}" mongodb_storage_capacity_logs: "{{ existing_mongodb_storage_capacity_logs }}" -# Perform upgrade from 4.2 to 4.4 if necessary +# 3. Perform upgrade from 4.2 to 4.4 if necessary # ----------------------------------------------------------------------------- # We will ALWAYS upgrade from 4.2 to 4.4, no matter what. v4.4 is the minimum version everyone should be running at - name: "community : install : Upgrade from v4.2 to v4.4" @@ -62,20 +51,12 @@ - existing_mongo_minor_version is defined - existing_mongo_minor_version == '4.2' -# Check for existing MongoDb install (again) +# 4. Check for existing MongoDb install (again) # ----------------------------------------------------------------------------- - name: "community : install : Lookup existing mongo install (again)" include_tasks: tasks/providers/community/check-mongo-exists.yml -# Install MongoDB +# 5. Install the selected MongoDb version # ----------------------------------------------------------------------------- - name: "community : install : Install/upgrade to chosen mongo version" include_tasks: tasks/providers/community/install-mongo.yml - -# Restore MongoDB database from backup -# ----------------------------------------------------------------------------- -- name: "community : install : Restore MongoDB database from backup" - include_tasks: tasks/providers/community/backup-restore/restore-database.yml - when: - - install_from_backup is defined - - install_from_backup | bool diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml new file mode 100644 index 0000000000..b5717a1a6f --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml @@ -0,0 +1,9 @@ +--- +# Task to restore instance when br_skip_instance is set to false +# ----------------------------------------------------------------------------- +- include_tasks: "tasks/providers/{{ mongodb_provider }}/restore_instance.yml" + when: not br_skip_instance | bool + +# Task to restore database +# ----------------------------------------------------------------------------- +- include_tasks: "tasks/providers/{{ mongodb_provider }}/restore_database.yml" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml new file mode 100644 index 0000000000..673b53c6a5 --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml @@ -0,0 +1,256 @@ +--- +# Check mongodb restore required variables +# ----------------------------------------------------------------------------- +- name: "Fail if required variables for Mongodb database restore are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_backup_version: "{{ mongodb_backup_version }}" + action: "restore" + component: "mongodb" + +# Set backup path facts +- name: "Set fact: backup dir paths" + set_fact: + mongodb_backup_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" + mongodb_resources_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce/resources" + +- name: "Check mongodb resource path exist" + stat: + path: "{{ mongodb_resources_path }}" + register: resources_backup_path_stat + +- name: "Fail if backup archive does not exist" + fail: + msg: "Mongodb resources archive not found at: {{ mongodb_resources_path }}" + when: not resources_backup_path_stat.stat.exists or not resources_backup_path_stat.stat.isdir + +# Verify cert-manager exists +# ----------------------------------------------------------------------------- +- name: Detect Certificate Manager installation + include_tasks: "{{ role_path }}/../../common_tasks/detect_cert_manager.yml" + +# Verify only one MongoCE instance file is present in backup archive +# ----------------------------------------------------------------------------- +- name: Get files from {{ mongodb_resources_path }}/mongodbcommunitys directory + set_fact: + instance_files: "{{ lookup('fileglob', '{{ mongodb_resources_path }}/mongodbcommunitys/*', wantlist=True) }}" + +- name: Assert exactly one MongoDBCommunity CR exists + assert: + that: + - instance_files | length == 1 + fail_msg: "MongoDBCommunity Directory must contain exactly one file" + +- name: Set fact mongodb cr + set_fact: + mongodbcr_cfg: "{{ lookup('file', '{{ instance_files[0] }}') | from_yaml }}" + +# Get Mongodb details from backup CR +# ----------------------------------------------------------------------------- +- name: Set fact mongo namespace and instance name from backup CR + set_fact: + mongodb_namespace: "{{ mongodbcr_cfg.metadata.namespace }}" + mongodb_instance_name: "{{ mongodbcr_cfg.metadata.name }}" + source_mongodb_replicas_check: "{{ mongodbcr_cfg.spec.members }}" + source_mongodb_version: "{{ mongodbcr_cfg.spec.version }}" + +- name: "Mongo restore information" + debug: + msg: + - "MAS Instance ID ................ {{ mas_instance_id }}" + - "MongoDB Namespace .............. {{ mongodb_namespace }}" + - "MongoDB Instance Name .......... {{ mongodb_instance_name }}" + - "Backup Version ................. {{ mongodb_backup_version }}" + - "Backup Path .................... {{ mongodb_backup_path }}" + +# 1. Restore Namespace +# ----------------------------------------------------------------------------- +- name: Restore namespace + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - Project + replace_resource: false + register: namespace_result + +# 2. Restore Secrets and ConfigMaps +# ----------------------------------------------------------------------------- +- name: Restore Secrets and ConfigMaps + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - Secret + - ConfigMap + register: secrets_configmaps_result + when: namespace_result.success + +# 3. Restore CRD +# ----------------------------------------------------------------------------- +- name: Restore CRD + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - CustomResourceDefinition + register: crd_result + when: secrets_configmaps_result.success + +# 4. Restore RBAC +# ----------------------------------------------------------------------------- +- name: Restore ServiceAccount, Role, RoleBinding + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - ServiceAccount + - Role + - RoleBinding + register: rbac_result + when: crd_result.success + +# 5. Configure anyuid permissions in the MongoDb namespace +# ----------------------------------------------------------------------------- +- name: "Configure anyuid permissions in the MongoDb namespace(s)" + shell: | + oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mongodb_namespace }}:default + oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mongodb_namespace }}:mongodb-kubernetes-operator + oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mongodb_namespace }}:mongodb-database + when: + - mongodb_namespace is defined + +# 6. Restore the MongoDb Operator deployment +# ----------------------------------------------------------------------------- +- name: Restore the MongoDb Operator + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - Deployment + register: operator_result + when: rbac_result.success + +- name: "Wait for Mongodb operator to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + name: mongodb-kubernetes-operator + namespace: "{{ mongodb_namespace }}" + register: mongodb_operator_status + retries: 60 + delay: 60 + until: + - mongodb_operator_status.resources is defined + - mongodb_operator_status.resources | length > 0 + - mongodb_operator_status.resources[0].status is defined + - mongodb_operator_status.resources[0].status.conditions is defined + - mongodb_operator_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Available') | selectattr('status', 'equalto', 'True') | list | length > 0 + +# 7. Restore Certficate Manager resources +# ----------------------------------------------------------------------------- +- name: "Restore Certificate Manager resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - Issuer + - Certificate + register: certmanager_result + when: operator_result.success + +# Load default storage classes (if not provided by the user and not an update) +# ----------------------------------------------------------------------------- +- name: Use chosen (or default) storage class + when: existing_mongo_storage_class is not defined + include_tasks: tasks/providers/community/determine-storage-classes.yml + +# 8. Restore MongodbCE CR resource +# ----------------------------------------------------------------------------- +- name: "Restore MongodbCE CR resource" + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - MongoDBCommunity +# override_values: +# MongoDBCommunity: +# - spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName: mongodb_storage_class +# - spec.statefulSet.spec.volumeClaimTemplates[1].spec.storageClassName: mongodb_storage_class + register: cr_result + when: certmanager_result.success + +- name: "Wait for {{ mongodb_instance_name }} stateful set to be ready" + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: StatefulSet + name: "{{ mongodb_instance_name }}" + namespace: "{{ mongodb_namespace }}" + vars: + mongodb_replicas_check: "{{ source_mongodb_replicas_check }}" + register: mongodb_statefulset + retries: 45 # Approx 90 minutes + delay: 120 # 2 minutes + until: + - mongodb_statefulset.resources is defined + - mongodb_statefulset.resources | length > 0 + - mongodb_statefulset.resources[0].status.readyReplicas is defined + - mongodb_statefulset.resources[0].status.readyReplicas == (mongodb_replicas_check|int) + +- name: "Wait for {{ mongodb_instance_name }}-arb stateful set to be ready" + when: source_mongodb_version is version('4.4.0','>=') # this statefulset will only exist in Mongo v4.4+ + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: StatefulSet + name: "{{ mongodb_instance_name }}-arb" + namespace: "{{ mongodb_namespace }}" + register: mongodb_arb_statefulset + retries: 45 # Approx 90 minutes + delay: 120 # 2 minutes + until: + - mongodb_arb_statefulset.resources is defined + - mongodb_arb_statefulset.resources | length > 0 + - mongodb_arb_statefulset.resources[0].status.availableReplicas is defined + - mongodb_arb_statefulset.resources[0].status.availableReplicas == 0 + +- name: "Wait for Mongo CR to report expected version {{ source_mongodb_version }}" + kubernetes.core.k8s_info: + api_version: mongodbcommunity.mongodb.com/v1 + kind: MongoDBCommunity + name: "{{ mongodb_instance_name }}" + namespace: "{{ mongodb_namespace }}" + register: mongodb_cr + retries: 45 # Approx 45 minutes + delay: 60 # 1 minute + until: + - mongodb_cr.resources[0].status.version is defined + - mongodb_cr.resources[0].status.version == source_mongodb_version + +# Restore MongoDb service monitor +# ----------------------------------------------------------------------------- +- name: "Restore Service Monitor resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - ServiceMonitor + register: servicemonitor_result + when: cr_result.success + +# Restore Grafana Dashboard if cluster has grafana v5 capability +# ----------------------------------------------------------------------------- +- name: Get cluster info + kubernetes.core.k8s_cluster_info: + register: api_status + no_log: true + +- name: Determine cluster grafana capabilities + set_fact: + supports_grafanav5: "{{ + api_status is defined and + api_status.apis is defined and + api_status.apis['grafana.integreatly.org/v1beta1'] is defined }}" + +- name: "Restore Grafana Dashboards" + ibm.mas_devops.restore_resource: + backup_path: "{{ mongodb_backup_path }}" + resource_kinds: + - GrafanaDashboard + register: grafdash_result + when: + - servicemonitor_result.success + - supports_grafanav5 + diff --git a/ibm/mas_devops/roles/suite_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_backup/tasks/main.yml index 0473c9fdf0..7e4c62deeb 100644 --- a/ibm/mas_devops/roles/suite_backup/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_backup/tasks/main.yml @@ -79,13 +79,19 @@ name: "{{ mas_instance_id }}-cert-public-ca" - namespace: "{{ mas_core_namespace }}" resources: + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ mas_core_namespace }}" + - kind: Secret + api_version: v1 + name: "{{ mas_instance_id }}-credentials-superuser" # subscription - kind: Subscription api_version: operators.coreos.com/v1alpha1 - name: ibm-mas-operator + name: ibm-mas - kind: OperatorGroup api_version: operators.coreos.com/v1 - name: ibm-mas-operator-group + name: operatorgroup # secrets - kind: Secret api_version: v1 diff --git a/ibm/mas_devops/roles/suite_restore/README.md b/ibm/mas_devops/roles/suite_restore/README.md new file mode 100644 index 0000000000..fdf62c6ed2 --- /dev/null +++ b/ibm/mas_devops/roles/suite_restore/README.md @@ -0,0 +1,62 @@ +Restore MAS Core +=============================================================================== + +Overview +------------------------------------------------------------------------------- +This role supports restoring the MAS Core namespace resources when provided the +backup arhive generated from `suite_backup` role. + +!!! important + Restore can only be made to the an instance with the same MAS instance ID as the backup. + + +Role Variables +------------------------------------------------------------------------------- + +### mas_instance_id +The instance ID of the Maximo Application Suite installation to restore. + +- **Required** +- Environment Variable: `MAS_INSTANCE_ID` +- Default Value: None + +### mas_backup_dir +The local directory path where backup files to restore are stored. + +- **Required** +- Environment Varia` +- Default: None +- Example: `/tmp/mas_backups` + +### suite_backup_version +The version of the backup file located in the `MAS_BACKUP_DIR` to be used +in the restore + +- **Required** +- Default: None +- Environment Variable: `SUITE_BACKUP_VERSION` +- Example: `260116-130937` + +### mas_domain +The domain to use for the MAS Suite instance. If not provided, the domain from the backup will be used. + +- Optional +- Environment Variable: `MAS_DOMAIN` +- Default: `NO_OVERRIDE` (uses value from backup) +- Example: `mydomain.example.com` + +### sls_url +The URL for the Suite License Service (SLS). If not provided, the URL from the backup will be used. + +- Optional +- Environment Variable: `SLS_URL` +- Default: `NO_OVERRIDE` (uses value from backup) +- Example: `https://sls.example.com` + +### bas_url +The URL for the Behavior Analytics Service (BAS). If not provided, the URL from the backup will be used. + +- Optional +- Environment Variable: `BAS_URL` +- Default: `NO_OVERRIDE` (uses value from backup) +- Example: `https://bas.example.com` diff --git a/ibm/mas_devops/roles/suite_restore/defaults/main.yml b/ibm/mas_devops/roles/suite_restore/defaults/main.yml new file mode 100644 index 0000000000..c44c12d7a5 --- /dev/null +++ b/ibm/mas_devops/roles/suite_restore/defaults/main.yml @@ -0,0 +1,10 @@ +--- +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" +suite_backup_version: "{{ lookup('env', 'SUITE_BACKUP_VERSION') }}" + +# Override values for restore - set to NO_OVERRIDE to use values from backup +mas_domain: "{{ lookup('env', 'MAS_DOMAIN') | default('NO_OVERRIDE', true) }}" +sls_url: "{{ lookup('env', 'SLS_URL') | default('NO_OVERRIDE', true) }}" +bas_url: "{{ lookup('env', 'BAS_URL') | default('NO_OVERRIDE', true) }}" diff --git a/ibm/mas_devops/roles/suite_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_restore/tasks/main.yml new file mode 100644 index 0000000000..68cb6462af --- /dev/null +++ b/ibm/mas_devops/roles/suite_restore/tasks/main.yml @@ -0,0 +1,294 @@ +--- +# 1. Check mas core restore required variables +# ----------------------------------------------------------------------------- +- name: "Fail if mas_instance_id is not provided" + ansible.builtin.assert: + that: mas_instance_id is defined and mas_instance_id != "" + fail_msg: "mas_instance_id is required" + +- name: "Fail if mas_backup_dir is not provided" + ansible.builtin.assert: + that: mas_backup_dir is defined and mas_backup_dir != "" + fail_msg: "mas_backup_dir is required" + +- name: "Fail if suite_backup_version is not provided" + ansible.builtin.assert: + that: suite_backup_version is defined and suite_backup_version != "" + fail_msg: "suite_backup_version is required" +- name: "Set fact: mas core namespace name" + set_fact: + mas_core_namespace: "mas-{{ mas_instance_id }}-core" + +- name: "Set fact: mas suite backup path" + set_fact: + suite_backup_path: "{{ mas_backup_dir }}/backup-{{ suite_backup_version }}-suite" + +# 2. Verify backup archive exists +# ----------------------------------------------------------------------------- +- name: "Check if backup archive exists" + stat: + path: "{{ suite_backup_path }}" + register: backup_path_stat + +- name: "Fail if backup archive does not exist" + fail: + msg: "Backup archive not found at: {{ suite_backup_path }}" + when: not backup_path_stat.stat.exists or not backup_path_stat.stat.isdir + +- name: "Check if backup resources directory exists" + stat: + path: "{{ suite_backup_path }}/resources" + register: backup_resources_stat + +- name: "Fail if backup resources directory does not exist" + fail: + msg: "Backup resources directory not found at: {{ suite_backup_path }}/resources" + when: not backup_resources_stat.stat.exists or not backup_resources_stat.stat.isdir + +# 3. Verify cert-manager exists +# ----------------------------------------------------------------------------- +- name: Detect Certificate Manager installation + include_tasks: "{{ role_path }}/../../common_tasks/detect_cert_manager.yml" + +# 4. Display restore information +# ----------------------------------------------------------------------------- +- name: "Display restore information" + debug: + msg: + - "MAS Instance ID: {{ mas_instance_id }}" + - "MAS Core Namespace: {{ mas_core_namespace }}" + - "Backup Version: {{ suite_backup_version }}" + - "Backup Path: {{ suite_backup_path }}" + +# 5. Restore resources in correct order +# ----------------------------------------------------------------------------- +# Step 1: Restore Projects first +- name: "Restore Projects" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - Project + replace_resource: false + register: projects_result + +- name: "Display Projects restore results" + debug: + msg: "Projects: {{ projects_result.created_count }} created, {{ projects_result.updated_count }} updated, {{ projects_result.skipped_count }} skipped, {{ projects_result.failed_count }} failed" + +# Step 2: Restore Secrets and ConfigMaps +- name: "Restore Secrets and ConfigMaps" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - Secret + - ConfigMap + register: secrets_configmaps_result + when: projects_result.success + +- name: "Display Secrets and ConfigMaps restore results" + debug: + msg: "Secrets and ConfigMaps: {{ secrets_configmaps_result.created_count }} created, {{ secrets_configmaps_result.updated_count }} updated, {{ secrets_configmaps_result.skipped_count }} skipped, {{ secrets_configmaps_result.failed_count }} failed" + when: projects_result.success + +# Step 3: Restore OperatorGroups and Subscriptions +- name: "Restore OperatorGroups" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - OperatorGroup + register: operatorgroups_result + when: projects_result.success + +- name: "Display OperatorGroups restore results" + debug: + msg: "OperatorGroups: {{ operatorgroups_result.created_count }} created, {{ operatorgroups_result.updated_count }} updated, {{ operatorgroups_result.skipped_count }} skipped, {{ operatorgroups_result.failed_count }} failed" + when: projects_result.success + +- name: "Restore Subscriptions" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - Subscription + register: subscriptions_result + when: projects_result.success + +- name: "Display Subscriptions restore results" + debug: + msg: "Subscriptions: {{ subscriptions_result.created_count }} created, {{ subscriptions_result.updated_count }} updated, {{ subscriptions_result.skipped_count }} skipped, {{ subscriptions_result.failed_count }} failed" + when: projects_result.success + +# Wait for subscriptions to be ready +- name: "Wait for Subscriptions to be ready (60s delay)" + ibm.mas_devops.verify_subscriptions: + retries: 30 + delay: 60 + when: + - projects_result.success + - subscriptions_result.created_count > 0 or subscriptions_result.updated_count > 0 or subscriptions_result.skipped_count > 0 + +# Step 4: Restore Certificate Manager resources (ClusterIssuers, Issuers, Certificates) +- name: "Restore Certificate Manager resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - ClusterIssuer + - Issuer + - Certificate + register: certmanager_result + when: projects_result.success + +- name: "Display Certificate Manager restore results" + debug: + msg: "Certificate Manager resources: {{ certmanager_result.created_count }} created, {{ certmanager_result.updated_count }} updated, {{ certmanager_result.skipped_count }} skipped, {{ certmanager_result.failed_count }} failed" + when: projects_result.success + +# Step 5: Restore MAS Addon resources (addons.mas.ibm.com) +- name: "Restore MAS Addon resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - MVIEdge + - ReplicaDB + - GenericAddon + register: addons_result + when: projects_result.success + +- name: "Display MAS Addons restore results" + debug: + msg: "MAS Addons: {{ addons_result.created_count }} created, {{ addons_result.updated_count }} updated, {{ addons_result.skipped_count }} skipped, {{ addons_result.failed_count }} failed" + when: projects_result.success + +# Step 6: Restore MAS Config resources (config.mas.ibm.com) +- name: "Restore MAS Config resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - AppCfg + - BasCfg + - IDPCfg + - JdbcCfg + - KafkaCfg + - MongoCfg + - ObjectStorageCfg + - PushNotificationCfg + - ScimCfg + - SlsCfg + - SmtpCfg + - WatsonStudioCfg + override_values: + SlsCfg: + - spec.config.url: "{{ sls_url }}" + BasCfg: + - spec.config.url: "{{ bas_url }}" + register: configs_result + when: projects_result.success + +- name: "Display MAS Configs restore results" + debug: + msg: "MAS Configs: {{ configs_result.created_count }} created, {{ configs_result.updated_count }} updated, {{ configs_result.skipped_count }} skipped, {{ configs_result.failed_count }} failed" + when: projects_result.success + +# Step 7: Restore MAS Internal resources (internal.mas.ibm.com) +- name: "Restore MAS Internal resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - CoreIDP + register: internal_result + when: projects_result.success + +- name: "Display MAS Internal restore results" + debug: + msg: "MAS Internal: {{ internal_result.created_count }} created, {{ internal_result.updated_count }} updated, {{ internal_result.skipped_count }} skipped, {{ internal_result.failed_count }} failed" + when: projects_result.success + +# Step 8: Restore Suite +- name: "Restore Suite" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - Suite + override_values: + Suite: + - spec.domain: "{{ mas_domain }}" + register: suite_result + when: projects_result.success + +- name: "Display Suite restore results" + debug: + msg: "Suite: {{ suite_result.created_count }} created, {{ suite_result.updated_count }} updated, {{ suite_result.skipped_count }} skipped, {{ suite_result.failed_count }} failed" + when: projects_result.success + +# Wait for Suite to be ready before restoring Workspaces +- name: "Wait for Suite to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: core.mas.ibm.com/v1 + kind: Suite + name: "{{ mas_instance_id }}" + namespace: "{{ mas_core_namespace }}" + register: suite_status + retries: 60 + delay: 60 + until: + - suite_status.resources is defined + - suite_status.resources | length > 0 + - suite_status.resources[0].status is defined + - suite_status.resources[0].status.conditions is defined + - suite_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | selectattr('status', 'equalto', 'True') | list | length > 0 + when: + - projects_result.success + - suite_result.created_count > 0 or suite_result.updated_count > 0 or suite_result.skipped_count > 0 + +# Step 9: Restore Workspaces +- name: "Restore Workspaces" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: + - Workspace + register: workspace_result + when: projects_result.success + +- name: "Display Workspace restore results" + debug: + msg: "Workspaces: {{ workspace_result.created_count }} created, {{ workspace_result.updated_count }} updated, {{ workspace_result.skipped_count }} skipped, {{ workspace_result.failed_count }} failed" + when: projects_result.success + +# 6. Calculate total results +# ----------------------------------------------------------------------------- +- name: "Calculate total restore results" + set_fact: + total_created: "{{ (projects_result.created_count | default(0)) + (secrets_configmaps_result.created_count | default(0)) + (operatorgroups_result.created_count | default(0)) + (subscriptions_result.created_count | default(0)) + (certmanager_result.created_count | default(0)) + (addons_result.created_count | default(0)) + (configs_result.created_count | default(0)) + (internal_result.created_count | default(0)) + (suite_result.created_count | default(0)) + (workspace_result.created_count | default(0)) }}" + total_updated: "{{ (projects_result.updated_count | default(0)) + (secrets_configmaps_result.updated_count | default(0)) + (operatorgroups_result.updated_count | default(0)) + (subscriptions_result.updated_count | default(0)) + (certmanager_result.updated_count | default(0)) + (addons_result.updated_count | default(0)) + (configs_result.updated_count | default(0)) + (internal_result.updated_count | default(0)) + (suite_result.updated_count | default(0)) + (workspace_result.updated_count | default(0)) }}" + total_skipped: "{{ (projects_result.skipped_count | default(0)) + (secrets_configmaps_result.skipped_count | default(0)) + (operatorgroups_result.skipped_count | default(0)) + (subscriptions_result.skipped_count | default(0)) + (certmanager_result.skipped_count | default(0)) + (addons_result.skipped_count | default(0)) + (configs_result.skipped_count | default(0)) + (internal_result.skipped_count | default(0)) + (suite_result.skipped_count | default(0)) + (workspace_result.skipped_count | default(0)) }}" + total_failed: "{{ (projects_result.failed_count | default(0)) + (secrets_configmaps_result.failed_count | default(0)) + (operatorgroups_result.failed_count | default(0)) + (subscriptions_result.failed_count | default(0)) + (certmanager_result.failed_count | default(0)) + (addons_result.failed_count | default(0)) + (configs_result.failed_count | default(0)) + (internal_result.failed_count | default(0)) + (suite_result.failed_count | default(0)) + (workspace_result.failed_count | default(0)) }}" + +- name: "Display total restore results" + debug: + msg: + - "Restore completed{{ ' with failures' if total_failed | int > 0 else ' successfully' }}" + - "Total resources created: {{ total_created }}" + - "Total resources updated: {{ total_updated }}" + - "Total resources skipped: {{ total_skipped }}" + - "Total resources failed: {{ total_failed }}" + +# 7. Fail task if any errors occurred +# ----------------------------------------------------------------------------- +- name: "Collect all failed resources" + set_fact: + all_failed_resources: "{{ (projects_result.failed_resources | default([])) + (secrets_configmaps_result.failed_resources | default([])) + (operatorgroups_result.failed_resources | default([])) + (subscriptions_result.failed_resources | default([])) + (certmanager_result.failed_resources | default([])) + (addons_result.failed_resources | default([])) + (configs_result.failed_resources | default([])) + (internal_result.failed_resources | default([])) + (suite_result.failed_resources | default([])) + (workspace_result.failed_resources | default([])) }}" + +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ all_failed_resources | to_nice_yaml }}" + when: total_failed | int > 0 + +- name: "Fail if restore had errors" + fail: + msg: | + Restore failed for {{ total_failed }} resource(s): + {% for resource in all_failed_resources %} + - {{ resource.description }}: {{ resource.error }} + {% endfor %} + when: total_failed | int > 0 From ed33d687e25605c8754074070c2c291babeb1cb4 Mon Sep 17 00:00:00 2001 From: whitfiea Date: Tue, 20 Jan 2026 13:28:13 +0000 Subject: [PATCH 22/61] refactoring --- .../restore_database/restore-db-from-disk.yml | 4 +- .../restore_database/restore-db-from-s3.yml | 4 +- .../backup-restore/restore-database.yml | 4 +- .../roles/suite_restore/tasks/main.yml | 181 ++++++++++++++++-- 4 files changed, 168 insertions(+), 25 deletions(-) diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml index c3fa09343e..7136660f91 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml @@ -136,7 +136,7 @@ delay: 15 # seconds until: - prepare_scripts_result.rc == 0 - - prepare_scripts_result.stdout | regex_search('PrepareSuccess', multiline=True) + - prepare_scripts_result.stdout | regex_search('PrepareSuccess', multiline=True) is not none - name: "Debug prepare_backup_scripts.sh output" ansible.builtin.debug: @@ -195,5 +195,5 @@ assert: that: - restore_db_result.stdout is defined - - restore_db_result.stdout | regex_search('RestoreSuccess', multiline=True) + - restore_db_result.stdout | regex_search('RestoreSuccess', multiline=True) is not none fail_msg: "DB2 restore failed. Check /tmp/db2_restore_disk.log in DB2 pod {{ db2_pod_name }} for more details." diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml index bc3ccd4973..9da39e0e88 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml @@ -96,7 +96,7 @@ delay: 15 # seconds until: - prepare_scripts_result.rc == 0 - - prepare_scripts_result.stdout | regex_search('PrepareSuccess', multiline=True) + - prepare_scripts_result.stdout | regex_search('PrepareSuccess', multiline=True) is not none - name: "Debug prepare_backup_scripts.sh output" ansible.builtin.debug: @@ -138,5 +138,5 @@ assert: that: - restore_db_result.stdout is defined - - restore_db_result.stdout | regex_search('RestoreSuccess', multiline=True) + - restore_db_result.stdout | regex_search('RestoreSuccess', multiline=True) is not none fail_msg: "DB2 restore failed. Check /tmp/db2_restore_s3.log in DB2 pod {{ db2_pod_name }} for more details." diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index d0cc551bc6..4358244f78 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -103,7 +103,7 @@ assert: that: - create_roleuser_output.rc == 0 - - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) + - create_roleuser_output.stdout | regex_search('ROLEUSERstatus-SUCCESS', multiline=True) is not none fail_msg: "Failed to create role and user for restore" # Create temporary restore directory @@ -125,7 +125,7 @@ assert: that: - database_restore_output.rc == 0 - - database_restore_output.stdout | regex_search('DATABASERESTOREstatus-SUCCESS', multiline=True) + - database_restore_output.stdout | regex_search('DATABASERESTOREstatus-SUCCESS', multiline=True) is not none fail_msg: "Failed to restore database from backup" - name: "Debug database-restore logs" diff --git a/ibm/mas_devops/roles/suite_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_restore/tasks/main.yml index 68cb6462af..2ce997e685 100644 --- a/ibm/mas_devops/roles/suite_restore/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_restore/tasks/main.yml @@ -73,7 +73,11 @@ - name: "Display Projects restore results" debug: - msg: "Projects: {{ projects_result.created_count }} created, {{ projects_result.updated_count }} updated, {{ projects_result.skipped_count }} skipped, {{ projects_result.failed_count }} failed" + msg: >- + Projects: {{ projects_result.created_count }} created, + {{ projects_result.updated_count }} updated, + {{ projects_result.skipped_count }} skipped, + {{ projects_result.failed_count }} failed # Step 2: Restore Secrets and ConfigMaps - name: "Restore Secrets and ConfigMaps" @@ -87,7 +91,11 @@ - name: "Display Secrets and ConfigMaps restore results" debug: - msg: "Secrets and ConfigMaps: {{ secrets_configmaps_result.created_count }} created, {{ secrets_configmaps_result.updated_count }} updated, {{ secrets_configmaps_result.skipped_count }} skipped, {{ secrets_configmaps_result.failed_count }} failed" + msg: >- + Secrets and ConfigMaps: {{ secrets_configmaps_result.created_count }} created, + {{ secrets_configmaps_result.updated_count }} updated, + {{ secrets_configmaps_result.skipped_count }} skipped, + {{ secrets_configmaps_result.failed_count }} failed when: projects_result.success # Step 3: Restore OperatorGroups and Subscriptions @@ -101,7 +109,11 @@ - name: "Display OperatorGroups restore results" debug: - msg: "OperatorGroups: {{ operatorgroups_result.created_count }} created, {{ operatorgroups_result.updated_count }} updated, {{ operatorgroups_result.skipped_count }} skipped, {{ operatorgroups_result.failed_count }} failed" + msg: >- + OperatorGroups: {{ operatorgroups_result.created_count }} created, + {{ operatorgroups_result.updated_count }} updated, + {{ operatorgroups_result.skipped_count }} skipped, + {{ operatorgroups_result.failed_count }} failed when: projects_result.success - name: "Restore Subscriptions" @@ -114,7 +126,11 @@ - name: "Display Subscriptions restore results" debug: - msg: "Subscriptions: {{ subscriptions_result.created_count }} created, {{ subscriptions_result.updated_count }} updated, {{ subscriptions_result.skipped_count }} skipped, {{ subscriptions_result.failed_count }} failed" + msg: >- + Subscriptions: {{ subscriptions_result.created_count }} created, + {{ subscriptions_result.updated_count }} updated, + {{ subscriptions_result.skipped_count }} skipped, + {{ subscriptions_result.failed_count }} failed when: projects_result.success # Wait for subscriptions to be ready @@ -124,7 +140,10 @@ delay: 60 when: - projects_result.success - - subscriptions_result.created_count > 0 or subscriptions_result.updated_count > 0 or subscriptions_result.skipped_count > 0 + - >- + subscriptions_result.created_count > 0 or + subscriptions_result.updated_count > 0 or + subscriptions_result.skipped_count > 0 # Step 4: Restore Certificate Manager resources (ClusterIssuers, Issuers, Certificates) - name: "Restore Certificate Manager resources" @@ -139,7 +158,11 @@ - name: "Display Certificate Manager restore results" debug: - msg: "Certificate Manager resources: {{ certmanager_result.created_count }} created, {{ certmanager_result.updated_count }} updated, {{ certmanager_result.skipped_count }} skipped, {{ certmanager_result.failed_count }} failed" + msg: >- + Certificate Manager resources: {{ certmanager_result.created_count }} created, + {{ certmanager_result.updated_count }} updated, + {{ certmanager_result.skipped_count }} skipped, + {{ certmanager_result.failed_count }} failed when: projects_result.success # Step 5: Restore MAS Addon resources (addons.mas.ibm.com) @@ -155,7 +178,11 @@ - name: "Display MAS Addons restore results" debug: - msg: "MAS Addons: {{ addons_result.created_count }} created, {{ addons_result.updated_count }} updated, {{ addons_result.skipped_count }} skipped, {{ addons_result.failed_count }} failed" + msg: >- + MAS Addons: {{ addons_result.created_count }} created, + {{ addons_result.updated_count }} updated, + {{ addons_result.skipped_count }} skipped, + {{ addons_result.failed_count }} failed when: projects_result.success # Step 6: Restore MAS Config resources (config.mas.ibm.com) @@ -185,7 +212,11 @@ - name: "Display MAS Configs restore results" debug: - msg: "MAS Configs: {{ configs_result.created_count }} created, {{ configs_result.updated_count }} updated, {{ configs_result.skipped_count }} skipped, {{ configs_result.failed_count }} failed" + msg: >- + MAS Configs: {{ configs_result.created_count }} created, + {{ configs_result.updated_count }} updated, + {{ configs_result.skipped_count }} skipped, + {{ configs_result.failed_count }} failed when: projects_result.success # Step 7: Restore MAS Internal resources (internal.mas.ibm.com) @@ -199,7 +230,11 @@ - name: "Display MAS Internal restore results" debug: - msg: "MAS Internal: {{ internal_result.created_count }} created, {{ internal_result.updated_count }} updated, {{ internal_result.skipped_count }} skipped, {{ internal_result.failed_count }} failed" + msg: >- + MAS Internal: {{ internal_result.created_count }} created, + {{ internal_result.updated_count }} updated, + {{ internal_result.skipped_count }} skipped, + {{ internal_result.failed_count }} failed when: projects_result.success # Step 8: Restore Suite @@ -216,7 +251,11 @@ - name: "Display Suite restore results" debug: - msg: "Suite: {{ suite_result.created_count }} created, {{ suite_result.updated_count }} updated, {{ suite_result.skipped_count }} skipped, {{ suite_result.failed_count }} failed" + msg: >- + Suite: {{ suite_result.created_count }} created, + {{ suite_result.updated_count }} updated, + {{ suite_result.skipped_count }} skipped, + {{ suite_result.failed_count }} failed when: projects_result.success # Wait for Suite to be ready before restoring Workspaces @@ -234,10 +273,17 @@ - suite_status.resources | length > 0 - suite_status.resources[0].status is defined - suite_status.resources[0].status.conditions is defined - - suite_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | selectattr('status', 'equalto', 'True') | list | length > 0 + - >- + suite_status.resources[0].status.conditions | + selectattr('type', 'equalto', 'Ready') | + selectattr('status', 'equalto', 'True') | + list | length > 0 when: - projects_result.success - - suite_result.created_count > 0 or suite_result.updated_count > 0 or suite_result.skipped_count > 0 + - >- + suite_result.created_count > 0 or + suite_result.updated_count > 0 or + suite_result.skipped_count > 0 # Step 9: Restore Workspaces - name: "Restore Workspaces" @@ -250,22 +296,107 @@ - name: "Display Workspace restore results" debug: - msg: "Workspaces: {{ workspace_result.created_count }} created, {{ workspace_result.updated_count }} updated, {{ workspace_result.skipped_count }} skipped, {{ workspace_result.failed_count }} failed" + msg: >- + Workspaces: {{ workspace_result.created_count }} created, + {{ workspace_result.updated_count }} updated, + {{ workspace_result.skipped_count }} skipped, + {{ workspace_result.failed_count }} failed when: projects_result.success +# Wait for all Workspaces to be ready before finishing +- name: "Wait for all Workspaces to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: core.mas.ibm.com/v1 + kind: Workspace + namespace: "{{ mas_core_namespace }}" + label_selectors: + - "mas.ibm.com/instanceId={{ mas_instance_id }}" + register: workspace_status + retries: 60 + delay: 60 + until: + - workspace_status.resources is defined + - workspace_status.resources | length > 0 + - >- + workspace_status.resources | + selectattr('status.conditions', 'defined') | + list | length == workspace_status.resources | length + - >- + workspace_status.resources | + map(attribute='status.conditions') | + map('selectattr', 'type', 'equalto', 'Ready') | + map('selectattr', 'status', 'equalto', 'True') | + map('list') | select | list | length == workspace_status.resources | length + when: + - projects_result.success + - >- + workspace_result.created_count > 0 or + workspace_result.updated_count > 0 or + workspace_result.skipped_count > 0 + # 6. Calculate total results # ----------------------------------------------------------------------------- - name: "Calculate total restore results" set_fact: - total_created: "{{ (projects_result.created_count | default(0)) + (secrets_configmaps_result.created_count | default(0)) + (operatorgroups_result.created_count | default(0)) + (subscriptions_result.created_count | default(0)) + (certmanager_result.created_count | default(0)) + (addons_result.created_count | default(0)) + (configs_result.created_count | default(0)) + (internal_result.created_count | default(0)) + (suite_result.created_count | default(0)) + (workspace_result.created_count | default(0)) }}" - total_updated: "{{ (projects_result.updated_count | default(0)) + (secrets_configmaps_result.updated_count | default(0)) + (operatorgroups_result.updated_count | default(0)) + (subscriptions_result.updated_count | default(0)) + (certmanager_result.updated_count | default(0)) + (addons_result.updated_count | default(0)) + (configs_result.updated_count | default(0)) + (internal_result.updated_count | default(0)) + (suite_result.updated_count | default(0)) + (workspace_result.updated_count | default(0)) }}" - total_skipped: "{{ (projects_result.skipped_count | default(0)) + (secrets_configmaps_result.skipped_count | default(0)) + (operatorgroups_result.skipped_count | default(0)) + (subscriptions_result.skipped_count | default(0)) + (certmanager_result.skipped_count | default(0)) + (addons_result.skipped_count | default(0)) + (configs_result.skipped_count | default(0)) + (internal_result.skipped_count | default(0)) + (suite_result.skipped_count | default(0)) + (workspace_result.skipped_count | default(0)) }}" - total_failed: "{{ (projects_result.failed_count | default(0)) + (secrets_configmaps_result.failed_count | default(0)) + (operatorgroups_result.failed_count | default(0)) + (subscriptions_result.failed_count | default(0)) + (certmanager_result.failed_count | default(0)) + (addons_result.failed_count | default(0)) + (configs_result.failed_count | default(0)) + (internal_result.failed_count | default(0)) + (suite_result.failed_count | default(0)) + (workspace_result.failed_count | default(0)) }}" + total_created: >- + {{ + (projects_result.created_count | default(0)) + + (secrets_configmaps_result.created_count | default(0)) + + (operatorgroups_result.created_count | default(0)) + + (subscriptions_result.created_count | default(0)) + + (certmanager_result.created_count | default(0)) + + (addons_result.created_count | default(0)) + + (configs_result.created_count | default(0)) + + (internal_result.created_count | default(0)) + + (suite_result.created_count | default(0)) + + (workspace_result.created_count | default(0)) + }} + total_updated: >- + {{ + (projects_result.updated_count | default(0)) + + (secrets_configmaps_result.updated_count | default(0)) + + (operatorgroups_result.updated_count | default(0)) + + (subscriptions_result.updated_count | default(0)) + + (certmanager_result.updated_count | default(0)) + + (addons_result.updated_count | default(0)) + + (configs_result.updated_count | default(0)) + + (internal_result.updated_count | default(0)) + + (suite_result.updated_count | default(0)) + + (workspace_result.updated_count | default(0)) + }} + total_skipped: >- + {{ + (projects_result.skipped_count | default(0)) + + (secrets_configmaps_result.skipped_count | default(0)) + + (operatorgroups_result.skipped_count | default(0)) + + (subscriptions_result.skipped_count | default(0)) + + (certmanager_result.skipped_count | default(0)) + + (addons_result.skipped_count | default(0)) + + (configs_result.skipped_count | default(0)) + + (internal_result.skipped_count | default(0)) + + (suite_result.skipped_count | default(0)) + + (workspace_result.skipped_count | default(0)) + }} + total_failed: >- + {{ + (projects_result.failed_count | default(0)) + + (secrets_configmaps_result.failed_count | default(0)) + + (operatorgroups_result.failed_count | default(0)) + + (subscriptions_result.failed_count | default(0)) + + (certmanager_result.failed_count | default(0)) + + (addons_result.failed_count | default(0)) + + (configs_result.failed_count | default(0)) + + (internal_result.failed_count | default(0)) + + (suite_result.failed_count | default(0)) + + (workspace_result.failed_count | default(0)) + }} - name: "Display total restore results" debug: msg: - - "Restore completed{{ ' with failures' if total_failed | int > 0 else ' successfully' }}" + - >- + Restore completed{{ ' with failures' if total_failed | int > 0 + else ' successfully' }} - "Total resources created: {{ total_created }}" - "Total resources updated: {{ total_updated }}" - "Total resources skipped: {{ total_skipped }}" @@ -275,7 +406,19 @@ # ----------------------------------------------------------------------------- - name: "Collect all failed resources" set_fact: - all_failed_resources: "{{ (projects_result.failed_resources | default([])) + (secrets_configmaps_result.failed_resources | default([])) + (operatorgroups_result.failed_resources | default([])) + (subscriptions_result.failed_resources | default([])) + (certmanager_result.failed_resources | default([])) + (addons_result.failed_resources | default([])) + (configs_result.failed_resources | default([])) + (internal_result.failed_resources | default([])) + (suite_result.failed_resources | default([])) + (workspace_result.failed_resources | default([])) }}" + all_failed_resources: >- + {{ + (projects_result.failed_resources | default([])) + + (secrets_configmaps_result.failed_resources | default([])) + + (operatorgroups_result.failed_resources | default([])) + + (subscriptions_result.failed_resources | default([])) + + (certmanager_result.failed_resources | default([])) + + (addons_result.failed_resources | default([])) + + (configs_result.failed_resources | default([])) + + (internal_result.failed_resources | default([])) + + (suite_result.failed_resources | default([])) + + (workspace_result.failed_resources | default([])) + }} - name: "Display failed resources" debug: From 75638932dd2ee9b6fb64ac24c8dcb56fc158dcd6 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 20 Jan 2026 15:51:15 +0000 Subject: [PATCH 23/61] [patch] add results for mongo instance restore --- .../providers/community/restore_instance.yml | 152 ++++++++++++++++-- 1 file changed, 142 insertions(+), 10 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml index 673b53c6a5..41c303ea81 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml @@ -93,7 +93,9 @@ resource_kinds: - CustomResourceDefinition register: crd_result - when: secrets_configmaps_result.success + when: + - namespace_result.success + - secrets_configmaps_result is defined and secrets_configmaps_result.success # 4. Restore RBAC # ----------------------------------------------------------------------------- @@ -105,7 +107,9 @@ - Role - RoleBinding register: rbac_result - when: crd_result.success + when: + - namespace_result.success + - crd_result is defined and crd_result.success # 5. Configure anyuid permissions in the MongoDb namespace # ----------------------------------------------------------------------------- @@ -115,7 +119,7 @@ oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mongodb_namespace }}:mongodb-kubernetes-operator oc adm policy add-scc-to-user anyuid system:serviceaccount:{{ mongodb_namespace }}:mongodb-database when: - - mongodb_namespace is defined + - mongodb_namespace is defined and namespace_result.success # 6. Restore the MongoDb Operator deployment # ----------------------------------------------------------------------------- @@ -125,7 +129,9 @@ resource_kinds: - Deployment register: operator_result - when: rbac_result.success + when: + - namespace_result.success + - rbac_result is defined and rbac_result.success - name: "Wait for Mongodb operator to be ready (60s delay)" kubernetes.core.k8s_info: @@ -142,6 +148,10 @@ - mongodb_operator_status.resources[0].status is defined - mongodb_operator_status.resources[0].status.conditions is defined - mongodb_operator_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Available') | selectattr('status', 'equalto', 'True') | list | length > 0 + when: + - namespace_result.success + - operator_result is defined and operator_result.success + # 7. Restore Certficate Manager resources # ----------------------------------------------------------------------------- @@ -152,13 +162,18 @@ - Issuer - Certificate register: certmanager_result - when: operator_result.success + when: + - namespace_result.success + - operator_result is defined and operator_result.success # Load default storage classes (if not provided by the user and not an update) # ----------------------------------------------------------------------------- - name: Use chosen (or default) storage class - when: existing_mongo_storage_class is not defined include_tasks: tasks/providers/community/determine-storage-classes.yml + when: + - namespace_result.success + - certmanager_result is defined and certmanager_result.success + - existing_mongo_storage_class is not defined # 8. Restore MongodbCE CR resource # ----------------------------------------------------------------------------- @@ -172,7 +187,9 @@ # - spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName: mongodb_storage_class # - spec.statefulSet.spec.volumeClaimTemplates[1].spec.storageClassName: mongodb_storage_class register: cr_result - when: certmanager_result.success + when: + - namespace_result.success + - certmanager_result is defined and certmanager_result.success - name: "Wait for {{ mongodb_instance_name }} stateful set to be ready" kubernetes.core.k8s_info: @@ -190,9 +207,11 @@ - mongodb_statefulset.resources | length > 0 - mongodb_statefulset.resources[0].status.readyReplicas is defined - mongodb_statefulset.resources[0].status.readyReplicas == (mongodb_replicas_check|int) + when: + - namespace_result.success + - cr_result is defined and cr_result.success - name: "Wait for {{ mongodb_instance_name }}-arb stateful set to be ready" - when: source_mongodb_version is version('4.4.0','>=') # this statefulset will only exist in Mongo v4.4+ kubernetes.core.k8s_info: api_version: apps/v1 kind: StatefulSet @@ -206,6 +225,10 @@ - mongodb_arb_statefulset.resources | length > 0 - mongodb_arb_statefulset.resources[0].status.availableReplicas is defined - mongodb_arb_statefulset.resources[0].status.availableReplicas == 0 + when: + - namespace_result.success + - cr_result is defined and cr_result.success + - source_mongodb_version is version('4.4.0','>=') # this statefulset will only exist in Mongo v4.4+ - name: "Wait for Mongo CR to report expected version {{ source_mongodb_version }}" kubernetes.core.k8s_info: @@ -219,6 +242,9 @@ until: - mongodb_cr.resources[0].status.version is defined - mongodb_cr.resources[0].status.version == source_mongodb_version + when: + - namespace_result.success + - cr_result is defined and cr_result.success # Restore MongoDb service monitor # ----------------------------------------------------------------------------- @@ -228,7 +254,9 @@ resource_kinds: - ServiceMonitor register: servicemonitor_result - when: cr_result.success + when: + - namespace_result.success + - cr_result is defined and cr_result.success # Restore Grafana Dashboard if cluster has grafana v5 capability # ----------------------------------------------------------------------------- @@ -236,6 +264,9 @@ kubernetes.core.k8s_cluster_info: register: api_status no_log: true + when: + - namespace_result.success + - servicemonitor_result is defined and servicemonitor_result.success - name: Determine cluster grafana capabilities set_fact: @@ -243,6 +274,9 @@ api_status is defined and api_status.apis is defined and api_status.apis['grafana.integreatly.org/v1beta1'] is defined }}" + when: + - namespace_result.success + - servicemonitor_result is defined and servicemonitor_result.success - name: "Restore Grafana Dashboards" ibm.mas_devops.restore_resource: @@ -251,6 +285,104 @@ - GrafanaDashboard register: grafdash_result when: - - servicemonitor_result.success + - namespace_result.success + - servicemonitor_result is defined and servicemonitor_result.success - supports_grafanav5 +# Calculate total results +# ----------------------------------------------------------------------------- +- name: "Calculate total restore results" + set_fact: + total_created: >- + {{ + (namespace_result.created_count | default(0)) + + (secrets_configmaps_result.created_count | default(0)) + + (crd_result.created_count | default(0)) + + (operator_result.created_count | default(0)) + + (rbac_result.created_count | default(0)) + + (certmanager_result.created_count | default(0)) + + (cr_result.created_count | default(0)) + + (servicemonitor_result.created_count | default(0)) + + (grafdash_result.created_count | default(0)) + }} + total_updated: >- + {{ + (namespace_result.updated_count | default(0)) + + (secrets_configmaps_result.updated_count | default(0)) + + (crd_result.updated_count | default(0)) + + (operator_result.updated_count | default(0)) + + (rbac_result.updated_count | default(0)) + + (certmanager_result.updated_count | default(0)) + + (cr_result.updated_count | default(0)) + + (servicemonitor_result.updated_count | default(0)) + + (grafdash_result.updated_count | default(0)) + }} + total_skipped: >- + {{ + (namespace_result.skipped_count | default(0)) + + (secrets_configmaps_result.skipped_count | default(0)) + + (crd_result.skipped_count | default(0)) + + (operator_result.skipped_count | default(0)) + + (rbac_result.skipped_count | default(0)) + + (certmanager_result.skipped_count | default(0)) + + (cr_result.skipped_count | default(0)) + + (servicemonitor_result.skipped_count | default(0)) + + (grafdash_result.skipped_count | default(0)) + }} + total_failed: >- + {{ + (namespace_result.failed_count | default(0)) + + (secrets_configmaps_result.failed_count | default(0)) + + (crd_result.failed_count | default(0)) + + (operator_result.failed_count | default(0)) + + (rbac_result.failed_count | default(0)) + + (certmanager_result.failed_count | default(0)) + + (cr_result.failed_count | default(0)) + + (servicemonitor_result.failed_count | default(0)) + + (grafdash_result.failed_count | default(0)) + }} + +- name: "Display total restore results" + debug: + msg: + - >- + Restore completed{{ ' with failures' if total_failed | int > 0 + else ' successfully' }} + - "Total resources created: {{ total_created }}" + - "Total resources updated: {{ total_updated }}" + - "Total resources skipped: {{ total_skipped }}" + - "Total resources failed: {{ total_failed }}" + +# Fail task if any errors occurred +# ----------------------------------------------------------------------------- +- name: "Collect all failed resources" + set_fact: + all_failed_resources: >- + {{ + (namespace_result.failed_resources | default([])) + + (secrets_configmaps_result.failed_resources | default([])) + + (crd_result.failed_resources | default([])) + + (operator_result.failed_resources | default([])) + + (rbac_result.failed_resources | default([])) + + (certmanager_result.failed_resources | default([])) + + (cr_result.failed_resources | default([])) + + (servicemonitor_result.failed_resources | default([])) + + (grafdash_result.failed_resources | default([])) + }} + +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ all_failed_resources | to_nice_yaml }}" + when: total_failed | int > 0 + +- name: "Fail if restore had errors" + fail: + msg: | + Restore failed for {{ total_failed }} resource(s): + {% for resource in all_failed_resources %} + - {{ resource.description }}: {{ resource.error }} + {% endfor %} + when: total_failed | int > 0 + From 5f2de52a15c553aac19ba5a4fb672b6f659f7b6d Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 22 Jan 2026 08:10:47 +0000 Subject: [PATCH 24/61] [minor] B&R for catalog, certmanager, db2, mongo (#2073) --- docs/playbooks/backup-restore.md | 119 +--- ibm/mas_devops/playbooks/br_db2.yml | 7 +- ibm/mas_devops/playbooks/br_mongodb.yml | 5 +- .../plugins/action/restore_resource.py | 83 ++- .../action/verify_backup_restore_vars.py | 6 +- ibm/mas_devops/plugins/filter/filters.py | 100 +++- ibm/mas_devops/plugins/filter/test_filters.py | 202 +++++++ .../tasks/provider/redhat/backup.yml | 7 + .../tasks/provider/redhat/restore.yml | 179 ++++++ ibm/mas_devops/roles/db2/README.md | 329 ++++++++--- ibm/mas_devops/roles/db2/defaults/main.yml | 13 +- .../db2/tasks/backup/backup-instance.yml | 17 + .../roles/db2/tasks/backup/main.yml | 3 +- .../backup-database.yml | 0 .../roles/db2/tasks/backup_database/main.yml | 33 ++ .../roles/db2/tasks/install/install.yml | 467 ---------------- .../roles/db2/tasks/install/main.yml | 509 +++++++++++++++++- ibm/mas_devops/roles/db2/tasks/main.yml | 7 + .../restore/determine-storage-classes.yml | 15 + .../roles/db2/tasks/restore/main.yml | 7 + .../db2/tasks/restore/restore-instance.yml | 338 ++++++++++++ .../tasks/restore/retrieve_db2_passwords.yml | 66 +++ .../roles/ibm_catalogs/tasks/backup/main.yml | 9 +- .../tasks/restore/create-wiotp-secret.yml | 27 + .../roles/ibm_catalogs/tasks/restore/main.yml | 151 ++++++ ibm/mas_devops/roles/mongodb/README.md | 41 +- .../roles/mongodb/defaults/main.yml | 8 +- ibm/mas_devops/roles/mongodb/tasks/main.yml | 4 +- .../tasks/providers/community/backup.yml | 6 +- .../providers/community/backup_database.yml | 37 ++ .../tasks/providers/community/restore.yml | 3 +- .../providers/community/restore_instance.yml | 27 +- 32 files changed, 2083 insertions(+), 742 deletions(-) create mode 100644 ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml rename ibm/mas_devops/roles/db2/tasks/{backup => backup_database}/backup-database.yml (100%) create mode 100644 ibm/mas_devops/roles/db2/tasks/backup_database/main.yml delete mode 100644 ibm/mas_devops/roles/db2/tasks/install/install.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/main.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml create mode 100644 ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml create mode 100644 ibm/mas_devops/roles/ibm_catalogs/tasks/restore/create-wiotp-secret.yml create mode 100644 ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml create mode 100644 ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 89cd75f872..19fb6edc37 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -12,125 +12,8 @@ MAS Devops Collection includes playbooks for backing up and restoring of the fol # MongoDB Community Edition Backup and Restore Playbook ------------------------------------------------------------------------------- -This playbook `ibm.mas_devops.br_mongodb` will invoke the role [mongodb](../roles/mongodb.md) to backup/restore the MongoDB cluster instance and databases. - -### Overview - -This playbook supports backup and restore operations for a MongoDB Community Edition deployment used by a MAS instance. -If you are using other MongoDB venders, such as IBM Cloud Databases for MongoDB, Amazon DocumentDB or MongoDB Atlas Database, please refer to the corresponding vender's documentation for more information about their provided backup/restore service. - -The process is split into instance-level and database-level operations to provide flexibility depending on the target environment and recovery scenario. - -## Backup Workflow - -The backup operation consists of **two independent phases**. - -### 1. MongoDB Instance Backup (Optional) - -This phase captures the **Kubernetes resources** required to recreate a MongoDB Community Edition cluster, including: - -- MongoDBCommunity custom resource -- Configuration parameters -- User credentials and authentication settings -- TLS and secret references (if applicable) - -This backup enables reinstallation of a MongoDB instance while **preserving configuration and users**. - -#### Skipping Instance Backup - -If instance-level backup is not required, it can be skipped by setting the following environment variable: - -```bash -export BR_SKIP_INSTANCE=true -``` - -2. MongoDB Database Backup - -This phase performs a backup of the MongoDB databases associated with the specified MAS instance ID. - -- Only databases belonging to the MAS instance are included -- The backup is independent of the MongoDB installation method -- Can be used for restore into a new or existing MongoDB instance - -### Restore Workflow - -The restore operation also consists of two phases, with behavior determined by the state of the target environment. - -### 1. MongoDB Instance Restore (Conditional) - -- If no MongoDB instance is currently running in the target namespace: - - The playbook restores the MongoDB Community Edition instance using the previously backed-up Kubernetes resources - - Configuration, users, and credentials are recreated - - Once the instance is ready, database restore begins -- If a MongoDB instance already exists: - - Instance restore is skipped automatically - - The playbook proceeds directly to database restore - - -### 2. MongoDB Database Restore - -- Restores the MongoDB databases from the backup -- Databases are restored into the target MongoDB instance -- Supports restoring into: - - A newly recreated MongoDB instance - - An existing, running MongoDB instance - - -| Scenario | Instance Restore | Database Restore | -| -------------------------------------- | ---------------- | ---------------- | -| Backup with `BR_SKIP_INSTANCE=true` | Skipped | Executed | -| Restore with no MongoDB instance | Executed | Executed | -| Restore with existing MongoDB instance | Skipped | Executed | - -### Notes for Operators -- Ensure the correct MAS instance ID is provided before running the playbook -- Verify namespace access and required permissions for Kubernetes resources and secrets -- Instance restore is idempotent and safely skipped when not applicable - -| Variable Name | Required | Default | Description | -| ----------------------- | -------- | --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `MASBR_ACTION` | Yes | N/A | Specifies whether the playbook should perform a `backup` or a `restore`. | -| `MASBR_BACKUP_VERSION` | Conditional | `YYMMDD-HHMMSS` | Identifies the backup version. For **backup**, this value is optional and defaults to a timestamp in the format YYMMDD-HHMMSS if not provided. For **restore**, this value is mandatory and must match an existing backup version. | -| `MAS_INSTANCE_ID` | Yes | N/A | Identifies the MAS instance whose MongoDB databases should be backed up or restored. To back up multiple MAS instances that share the same MongoDB CE instance, run the playbook multiple times with different values. | -| `MONGODB_NAMESPACE` | No | `mongoce` | Namespace where MongoDB Community Edition is installed. Set this if MongoDB CE is deployed in a custom namespace. | -| `MONGODB_INSTANCE_NAME` | No | `mas-mongo-ce` | Name of the MongoDB Community Edition instance. For backup, this value is used to locate the instance. For restore, the value is taken from the backup data. | -| `BR_SKIP_INSTANCE` | No | `true` | Skips MongoDB instance backup or restore. Set to `false` to back up or restore instances. | - - -### Examples -```bash -# Backup of MongoDB CE cluster instance and database for the dev1 instance -export MASBR_ACTION=backup -export MAS_BACKUP_DIR=/tmp/backup -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_mongodb - - -# Backup of ONLY MongoDB database for the dev1 instance -export MASBR_ACTION=backup -export MAS_BACKUP_DIR=/tmp/backup -export MAS_INSTANCE_ID=dev -export BR_SKIP_INSTANCE=true -ansible-playbook ibm.mas_devops.br_mongodb - - -# Restore MongoDB cluster instance and database -export MASBR_ACTION=restore -export MAS_BACKUP_DIR=/tmp/backup -export MASBR_BACKUP_VERSION=251212-101010 -export MAS_INSTANCE_ID=dev -ansible-playbook ibm.mas_devops.br_mongodb - -# Restore ONLY MongoDB database -export MASBR_ACTION=restore -export MAS_BACKUP_DIR=/tmp/backup -export MASBR_BACKUP_VERSION=251212-101010 -export MAS_INSTANCE_ID=dev -export BR_SKIP_INSTANCE=true -ansible-playbook ibm.mas_devops.br_mongodb -``` - +Coming soon... Backup/Restore for Db2 ------------------------------------------------------------------------------- diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index 1601056d4a..f393957c33 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -7,9 +7,8 @@ mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or restore_database or install + db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or backup_database or restore or restore_database db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" # Required for restore action - br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false to enable DB2 instance backup/restore backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline and required for backup backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) }}" backup_s3_alias: "{{ lookup('env', 'BACKUP_S3_ALIAS') | default('S3DB2COS', true) }}" # Required for backup_vendor=s3 @@ -22,8 +21,8 @@ pre_tasks: - name: "Fail if DB2_ACTION is not set to backup|restore" assert: - that: db2_action in ["backup", "restore_database", "install"] - fail_msg: "DB2_ACTION is required and must be set to 'backup' or 'restore'" + that: db2_action in ["backup", "backup_database", "restore_database", "restore"] + fail_msg: "DB2_ACTION is required and must be set to 'backup' or 'backup_database' or 'restore_database' or 'restore'" - name: "Fail if BACKUP_VENDOR is not set to s3|disk" assert: diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index e1d61d0ab1..490661d42b 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -10,14 +10,13 @@ mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" mongodb_namespace: "{{ lookup('env', 'MONGODB_NAMESPACE') | default('mongoce', true) }}" mongodb_provider: "community" # only community is supported currently - br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set to false for mongodb instance backup mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Optional pre_tasks: - name: "Fail if mongodb_action is not set to backup|install|restore_database" assert: - that: mongodb_action in ["backup", "restore_database", "install"] - fail_msg: "mongodb_action is required and must be set to 'backup' or 'restore_database' or 'install'" + that: mongodb_action in ["backup", "backup_database", "restore_database", "restore"] + fail_msg: "mongodb_action is required and must be set to 'backup' or 'backup_database' or 'restore' or 'restore_database'" roles: - role: ibm.mas_devops.mongodb diff --git a/ibm/mas_devops/plugins/action/restore_resource.py b/ibm/mas_devops/plugins/action/restore_resource.py index 24f9cdbadc..954750394a 100644 --- a/ibm/mas_devops/plugins/action/restore_resource.py +++ b/ibm/mas_devops/plugins/action/restore_resource.py @@ -59,6 +59,54 @@ def setup_logging(): # Initialize logging setup_logging() +def filter_fields(resource_data: dict, fields_to_filter: dict, resource_kind: str) -> dict: + """ + Filter out specified fields from a resource based on key paths, filtered by resource kind. + + Args: + + resource_data: The resource dictionary to modify + fields_to_filter: Dictionary mapping resource kinds to lists of field paths to filter out + resource_kind: The kind of the current resource (e.g., 'Suite', 'Secret', 'ConfigMap') + + Returns: + dict: Modified resource data with specified fields removed + + Example: + fields_to_filter = { + 'Suite': [ + 'spec.domain', 'spec.clusterIssuer.name' + + ] + } + """ + + if not fields_to_filter or resource_kind not in fields_to_filter: + return resource_data + + kind_fields = fields_to_filter[resource_kind] + + if not kind_fields: + return resource_data + + for field_path in kind_fields: + # Split the field path into parts + keys = field_path.split('.') + # Traverse the dictionary to find the parent of the field to remove + current = resource_data + + for key in keys[:-1]: + if key not in current: + # If any part of the path doesn't exist, skip this field + break + current = current[key] + + # Remove the field if it exists + if keys[-1] in current: + del current[keys[-1]] + + return resource_data + def apply_overrides(resource_data: dict, override_values: dict, resource_kind: str) -> dict: """ @@ -146,7 +194,7 @@ class ActionModule(ActionBase): backup_path: "/backup/backup-250115-120000-suite" replace_resource: false - - name: "Restore resources with overrides" + - name: "Restore resources with overrides, filter_values and skip_files" ibm.mas_devops.restore_resource: backup_path: "/backup/backup-250115-120000-suite" resource_kinds: @@ -154,12 +202,19 @@ class ActionModule(ActionBase): - Secret - ConfigMap replace_resource: true + filter_values: + Suite: + - spec.domain + - spec.clusterIssuer.name override_values: Suite: - spec.domain: mydomain.com - spec.clusterIssuer.name: bob Secret: - data.value: newvalue + skip_files: #skip applying these files + Secret: + - jdbc-credentials.yaml """ def run(self, tmp=None, task_vars=None): super(ActionModule, self).run(tmp, task_vars) @@ -190,6 +245,8 @@ def run(self, tmp=None, task_vars=None): replace_resource = self._task.args.get('replace_resource', True) resource_kinds = self._task.args.get('resource_kinds', None) override_values = self._task.args.get('override_values', None) + filter_values = self._task.args.get('filter_values', None) + skip_files = self._task.args.get('skip_files', None) if backup_path is None or backup_path == "": raise AnsibleError(f"Error: backup_path argument was not provided") @@ -208,6 +265,15 @@ def run(self, tmp=None, task_vars=None): if override_values: override_kinds = ', '.join(override_values.keys()) display.v(f"Override values will be applied for resource kinds: {override_kinds}") + + if filter_values: + filter_kinds = ', '.join(filter_values.keys()) + display.v(f"Filter values will be applied for resource kinds: {filter_kinds}") + + skip_files_lower = None + if skip_files: + display.v(f"Skip files will be applied for resource kinds: {', '.join(skip_files.keys())}") + skip_files_lower = {f"{k.lower()}s": v for k, v in skip_files.items()} total_created = 0 total_updated = 0 @@ -271,6 +337,10 @@ def run(self, tmp=None, task_vars=None): # Process each resource directory for resource_dir in sorted(resource_dirs): resource_dir_path = os.path.join(resources_path, resource_dir) + files_to_skip= [] + + if skip_files_lower: + files_to_skip = skip_files_lower[resource_dir] # Get all YAML files in this directory try: @@ -287,6 +357,12 @@ def run(self, tmp=None, task_vars=None): # Process each YAML file for yaml_file in sorted(yaml_files): + + if yaml_file in files_to_skip: + display.v(f"Skipping {yaml_file} as it is in the skip list") + total_skipped += 1 + continue + yaml_file_path = os.path.join(resource_dir_path, yaml_file) # Load the resource data @@ -307,6 +383,11 @@ def run(self, tmp=None, task_vars=None): resource_kind = resource_data.get('kind', 'Unknown') resource_data = apply_overrides(resource_data, override_values, resource_kind) + # Filter values if provided + if filter_values: + resource_kind = resource_data.get('kind', 'Unknown') + resource_data = filter_fields(resource_data, filter_values, resource_kind) + # Restore the resource success, resource_name, status_msg = restoreResource( dynClient, resource_data, namespace=None, replace_resource=replace_resource diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index 7bad7d847a..feb0c55e72 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -8,10 +8,12 @@ class ActionModule(ActionBase): REQUIRED = { "catalog": { - "backup": ["mas_backup_dir"] + "backup": ["mas_backup_dir"], + "restore": ["mas_backup_dir", "catalog_backup_version"] }, "certmanager": { - "backup": ["mas_backup_dir"] + "backup": ["mas_backup_dir"], + "restore": ["mas_backup_dir", "certmanager_backup_version"] }, "grafana": { "backup": ["mas_backup_dir"] diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index 7c104131bc..26b72fb32e 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -430,26 +430,6 @@ def get_ecr_repositories(image_mirror_output): repositories.append(repo_to_add) return repositories -def get_tlscert_configmapname_from_mongoce(mongoDBCommunityCR): - """ - filter: get_tlscert_configmapname_from_mongoce - author: Sanjay Prabhakar - version_added: 0.1 - short_description: Get the name of TLS Cert configmap from MongoDBCommunity CR - description: - - This filter returns the name of TLS Cert configmap from MongoDBCommunity CR - options: - mongoDBCommunityCR: - description: MongoDBCommunity CR definition - required: True - """ - if 'security' in mongoDBCommunityCR['spec']: - if 'tls' in mongoDBCommunityCR['spec']['security']: - if 'caConfigMapRef' in mongoDBCommunityCR['spec']['security']['tls']: - return mongoDBCommunityCR['spec']['security']['tls']['caConfigMapRef']['name'] - else: - return None - def is_channel_upgrade_path_valid(current: str, target: str, valid_paths: dict) -> bool: """ Checks if a given current channel version can be upgraded to a target channel version. @@ -494,6 +474,81 @@ def get_default_upgrade_channel(current: str, valid_paths: dict) -> str: print(f'Error: channel upgrade compatibility matrix is incorrectly defined') return default +def set_storage_classes_names(storage_list: list, storage_class_name_rwo: str, storage_class_name_rwx: str): + """ + Iterate through the storage_list list and set the storage_class_name for each storage item based on the access mode. + Expects data to be + storage: + - name: meta + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 20Gi + storageClassName: nfs-client + type: create + - name: backup + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: nfs-client + type: create + """ + + for storage_item in storage_list: + if 'spec' in storage_item and 'accessModes' in storage_item['spec'] and 'storageClassName' in storage_item['spec']: + if storage_item['spec']['accessModes'][0] == 'ReadWriteMany': + storage_item['spec']['storageClassName'] = storage_class_name_rwx + else: + storage_item['spec']['storageClassName'] = storage_class_name_rwo + return storage_list + +def override_db2_storage_classes_names(storage_list: list, storage_class_name_meta: str, storage_class_name_data: str, storage_class_name_backup: str, storage_class_name_logs: str, storage_class_name_temp: str): + """ + Iterate through the storage_list list and set the storage_class_name for each storage item based on the storage name. + Expects data to be + storage: + - name: meta + spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 20Gi + storageClassName: nfs-client + type: create + - name: backup + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: nfs-client + type: create + """ + for storage_item in storage_list: + if 'name' in storage_item and 'spec' in storage_item and 'storageClassName' in storage_item['spec']: + if storage_item['name'] == 'meta': + storage_item['spec']['storageClassName'] = storage_class_name_meta + elif storage_item['name'] == 'data': + storage_item['spec']['storageClassName'] = storage_class_name_data + elif storage_item['name'] == 'backup': + storage_item['spec']['storageClassName'] = storage_class_name_backup + elif storage_item['name'] == 'tempts': + storage_item['spec']['storageClassName'] = storage_class_name_temp + elif storage_item['name'] == 'activelogs': + storage_item['spec']['storageClassName'] = storage_class_name_logs + else: + print(f'WARNING: Unhandled db2 storage name for {storage_item["name"]}') + + return storage_list + + class FilterModule(object): def filters(self): return { @@ -516,7 +571,8 @@ def filters(self): 'format_pre_version_with_buildid': format_pre_version_with_buildid, 'get_db2_instance_name': get_db2_instance_name, 'get_ecr_repositories': get_ecr_repositories, - 'get_tlscert_configmapname_from_mongoce': get_tlscert_configmapname_from_mongoce, 'is_channel_upgrade_path_valid': is_channel_upgrade_path_valid, - 'get_default_upgrade_channel': get_default_upgrade_channel + 'get_default_upgrade_channel': get_default_upgrade_channel, + 'set_storage_classes_names': set_storage_classes_names, + 'override_db2_storage_classes_names': override_db2_storage_classes_names } diff --git a/ibm/mas_devops/plugins/filter/test_filters.py b/ibm/mas_devops/plugins/filter/test_filters.py index f10ef5ecf4..b4c31c5f9c 100644 --- a/ibm/mas_devops/plugins/filter/test_filters.py +++ b/ibm/mas_devops/plugins/filter/test_filters.py @@ -60,3 +60,205 @@ def test_get_default_upgrade_channel_for_invalid_paths(): ########################################################################## + + +########################################################################## +# Tests for set_storage_class_name + + +def test_set_storage_class_name_with_rwx_access_mode(): + """Test that ReadWriteMany access mode sets RWX storage class""" + storage_data = [ + { + 'name': 'meta', + 'spec': { + 'accessModes': ['ReadWriteMany'], + 'resources': {'requests': {'storage': '20Gi'}}, + 'storageClassName': 'nfs-client' + }, + 'type': 'create' + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['spec']['storageClassName'] == 'rwx-class' + + +def test_set_storage_class_name_with_rwo_access_mode(): + """Test that ReadWriteOnce access mode sets RWO storage class""" + storage_data = [ + { + 'name': 'data', + 'spec': { + 'accessModes': ['ReadWriteOnce'], + 'resources': {'requests': {'storage': '10Gi'}}, + 'storageClassName': 'default' + }, + 'type': 'create' + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['spec']['storageClassName'] == 'rwo-class' + + +def test_set_storage_class_name_with_multiple_items(): + """Test processing multiple storage items with different access modes""" + storage_data = [ + { + 'name': 'meta', + 'spec': { + 'accessModes': ['ReadWriteMany'], + 'storageClassName': 'old-class' + } + }, + { + 'name': 'data', + 'spec': { + 'accessModes': ['ReadWriteOnce'], + 'storageClassName': 'old-class' + } + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['spec']['storageClassName'] == 'rwx-class' + assert result[1]['spec']['storageClassName'] == 'rwo-class' + + +def test_set_storage_class_name_with_empty_list(): + """Test that empty list returns empty list""" + storage_data = [] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result == [] + + +def test_set_storage_class_name_with_missing_spec(): + """Test that items without spec field are skipped""" + storage_data = [ + {'name': 'invalid', 'type': 'create'}, + { + 'name': 'valid', + 'spec': { + 'accessModes': ['ReadWriteMany'], + 'storageClassName': 'old-class' + } + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert len(result) == 2 + assert 'spec' not in result[0] + assert result[1]['spec']['storageClassName'] == 'rwx-class' + + +def test_set_storage_class_name_with_missing_access_modes(): + """Test that items without accessModes are skipped""" + storage_data = [ + { + 'name': 'invalid', + 'spec': { + 'storageClassName': 'old-class' + } + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['spec']['storageClassName'] == 'old-class' + + +def test_set_storage_class_name_with_non_list_access_modes(): + """Test that items with non-list accessModes are skipped""" + storage_data = [ + { + 'name': 'invalid', + 'spec': { + 'accessModes': 'ReadWriteMany', + 'storageClassName': 'old-class' + } + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['spec']['storageClassName'] == 'old-class' + + +def test_set_storage_class_name_with_missing_storage_class_name(): + """Test that items without storageClassName are skipped""" + storage_data = [ + { + 'name': 'invalid', + 'spec': { + 'accessModes': ['ReadWriteMany'] + } + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert 'storageClassName' not in result[0]['spec'] + + +def test_set_storage_class_name_with_empty_access_modes_list(): + """Test that items with empty accessModes list cause IndexError""" + storage_data = [ + { + 'name': 'invalid', + 'spec': { + 'accessModes': [], + 'storageClassName': 'old-class' + } + } + ] + # This will raise IndexError due to accessing [0] on empty list + with pytest.raises(IndexError): + set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + + +def test_set_storage_class_name_with_read_write_once_pod(): + """Test that ReadWriteOncePod access mode sets RWO storage class""" + storage_data = [ + { + 'name': 'data', + 'spec': { + 'accessModes': ['ReadWriteOncePod'], + 'storageClassName': 'old-class' + } + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['spec']['storageClassName'] == 'rwo-class' + + +def test_set_storage_class_name_with_read_only_many(): + """Test that ReadOnlyMany access mode sets RWO storage class (not RWX)""" + storage_data = [ + { + 'name': 'data', + 'spec': { + 'accessModes': ['ReadOnlyMany'], + 'storageClassName': 'old-class' + } + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['spec']['storageClassName'] == 'rwo-class' + + +def test_set_storage_class_name_preserves_other_fields(): + """Test that other fields in storage items are preserved""" + storage_data = [ + { + 'name': 'meta', + 'spec': { + 'accessModes': ['ReadWriteMany'], + 'resources': {'requests': {'storage': '20Gi'}}, + 'storageClassName': 'old-class', + 'volumeMode': 'Filesystem' + }, + 'type': 'create', + 'custom_field': 'value' + } + ] + result = set_storage_class_name(storage_data, 'rwo-class', 'rwx-class') + assert result[0]['name'] == 'meta' + assert result[0]['type'] == 'create' + assert result[0]['custom_field'] == 'value' + assert result[0]['spec']['resources'] == {'requests': {'storage': '20Gi'}} + assert result[0]['spec']['volumeMode'] == 'Filesystem' + assert result[0]['spec']['storageClassName'] == 'rwx-class' + + +########################################################################## diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml index f4babbcb38..e7200d8318 100644 --- a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml @@ -19,6 +19,13 @@ certmanager_backup_resources: - namespace: "{{ cert_manager_operator_namespace }}" resources: + # Projects + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ cert_manager_namespace }}" + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ cert_manager_operator_namespace }}" # certmanager source - kind: Subscription api_version: operators.coreos.com/v1alpha1 diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml new file mode 100644 index 0000000000..6ae5085506 --- /dev/null +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml @@ -0,0 +1,179 @@ +--- +- name: "Fail if require variables for Redhat cert-manager backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_backup_dir: "{{ mas_backup_dir }}" + certmanager_backup_version: "{{ certmanager_backup_version }}" + action: "restore" + component: "certmanager" + +- name: "Set fact: cert-manager backup base directory path" + set_fact: + certmgr_backup_path="{{ mas_backup_dir }}/backup-{{ certmanager_backup_version }}-certmanager" + certmgr_backup_resource_path="{{ mas_backup_dir }}/backup-{{ certmanager_backup_version }}-certmanager/resources" + +- name: "Check cert-manager backup resource path exist" + stat: + path: "{{ certmgr_backup_resource_path }}" + register: resources_backup_path_stat + +- name: "Fail if backup archive does not exist" + fail: + msg: "Cert-manager resources archive not found at: {{ certmgr_backup_resource_path }}" + when: not resources_backup_path_stat.stat.exists or not resources_backup_path_stat.stat.isdir + +- name: "Cert-manager restore information" + debug: + msg: + - "Backup Version ................. {{ certmanager_backup_version }}" + - "Backup Path .................... {{ certmgr_backup_path }}" + +# Restore Projects +# ------------------------------------------------------------------------- +- name: "Restore Projects" + ibm.mas_devops.restore_resource: + backup_path: "{{ certmgr_backup_path }}" + resource_kinds: + - Project + register: project_result + +# Restore OperatorGroups and Subscriptions +# ------------------------------------------------------------------------- +- name: "Restore OperatorGroups" + ibm.mas_devops.restore_resource: + backup_path: "{{ certmgr_backup_path }}" + resource_kinds: + - OperatorGroup + register: operatorgroups_result + when: project_result.success + +- name: "Restore Subscriptions" + ibm.mas_devops.restore_resource: + backup_path: "{{ certmgr_backup_path }}" + resource_kinds: + - Subscription + register: subscriptions_result + when: operatorgroups_result.success + +# Wait for Subscription to be processed +# ----------------------------------------------------------------------------- +- name: "Wait for Red Hat cert-manager-operator-controller-manager to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: apps/v1 + name: cert-manager-operator-controller-manager + namespace: "{{ cert_manager_operator_namespace }}" + kind: Deployment + register: certmanager_deployment + until: + - certmanager_deployment.resources is defined + - certmanager_deployment.resources | length > 0 + - certmanager_deployment.resources[0].status is defined + - certmanager_deployment.resources[0].status.replicas is defined + - certmanager_deployment.resources[0].status.readyReplicas is defined + - certmanager_deployment.resources[0].status.readyReplicas == certmanager_deployment.resources[0].status.replicas + retries: 30 # Approximately 1/2 hour before we give up + delay: 60 # 1 minute + when: subscriptions_result.success + +# Wait for CertManager instance to be created +# ----------------------------------------------------------------------------- +- name: "Wait for CertManager Cluster Custom Resource to be created" + kubernetes.core.k8s_info: + api_version: operator.openshift.io/v1alpha1 + name: cluster + kind: CertManager + register: certmanager_cluster_cr + until: + - certmanager_cluster_cr.resources is defined + - certmanager_cluster_cr.resources | length > 0 + retries: 10 # Approximately 5 minutes before we give up + delay: 30 # 30 seconds + when: subscriptions_result.success + + +# Wait for Cert Manager's webhook to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for cert-manager-webhook deployment to be ready (60s delay)" + kubernetes.core.k8s_info: + api_version: apps/v1 + name: cert-manager-webhook + namespace: "{{ cert_manager_namespace }}" + kind: Deployment + register: certmanager_webhook_deployment + until: + - certmanager_webhook_deployment.resources is defined + - certmanager_webhook_deployment.resources | length > 0 + - certmanager_webhook_deployment.resources[0].status is defined + - certmanager_webhook_deployment.resources[0].status.replicas is defined + - certmanager_webhook_deployment.resources[0].status.readyReplicas is defined + - certmanager_webhook_deployment.resources[0].status.readyReplicas == certmanager_webhook_deployment.resources[0].status.replicas + retries: 60 # Approximately 1/2 hour before we give up + delay: 60 # 1 minute + when: subscriptions_result.success + +# Calculate total results +# ----------------------------------------------------------------------------- +- name: "Calculate total restore results" + set_fact: + total_created: >- + {{ + (project_result.created_count | default(0)) + + (operatorgroups_result.created_count | default(0)) + + (subscriptions_result.created_count | default(0)) + }} + total_updated: >- + {{ + (project_result.updated_count | default(0)) + + (operatorgroups_result.updated_count | default(0)) + + (subscriptions_result.updated_count | default(0)) + }} + total_skipped: >- + {{ + (project_result.skipped_count | default(0)) + + (operatorgroups_result.skipped_count | default(0)) + + (subscriptions_result.skipped_count | default(0)) + }} + total_failed: >- + {{ + (project_result.failed_count | default(0)) + + (operatorgroups_result.failed_count | default(0)) + + (subscriptions_result.failed_count | default(0)) + }} + +- name: "Display total restore results" + debug: + msg: + - >- + Restore completed{{ ' with failures' if total_failed | int > 0 + else ' successfully' }} + - "Total resources created: {{ total_created }}" + - "Total resources updated: {{ total_updated }}" + - "Total resources skipped: {{ total_skipped }}" + - "Total resources failed: {{ total_failed }}" + +# Fail task if any errors occurred +# ----------------------------------------------------------------------------- +- name: "Collect all failed resources" + set_fact: + all_failed_resources: >- + {{ + (project_result.failed_resources | default([])) + + (operatorgroups_result.failed_resources | default([])) + + (subscriptions_result.failed_resources | default([])) + }} + +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ all_failed_resources | to_nice_yaml }}" + when: total_failed | int > 0 + +- name: "Fail if restore had errors" + fail: + msg: | + Restore failed for {{ total_failed }} resource(s): + {% for resource in all_failed_resources %} + - {{ resource.description }}: {{ resource.error }} + {% endfor %} + when: total_failed | int > 0 + diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index f9f8dd7794..bb596f7fab 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -56,20 +56,24 @@ Specifies which operation to perform on the Db2 database. **When to use**: - Use `install` (default) for initial Db2 deployment - Use `upgrade` to upgrade all Db2 instances in the namespace to a new version -- Use `backup` to create a backup of Db2 data -- Use `restore` to restore Db2 from a backup +- Use `backup` to create a backup of Db2 instance and/or database +- Use `restore` to restore a backup of db2 instance and database +- Use `restore_database` to restore only the database to an existing Db2 instance -**Valid values**: `install`, `upgrade`, `backup`, `restore` +**Valid values**: `install`, `upgrade`, `backup`, `restore`, `restore_database` -**Impact**: -- `install`: Creates new Db2 operator and instance +**Impact**: +- `install`: Creates new Db2 operator and instance. When `db2_backup_version` is provided, installs from backup (instance + database) - `upgrade`: Upgrades ALL instances in `db2_namespace` to `db2_version` (affects all instances in namespace) -- `backup`: Creates backup of Db2 data -- `restore`: Restores Db2 from backup +- `backup`: Creates backup of Db2 instance resources and/or database data +- `restore`: Creates new Db2 operator and instance from the backup of Db2 instance resources and restores database data to the created instance +- `restore_database`: Restores database to an existing running Db2 instance (does not restore instance resources) **Related variables**: - `db2_version`: Required for upgrade action to specify target version - `db2_namespace`: All instances in this namespace are affected by upgrade +- `db2_backup_version`: Required for restore/restore_database action; optional for backup, defaults to YYMMDD-HHMMSS +- `override_storageclass`: In Restore, controls whether storage classes are overridden **Note**: **WARNING** - When using `upgrade`, ALL Db2 instances in the specified namespace will be upgraded. Plan accordingly and ensure `db2_version` matches the operator channel. @@ -863,80 +867,266 @@ Local directory path where backups will be stored or restored from. - **Required** for backup and restore operations - Environment Variable: `MAS_BACKUP_DIR` - Default: None -- Example: `/tmp/mas_backups` + +**Purpose**: Specifies the local filesystem directory for storing Db2 backup files and metadata. This directory serves as the staging area for all backup and restore operations. + +**When to use**: +- Always required when performing backup or restore operations +- Must be accessible from the system running the Ansible playbook +- Should have sufficient disk space for database backups + +**Valid values**: Any valid local filesystem path (e.g., `/tmp/mas_backups`, `/backup/db2`) + +**Impact**: +- Backup files and metadata are stored in subdirectories under this path +- Directory structure: `/backup--db2u/` +- Insufficient space will cause backup failures + +**Related variables**: +- `db2_backup_version`: Used to create versioned backup subdirectories +- `backup_vendor`: When set to `s3`, database backups go to S3 but instance resources still use this directory + +**Example**: `/tmp/masbr` ### db2_backup_version -The backup version timestamp to restore from. This is automatically generated during backup in the format `YYMMDD-HHMMSS`. +The backup version timestamp identifier for backup and restore operations. -- **Required** for restore operations +- **Required** for `restore` and `restore_database` actions +- **Auto-generated** for backup operations - Environment Variable: `DB2_BACKUP_VERSION` -- Default: Auto-generated for backup operations -- Example: `251212-021316` +- Default: Auto-generated in format `YYMMDD-HHMMSS` + +**Purpose**: Uniquely identifies a specific backup version using a timestamp. This allows multiple backups to coexist and enables point-in-time restore operations. + +**When to use**: +- Automatically generated during backup (no need to set manually) +- Must be specified when restoring to identify which backup to use +- Must be specified when installing Db2 from an existing backup + +**Valid values**: Timestamp string in format `YYMMDD-HHMMSS` (e.g., `251212-021316` for December 12, 2025 at 02:13:16) + +**Impact**: +- Determines the backup directory name: `backup--db2u` +- Used to locate backup files during restore operations +- Recorded in backup metadata file for verification + +**Related variables**: +- `mas_backup_dir`: Parent directory containing versioned backups +- `db2_action`: Required when action is `restore_database` or `restore`(instance & database) + +**Example**: `251212-021316` + +### override_storageclass +Controls whether to override storage classes during Db2 installation from backup. +Only used in Db2 instance restore. + +- **Optional** +- Environment Variable: `OVERRIDE_STORAGECLASS` +- Default: `false` + +**Purpose**: Allows changing storage classes when restoring Db2 to a different cluster or when the original storage classes are not available. When enabled, uses specified storage class variables or cluster defaults instead of backup metadata values. + +**When to use**: +- Set to `true` when restoring to a cluster with different storage classes +- Set to `true` when original storage classes are not available in target cluster +- Leave as `false` to use the same storage classes as the original instance + +**Valid values**: `true`, `false` + +**Impact**: +- When `true`: Uses `DB2_META_STORAGE_CLASS`, `DB2_DATA_STORAGE_CLASS`, `DB2_BACKUP_STORAGE_CLASS`, `DB2_LOGS_STORAGE_CLASS`, `DB2_TEMP_STORAGE_CLASS` if set, otherwise uses cluster default storage classes +- When `false`: Uses storage classes from backup metadata (original instance configuration) + +**Related variables**: +- `db2_meta_storage_class`: Override for metadata storage +- `db2_data_storage_class`: Override for data storage +- `db2_backup_storage_class`: Override for backup storage +- `db2_logs_storage_class`: Override for logs storage +- `db2_temp_storage_class`: Override for temp storage ### backup_type -Type of backup to perform. Online backups keep the database available during backup, while offline backups require database downtime but are faster. -If your DB2 instance has got circular logging enabled i.e `LOGARCHMETH1: OFF or/and LOGARCHMETH2: OFF`, you can only use `offline` backup type. -If your DB2 instance has got circular logging disabled, you can use either `online` or `offline` backup type. -If you are unsure, you can use default `online` backup type. +Type of backup operation to perform on the Db2 database. -- Optional +- **Optional** - Environment Variable: `DB2_BACKUP_TYPE` - Default: `online` -- Supported values: `online`, `offline` + +**Purpose**: Determines whether the database remains available during backup. Online backups allow continued database access but may impact performance, while offline backups require downtime but complete faster. + +**When to use**: +- Use `online` (default) for production systems requiring high availability +- Use `offline` when downtime is acceptable and faster backup is desired +- **Must use `offline`** if circular logging is enabled (`LOGARCHMETH1: OFF` and/or `LOGARCHMETH2: OFF`) + +**Valid values**: `online`, `offline` + +**Impact**: +- `online`: Database remains accessible during backup; requires archive logging enabled; may impact performance +- `offline`: Database is unavailable during backup; faster completion; works with circular logging + +**Related variables**: +- `db2_database_db_config`: Check `LOGARCHMETH1` and `LOGARCHMETH2` settings to determine if online backup is supported + +**Important**: If your Db2 instance has circular logging enabled (default configuration), you can only use `offline` backup type. If archive logging is enabled, you can use either type. ### backup_vendor -Storage vendor for backup files. Use `disk` for local storage or `s3` for S3-compatible object storage. -*Note* : Only database backup is stored in S3, instance backup is always stored in local disk. +Storage backend for database backup files only. -- Optional +- **Optional** - Environment Variable: `BACKUP_VENDOR` - Default: `disk` -- Supported values: `disk`, `s3` -### br_skip_instance -When set to `false`, includes Db2 instance resources (secrets, certificates, custom resources) in the backup. +**Purpose**: Determines where database backup files are stored. Disk storage keeps backups locally, while S3 storage sends them directly to S3-compatible object storage. -- Optional -- Environment Variable: `BR_SKIP_INSTANCE` -- Default: `true` +**When to use**: +- Use `disk` (default) for local backups or when S3 is not available +- Use `s3` for cloud-based backups, long-term retention, or disaster recovery scenarios + +**Valid values**: `disk`, `s3` + +**Impact**: +- `disk`: Database Backup files stored locally and copied to `mas_backup_dir`; requires sufficient local storage +- `s3`: Database backup sent directly to S3 bucket; instance resources still stored locally; requires S3 credentials + +**Related variables**: +- When `s3`: Requires `backup_s3_endpoint`, `backup_s3_bucket`, `backup_s3_access_key`, `backup_s3_secret_key` +- `mas_backup_dir`: Always required for metadata and instance resources + +**Note**: Instance resources (secrets, certificates, CRs) are always stored locally in `mas_backup_dir`, regardless of vendor setting. Only database backup files go to S3. + +**Purpose**: Determines if Kubernetes resources (secrets, certificates, Db2uCluster CR, etc.) are backed up along with the database. When `false`, enables full disaster recovery by backing up both instance configuration and data. + +**When to use**: +- Set to `false` when you need complete disaster recovery capability (instance + database) +- Set to `false` when migrating Db2 to a new cluster +- Leave as `true` (default) for database-only backups when instance already exists + +**Valid values**: `true`, `false` + +**Impact**: +- `true`: Only database data is backed up; faster backup; requires existing Db2 instance for restore +- `false`: Both instance resources and database are backed up; enables full recovery; allows install from backup + +**Note**: Instance resources include: Db2uCluster CR, secrets (passwords, certificates), ConfigMaps, and other Kubernetes resources needed to recreate the Db2 instance. ### backup_s3_endpoint -S3 endpoint URL for S3-compatible object storage. +S3-compatible object storage endpoint URL. - **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_ENDPOINT` - Default: None -- Example: `https://s3.us-east.cloud-object-storage.appdomain.cloud` + +**Purpose**: Specifies the S3 API endpoint for storing database backups. Supports AWS S3, IBM Cloud Object Storage, MinIO, and other S3-compatible services. + +**When to use**: +- Required when using S3 storage for backups (`backup_vendor: s3`) +- Must be accessible from the Db2 pod + +**Valid values**: HTTPS URL to S3-compatible endpoint (e.g., `https://s3.us-east.cloud-object-storage.appdomain.cloud`, `https://s3.amazonaws.com`) + +**Impact**: Db2 connects to this endpoint to upload/download backup files. Incorrect endpoint will cause backup/restore failures. + +**Related variables**: +- `backup_vendor`: Must be set to `s3` +- `backup_s3_bucket`: Bucket at this endpoint +- `backup_s3_access_key`, `backup_s3_secret_key`: Credentials for this endpoint + +**Example**: `https://s3.us-east.cloud-object-storage.appdomain.cloud` ### backup_s3_bucket -S3 bucket name where backups will be stored. +S3 bucket name for storing database backups. - **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_BUCKET` - Default: None -- Example: `mas-db2-backups` + +**Purpose**: Specifies the S3 bucket where database backup files will be stored. The bucket must exist and credentials must have read/write permissions. + +**When to use**: +- Required when using S3 storage for backups (`backup_vendor: s3`) +- Bucket must be created before running backup + +**Valid values**: Valid S3 bucket name following S3 naming conventions + +**Impact**: Backup files are stored in this bucket under path `/`. Incorrect bucket name or insufficient permissions will cause failures. + +**Related variables**: +- `backup_vendor`: Must be set to `s3` +- `backup_s3_endpoint`: S3 service hosting this bucket +- `backup_s3_access_key`, `backup_s3_secret_key`: Must have permissions for this bucket + +**Example**: `mas-db2-backups` ### backup_s3_access_key -S3 access key for authentication. +S3 access key ID for authentication. - **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_ACCESS_KEY` - Default: None +**Purpose**: Provides the access key ID for authenticating to S3-compatible object storage. Used together with secret key for S3 API authentication. + +**When to use**: +- Required when using S3 storage for backups (`backup_vendor: s3`) +- Must have read/write permissions to the specified bucket + +**Valid values**: Valid S3 access key ID from your S3 provider + +**Impact**: Used for S3 authentication. Invalid credentials will cause backup/restore to fail with authentication errors. + +**Related variables**: +- `backup_vendor`: Must be set to `s3` +- `backup_s3_secret_key`: Corresponding secret key +- `backup_s3_bucket`: Bucket these credentials can access + +**Security**: Store securely using Ansible Vault or environment variables. Never commit to version control. + ### backup_s3_secret_key -S3 secret key for authentication. +S3 secret access key for authentication. - **Required** when `backup_vendor` is `s3` - Environment Variable: `BACKUP_S3_SECRET_KEY` - Default: None +**Purpose**: Provides the secret access key for authenticating to S3-compatible object storage. Used together with access key for S3 API authentication. + +**When to use**: +- Required when using S3 storage for backups (`backup_vendor: s3`) +- Must correspond to the access key ID + +**Valid values**: Valid S3 secret access key from your S3 provider + +**Impact**: Used for S3 authentication. Invalid credentials will cause backup/restore to fail with authentication errors. + +**Related variables**: +- `backup_vendor`: Must be set to `s3` +- `backup_s3_access_key`: Corresponding access key ID +- `backup_s3_bucket`: Bucket these credentials can access + +**Security**: Store securely using Ansible Vault or environment variables. Never commit to version control. + ### backup_s3_alias -S3 alias name used in Db2 configuration. +Db2 storage access alias name for S3 configuration. -- Optional +- **Optional** - Environment Variable: `BACKUP_S3_ALIAS` - Default: `S3DB2COS` +**Purpose**: Defines the alias name used in Db2's storage access configuration for S3. This is an internal Db2 identifier for the S3 connection. + +**When to use**: +- Usually leave as default unless you have specific Db2 storage access naming requirements +- Change only if you need to match existing Db2 storage access configurations + +**Valid values**: Valid Db2 storage access alias name (alphanumeric, no spaces) + +**Impact**: Used internally by Db2 to reference the S3 storage configuration. Changing this is rarely necessary. + +**Related variables**: +- `backup_vendor`: Only used when set to `s3` + +**Default**: `S3DB2COS` + Example Usage - Backup and Restore ------------------------------------------------------------------------------- @@ -948,7 +1138,7 @@ Example Usage - Backup and Restore vars: mas_instance_id: masinst1 mas_backup_dir: /tmp/masbr - db2_action: backup + db2_action: backup_database db2_instance_name: db2u-manage db2_namespace: db2u backup_type: online @@ -964,7 +1154,7 @@ Example Usage - Backup and Restore vars: mas_instance_id: masinst1 mas_backup_dir: /tmp/masbr - db2_action: backup + db2_action: backup_database db2_instance_name: db2u-manage backup_type: online backup_vendor: s3 @@ -985,7 +1175,7 @@ Example Usage - Backup and Restore mas_backup_dir: /tmp/masbr db2_action: backup db2_instance_name: db2u-manage - br_skip_instance: false # Include instance resources + db2_namespace: db2u backup_vendor: disk roles: - ibm.mas_devops.db2 @@ -1026,26 +1216,50 @@ Example Usage - Backup and Restore - ibm.mas_devops.db2 ``` -### Install Db2 from Backup (Instance + Database) +### Restore Db2 from Backup (Instance + Database) +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + db2_action: restore + mas_instance_id: masinst1 + db2_backup_version: 251212-021316 + mas_backup_dir: /tmp/masbr + backup_vendor: disk + roles: + - ibm.mas_devops.db2 +``` + +### Restore Db2 from Backup (Instance + Database) w/ storage class override +# This will override the storage class for all Db2 PVCs +# If you want to override specific PVCs, use the following variables: +# db2_meta_storage_class, db2_data_storage_class, db2_backup_storage_class, db2_logs_storage_class, db2_temp_storage_class +# or cluster's default storage class will be used to override. ```yaml - hosts: localhost any_errors_fatal: true vars: - db2_action: install + db2_action: restore mas_instance_id: masinst1 db2_backup_version: 251212-021316 mas_backup_dir: /tmp/masbr backup_vendor: disk + override_storageclass: true + db2_meta_storage_class: nfs-client # optional + db2_data_storage_class: nfs-client # optional + db2_backup_storage_class: nfs-client # optional + db2_logs_storage_class: nfs-client # optional + db2_temp_storage_class: nfs-client # optional roles: - ibm.mas_devops.db2 ``` -### Install Db2 from Backup (Instance + Database(S3)) +### Restore Db2 from Backup (Instance + Database(S3)) ```yaml - hosts: localhost any_errors_fatal: true vars: - db2_action: install + db2_action: restore mas_instance_id: masinst1 db2_backup_version: 251212-021316 mas_backup_dir: /tmp/masbr @@ -1058,34 +1272,6 @@ Example Usage - Backup and Restore - ibm.mas_devops.db2 ``` -Backup and Restore Details -------------------------------------------------------------------------------- - -### Backup Process -1. Validates Db2 instance is running -2. Prepares backup scripts and copies them to the Db2 pod -3. Configures S3 storage access (if using S3) -4. Executes Db2 backup command (online or offline) -5. Compresses and transfers backup files (for disk storage) -6. Creates metadata file (`db2-backup-info.yaml`) with backup details - -### Database Restore Process -1. Validates backup files and Db2 version compatibility -2. Prepares restore scripts and copies them to the Db2 pod -3. Configures S3 storage access (if restoring from S3) -4. Copies backup files to the Db2 pod (for disk restores) -5. Executes Db2 restore command -6. Verifies restore completion - -### Install From Backup Process -1. Validates backup files -2. Creates namespace and copies resources to namespace -3. Gets Db2 instance details from backup metadata -4. Installs Db2 instance using the backup details -5. Waits for Db2 instance to be ready -6. Performs post deployment actions like restoring instance password -7. Performs Database restore process as mentioned above - ### Backup Directory Structure (Disk) ``` /tmp/masbr/ @@ -1094,10 +1280,11 @@ Backup and Restore Details │ ├── db2-BLUDB-backup-.tar.gz │ └── db2-backup-info.yaml └── resources/ - ├── cr.yaml + ├── db2uclusters/ ├── secrets/ ├── certificates/ └── issuers/ + └── {kind}s/ ``` ### Database backup Metadata (db2-backup-info.yaml) diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index f2a2511dc8..81eb210e4f 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -1,7 +1,7 @@ --- # db2 action # --------------------------------------------------------------------------- -# Supported actions: install, uninstall, backup, restore_database +# Supported actions: install, uninstall, backup, backup_database, restore, restore_database db2_action: "{{ lookup('env', 'DB2_ACTION') | default('install', true) }}" # Configure Db2 instance @@ -129,7 +129,10 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # ----------------------------------------------------------------------------- mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" -br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(true, true) }}" # set flag to false, to backup db2 instance + +# Set flag to true, to use cluster's default storage classes +override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default(false, true) }}" + backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) }}" # Supported values are s3 and disk backup_s3_alias: "{{ lookup('env', 'BACKUP_S3_ALIAS') | default('S3DB2COS', true) }}" backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" @@ -138,3 +141,9 @@ backup_s3_access_key: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY') }}" backup_s3_secret_key: "{{ lookup('env', 'BACKUP_S3_SECRET_KEY') }}" backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline +_db2_storages: "NO_OVERRIDE" +_db2_instance_password: "NO_OVERRIDE" +_db2_ldapblueadmin_password: "NO_OVERRIDE" +_db2_ldappassword: "NO_OVERRIDE" + + diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml index 7c6d778ab9..03bcec5367 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml @@ -4,6 +4,10 @@ db2_backup_resources: - namespace: "{{ db2_namespace }}" resources: + # Namespace + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ db2_namespace }}" # db2u.databases.ibm.com - kind: Db2uCluster api_version: db2u.databases.ibm.com/v1 @@ -26,6 +30,15 @@ - kind: Secret api_version: v1 name: "c-{{ db2_instance_name }}-instancepassword" + - kind: Secret + api_version: v1 + name: "c-{{ db2_instance_name }}-ldapblueadminpassword" + - kind: Secret + api_version: v1 + name: "c-{{ db2_instance_name }}-ldappassword" + - kind: Secret + api_version: v1 + name: "db2u-certificate-{{ db2_instance_name }}" # Issuers - kind: Issuer api_version: cert-manager.io/v1 @@ -40,6 +53,10 @@ - kind: Certificate api_version: cert-manager.io/v1 name: db2u-ca-certificate + # Route + - kind: Route + api_version: route.openshift.io/v1 + name: "db2u-{{ db2_instance_name }}-tls-route" - namespace: "mas-{{ mas_instance_id }}-core" resources: # secrets diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 943779b10c..3504676756 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -31,9 +31,8 @@ # ------------------------------------------------------------------------- - name: "Start Db2 Universal operator Instance backup process." include_tasks: "{{ role_path }}/tasks/backup/backup-instance.yml" - when: not br_skip_instance | bool # Backup Db2 database Data using Db2Backup CR # ------------------------------------------------------------------------- - name: "Start Database backup process." - include_tasks: "{{ role_path }}/tasks/backup/backup-database.yml" + include_tasks: "{{ role_path }}/tasks/backup_database/backup-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml similarity index 100% rename from ibm/mas_devops/roles/db2/tasks/backup/backup-database.yml rename to ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml new file mode 100644 index 0000000000..80000c5d5c --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml @@ -0,0 +1,33 @@ +--- +# Check db2 backup required variables +# ----------------------------------------------------------------------------- +- name: Verify DB2 backup variables + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "{{ db2_action }}" + db2_instance_name: "{{ db2_instance_name }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mas_instance_id: "{{ mas_instance_id }}" + db2_namespace: "{{ db2_namespace }}" + +# Set DB2 backup version if not provided +# ----------------------------------------------------------------------------- +- name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + db2_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: db2_backup_version is not defined or db2_backup_version == "" or db2_backup_version == "None" + +- name: "Set fact: DB2 backup base directory path" + set_fact: + db2_backup_path: "{{ mas_backup_dir }}/{{ db2_action }}-{{ db2_backup_version }}-db2u" + +- name: "Create {{ db2_backup_path }} directory for Db2 backup" + file: + path: "{{ db2_backup_path }}" + state: directory + mode: "0755" + +# Backup Db2 database Data using Db2Backup CR +# ------------------------------------------------------------------------- +- name: "Start Database backup process." + include_tasks: "tasks/backup_database/backup-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/install/install.yml b/ibm/mas_devops/roles/db2/tasks/install/install.yml deleted file mode 100644 index 0be39b86e9..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/install/install.yml +++ /dev/null @@ -1,467 +0,0 @@ ---- -- name: "Fail if required db2_dbname is over 8 characters" - assert: - that: - - db2_dbname is defined and db2_dbname != "" - - db2_dbname | length <= 8 - fail_msg: "Property value of db2_dbname is set to '{{ db2_dbname }}' and is greater than 8 character long." - -# 2. Load default storage classes (if not provided by the user) -# ----------------------------------------------------------------------------- -- include_tasks: tasks/install/determine-storage-classes.yml - -# 3. Setup the norootsquash daemonsets for db2u pods to work with NFS backed storage -# ----------------------------------------------------------------------------- -- include_tasks: tasks/install/setup_norootsquash.yml - when: - - db2_meta_storage_class is defined - - db2_data_storage_class is defined - - '"ibmc-file" in db2_data_storage_class or "ibmc-file" in db2_meta_storage_class or "ibmc-vpc-file" in db2_data_storage_class or "ibmc-vpc-file" in db2_meta_storage_class' - -# 4. Provide debug information to the user -# ----------------------------------------------------------------------------- -- name: "Debug - MAS" - debug: - msg: - - "MAS Instance ID ........................ {{ mas_instance_id }}" - - "MAS Config directory ................... {{ mas_config_dir }}" - - "MAS Instance ID ........................ {{ mas_instance_id | default('', true) }}" - - "MAS Workspace ID ....................... {{ mas_workspace_id | default('', true) }}" - - "MAS Application ID ..................... {{ mas_application_id | default('', true) }}" - - "MAS Config Directory ................... {{ mas_config_dir | default('', true) }}" - - "MAS Config Scope ....................... {{ mas_config_scope | default('', true) }}" - -- name: "Debug - Affinity & Tolerations" - debug: - msg: - - "Affinity key ........................... {{ db2_affinity_key | default('', true) }}" - - "Affinity value ......................... {{ db2_affinity_value | default('', true) }}" - - - "Toleration key ......................... {{ db2_tolerate_key | default('', true) }}" - - "Toleration value ....................... {{ db2_tolerate_value | default('', true) }}" - - "Toleration effect ...................... {{ db2_tolerate_effect | default('', true) }}" - -- name: "Debug - Db2 Instance" - debug: - msg: - - "Namespace .............................. {{ db2_namespace }}" - - "Db2 Instance ........................... {{ db2_instance_name }}" - -- name: "Debug - Database Settings" - debug: - msg: - - "Database Name .......................... {{ db2_dbname }}" - - "4K Device Support ...................... {{ db2_4k_device_support }}" - - "Table Organization ..................... {{ db2_table_org }}" - - "TLS Version ............................ {{ tls_version }}" - - "Workload ............................... {{ db2_workload }}" - -- name: "Debug - Resources" - debug: - msg: - - "CPU Request ............................ {{ db2_cpu_requests }}" - - "CPU Limit .............................. {{ db2_cpu_limits }}" - - "Memory Request ......................... {{ db2_memory_requests }}" - - "Memory Limit ........................... {{ db2_memory_limits }}" - -- name: "Debug - Storage" - debug: - msg: - - "Meta ................................... {{ db2_meta_storage_class }} - {{ db2_meta_storage_size }} @ {{ db2_meta_storage_accessmode }}" - - "Data ................................... {{ db2_data_storage_class }} - {{ db2_data_storage_size }} @ {{ db2_data_storage_accessmode }}" - - "Backup ................................. {{ db2_backup_storage_class }} - {{ db2_backup_storage_size }} @ {{ db2_backup_storage_accessmode }}" - - "Logs ................................... {{ db2_logs_storage_class }} - {{ db2_logs_storage_size }} @ {{ db2_logs_storage_accessmode }}" - - "Temp ................................... {{ db2_temp_storage_class }} - {{ db2_temp_storage_size }} @ {{ db2_temp_storage_accessmode }}" - -# Lookup db2 operator group -- name: "Check if operator group is present in {{ db2_namespace }} namespace already" - kubernetes.core.k8s_info: - namespace: "{{ db2_namespace }}" - kind: OperatorGroup - register: db2_og_info - -# Look up the default channel for the db2u-operator package manifest -- name: Lookup db2u-operator packagemanifest - kubernetes.core.k8s_info: - api_version: packages.operators.coreos.com/v1 - kind: PackageManifest - name: db2u-operator - namespace: "{{ ibm_common_services_namespace }}" - register: db2u_manifest_info - until: db2u_manifest_info.resources[0].status.defaultChannel is defined - retries: 60 # Approximately 30 minutes before we give up - delay: 30 # seconds - when: db2_channel is not defined or db2_channel == "" or db2_version is not defined or db2_version == "" - -- name: Set db2u-operator channel - ansible.builtin.set_fact: - db2_channel: "{{ db2u_manifest_info.resources[0].status.defaultChannel }}" - when: db2_channel is not defined or db2_channel == "" - -# 5. Fail if required parameters are not set -# ----------------------------------------------------------------------------- -- name: "Verify db2_channel is set" - assert: - that: - - db2_channel is defined and db2_channel != "" - fail_msg: "Unable to determine db2_channel from catalog" - -- name: Debug DB2 upgrade channel - ansible.builtin.debug: - msg: - - "Db2 Channel ............................ {{ db2_channel }}" - -# 6. Install a Db2u Operator -# ----------------------------------------------------------------------------- -- name: "Create db2u Namespace" - kubernetes.core.k8s: - apply: yes - template: "templates/db2u_namespace.yaml" - register: _db2_namespace_result - -- name: Check if ibm-registry secret exists - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: ibm-registry - namespace: "{{ db2_namespace }}" - register: ibm_registry_secret_info - -- name: "Create ibm-registry secret if not present" - when: - - ibm_registry_secret_info.resources is defined - - ibm_registry_secret_info.resources | length == 0 - block: - - name: Set 'ibm-registry' secret content - no_log: true - vars: - entitledAuthStr: "{{ registry_user }}:{{ ibm_entitlement_key }}" - entitledAuth: "{{ entitledAuthStr | b64encode }}" - content: - - '{"auths":{"{{ registry }}/cp/cpd":{"username":"{{ registry_user }}","password":"{{ ibm_entitlement_key }}","email":"{{ registry_user }}","auth":"{{ entitledAuth }}"}' - - "}" - - "}" - set_fact: - new_secret: "{{ content | join('') }}" - - # Note: We use "new_secret | to_json " because older versions of Ansible create - # invalid json representations containing single quotes - # However, in newer versions of Ansible to_json returns an object, which can't be passed to b64encode - - name: "Generate docker secret (old Ansible versions)" - when: ansible_version.full is version_compare(2.20, '<') - set_fact: - dockerconfigjsonB64: "{{ new_secret | to_json | b64encode }}" - - - name: "Generate docker secret (new Ansible versions)" - when: ansible_version.full is version_compare(2.20, '>=') - set_fact: - dockerconfigjsonB64: "{{ new_secret | b64encode }}" - - - name: "Generate 'ibm-registry' secret" - no_log: true - kubernetes.core.k8s: - definition: - apiVersion: v1 - kind: Secret - type: kubernetes.io/dockerconfigjson - metadata: - name: ibm-registry - namespace: "{{ db2_namespace }}" - data: - .dockerconfigjson: "{{ dockerconfigjsonB64 }}" - register: secretUpdateResult - -- name: "Delete old db2 subscription, operand request and csv from {{ ibm_common_services_namespace }}" - include_tasks: "tasks/delete_db2_operand_request.yml" - -- name: "Create Db2 Universal Operator Subscription in {{ db2_namespace }} namespace" - ibm.mas_devops.apply_subscription: - namespace: "{{ db2_namespace }}" - package_name: db2u-operator - package_channel: "{{ db2_channel }}" - register: subscription - -# 7. Get the cluster subdomain to be used for the certificate and route creation -# ----------------------------------------------------------------------------- -- name: "Get cluster subdomain" - kubernetes.core.k8s_info: - api_version: config.openshift.io/v1 - kind: Ingress - name: cluster - register: _cluster_subdomain - -# 8. Create self-signed certificate for Db2u SSL -# ----------------------------------------------------------------------------- -- name: "Create internal CA certificate issuer" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/ca_issuer.yml.j2" - register: createCaIssuer - -- name: "Create and wait for CA certificate" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/ca_certificate.yml.j2" - wait: yes - wait_timeout: 600 #10 minutes - wait_condition: - type: Ready - status: True - register: createCaCert - -- name: "Create certificate issuer" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/issuer.yml.j2" - register: createIssuer - -- name: "Create db2u certificate" - kubernetes.core.k8s: - apply: yes - template: "templates/certs/certificate.yml.j2" - register: createCertificate - -# 9. Wait until the Db2uCluster CRD is available -# ----------------------------------------------------------------------------- -- name: "Wait until the Db2uCluster CRD is available" - include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" - vars: - crd_name: "db2uclusters.db2u.databases.ibm.com" - -# 10. Get information from the db2u-release ConfigMap -# ----------------------------------------------------------------------------- -# if db2_version is not set, then we define it based on the latest version supported by the db2u-license-keys secret -# Starting with s11.5.8.0-cn3, the 's' prefix is removed in db2u-license-keys, it is recommended to use db2u-release configmap. -- block: - - name: "Wait until the db2u-release configmap is available" - no_log: true - kubernetes.core.k8s_info: - api_version: v1 - name: db2u-release - namespace: "{{ db2_namespace }}" - kind: ConfigMap - register: db2_release_info - retries: 20 # ~approx 10 minutes before we give up waiting for the CRD to be created - delay: 30 # seconds - until: - - db2_release_info.resources is defined - - db2_release_info.resources | length > 0 - - db2_release_info.resources[0].data is defined - - db2_release_info.resources[0].data | length > 0 - - - name: Set db2u-release configmap content - no_log: true - set_fact: - db2_releases_content: "{{ db2_release_info.resources[0].data.json | from_json }}" - - - name: Extract major version from channel - set_fact: - db2_major_version: "{{ db2_channel | regex_replace('^v(\\d{2})(\\d+).*', '\\1') }}" - - - name: Debug extracted version - ansible.builtin.debug: - msg: - - "DB2 Channel ........................... {{ db2_channel }}" - - "Extracted Major Version ............... {{ db2_major_version }}" - - "Filter Pattern ........................ ^s{{ db2_major_version }}" - - - name: Determine DB2 version based on channel - set_fact: - db2_version: >- - {{ - (db2_releases_content.databases.db2u.keys() - | select('match', '^s' + db2_major_version) - | sort) - | last - }} - when: - - (db2_releases_content.databases.db2u.keys() | select('match', '^s' + db2_major_version) | list | length) > 0 - - when: db2_version is not defined or db2_version == "" - -- name: Debug DB2 channel and determined version - ansible.builtin.debug: - msg: - - "DB2 Channel ........................... {{ db2_channel }}" - - "DB2 Version (determined) .............. {{ db2_version }}" - -- name: Debug available DB2 versions - ansible.builtin.debug: - msg: - - "Available DB2 Versions ................ {{ db2_releases_content.databases.db2u.keys() | list }}" - when: db2_releases_content is defined - -# 11. Fail if required parameters are not set -# ----------------------------------------------------------------------------- -- name: "Verify db2_channel and db2_version set" - assert: - that: - - db2_channel is defined and db2_channel != "" - - db2_version is defined and db2_version != "" - fail_msg: "Unable to determine db2_channel and/or db2_version" - -- name: Debug DB2 upgrade channel and version - ansible.builtin.debug: - msg: - - "Db2 Channel ............................ {{ db2_channel }}" - - "Db2 Version ............................ {{ db2_version }}" - -# 12. Lookup db2 instance to see if it exists already -# ----------------------------------------------------------------------------- -- name: "See if db2u instance already exists" - kubernetes.core.k8s_info: - api_version: db2u.databases.ibm.com/v1 - name: "{{ db2_instance_name | lower }}" - namespace: "{{db2_namespace}}" - kind: Db2uCluster - register: initial_db2_cluster_lookup - -# 13. Create a Db2 instance -# ----------------------------------------------------------------------------- -- name: "Create db2 instance" - kubernetes.core.k8s: - apply: yes - template: "templates/db2ucluster.yml.j2" - -- name: "Set db2 instance timezone" - include_tasks: "tasks/install/setup_timezone.yml" - when: - - db2_timezone is defined - - db2_timezone != "" - -# 14. Wait for the cluster to be ready -# ----------------------------------------------------------------------------- -- name: "Wait for db2u instance to be ready (5m delay)" - kubernetes.core.k8s_info: - api_version: db2u.databases.ibm.com/v1 - name: "{{ db2_instance_name | lower }}" - namespace: "{{db2_namespace}}" - kind: Db2uCluster - register: db2_cluster_lookup - until: - - db2_cluster_lookup.resources is defined - - db2_cluster_lookup.resources | length == 1 - - db2_cluster_lookup.resources[0].status is defined - - db2_cluster_lookup.resources[0].status.state is defined - - db2_cluster_lookup.resources[0].status.state == "Ready" - retries: 24 # Approximately 2 hours before we give up - delay: 300 # 5 minutes - -# 15. Configure a public route for Db2 -# ----------------------------------------------------------------------------- -- name: Lookup db2u Engn Service - kubernetes.core.k8s_info: - api_version: v1 - kind: Service - name: "c-{{db2_instance_name | lower}}-db2u-engn-svc" - namespace: "{{db2_namespace}}" - register: _db2_instance_engn_svc - until: - - _db2_instance_engn_svc.resources[0] is defined - retries: 15 # approx 5 minutes before we give up - delay: 20 # seconds - -- name: Lookup db2u TLS certificates - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: "db2u-certificate-{{db2_instance_name}}" - namespace: "{{db2_namespace}}" - register: _db2_instance_certificates - -- name: Set Db2u certificates as Facts - set_fact: - db2_ca_pem: "{{ _db2_instance_certificates.resources[0].data['ca.crt'] | b64decode }}" - db2_tls_crt: "{{ _db2_instance_certificates.resources[0].data['tls.crt'] | b64decode }}" - db2_tls_key: "{{ _db2_instance_certificates.resources[0].data['tls.key'] | b64decode }}" - when: - - _db2_instance_certificates is defined - - (_db2_instance_certificates.resources | length > 0) - -- name: Set Db2u TLS port - set_fact: - db2_tls_serviceport: "{{item.targetPort}}" - when: "item.name == 'ssl-server'" - loop: "{{_db2_instance_engn_svc.resources[0].spec.ports}}" - -- name: "Create dedicated route: db2u-{{ db2_instance_name }}-tls-route" - kubernetes.core.k8s: - apply: yes - template: "templates/tlsroute.yml.j2" - -- name: Lookup existing db2u-tls-route - kubernetes.core.k8s_info: - api_version: v1 - kind: Route - name: "db2u-tls-route" - namespace: "{{db2_namespace}}" - register: _db2_tls_route - -# delete existing db2u-tls-route if that exists and matches with the same host/location from route created by previous step. -# that way we ensure that clean up should only happen if there's a conflicting route present. -- name: Clean up db2u-tls-route if needed - vars: - expected_host: "{{db2_instance_name | lower }}-{{db2_namespace}}.{{_cluster_subdomain.resources[0].spec.domain}}" - when: - - _db2_tls_route.resources | length > 0 - - _db2_tls_route.resources[0].spec.host == expected_host - kubernetes.core.k8s: - api_version: v1 - kind: Route - name: "db2u-tls-route" - namespace: "{{db2_namespace}}" - state: absent - - -## 16. create an LDAP user if db2_ldap_username specified -# ----------------------------------------------------------------------------- -- name: Create LDAP user if username and password is provided - include_tasks: tasks/install/create_ldap_user.yml - when: - - db2_ldap_username is defined - - db2_ldap_username != "" - - db2_ldap_password is defined - - db2_ldap_password != "" - - db2_rotate_password == false - -- debug: - msg: - - "{{db2_ldap_username}}" - - "{{db2_rotate_password}}" - -# 17. Rotate db2 ldap password -# ----------------------------------------------------------------------------- -- name: Rotate Db2 LDAP password if db2_rotate_password is True and username is provided - include_tasks: tasks/install/rotate_ldap_user_password.yml - when: - - db2_ldap_username is defined - - db2_ldap_username != "" - - db2_rotate_password == true - -# 18. Wait for the statefulset to be ready -# ----------------------------------------------------------------------------- -- name: "Wait for Db2 Stateful set to be ready" - kubernetes.core.k8s_info: - api_version: apps/v1 - kind: StatefulSet - name: "c-{{ db2_instance_name | lower }}-db2u" - namespace: "{{ db2_namespace }}" - register: db2_sts - until: - - db2_sts.resources is defined - - db2_sts.resources | length > 0 - - db2_sts.resources[0].status is defined - - db2_sts.resources[0].status.replicas is defined - - db2_sts.resources[0].status.readyReplicas is defined - - db2_sts.resources[0].status.readyReplicas == db2_sts.resources[0].status.replicas - retries: 20 # approx 10 minutes before we give up - delay: 30 # seconds - -# 19. Generate a JdbcCfg for MAS configuration -# ----------------------------------------------------------------------------- -- include_tasks: tasks/install/suite_jdbccfg.yml - when: - - mas_instance_id is defined - - mas_instance_id != "" - - mas_config_dir is defined - - mas_config_dir != "" diff --git a/ibm/mas_devops/roles/db2/tasks/install/main.yml b/ibm/mas_devops/roles/db2/tasks/install/main.yml index 14bcb60364..6b8a4e91b5 100644 --- a/ibm/mas_devops/roles/db2/tasks/install/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/install/main.yml @@ -1,14 +1,5 @@ --- -# Detect if this is an install-from-backup scenario -- name: "Check for backup restore variables" - set_fact: - install_from_backup: "{{ (db2_backup_version is defined and db2_backup_version != '' and db2_backup_version != 'None') and (mas_backup_dir is defined and mas_backup_dir != '') }}" - -- include_tasks: tasks/install/get-backup-info.yml - when: - - install_from_backup | bool - -# Fail if required parameters are not set +# 1. Fail if required parameters are not set # ----------------------------------------------------------------------------- - name: "Fail if required properties have not been provided" assert: @@ -16,13 +7,499 @@ - db2_instance_name is defined and db2_instance_name != "" - ibm_entitlement_key is defined and ibm_entitlement_key != "" fail_msg: "One or more required properties have not been set" - when: not install_from_backup | bool -# Start the installation task +- name: "Fail if required db2_dbname is over 8 characters" + assert: + that: + - db2_dbname is defined and db2_dbname != "" + - db2_dbname | length <= 8 + fail_msg: "Property value of db2_dbname is set to '{{ db2_dbname }}' and is greater than 8 character long." + +# 2. Load default storage classes (if not provided by the user) +# ----------------------------------------------------------------------------- +- include_tasks: tasks/install/determine-storage-classes.yml + +# 3. Setup the norootsquash daemonsets for db2u pods to work with NFS backed storage +# ----------------------------------------------------------------------------- +- include_tasks: tasks/install/setup_norootsquash.yml + when: + - db2_meta_storage_class is defined + - db2_data_storage_class is defined + - '"ibmc-file" in db2_data_storage_class or "ibmc-file" in db2_meta_storage_class or "ibmc-vpc-file" in db2_data_storage_class or "ibmc-vpc-file" in db2_meta_storage_class' + +# 4. Provide debug information to the user +# ----------------------------------------------------------------------------- +- name: "Debug - MAS" + debug: + msg: + - "MAS Instance ID ........................ {{ mas_instance_id }}" + - "MAS Config directory ................... {{ mas_config_dir }}" + - "MAS Instance ID ........................ {{ mas_instance_id | default('', true) }}" + - "MAS Workspace ID ....................... {{ mas_workspace_id | default('', true) }}" + - "MAS Application ID ..................... {{ mas_application_id | default('', true) }}" + - "MAS Config Directory ................... {{ mas_config_dir | default('', true) }}" + - "MAS Config Scope ....................... {{ mas_config_scope | default('', true) }}" + +- name: "Debug - Affinity & Tolerations" + debug: + msg: + - "Affinity key ........................... {{ db2_affinity_key | default('', true) }}" + - "Affinity value ......................... {{ db2_affinity_value | default('', true) }}" + + - "Toleration key ......................... {{ db2_tolerate_key | default('', true) }}" + - "Toleration value ....................... {{ db2_tolerate_value | default('', true) }}" + - "Toleration effect ...................... {{ db2_tolerate_effect | default('', true) }}" + +- name: "Debug - Db2 Instance" + debug: + msg: + - "Namespace .............................. {{ db2_namespace }}" + - "Db2 Instance ........................... {{ db2_instance_name }}" + +- name: "Debug - Database Settings" + debug: + msg: + - "Database Name .......................... {{ db2_dbname }}" + - "4K Device Support ...................... {{ db2_4k_device_support }}" + - "Table Organization ..................... {{ db2_table_org }}" + - "TLS Version ............................ {{ tls_version }}" + - "Workload ............................... {{ db2_workload }}" + +- name: "Debug - Resources" + debug: + msg: + - "CPU Request ............................ {{ db2_cpu_requests }}" + - "CPU Limit .............................. {{ db2_cpu_limits }}" + - "Memory Request ......................... {{ db2_memory_requests }}" + - "Memory Limit ........................... {{ db2_memory_limits }}" + +- name: "Debug - Storage" + debug: + msg: + - "Meta ................................... {{ db2_meta_storage_class }} - {{ db2_meta_storage_size }} @ {{ db2_meta_storage_accessmode }}" + - "Data ................................... {{ db2_data_storage_class }} - {{ db2_data_storage_size }} @ {{ db2_data_storage_accessmode }}" + - "Backup ................................. {{ db2_backup_storage_class }} - {{ db2_backup_storage_size }} @ {{ db2_backup_storage_accessmode }}" + - "Logs ................................... {{ db2_logs_storage_class }} - {{ db2_logs_storage_size }} @ {{ db2_logs_storage_accessmode }}" + - "Temp ................................... {{ db2_temp_storage_class }} - {{ db2_temp_storage_size }} @ {{ db2_temp_storage_accessmode }}" + +# Lookup db2 operator group +- name: "Check if operator group is present in {{ db2_namespace }} namespace already" + kubernetes.core.k8s_info: + namespace: "{{ db2_namespace }}" + kind: OperatorGroup + register: db2_og_info + +# Check if db2_channel is avaialble in the catalog and is fetched correctly +- name: Debug DB2 upgrade channel + ansible.builtin.debug: + msg: + - "Db2 Channel (check if default got fetched from CLI)............................ {{ db2_channel }}" + +# Load catalog metadata as additional fallback (if CLI didn't set it) +- name: "Determine Version of Maximo Operator Catalog" + include_tasks: "{{ role_path }}/../../common_tasks/determine-ibmcatalog-tag.yml" + when: db2_channel is not defined or db2_channel == "" + +- name: Load Catalog Metadata for db2_channel + when: + - db2_channel is not defined or db2_channel == "" + - catalog_tag is defined and catalog_tag != "" + block: + - ibm.mas_devops.get_catalog_info: + mas_catalog_version: "{{ catalog_tag }}" + register: _mas_catalog + + - name: "Set db2_channel from catalog metadata" + when: _mas_catalog.db2_channel_default is defined + set_fact: + db2_channel: "{{ _mas_catalog.db2_channel_default }}" + +# Look up the default channel for the db2u-operator package manifest +- name: Lookup db2u-operator packagemanifest + kubernetes.core.k8s_info: + api_version: packages.operators.coreos.com/v1 + kind: PackageManifest + name: db2u-operator + namespace: "{{ ibm_common_services_namespace }}" + register: db2u_manifest_info + until: db2u_manifest_info.resources[0].status.defaultChannel is defined + retries: 60 # Approximately 30 minutes before we give up + delay: 30 # seconds + when: db2_channel is not defined or db2_channel == "" + #when: db2_channel is not defined or db2_channel == "" or db2_version is not defined or db2_version == "" + + # Rotate DB2 channel by day of the week +- name: "Select DB2 channel based on day of the week" + when: db2_channel is defined and db2_channel == "rotate" + set_fact: + db2_channel: "{{ rotate_db2_channel[ansible_date_time['weekday']] }}" + vars: + rotate_db2_channel: + Monday: "v110509.0" + Tuesday: "v120101.0" + Wednesday: "v110509.0" + Thursday: "v120101.0" + Friday: "v120101.0" + Saturday: "v120101.0" + Sunday: "v120101.0" + +- name: "Debug DB2 channel after rotation" + when: db2_channel is defined and db2_channel != "" + ansible.builtin.debug: + msg: + - "DB2 Channel (after rotation check) ........ {{ db2_channel }}" + +- name: Set db2u-operator channel + ansible.builtin.set_fact: + db2_channel: "{{ db2u_manifest_info.resources[0].status.defaultChannel }}" + when: db2_channel is not defined or db2_channel == "" + +# 5. Fail if required parameters are not set +# ----------------------------------------------------------------------------- +- name: "Verify db2_channel is set" + assert: + that: + - db2_channel is defined and db2_channel != "" + fail_msg: "Unable to determine db2_channel from catalog" + +- name: Debug DB2 upgrade channel + ansible.builtin.debug: + msg: + - "Db2 Channel ............................ {{ db2_channel }}" + +# 6. Install a Db2u Operator +# ----------------------------------------------------------------------------- +- name: "Create db2u Namespace" + kubernetes.core.k8s: + apply: yes + template: "templates/db2u_namespace.yaml" + register: _db2_namespace_result + +- name: Set 'ibm-registry' secret content + no_log: true + vars: + entitledAuthStr: "{{ registry_user }}:{{ ibm_entitlement_key }}" + entitledAuth: "{{ entitledAuthStr | b64encode }}" + content: + - '{"auths":{"{{ registry }}/cp/cpd":{"username":"{{ registry_user }}","password":"{{ ibm_entitlement_key }}","email":"{{ registry_user }}","auth":"{{ entitledAuth }}"}' + - "}" + - "}" + set_fact: + new_secret: "{{ content | join('') }}" + +# Note: We use "new_secret | to_json " because older versions of Ansible create +# invalid json representations containing single quotes +# However, in newer versions of Ansible to_json returns an object, which can't be passed to b64encode +- name: "Generate docker secret (old Ansible versions)" + when: ansible_version.full is version_compare(2.20, '<') + set_fact: + dockerconfigjsonB64: "{{ new_secret | to_json | b64encode }}" + +- name: "Generate docker secret (new Ansible versions)" + when: ansible_version.full is version_compare(2.20, '>=') + set_fact: + dockerconfigjsonB64: "{{ new_secret | b64encode }}" + +- name: "Generate 'ibm-registry' secret" + no_log: true + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: kubernetes.io/dockerconfigjson + metadata: + name: ibm-registry + namespace: "{{ db2_namespace }}" + data: + .dockerconfigjson: "{{ dockerconfigjsonB64 }}" + register: secretUpdateResult + +- name: "Delete old db2 subscription, operand request and csv from {{ ibm_common_services_namespace }}" + include_tasks: "tasks/delete_db2_operand_request.yml" + +- name: "Create Db2 Universal Operator Subscription in {{ db2_namespace }} namespace" + ibm.mas_devops.apply_subscription: + namespace: "{{ db2_namespace }}" + package_name: db2u-operator + package_channel: "{{ db2_channel }}" + register: subscription + +# 7. Get the cluster subdomain to be used for the certificate and route creation +# ----------------------------------------------------------------------------- +- name: "Get cluster subdomain" + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + kind: Ingress + name: cluster + register: _cluster_subdomain + +# 8. Create self-signed certificate for Db2u SSL # ----------------------------------------------------------------------------- -- include_tasks: tasks/install/install.yml +- name: "Create internal CA certificate issuer" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/ca_issuer.yml.j2" + register: createCaIssuer -# Start database restore task +- name: "Create and wait for CA certificate" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/ca_certificate.yml.j2" + wait: yes + wait_timeout: 600 #10 minutes + wait_condition: + type: Ready + status: True + register: createCaCert + +- name: "Create certificate issuer" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/issuer.yml.j2" + register: createIssuer + +- name: "Create db2u certificate" + kubernetes.core.k8s: + apply: yes + template: "templates/certs/certificate.yml.j2" + register: createCertificate + +# 9. Wait until the Db2uCluster CRD is available +# ----------------------------------------------------------------------------- +- name: "Wait until the Db2uCluster CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: "db2uclusters.db2u.databases.ibm.com" + +# 10. Get information from the db2u-release ConfigMap +# ----------------------------------------------------------------------------- +# if db2_version is not set, then we define it based on the latest version supported by the db2u-license-keys secret +# Starting with s11.5.8.0-cn3, the 's' prefix is removed in db2u-license-keys, we are recommeded to use db2u-release configmap. +- block: + - name: "Wait until the db2u-release configmap is available" + no_log: true + kubernetes.core.k8s_info: + api_version: v1 + name: db2u-release + namespace: "{{ db2_namespace }}" + kind: ConfigMap + register: db2_release_info + retries: 20 # ~approx 10 minutes before we give up waiting for the CRD to be created + delay: 30 # seconds + until: + - db2_release_info.resources is defined + - db2_release_info.resources | length > 0 + - db2_release_info.resources[0].data is defined + - db2_release_info.resources[0].data | length > 0 + + - name: Set db2u-release configmap content + no_log: true + set_fact: + db2_releases_content: "{{ db2_release_info.resources[0].data.json | from_json }}" + + - name: Extract major version from channel + set_fact: + db2_major_version: "{{ db2_channel | regex_replace('^v(\\d{2})(\\d+).*', '\\1') }}" + + - name: Debug extracted version + ansible.builtin.debug: + msg: + - "DB2 Channel ........................... {{ db2_channel }}" + - "Extracted Major Version ............... {{ db2_major_version }}" + - "Filter Pattern ........................ ^s{{ db2_major_version }}" + + - name: Determine DB2 version based on channel + set_fact: + db2_version: >- + {{ + (db2_releases_content.databases.db2u.keys() + | select('match', '^s' + db2_major_version) + | sort) + | last + }} + when: + - (db2_releases_content.databases.db2u.keys() | select('match', '^s' + db2_major_version) | list | length) > 0 + + when: db2_version is not defined or db2_version == "" + +- name: Debug DB2 channel and determined version + ansible.builtin.debug: + msg: + - "DB2 Channel ........................... {{ db2_channel }}" + - "DB2 Version (determined) .............. {{ db2_version }}" + - "Available versions .................... {{ db2_releases_content.databases.db2u.keys() | list }}" + +# 11. Fail if required parameters are not set # ----------------------------------------------------------------------------- -- include_tasks: tasks/restore_database/restore-database.yml - when: install_from_backup | bool +- name: "Verify db2_channel and db2_version set" + assert: + that: + - db2_channel is defined and db2_channel != "" + - db2_version is defined and db2_version != "" + fail_msg: "Unable to determine db2_channel and/or db2_version" + +- name: Debug DB2 upgrade channel and version + ansible.builtin.debug: + msg: + - "Db2 Channel ............................ {{ db2_channel }}" + - "Db2 Version ............................ {{ db2_version }}" + +# 12. Lookup db2 instance to see if it exists already +# ----------------------------------------------------------------------------- +- name: "See if db2u instance already exists" + kubernetes.core.k8s_info: + api_version: db2u.databases.ibm.com/v1 + name: "{{ db2_instance_name | lower }}" + namespace: "{{db2_namespace}}" + kind: Db2uCluster + register: initial_db2_cluster_lookup + +# 13. Create a Db2 instance +# ----------------------------------------------------------------------------- +- name: "Create db2 instance" + kubernetes.core.k8s: + apply: yes + template: "templates/db2ucluster.yml.j2" + +- name: "Set db2 instance timezone" + include_tasks: "tasks/install/setup_timezone.yml" + when: + - db2_timezone is defined + - db2_timezone != "" + +# 14. Wait for the cluster to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for db2u instance to be ready (5m delay)" + kubernetes.core.k8s_info: + api_version: db2u.databases.ibm.com/v1 + name: "{{ db2_instance_name | lower }}" + namespace: "{{db2_namespace}}" + kind: Db2uCluster + register: db2_cluster_lookup + until: + - db2_cluster_lookup.resources is defined + - db2_cluster_lookup.resources | length == 1 + - db2_cluster_lookup.resources[0].status is defined + - db2_cluster_lookup.resources[0].status.state is defined + - db2_cluster_lookup.resources[0].status.state == "Ready" + retries: 24 # Approximately 2 hours before we give up + delay: 300 # 5 minutes + +# 15. Configure a public route for Db2 +# ----------------------------------------------------------------------------- +- name: Lookup db2u Engn Service + kubernetes.core.k8s_info: + api_version: v1 + kind: Service + name: "c-{{db2_instance_name | lower}}-db2u-engn-svc" + namespace: "{{db2_namespace}}" + register: _db2_instance_engn_svc + until: + - _db2_instance_engn_svc.resources[0] is defined + retries: 15 # approx 5 minutes before we give up + delay: 20 # seconds + +- name: Lookup db2u TLS certificates + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "db2u-certificate-{{db2_instance_name}}" + namespace: "{{db2_namespace}}" + register: _db2_instance_certificates + +- name: Set Db2u certificates as Facts + set_fact: + db2_ca_pem: "{{ _db2_instance_certificates.resources[0].data['ca.crt'] | b64decode }}" + db2_tls_crt: "{{ _db2_instance_certificates.resources[0].data['tls.crt'] | b64decode }}" + db2_tls_key: "{{ _db2_instance_certificates.resources[0].data['tls.key'] | b64decode }}" + when: + - _db2_instance_certificates is defined + - (_db2_instance_certificates.resources | length > 0) + +- name: Set Db2u TLS port + set_fact: + db2_tls_serviceport: "{{item.targetPort}}" + when: "item.name == 'ssl-server'" + loop: "{{_db2_instance_engn_svc.resources[0].spec.ports}}" + +- name: "Create dedicated route: db2u-{{ db2_instance_name }}-tls-route" + kubernetes.core.k8s: + apply: yes + template: "templates/tlsroute.yml.j2" + +- name: Lookup existing db2u-tls-route + kubernetes.core.k8s_info: + api_version: v1 + kind: Route + name: "db2u-tls-route" + namespace: "{{db2_namespace}}" + register: _db2_tls_route + +# delete existing db2u-tls-route if that exists and matches with the same host/location from route created by previous step. +# that way we ensure that clean up should only happen if there's a conflicting route present. +- name: Clean up db2u-tls-route if needed + vars: + expected_host: "{{db2_instance_name | lower }}-{{db2_namespace}}.{{_cluster_subdomain.resources[0].spec.domain}}" + when: + - _db2_tls_route.resources | length > 0 + - _db2_tls_route.resources[0].spec.host == expected_host + kubernetes.core.k8s: + api_version: v1 + kind: Route + name: "db2u-tls-route" + namespace: "{{db2_namespace}}" + state: absent + + +## 16. create an LDAP user if db2_ldap_username specified +# ----------------------------------------------------------------------------- +- name: Create LDAP user if username and password is provided + include_tasks: tasks/install/create_ldap_user.yml + when: + - db2_ldap_username is defined + - db2_ldap_username != "" + - db2_ldap_password is defined + - db2_ldap_password != "" + - db2_rotate_password == false + +- debug: + msg: + - "{{db2_ldap_username}}" + - "{{db2_rotate_password}}" + +# 17. Rotate db2 ldap password +# ----------------------------------------------------------------------------- +- name: Rotate Db2 LDAP password if db2_rotate_password is True and username is provided + include_tasks: tasks/install/rotate_ldap_user_password.yml + when: + - db2_ldap_username is defined + - db2_ldap_username != "" + - db2_rotate_password == true + +# 18. Wait for the statefulset to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for Db2 Stateful set to be ready" + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: StatefulSet + name: "c-{{ db2_instance_name | lower }}-db2u" + namespace: "{{ db2_namespace }}" + register: db2_sts + until: + - db2_sts.resources is defined + - db2_sts.resources | length > 0 + - db2_sts.resources[0].status is defined + - db2_sts.resources[0].status.replicas is defined + - db2_sts.resources[0].status.readyReplicas is defined + - db2_sts.resources[0].status.readyReplicas == db2_sts.resources[0].status.replicas + retries: 20 # approx 10 minutes before we give up + delay: 30 # seconds + +# 19. Generate a JdbcCfg for MAS configuration +# ----------------------------------------------------------------------------- +- include_tasks: tasks/install/suite_jdbccfg.yml + when: + - mas_instance_id is defined + - mas_instance_id != "" + - mas_config_dir is defined + - mas_config_dir != "" diff --git a/ibm/mas_devops/roles/db2/tasks/main.yml b/ibm/mas_devops/roles/db2/tasks/main.yml index 113dcb8d30..5b025b1c13 100644 --- a/ibm/mas_devops/roles/db2/tasks/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/main.yml @@ -3,3 +3,10 @@ include_tasks: "tasks/{{ db2_action }}/main.yml" when: - db2_action != "none" + - db2_action in ["install", "upgrade", "backup", "restore_database", "backup_database", "restore"] + +- name: "Fail if db2_action is invalid" + fail: + msg: "db2_action must be one of ['install', 'upgrade', 'backup', 'restore_database', 'backup_database', 'restore']" + when: + - db2_action not in ["install", "upgrade", "backup", "restore_database", "backup_database", "restore"] diff --git a/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml b/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml new file mode 100644 index 0000000000..605b839b60 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml @@ -0,0 +1,15 @@ +--- +- name: "Determining storage class name to override" + include_tasks: "tasks/install/determine-storage-classes.yml" + +# Get Storage from DB2 CR +- name: Override storage class names in the db2 spec.storage + set_fact: + _db2_storages: "{{ db2ucluster_cr_cfg.spec.storage | ibm.mas_devops.override_db2_storage_classes_names(db2_meta_storage_class, db2_data_storage_class, db2_backup_storage_class, db2_logs_storage_class, db2_temp_storage_class) }}" + when: + - db2_meta_storage_class is defined + - db2_data_storage_class is defined + - db2_backup_storage_class is defined + - db2_logs_storage_class is defined + - db2_temp_storage_class is defined + diff --git a/ibm/mas_devops/roles/db2/tasks/restore/main.yml b/ibm/mas_devops/roles/db2/tasks/restore/main.yml new file mode 100644 index 0000000000..4389c5809a --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/main.yml @@ -0,0 +1,7 @@ +--- +# Restore Db2 Universal operator instance +- name: "Start DB2 Instance restore process." + include_tasks: "{{ role_path }}/tasks/restore/restore-instance.yml" + +# Restore Db2 database +- include_tasks: tasks/restore_database/main.yml diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml new file mode 100644 index 0000000000..035930f1b6 --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml @@ -0,0 +1,338 @@ +--- +# Check db2 Restore required variables +# ----------------------------------------------------------------------------- +- name: Verify DB2 restore variables + ibm.mas_devops.verify_backup_restore_vars: + component: "db2" + action: "restore_instance" + db2_backup_version: "{{ db2_backup_version }}" + mas_backup_dir: "{{ mas_backup_dir }}" + +# Set backup path facts +- name: "Set fact: backup dir paths" + set_fact: + db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u" + db2_resources_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u/resources" + +- name: "Check Db2u resource path exist" + stat: + path: "{{ db2_resources_path }}" + register: resources_backup_path_stat + +- name: "Fail if backup archive does not exist" + fail: + msg: "Db2 backup resources archive not found at: {{ db2_resources_path }}" + when: not resources_backup_path_stat.stat.exists or not resources_backup_path_stat.stat.isdir + +# Verify cert-manager exists +# ----------------------------------------------------------------------------- +- name: Detect Certificate Manager installation + include_tasks: "{{ role_path }}/../../common_tasks/detect_cert_manager.yml" + +# Verify only one db2ucluster instance file is present in backup archive +# ----------------------------------------------------------------------------- +- name: Get files from {{ db2_resources_path }}/db2uclusters directory + set_fact: + instance_files: "{{ lookup('fileglob', '{{ db2_resources_path }}/db2uclusters/*', wantlist=True) }}" + +- name: Assert exactly one Db2uCluster CR exists + assert: + that: + - instance_files | length == 1 + fail_msg: "Db2uCluster Directory must contain exactly one file" + +- name: Set fact db2ucluster cr + set_fact: + db2ucluster_cr_cfg: "{{ lookup('file', '{{ instance_files[0] }}') | from_yaml }}" + +# Get Db2u details from backup CR +# ----------------------------------------------------------------------------- +- name: Set fact db2u namespace and instance name from backup CR + set_fact: + db2_namespace: "{{ db2ucluster_cr_cfg.metadata.namespace }}" + db2_instance_name: "{{ db2ucluster_cr_cfg.metadata.name }}" + db2_version: "{{ db2ucluster_cr_cfg.spec.version }}" + db2_dbname: "{{ db2ucluster_cr_cfg.spec.environment.database.name }}" + db2_type: "{{ db2ucluster_cr_cfg.spec.environment.dbType }}" + +- name: "Db2uCluster restore information" + debug: + msg: + - "Db2u Namespace ................. {{ db2_namespace }}" + - "Db2u Instance Name ............. {{ db2_instance_name }}" + - "Db2u version ................... {{ db2_version }}" + - "Db2u Database Name ............. {{ db2_dbname }}" + - "Db2u Database Type ............. {{ db2_type }}" + - "MAS Instance ID ................ {{ mas_instance_id | default('undefined') }}" + - "Backup Version ................. {{ db2_backup_version }}" + - "Backup Path .................... {{ db2_backup_path }}" + +# 1. Restore Namespace +# ----------------------------------------------------------------------------- +- name: Restore namespace + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - Project + replace_resource: false + register: namespace_result + +# 2. Restore Secrets & configmaps +# ----------------------------------------------------------------------------- +- name: Restore Secrets and configmaps + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - Secret + - ConfigMap + replace_resource: false + register: secrets_configmaps_result + when: + - namespace_result.success + + +# 3. Restore Operatorgroups +# ----------------------------------------------------------------------------- +- name: Restore Operatorgroups + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - OperatorGroup + replace_resource: false + register: operatorgroups_result + when: + - namespace_result.success + - secrets_configmaps_result is defined and secrets_configmaps_result.success + +# 4. Restore Subscription +# ----------------------------------------------------------------------------- +- name: Restore Subscriptions + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - Subscription + replace_resource: false + register: subscriptions_result + when: + - namespace_result.success + - operatorgroups_result is defined and operatorgroups_result.success + +# 5. Restore Certs and Issuers +# ----------------------------------------------------------------------------- +- name: "Restore Certificate Manager resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - Issuer + - Certificate + register: certmanager_result + when: + - namespace_result.success + - subscriptions_result is defined and subscriptions_result.success + +# 6. Wait until the Db2uCluster CRD is available +# ----------------------------------------------------------------------------- +- name: "Wait until the Db2uCluster CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: "db2uclusters.db2u.databases.ibm.com" + +# 7. Restore Db2uCluster +# ----------------------------------------------------------------------------- + +- name: "Overriding storage class name with default storage class in CR" + include_tasks: "tasks/restore/determine-storage-classes.yml" + when: + - override_storageclass | bool + +- name: "Retrieve db2 passwords from secrets to restore via Db2uCluster CR" + include_tasks: "tasks/restore/retrieve_db2_passwords.yml" + no_log: true + +- name: "Restore Db2uCluster resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - Db2uCluster + override_values: + Db2uCluster: + - spec.environment.database.ssl.secretName: "db2u-certificate-{{ db2_instance_name | lower }}" + - spec.environment.instance.password: "{{ _db2_instance_password }}" + - spec.environment.ldap.password: "{{ _db2_ldappassword }}" + - spec.environment.ldap.blueAdminPassword: "{{ _db2_ldapblueadmin_password }}" + - spec.storage: "{{ _db2_storages }}" + register: cr_result + when: + - namespace_result.success + +# 8. Set timezone +# ----------------------------------------------------------------------------- +- name: "Determine if timezone is set in the CR" + set_fact: + db2_timezone: "{{ db2ucluster_cr_cfg.spec.advOpts.timezone | default('') }}" + +- name: "Set db2 instance timezone" + include_tasks: "tasks/install/setup_timezone.yml" + when: + - db2_timezone is defined and db2_timezone != "" + +# 9. Wait for the cluster to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for db2u instance to be ready (5m delay)" + kubernetes.core.k8s_info: + api_version: db2u.databases.ibm.com/v1 + name: "{{ db2_instance_name | lower }}" + namespace: "{{db2_namespace}}" + kind: Db2uCluster + register: db2_cluster_lookup + until: + - db2_cluster_lookup.resources is defined + - db2_cluster_lookup.resources | length == 1 + - db2_cluster_lookup.resources[0].status is defined + - db2_cluster_lookup.resources[0].status.state is defined + - db2_cluster_lookup.resources[0].status.state == "Ready" + retries: 24 # Approximately 2 hours before we give up + delay: 300 # 5 minutes + when: + - namespace_result.success + - cr_result.success + no_log: true # no_log because spits out big json object + +# 10. Restore route +- name: "Get cluster subdomain" + kubernetes.core.k8s_info: + api_version: config.openshift.io/v1 + kind: Ingress + name: cluster + register: _cluster_subdomain + no_log: true # no_log because spits out big json object + +# Apply cluster domain in tls route +- name: "Restore routes" + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - Route + override_values: + Route: + - spec.host: "{{db2_instance_name | lower }}-{{db2_namespace}}.{{_cluster_subdomain.resources[0].spec.domain}}" + register: routes_result + when: + - namespace_result.success + - subscriptions_result is defined and subscriptions_result.success + +# 11. Restore LDAP user +# ----------------------------------------------------------------------------- +- name: Restore LDAP user if username and password is provided + include_tasks: tasks/install/create_ldap_user.yml + when: + - db2_ldap_username is defined and db2_ldap_username != "" + - db2_ldap_password is defined and db2_ldap_password != "" + +# 12. Wait for the statefulset to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for Db2 Stateful set to be ready" + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: StatefulSet + name: "c-{{ db2_instance_name | lower }}-db2u" + namespace: "{{ db2_namespace }}" + register: db2_sts + until: + - db2_sts.resources is defined + - db2_sts.resources | length > 0 + - db2_sts.resources[0].status is defined + - db2_sts.resources[0].status.replicas is defined + - db2_sts.resources[0].status.readyReplicas is defined + - db2_sts.resources[0].status.readyReplicas == db2_sts.resources[0].status.replicas + retries: 20 # approx 10 minutes before we give up + delay: 30 # seconds + no_log: true # no_log because spits out big json object + +# Calculate total results +# ----------------------------------------------------------------------------- +- name: "Calculate total restore results" + set_fact: + total_created: >- + {{ + (namespace_result.created_count | default(0)) + + (secrets_configmaps_result.created_count | default(0)) + + (operatorgroups_result.created_count | default(0)) + + (subscriptions_result.created_count | default(0)) + + (certmanager_result.created_count | default(0)) + + (cr_result.created_count | default(0)) + + (routes_result.created_count | default(0)) + }} + total_updated: >- + {{ + (namespace_result.updated_count | default(0)) + + (secrets_configmaps_result.updated_count | default(0)) + + (operatorgroups_result.updated_count | default(0)) + + (subscriptions_result.updated_count | default(0)) + + (certmanager_result.updated_count | default(0)) + + (cr_result.updated_count | default(0)) + + (routes_result.updated_count | default(0)) + }} + total_skipped: >- + {{ + (namespace_result.skipped_count | default(0)) + + (secrets_configmaps_result.skipped_count | default(0)) + + (operatorgroups_result.skipped_count | default(0)) + + (subscriptions_result.skipped_count | default(0)) + + (certmanager_result.skipped_count | default(0)) + + (cr_result.skipped_count | default(0)) + + (routes_result.skipped_count | default(0)) + }} + total_failed: >- + {{ + (namespace_result.failed_count | default(0)) + + (secrets_configmaps_result.failed_count | default(0)) + + (operatorgroups_result.failed_count | default(0)) + + (subscriptions_result.failed_count | default(0)) + + (certmanager_result.failed_count | default(0)) + + (cr_result.failed_count | default(0)) + + (routes_result.failed_count | default(0)) + }} + +- name: "Display total restore results" + debug: + msg: + - >- + Restore completed{{ ' with failures' if total_failed | int > 0 + else ' successfully' }} + - "Total resources created: {{ total_created }}" + - "Total resources updated: {{ total_updated }}" + - "Total resources skipped: {{ total_skipped }}" + - "Total resources failed: {{ total_failed }}" + +# Fail task if any errors occurred +# ----------------------------------------------------------------------------- +- name: "Collect all failed resources" + set_fact: + all_failed_resources: >- + {{ + (namespace_result.failed_resources | default([])) + + (secrets_configmaps_result.failed_resources | default([])) + + (operatorgroups_result.failed_resources | default([])) + + (subscriptions_result.failed_resources | default([])) + + (certmanager_result.failed_resources | default([])) + + (cr_result.failed_resources | default([])) + + (routes_result.failed_resources | default([])) + }} + +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ all_failed_resources | to_nice_yaml }}" + when: total_failed | int > 0 + +- name: "Fail if restore had errors" + fail: + msg: | + Restore failed for {{ total_failed }} resource(s): + {% for resource in all_failed_resources %} + - {{ resource.description }}: {{ resource.error }} + {% endfor %} + when: total_failed | int > 0 + diff --git a/ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml b/ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml new file mode 100644 index 0000000000..fb821bb1df --- /dev/null +++ b/ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml @@ -0,0 +1,66 @@ +--- +# Retrieving instance password from secret to restore via Db2uCluster CR +- name: "Check instance password secret is present in /secrets/ dir" + no_log: true + stat: + path: "{{ db2_resources_path }}/secrets/c-{{ db2_instance_name }}-instancepassword.yaml" + register: instancepassword_stat + +- name: Read YAML file and get password value + no_log: true + set_fact: + _db2_instance_password: "{{ lookup('file', '{{ db2_resources_path }}/secrets/c-{{ db2_instance_name }}-instancepassword.yaml') | from_yaml | json_query('data.password') | b64decode }}" + when: instancepassword_stat.stat.exists + +# Retrieving ldapblueadminpassword from secrets to restore via Db2uCluster CR +- name: "Check ldapblueadminpassword password secret is present in /secrets/ dir" + no_log: true + stat: + path: "{{ db2_resources_path }}/secrets/c-{{ db2_instance_name }}-ldapblueadminpassword.yaml" + register: ldapblueadminpassword_stat + +- name: Read YAML file and get password value + no_log: true + set_fact: + _db2_ldapblueadmin_password: "{{ lookup('file', '{{ db2_resources_path }}/secrets/c-{{ db2_instance_name }}-ldapblueadminpassword.yaml') | from_yaml | json_query('data.password') | b64decode }}" + when: ldapblueadminpassword_stat.stat.exists + + +# Retrieving ldappassword from secrets to restore via Db2uCluster CR +- name: "Check ldappassword password secret is present in /secrets/ dir" + no_log: true + stat: + path: "{{ db2_resources_path }}/secrets/c-{{ db2_instance_name }}-ldappassword.yaml" + register: ldappassword_stat + +- name: Read YAML file and get password value + no_log: true + set_fact: + _db2_ldappassword: "{{ lookup('file', '{{ db2_resources_path }}/secrets/c-{{ db2_instance_name }}-ldappassword.yaml') | from_yaml | json_query('data.password') | b64decode }}" + when: ldappassword_stat.stat.exists + +# Retrieving jdbc username/password from secrets to restore via Db2uCluster CR +- name: "Check jdbc-{{ db2_instance_name }}-credentials password secret is present in /secrets/ dir" + no_log: true + stat: + path: "{{ db2_resources_path }}/secrets/jdbc-{{ db2_instance_name }}-credentials.yaml" + register: jdbcsecret_stat + +- name: Read YAML file and get username value + no_log: true + set_fact: + _jdbc_username: "{{ lookup('file', '{{ db2_resources_path }}/secrets/jdbc-{{ db2_instance_name }}-credentials.yaml') | from_yaml | json_query('data.username') | b64decode }}" + when: jdbcsecret_stat.stat.exists + +- name: Read YAML file and get password value + no_log: true + set_fact: + db2_ldap_username: "{{ _jdbc_username}}" + db2_ldap_password: "{{ lookup('file', '{{ db2_resources_path }}/secrets/jdbc-{{ db2_instance_name }}-credentials.yaml') | from_yaml | json_query('data.password') | b64decode }}" + when: + - _jdbc_username is defined and _jdbc_username != "" + - _jdbc_username != "db2inst1" + - jdbcsecret_stat.stat.exists + + + diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml index df369eeebc..2e7eb4473e 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml @@ -23,10 +23,17 @@ - kind: CatalogSource api_version: operators.coreos.com/v1alpha1 name: ibm-operator-catalog - # Secret (in dev envs) + # Secret (for dev catalogs) - kind: Secret api_version: v1 name: wiotp-docker-local + # Service Accounts (this is reqd for dev catalogs) + - kind: ServiceAccount + api_version: v1 + name: ibm-operator-catalog + - kind: ServiceAccount + api_version: v1 + name: default # Call the backup_resources plugin to execute the backup to the path provided # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/create-wiotp-secret.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/create-wiotp-secret.yml new file mode 100644 index 0000000000..ebda6fba25 --- /dev/null +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/create-wiotp-secret.yml @@ -0,0 +1,27 @@ +--- + +# Create an image pull secret for local artifactory so that we can install the development catalog +# --------------------------------------------------------------------------------------------------------------------- +- name: "Create wiotp-docker-local secret" + no_log: true + vars: + artifactoryAuthStr: "{{artifactory_username}}:{{artifactory_token}}" + artifactoryAuth: "{{ artifactoryAuthStr | b64encode }}" + content: + - '{"auths":{"docker-na-public.artifactory.swg-devops.com/wiotp-docker-local": {"username":"{{artifactory_username}}","password":"{{artifactory_token}}","auth":"{{artifactoryAuth}}"}' + - ',"docker-na-proxy-svl.artifactory.swg-devops.com/wiotp-docker-local": {"username":"{{artifactory_username}}","password":"{{artifactory_token}}","auth":"{{artifactoryAuth}}"}' + - ',"docker-na-proxy-rtp.artifactory.swg-devops.com/wiotp-docker-local": {"username":"{{artifactory_username}}","password":"{{artifactory_token}}","auth":"{{artifactoryAuth}}"}' + - "}" + - "}" + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Secret + type: kubernetes.io/dockerconfigjson + metadata: + name: wiotp-docker-local + namespace: openshift-marketplace + stringData: + # Only way I could get three consecutive "}" into a string :) + .dockerconfigjson: "{{ content | join('') | string }}" + register: result diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml new file mode 100644 index 0000000000..f2e37e3913 --- /dev/null +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml @@ -0,0 +1,151 @@ +--- +- name: "Fail if require variables for IBM operator catalog backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_backup_dir: "{{ mas_backup_dir }}" + catalog_backup_version: "{{ catalog_backup_version }}" + action: "restore" + component: "catalog" + +- name: "Set fact: Catalog backup base directory path" + set_fact: + catalog_backup_path: "{{ mas_backup_dir }}/backup-{{ catalog_backup_version }}-catalog" + catalog_resources_path: "{{ mas_backup_dir }}/backup-{{ catalog_backup_version }}-catalog/resources" + +- name: "Check catalog backup resource path exist" + stat: + path: "{{ catalog_resources_path }}" + register: resources_backup_path_stat + +- name: "Fail if backup archive does not exist" + fail: + msg: "Catalog resources archive not found at: {{ catalog_resources_path }}" + when: not resources_backup_path_stat.stat.exists or not resources_backup_path_stat.stat.isdir + +- name: "MAS Catalog restore information" + debug: + msg: + - "Backup Version ................. {{ catalog_backup_version }}" + - "Backup Path .................... {{ catalog_backup_path }}" + +# 1. Restore Secrets (required for dev catalog) +# ----------------------------------------------------------------------------- +- name: Create wiotp-docker-local secret if artifactory details are provided. + when: + - artifactory_username is defined and artifactory_username != "" + - artifactory_token is defined and artifactory_token != "" + include_tasks: tasks/restore/create-wiotp-secret.yml + +- name: Restore secrets + ibm.mas_devops.restore_resource: + backup_path: "{{ catalog_backup_path }}" + resource_kinds: + - Secret + register: secret_result + when: + - artifactory_username is not defined + - artifactory_token is not defined + +# 2. Restore Service Accounts (required for dev catalog) +# ----------------------------------------------------------------------------- +- name: Restore ServiceAccounts + ibm.mas_devops.restore_resource: + backup_path: "{{ catalog_backup_path }}" + resource_kinds: + - ServiceAccount + register: sa_result + +# 3. Restore CatalogSource +# ----------------------------------------------------------------------------- +- name: Restore CatalogSource + ibm.mas_devops.restore_resource: + backup_path: "{{ catalog_backup_path }}" + resource_kinds: + - CatalogSource + register: catalog_result + +# Calculate total results +# ----------------------------------------------------------------------------- +- name: "Calculate total restore results" + set_fact: + total_created: >- + {{ + (secret_result.created_count | default(0)) + + (sa_result.created_count | default(0)) + + (catalog_result.created_count | default(0)) + }} + total_updated: >- + {{ + (secret_result.updated_count | default(0)) + + (sa_result.updated_count | default(0)) + + (catalog_result.updated_count | default(0)) + }} + total_skipped: >- + {{ + (secret_result.skipped_count | default(0)) + + (sa_result.skipped_count | default(0)) + + (catalog_result.skipped_count | default(0)) + }} + total_failed: >- + {{ + (secret_result.failed_count | default(0)) + + (sa_result.failed_count | default(0)) + + (catalog_result.failed_count | default(0)) + }} + +- name: "Display total restore results" + debug: + msg: + - >- + Restore completed{{ ' with failures' if total_failed | int > 0 + else ' successfully' }} + - "Total resources created: {{ total_created }}" + - "Total resources updated: {{ total_updated }}" + - "Total resources skipped: {{ total_skipped }}" + - "Total resources failed: {{ total_failed }}" + +# Fail task if any errors occurred +# ----------------------------------------------------------------------------- +- name: "Collect all failed resources" + set_fact: + all_failed_resources: >- + {{ + (secret_result.failed_resources | default([])) + + (sa_result.failed_resources | default([])) + + (catalog_result.failed_resources | default([])) + }} + +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ all_failed_resources | to_nice_yaml }}" + when: total_failed | int > 0 + +- name: "Fail if restore had errors" + fail: + msg: | + Restore failed for {{ total_failed }} resource(s): + {% for resource in all_failed_resources %} + - {{ resource.description }}: {{ resource.error }} + {% endfor %} + when: total_failed | int > 0 + +# 4. Wait until ready +# ----------------------------------------------------------------------------- +- name: "Wait until CatalogSource/ibm-operator-catalog is Ready" + when: catalog_result.success + kubernetes.core.k8s_info: + api_version: operators.coreos.com/v1alpha1 + kind: CatalogSource + name: ibm-operator-catalog + namespace: openshift-marketplace + register: ibm_catalog_lookup + retries: 30 # Up to 30 min + delay: 60 # Every minute + until: + - ibm_catalog_lookup.resources is defined + - ibm_catalog_lookup.resources | length == 1 + - ibm_catalog_lookup.resources[0].status is defined + - ibm_catalog_lookup.resources[0].status.connectionState is defined + - ibm_catalog_lookup.resources[0].status.connectionState.lastObservedState is defined + - ibm_catalog_lookup.resources[0].status.connectionState.lastObservedState == "READY" diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index 95bb6ae81f..642c98a62e 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -581,18 +581,19 @@ For backup and restore operations, set `mongodb_action` to one of the following: - Default Value: YYMMDD-HHMMSS - Example: `251212-021316` -### br_skip_instance -Controls whether to backup MongoDB instance resources (secrets, certificates, issuers) along with database data. Set to `true` to skip instance resources in the backup. - -- Environment Variable: `BR_SKIP_INSTANCE` -- Default Value: `false` - ### mongodb_instance_name The name of the MongoDB instance to backup. - Environment Variable: `MONGODB_INSTANCE_NAME` - Default Value: `mas-mongo-ce` +### override_storageclass +Controls whether to override storage class for the MongoDB instance during restore. +Set to `true` to override the storage class with `MONGODB_STORAGE_CLASS` value or cluster's default storageclass. + +- Environment Variable: `OVERRIDE_STORAGECLASS` +- Default Value: `false` + ### mas_app_id Optional. Specific MAS application ID for targeted backup/restore operations. @@ -609,7 +610,8 @@ This section provides comprehensive information about MongoDB backup and restore | Action | Purpose | Instance Resources | Database Data | Prerequisites | Use Case | |--------|---------|-------------------|---------------|---------------|----------| -| `backup` | Create backup | Optional (controlled by `br_skip_instance`) | Yes | Running MongoDB instance | Regular backups, disaster recovery preparation | +| `backup` | Create backup | Yes | Yes | Running MongoDB instance | Regular backups, disaster recovery preparation | +| `backup_database` | Database-only backup | No | Yes | Running MongoDB instance | Regular backups, disaster recovery preparation | | `restore` | Full restore | Yes (recreates instance) | Yes | Backup with instance resources | Disaster recovery, cluster migration, complete restoration | | `restore_database` | Database-only restore | No (preserves existing) | Yes | Running MongoDB instance with matching version | Data recovery, selective restore, testing | @@ -660,7 +662,7 @@ Performs a complete restoration of both the MongoDB instance and its databases: 7. Restores the MongoDB Operator Deployment 8. Waits for MongoDB operator to be ready 9. Restores Certificate Manager resources (Issuer, Certificate) -10. Restores the MongoDBCommunity Custom Resource +10. Restores the MongoDBCommunity Custom Resource (Overrides storageclass if flag is set) 11. Waits for MongoDB StatefulSets to be ready 12. Restores ServiceMonitor and GrafanaDashboard resources 13. Restores database data using `mongorestore` @@ -675,7 +677,7 @@ Performs a complete restoration of both the MongoDB instance and its databases: - `mas_instance_id`: MAS instance identifier - `mas_backup_dir`: Directory containing the backup - `mongodb_backup_version`: Timestamp of the backup to restore -- `br_skip_instance`: Set to `false` to restore instance resources +- `override_storageclass`: Set to `true` to override storage class during instance restore #### 2. Database-Only Restore (`restore_database` action) Restores only the database data to an existing MongoDB instance: @@ -729,9 +731,8 @@ Restores only the database data to an existing MongoDB instance: - Network bandwidth may affect backup/restore speed **Restore Action Differences:** -- **`restore` action**: Recreates the entire MongoDB instance from scratch, including all Kubernetes resources +- **`restore` action**: Recreates the entire MongoDB instance from scratch, including all Kubernetes resources and restores database - **`restore_database` action**: Only restores database data to an existing instance, preserving current configuration -- Use `br_skip_instance` variable to control whether instance resources are included in backup/restore ### Backup and Restore Best Practices @@ -799,7 +800,20 @@ Create a complete backup including both database data and instance resources (se mas_instance_id: masinst1 mas_backup_dir: /tmp/masbr mongodb_action: backup - br_skip_instance: false + roles: + - ibm.mas_devops.mongodb +``` + +### Backup Database (CE Operator) +Create a backup of database data. + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: masinst1 + mas_backup_dir: /tmp/masbr + mongodb_action: backup_database roles: - ibm.mas_devops.mongodb ``` @@ -844,7 +858,7 @@ Perform a complete restoration of MongoDB instance including all Kubernetes reso **Prerequisites**: - Backup files must be available in the specified backup directory -- Backup must include instance resources (created with `br_skip_instance: false`) +- Backup must include instance resources (secrets, certificates, CRs) - Target cluster must have cert-manager installed - Sufficient storage and resources available @@ -856,7 +870,6 @@ Perform a complete restoration of MongoDB instance including all Kubernetes reso mas_instance_id: masinst1 mongodb_backup_version: 251212-021316 mas_backup_dir: /tmp/masbr - br_skip_instance: false roles: - ibm.mas_devops.mongodb ``` diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index ddc1d20ffa..68b3807f9d 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -124,7 +124,13 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # ----------------------------------------------------------------------------- mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" -br_skip_instance: "{{ lookup('env', 'BR_SKIP_INSTANCE') | default(false, true) }}" + +# set flag to true, to override storage class with cluster's default +override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default(false, true) }}" +_mongodb_storage: "NO_OVERRIDE" + +# ----------------------------------------------------------------------------- + # Mongo upgrade flags # If identified that there's an existing Mongo that might lead to a v5 or v6 upgrade diff --git a/ibm/mas_devops/roles/mongodb/tasks/main.yml b/ibm/mas_devops/roles/mongodb/tasks/main.yml index 1d7cc18132..2b191896af 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/main.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/main.yml @@ -10,8 +10,8 @@ assert: that: - mongodb_action is defined - - mongodb_action in ["install", "uninstall", "backup", "restore_database", "restore"] - fail_msg: "mongodb_action property is required and must be one of 'install', 'uninstall', 'backup', 'restore_database', 'restore'" + - mongodb_action in ["install", "uninstall", "backup", "backup_database", "restore_database", "restore"] + fail_msg: "mongodb_action property is required and must be one of 'install', 'uninstall', 'backup', 'backup_database','restore_database', 'restore'" # 2. Run the install / uninstall for specified provider # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index 3f7d000303..7b384730b5 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -7,7 +7,7 @@ mas_instance_id: "{{ mas_instance_id }}" mas_backup_dir: "{{ mas_backup_dir }}" mongodb_instance_name: "{{ mongodb_instance_name }}" - action: "{{ mongodb_action }}" + action: "backup" component: "mongodb" - name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" @@ -27,20 +27,16 @@ # Backup Mongodb Cluster Instance Kubernetes Resources # ------------------------------------------------------------------------- - - name: "Start MongoCE Instance backup process." include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/backup-instance.yml" - when: not br_skip_instance | bool # Backup Mongodb database Data using mondodump # ------------------------------------------------------------------------- - - name: "Debug information - Mongodb database Data backup" debug: msg: - "MongoCE namespace .......................... {{ mongodb_namespace }}" - "MongoCE instance name ...................... {{ mongodb_instance_name }}" - - "{{ 'MongoCE version ............................ ' ~ mongodb_version if mongodb_version is defined else omit }}" - name: "Start Database backup process." include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/backup-database.yml" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml new file mode 100644 index 0000000000..c8e408f59c --- /dev/null +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml @@ -0,0 +1,37 @@ +--- +# Check mongodb backup variables +# ----------------------------------------------------------------------------- + +- name: "Fail if require variables for Mongodb backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mongodb_instance_name: "{{ mongodb_instance_name }}" + action: "backup" + component: "mongodb" + +- name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + mongodb_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: mongodb_backup_version is not defined or mongodb_backup_version == "" or mongodb_backup_version == "None" + +- name: "Set fact: Create Mongodb backup base directory path" + set_fact: + mongodb_backup_path: "{{ mas_backup_dir }}/{{ mongodb_action }}-{{ mongodb_backup_version }}-mongoce" + +- name: "Create {{ mongodb_backup_path }} directory for Mongodb backup" + file: + path: "{{ mongodb_backup_path }}" + state: directory + mode: "0755" + +# Backup Mongodb database Data using mondodump +# ------------------------------------------------------------------------- +- name: "Debug information - Mongodb database Data backup" + debug: + msg: + - "MongoCE namespace .......................... {{ mongodb_namespace }}" + - "MongoCE instance name ...................... {{ mongodb_instance_name }}" + +- name: "Start Database backup process." + include_tasks: "{{ role_path }}/tasks/providers/{{ mongodb_provider }}/backup-restore/backup-database.yml" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml index b5717a1a6f..df8ca2aebb 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml @@ -1,8 +1,7 @@ --- -# Task to restore instance when br_skip_instance is set to false +# Task to restore instance # ----------------------------------------------------------------------------- - include_tasks: "tasks/providers/{{ mongodb_provider }}/restore_instance.yml" - when: not br_skip_instance | bool # Task to restore database # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml index 41c303ea81..1cb366083c 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml @@ -168,12 +168,19 @@ # Load default storage classes (if not provided by the user and not an update) # ----------------------------------------------------------------------------- -- name: Use chosen (or default) storage class +- name: Use chosen (or default) storage class when override_storageclass is true include_tasks: tasks/providers/community/determine-storage-classes.yml when: - namespace_result.success - certmanager_result is defined and certmanager_result.success - - existing_mongo_storage_class is not defined + - override_storageclass | bool + +- name: Replace mongo storageclass names when override_storageclass is true + when: + - override_storageclass | bool + - mongodb_storage_class is defined + set_fact: + _mongodb_storage: "{{ mongodbcr_cfg.spec.statefulSet.spec.volumeClaimTemplates | ibm.mas_devops.set_storage_classes_names(mongodb_storage_class, mongodb_storage_class) }}" # 8. Restore MongodbCE CR resource # ----------------------------------------------------------------------------- @@ -182,10 +189,12 @@ backup_path: "{{ mongodb_backup_path }}" resource_kinds: - MongoDBCommunity -# override_values: -# MongoDBCommunity: -# - spec.statefulSet.spec.volumeClaimTemplates[0].spec.storageClassName: mongodb_storage_class -# - spec.statefulSet.spec.volumeClaimTemplates[1].spec.storageClassName: mongodb_storage_class + filter_values: + MongoDBCommunity: + - spec.replicaSetHorizons + override_values: + MongoDBCommunity: + - spec.statefulSet.spec.volumeClaimTemplates: "{{ _mongodb_storage }}" register: cr_result when: - namespace_result.success @@ -315,7 +324,7 @@ (certmanager_result.updated_count | default(0)) + (cr_result.updated_count | default(0)) + (servicemonitor_result.updated_count | default(0)) + - (grafdash_result.updated_count | default(0)) + (grafdash_result.updated_count | default(0)) }} total_skipped: >- {{ @@ -327,7 +336,7 @@ (certmanager_result.skipped_count | default(0)) + (cr_result.skipped_count | default(0)) + (servicemonitor_result.skipped_count | default(0)) + - (grafdash_result.skipped_count | default(0)) + (grafdash_result.skipped_count | default(0)) }} total_failed: >- {{ @@ -339,7 +348,7 @@ (certmanager_result.failed_count | default(0)) + (cr_result.failed_count | default(0)) + (servicemonitor_result.failed_count | default(0)) + - (grafdash_result.failed_count | default(0)) + (grafdash_result.failed_count | default(0)) }} - name: "Display total restore results" From 22a8e2e311c561929e0f60cf0851b8b9cffd9c3c Mon Sep 17 00:00:00 2001 From: Andrew Whitfield Date: Thu, 22 Jan 2026 13:20:16 +0000 Subject: [PATCH 25/61] [minor] Add in br_core playbook (#2075) --- .../backup-namespace-resources.sh.j2 | 72 ------- ibm/mas_devops/playbooks/br_core.yml | 144 ++++++++++++++ .../plugins/action/backup_resource.py | 2 +- .../action/restore_mongoce_resources.py | 1 - .../plugins/action/restore_resource.py | 1 - .../action/verify_backup_restore_vars.py | 20 +- .../tasks/provider/redhat/backup.yml | 1 - .../tasks/provider/redhat/restore.yml | 1 - ibm/mas_devops/roles/db2/defaults/main.yml | 2 - .../db2/tasks/backup/backup-instance.yml | 2 - .../tasks/backup_database/backup-database.yml | 1 - .../restore/determine-storage-classes.yml | 1 - .../db2/tasks/restore/restore-instance.yml | 1 - .../tasks/restore/retrieve_db2_passwords.yml | 3 - .../restore_database/restore-db-from-disk.yml | 1 - .../restore_database/restore-db-from-s3.yml | 1 - .../roles/grafana/tasks/backup/main.yml | 1 - .../roles/ibm_catalogs/tasks/backup/main.yml | 1 - .../backup-restore/backup-database.yml | 2 - .../backup-restore/restore-database.yml | 50 ++--- .../providers/community/restore_instance.yml | 1 - .../roles/sls/tasks/backup/main.yml | 1 - .../roles/suite_app_backup/tasks/main.yml | 1 - ibm/mas_devops/roles/suite_backup/README.md | 16 ++ .../roles/suite_backup/defaults/main.yml | 6 + .../roles/suite_backup/tasks/main.yml | 186 ++++++++++-------- ibm/mas_devops/roles/suite_restore/README.md | 44 ++++- .../roles/suite_restore/defaults/main.yml | 12 ++ .../roles/suite_restore/tasks/main.yml | 116 ++++++++--- 29 files changed, 444 insertions(+), 247 deletions(-) delete mode 100644 ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 create mode 100644 ibm/mas_devops/playbooks/br_core.yml diff --git a/ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 b/ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 deleted file mode 100644 index 89bf2fd25a..0000000000 --- a/ibm/mas_devops/common_tasks/templates/backup_restore/backup-namespace-resources.sh.j2 +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -BACKUP_FOLDER="{{ masbr_ns_backup_folder }}" -LOG_FILE="{{ masbr_ns_backup_log }}" - - -function backupSingleResource { - local RESOURCE_NAMESPACE=$1 - local RESOURCE_KIND=$2 - local RESOURCE_NAME=$3 - - echo "Backing up $RESOURCE_KIND/$RESOURCE_NAME in the $RESOURCE_NAMESPACE namespace..." | tee -a $LOG_FILE - - if [[ "$(oc get $RESOURCE_KIND/$RESOURCE_NAME -n $RESOURCE_NAMESPACE --ignore-not-found=true --no-headers=true | wc -l)" == "0" ]]; then - echo "Not found $RESOURCE_KIND/$RESOURCE_NAME in the $RESOURCE_NAMESPACE namespace!" | tee -a $LOG_FILE - - else - local resourceYaml=$(oc get $RESOURCE_KIND/$RESOURCE_NAME -n $RESOURCE_NAMESPACE -o yaml) - hasCredentials=$(echo "$resourceYaml" | yq '.spec.config.credentials | has("secretName")') - - if [ "$hasCredentials" == "true" ]; then - credentialsName=`(echo "$resourceYaml" | yq .spec.config.credentials.secretName)` - echo "The credentials $credentialsName will be backed up for the resource $RESOURCE_NAME" | tee -a $LOG_FILE - backupSingleResource $RESOURCE_NAMESPACE Secret $credentialsName - fi - - echo "$resourceYaml" | yq 'del(.metadata.creationTimestamp, .metadata.ownerReferences, .metadata.generation, .metadata.resourceVersion, .metadata.uid, .metadata.annotations["kubectl.kubernetes.io/last-applied-configuration"], .status)' > $BACKUP_FOLDER/$RESOURCE_KIND-$RESOURCE_NAME.yaml | tee -a $LOG_FILE - echo "Saved $RESOURCE_KIND/$RESOURCE_NAME to $BACKUP_FOLDER/$RESOURCE_KIND-$RESOURCE_NAME.yaml" | tee -a $LOG_FILE - fi -} - -function backupResources { - local RESOURCE_NAMESPACE=$1 - local RESOURCE_KIND=$2 - local RESOURCE_KEYWORDS=$3 - - if [[ $RESOURCE_KEYWORDS == "" ]]; then - echo "Backing up all $RESOURCE_KIND resources in the $RESOURCE_NAMESPACE namespace..." | tee -a $LOG_FILE - resourceNames=($(oc get $RESOURCE_KIND -n $RESOURCE_NAMESPACE --no-headers=true | awk '{print $1}')) - else - echo "Backing up $RESOURCE_KIND resources containing keywords \"$RESOURCE_KEYWORDS\" in the $RESOURCE_NAMESPACE namespace..." | tee -a $LOG_FILE - resourceNames=($(oc get $RESOURCE_KIND -n $RESOURCE_NAMESPACE --no-headers=true | awk '{print $1}' | grep "$RESOURCE_KEYWORDS")) - fi - - for resourceName in ${resourceNames[@]}; do - backupSingleResource $RESOURCE_NAMESPACE $RESOURCE_KIND $resourceName - done -} - -if command -v yq &> /dev/null; then - echo "yq is installed, continuing" -else - echo "yq not found! please install yq before performing a backup" - exit 1 -fi - - -{% if masbr_ns_backup_resources is defined and masbr_ns_backup_resources | length > 0 %} -{% for ns_resources in masbr_ns_backup_resources %} - -{% for resource in ns_resources.resources %} - -{% if resource.name is not defined or resource.name == "*" %} -backupResources "{{ ns_resources.namespace }}" "{{ resource.kind }}" "{{ resource.keywords|default('') }}" -{% else %} -backupSingleResource "{{ ns_resources.namespace }}" "{{ resource.kind }}" "{{ resource.name }}" -{% endif %} - -{% endfor %} # ns_resources.resources - -{% endfor %} # masbr_ns_backup_resources -{% endif %} \ No newline at end of file diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml new file mode 100644 index 0000000000..b59bab9e9d --- /dev/null +++ b/ibm/mas_devops/playbooks/br_core.yml @@ -0,0 +1,144 @@ +--- +- name: "MAS Core Backup and Restore" + hosts: localhost + any_errors_fatal: true + vars: + # Define the target for backup/restore + mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + + # Set the action for this playbook of either backup or restore + br_action: "{{ lookup('env', 'BR_ACTION') | default('backup', true) }}" + + # The top level location to save (on backup) or retrieve (on restore) the archive files + mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups + + # Backup version to use for all during backup action + # Note: The default timestamp will be set in pre_tasks to ensure consistency across all roles + backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" + + # Required for restore action, the backup version to use on restore + backup_version_to_restore: "{{ lookup('env', 'BACKUP_VERSION_TO_RESTORE') }}" + + # Computed variable: use backup_version for backup action, backup_version_to_restore for restore action + br_action_version: "{{ backup_version if br_action == 'backup' else backup_version_to_restore }}" + + # Configurable flags for SLS and DRO. You should not include SLS or DRO if theses are configured externally + # the the cluster running the Suite. When these values are false then the slscfg and bascfg (for DRO) will + # be restored from what is defined in the backup config for the suite. + include_sls: "{{ lookup('env', 'INCLUDE_SLS') | default('true', true) | bool }}" # Set to false to skip SLS + include_dro: "{{ lookup('env', 'INCLUDE_DRO') | default('true', true) | bool }}" # Set to false to skip DRO + + # Required for DRO install + ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" + dro_contact_email: "{{ lookup('env', 'DRO_CONTACT_EMAIL') }}" + dro_contact_firstname: "{{ lookup('env', 'DRO_CONTACT_FIRSTNAME') }}" + dro_contact_lastname: "{{ lookup('env', 'DRO_CONTACT_LASTNAME') }}" + + # Set the following vars to control the domain and urls used on the restore. + # This is used when you are restoring to a different cluster than the one that was backed up, + # and the domain will be different. + mas_domain_on_restore: "{{ lookup('env', 'MAS_DOMAIN_ON_RESTORE') }}" + # If `include_sls` is false. The URL for the Suite License Service (SLS). If not provided, the + # URL from the backup will be used. This is used when the domain has changed for SLS but SLS was restored + # from a backup and so the regristration key is the same. + sls_url_on_restore: "{{ lookup('env', 'SLS_URL_ON_RESTORE') }}" + # Same setting as above but for dro when `include_dro` is false. + dro_url_on_restore: "{{ lookup('env', 'MAS_DOMAIN_ON_RESTORE') }}" + + # For DRO and SLS set the mas_config_dir for when these are either restored or installed new. + mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" + + pre_tasks: + - name: "Set backup_version with timestamp if not provided" + ansible.builtin.set_fact: + backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: backup_version is not defined or backup_version == '' + + - name: "Fail if mas_instance_id is not set to" + ansible.builtin.assert: + that: + - mas_instance_id is defined + fail_msg: "mas_instance_id is required and must be set" + + - name: "Fail if mas_backup_dir is not set to" + ansible.builtin.assert: + that: + - mas_backup_dir is defined + fail_msg: "mas_backup_dir is required and must be set" + + - name: "Fail if br_action is not set to backup|restore" + ansible.builtin.assert: + that: + - br_action is defined + - br_action in ["backup", "restore"] + fail_msg: "br_action is required and must be set to 'backup' or 'restore'" + + - name: "Fail if backup_version_to_restore is not set on restore" + ansible.builtin.assert: + that: + - backup_version_to_restore is defined + fail_msg: "backup_version_to_restore is required when running 'restore'" + when: br_action == "restore" + + - name: Check for required DRO environment variables + assert: + that: + # DRO. + - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" + - lookup('env', 'DRO_CONTACT_EMAIL') != "" + - lookup('env', 'DRO_CONTACT_FIRSTNAME') != "" + - lookup('env', 'DRO_CONTACT_LASTNAME') != "" + fail_msg: "One or more required environment variables are not defined for DRO" + when: + - include_dro | bool + - br_action == "restore" + + roles: + - role: ibm.mas_devops.ibm_catalogs + vars: + ibm_catalogs_action: "{{ br_action }}" + catalog_backup_version: "{{ br_action_version }}" + + - role: ibm.mas_devops.cert_manager + vars: + cert_manager_action: "{{ br_action }}" + certmanager_backup_version: "{{ br_action_version }}" + + - role: ibm.mas_devops.mongodb + vars: + mongodb_action: "{{ br_action }}" + mongodb_backup_version: "{{ br_action_version }}" + mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" + mongodb_provider: "community" # only community is supported + + - role: ibm.mas_devops.sls + when: include_sls | bool + vars: + sls_action: "{{ br_action }}" + sls_backup_version: "{{ br_action_version }}" + + - role: ibm.mas_devops.suite_backup + when: br_action == "backup" + vars: + suite_backup_version: "{{ br_action_version }}" + + - role: ibm.mas_devops.dro + when: + - include_dro | bool + - br_action == "restore" + vars: + dro_action: "install" + + - role: ibm.mas_devops.suite_restore + when: br_action == "restore" + vars: + # SLS skiped in this play so include the config in the restore from the backup + include_sls_from_backup: "{{ not (include_sls | bool) }}" + # DRO skiped in this play so include the config in the restore from the backup + include_dro_from_backup: "{{ not (include_dro | bool) }}" + suite_backup_version: "{{ br_action_version }}" + sls_cfg_file: "{{ (mas_config_dir + '/sls.yml') if (include_sls | bool) else omit }}" + dro_cfg_file: "{{ (mas_config_dir + '/dro.yml') if (include_dro | bool) else omit }}" + mas_domain: "{{ mas_domain_on_restore if (mas_domain_on_restore is defined and mas_domain_on_restore != '') else omit }}" + sls_url: "{{ sls_url_on_restore if (sls_url_on_restore is defined and sls_url_on_restore != '') else omit }}" + dro_url: "{{ dro_url_on_restore if (dro_url_on_restore is defined and dro_url_on_restore != '') else omit }}" diff --git a/ibm/mas_devops/plugins/action/backup_resource.py b/ibm/mas_devops/plugins/action/backup_resource.py index 36cf8ce1ba..9deb0e9de0 100644 --- a/ibm/mas_devops/plugins/action/backup_resource.py +++ b/ibm/mas_devops/plugins/action/backup_resource.py @@ -9,7 +9,7 @@ from kubernetes import client, config from kubernetes.dynamic import DynamicClient -from mas.devops.backup import createBackupDirectories, backupResources +from mas.devops.backup import backupResources urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') diff --git a/ibm/mas_devops/plugins/action/restore_mongoce_resources.py b/ibm/mas_devops/plugins/action/restore_mongoce_resources.py index c69b50d396..e42dc46547 100644 --- a/ibm/mas_devops/plugins/action/restore_mongoce_resources.py +++ b/ibm/mas_devops/plugins/action/restore_mongoce_resources.py @@ -10,7 +10,6 @@ from kubernetes.dynamic import DynamicClient from kubernetes.dynamic.exceptions import NotFoundError -import yaml import os from mas.devops.ocp import createNamespace, apply_resource diff --git a/ibm/mas_devops/plugins/action/restore_resource.py b/ibm/mas_devops/plugins/action/restore_resource.py index 954750394a..3a30d8cab9 100644 --- a/ibm/mas_devops/plugins/action/restore_resource.py +++ b/ibm/mas_devops/plugins/action/restore_resource.py @@ -2,7 +2,6 @@ import logging import os -import yaml import urllib3 from ansible.plugins.action import ActionBase diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index feb0c55e72..2c1e79c793 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -15,21 +15,25 @@ class ActionModule(ActionBase): "backup": ["mas_backup_dir"], "restore": ["mas_backup_dir", "certmanager_backup_version"] }, + "db2": { + "backup": ["db2_instance_name", "db2_namespace", "mas_backup_dir", "mas_instance_id"], + "restore_instance": ["mas_backup_dir", "db2_backup_version"], + "restore_database": ["mas_backup_dir", "db2_backup_version", "db2_instance_name", "backup_vendor"], + "s3_setup": ["backup_vendor","backup_s3_alias", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] + }, "grafana": { "backup": ["mas_backup_dir"] }, - "sls": { - "backup": ["mas_backup_dir", "sls_namespace", "sls_instance_name"] - }, "mongodb": { "backup": ["mongodb_instance_name", "mas_backup_dir", "mas_instance_id"], # mongodb_instance_name has a default value "restore": ["mas_backup_dir", "mongodb_backup_version"] }, - "db2": { - "backup": ["db2_instance_name", "db2_namespace", "mas_backup_dir", "mas_instance_id"], - "restore_instance": ["mas_backup_dir", "db2_backup_version"], - "restore_database": ["mas_backup_dir", "db2_backup_version", "db2_instance_name", "backup_vendor"], - "s3_setup": ["backup_vendor","backup_s3_alias", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] + "sls": { + "backup": ["mas_backup_dir", "sls_namespace", "sls_instance_name"] + }, + "suite": { + "backup": ["mas_instance_id", "mas_backup_dir"], + "restore": ["mas_instance_id", "mas_backup_dir", "suite_backup_version"] } } diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml index e7200d8318..495fa54e19 100644 --- a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml @@ -73,4 +73,3 @@ - {{ resource.description }} in {{ resource.scope }} {% endfor %} when: backup_result.failed_count > 0 - diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml index 6ae5085506..e8266b2218 100644 --- a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/restore.yml @@ -176,4 +176,3 @@ - {{ resource.description }}: {{ resource.error }} {% endfor %} when: total_failed | int > 0 - diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index 81eb210e4f..48ac80995e 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -145,5 +145,3 @@ _db2_storages: "NO_OVERRIDE" _db2_instance_password: "NO_OVERRIDE" _db2_ldapblueadmin_password: "NO_OVERRIDE" _db2_ldappassword: "NO_OVERRIDE" - - diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml index 03bcec5367..1ed4710de5 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml @@ -101,5 +101,3 @@ - {{ resource.description }} in {{ resource.scope }} {% endfor %} when: backup_result.failed_count > 0 - - diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml index 3f21625a9f..35f9bf6088 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml @@ -250,4 +250,3 @@ - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" - "STATUS : SUCCESS" - "================================================================================" - diff --git a/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml b/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml index 605b839b60..8989c2d8da 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml @@ -12,4 +12,3 @@ - db2_backup_storage_class is defined - db2_logs_storage_class is defined - db2_temp_storage_class is defined - diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml index 035930f1b6..e335608a71 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml @@ -335,4 +335,3 @@ - {{ resource.description }}: {{ resource.error }} {% endfor %} when: total_failed | int > 0 - diff --git a/ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml b/ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml index fb821bb1df..dcb59d28c8 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/retrieve_db2_passwords.yml @@ -61,6 +61,3 @@ - _jdbc_username is defined and _jdbc_username != "" - _jdbc_username != "db2inst1" - jdbcsecret_stat.stat.exists - - - diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml index 7136660f91..a83f11588f 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml @@ -1,4 +1,3 @@ - - name: "Set fact backup_path for the backup script when backup vendor is disk" set_fact: pod_full_backup_path: "/mnt/backup/{{ db2_backup_version }}" diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml index 9da39e0e88..5fe12c5ab3 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml @@ -1,4 +1,3 @@ - - name: "Set fact backup_path for the backup script when backup vendor is disk" set_fact: pod_full_backup_path: "/mnt/backup/{{ db2_backup_version }}" diff --git a/ibm/mas_devops/roles/grafana/tasks/backup/main.yml b/ibm/mas_devops/roles/grafana/tasks/backup/main.yml index 820d586e1a..96c5db7e4b 100644 --- a/ibm/mas_devops/roles/grafana/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/grafana/tasks/backup/main.yml @@ -96,4 +96,3 @@ - {{ resource.description }} in {{ resource.scope }} {% endfor %} when: backup_result.failed_count > 0 - diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml index 2e7eb4473e..39423dc402 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml @@ -72,4 +72,3 @@ - {{ resource.description }} in {{ resource.scope }} {% endfor %} when: backup_result.failed_count > 0 - diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml index ac49f665d5..14f1a1929c 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-database.yml @@ -109,5 +109,3 @@ shell: | oc exec -n {{mongodb_namespace}} {{mongoce_pod_name}} -c mongod -- rm -rf /tmp/masbr/{{ mongodb_backup_version }} register: database_backup_cleanup_output - - diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml index 4358244f78..d759578006 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/restore-database.yml @@ -63,31 +63,31 @@ # Prepare shell scripts for restore and transfer scripts to mongo pod # ----------------------------------------------------------------------------- - block: - - name: Create create-role-user.sh script in local /tmp - ansible.builtin.template: - src: community/backup-restore/create-role-user.sh.j2 - dest: /tmp/create-role-user.sh - mode: '777' - - - name: Create database-restore.sh script in local /tmp - ansible.builtin.template: - src: community/backup-restore/database-restore.sh.j2 - dest: /tmp/database-restore.sh - mode: '777' - - - name: Copy the create-role-user.sh script into the mongodb pod - shell: "oc cp --retries=50 /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" - register: copy_result - retries: 2 - delay: 15 # seconds - until: copy_result.rc == 0 - - - name: Copy the database-restore.sh script into the mongodb pod - shell: "oc cp --retries=50 /tmp/database-restore.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-restore.sh -c mongod" - register: copy_result - retries: 2 - delay: 15 # seconds - until: copy_result.rc == 0 + - name: Create create-role-user.sh script in local /tmp + ansible.builtin.template: + src: community/backup-restore/create-role-user.sh.j2 + dest: /tmp/create-role-user.sh + mode: '777' + + - name: Create database-restore.sh script in local /tmp + ansible.builtin.template: + src: community/backup-restore/database-restore.sh.j2 + dest: /tmp/database-restore.sh + mode: '777' + + - name: Copy the create-role-user.sh script into the mongodb pod + shell: "oc cp --retries=50 /tmp/create-role-user.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/create-role-user.sh -c mongod" + register: copy_result + retries: 2 + delay: 15 # seconds + until: copy_result.rc == 0 + + - name: Copy the database-restore.sh script into the mongodb pod + shell: "oc cp --retries=50 /tmp/database-restore.sh {{ mongodb_namespace }}/{{ mongoce_pod_name }}:/tmp/database-restore.sh -c mongod" + register: copy_result + retries: 2 + delay: 15 # seconds + until: copy_result.rc == 0 # The log file will also be available inside the pod /tmp/create-role-user.log - name: Exec into mongo pod and run create-role-user.sh to setup restore role and user in Mongodb. diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml index 1cb366083c..fae1ffea5c 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml @@ -394,4 +394,3 @@ - {{ resource.description }}: {{ resource.error }} {% endfor %} when: total_failed | int > 0 - diff --git a/ibm/mas_devops/roles/sls/tasks/backup/main.yml b/ibm/mas_devops/roles/sls/tasks/backup/main.yml index f961ac05e0..091332d4f3 100644 --- a/ibm/mas_devops/roles/sls/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/backup/main.yml @@ -100,4 +100,3 @@ fail: msg: "{{ sls_registration_result.msg }}" when: sls_registration_result.failed - diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml index b3e958d33b..b060674ce2 100644 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml @@ -15,4 +15,3 @@ assert: that: mas_app_id is defined and mas_app_id != "" fail_msg: "mas_app_id is required" - diff --git a/ibm/mas_devops/roles/suite_backup/README.md b/ibm/mas_devops/roles/suite_backup/README.md index 3791bc45db..23609825fc 100644 --- a/ibm/mas_devops/roles/suite_backup/README.md +++ b/ibm/mas_devops/roles/suite_backup/README.md @@ -33,3 +33,19 @@ Set version to override the default `YYMMDD-HHMMSS` timestamp version used in th - **Optional** - Default: `YYMMDD-HHMMSS` timestamp. - Environment Variable: `SUITE_BACKUP_VERSION` + +### include_sls +Controls whether to include the Suite SLS (Suite License Service) configuration in the backup archive. If you plan to install a new SLS in any +recovery action then you should set this to `false`. + +- **Optional** +- Default: `true` +- Environment Variable: `INCLUDE_SLS` + +### include_dro +Controls whether to include the Suite DRO configuration in the backup archive. If you plan to install a new DRO in any +recovery action then you should set this to `false`. + +- **Optional** +- Default: `true` +- Environment Variable: `INCLUDE_DRO` diff --git a/ibm/mas_devops/roles/suite_backup/defaults/main.yml b/ibm/mas_devops/roles/suite_backup/defaults/main.yml index 1483756d8b..7a7408afd4 100644 --- a/ibm/mas_devops/roles/suite_backup/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_backup/defaults/main.yml @@ -3,3 +3,9 @@ mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" suite_backup_version: "{{ lookup('env', 'SUITE_BACKUP_VERSION') }}" + +# Include SLS configuration from backup in backup +include_sls: "{{ lookup('env', 'INCLUDE_SLS') | default('true', true) | bool }}" + +# Include DRO (BAS) configuration from backup in backup +include_dro: "{{ lookup('env', 'INCLUDE_DRO') | default('true', true) | bool }}" diff --git a/ibm/mas_devops/roles/suite_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_backup/tasks/main.yml index 7e4c62deeb..f079d9b5cb 100644 --- a/ibm/mas_devops/roles/suite_backup/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_backup/tasks/main.yml @@ -1,15 +1,12 @@ --- # 1. Check mas core backup required variables # ----------------------------------------------------------------------------- -- name: "Fail if mas_instance_id is not provided" - assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - -- name: "Fail if mas_backup_dir is not provided" - assert: - that: mas_backup_dir is defined and mas_backup_dir != "" - fail_msg: "mas_backup_dir is required" +- name: "Verify required variables for suite backup" + ibm.mas_devops.verify_backup_restore_vars: + component: suite + action: backup + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" - name: "Check if SUITE_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: @@ -46,7 +43,97 @@ msg: "Failed to detect MAS cluster issuer" when: mas_cluster_issuer.failed -# 4. Set the backup_resources we want to backup. Note that if the resource doesn't +# 4. Build the config.mas.ibm.com resource list based on include flags +# ----------------------------------------------------------------------------- +- name: "Set fact: base config resources list" + set_fact: + config_resources: + - kind: AppCfg + api_version: config.mas.ibm.com/v1 + - kind: IDPCfg + api_version: config.mas.ibm.com/v1 + - kind: JdbcCfg + api_version: config.mas.ibm.com/v1 + - kind: KafkaCfg + api_version: config.mas.ibm.com/v1 + - kind: MongoCfg + api_version: config.mas.ibm.com/v1 + - kind: ObjectStorageCfg + api_version: config.mas.ibm.com/v1 + - kind: PushNotificationCfg + api_version: config.mas.ibm.com/v1 + - kind: ScimCfg + api_version: config.mas.ibm.com/v1 + - kind: SmtpCfg + api_version: config.mas.ibm.com/v1 + - kind: WatsonStudioCfg + api_version: config.mas.ibm.com/v1 + +- name: "Add BasCfg to config resources if include_dro is true" + set_fact: + config_resources: "{{ config_resources + [{'kind': 'BasCfg', 'api_version': 'config.mas.ibm.com/v1'}] }}" + when: include_dro | bool + +- name: "Add SlsCfg to config resources if include_sls is true" + set_fact: + config_resources: "{{ config_resources + [{'kind': 'SlsCfg', 'api_version': 'config.mas.ibm.com/v1'}] }}" + when: include_sls | bool + +# 5. Build the core namespace resources list +# ----------------------------------------------------------------------------- +- name: "Set fact: mas core namespace resources" + set_fact: + mas_core_namespace_resources: + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ mas_core_namespace }}" + - kind: Secret + api_version: v1 + name: "{{ mas_instance_id }}-credentials-superuser" + # subscription + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: ibm-mas + - kind: OperatorGroup + api_version: operators.coreos.com/v1 + name: operatorgroup + # secrets + - kind: Secret + api_version: v1 + name: ibm-entitlement + - kind: Secret + api_version: v1 + name: "{{ mas_instance_id }}-credentials-superuser" + # public cert in case of manual managed certs + - kind: Secret + api_version: v1 + name: "{{ mas_instance_id }}-cert-public" + # certificates + - kind: Certificate + api_version: cert-manager.io/v1 + labels: + - "mas.ibm.com/instanceId={{ mas_instance_id }}" + # addons.mas.ibm.com + - kind: MVIEdge + api_version: addons.mas.ibm.com/v1 + - kind: ReplicaDB + api_version: addons.mas.ibm.com/v1 + - kind: GenericAddon + api_version: addons.mas.ibm.com/v1 + # core.mas.ibm.com + - kind: Suite + api_version: core.mas.ibm.com/v1 + - kind: Workspace + api_version: core.mas.ibm.com/v1 + # internal.mas.ibm.com + - kind: CoreIDP + api_version: internal.mas.ibm.com/v1 + +- name: "Combine core namespace resources with config resources" + set_fact: + mas_core_namespace_resources: "{{ mas_core_namespace_resources + config_resources }}" + +# 6. Set the backup_resources we want to backup. Note that if the resource doesn't # exist, the backup will still succeed. If a resource contains `secretName` key # then the secret will be backed up as well. # ----------------------------------------------------------------------------- @@ -78,78 +165,9 @@ api_version: cert-manager.io/v1 name: "{{ mas_instance_id }}-cert-public-ca" - namespace: "{{ mas_core_namespace }}" - resources: - - kind: Project - api_version: project.openshift.io/v1 - name: "{{ mas_core_namespace }}" - - kind: Secret - api_version: v1 - name: "{{ mas_instance_id }}-credentials-superuser" - # subscription - - kind: Subscription - api_version: operators.coreos.com/v1alpha1 - name: ibm-mas - - kind: OperatorGroup - api_version: operators.coreos.com/v1 - name: operatorgroup - # secrets - - kind: Secret - api_version: v1 - name: ibm-entitlement - - kind: Secret - api_version: v1 - name: "{{ mas_instance_id }}-credentials-superuser" - # public cert in case of manual managed certs - - kind: Secret - api_version: v1 - name: "{{ mas_instance_id }}-cert-public" - # certificates - - kind: Certificate - api_version: cert-manager.io/v1 - labels: - - "mas.ibm.com/instanceId={{ mas_instance_id }}" - # addons.mas.ibm.com - - kind: MVIEdge - api_version: addons.mas.ibm.com/v1 - - kind: ReplicaDB - api_version: addons.mas.ibm.com/v1 - - kind: GenericAddon - api_version: addons.mas.ibm.com/v1 - # config.mas.ibm.com - - kind: AppCfg - api_version: config.mas.ibm.com/v1 - - kind: BasCfg - api_version: config.mas.ibm.com/v1 - - kind: IDPCfg - api_version: config.mas.ibm.com/v1 - - kind: JdbcCfg - api_version: config.mas.ibm.com/v1 - - kind: KafkaCfg - api_version: config.mas.ibm.com/v1 - - kind: MongoCfg - api_version: config.mas.ibm.com/v1 - - kind: ObjectStorageCfg - api_version: config.mas.ibm.com/v1 - - kind: PushNotificationCfg - api_version: config.mas.ibm.com/v1 - - kind: ScimCfg - api_version: config.mas.ibm.com/v1 - - kind: SlsCfg - api_version: config.mas.ibm.com/v1 - - kind: SmtpCfg - api_version: config.mas.ibm.com/v1 - - kind: WatsonStudioCfg - api_version: config.mas.ibm.com/v1 - # core.mas.ibm.com - - kind: Suite - api_version: core.mas.ibm.com/v1 - - kind: Workspace - api_version: core.mas.ibm.com/v1 - # internal.mas.ibm.com - - kind: CoreIDP - api_version: internal.mas.ibm.com/v1 - -# 5. Call the backup_resources plugin to execute the backup to the path provided + resources: "{{ mas_core_namespace_resources }}" + +# 7. Call the backup_resources plugin to execute the backup to the path provided # ----------------------------------------------------------------------------- - name: "Backup MAS Suite resources (referenced secrets are auto-discovered)" ibm.mas_devops.backup_resource: @@ -157,7 +175,7 @@ backup_path: "{{ suite_backup_path }}" register: backup_result -# 6. Show the results +# 8. Show the results # ----------------------------------------------------------------------------- - name: "Display backup results" debug: @@ -169,7 +187,7 @@ - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" - "Backup location: {{ suite_backup_path }}" -# 7. Fail task if any errors occurred. +# 9. Fail task if any errors occurred. # ----------------------------------------------------------------------------- - name: "Display failed resources" debug: diff --git a/ibm/mas_devops/roles/suite_restore/README.md b/ibm/mas_devops/roles/suite_restore/README.md index fdf62c6ed2..b0f32173b6 100644 --- a/ibm/mas_devops/roles/suite_restore/README.md +++ b/ibm/mas_devops/roles/suite_restore/README.md @@ -40,23 +40,57 @@ in the restore ### mas_domain The domain to use for the MAS Suite instance. If not provided, the domain from the backup will be used. -- Optional +- **Optional** - Environment Variable: `MAS_DOMAIN` - Default: `NO_OVERRIDE` (uses value from backup) - Example: `mydomain.example.com` +### include_sls_from_backup +Controls whether to restore the Suite SLS (Suite License Service) configuration from the backup archive. + +- **Optional** +- Default: `true` +- Environment Variable: `INCLUDE_SLS_FROM_BACKUP` + ### sls_url -The URL for the Suite License Service (SLS). If not provided, the URL from the backup will be used. +If `include_sls_from_backup` is true. The URL for the Suite License Service (SLS). If not provided, the URL from the backup will be used. +This is used when the domain has changed for SLS but SLS was restored from a backup and so the regristration key is the same. -- Optional +- **Optional** - Environment Variable: `SLS_URL` - Default: `NO_OVERRIDE` (uses value from backup) - Example: `https://sls.example.com` +### sls_cfg_file +If `include_sls_from_backup` is false. Path to the file containing external SLS configuration YAML file to apply. +This is used when you want to use SLS configuration from outside the backup archive (e.g., from a separate SLS role execution). + +- **Optional** +- Environment Variable: `SLS_CFG_FILE` +- Default: None +- Example: `/tmp/sls_config/sls.yml` + +### include_dro_from_backup +Controls whether to restore the Suite DRO configuration from the backup archive. + +- **Optional** +- Default: `true` +- Environment Variable: `INCLUDE_DRO_FROM_BACKUP` + ### bas_url -The URL for the Behavior Analytics Service (BAS). If not provided, the URL from the backup will be used. +If `include_dro_from_backup` is true. The URL for the Behavior Analytics Service (BAS). If not provided, the URL from the backup will be used. +This is used when the domain has changed for DRO but DRO was restored from a backup and so the api key is the same. -- Optional +- **Optional** - Environment Variable: `BAS_URL` - Default: `NO_OVERRIDE` (uses value from backup) - Example: `https://bas.example.com` + +### dro_cfg_file +If `include_dro_from_backup` is false. Path to the file containing external DRO (BAS) configuration YAML file to apply. +This is used when you want to use DRO configuration from outside the backup archive (e.g., from a separate DRO role execution). + +- **Optional** +- Environment Variable: `DRO_CFG_FILE` +- Default: None +- Example: `/tmp/dro_config/dro.yml` diff --git a/ibm/mas_devops/roles/suite_restore/defaults/main.yml b/ibm/mas_devops/roles/suite_restore/defaults/main.yml index c44c12d7a5..a5ce7a0b4a 100644 --- a/ibm/mas_devops/roles/suite_restore/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_restore/defaults/main.yml @@ -8,3 +8,15 @@ suite_backup_version: "{{ lookup('env', 'SUITE_BACKUP_VERSION') }}" mas_domain: "{{ lookup('env', 'MAS_DOMAIN') | default('NO_OVERRIDE', true) }}" sls_url: "{{ lookup('env', 'SLS_URL') | default('NO_OVERRIDE', true) }}" bas_url: "{{ lookup('env', 'BAS_URL') | default('NO_OVERRIDE', true) }}" + +# Include SLS configuration from backup in restore +include_sls_from_backup: "{{ lookup('env', 'INCLUDE_SLS_FROM_BACKUP') | default('true', true) | bool }}" + +# Path to external SLS config file (used when include_sls_from_backup is false) +sls_cfg_file: "{{ lookup('env', 'SLS_CFG_FILE') | default('', true) }}" + +# Include DRO (BAS) configuration from backup in restore +include_dro_from_backup: "{{ lookup('env', 'INCLUDE_DRO_FROM_BACKUP') | default('true', true) | bool }}" + +# Path to external DRO config file (used when include_dro_from_backup is false) +dro_cfg_file: "{{ lookup('env', 'DRO_CFG_FILE') | default('', true) }}" diff --git a/ibm/mas_devops/roles/suite_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_restore/tasks/main.yml index 2ce997e685..46ee241428 100644 --- a/ibm/mas_devops/roles/suite_restore/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_restore/tasks/main.yml @@ -1,20 +1,14 @@ --- # 1. Check mas core restore required variables # ----------------------------------------------------------------------------- -- name: "Fail if mas_instance_id is not provided" - ansible.builtin.assert: - that: mas_instance_id is defined and mas_instance_id != "" - fail_msg: "mas_instance_id is required" - -- name: "Fail if mas_backup_dir is not provided" - ansible.builtin.assert: - that: mas_backup_dir is defined and mas_backup_dir != "" - fail_msg: "mas_backup_dir is required" - -- name: "Fail if suite_backup_version is not provided" - ansible.builtin.assert: - that: suite_backup_version is defined and suite_backup_version != "" - fail_msg: "suite_backup_version is required" +- name: "Verify required variables for suite restore" + ibm.mas_devops.verify_backup_restore_vars: + component: suite + action: restore + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + suite_backup_version: "{{ suite_backup_version }}" + - name: "Set fact: mas core namespace name" set_fact: mas_core_namespace: "mas-{{ mas_instance_id }}-core" @@ -185,13 +179,11 @@ {{ addons_result.failed_count }} failed when: projects_result.success -# Step 6: Restore MAS Config resources (config.mas.ibm.com) -- name: "Restore MAS Config resources" - ibm.mas_devops.restore_resource: - backup_path: "{{ suite_backup_path }}" - resource_kinds: +# Step 6: Build MAS Config resource kinds list based on include flags +- name: "Set fact: base config resource kinds" + set_fact: + config_resource_kinds: - AppCfg - - BasCfg - IDPCfg - JdbcCfg - KafkaCfg @@ -199,14 +191,80 @@ - ObjectStorageCfg - PushNotificationCfg - ScimCfg - - SlsCfg - SmtpCfg - WatsonStudioCfg - override_values: - SlsCfg: - - spec.config.url: "{{ sls_url }}" - BasCfg: - - spec.config.url: "{{ bas_url }}" + config_override_values: {} + +- name: "Add BasCfg to restore if include_dro_from_backup is true" + block: + - name: "Add BasCfg to config resource kinds" + set_fact: + config_resource_kinds: "{{ config_resource_kinds + ['BasCfg'] }}" + + - name: "Add BasCfg override values" + set_fact: + config_override_values: "{{ config_override_values | combine({'BasCfg': [{'spec.config.url': bas_url}]}) }}" + when: include_dro_from_backup | bool + +- name: "Apply external DRO config if include_dro_from_backup is false and dro_cfg_file is provided" + block: + - name: "Verify dro_cfg_file exists" + stat: + path: "{{ dro_cfg_file }}" + register: dro_cfg_file_stat + + - name: "Fail if dro_cfg_file does not exist" + fail: + msg: "DRO config file not found at: {{ dro_cfg_file }}" + when: not dro_cfg_file_stat.stat.exists or not dro_cfg_file_stat.stat.isreg + + - name: "Apply DRO config file" + kubernetes.core.k8s: + state: present + src: "{{ dro_cfg_file }}" + when: + - not (include_dro_from_backup | bool) + - dro_cfg_file is defined + - dro_cfg_file != "" + +- name: "Add SlsCfg to restore if include_sls_from_backup is true" + block: + - name: "Add SlsCfg to config resource kinds" + set_fact: + config_resource_kinds: "{{ config_resource_kinds + ['SlsCfg'] }}" + + - name: "Add SlsCfg override values" + set_fact: + config_override_values: "{{ config_override_values | combine({'SlsCfg': [{'spec.config.url': sls_url}]}) }}" + when: include_sls_from_backup | bool + +- name: "Apply external SLS config if include_sls_from_backup is false and sls_cfg_file is provided" + block: + - name: "Verify sls_cfg_file exists" + stat: + path: "{{ sls_cfg_file }}" + register: sls_cfg_file_stat + + - name: "Fail if sls_cfg_file does not exist" + fail: + msg: "SLS config file not found at: {{ sls_cfg_file }}" + when: not sls_cfg_file_stat.stat.exists or not sls_cfg_file_stat.stat.isreg + + - name: "Apply SLS config file" + kubernetes.core.k8s: + state: present + src: "{{ sls_cfg_file }}" + when: + - not (include_sls_from_backup | bool) + - sls_cfg_file is defined + - sls_cfg_file != "" + +# Step 7: Restore MAS Config resources (config.mas.ibm.com) +- name: "Restore MAS Config resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ suite_backup_path }}" + resource_kinds: "{{ config_resource_kinds }}" + override_values: "{{ config_override_values }}" register: configs_result when: projects_result.success @@ -219,7 +277,7 @@ {{ configs_result.failed_count }} failed when: projects_result.success -# Step 7: Restore MAS Internal resources (internal.mas.ibm.com) +# Step 8: Restore MAS Internal resources (internal.mas.ibm.com) - name: "Restore MAS Internal resources" ibm.mas_devops.restore_resource: backup_path: "{{ suite_backup_path }}" @@ -237,7 +295,7 @@ {{ internal_result.failed_count }} failed when: projects_result.success -# Step 8: Restore Suite +# Step 9: Restore Suite - name: "Restore Suite" ibm.mas_devops.restore_resource: backup_path: "{{ suite_backup_path }}" @@ -285,7 +343,7 @@ suite_result.updated_count > 0 or suite_result.skipped_count > 0 -# Step 9: Restore Workspaces +# Step 10: Restore Workspaces - name: "Restore Workspaces" ibm.mas_devops.restore_resource: backup_path: "{{ suite_backup_path }}" From 2b18faafc4c2bf510d8b8ddf305f8a34974d871d Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 22 Jan 2026 14:03:51 +0000 Subject: [PATCH 26/61] [minor] sls backup and restore (#2076) --- .../action/verify_backup_restore_vars.py | 3 +- ibm/mas_devops/roles/sls/README.md | 249 ++++++++++++++++- ibm/mas_devops/roles/sls/defaults/main.yml | 1 + .../roles/sls/tasks/backup/main.yml | 19 +- ibm/mas_devops/roles/sls/tasks/main.yml | 2 +- .../roles/sls/tasks/restore/main.yml | 254 ++++++++++++++++++ 6 files changed, 520 insertions(+), 8 deletions(-) create mode 100644 ibm/mas_devops/roles/sls/tasks/restore/main.yml diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index 2c1e79c793..7b85e8a90d 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -29,7 +29,8 @@ class ActionModule(ActionBase): "restore": ["mas_backup_dir", "mongodb_backup_version"] }, "sls": { - "backup": ["mas_backup_dir", "sls_namespace", "sls_instance_name"] + "backup": ["mas_backup_dir", "sls_namespace", "sls_instance_name"], + "restore": ["mas_backup_dir", "sls_backup_version"] }, "suite": { "backup": ["mas_instance_id", "mas_backup_dir"], diff --git a/ibm/mas_devops/roles/sls/README.md b/ibm/mas_devops/roles/sls/README.md index 04a49cc1fb..b9e32c61df 100644 --- a/ibm/mas_devops/roles/sls/README.md +++ b/ibm/mas_devops/roles/sls/README.md @@ -15,21 +15,28 @@ Specifies which operation to perform on the Suite License Service (SLS) instance - Environment Variable: `SLS_ACTION` - Default: `install` -**Purpose**: Controls what action the role executes against the SLS instance. This allows the same role to handle installation, configuration generation, and removal of SLS. +**Purpose**: Controls what action the role executes against the SLS instance. This allows the same role to handle installation, configuration generation, backup, restore, and removal of SLS. **When to use**: - Use `install` for initial SLS deployment or updates - Use `gencfg` to generate SLS configuration for MAS without installing SLS (when using existing SLS) +- Use `backup` to create a backup of SLS instance and its configuration +- Use `restore` to restore SLS instance from a backup - Use `uninstall` to remove SLS instance (use with caution) -**Valid values**: `install`, `gencfg`, `uninstall` +**Valid values**: `install`, `gencfg`, `backup`, `restore`, `uninstall` -**Impact**: +**Impact**: - `install`: Deploys or updates SLS operator and instance - `gencfg`: Only generates SLSCfg resource for MAS integration +- `backup`: Creates a backup of all SLS resources, secrets, and registration data +- `restore`: Restores SLS from a previously created backup - `uninstall`: Removes SLS instance and operator (destructive operation) -**Related variables**: When using `gencfg`, requires `sls_url` to be set to point to existing SLS instance. +**Related variables**: +- When using `gencfg`, requires `sls_url` to be set to point to existing SLS instance +- When using `backup`, requires `mas_backup_dir` and optionally `sls_backup_version` +- When using `restore`, requires `mas_backup_dir` and `sls_backup_version` **Note**: Always backup license data before using `uninstall`. The `gencfg` action is useful when SLS is shared across multiple MAS instances. @@ -689,6 +696,240 @@ For examples refer to the [BestEfforts reference configuration in the MAS CLI](h - Environment Variable: `MAS_POD_TEMPLATES_DIR` - Default: None + +## Backup and Restore + +The SLS role supports backup and restore operations to protect your license service configuration and data. This is essential for disaster recovery, migration, and upgrade scenarios. + +### Backup Variables + +#### mas_backup_dir +Directory path where SLS backup files will be stored. + +- **Required** for backup and restore operations +- Environment Variable: `MAS_BACKUP_DIR` +- Default: None + +**Purpose**: Specifies the local filesystem directory where backup archives will be created (for backup) or read from (for restore). This directory serves as the central location for all SLS backup data. + +**When to use**: +- Required when `sls_action` is set to `backup` or `restore` +- Should be a persistent location with sufficient storage space +- Ensure the directory is accessible and has appropriate permissions + +**Valid values**: Any valid local filesystem path (e.g., `/backup/mas`, `/home/user/sls-backups`) + +**Impact**: All backup files and metadata will be stored in subdirectories under this path. The backup creates a timestamped directory structure: `{mas_backup_dir}/backup-{version}-sls/` + +**Related variables**: Works with `sls_backup_version` to create unique backup directories. + +**Note**: Ensure this directory has sufficient space for backup data and is regularly backed up to external storage for disaster recovery. + +#### sls_backup_version +Version identifier for the backup, used to create unique backup directories. + +- **Optional** for backup (auto-generated if not provided) +- **Required** for restore +- Environment Variable: `SLS_BACKUP_VERSION` +- Default: Auto-generated timestamp in format `YYMMDD-HHMMSS` + +**Purpose**: Provides a unique identifier for each backup, allowing multiple backups to coexist and enabling point-in-time restore operations. + +**When to use**: +- For backup: Leave unset to auto-generate a timestamp-based version, or provide a custom identifier +- For restore: Must specify the exact version identifier of the backup to restore + +**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYMMDD-HHMMSS` (e.g., `260122-131500`) + +**Impact**: +- For backup: Creates directory `{mas_backup_dir}/backup-{version}-sls/` +- For restore: Looks for backup in `{mas_backup_dir}/backup-{version}-sls/` + +**Related variables**: Works with `mas_backup_dir` to determine backup location. + +**Note**: When restoring, you must know the exact backup version identifier. List the contents of `mas_backup_dir` to see available backups. + +### What Gets Backed Up + +The SLS backup operation captures all critical resources needed to restore a complete SLS instance: + +**Kubernetes Resources:** +- **Project/Namespace**: The SLS namespace configuration +- **Secrets**: + - IBM entitlement credentials (`ibm-entitlement`) + - MongoDB connection credentials (`ibm-sls-mongo-credentials`) + - Bootstrap configuration (`{instance-name}-bootstrap`) + - License entitlement data (`ibm-sls-{instance-name}-entitlement`) + - All referenced secrets (auto-discovered) +- **ConfigMaps**: Suite registration configuration +- **Operator Resources**: + - Subscription (`ibm-sls`) + - OperatorGroup +- **SLS Custom Resources**: + - LicenseService CR (the main SLS instance) +- **Certificate Manager Resources**: + - Issuers (self-signed and CA issuers) + - Certificates (CA certificate) + +**Registration Data:** +- License ID +- Registration key +- Suite registration information + +### Backup Process + +The backup operation performs the following steps: + +1. **Validation**: Verifies required variables (`mas_backup_dir`, `sls_namespace`, `sls_instance_name`) +2. **Version Generation**: Creates or uses provided backup version identifier +3. **Resource Discovery**: Identifies all SLS resources and auto-discovers referenced secrets +4. **Backup Execution**: Exports all resources to YAML files in the backup directory +5. **Registration Capture**: Saves SLS registration details for restore +6. **Verification**: Reports backup statistics and any failures + +**Backup Directory Structure:** +``` +{mas_backup_dir}/ +└── backup-{version}-sls/ + ├── resources/ + │ ├── projects/ + │ ├── secrets/ + │ ├── configmaps/ + │ ├── subscriptions/ + │ ├── operatorgroups/ + │ ├── licenseservices/ + │ ├── issuers/ + │ └── certificates/ + └── sls-registration.yaml +``` + +### Restore Process + +The restore operation performs the following steps: + +1. **Validation**: Verifies required variables and backup existence +2. **Backup Verification**: Checks for valid backup structure and required files +3. **Certificate Manager Check**: Ensures cert-manager is installed in the cluster +4. **Sequential Restoration**: + - Projects/Namespaces + - Secrets and ConfigMaps + - OperatorGroups + - Subscriptions (triggers operator installation) + - Certificate Manager resources + - Bootstrap secret (recreated from registration data) + - LicenseService CR (with optional domain override) +5. **Verification**: Waits for SLS to become ready and reports restore statistics + +**Important Restore Considerations:** +- The target cluster must have Certificate Manager installed +- MongoDB must be available and accessible (not restored by this role) +- The restore can optionally override the `sls_domain` if deploying to a different cluster +- All secrets and credentials are restored exactly as backed up + +### Backup Example + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + sls_action: backup + mas_backup_dir: /backup/mas + sls_namespace: ibm-sls + sls_instance_name: sls + # Optional: specify custom backup version + # sls_backup_version: "pre-upgrade-backup" + + roles: + - ibm.mas_devops.sls +``` + +**Using environment variables:** +```bash +export SLS_ACTION=backup +export MAS_BACKUP_DIR=/backup/mas +export SLS_NAMESPACE=ibm-sls +export SLS_INSTANCE_NAME=sls +# Optional: export SLS_BACKUP_VERSION=pre-upgrade-backup + +ansible-playbook ibm.mas_devops.run_role +``` + +### Restore Example + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + sls_action: restore + mas_backup_dir: /backup/mas + sls_backup_version: "260122-131500" + # Optional: override domain for different cluster + # sls_domain: sls.newcluster.example.com + + roles: + - ibm.mas_devops.sls +``` + +**Using environment variables:** +```bash +export SLS_ACTION=restore +export MAS_BACKUP_DIR=/backup/mas +export SLS_BACKUP_VERSION=260122-131500 +# Optional: export SLS_DOMAIN=sls.newcluster.example.com + +ansible-playbook ibm.mas_devops.run_role +``` + +### Best Practices + +1. **Regular Backups**: Schedule regular backups before: + - SLS upgrades + - License entitlement changes + - Cluster maintenance + - MAS upgrades + +2. **Backup Storage**: + - Store backups in a location separate from the cluster + - Implement backup retention policies + - Test restore procedures regularly + +3. **MongoDB Backup**: + - SLS backup does NOT include MongoDB data + - Ensure MongoDB is backed up separately using the `mongodb` role backup functionality + - Coordinate SLS and MongoDB backups for consistency + +4. **Pre-Restore Checklist**: + - Verify Certificate Manager is installed + - Ensure MongoDB is available and accessible + - Confirm backup version exists and is complete + - Review and adjust `sls_domain` if restoring to different cluster + +5. **Disaster Recovery**: + - Document backup locations and procedures + - Test restore process in non-production environment + - Keep backup version identifiers in a safe location + - Maintain MongoDB connection details separately + +6. **Migration Scenarios**: + - When migrating to a new cluster, restore MongoDB first + - Update `sls_domain` if cluster ingress domain changed + - Verify network connectivity between restored SLS and MAS instances + +### Troubleshooting + +**Backup Issues:** +- **"Required variables not provided"**: Ensure `mas_backup_dir`, `sls_namespace`, and `sls_instance_name` are set +- **"Resource not found"**: Some resources may not exist in your deployment (this is normal) +- **Permission errors**: Ensure write access to `mas_backup_dir` + +**Restore Issues:** +- **"Backup archive not found"**: Verify `sls_backup_version` matches an existing backup directory +- **"Certificate Manager not found"**: Install cert-manager before restoring SLS +- **"LicenseService not ready"**: Check operator logs and ensure MongoDB is accessible +- **Domain mismatch**: Use `sls_domain` variable to override domain for new cluster + +For more information on backup and restore, refer to the [IBM Documentation on Backing up and Restoring SLS](https://www.ibm.com/docs/en/masv-and-l/cd?topic=service-backing-up-restoring). + ## Example Playbook ### Install and generate a configuration [up to SLS 3.6.0] diff --git a/ibm/mas_devops/roles/sls/defaults/main.yml b/ibm/mas_devops/roles/sls/defaults/main.yml index 7b30689e50..3f2358d428 100644 --- a/ibm/mas_devops/roles/sls/defaults/main.yml +++ b/ibm/mas_devops/roles/sls/defaults/main.yml @@ -66,3 +66,4 @@ mas_pod_templates_dir: "{{ lookup('env', 'MAS_POD_TEMPLATES_DIR') | default('', # Backup and restore variables sls_backup_version: "{{ lookup('env', 'SLS_BACKUP_VERSION') }}" mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" + diff --git a/ibm/mas_devops/roles/sls/tasks/backup/main.yml b/ibm/mas_devops/roles/sls/tasks/backup/main.yml index 091332d4f3..8ac84fc3d0 100644 --- a/ibm/mas_devops/roles/sls/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/backup/main.yml @@ -12,15 +12,19 @@ sls_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" when: sls_backup_version is not defined or sls_backup_version == "" or sls_backup_version == "None" -- name: "Set fact: cert-manager backup base directory path" +- name: "Set fact: SLS backup base directory path" set_fact: sls_backup_path: "{{ mas_backup_dir }}/backup-{{ sls_backup_version }}-sls" -- name: "Set fact: cert-manager backup resources" +- name: "Set fact: SLS backup resources" set_fact: sls_backup_resources: - namespace: "{{ sls_namespace }}" resources: + # project + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ sls_namespace }}" # secret - kind: Secret api_version: v1 @@ -50,6 +54,17 @@ - kind: LicenseService api_version: sls.ibm.com/v1 name: "{{ sls_instance_name }}" + # Issuers + - kind: Issuer + api_version: cert-manager.io/v1 + name: "{{ sls_instance_name }}-issuer" + - kind: Issuer + api_version: cert-manager.io/v1 + name: "{{ sls_instance_name }}-ca-issuer" + # Certificates + - kind: Certificate + api_version: cert-manager.io/v1 + name: "{{ sls_instance_name }}-cert-ca" # Call the backup_resources plugin to execute the backup to the path provided # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/sls/tasks/main.yml b/ibm/mas_devops/roles/sls/tasks/main.yml index 3d2f5b4b20..81ebce0653 100644 --- a/ibm/mas_devops/roles/sls/tasks/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/main.yml @@ -7,7 +7,7 @@ - name: Run the specified action ansible.builtin.include_tasks: "tasks/{{ sls_action }}/main.yml" when: - - sls_action in ["install", "uninstall", "backup"] + - sls_action in ["install", "uninstall", "backup", "restore"] - sls_url is not defined or sls_url == "" # TODO: We should take a bigger look at how the "only generate a cfg" mode works diff --git a/ibm/mas_devops/roles/sls/tasks/restore/main.yml b/ibm/mas_devops/roles/sls/tasks/restore/main.yml new file mode 100644 index 0000000000..753d580f54 --- /dev/null +++ b/ibm/mas_devops/roles/sls/tasks/restore/main.yml @@ -0,0 +1,254 @@ +--- +- name: "Fail if require variables for SLS backup are not provided" + ibm.mas_devops.verify_backup_restore_vars: + mas_backup_dir: "{{ mas_backup_dir }}" + sls_backup_version: "{{ sls_backup_version }}" + action: "restore" + component: "sls" + +- name: "Set fact: SLS backup base directory path" + set_fact: + sls_backup_path: "{{ mas_backup_dir }}/backup-{{ sls_backup_version }}-sls" + sls_backup_resource_path: "{{ mas_backup_dir }}/backup-{{ sls_backup_version }}-sls/resources" + +- name: "Check SLS backup resource path exist" + stat: + path: "{{ sls_backup_resource_path }}" + register: resources_backup_path_stat + +- name: "Fail if backup archive does not exist" + fail: + msg: "SLS resources archive not found at: {{ sls_backup_resource_path }}" + when: not resources_backup_path_stat.stat.exists or not resources_backup_path_stat.stat.isdir + +# Verify only one LicenseService instance file is present in backup archive +# ----------------------------------------------------------------------------- +- name: Get files from {{ sls_backup_resource_path }}/licenseservices directory + set_fact: + instance_files: "{{ lookup('fileglob', '{{ sls_backup_resource_path }}/licenseservices/*', wantlist=True) }}" + +- name: Assert exactly one LicenseService CR exists + assert: + that: + - instance_files | length == 1 + fail_msg: "LicenseService Directory must contain exactly one file" + +- name: Set fact LicenseService cr + set_fact: + sls_cr_cfg: "{{ lookup('file', '{{ instance_files[0] }}') | from_yaml }}" + +- name: "Set fact: SLS details" + set_fact: + sls_instance_name: "{{ sls_cr_cfg.metadata.name }}" + sls_namespace: "{{ sls_cr_cfg.metadata.namespace }}" + +# Verify sls-registration.yaml exists in backup archive +# ----------------------------------------------------------------------------- +- name: "Check sls-registration.yaml exists in {{ sls_backup_path }}" + stat: + path: "{{ sls_backup_path }}/sls-registration.yaml" + register: registrationfile_stat + +- name: "Fail if sls-registration.yaml does not exist" + fail: + msg: "sls-registration.yaml not found at: {{ sls_backup_path }}" + when: not registrationfile_stat.stat.exists + +- name: "SLS restore information" + debug: + msg: + - "Backup Version ................. {{ sls_backup_version }}" + - "Backup Path .................... {{ sls_backup_path }}" + - "SLS Instance Name .............. {{ sls_instance_name }}" + - "SLS Namespace .................. {{ sls_namespace }}" + +# Verify cert-manager exists +# ----------------------------------------------------------------------------- +- name: Detect Certificate Manager installation + include_tasks: "{{ role_path }}/../../common_tasks/detect_cert_manager.yml" + +# Restore Projects +# ------------------------------------------------------------------------- +- name: "Restore Projects" + ibm.mas_devops.restore_resource: + backup_path: "{{ sls_backup_path }}" + resource_kinds: + - Project + register: projects_result + +# Restore Secrets and ConfigMaps +- name: "Restore Secrets and ConfigMaps" + ibm.mas_devops.restore_resource: + backup_path: "{{ sls_backup_path }}" + resource_kinds: + - Secret + - ConfigMap + register: secrets_configmaps_result + when: projects_result.success + +# Restore OperatorGroups and Subscriptions +- name: "Restore OperatorGroups" + ibm.mas_devops.restore_resource: + backup_path: "{{ sls_backup_path }}" + resource_kinds: + - OperatorGroup + register: operatorgroups_result + when: projects_result.success + +- name: "Restore Subscriptions" + ibm.mas_devops.restore_resource: + backup_path: "{{ sls_backup_path }}" + resource_kinds: + - Subscription + register: subscriptions_result + when: projects_result.success + +# Wait until the LicenseService CRD is available +# ----------------------------------------------------------------------------- +- name: "Wait until the LicenseService CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: "licenseservices.sls.ibm.com" + +# Restore Certificate Manager resources (Issuers, Certificates) +- name: "Restore Certificate Manager resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ sls_backup_path }}" + resource_kinds: + - Issuer + - Certificate + register: certmanager_result + when: projects_result.success + +# Create Bootstrap secret +# ----------------------------------------------------------------------------- +- name: "Load sls-registration.yaml" + include_vars: + file: "{{ sls_backup_path }}/sls-registration.yaml" + name: sls_registration + +# refer: https://www.ibm.com/docs/en/masv-and-l/cd?topic=service-backing-up-restoring +- name: Create SLS Bootstrap secret + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ sls_instance_name }}-bootstrap" + namespace: "{{ sls_namespace }}" + stringData: + licensingId: "{{ (sls_registration.licenseId is defined and sls_registration.licenseId != '') | ternary(sls_registration.licenseId, omit) }}" + registrationKey: "{{ (sls_registration.registrationKey is defined and sls_registration.registrationKey != '') | ternary(sls_registration.registrationKey, omit) }}" + when: projects_result.success + +# Restore SLS resources +# ----------------------------------------------------------------------------- +# Setting NO_OVERRIDE to sls_domain here since its being used by other actions. +- name: "Set NO_OVERRIDE to sls_domain if its not defined" + set_fact: + _sls_domain: "{{ sls_domain | default('NO_OVERRIDE', true) }}" + +- name: "Restore LicenseService" + ibm.mas_devops.restore_resource: + backup_path: "{{ sls_backup_path }}" + resource_kinds: + - LicenseService + override_values: + LicenseService: + - spec.domain: "{{ _sls_domain }}" + - ca.secretName: "{{ sls_instance_name }}-cert-ca" + register: sls_result + when: projects_result.success + +# Verify SLS +# ----------------------------------------------------------------------------- +- name: Verify LicenseService CR + ansible.builtin.include_tasks: "tasks/install/sls-verify.yml" + when: + - projects_result.success + - sls_result is defined and sls_result.success + +# Calculate total results +# ----------------------------------------------------------------------------- +- name: "Calculate total restore results" + set_fact: + total_created: >- + {{ + (projects_result.created_count | default(0)) + + (secrets_configmaps_result.created_count | default(0)) + + (operatorgroups_result.created_count | default(0)) + + (subscriptions_result.created_count | default(0)) + + (certmanager_result.created_count | default(0)) + + (sls_result.created_count | default(0)) + }} + total_updated: >- + {{ + (projects_result.updated_count | default(0)) + + (secrets_configmaps_result.updated_count | default(0)) + + (operatorgroups_result.updated_count | default(0)) + + (subscriptions_result.updated_count | default(0)) + + (certmanager_result.updated_count | default(0)) + + (sls_result.updated_count | default(0)) + }} + total_skipped: >- + {{ + (projects_result.skipped_count | default(0)) + + (secrets_configmaps_result.skipped_count | default(0)) + + (operatorgroups_result.skipped_count | default(0)) + + (subscriptions_result.skipped_count | default(0)) + + (certmanager_result.skipped_count | default(0)) + + (sls_result.skipped_count | default(0)) + }} + total_failed: >- + {{ + (projects_result.failed_count | default(0)) + + (secrets_configmaps_result.failed_count | default(0)) + + (operatorgroups_result.failed_count | default(0)) + + (subscriptions_result.failed_count | default(0)) + + (certmanager_result.failed_count | default(0)) + + (sls_result.failed_count | default(0)) + }} + +- name: "Display total restore results" + debug: + msg: + - >- + Restore completed{{ ' with failures' if total_failed | int > 0 + else ' successfully' }} + - "Total resources created: {{ total_created }}" + - "Total resources updated: {{ total_updated }}" + - "Total resources skipped: {{ total_skipped }}" + - "Total resources failed: {{ total_failed }}" + +# Fail task if any errors occurred +# ----------------------------------------------------------------------------- +- name: "Collect all failed resources" + set_fact: + all_failed_resources: >- + {{ + (projects_result.failed_resources | default([])) + + (secrets_configmaps_result.failed_resources | default([])) + + (operatorgroups_result.failed_resources | default([])) + + (subscriptions_result.failed_resources | default([])) + + (certmanager_result.failed_resources | default([])) + + (sls_result.failed_resources | default([])) + }} + +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ all_failed_resources | to_nice_yaml }}" + when: total_failed | int > 0 + +- name: "Fail if restore had errors" + fail: + msg: | + Restore failed for {{ total_failed }} resource(s): + {% for resource in all_failed_resources %} + - {{ resource.description }}: {{ resource.error }} + {% endfor %} + when: total_failed | int > 0 + + From 5c96b187e4fad7cdc5e4aaf0b6b498c6e6969620 Mon Sep 17 00:00:00 2001 From: whitfiea Date: Thu, 22 Jan 2026 14:04:54 +0000 Subject: [PATCH 27/61] fix lint issues --- ibm/mas_devops/roles/sls/defaults/main.yml | 1 - ibm/mas_devops/roles/sls/tasks/restore/main.yml | 2 -- 2 files changed, 3 deletions(-) diff --git a/ibm/mas_devops/roles/sls/defaults/main.yml b/ibm/mas_devops/roles/sls/defaults/main.yml index 3f2358d428..7b30689e50 100644 --- a/ibm/mas_devops/roles/sls/defaults/main.yml +++ b/ibm/mas_devops/roles/sls/defaults/main.yml @@ -66,4 +66,3 @@ mas_pod_templates_dir: "{{ lookup('env', 'MAS_POD_TEMPLATES_DIR') | default('', # Backup and restore variables sls_backup_version: "{{ lookup('env', 'SLS_BACKUP_VERSION') }}" mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" - diff --git a/ibm/mas_devops/roles/sls/tasks/restore/main.yml b/ibm/mas_devops/roles/sls/tasks/restore/main.yml index 753d580f54..6af345729f 100644 --- a/ibm/mas_devops/roles/sls/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/restore/main.yml @@ -250,5 +250,3 @@ - {{ resource.description }}: {{ resource.error }} {% endfor %} when: total_failed | int > 0 - - From 69d6e9d7f73d36cd88e7c7caf8f6628e731510ce Mon Sep 17 00:00:00 2001 From: Andrew Whitfield Date: Mon, 26 Jan 2026 09:22:47 +0000 Subject: [PATCH 28/61] [skip ee] Add doc updates (#2082) --- build/bin/gen_role_docs.py | 2 - build/scripts/validate_readme.py | 2 - docs/playbooks/backup-restore.md | 259 ++++++++++++++++-- .../plugins/action/backup_resource.py | 1 - .../plugins/action/restore_resource.py | 2 - ibm/mas_devops/roles/cert_manager/README.md | 137 ++++++++- ibm/mas_devops/roles/ibm_catalogs/README.md | 131 +++++++++ ibm/mas_devops/roles/mongodb/README.md | 10 +- ibm/mas_devops/roles/sls/README.md | 14 +- ibm/mas_devops/roles/suite_backup/README.md | 153 ++++++++++- ibm/mas_devops/roles/suite_restore/README.md | 145 +++++++++- 11 files changed, 810 insertions(+), 46 deletions(-) diff --git a/build/bin/gen_role_docs.py b/build/bin/gen_role_docs.py index ad404b7177..1fe3b79da0 100644 --- a/build/bin/gen_role_docs.py +++ b/build/bin/gen_role_docs.py @@ -136,5 +136,3 @@ print(f" ✗ Warning: {src_file} not found, skipping {role}") print(f"Role documentation generation complete!") - -# Made with Bob diff --git a/build/scripts/validate_readme.py b/build/scripts/validate_readme.py index 2599ce9bf5..344e8ad3a6 100644 --- a/build/scripts/validate_readme.py +++ b/build/scripts/validate_readme.py @@ -443,5 +443,3 @@ def main(): if __name__ == '__main__': main() - -# Made with Bob diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 19fb6edc37..9c683b4fb1 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -1,31 +1,256 @@ Backup and Restore =============================================================================== +!!! important + These playbooks are samples to demonstrate how to use the roles in this collection. + + They are **not intended for production use** as-is, they are a starting point for power users to aid in the development of their own Ansible playbooks using the roles in this collection. + + The recommended way to perform backup and restore operations for MAS is to use the [MAS CLI](https://ibm-mas.github.io/cli/), which uses this Ansible Collection to deliver a complete managed lifecycle for your MAS instance. + Overview ------------------------------------------------------------------------------- -MAS Devops Collection includes playbooks for backing up and restoring of the following MAS components and their dependencies: +MAS Devops Collection includes playbooks for backing up and restoring MAS components and their dependencies. The backup and restore operations are designed to protect your MAS installation and enable disaster recovery, cluster migration, and testing scenarios. -- [MongoDB](#backuprestore-for-mongodb) -- [Db2](#backuprestore-for-db2) -- [MAS Core](#backuprestore-for-mas-core) -- [Manage](#backuprestore-for-manage) +**Supported Components:** +- [MAS Core](#mas-core-backup-and-restore) +- MongoDB Community Edition +- Db2 (Coming soon) +- Manage (Coming soon) -# MongoDB Community Edition Backup and Restore Playbook -------------------------------------------------------------------------------- +**Roles Supporting Backup/Restore:** +- [`ibm.mas_devops.cert_manager`](../roles/cert_manager.md) +- [`ibm.mas_devops.db2`](../roles/db2.md) +- [`ibm.mas_devops.ibm_catalogs`](../roles/ibm_catalogs.md) +- [`ibm.mas_devops.mongodb`](../roles/mongodb.md) +- [`ibm.mas_devops.sls`](../roles/sls.md) +- [`ibm.mas_devops.suite_backup`](../roles/suite_backup.md) +- [`ibm.mas_devops.suite_restore`](../roles/suite_restore.md) -Coming soon... +MAS Core Backup and Restore +=============================================================================== -Backup/Restore for Db2 -------------------------------------------------------------------------------- +## Overview +This playbook performs comprehensive backup and restore operations for IBM Maximo Application Suite Core and its dependencies. The playbook can be run against any OCP cluster regardless of its type; whether it's running in IBM Cloud, Azure, AWS, or your local datacenter. -Coming soon... +**Important**: Backup can only be restored to an instance with the same MAS instance ID. -Backup/Restore for MAS Core -------------------------------------------------------------------------------- -Coming soon... +## Playbook Content +The playbook executes the following roles in sequence: -Backup/Restore for Manage -------------------------------------------------------------------------------- -Coming soon... +### Backup Operation +1. [Backup IBM Operator Catalogs](../roles/ibm_catalogs.md) (~1 minute) +2. [Backup Certificate Manager](../roles/cert_manager.md) (~1 minute) +3. [Backup MongoDB Community Edition](../roles/mongodb.md) (~5-30 minutes depending on database size) +4. [Backup Suite License Service](../roles/sls.md) (~2 minutes, optional) +5. [Backup MAS Core](../roles/suite_backup.md) (~5 minutes) + +### Restore Operation +1. [Restore IBM Operator Catalogs](../roles/ibm_catalogs.md) (~2 minutes) +2. [Restore Certificate Manager](../roles/cert_manager.md) (~5 minutes) +3. [Restore MongoDB Community Edition](../roles/mongodb.md) (~10-60 minutes depending on database size) +4. [Restore Suite License Service](../roles/sls.md) (~10 minutes, optional) +5. [Install Data Reporter Operator](../roles/dro.md) (~10 minutes, optional) +6. [Restore MAS Core](../roles/suite_restore.md) (~30 minutes) + +All timings are estimates. See the individual role documentation for more information and full details of all configuration options. + +## Required Environment Variables + +### Common Variables (Backup and Restore) +- `MAS_INSTANCE_ID` - The instance ID of the MAS installation +- `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) +- `BR_ACTION` - Set to `backup` or `restore` + +### Backup-Specific Variables +- `BACKUP_VERSION` - (Optional) Custom version identifier for the backup. If not provided, defaults to timestamp format `YYMMDD-HHMMSS` + +### Restore-Specific Variables +- `BACKUP_VERSION_TO_RESTORE` - (Required) The backup version identifier to restore +- `IBM_ENTITLEMENT_KEY` - Your IBM Entitlement key (required if `INCLUDE_DRO=true`) +- `DRO_CONTACT_EMAIL` - Primary contact email (required if `INCLUDE_DRO=true`) +- `DRO_CONTACT_FIRSTNAME` - Primary contact first name (required if `INCLUDE_DRO=true`) +- `DRO_CONTACT_LASTNAME` - Primary contact last name (required if `INCLUDE_DRO=true`) + +## Optional Environment Variables + +### SLS and DRO Configuration +- `INCLUDE_SLS` - Set to `false` to skip SLS backup/restore (default: `true`) + - Use `false` if SLS is configured externally to the cluster +- `INCLUDE_DRO` - Set to `false` to skip DRO installation on restore (default: `true`) + - Use `false` if DRO is configured externally to the cluster + +### Restore to Different Cluster +When restoring to a different cluster with a different domain: +- `MAS_DOMAIN_ON_RESTORE` - Override the MAS domain for the restored instance +- `SLS_URL_ON_RESTORE` - Override the SLS URL in the SLS Config (when `INCLUDE_SLS=false`) +- `DRO_URL_ON_RESTORE` - Override the DRO URL in the DRO Config (when `INCLUDE_DRO=false`) + +### MongoDB Configuration +- `MONGODB_INSTANCE_NAME` - MongoDB instance name (default: `mas-mongo-ce`) + +### Configuration Directory +- `MAS_CONFIG_DIR` - Directory for SLS/DRO configuration files (required when `INCLUDE_SLS=true` or `INCLUDE_DRO=true` on restore) + +## Usage Examples + +### Backup MAS Core +Create a complete backup of MAS Core and all dependencies: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export BR_ACTION=backup + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_core +``` + +### Backup with Custom Version +Create a backup with a custom version identifier: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export BR_ACTION=backup +export BACKUP_VERSION=pre-upgrade-backup + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_core +``` + +### Backup Without SLS +Create a backup excluding SLS (when using external SLS): + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export BR_ACTION=backup +export INCLUDE_SLS=false + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_core +``` + +### Restore MAS Core +Restore MAS Core from a backup to the same cluster: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export BR_ACTION=restore +export BACKUP_VERSION_TO_RESTORE=260122-131500 + +export IBM_ENTITLEMENT_KEY=xxx +export DRO_CONTACT_EMAIL=user@example.com +export DRO_CONTACT_FIRSTNAME=John +export DRO_CONTACT_LASTNAME=Doe +export MAS_CONFIG_DIR=~/masconfig + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_core +``` + +### Restore to Different Cluster +Restore MAS Core to a different cluster with a different domain: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export BR_ACTION=restore +export BACKUP_VERSION_TO_RESTORE=260122-131500 + +export IBM_ENTITLEMENT_KEY=xxx +export DRO_CONTACT_EMAIL=user@example.com +export DRO_CONTACT_FIRSTNAME=John +export DRO_CONTACT_LASTNAME=Doe +export MAS_CONFIG_DIR=~/masconfig + +# Override domain for new cluster +export MAS_DOMAIN_ON_RESTORE=mas.newcluster.example.com + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_core +``` + +### Restore with External SLS and DRO +Restore MAS Core using external SLS and DRO services: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export BR_ACTION=restore +export BACKUP_VERSION_TO_RESTORE=260122-131500 + +# Skip SLS and DRO installation +export INCLUDE_SLS=false +export INCLUDE_DRO=false + +# Provide external service URLs +export SLS_URL_ON_RESTORE=https://sls.external.example.com +export DRO_URL_ON_RESTORE=https://dro.external.example.com + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_core +``` + +## Important Considerations + +### Prerequisites for Restore +- Target cluster must have sufficient resources (CPU, memory, storage) +- Certificate Manager must be installed (handled by playbook) +- Target cluster must use the same MAS instance ID as the backup +- Backup files must be accessible from the restore environment + +### Backup Best Practices +1. **Regular Schedule**: Perform backups regularly, especially before: + - MAS upgrades + - Configuration changes + - Application installations + - Cluster maintenance +2. **Test Restores**: Periodically test restore procedures in non-production environments +3. **Secure Storage**: Store backups in a secure location separate from the cluster +4. **Retention Policy**: Implement and document backup retention policies +5. **Verify Integrity**: Verify backup integrity after completion + +### Restore Best Practices +1. **Pre-Restore Validation**: + - Verify backup archive exists and is complete + - Confirm target cluster has sufficient resources + - Verify MAS instance ID matches the backup +2. **Dependency Coordination**: + - Ensure all external services (SLS, DRO, databases) are accessible + - Verify network connectivity to external services +3. **Post-Restore Verification**: + - Verify Suite status is Ready + - Verify all Workspaces are Ready + - Test application connectivity + - Test user authentication + +### Storage Requirements +- Ensure sufficient storage in the backup directory +- Plan for at least 2x the database size for MongoDB backups +- Monitor disk space during backup operations +- Backup directory structure: `{mas_backup_dir}/backup-{version}-{component}/` + +### Security Considerations +- Backup files contain sensitive data including credentials and certificates +- Secure backup directory with appropriate permissions (chmod 700 recommended) +- Consider encrypting backups for long-term storage +- Restrict access to backup files to authorized personnel only +- Ensure secure transfer of backup files to restore environment + +!!! tip + If you do not want to set up all the dependencies on your local system, you can run the playbook inside our docker image: `docker run -ti --pull always quay.io/ibmmas/cli` + +## Additional Resources + +For detailed information about individual backup and restore operations, refer to the role documentation: +- [IBM Operator Catalogs Backup/Restore](../roles/ibm_catalogs.md) +- [Certificate Manager Backup/Restore](../roles/cert_manager.md) +- [MongoDB Backup/Restore](../roles/mongodb.md) +- [SLS Backup/Restore](../roles/sls.md) +- [MAS Core Backup](../roles/suite_backup.md) +- [MAS Core Restore](../roles/suite_restore.md) diff --git a/ibm/mas_devops/plugins/action/backup_resource.py b/ibm/mas_devops/plugins/action/backup_resource.py index 9deb0e9de0..c09a1d3e79 100644 --- a/ibm/mas_devops/plugins/action/backup_resource.py +++ b/ibm/mas_devops/plugins/action/backup_resource.py @@ -184,4 +184,3 @@ def run(self, tmp=None, task_vars=None): failed_resources=failed_resources ) -# Made with Bob diff --git a/ibm/mas_devops/plugins/action/restore_resource.py b/ibm/mas_devops/plugins/action/restore_resource.py index 3a30d8cab9..08caadd778 100644 --- a/ibm/mas_devops/plugins/action/restore_resource.py +++ b/ibm/mas_devops/plugins/action/restore_resource.py @@ -430,5 +430,3 @@ def run(self, tmp=None, task_vars=None): failed_count=total_failed, failed_resources=failed_resources ) - -# Made with Bob \ No newline at end of file diff --git a/ibm/mas_devops/roles/cert_manager/README.md b/ibm/mas_devops/roles/cert_manager/README.md index b209ac825e..1db614f013 100644 --- a/ibm/mas_devops/roles/cert_manager/README.md +++ b/ibm/mas_devops/roles/cert_manager/README.md @@ -23,19 +23,154 @@ Specifies which operation to perform on the Certificate Manager operator. **When to use**: - Use `install` (default) for initial deployment or to ensure cert-manager is present - Use `uninstall` to remove cert-manager (use with extreme caution) +- Use `backup` to backup the cert-manager installation resources to an archive +- Use `restore` to restore the cert-manager installation resources from an archive created from the `backup` action - Use `none` to skip cert-manager operations while running broader playbooks -**Valid values**: `install`, `uninstall`, `none` +**Valid values**: `install`, `uninstall`, `backup`, `restore`, `none` **Impact**: - `install`: Deploys Red Hat Certificate Manager Operator to `cert-manager-operator` namespace and creates operand in `cert-manager` namespace - `uninstall`: Removes cert-manager operator and operand (destructive operation) +- `backup`: Stores the resources used for the installation (not certificate or secrets) in an archive location +- `restore`: Restores the resources used for the installation (not certificate or secrets) from an archive location - `none`: Role takes no action **Related variables**: None **Note**: **WARNING** - Certificate Manager is a cluster-wide dependency used by MAS, SLS, and other components. Uninstalling it will break certificate management for all dependent applications. Only use `uninstall` if you are certain no applications depend on it. +### Backup and Restore Variables + +#### mas_backup_dir +Directory path where Certificate Manager backup files will be stored. + +- **Required** for backup and restore operations +- Environment Variable: `MAS_BACKUP_DIR` +- Default: None + +**Purpose**: Specifies the local filesystem directory where backup archives will be created (for backup) or read from (for restore). This directory serves as the central location for all Certificate Manager backup data. + +**When to use**: +- Required when `cert_manager_action` is set to `backup` or `restore` +- Should be a persistent location with sufficient storage space +- Ensure the directory is accessible and has appropriate permissions + +**Valid values**: Any valid local filesystem path (e.g., `/backup/mas`, `/home/user/certmanager-backups`) + +**Impact**: All backup files and metadata will be stored in subdirectories under this path. The backup creates a timestamped directory structure: `{mas_backup_dir}/backup-{version}-certmanager/` + +**Related variables**: Works with `certmanager_backup_version` to create unique backup directories. + +**Note**: Ensure this directory has sufficient space for backup data and is regularly backed up to external storage for disaster recovery. + +#### certmanager_backup_version +Version identifier for the backup, used to create unique backup directories. + +- **Optional** for backup (auto-generated if not provided) +- **Required** for restore +- Environment Variable: `CERTMANAGER_BACKUP_VERSION` +- Default: Auto-generated timestamp in format `YYMMDD-HHMMSS` + +**Purpose**: Provides a unique identifier for each backup, allowing multiple backups to coexist and enabling point-in-time restore operations. + +**When to use**: +- For backup: Leave unset to auto-generate a timestamp-based version, or provide a custom identifier +- For restore: Must specify the exact version identifier of the backup to restore + +**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYMMDD-HHMMSS` (e.g., `260122-131500`) + +**Impact**: +- For backup: Creates directory `{mas_backup_dir}/backup-{version}-certmanager/` +- For restore: Looks for backup in `{mas_backup_dir}/backup-{version}-certmanager/` + +**Related variables**: Works with `mas_backup_dir` to determine backup location. + +**Note**: When restoring, you must know the exact backup version identifier. List the contents of `mas_backup_dir` to see available backups. + +## Backup and Restore Operations +------------------------------------------------------------------------------- + +This section provides comprehensive information about Certificate Manager backup and restore operations. + +### Action Comparison + +| Action | Purpose | Instance Resources | Prerequisites | Use Case | +|--------|---------|-------------------|---------------|----------| +| `backup` | Create backup | Yes (operator and operand resources) | Running Certificate Manager instance | Regular backups, disaster recovery preparation | +| `restore` | Full restore | Yes (recreates operator and operand) | Backup archive | Disaster recovery, cluster migration, complete restoration | + +### Backup Process + +The Certificate Manager backup operation creates a backup of your Certificate Manager installation resources: + +1. **Operator Resources**: Backs up Kubernetes resources including: + - Projects/Namespaces (`cert-manager-operator` and `cert-manager`) + - Subscription (`openshift-cert-manager-operator`) + - OperatorGroup +2. **Auto-discovered Secrets**: Any secrets referenced by the backed-up resources are automatically discovered and included + +**Note**: The backup does NOT include individual certificates or secrets created by Certificate Manager for applications. Those are backed up as part of the specific service (e.g., MongoDB, SLS) that uses them. + +**Backup Directory Structure:** +``` +{mas_backup_dir}/ +└── backup-{version}-certmanager/ + └── resources/ + ├── projects/ + ├── subscriptions/ + ├── operatorgroups/ + └── secrets/ +``` + +### Restore Process + +The Certificate Manager restore operation performs a complete restoration of the Certificate Manager operator and operand: + +**Steps:** +1. Validates backup files and required variables +2. Restores Projects/Namespaces +3. Restores OperatorGroups +4. Restores Subscriptions (triggers operator installation) +5. Waits for cert-manager-operator-controller-manager deployment to be ready (up to 30 minutes) +6. Waits for CertManager cluster Custom Resource to be created (up to 5 minutes) +7. Waits for cert-manager-webhook deployment to be ready (up to 30 minutes) + +**When to use:** +- Disaster recovery scenarios +- Migrating Certificate Manager to a new cluster +- Recreating a deleted Certificate Manager instance +- Setting up a new environment from backup + +### Important Considerations + +**Version Compatibility:** +- Target Certificate Manager version should match the backup version +- Version upgrades should be performed separately, not during restore +- The restore process validates version compatibility before proceeding + +**Storage Requirements:** +- Ensure sufficient storage in the backup directory +- Backup directory structure: `{mas_backup_dir}/backup-{version}-certmanager/` +- Monitor disk space during backup operations + +**Security:** +- Backup files contain operator configuration and auto-discovered secrets +- Secure backup directory with appropriate permissions (chmod 700 recommended) +- Consider encrypting backups for long-term storage +- Restrict access to backup files to authorized personnel only + +### Backup and Restore Best Practices + +1. **Regular Backups**: Schedule automated backups at regular intervals, especially before upgrades +2. **Test Restores**: Periodically test restore procedures in non-production environments +3. **Monitor Operations**: Implement monitoring and alerting for backup failures +4. **Backup Validation**: Verify backup integrity after completion +5. **Retention Policy**: Implement and document backup retention policies +6. **Disaster Recovery**: Include Certificate Manager backup/restore in your DR plan +7. **Coordinate with Services**: Coordinate Certificate Manager backups with dependent service backups + + ## Example Playbook After installing the Ansible Collection you can include this role in your own custom playbooks. diff --git a/ibm/mas_devops/roles/ibm_catalogs/README.md b/ibm/mas_devops/roles/ibm_catalogs/README.md index b58774c95e..7ff93b106b 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/README.md +++ b/ibm/mas_devops/roles/ibm_catalogs/README.md @@ -83,6 +83,137 @@ Artifactory API token for accessing pre-release development catalogs (IBM employ **Note**: **IBM EMPLOYEES ONLY** - This is for pre-release testing only. Never use development catalogs in production. Keep this token secure and do not commit to source control. Generate tokens from IBM Artifactory. +### Backup and Restore Variables + +#### mas_backup_dir +Directory path where IBM Operator Catalog backup files will be stored. + +- **Required** for backup and restore operations +- Environment Variable: `MAS_BACKUP_DIR` +- Default: None + +**Purpose**: Specifies the local filesystem directory where backup archives will be created (for backup) or read from (for restore). This directory serves as the central location for all IBM Operator Catalog backup data. + +**When to use**: +- Required when `ibm_catalogs_action` is set to `backup` or `restore` +- Should be a persistent location with sufficient storage space +- Ensure the directory is accessible and has appropriate permissions + +**Valid values**: Any valid local filesystem path (e.g., `/backup/mas`, `/home/user/catalog-backups`) + +**Impact**: All backup files and metadata will be stored in subdirectories under this path. The backup creates a timestamped directory structure: `{mas_backup_dir}/backup-{version}-catalog/` + +**Related variables**: Works with `catalog_backup_version` to create unique backup directories. + +**Note**: Ensure this directory has sufficient space for backup data and is regularly backed up to external storage for disaster recovery. + +#### catalog_backup_version +Version identifier for the backup, used to create unique backup directories. + +- **Optional** for backup (auto-generated if not provided) +- **Required** for restore +- Environment Variable: `CATALOG_BACKUP_VERSION` +- Default: Auto-generated timestamp in format `YYMMDD-HHMMSS` + +**Purpose**: Provides a unique identifier for each backup, allowing multiple backups to coexist and enabling point-in-time restore operations. + +**When to use**: +- For backup: Leave unset to auto-generate a timestamp-based version, or provide a custom identifier +- For restore: Must specify the exact version identifier of the backup to restore + +**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYMMDD-HHMMSS` (e.g., `260122-131500`) + +**Impact**: +- For backup: Creates directory `{mas_backup_dir}/backup-{version}-catalog/` +- For restore: Looks for backup in `{mas_backup_dir}/backup-{version}-catalog/` + +**Related variables**: Works with `mas_backup_dir` to determine backup location. + +**Note**: When restoring, you must know the exact backup version identifier. List the contents of `mas_backup_dir` to see available backups. + +Backup and Restore Operations +------------------------------------------------------------------------------- + +This section provides comprehensive information about IBM Operator Catalog backup and restore operations. + +### Action Comparison + +| Action | Purpose | Instance Resources | Prerequisites | Use Case | +|--------|---------|-------------------|---------------|----------| +| `backup` | Create backup | Yes (catalog and related resources) | Running IBM Operator Catalog | Regular backups, disaster recovery preparation | +| `restore` | Full restore | Yes (recreates catalog and related resources) | Backup archive | Disaster recovery, cluster migration, complete restoration | + +### Backup Process + +The IBM Operator Catalog backup operation creates a backup of your catalog installation resources: + +1. **Catalog Resources**: Backs up Kubernetes resources including: + - CatalogSource (`ibm-operator-catalog`) + - Secrets (for development catalogs: `wiotp-docker-local`) + - ServiceAccounts (`ibm-operator-catalog`, `default`) +2. **Auto-discovered Secrets**: Any secrets referenced by the backed-up resources are automatically discovered and included + +**Note**: The backup includes development catalog credentials if they were configured during installation. + +**Backup Directory Structure:** +``` +{mas_backup_dir}/ +└── backup-{version}-catalog/ + └── resources/ + ├── catalogsources/ + ├── secrets/ + └── serviceaccounts/ +``` + +### Restore Process + +The IBM Operator Catalog restore operation performs a complete restoration of the catalog: + +**Steps:** +1. Validates backup files and required variables +2. Restores Secrets (or creates new `wiotp-docker-local` secret if `artifactory_username` and `artifactory_token` are provided) +3. Restores ServiceAccounts +4. Restores CatalogSource +5. Waits for CatalogSource to be ready (up to 30 minutes) + +**When to use:** +- Disaster recovery scenarios +- Migrating IBM Operator Catalog to a new cluster +- Recreating a deleted catalog +- Setting up a new environment from backup + +### Important Considerations + +**Version Compatibility:** +- Target catalog version should match the backup version +- The restore process validates version compatibility before proceeding + +**Storage Requirements:** +- Ensure sufficient storage in the backup directory +- Backup directory structure: `{mas_backup_dir}/backup-{version}-catalog/` +- Monitor disk space during backup operations + +**Security:** +- Backup files contain catalog configuration and credentials (for development catalogs) +- Secure backup directory with appropriate permissions (chmod 700 recommended) +- Consider encrypting backups for long-term storage +- Restrict access to backup files to authorized personnel only + +**Development Catalog Credentials:** +- If restoring with new `artifactory_username` and `artifactory_token`, the restore will create a new secret instead of using the backed-up one +- This allows updating credentials during restore if needed + +### Backup and Restore Best Practices + +1. **Regular Backups**: Schedule automated backups at regular intervals, especially before upgrades +2. **Test Restores**: Periodically test restore procedures in non-production environments +3. **Monitor Operations**: Implement monitoring and alerting for backup failures +4. **Backup Validation**: Verify backup integrity after completion +5. **Retention Policy**: Implement and document backup retention policies +6. **Disaster Recovery**: Include IBM Operator Catalog backup/restore in your DR plan +7. **Coordinate with Operators**: Coordinate catalog backups with operator backups that depend on it + + ## Example Playbook After installing the Ansible Collection you can include this role in your own custom playbooks. diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index 642c98a62e..d29d144df8 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -550,12 +550,13 @@ Confirmation flag to upgrade MongoDB from version 7 to version 8. - Environment Variable: `MONGODB_V8_UPGRADE` - Default Value: `false` -Role Variables - Backup and Restore (CE Operator) - (authored by IBM Bob) +Role Variables - Backup and Restore (CE Operator) ------------------------------------------------------------------------------- ### mongodb_action For backup and restore operations, set `mongodb_action` to one of the following: -- `backup`: Create a backup of MongoDB databases and optionally instance resources +- `backup`: Create a backup of MongoDB databases and instance resources +- `backup_database`: Create a backup of MongoDB databases only - `restore`: Restore both MongoDB instance resources and databases from a backup (full restore) - `restore_database`: Restore only MongoDB databases from a backup to an existing instance (database-only restore) @@ -563,7 +564,8 @@ For backup and restore operations, set `mongodb_action` to one of the following: - Default Value: `install` **Action Details:** -- **backup**: Creates a complete backup including database data and optionally Kubernetes resources (secrets, certificates, CRs) +- **backup**: Creates a complete backup including database data and Kubernetes resources (secrets, certificates, CRs) +- **backup_database**: Creates a complete backup including database data only - **restore**: Performs a full restore by recreating the MongoDB instance from backup resources and then restoring database data - **restore_database**: Restores only the database data to an already running MongoDB instance without touching instance resources @@ -601,7 +603,7 @@ Optional. Specific MAS application ID for targeted backup/restore operations. - Default Value: None -Backup and Restore Operations - (authored by IBM Bob) +Backup and Restore Operations ------------------------------------------------------------------------------- This section provides comprehensive information about MongoDB backup and restore operations for the Community Edition (CE) operator. diff --git a/ibm/mas_devops/roles/sls/README.md b/ibm/mas_devops/roles/sls/README.md index b9e32c61df..f93931f0d5 100644 --- a/ibm/mas_devops/roles/sls/README.md +++ b/ibm/mas_devops/roles/sls/README.md @@ -696,13 +696,8 @@ For examples refer to the [BestEfforts reference configuration in the MAS CLI](h - Environment Variable: `MAS_POD_TEMPLATES_DIR` - Default: None - -## Backup and Restore - -The SLS role supports backup and restore operations to protect your license service configuration and data. This is essential for disaster recovery, migration, and upgrade scenarios. - -### Backup Variables - +Role Variables - Backup and Restore Variables +------------------------------------------------------------------------------- #### mas_backup_dir Directory path where SLS backup files will be stored. @@ -749,6 +744,11 @@ Version identifier for the backup, used to create unique backup directories. **Note**: When restoring, you must know the exact backup version identifier. List the contents of `mas_backup_dir` to see available backups. +Backup and Restore Operations +------------------------------------------------------------------------------- + +The SLS role supports backup and restore operations to protect your license service configuration and data. This is essential for disaster recovery, migration, and upgrade scenarios. + ### What Gets Backed Up The SLS backup operation captures all critical resources needed to restore a complete SLS instance: diff --git a/ibm/mas_devops/roles/suite_backup/README.md b/ibm/mas_devops/roles/suite_backup/README.md index 23609825fc..2c7a7aef6a 100644 --- a/ibm/mas_devops/roles/suite_backup/README.md +++ b/ibm/mas_devops/roles/suite_backup/README.md @@ -3,7 +3,8 @@ Backup MAS Core Overview ------------------------------------------------------------------------------- -This role supports backing up MAS Core namespace resources; supports creating on-demand full backups. +This role supports backing up MAS Core namespace resources and supporting resources +in other namespaces; supports creating on-demand full backups. !!! important Backup can only be restored to an instance with the same MAS instance ID. @@ -35,17 +36,159 @@ Set version to override the default `YYMMDD-HHMMSS` timestamp version used in th - Environment Variable: `SUITE_BACKUP_VERSION` ### include_sls -Controls whether to include the Suite SLS (Suite License Service) configuration in the backup archive. If you plan to install a new SLS in any -recovery action then you should set this to `false`. +Controls whether to include the Suite SLS (Suite License Service) configuration in the backup archive. +If you plan to install a new SLS in any recovery action then you should set this to `false`. - **Optional** - Default: `true` - Environment Variable: `INCLUDE_SLS` ### include_dro -Controls whether to include the Suite DRO configuration in the backup archive. If you plan to install a new DRO in any -recovery action then you should set this to `false`. +Controls whether to include the Suite DRO configuration in the backup archive. +If you plan to install a new DRO in any recovery action then you should set this to `false`. - **Optional** - Default: `true` - Environment Variable: `INCLUDE_DRO` + + +## Backup Operations +------------------------------------------------------------------------------- + +This section provides comprehensive information about MAS Core backup operations. + +### Overview + +The MAS Core backup operation creates a comprehensive backup of your MAS Core installation, including all namespace resources and supporting resources in other namespaces. This backup can be restored using the [`suite_restore`](suite_restore.md) role. + +**Important**: Backup can only be restored to an instance with the same MAS instance ID. + +### What Gets Backed Up + +The MAS Core backup operation captures all critical resources needed to restore a complete MAS Core instance: + +**Core Namespace Resources (`mas-{instance-id}-core`):** +- **Projects/Namespaces**: The MAS Core namespace +- **Secrets**: + - Superuser credentials (`{instance-id}-credentials-superuser`) + - IBM entitlement key (`ibm-entitlement`) + - Public certificates (`{instance-id}-cert-public`) + - All auto-discovered secrets referenced by other resources +- **Operator Resources**: + - Subscription (`ibm-mas`) + - OperatorGroup +- **Certificate Manager Resources**: + - Certificates (with label `mas.ibm.com/instanceId={instance-id}`) +- **MAS Addon Resources** (addons.mas.ibm.com): + - MVIEdge + - ReplicaDB + - GenericAddon +- **MAS Core Resources** (core.mas.ibm.com): + - Suite CR + - Workspace CRs +- **MAS Internal Resources** (internal.mas.ibm.com): + - CoreIDP +- **MAS Configuration Resources** (config.mas.ibm.com): + - AppCfg, IDPCfg, JdbcCfg, KafkaCfg, MongoCfg + - ObjectStorageCfg, PushNotificationCfg, ScimCfg + - SmtpCfg, WatsonStudioCfg + - BasCfg (if `include_dro` is true) + - SlsCfg (if `include_sls` is true) + +**Certificate Manager Resources:** +- **ClusterIssuers**: + - Public cluster issuer (detected automatically) + - `mas-{instance-id}-core-internal-issuer` + - `mas-{instance-id}-ca` +- **Issuers** (in cert-manager namespace): + - `mas-{instance-id}-core-internal-ca-issuer` + - `mas-{instance-id}-core-public-ca-issuer` +- **Certificates** (in cert-manager namespace): + - `{instance-id}-cert-internal-ca` + - `{instance-id}-cert-public-ca` + +### Backup Process + +The MAS Core backup operation performs the following steps: + +1. **Validation**: Verifies required variables (`mas_instance_id`, `mas_backup_dir`) +2. **Version Generation**: Creates or uses provided backup version identifier +3. **Certificate Manager Detection**: Detects the Certificate Manager installation and namespace +4. **Cluster Issuer Detection**: Identifies the public cluster issuer in use +5. **Resource Discovery**: Identifies all MAS Core resources and auto-discovers referenced secrets +6. **Backup Execution**: Exports all resources to YAML files in the backup directory +7. **Verification**: Reports backup statistics and any failures + +**Backup Directory Structure:** +``` +{mas_backup_dir}/ +└── backup-{version}-suite/ + └── resources/ + ├── projects/ + ├── secrets/ + ├── configmaps/ + ├── subscriptions/ + ├── operatorgroups/ + ├── clusterissuers/ + ├── issuers/ + ├── certificates/ + ├── mviedges/ + ├── replicadbs/ + ├── genericaddons/ + ├── suites/ + ├── workspaces/ + ├── coreidps/ + ├── appcfgs/ + ├── idpcfgs/ + ├── jdbccfgs/ + ├── kafkacfgs/ + ├── mongocfgs/ + ├── objectstoragecfgs/ + ├── pushnotificationcfgs/ + ├── scimcfgs/ + ├── smtpcfgs/ + ├── watsonstudiocfgs/ + ├── bascfgs/ (if include_dro is true) + └── slscfgs/ (if include_sls is true) +``` + +### Important Considerations + +**Instance ID Requirement:** +- Backup can only be restored to an instance with the same MAS instance ID +- The instance ID is embedded in resource names and cannot be changed during restore + +**Storage Requirements:** +- Ensure sufficient storage in the backup directory +- Backup directory structure: `{mas_backup_dir}/backup-{version}-suite/` +- Monitor disk space during backup operations + +**Security:** +- Backup files contain sensitive data including credentials and certificates +- Secure backup directory with appropriate permissions (chmod 700 recommended) +- Consider encrypting backups for long-term storage +- Restrict access to backup files to authorized personnel only + +**SLS and DRO Configuration:** +- Use `include_sls=false` if you plan to install a new SLS during recovery +- Use `include_dro=false` if you plan to install a new DRO during recovery +- Default is `true` for both, which includes the configuration in the backup + +### Backup Best Practices + +1. **Regular Backups**: Schedule automated backups at regular intervals, especially before: + - MAS upgrades + - Configuration changes + - Application installations + - Cluster maintenance +2. **Test Restores**: Periodically test restore procedures in non-production environments +3. **Monitor Operations**: Implement monitoring and alerting for backup failures +4. **Backup Validation**: Verify backup integrity after completion +5. **Retention Policy**: Implement and document backup retention policies +6. **Disaster Recovery**: Include MAS Core backup/restore in your DR plan +7. **Coordinate Backups**: Coordinate MAS Core backups with: + - Database backups (MongoDB, Db2) + - SLS backups (if using separate SLS) + - DRO backups (if using separate DRO) + - Application-specific backups + diff --git a/ibm/mas_devops/roles/suite_restore/README.md b/ibm/mas_devops/roles/suite_restore/README.md index b0f32173b6..3cdc710d5b 100644 --- a/ibm/mas_devops/roles/suite_restore/README.md +++ b/ibm/mas_devops/roles/suite_restore/README.md @@ -3,8 +3,9 @@ Restore MAS Core Overview ------------------------------------------------------------------------------- -This role supports restoring the MAS Core namespace resources when provided the -backup arhive generated from `suite_backup` role. +This role supports restoring the MAS Core namespace resources and supporting +resources in other namespace when provided the backup archive generated from +`suite_backup` role. !!! important Restore can only be made to the an instance with the same MAS instance ID as the backup. @@ -14,7 +15,8 @@ Role Variables ------------------------------------------------------------------------------- ### mas_instance_id -The instance ID of the Maximo Application Suite installation to restore. +The instance ID of the Maximo Application Suite installation to restore. This +should match the instance ID of the backup. - **Required** - Environment Variable: `MAS_INSTANCE_ID` @@ -24,13 +26,13 @@ The instance ID of the Maximo Application Suite installation to restore. The local directory path where backup files to restore are stored. - **Required** -- Environment Varia` +- Environment Variaable: `MAS_BACKUP_DIR` - Default: None - Example: `/tmp/mas_backups` ### suite_backup_version The version of the backup file located in the `MAS_BACKUP_DIR` to be used -in the restore +in the restore. - **Required** - Default: None @@ -47,6 +49,10 @@ The domain to use for the MAS Suite instance. If not provided, the domain from t ### include_sls_from_backup Controls whether to restore the Suite SLS (Suite License Service) configuration from the backup archive. +This should be used when the registration key stays the same, either due to also restoring the same +SLS service, or you are using a centralized SLS service that has not changed. If you plan to install +and use a new SLS service then set this value to `false` and use the `sls_cfg_file` variable to point +to the new SLS configuration file. - **Optional** - Default: `true` @@ -72,6 +78,9 @@ This is used when you want to use SLS configuration from outside the backup arch ### include_dro_from_backup Controls whether to restore the Suite DRO configuration from the backup archive. +This should be used when the DRO details stay the same as you are using a centralized DRO service that has not changed. +If you plan to install and use a new DRO service then set this value to `false` and use the `dro_cfg_file` variable to point +to the new DRO configuration file. - **Optional** - Default: `true` @@ -94,3 +103,129 @@ This is used when you want to use DRO configuration from outside the backup arch - Environment Variable: `DRO_CFG_FILE` - Default: None - Example: `/tmp/dro_config/dro.yml` + + +Restore Operations +------------------------------------------------------------------------------- + +This section provides comprehensive information about MAS Core restore operations. + +### Overview + +The MAS Core restore operation performs a complete restoration of a MAS Core installation from a backup created by the [`suite_backup`](suite_backup.md) role. The restore process recreates all namespace resources and supporting resources in the correct order. + +**Important**: Restore can only be made to an instance with the same MAS instance ID as the backup. + +### Restore Process + +The MAS Core restore operation performs the following steps in sequence: + +1. **Validation**: Verifies required variables and backup archive existence +2. **Certificate Manager Check**: Ensures cert-manager is installed in the cluster +3. **Projects Restoration**: Restores the MAS Core namespace +4. **Secrets and ConfigMaps**: Restores all secrets and configuration maps +5. **Operator Resources**: Restores OperatorGroups and Subscriptions +6. **Subscription Wait**: Waits for subscriptions to be ready (up to 30 minutes) +7. **Certificate Manager Resources**: Restores ClusterIssuers, Issuers, and Certificates +8. **MAS Addon Resources**: Restores MVIEdge, ReplicaDB, and GenericAddon resources +9. **MAS Configuration Resources**: Restores all config.mas.ibm.com resources with optional overrides: + - BasCfg (if `include_dro_from_backup` is true, with optional `bas_url` override) + - SlsCfg (if `include_sls_from_backup` is true, with optional `sls_url` override) + - AppCfg, IDPCfg, JdbcCfg, KafkaCfg, MongoCfg, ObjectStorageCfg, etc. +10. **MAS Internal Resources**: Restores CoreIDP resources +11. **Suite Restoration**: Restores the Suite CR with optional `mas_domain` override +12. **Suite Wait**: Waits for Suite to be ready (up to 60 minutes) +13. **Workspace Restoration**: Restores all Workspace CRs +14. **Workspace Wait**: Waits for all Workspaces to be ready (up to 60 minutes) + +### Configuration Override Options + +The restore process supports several override options to adapt the backup to a new environment: + +**Domain Override:** +- Use `mas_domain` to change the domain when restoring to a different cluster +- Default: Uses the domain from the backup + +**SLS Configuration:** +- If `include_sls_from_backup=true`: Restores SlsCfg from backup + - Use `sls_url` to override the SLS URL if the domain changed +- If `include_sls_from_backup=false`: Use `sls_cfg_file` to provide external SLS configuration + +**DRO Configuration:** +- If `include_dro_from_backup=true`: Restores BasCfg from backup + - Use `bas_url` to override the BAS URL if the domain changed +- If `include_dro_from_backup=false`: Use `dro_cfg_file` to provide external DRO configuration + +### When to Use + +**Full Restore Scenarios:** +- Disaster recovery after cluster failure +- Migrating MAS Core to a new cluster +- Recreating a deleted MAS Core instance +- Setting up a new environment from backup +- Testing backup integrity in non-production + +**Partial Configuration Scenarios:** +- Restoring with new SLS service (set `include_sls_from_backup=false`) +- Restoring with new DRO service (set `include_dro_from_backup=false`) +- Restoring to cluster with different domain (use `mas_domain` override) + +### Important Considerations + +**Prerequisites:** +- Target cluster must have Certificate Manager installed +- Target cluster must have the same MAS instance ID as the backup +- Required dependencies (MongoDB, Db2, etc.) must be available and accessible +- Sufficient cluster resources (CPU, memory, storage) must be available + +**Instance ID Requirement:** +- Restore can only be made to an instance with the same MAS instance ID +- The instance ID is embedded in resource names and cannot be changed + +**Storage Requirements:** +- Ensure backup directory is accessible from the restore environment +- Verify backup archive integrity before starting restore + +**Security:** +- Backup files contain sensitive data including credentials and certificates +- Ensure secure transfer of backup files to restore environment +- Verify backup file permissions and access controls + +**Configuration Dependencies:** +- If using external SLS/DRO configuration files, ensure they are valid and accessible +- Coordinate with database restore operations to ensure data consistency +- Verify network connectivity to external services (SLS, DRO, databases) + +### Restore Best Practices + +1. **Pre-Restore Validation**: + - Verify backup archive exists and is complete + - Confirm Certificate Manager is installed + - Ensure target cluster has sufficient resources + - Verify MAS instance ID matches the backup + +2. **Dependency Coordination**: + - Restore databases (MongoDB, Db2) before MAS Core + - Restore SLS before MAS Core if using separate SLS + - Restore DRO before MAS Core if using separate DRO + - Ensure all external services are accessible + +3. **Configuration Planning**: + - Determine if domain override is needed + - Decide whether to use backup SLS/DRO or new services + - Prepare external configuration files if needed + - Document any configuration changes + +4. **Post-Restore Verification**: + - Verify Suite status is Ready + - Verify all Workspaces are Ready + - Test application connectivity + - Verify database connections + - Test user authentication + +5. **Disaster Recovery**: + - Test restore procedures regularly in non-production + - Document restore procedures and configuration + - Maintain backup version identifiers + - Keep external configuration files secure and accessible + From 30ea28c533cc568b8b4f0e0913128758920c9f0f Mon Sep 17 00:00:00 2001 From: whitfiea Date: Mon, 26 Jan 2026 09:29:26 +0000 Subject: [PATCH 29/61] [skip ee] Update actions confdition --- .github/workflows/ansible.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible.yml b/.github/workflows/ansible.yml index 9266c6987e..55bdf5ad8b 100644 --- a/.github/workflows/ansible.yml +++ b/.github/workflows/ansible.yml @@ -71,7 +71,7 @@ jobs: build-ee: name: Build Ansible Execution Environment runs-on: ubuntu-latest - if: ${{ !contains(github.event.head_commit.message, '[doc]') }} && ${{ !contains(github.event.head_commit.message, '[skip ee]') }} + if: ${{ !contains(github.event.head_commit.message, '[doc]') && !contains(github.event.head_commit.message, '[skip ee]') }} steps: - name: Checkout uses: actions/checkout@v2.3.1 From ef42bb93676b8b7b9a89059363910bc6398a8545 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 26 Jan 2026 12:01:47 +0000 Subject: [PATCH 30/61] [skip ee] include install grafana in br_core (#2085) --- docs/playbooks/backup-restore.md | 34 ++++++- ibm/mas_devops/playbooks/br_core.yml | 10 ++ .../roles/grafana/tasks/backup/main.yml | 98 ------------------- ibm/mas_devops/roles/grafana/tasks/main.yml | 5 +- 4 files changed, 41 insertions(+), 106 deletions(-) delete mode 100644 ibm/mas_devops/roles/grafana/tasks/backup/main.yml diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 9c683b4fb1..48dff3dd63 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -49,10 +49,11 @@ The playbook executes the following roles in sequence: ### Restore Operation 1. [Restore IBM Operator Catalogs](../roles/ibm_catalogs.md) (~2 minutes) 2. [Restore Certificate Manager](../roles/cert_manager.md) (~5 minutes) -3. [Restore MongoDB Community Edition](../roles/mongodb.md) (~10-60 minutes depending on database size) -4. [Restore Suite License Service](../roles/sls.md) (~10 minutes, optional) -5. [Install Data Reporter Operator](../roles/dro.md) (~10 minutes, optional) -6. [Restore MAS Core](../roles/suite_restore.md) (~30 minutes) +3. [Install Grafana](../roles/grafana.md) (~10 minutes, optional) +4. [Restore MongoDB Community Edition](../roles/mongodb.md) (~10-60 minutes depending on database size) +5. [Restore Suite License Service](../roles/sls.md) (~10 minutes, optional) +6. [Install Data Reporter Operator](../roles/dro.md) (~10 minutes, optional) +7. [Restore MAS Core](../roles/suite_restore.md) (~30 minutes) All timings are estimates. See the individual role documentation for more information and full details of all configuration options. @@ -81,6 +82,9 @@ All timings are estimates. See the individual role documentation for more inform - `INCLUDE_DRO` - Set to `false` to skip DRO installation on restore (default: `true`) - Use `false` if DRO is configured externally to the cluster +### GRAFANA Configuration +- `INCLUDE_GRAFANA` - Set to ``false`` to skip Grafana install (default: `true`) + ### Restore to Different Cluster When restoring to a different cluster with a different domain: - `MAS_DOMAIN_ON_RESTORE` - Override the MAS domain for the restored instance @@ -152,6 +156,28 @@ oc login --token=xxxx --server=https://myocpserver ansible-playbook ibm.mas_devops.br_core ``` +### Restore MAS Core without Grafana +Restore MAS Core from a backup without grafana: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export BR_ACTION=restore +export BACKUP_VERSION_TO_RESTORE=260122-131500 + +export INCLUDE_GRAFANA=false + +export IBM_ENTITLEMENT_KEY=xxx +export DRO_CONTACT_EMAIL=user@example.com +export DRO_CONTACT_FIRSTNAME=John +export DRO_CONTACT_LASTNAME=Doe +export MAS_CONFIG_DIR=~/masconfig + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_core +``` + + ### Restore to Different Cluster Restore MAS Core to a different cluster with a different domain: diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml index b59bab9e9d..b2b4b4e70f 100644 --- a/ibm/mas_devops/playbooks/br_core.yml +++ b/ibm/mas_devops/playbooks/br_core.yml @@ -22,6 +22,9 @@ # Computed variable: use backup_version for backup action, backup_version_to_restore for restore action br_action_version: "{{ backup_version if br_action == 'backup' else backup_version_to_restore }}" + # Configurable flags for Grafana. Set to false to skip Grafana + include_grafana: "{{ lookup('env', 'INCLUDE_GRAFANA') | default('true', true) | bool }}" + # Configurable flags for SLS and DRO. You should not include SLS or DRO if theses are configured externally # the the cluster running the Suite. When these values are false then the slscfg and bascfg (for DRO) will # be restored from what is defined in the backup config for the suite. @@ -104,6 +107,13 @@ cert_manager_action: "{{ br_action }}" certmanager_backup_version: "{{ br_action_version }}" + - role: ibm.mas_devops.grafana + when: + - include_grafana | bool + - br_action == "restore" + vars: + grafana_action: "install" + - role: ibm.mas_devops.mongodb vars: mongodb_action: "{{ br_action }}" diff --git a/ibm/mas_devops/roles/grafana/tasks/backup/main.yml b/ibm/mas_devops/roles/grafana/tasks/backup/main.yml deleted file mode 100644 index 96c5db7e4b..0000000000 --- a/ibm/mas_devops/roles/grafana/tasks/backup/main.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- -- name: "Fail if require variables for Grafana backup are not provided" - ibm.mas_devops.verify_backup_restore_vars: - mas_backup_dir: "{{ mas_backup_dir }}" - action: "backup" - component: "grafana" - -- name: "Check if GRAFANA_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" - set_fact: - grafana_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" - when: grafana_backup_version is not defined or grafana_backup_version == "" or grafana_backup_version == "None" - -- name: "Set fact: Grafana backup base directory path" - set_fact: - grafana_backup_path: "{{ mas_backup_dir }}/backup-{{ grafana_backup_version }}-grafana" - -- name: "Set fact: cert-manager backup resources" - set_fact: - grafana_backup_resources: - - namespace: "{{ grafana_v5_namespace }}" - resources: - # Clusterrole - - kind: ClusterRole - api_version: rbac.authorization.k8s.io/v1 - name: grafana-operator - - kind: ClusterRole - api_version: rbac.authorization.k8s.io/v1 - name: prometheus-role - # Clusterrolebinding - - kind: ClusterRoleBinding - api_version: rbac.authorization.k8s.io/v1 - name: prometheus-rolebinding - - kind: ClusterRoleBinding - api_version: rbac.authorization.k8s.io/v1 - name: grafana-operator - # Secret - - kind: Secret - api_version: v1 - name: prometheus-serviceaccount-token - # Service account - - kind: ServiceAccount - api_version: v1 - name: prometheus-serviceaccount - # Subscription - - kind: Subscription - api_version: operators.coreos.com/v1alpha1 - name: grafana-operator - # Grafana CR - - kind: Grafana - api_version: grafana.integreatly.org/v1beta1 - name: mas-grafana - # Dashboard - - kind: GrafanaDashboard - api_version: grafana.integreatly.org/v1beta1 - - namespace: openshift-monitoring - resources: - # Configmap - - kind: ConfigMap - api_version: v1 - name: cluster-monitoring-config - -# Call the backup_resources plugin to execute the backup to the path provided -# ----------------------------------------------------------------------------- -- name: "Backup Grafana resources (referenced secrets are auto-discovered)" - ibm.mas_devops.backup_resource: - backup_resources: "{{ grafana_backup_resources }}" - backup_path: "{{ grafana_backup_path }}" - register: backup_result - -# Show the results -# ----------------------------------------------------------------------------- -- name: "Display backup results" - debug: - msg: - - "Backup completed{{ ' with failures' if backup_result.failed_count > 0 else ' successfully' }}" - - "Total resources backed up: {{ backup_result.backed_up_count }}" - - "Total resources failed: {{ backup_result.failed_count }}" - - "Resources not found: {{ backup_result.not_found_count }}" - - "Secrets auto-discovered: {{ backup_result.discovered_secrets_count }}" - - "Backup location: {{ grafana_backup_path }}" - -# Fail task if any errors occurred. -# ----------------------------------------------------------------------------- -- name: "Display failed resources" - debug: - msg: - - "Failed resources:" - - "{{ backup_result.failed_resources | to_nice_yaml }}" - when: backup_result.failed_count > 0 - -- name: "Fail if backup had errors" - fail: - msg: | - Backup failed for {{ backup_result.failed_count }} resource(s): - {% for resource in backup_result.failed_resources %} - - {{ resource.description }} in {{ resource.scope }} - {% endfor %} - when: backup_result.failed_count > 0 diff --git a/ibm/mas_devops/roles/grafana/tasks/main.yml b/ibm/mas_devops/roles/grafana/tasks/main.yml index cd5b6faee6..cc807a184c 100644 --- a/ibm/mas_devops/roles/grafana/tasks/main.yml +++ b/ibm/mas_devops/roles/grafana/tasks/main.yml @@ -6,8 +6,6 @@ - "Grafana major version................... {{ grafana_major_version }}" - "Grafana v4 namespace ................... {{ grafana_v4_namespace }}" - "Grafana v5 namespace ................... {{ grafana_v5_namespace }}" - when: - - grafana_action in ['install', 'uninstall', 'update'] # 1. Perform the selected action # ----------------------------------------------------------------------------- @@ -15,6 +13,5 @@ # - install # - uninstall # - update -# - backup - include_tasks: "{{ grafana_action }}/main.yml" - when: grafana_action in ['install', 'uninstall', 'update', 'backup'] + when: grafana_action in ['install', 'uninstall', 'update'] From a61a0fa48c519e2b0b421ca8ee960161e8c359ea Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Wed, 28 Jan 2026 08:48:24 +0000 Subject: [PATCH 31/61] [patch] new upload_backup_archive role (#2089) --- ibm/mas_devops/meta/runtime.yml | 3 - .../action/get_mongodb_cr_to_restore.py | 75 ----- .../plugins/action/restore_db2_resources.py | 217 -------------- .../action/restore_mongoce_resources.py | 102 ------- ibm/mas_devops/plugins/action/upload_to_s3.py | 63 ++++ .../plugins/module_utils/backuprestore.py | 78 +---- .../roles/db2/tasks/backup/main.yml | 4 +- .../roles/db2/tasks/backup_database/main.yml | 4 +- .../db2/tasks/install/get-backup-info.yml | 59 ---- .../tasks/providers/community/backup.yml | 2 +- .../providers/community/backup_database.yml | 2 +- .../roles/upload_backup_archive/README.md | 279 ++++++++++++++++++ .../upload_backup_archive/defaults/main.yml | 34 +++ .../roles/upload_backup_archive/meta/main.yml | 19 ++ .../tasks/create_archive.yml | 68 +++++ .../upload_backup_archive/tasks/main.yml | 49 +++ .../tasks/upload_to_artifactory.yml | 69 +++++ .../tasks/upload_to_s3.yml | 34 +++ 18 files changed, 622 insertions(+), 539 deletions(-) delete mode 100644 ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py delete mode 100644 ibm/mas_devops/plugins/action/restore_db2_resources.py delete mode 100644 ibm/mas_devops/plugins/action/restore_mongoce_resources.py create mode 100644 ibm/mas_devops/plugins/action/upload_to_s3.py delete mode 100644 ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml create mode 100644 ibm/mas_devops/roles/upload_backup_archive/README.md create mode 100644 ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml create mode 100644 ibm/mas_devops/roles/upload_backup_archive/meta/main.yml create mode 100644 ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml create mode 100644 ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml create mode 100644 ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml create mode 100644 ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml diff --git a/ibm/mas_devops/meta/runtime.yml b/ibm/mas_devops/meta/runtime.yml index be19d92986..4500be7bed 100644 --- a/ibm/mas_devops/meta/runtime.yml +++ b/ibm/mas_devops/meta/runtime.yml @@ -12,11 +12,8 @@ action_groups: - verify_subscriptions - verify_workloads - verify_storage_class - - backup_mongo_instance - get_mongoce_info - verify_backup_restore_vars - - get_mongodb_cr_to_restore - - restore_mongoce_resources - verify_mongoce_version - wait_for_app_ready - wait_for_conditions diff --git a/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py b/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py deleted file mode 100644 index 732de926d8..0000000000 --- a/ibm/mas_devops/plugins/action/get_mongodb_cr_to_restore.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 - -import logging -import urllib3 - -from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client -from ansible.plugins.action import ActionBase -from ansible.errors import AnsibleError -from ansible.utils.display import Display -from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic.exceptions import NotFoundError - -import yaml -import base64 -import os - -urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - -display = Display() - -def display_information(mongoDBCommunityCR : dict): - display.v(f"MongoCE instance name .......................... {mongoDBCommunityCR['metadata']['name']}") - display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") - display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") - - -class ActionModule(ActionBase): - """ - Usage Example - ------------- - tasks: - - name: "Retrieve and Set facts from MongoDB instance CR and resources" - ibm.mas_devops.get_mongodb_cr_to_restore: - """ - def run(self, tmp=None, task_vars=None): - super(ActionModule, self).run(tmp, task_vars) - - mongodb_resource_path = self._task.args.get('mongodb_resource_path') - mongodb_backup_version = self._task.args.get('mongodb_backup_version') - - if mongodb_resource_path is None or mongodb_resource_path == "": - raise AnsibleError(f"Error: mongodb_resource_path argument was not provided") - - if mongodb_backup_version is None or mongodb_backup_version == "": - raise AnsibleError(f"Error: mongodb_backup_version argument was not provided") - - # check if backup directory exists - display.v(f"Checking if MongoDB backup directory exists at path: {mongodb_resource_path}") - if not os.path.isdir(mongodb_resource_path): - raise AnsibleError(f"Directory {mongodb_resource_path} does NOT exist!") - display.v(f"MongoDB backup directory exists at path: {mongodb_resource_path}") - - # check if cr.yaml exists - mongodb_cr_file = f"{mongodb_resource_path}/cr.yaml" - display.v(f"Checking if MongoDB backup CR file exists at path: {mongodb_cr_file}") - if not os.path.isfile(mongodb_cr_file): - raise AnsibleError(f"MongoDB backup CR file does NOT exist at path: {mongodb_cr_file}") - display.v(f"MongoDB backup CR file exists at path: {mongodb_cr_file}") - - # read cr.yaml - with open(mongodb_cr_file, 'r') as cr_file: - mongodb_cr = yaml.safe_load(cr_file) - display.v("Successfully read MongoDB backup CR file") - - - return dict( - message=f"Successfully set mongodb CR as facts from backup at '{mongodb_resource_path}'", - failed=False, - changed=False, - success=True, - mongodb_cr= mongodb_cr - ) - - diff --git a/ibm/mas_devops/plugins/action/restore_db2_resources.py b/ibm/mas_devops/plugins/action/restore_db2_resources.py deleted file mode 100644 index d31cb03109..0000000000 --- a/ibm/mas_devops/plugins/action/restore_db2_resources.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -import logging -import urllib3 - -from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client -from ansible.plugins.action import ActionBase -from ansible.errors import AnsibleError -from ansible.utils.display import Display - -from ansible_collections.ibm.mas_devops.plugins.module_utils.backuprestore import getDb2VersionFromCR - -import yaml -import os -import base64 - -from mas.devops.ocp import createNamespace, apply_resource - -urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - -display = Display() - -def checkBackupDirectoryExists(db2_backup_path: str): - - if not os.path.exists(db2_backup_path): - raise AnsibleError(f"DB2 Backup path {db2_backup_path} does not exist.") - - db2_resources_path = os.path.join(db2_backup_path, "resources") - - if not os.path.exists(db2_resources_path): - raise AnsibleError(f"Db2 instance resources backup path {db2_resources_path} does not exist.") - - cr_path = os.path.join(db2_resources_path, "cr.yaml") - if not os.path.exists(cr_path): - raise AnsibleError(f"Db2 instance CR backup file {cr_path} does not exist.") - - db2_secrets_path = os.path.join(db2_resources_path, "secrets") - if not os.path.exists(db2_secrets_path): - raise AnsibleError(f"Db2 instance resources secrets backup path {db2_secrets_path} does not exist.") - - return True - - -class ActionModule(ActionBase): - - """ - Usage Example - ------------- - tasks: - - name: "Restore Db2u instance resources (CR, secrets, configmaps, issuers, certificates)" - ibm.mas_devops.restore_db2_instance: - """ - def run(self, tmp=None, task_vars=None): - super(ActionModule, self).run(tmp, task_vars) - - # Initialize DynamicClient and grab the task args - host = self._task.args.get('host', None) - api_key = self._task.args.get('api_key', None) - - dynClient = get_api_client(api_key=api_key, host=host) - - mas_backup_dir = self._task.args.get('mas_backup_dir', None) - db2_backup_version = self._task.args.get('db2_backup_version', None) - - if not mas_backup_dir or mas_backup_dir == '': - raise AnsibleError("mas_backup_dir is a required parameter and cannot be empty") - if not db2_backup_version or db2_backup_version == '': - raise AnsibleError("db2_backup_version is a required parameter and cannot be empty") - - try: - - # Check if backup directory exists - db2_backup_path = os.path.join(mas_backup_dir, f"backup-{db2_backup_version}-db2u") - checkBackupDirectoryExists(db2_backup_path) - display.v(f"- Db2 backup path {db2_backup_path} exists. Proceeding with restore...") - - db2_backup_resource_path = os.path.join(db2_backup_path, "resources") - # Get DB2 instance name from backed up CR - cr_path = os.path.join(db2_backup_resource_path, "cr.yaml") - - # read cr.yaml - with open(cr_path, 'r') as cr_file: - backup_db2u_cr = yaml.safe_load(cr_file) - display.v("- Successfully read DB2 backup CR file") - - db2_info = {} - - db2_instance_name = backup_db2u_cr['metadata']['name'] - db2_namespace = backup_db2u_cr['metadata']['namespace'] - db2_info['db2_instance_name'] = db2_instance_name - db2_info['db2_namespace'] = db2_namespace - - # ======================================================= - # 1. Create DB2 namespace if not exists - # ======================================================= - display.v(f"- Creating Db2 namespace '{db2_namespace}' if it does not already exist") - createNamespace(dynClient, db2_namespace) - - # ======================================================= - # 2. Restore Db2 Secret resources from backup - # ======================================================= - db2_secrets_path = os.path.join(db2_backup_resource_path, "secrets") - display.v(f"- Restoring Db2 Secret resources from backup path '{db2_secrets_path}'") - secret_files = os.listdir(db2_secrets_path) - for secret_file in secret_files: - # Some info files will be kept in secrets folder with NOT_SECRET in the filename - if "NOT_SECRET" in secret_file: - continue - display.v(f"- Restoring Db2 Secret resource from backup file '{secret_file}'") - with open(os.path.join(db2_secrets_path, secret_file), 'r') as f: - secret_yaml = f.read() - apply_resource(dynClient, secret_yaml, db2_namespace) - - # Get instance password to use for post-install restoration - inst_pwd_filepath = os.path.join(db2_secrets_path, f"c-{db2_instance_name}-instancepassword.yaml") - if os.path.exists(inst_pwd_filepath): - with open(inst_pwd_filepath, 'r') as instpwd_file: - instpwd_data = yaml.safe_load(instpwd_file) - if 'data' in instpwd_data and 'password' in instpwd_data['data']: - pwd_encoded = instpwd_data['data']['password'] # base64 encoded - db2_info['db2_instance_password'] = base64.b64decode(pwd_encoded).decode('utf-8') - else: - display.v(f"Unable to find instance password in {inst_pwd_filepath}") - else: - display.v(f"Unable to find instance password file {inst_pwd_filepath}") - - # Get LDAP user info if present - ldap_info_file_path = os.path.join(db2_secrets_path, "ldapuser-NOT_SECRET.yaml") - if os.path.exists(ldap_info_file_path): - with open(ldap_info_file_path, 'r') as ldap_info_file: - ldap_info = yaml.safe_load(ldap_info_file) - db2_info['db2_ldap_username'] = base64.b64decode(ldap_info['db2_ldap_username']).decode('utf-8') - db2_info['db2_ldap_password'] = base64.b64decode(ldap_info['db2_ldap_password']).decode('utf-8') - display.v(f"- Successfully read LDAP user info from {ldap_info_file_path}") - - - # ======================================================= - # 3. Restore DB2 Issuer resources from backup - # ======================================================= - db2_issuers_path = os.path.join(db2_backup_resource_path, "issuers") - display.v(f"- Restoring DB2 Issuer resources from backup path '{db2_issuers_path}'") - issuer_files = os.listdir(db2_issuers_path) - for issuer_file in issuer_files: - with open(os.path.join(db2_issuers_path, issuer_file), 'r') as f: - issuer_yaml = f.read() - apply_resource(dynClient, issuer_yaml, db2_namespace) - - # ======================================================= - # 4. Restore DB2 Certificate resources from backup - # ======================================================= - db2_certs_path = os.path.join(db2_backup_resource_path, "certificates") - display.v(f"- Restoring DB2 Certificate resources from backup path '{db2_certs_path}'") - cert_files = os.listdir(db2_certs_path) - for cert_file in cert_files: - with open(os.path.join(db2_certs_path, cert_file), 'r') as f: - cert_yaml = f.read() - apply_resource(dynClient, cert_yaml, db2_namespace) - - # ======================================================= - # 5. Gather info from backup files to recreate new Db2u instance - # ======================================================= - - db2_info['db2_type'] = backup_db2u_cr['spec'].get('environment', {}).get('dbType', 'db2wh') - # Get DB name from backup CR - db2_info['db2_database_name'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('name', 'BLUDB') - - # Get Db2 configs from backup CR - db2_info['db2_database_db_config'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('dbConfig', {}) - db2_info['db2_instance_dbm_config'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('dbmConfig', {}) - db2_info['db2_instance_registry'] = backup_db2u_cr['spec'].get('environment', {}).get('instance', {}).get('registry', {}) - - # Get dftTableOrg - db2_info['db2_table_org'] = backup_db2u_cr['spec'].get('environment', {}).get('database', {}).get('settings', {}).get('dftTableOrg', 'ROW') - - # Get Pod config - db2_pod_config = backup_db2u_cr['spec'].get('podConfig', {}).get('db2u', {}).get('resource', {}).get('db2u', {}) - db2_info['db2_cpu_requests'] = db2_pod_config.get('requests', {}).get('cpu', None) - db2_info['db2_memory_requests'] = db2_pod_config.get('requests', {}).get('memory', None) - db2_info['db2_cpu_limits'] = db2_pod_config.get('limits', {}).get('cpu', None) - db2_info['db2_memory_limits'] = db2_pod_config.get('limits', {}).get('memory', None) - - # Get timezone if present - db2_info['db2_timezone'] = backup_db2u_cr['spec'].get('advOpts', {}).get('timezone', '') - - # Get number of db2 pods - db2_info['db2_num_pods'] = backup_db2u_cr['spec']['size'] - - # Get DB2 backup info file - db2_info_file_path = os.path.join(db2_backup_resource_path, "db2-backup-info.yaml") - if os.path.exists(db2_info_file_path): - with open(db2_info_file_path, 'r') as info_file: - backup_db2_info = yaml.safe_load(info_file) - db2_info['db2_version'] = backup_db2_info['db2_version'] - db2_info['db2_channel'] = backup_db2_info['db2_channel'] - display.v(f"- Successfully read Db2 backup info from {db2_info_file_path}") - else: - db2_info['db2_version'] = getDb2VersionFromCR(backup_db2u_cr) - if "s11.5.9.0" in db2_info['db2_version']: - db2_info['db2_channel'] = "v110509.0" - else: - display.v("- Warning: Could not find Db2 backup info file. Using default channel for the version from CR.") - - return dict( - message=f"Successfully restored Db2 instance's resources from backup paths.", - failed=False, - changed=False, - success=True, - **db2_info - ) - except Exception as e: - display.v(f"- Failed to restore Db2 instance. Exception occurred: {e}") - return dict( - message=f"Exception occurred while restoring DB2 instance's resources from backup paths. {e}", - failed=True, - changed=False, - success=False - ) diff --git a/ibm/mas_devops/plugins/action/restore_mongoce_resources.py b/ibm/mas_devops/plugins/action/restore_mongoce_resources.py deleted file mode 100644 index e42dc46547..0000000000 --- a/ibm/mas_devops/plugins/action/restore_mongoce_resources.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 - -import logging -import urllib3 - -from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client -from ansible.plugins.action import ActionBase -from ansible.errors import AnsibleError -from ansible.utils.display import Display -from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic.exceptions import NotFoundError - -import os -from mas.devops.ocp import createNamespace, apply_resource - -urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient -logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') - -display = Display() - -def display_information(mongoDBCommunityCR : dict): - display.v(f"MongoCE instance name .......................... {mongoDBCommunityCR['metadata']['name']}") - display.v(f"MongoCE namespace .............................. {mongoDBCommunityCR['metadata']['namespace']}") - display.v(f"MongoDB Version ................................ {mongoDBCommunityCR['spec']['version']}") - -class ActionModule(ActionBase): - """ - Usage Example - ------------- - tasks: - - name: "Restore MongoDB instance resources (secrets, issuers, certificates)" - ibm.mas_devops.restore_mongoce_resources: - """ - def run(self, tmp=None, task_vars=None): - super(ActionModule, self).run(tmp, task_vars) - - # Initialize DynamicClient and grab the task args - host = self._task.args.get('host', None) - api_key = self._task.args.get('api_key', None) - - dynClient = get_api_client(api_key=api_key, host=host) - - mongodb_namespace = self._task.args.get('mongodb_namespace') - mongodb_backup_secrets_path = self._task.args.get('backup_secrets_path') - mongodb_backup_issuers_path = self._task.args.get('backup_issuers_path') - mongodb_backup_certs_path = self._task.args.get('backup_certificates_path') - - if mongodb_namespace is None: - raise AnsibleError(f"Error: mongodb_namespace argument was not provided") - if mongodb_backup_secrets_path is None: - raise AnsibleError(f"Error: mongodb_backup_secrets_path argument was not provided") - if mongodb_backup_issuers_path is None: - raise AnsibleError(f"Error: mongodb_backup_issuers_path argument was not provided") - if mongodb_backup_certs_path is None: - raise AnsibleError(f"Error: mongodb_backup_certs_path argument was not provided") - - # ======================================================= - # 1. Create MongoDB namespace if not exists - # ======================================================= - display.v(f"Creating MongoDB namespace '{mongodb_namespace}' if it does not already exist") - createNamespace(dynClient, mongodb_namespace) - - # ======================================================= - # 2. Restore MongoDB Secret resources from backup - # ======================================================= - display.v(f"Restoring MongoDB Secret resources from backup path '{mongodb_backup_secrets_path}'") - secret_files = os.listdir(mongodb_backup_secrets_path) - for secret_file in secret_files: - with open(os.path.join(mongodb_backup_secrets_path, secret_file), 'r') as f: - secret_yaml = f.read() - apply_resource(dynClient, secret_yaml, mongodb_namespace) - - # ======================================================= - # 3. Restore MongoDB Issuer resources from backup - # ======================================================= - display.v(f"Restoring MongoDB Issuer resources from backup path '{mongodb_backup_issuers_path}'") - issuer_files = os.listdir(mongodb_backup_issuers_path) - for issuer_file in issuer_files: - with open(os.path.join(mongodb_backup_issuers_path, issuer_file), 'r') as f: - issuer_yaml = f.read() - apply_resource(dynClient, issuer_yaml, mongodb_namespace) - - # ======================================================= - # 4. Restore MongoDB Certificate resources from backup - # ======================================================= - display.v(f"Restoring MongoDB Certificate resources from backup path '{mongodb_backup_certs_path}'") - cert_files = os.listdir(mongodb_backup_certs_path) - for cert_file in cert_files: - with open(os.path.join(mongodb_backup_certs_path, cert_file), 'r') as f: - cert_yaml = f.read() - apply_resource(dynClient, cert_yaml, mongodb_namespace) - - - return dict( - message=f"Successfully restored MongoDB instance's Secrets, Issuers and Certificates from backup paths.", - failed=False, - changed=False, - success=True, - restored=True - ) - - diff --git a/ibm/mas_devops/plugins/action/upload_to_s3.py b/ibm/mas_devops/plugins/action/upload_to_s3.py new file mode 100644 index 0000000000..c9bd6e87d2 --- /dev/null +++ b/ibm/mas_devops/plugins/action/upload_to_s3.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import logging +import os +import urllib3 +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +from mas.devops.backup import uploadToS3 + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient + +def normalize_endpoint_url(endpoint) -> str|None: + if not endpoint: + return endpoint + if not endpoint.startswith(("http://", "https://")): + return f"https://{endpoint}" + return endpoint + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Upload to S3 location" + ibm.mas_devops.upload_to_s3: + mas_catalog_version: "{{ catalog_tag }}" + fail_if_catalog_does_not_exist: true + register: mas_catalog_metadata + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + file_path = self._task.args.get('file_path', None) + bucket_name = self._task.args.get('bucket_name', None) + object_name = self._task.args.get('object_name', None) + aws_access_key_id = self._task.args.get('aws_access_key_id', None) + aws_secret_access_key = self._task.args.get('aws_secret_access_key', None) + endpoint_url = self._task.args.get('endpoint_url', None) + region_name = self._task.args.get('region_name', None) + + if file_path is None: + raise AnsibleError(f"Error: file_path argument was not provided") + if bucket_name is None: + raise AnsibleError(f"Error: bucket_name argument was not provided") + if object_name is None: + raise AnsibleError(f"Error: object_name argument was not provided") + if aws_access_key_id is None: + raise AnsibleError(f"Error: aws_access_key_id argument was not provided") + if aws_secret_access_key is None: + raise AnsibleError(f"Error: aws_secret_access_key argument was not provided") + + endpoint_url = normalize_endpoint_url(endpoint=endpoint_url) + + upload_status = uploadToS3( + file_path=file_path, bucket_name=bucket_name, object_name=object_name, + endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, region_name=region_name + ) + + return dict( + success=upload_status + ) + diff --git a/ibm/mas_devops/plugins/module_utils/backuprestore.py b/ibm/mas_devops/plugins/module_utils/backuprestore.py index c84c40509e..1b8fc65330 100644 --- a/ibm/mas_devops/plugins/module_utils/backuprestore.py +++ b/ibm/mas_devops/plugins/module_utils/backuprestore.py @@ -6,83 +6,7 @@ from kubernetes.dynamic import DynamicClient from kubernetes.dynamic.exceptions import NotFoundError -from mas.devops.ocp import getCR, getSecret - - -def createBackupDirectories(paths: list) -> bool: - """ - Create backup directories if they do not exist - """ - try: - for path in paths: - os.makedirs(path, exist_ok=True) - return True - except Exception as e: - return False - -def copyContentsToYamlFile(file_path: str, content: dict) -> bool: - """ - Write dictionary content to a YAML file - """ - try: - with open(file_path, 'w') as yaml_file: - yaml.dump(content, yaml_file, default_flow_style=False) - return True - except Exception as e: - return False - -def filterResourceData(data: dict) -> dict: - """ - filter metadata from Resource data and create minimal dict - """ - metadata_fields_to_remove = [ - 'annotations', - 'creationTimestamp', - 'generation', - 'resourceVersion', - 'selfLink', - 'uid', - 'ownerReferences' - 'managedFields' - ] - filteredCopy = data.copy() - if 'metadata' in filteredCopy: - for field in metadata_fields_to_remove: - if field in filteredCopy['metadata']: - del filteredCopy['metadata'][field] - - if 'status' in filteredCopy: - del filteredCopy['status'] - - return filteredCopy - -def backupSecret(dynClient: DynamicClient, namespace: str, secret_name: str, backup_path: str) -> bool: - """ - Backup a Secret to a YAML file - """ - secret = getSecret(dynClient, namespace, secret_name) - if secret: - secret_file_path = f"{backup_path}/{secret_name}.yaml" - filtered_secret = filterResourceData(secret) - if copyContentsToYamlFile(secret_file_path, filtered_secret): - return True - else: - return False - else: - return False - -def getSubscription(dynClient: DynamicClient, namespace: str, package_name: str): - """ - Retrieve Subscription resource by package name in the specified namespace - """ - try: - subscription_resource = dynClient.resources.get(api_version="operators.coreos.com/v1alpha1", kind="Subscription") - subscription = subscription_resource.get(namespace=namespace, name=package_name) - return subscription.to_dict() - except NotFoundError: - return None - except Exception as e: - return None +from mas.devops.ocp import getCR def getDb2VersionFromCR(db2uCR: dict) -> str: """ diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 3504676756..c7bbab3aac 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -4,7 +4,7 @@ - name: Verify DB2 backup variables ibm.mas_devops.verify_backup_restore_vars: component: "db2" - action: "{{ db2_action }}" + action: "backup" db2_instance_name: "{{ db2_instance_name }}" mas_backup_dir: "{{ mas_backup_dir }}" mas_instance_id: "{{ mas_instance_id }}" @@ -19,7 +19,7 @@ - name: "Set fact: DB2 backup base directory path" set_fact: - db2_backup_path: "{{ mas_backup_dir }}/{{ db2_action }}-{{ db2_backup_version }}-db2u" + db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u" - name: "Create {{ db2_backup_path }} directory for Db2 backup" file: diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml index 80000c5d5c..91be4ffb56 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml @@ -4,7 +4,7 @@ - name: Verify DB2 backup variables ibm.mas_devops.verify_backup_restore_vars: component: "db2" - action: "{{ db2_action }}" + action: "backup" db2_instance_name: "{{ db2_instance_name }}" mas_backup_dir: "{{ mas_backup_dir }}" mas_instance_id: "{{ mas_instance_id }}" @@ -19,7 +19,7 @@ - name: "Set fact: DB2 backup base directory path" set_fact: - db2_backup_path: "{{ mas_backup_dir }}/{{ db2_action }}-{{ db2_backup_version }}-db2u" + db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u" - name: "Create {{ db2_backup_path }} directory for Db2 backup" file: diff --git a/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml b/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml deleted file mode 100644 index 83760797b2..0000000000 --- a/ibm/mas_devops/roles/db2/tasks/install/get-backup-info.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -# Check db2 Restore required variables -# ----------------------------------------------------------------------------- -- name: Verify DB2 restore variables - ibm.mas_devops.verify_backup_restore_vars: - component: "db2" - action: "restore_instance" - db2_backup_version: "{{ db2_backup_version }}" - mas_backup_dir: "{{ mas_backup_dir }}" - -# Restore Db2 Universal operator Instance Kubernetes Resources -# ------------------------------------------------------------------------- -- name: "Start Db2 Universal operator Instance restore process." - ibm.mas_devops.restore_db2_resources: - db2_backup_version: "{{ db2_backup_version }}" - mas_backup_dir: "{{ mas_backup_dir }}" - register: restore_db2_instance_result - -- name: Assert if restoring DB2 instance resources succeeeded - assert: - that: - - restore_db2_instance_result is defined - - restore_db2_instance_result.success is defined - - restore_db2_instance_result.success == true - fail_msg: "Failed to restore DB2 instance resources" - -# Create Db2 Instance using info from backup CR if required -# ------------------------------------------------------------------------- -- name: "Set fact from DB2 backup" - set_fact: - db2_database_db_config: "{{ restore_db2_instance_result.db2_database_db_config }}" - db2_instance_dbm_config: "{{ restore_db2_instance_result.db2_instance_dbm_config }}" - db2_instance_registry: "{{ restore_db2_instance_result.db2_instance_registry }}" - db2_channel: "{{ restore_db2_instance_result.db2_channel }}" - db2_version: "{{ restore_db2_instance_result.db2_version }}" - db2_instance_name: "{{ restore_db2_instance_result.db2_instance_name }}" - db2_namespace: "{{ restore_db2_instance_result.db2_namespace }}" - db2_type: "{{ restore_db2_instance_result.db2_type }}" - db2_dbname: "{{ restore_db2_instance_result.db2_database_name }}" - db2_table_org: "{{ restore_db2_instance_result.db2_table_org }}" - db2_cpu_requests: "{{ restore_db2_instance_result.db2_cpu_requests }}" - db2_memory_requests: "{{ restore_db2_instance_result.db2_memory_requests }}" - db2_cpu_limits: "{{ restore_db2_instance_result.db2_cpu_limits }}" - db2_memory_limits: "{{ restore_db2_instance_result.db2_memory_limits }}" - db2_timezone: "{{ restore_db2_instance_result.db2_timezone }}" - db2_num_pods: "{{ restore_db2_instance_result.db2_num_pods }}" - -- name: Set fact Instance password - no_log: true - set_fact: - db2_instance_password: "{{ restore_db2_instance_result.db2_instance_password }}" - when: restore_db2_instance_result.db2_instance_password is defined - -- name: "Set fact LDAP credentials if exist." - no_log: true - set_fact: - db2_ldap_username: "{{ restore_db2_instance_result.db2_ldap_username }}" - db2_ldap_password: "{{ restore_db2_instance_result.db2_ldap_password }}" - when: restore_db2_instance_result.db2_ldap_username is defined and restore_db2_instance_result.db2_ldap_password is defined diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index 7b384730b5..7414258ef4 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -17,7 +17,7 @@ - name: "Set fact: Create Mongodb backup base directory path" set_fact: - mongodb_backup_path: "{{ mas_backup_dir }}/{{ mongodb_action }}-{{ mongodb_backup_version }}-mongoce" + mongodb_backup_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" - name: "Create {{ mongodb_backup_path }} directory for Mongodb backup" file: diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml index c8e408f59c..4efaff9102 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml @@ -17,7 +17,7 @@ - name: "Set fact: Create Mongodb backup base directory path" set_fact: - mongodb_backup_path: "{{ mas_backup_dir }}/{{ mongodb_action }}-{{ mongodb_backup_version }}-mongoce" + mongodb_backup_path: "{{ mas_backup_dir }}/backup-{{ mongodb_backup_version }}-mongoce" - name: "Create {{ mongodb_backup_path }} directory for Mongodb backup" file: diff --git a/ibm/mas_devops/roles/upload_backup_archive/README.md b/ibm/mas_devops/roles/upload_backup_archive/README.md new file mode 100644 index 0000000000..64b82c6537 --- /dev/null +++ b/ibm/mas_devops/roles/upload_backup_archive/README.md @@ -0,0 +1,279 @@ +# upload_backup_archive +Creates a compressed archive of MAS backup directories and uploads it to AWS S3 or Artifactory. + +This role automates the process of packaging MAS backup directories into a single tar.gz archive and uploading it to a remote storage location. It supports multiple backup components (catalog, cert-manager, SLS, MongoDB, Db2, and MAS Suite) and allows for component-specific backup versions. The role intelligently detects which backup directories exist and only archives those that are present. + +Key features: +- Creates compressed tar.gz archives of MAS backup directories +- Supports uploading to AWS S3 or S3-compatible storage +- Supports uploading to Artifactory repositories +- Handles component-specific backup versions +- Automatic cleanup of temporary files +- Configurable upload timeouts for large archives + +## Prerequisites + +### For S3 Upload +- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) or boto3 Python library must be installed +- `amazon.aws` Ansible collection must be installed +- AWS credentials with S3 write permissions +- S3 bucket must exist and be accessible + +### For Artifactory Upload +- `curl` command-line tool must be installed +- Artifactory API token with upload permissions +- Artifactory repository must exist and be accessible + +## Role Variables + +### Required Variables + +#### mas_backup_dir +Directory containing the MAS backup folders. This is the parent directory where all component backup directories are located. + +- **Required** +- Environment Variable: `MAS_BACKUP_DIR` +- Default Value: None + +#### backup_version +Version identifier for the backup. This is used as the default version for all component backups unless component-specific versions are provided. + +- **Required** +- Environment Variable: `BACKUP_VERSION` +- Default Value: None + +### Component-Specific Backup Versions + +These variables allow you to specify different backup versions for individual components. If not provided, they default to the value of `backup_version`. + +#### catalog_backup_version +Backup version for the catalog component. + +- **Optional** +- Environment Variable: `CATALOG_BACKUP_VERSION` +- Default Value: Value of `backup_version` + +#### certmanager_backup_version +Backup version for the cert-manager component. + +- **Optional** +- Environment Variable: `CERTMANAGER_BACKUP_VERSION` +- Default Value: Value of `backup_version` + +#### mongodb_backup_version +Backup version for the MongoDB component. + +- **Optional** +- Environment Variable: `MONGODB_BACKUP_VERSION` +- Default Value: Value of `backup_version` + +#### sls_backup_version +Backup version for the SLS (Suite License Service) component. + +- **Optional** +- Environment Variable: `SLS_BACKUP_VERSION` +- Default Value: Value of `backup_version` + +#### db2_backup_version +Backup version for the Db2 component. + +- **Optional** +- Environment Variable: `DB2_BACKUP_VERSION` +- Default Value: Value of `backup_version` + +#### suite_backup_version +Backup version for the MAS Suite component. + +- **Optional** +- Environment Variable: `SUITE_BACKUP_VERSION` +- Default Value: Value of `backup_version` + +### S3 Upload Variables + +Provide these variables to upload the backup archive to AWS S3 or S3-compatible storage. If S3 credentials are provided, S3 upload takes precedence over Artifactory. + +#### aws_access_key_id +AWS access key ID for authentication. + +- **Required for S3 upload** +- Environment Variable: `S3_ACCESS_KEY_ID` +- Default Value: None + +#### aws_secret_access_key +AWS secret access key for authentication. + +- **Required for S3 upload** +- Environment Variable: `S3_SECRET_ACCESS_KEY` +- Default Value: None + +#### s3_bucket_name +Name of the S3 bucket where the archive will be uploaded. + +- **Required for S3 upload** +- Environment Variable: `S3_BUCKET_NAME` +- Default Value: None + +#### s3_region +AWS region where the S3 bucket is located. + +- **Optional** +- Environment Variable: `S3_REGION` +- Default Value: `us-east-1` + +#### s3_endpoint_url +Custom S3 endpoint URL for S3-compatible storage services (e.g., MinIO, Wasabi, IBM Cloud Object Storage). + +- **Optional** +- Environment Variable: `S3_ENDPOINT_URL` +- Default Value: None (uses AWS S3 endpoints) + +### Artifactory Upload Variables + +Provide these variables to upload the backup archive to Artifactory. Artifactory upload is used only if S3 credentials are not provided. + +#### artifactory_username +Artifactory username for authentication. + +- **Required for Artifactory upload** +- Environment Variable: `ARTIFACTORY_USERNAME` +- Default Value: None + +#### artifactory_token +Artifactory API token for authentication. + +- **Required for Artifactory upload** +- Environment Variable: `ARTIFACTORY_TOKEN` +- Default Value: None + +#### artifactory_url +Base URL of the Artifactory server (e.g., `https://artifactory.example.com/artifactory`). + +- **Required for Artifactory upload** +- Environment Variable: `ARTIFACTORY_URL` +- Default Value: None + +#### artifactory_repository +Name of the Artifactory repository where the archive will be uploaded. + +- **Required for Artifactory upload** +- Environment Variable: `ARTIFACTORY_REPOSITORY` +- Default Value: None + +### General Configuration + +#### backup_archive_name +Name of the tar.gz archive file that will be created and uploaded. + +- **Optional** +- Environment Variable: None +- Default Value: `mas-backup-{{ backup_version }}.tar.gz` + +#### backup_temp_dir +Temporary directory where the archive will be created before upload. The directory is created if it doesn't exist and cleaned up after upload. + +- **Optional** +- Environment Variable: None +- Default Value: `/tmp/mas-backup-{{ backup_version }}` + +#### upload_timeout +Maximum time in seconds to wait for the upload to complete. Useful for large archives or slow network connections. + +- **Optional** +- Environment Variable: None +- Default Value: `3600` (1 hour) + +## Example Playbook + +### S3 Upload +After installing the Ansible Collection you can include this role in your own custom playbooks. + +```yaml +- hosts: localhost + vars: + mas_backup_dir: /backup/mas + backup_version: "260117-191500" + aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + s3_bucket_name: my-mas-backups + s3_region: us-west-2 + roles: + - ibm.mas_devops.upload_backup_archive +``` + +### Artifactory Upload + +```yaml +- hosts: localhost + vars: + mas_backup_dir: /backup/mas + backup_version: "260117-191500" + artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" + artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + artifactory_url: https://artifactory.example.com/artifactory + artifactory_repository: mas-backups + roles: + - ibm.mas_devops.upload_backup_archive +``` + +### S3-Compatible Storage (IBMcloud, MinIO, Wasabi, etc.) + +```yaml +- hosts: localhost + vars: + mas_backup_dir: /backup/mas + backup_version: "260117-191500" + aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY') }}" + aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_KEY') }}" + s3_bucket_name: mas-backups + s3_region: us-east-1 + s3_endpoint_url: https://s3.example.com + roles: + - ibm.mas_devops.upload_backup_archive +``` + +### Component-Specific Backup Versions + +```yaml +- hosts: localhost + vars: + mas_backup_dir: /backup/mas + backup_version: "260117-191500" + # Override specific component versions + mongodb_backup_version: "260116-120000" + db2_backup_version: "260115-180000" + aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + s3_bucket_name: my-mas-backups + roles: + - ibm.mas_devops.upload_backup_archive +``` + +## Run Role Playbook +After installing the Ansible Collection you can easily run the role standalone using the `run_role` playbook provided. + +### S3 Upload + +```bash +export MAS_BACKUP_DIR=/backup/mas +export BACKUP_VERSION=260117-191500 +export S3_ACCESS_KEY_ID=your_access_key +export S3_SECRET_ACCESS_KEY=your_secret_key +export S3_BUCKET_NAME=my-mas-backups +export S3_REGION=us-west-2 +ROLE_NAME=upload_backup_archive ansible-playbook ibm.mas_devops.run_role +``` + +### Artifactory Upload + +```bash +export MAS_BACKUP_DIR=/backup/mas +export BACKUP_VERSION=260117-191500 +export ARTIFACTORY_USERNAME=your_username +export ARTIFACTORY_TOKEN=your_token +export ARTIFACTORY_URL=https://artifactory.example.com/artifactory +export ARTIFACTORY_REPOSITORY=mas-backups +ROLE_NAME=upload_backup_archive ansible-playbook ibm.mas_devops.run_role +``` + +## License +EPL-2.0 \ No newline at end of file diff --git a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml new file mode 100644 index 0000000000..75ca7235f3 --- /dev/null +++ b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml @@ -0,0 +1,34 @@ +--- +# Required variables + +# Directory containing the backup folders +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" + +# Backup version to use for all component's backup +backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" + +# Specific backup versions for each component (optional - override backup_version) +catalog_backup_version: "{{ lookup('env', 'CATALOG_BACKUP_VERSION') | default (backup_version, true) }}" +certmanager_backup_version: "{{ lookup('env', 'CERTMANAGER_BACKUP_VERSION') | default (backup_version, true) }}" +mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') | default (backup_version, true) }}" +sls_backup_version: "{{ lookup('env', 'SLS_BACKUP_VERSION') | default (backup_version, true) }}" +db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') | default (backup_version, true) }}" +suite_backup_version: "{{ lookup('env', 'SUITE_BACKUP_VERSION') | default (backup_version, true) }}" + +# S3 Configuration (provide these to upload to S3) +aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY_ID') }}" +aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_ACCESS_KEY') }}" +s3_bucket_name: "{{ lookup('env', 'S3_BUCKET_NAME') }}" +s3_region: "{{ lookup('env', 'S3_REGION') }}" +s3_endpoint_url: "{{ lookup('env', 'S3_ENDPOINT_URL') }}" + +# Artifactory Configuration (provide these to upload to Artifactory) +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" +artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" +artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" + +# General settings +backup_archive_name: "mas-backup-{{ backup_version }}.tar.gz" +backup_temp_dir: "/tmp/mas-backup-{{ backup_version }}" +upload_timeout: 3600 # Upload timeout in seconds (1 hour) diff --git a/ibm/mas_devops/roles/upload_backup_archive/meta/main.yml b/ibm/mas_devops/roles/upload_backup_archive/meta/main.yml new file mode 100644 index 0000000000..ebe3aabfe8 --- /dev/null +++ b/ibm/mas_devops/roles/upload_backup_archive/meta/main.yml @@ -0,0 +1,19 @@ +--- +galaxy_info: + author: IBM + description: Upload MAS backup archive to S3 or Artifactory + company: IBM + license: EPL-2.0 + min_ansible_version: "2.9" + platforms: + - name: EL + versions: + - "8" + galaxy_tags: + - ibm + - mas + - backup + - s3 + - artifactory + +dependencies: [] diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml new file mode 100644 index 0000000000..29980f222f --- /dev/null +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml @@ -0,0 +1,68 @@ +--- +# Create tar archive of backup directories +- name: "Set backup directories to archive" + ansible.builtin.set_fact: + backup_directories: + - "backup-{{ catalog_backup_version }}-catalog" + - "backup-{{ certmanager_backup_version }}-certmanager" + - "backup-{{ sls_backup_version }}-sls" + - "backup-{{ mongodb_backup_version }}-mongoce" + - "backup-{{ db2_backup_version }}-db2u" + - "backup-{{ suite_backup_version }}-suite" + +- name: "Verify backup directory exists" + ansible.builtin.stat: + path: "{{ mas_backup_dir }}" + register: backup_dir_stat + +- name: "Fail if backup directory does not exist" + ansible.builtin.fail: + msg: "Backup directory {{ mas_backup_dir }} does not exist" + when: not backup_dir_stat.stat.exists or not backup_dir_stat.stat.isdir + +- name: "Check which backup directories exist" + ansible.builtin.stat: + path: "{{ mas_backup_dir }}/{{ item }}" + register: backup_dirs_stat + loop: "{{ backup_directories }}" + +- name: "Set list of existing backup directories" + ansible.builtin.set_fact: + existing_backup_dirs: "{{ backup_dirs_stat.results | selectattr('stat.exists', 'equalto', true) | map(attribute='item') | list }}" + +- name: "Display existing backup directories" + ansible.builtin.debug: + msg: "Found {{ existing_backup_dirs | length }} backup directories: {{ existing_backup_dirs }}" + +- name: "Fail if no backup directories found" + ansible.builtin.fail: + msg: "No backup directories found in {{ mas_backup_dir }} for version {{ backup_version }}" + when: existing_backup_dirs | length == 0 + +- name: "Create temporary directory for archive" + ansible.builtin.file: + path: "{{ backup_temp_dir }}" + state: directory + mode: '0755' + +- name: "Create tar.gz archive of backup directories" + ansible.builtin.command: + cmd: "tar -czf {{ backup_temp_dir }}/{{ backup_archive_name }} -C {{ mas_backup_dir }} {{ existing_backup_dirs | join(' ') }}" + register: tar_result + changed_when: tar_result.rc == 0 + +- name: "Verify archive was created" + ansible.builtin.stat: + path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + register: archive_stat + +- name: "Fail if archive was not created" + ansible.builtin.fail: + msg: "Failed to create archive {{ backup_temp_dir }}/{{ backup_archive_name }}" + when: not archive_stat.stat.exists + +- name: "Display archive information" + ansible.builtin.debug: + msg: + - "Archive created successfully: {{ backup_temp_dir }}/{{ backup_archive_name }}" + - "Archive size: {{ (archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml new file mode 100644 index 0000000000..7a26701e93 --- /dev/null +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml @@ -0,0 +1,49 @@ +--- +# Validate required variables +- name: "Fail if mas_backup_dir is not defined" + ansible.builtin.fail: + msg: "mas_backup_dir is required but not defined" + when: mas_backup_dir is not defined or mas_backup_dir == '' + +- name: "Fail if backup_version is not defined" + ansible.builtin.fail: + msg: "backup_version is required but not defined" + when: backup_version is not defined or backup_version == '' + +# Determine upload destination +- name: "Check if S3 credentials are provided" + ansible.builtin.set_fact: + upload_to_s3: "{{ (aws_access_key_id is defined and aws_access_key_id != '') and (aws_secret_access_key is defined and aws_secret_access_key != '') and (s3_bucket_name is defined and s3_bucket_name != '') }}" + +- name: "Check if Artifactory credentials are provided" + ansible.builtin.set_fact: + upload_to_artifactory: "{{ (artifactory_username is defined and artifactory_username != '') and (artifactory_token is defined and artifactory_token != '') and (artifactory_url is defined and artifactory_url != '') }}" + +- name: "Fail if neither S3 nor Artifactory credentials are provided" + ansible.builtin.fail: + msg: "Either S3 credentials (aws_access_key_id, aws_secret_access_key, s3_bucket_name) or Artifactory credentials (artifactory_username, artifactory_token, artifactory_url) must be provided" + when: not upload_to_s3 and not upload_to_artifactory + +- name: "Display upload destination" + ansible.builtin.debug: + msg: "Will upload to: {{ 'S3' if upload_to_s3 else 'Artifactory' }}" + +# Create tar archive +- name: "Create backup archive" + ansible.builtin.include_tasks: create_archive.yml + +# Upload to S3 or Artifactory +- name: "Upload to S3" + ansible.builtin.include_tasks: upload_to_s3.yml + when: upload_to_s3 + +- name: "Upload to Artifactory" + ansible.builtin.include_tasks: upload_to_artifactory.yml + when: upload_to_artifactory and not upload_to_s3 + +# Cleanup +- name: "Remove temporary archive file" + ansible.builtin.file: + path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + state: absent + when: backup_temp_dir is defined diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml new file mode 100644 index 0000000000..604d26a664 --- /dev/null +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml @@ -0,0 +1,69 @@ +--- +# Upload backup archive to Artifactory +- name: "Validate Artifactory repository is defined" + ansible.builtin.fail: + msg: "artifactory_repository is required when uploading to Artifactory" + when: artifactory_repository is not defined or artifactory_repository == '' + +- name: "Set Artifactory upload URL" + ansible.builtin.set_fact: + artifactory_upload_url: "{{ artifactory_url }}/{{ artifactory_repository }}/{{ backup_archive_name }}" + +- name: "Display Artifactory upload information" + ansible.builtin.debug: + msg: + - "Uploading to Artifactory: {{ artifactory_url }}" + - "Repository: {{ artifactory_repository }}" + - "Archive: {{ backup_archive_name }}" + - "Full URL: {{ artifactory_upload_url }}" + +- name: "Get archive file stats" + ansible.builtin.stat: + path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + register: archive_file_stat + +- name: "Upload archive to Artifactory using curl" + ansible.builtin.command: + cmd: > + curl -X PUT + -u {{ artifactory_username }}:{{ artifactory_token }} + -T {{ backup_temp_dir }}/{{ backup_archive_name }} + {{ artifactory_upload_url }} + --max-time {{ upload_timeout }} + --connect-timeout 60 + --fail + --silent + --show-error + --write-out '%{http_code}' + register: artifactory_upload_result + changed_when: artifactory_upload_result.rc == 0 + failed_when: artifactory_upload_result.rc != 0 + async: "{{ upload_timeout }}" + poll: 10 + no_log: true + +- name: "Check upload HTTP status code" + ansible.builtin.set_fact: + upload_http_code: "{{ artifactory_upload_result.stdout | regex_search('[0-9]{3}$') }}" + when: artifactory_upload_result is succeeded + +- name: "Display Artifactory upload result" + ansible.builtin.debug: + msg: + - "Successfully uploaded {{ backup_archive_name }} to Artifactory" + - "HTTP Status: {{ upload_http_code | default('N/A') }}" + - "Archive size: {{ (archive_file_stat.stat.size / 1024 / 1024) | round(2) }} MB" + when: artifactory_upload_result is succeeded + +- name: "Fail if Artifactory upload failed" + ansible.builtin.fail: + msg: "Failed to upload archive to Artifactory. HTTP Status: {{ upload_http_code | default('Unknown') }}" + when: + - artifactory_upload_result is succeeded + - upload_http_code is defined + - upload_http_code not in ['200', '201'] + +- name: "Handle upload command failure" + ansible.builtin.fail: + msg: "Failed to upload archive to Artifactory: {{ artifactory_upload_result.stderr | default('Unknown error') }}" + when: artifactory_upload_result is failed diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml new file mode 100644 index 0000000000..59685fbb1d --- /dev/null +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml @@ -0,0 +1,34 @@ +--- +# Upload backup archive to S3 +- name: "Set S3 region default" + ansible.builtin.set_fact: + region: "{{ s3_region | default('us-east-1') }}" + +- name: "Display S3 upload information" + ansible.builtin.debug: + msg: + - "Uploading to S3 bucket: {{ s3_bucket_name }}" + - "Region: {{ region }}" + - "Archive: {{ backup_archive_name }}" + +- name: "Upload archive to S3" + ibm.mas_devops.upload_to_s3: + aws_access_key_id: "{{ aws_access_key_id }}" + aws_secret_access_key: "{{ aws_secret_access_key }}" + bucket_name: "{{ s3_bucket_name }}" + object_name: "{{ backup_archive_name }}" + file_path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + region_name: "{{ region }}" + endpoint_url: "{{ s3_endpoint_url | default(omit) }}" + register: s3_upload_result + poll: 10 + +- name: "Display S3 upload result" + ansible.builtin.debug: + msg: "Successfully uploaded {{ backup_archive_name }} to S3 bucket {{ s3_bucket_name }}" + when: s3_upload_result.success | bool + +- name: "Fail if S3 upload failed" + ansible.builtin.fail: + msg: "Failed to upload archive to S3: {{ s3_upload_result.msg | default('Unknown error') }}" + when: not s3_upload_result.success | bool From c75e75d196ca2797751f6c34754c164e963d895d Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 29 Jan 2026 14:20:38 +0000 Subject: [PATCH 32/61] [patch] new download_backup_archive role (#2092) --- .../plugins/action/download_from_s3.py | 63 ++++ .../roles/download_backup_archive/README.md | 272 ++++++++++++++++++ .../download_backup_archive/defaults/main.yml | 28 ++ .../download_backup_archive/meta/main.yml | 20 ++ .../tasks/download_from_artifactory.yml | 78 +++++ .../tasks/download_from_s3.yml | 49 ++++ .../tasks/extract_archive.yml | 43 +++ .../download_backup_archive/tasks/main.yml | 79 +++++ 8 files changed, 632 insertions(+) create mode 100644 ibm/mas_devops/plugins/action/download_from_s3.py create mode 100644 ibm/mas_devops/roles/download_backup_archive/README.md create mode 100644 ibm/mas_devops/roles/download_backup_archive/defaults/main.yml create mode 100644 ibm/mas_devops/roles/download_backup_archive/meta/main.yml create mode 100644 ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml create mode 100644 ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml create mode 100644 ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml create mode 100644 ibm/mas_devops/roles/download_backup_archive/tasks/main.yml diff --git a/ibm/mas_devops/plugins/action/download_from_s3.py b/ibm/mas_devops/plugins/action/download_from_s3.py new file mode 100644 index 0000000000..e46024ba13 --- /dev/null +++ b/ibm/mas_devops/plugins/action/download_from_s3.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +import logging +import os +import urllib3 +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase +from mas.devops.backup import downloadFromS3 + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient + +def normalize_endpoint_url(endpoint) -> str|None: + if not endpoint: + return endpoint + if not endpoint.startswith(("http://", "https://")): + return f"https://{endpoint}" + return endpoint + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Upload to S3 location" + ibm.mas_devops.upload_to_s3: + mas_catalog_version: "{{ catalog_tag }}" + fail_if_catalog_does_not_exist: true + register: mas_catalog_metadata + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + local_dir = self._task.args.get('local_dir', None) + bucket_name = self._task.args.get('bucket_name', None) + object_name = self._task.args.get('object_name', None) + aws_access_key_id = self._task.args.get('aws_access_key_id', None) + aws_secret_access_key = self._task.args.get('aws_secret_access_key', None) + endpoint_url = self._task.args.get('endpoint_url', None) + region_name = self._task.args.get('region_name', None) + + if local_dir is None: + raise AnsibleError(f"Error: local_dir argument was not provided") + if bucket_name is None: + raise AnsibleError(f"Error: bucket_name argument was not provided") + if object_name is None: + raise AnsibleError(f"Error: object_name argument was not provided") + if aws_access_key_id is None: + raise AnsibleError(f"Error: aws_access_key_id argument was not provided") + if aws_secret_access_key is None: + raise AnsibleError(f"Error: aws_secret_access_key argument was not provided") + + endpoint_url = normalize_endpoint_url(endpoint=endpoint_url) + + upload_status = downloadFromS3( + local_dir=local_dir, bucket_name=bucket_name, object_name=object_name, + endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, region_name=region_name + ) + + return dict( + success=upload_status + ) + diff --git a/ibm/mas_devops/roles/download_backup_archive/README.md b/ibm/mas_devops/roles/download_backup_archive/README.md new file mode 100644 index 0000000000..f59b6be64e --- /dev/null +++ b/ibm/mas_devops/roles/download_backup_archive/README.md @@ -0,0 +1,272 @@ +# download_backup_archive +Downloads and extracts MAS backup archives from AWS S3 or Artifactory. + +This role automates the process of downloading MAS backup archives from remote storage locations and extracting them to a local directory for restore operations. It supports downloading from both AWS S3 (or S3-compatible storage) and Artifactory repositories. The role handles archive verification, extraction, and cleanup operations. + +Key features: +- Downloads compressed tar.gz archives from AWS S3 or S3-compatible storage +- Downloads compressed tar.gz archives from Artifactory repositories +- Automatic extraction of downloaded archives +- Configurable download timeouts for large archives +- Optional cleanup of temporary files after extraction +- Verification of downloaded archive integrity + +## Prerequisites + +### For S3 Download +- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) or boto3 Python library must be installed +- `amazon.aws` Ansible collection must be installed +- AWS credentials with S3 read permissions +- S3 bucket must exist and be accessible + +### For Artifactory Download +- `curl` command-line tool must be installed +- Artifactory API token with download permissions +- Artifactory repository must exist and be accessible + +## Role Variables + +### Required Variables + +#### mas_restore_dir +Directory where the backup archive will be downloaded and extracted. This is the parent directory where all component backup directories will be restored. + +- **Required** +- Environment Variable: `MAS_RESTORE_DIR` +- Default Value: None + +### Backup Archive Variables + +#### backup_version +Version identifier for the backup to download. This must match the version used when the backup was created. This is used to construct the value `mas-backup-.tar.gz` for `backup_archive_name`. +If you are using a custom archive name using `backup_archive_name`, you can omit this variable. + +- **optional** +- Environment Variable: `BACKUP_VERSION` +- Default Value: None + +#### backup_archive_name +Custom archive name of the tar.gz archive file to download, including the `.tar.gz` extension. +If you are using `backup_version`, you can omit this variable. + +- **Optional** +- Environment Variable: `BACKUP_ARCHIVE_NAME` +- Default Value: None + +### S3 Download Variables + +Provide these variables to download the backup archive from AWS S3 or S3-compatible storage. If S3 credentials are provided, S3 download takes precedence over Artifactory. + +#### aws_access_key_id +AWS access key ID for authentication. + +- **Required for S3 download** +- Environment Variable: `S3_ACCESS_KEY_ID` +- Default Value: None + +#### aws_secret_access_key +AWS secret access key for authentication. + +- **Required for S3 download** +- Environment Variable: `S3_SECRET_ACCESS_KEY` +- Default Value: None + +#### s3_bucket_name +Name of the S3 bucket where the archive is stored. + +- **Required for S3 download** +- Environment Variable: `S3_BUCKET_NAME` +- Default Value: None + +#### s3_region +AWS region where the S3 bucket is located. + +- **Optional** +- Environment Variable: `S3_REGION` +- Default Value: `us-east-1` + +#### s3_endpoint_url +Custom S3 endpoint URL for S3-compatible storage services (e.g., MinIO, Wasabi, IBM Cloud Object Storage). + +- **Optional** +- Environment Variable: `S3_ENDPOINT_URL` +- Default Value: None (uses AWS S3 endpoints) + +### Artifactory Download Variables + +Provide these variables to download the backup archive from Artifactory. Artifactory download is used only if S3 credentials are not provided. + +#### artifactory_username +Artifactory username for authentication. + +- **Required for Artifactory download** +- Environment Variable: `ARTIFACTORY_USERNAME` +- Default Value: None + +#### artifactory_token +Artifactory API token for authentication. + +- **Required for Artifactory download** +- Environment Variable: `ARTIFACTORY_TOKEN` +- Default Value: None + +#### artifactory_url +Base URL of the Artifactory server (e.g., `https://artifactory.example.com/artifactory`). + +- **Required for Artifactory download** +- Environment Variable: `ARTIFACTORY_URL` +- Default Value: None + +#### artifactory_repository +Name of the Artifactory repository where the archive is stored. + +- **Required for Artifactory download** +- Environment Variable: `ARTIFACTORY_REPOSITORY` +- Default Value: None + +### General Configuration + +#### backup_temp_dir +Temporary directory where the archive will be downloaded before extraction. The directory is created if it doesn't exist and can be cleaned up after extraction. + +- **Optional** +- Environment Variable: `BACKUP_TEMP_DIR` +- Default Value: `/tmp/mas-restore` + +#### download_timeout +Maximum time in seconds to wait for the download to complete. Useful for large archives or slow network connections. + +- **Optional** +- Environment Variable: `DOWNLOAD_TIMEOUT_SECS` +- Default Value: `3600` (1 hour) + +#### extract_archive +Whether to automatically extract the downloaded archive. Set to `false` if you only want to download the archive without extracting it. + +- **Optional** +- Environment Variable: `EXTRACT_ARCHIVE` +- Default Value: `true` + +#### cleanup_archive +Whether to remove the archive file and temporary directory after successful extraction. Set to `false` to keep the downloaded archive. + +- **Optional** +- Environment Variable: `CLEANUP_ARCHIVE` +- Default Value: `true` + +## Example Playbook + +### S3 Download +After installing the Ansible Collection you can include this role in your own custom playbooks. + +```yaml +- hosts: localhost + vars: + mas_restore_dir: /restore/mas + backup_version: "260117-191500" + aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + s3_bucket_name: my-mas-backups + s3_region: us-west-2 + roles: + - ibm.mas_devops.download_backup_archive +``` + +### Artifactory Download + +```yaml +- hosts: localhost + vars: + mas_restore_dir: /restore/mas + backup_version: "260117-191500" + artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" + artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" + artifactory_url: https://artifactory.example.com/artifactory + artifactory_repository: mas-backups + roles: + - ibm.mas_devops.download_backup_archive +``` + +### S3-Compatible Storage (IBMcloud, MinIO, Wasabi, etc.) + +```yaml +- hosts: localhost + vars: + mas_restore_dir: /restore/mas + backup_version: "260117-191500" + aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY') }}" + aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_KEY') }}" + s3_bucket_name: mas-backups + s3_region: us-east-1 + s3_endpoint_url: https://s3.example.com + roles: + - ibm.mas_devops.download_backup_archive +``` + +### Download Without Extraction + +```yaml +- hosts: localhost + vars: + mas_restore_dir: /restore/mas + backup_version: "260117-191500" + extract_archive: false + cleanup_archive: false + aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + s3_bucket_name: my-mas-backups + roles: + - ibm.mas_devops.download_backup_archive +``` + +## Run Role Playbook +After installing the Ansible Collection you can easily run the role standalone using the `run_role` playbook provided. + +### S3 Download + +```bash +export MAS_RESTORE_DIR=/restore/mas +export BACKUP_VERSION=260117-191500 +export S3_ACCESS_KEY_ID=your_access_key +export S3_SECRET_ACCESS_KEY=your_secret_key +export S3_BUCKET_NAME=my-mas-backups +export S3_REGION=us-west-2 +ROLE_NAME=download_backup_archive ansible-playbook ibm.mas_devops.run_role +``` + +### Artifactory Download + +```bash +export MAS_RESTORE_DIR=/restore/mas +export BACKUP_VERSION=260117-191500 +export ARTIFACTORY_USERNAME=your_username +export ARTIFACTORY_TOKEN=your_token +export ARTIFACTORY_URL=https://artifactory.example.com/artifactory +export ARTIFACTORY_REPOSITORY=mas-backups +ROLE_NAME=download_backup_archive ansible-playbook ibm.mas_devops.run_role +``` + +## Extracted Directory Structure + +After successful execution, the role will extract the backup archive to the restore directory with the following structure: + +``` +/restore/mas/ +├── backup-260117-191500-catalog/ +├── backup-260117-191500-certmanager/ +├── backup-260117-191500-sls/ +├── backup-260117-191500-mongoce/ +├── backup-260117-191500-db2u/ +└── backup-260117-191500-suite/ +``` + +The exact directories present will depend on which components were included in the original backup. + +## Related Roles + +- `upload_backup_archive` - Creates and uploads MAS backup archives to S3 or Artifactory +- `mongodb` - MongoDB backup and restore operations +- `db2` - Db2 backup and restore operations + +## License +EPL-2.0 \ No newline at end of file diff --git a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml new file mode 100644 index 0000000000..fd04438120 --- /dev/null +++ b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml @@ -0,0 +1,28 @@ +--- +# Required variables + +# Directory where the backup archive will be downloaded and extracted +mas_restore_dir: "{{ lookup('env', 'MAS_RESTORE_DIR') }}" + +# Backup version to download +backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" +backup_archive_name: "{{ lookup('env', 'BACKUP_ARCHIVE_NAME') }}" + +# S3 Configuration (provide these to download from S3) +aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY_ID') }}" +aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_ACCESS_KEY') }}" +s3_bucket_name: "{{ lookup('env', 'S3_BUCKET_NAME') }}" +s3_region: "{{ lookup('env', 'S3_REGION') }}" +s3_endpoint_url: "{{ lookup('env', 'S3_ENDPOINT_URL') }}" + +# Artifactory Configuration (provide these to download from Artifactory) +artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" +artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" +artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" +artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" + +# General settings +backup_temp_dir: "{{ lookup('env', 'BACKUP_TEMP_DIR') | default('/tmp/mas-restore, true) }}" +download_timeout: "{{ lookup('env', 'DOWNLOAD_TIMEOUT_SECS') | default(3600, true) }}" # Download timeout in seconds (1 hour) +extract_archive: "{{ lookup('env', 'EXTRACT_ARCHIVE') | default(true, true) }}" # Whether to extract the archive after download +cleanup_archive: "{{ lookup('env', 'CLEANUP_ARCHIVE') | default(true, true) }}" # Whether to remove the archive file after extraction diff --git a/ibm/mas_devops/roles/download_backup_archive/meta/main.yml b/ibm/mas_devops/roles/download_backup_archive/meta/main.yml new file mode 100644 index 0000000000..155b1d78a4 --- /dev/null +++ b/ibm/mas_devops/roles/download_backup_archive/meta/main.yml @@ -0,0 +1,20 @@ +--- +galaxy_info: + author: IBM + description: Download and extract MAS backup archive from S3 or Artifactory + company: IBM + license: EPL-2.0 + min_ansible_version: "2.9" + platforms: + - name: EL + versions: + - "8" + galaxy_tags: + - ibm + - mas + - backup + - restore + - s3 + - artifactory + +dependencies: [] diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml new file mode 100644 index 0000000000..f8a25c323d --- /dev/null +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml @@ -0,0 +1,78 @@ +--- +# Download backup archive from Artifactory +- name: "Validate Artifactory repository is defined" + ansible.builtin.fail: + msg: "artifactory_repository is required when downloading from Artifactory" + when: artifactory_repository is not defined or artifactory_repository == '' + +- name: "Set Artifactory download URL" + ansible.builtin.set_fact: + artifactory_download_url: "{{ artifactory_url }}/{{ artifactory_repository }}/{{ backup_archive_name }}" + +- name: "Display Artifactory download information" + ansible.builtin.debug: + msg: + - "Downloading from Artifactory: {{ artifactory_url }}" + - "Repository: {{ artifactory_repository }}" + - "Archive: {{ backup_archive_name }}" + - "Full URL: {{ artifactory_download_url }}" + - "Download to: {{ backup_temp_dir }}" + +- name: "Download archive from Artifactory using curl" + ansible.builtin.command: + cmd: > + curl -X GET + -u {{ artifactory_username }}:{{ artifactory_token }} + -o {{ backup_temp_dir }}/{{ backup_archive_name }} + {{ artifactory_download_url }} + --max-time {{ download_timeout }} + --connect-timeout 60 + --fail + --silent + --show-error + --location + --write-out '%{http_code}' + register: artifactory_download_result + changed_when: artifactory_download_result.rc == 0 + failed_when: artifactory_download_result.rc != 0 + async: "{{ download_timeout }}" + poll: 10 + no_log: true + +- name: "Check download HTTP status code" + ansible.builtin.set_fact: + download_http_code: "{{ artifactory_download_result.stdout | regex_search('[0-9]{3}$') }}" + when: artifactory_download_result is succeeded + +- name: "Verify downloaded archive exists" + ansible.builtin.stat: + path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + register: downloaded_archive_stat + +- name: "Display Artifactory download result" + ansible.builtin.debug: + msg: + - "Successfully downloaded {{ backup_archive_name }} from Artifactory" + - "HTTP Status: {{ download_http_code | default('N/A') }}" + - "Archive size: {{ (downloaded_archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + when: + - artifactory_download_result is succeeded + - downloaded_archive_stat.stat.exists + +- name: "Fail if Artifactory download failed" + ansible.builtin.fail: + msg: "Failed to download archive from Artifactory. HTTP Status: {{ download_http_code | default('Unknown') }}" + when: + - artifactory_download_result is succeeded + - download_http_code is defined + - download_http_code not in ['200', '201'] + +- name: "Handle download command failure" + ansible.builtin.fail: + msg: "Failed to download archive from Artifactory: {{ artifactory_download_result.stderr | default('Unknown error') }}" + when: artifactory_download_result is failed + +- name: "Fail if downloaded archive does not exist" + ansible.builtin.fail: + msg: "Downloaded archive {{ backup_temp_dir }}/{{ backup_archive_name }} does not exist" + when: not downloaded_archive_stat.stat.exists diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml new file mode 100644 index 0000000000..7a88fa4177 --- /dev/null +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml @@ -0,0 +1,49 @@ +--- +# Download backup archive from S3 +- name: "Set S3 region default" + ansible.builtin.set_fact: + region: "{{ s3_region | default('us-east-1') }}" + +- name: "Display S3 download information" + ansible.builtin.debug: + msg: + - "Downloading from S3 bucket: {{ s3_bucket_name }}" + - "Region: {{ region }}" + - "Archive: {{ backup_archive_name }}" + - "Download to: {{ backup_temp_dir }}" + +- name: "Download archive from S3" + ibm.mas_devops.download_from_s3: + aws_access_key_id: "{{ aws_access_key_id }}" + aws_secret_access_key: "{{ aws_secret_access_key }}" + bucket_name: "{{ s3_bucket_name }}" + object_name: "{{ backup_archive_name }}" + local_dir: "{{ backup_temp_dir }}" + region_name: "{{ region }}" + endpoint_url: "{{ s3_endpoint_url | default(omit) }}" + register: s3_download_result + poll: 10 + +- name: "Verify downloaded archive exists" + ansible.builtin.stat: + path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + register: downloaded_archive_stat + +- name: "Display S3 download result" + ansible.builtin.debug: + msg: + - "Successfully downloaded {{ backup_archive_name }} from S3 bucket {{ s3_bucket_name }}" + - "Archive size: {{ (downloaded_archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + when: + - s3_download_result.success | bool + - downloaded_archive_stat.stat.exists + +- name: "Fail if S3 download failed" + ansible.builtin.fail: + msg: "Failed to download archive from S3: {{ s3_download_result.msg | default('Unknown error') }}" + when: not s3_download_result.success | bool + +- name: "Fail if downloaded archive does not exist" + ansible.builtin.fail: + msg: "Downloaded archive {{ backup_temp_dir }}/{{ backup_archive_name }} does not exist" + when: not downloaded_archive_stat.stat.exists diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml new file mode 100644 index 0000000000..f98c124561 --- /dev/null +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml @@ -0,0 +1,43 @@ +--- +# Extract downloaded backup archive +- name: "Verify archive exists before extraction" + ansible.builtin.stat: + path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + register: archive_stat + +- name: "Fail if archive does not exist" + ansible.builtin.fail: + msg: "Archive {{ backup_temp_dir }}/{{ backup_archive_name }} does not exist" + when: not archive_stat.stat.exists + +- name: "Display extraction information" + ansible.builtin.debug: + msg: + - "Extracting archive: {{ backup_archive_name }}" + - "Archive size: {{ (archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + - "Extract to: {{ mas_restore_dir }}" + +- name: "Extract tar.gz archive to restore directory" + ansible.builtin.command: + cmd: "tar -xzf {{ backup_temp_dir }}/{{ backup_archive_name }} -C {{ mas_restore_dir }}" + register: tar_extract_result + changed_when: tar_extract_result.rc == 0 + +- name: "List extracted directories" + ansible.builtin.find: + paths: "{{ mas_restore_dir }}" + file_type: directory + patterns: "backup-*" + register: extracted_dirs + +- name: "Display extraction result" + ansible.builtin.debug: + msg: + - "Successfully extracted {{ backup_archive_name }}" + - "Extracted {{ extracted_dirs.files | length }} backup directories to {{ mas_restore_dir }}" + - "Directories: {{ extracted_dirs.files | map(attribute='path') | map('basename') | list }}" + +- name: "Fail if no backup directories were extracted" + ansible.builtin.fail: + msg: "No backup directories found after extraction in {{ mas_restore_dir }}" + when: extracted_dirs.files | length == 0 diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml new file mode 100644 index 0000000000..f74aae1eac --- /dev/null +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml @@ -0,0 +1,79 @@ +--- +# Validate required variables +- name: "Fail if mas_restore_dir is not defined" + ansible.builtin.fail: + msg: "mas_restore_dir is required but not defined" + when: mas_restore_dir is not defined or mas_restore_dir == '' + +- name: "Fail if either backup_version or backup_archive_name is not defined" + ansible.builtin.fail: + msg: "Either backup_version or backup_archive_name is required but not defined" + when: (backup_version is not defined or backup_version == '') and (backup_archive_name is not defined or backup_archive_name == '') + +- name: Set backup_archive_name if backup_version is provided + ansible.builtin.set_fact: + backup_archive_name: "mas-backup-{{ backup_version }}.tar.gz" # Default name if backup_version is provided + when: backup_version is defined and backup_version != '' and (backup_archive_name is not defined or backup_archive_name == '') + +# Determine download source +- name: "Check if S3 credentials are provided" + ansible.builtin.set_fact: + download_from_s3: "{{ (aws_access_key_id is defined and aws_access_key_id != '') and (aws_secret_access_key is defined and aws_secret_access_key != '') and (s3_bucket_name is defined and s3_bucket_name != '') }}" + +- name: "Check if Artifactory credentials are provided" + ansible.builtin.set_fact: + download_from_artifactory: "{{ (artifactory_username is defined and artifactory_username != '') and (artifactory_token is defined and artifactory_token != '') and (artifactory_url is defined and artifactory_url != '') }}" + +- name: "Fail if neither S3 nor Artifactory credentials are provided" + ansible.builtin.fail: + msg: "Either S3 credentials (aws_access_key_id, aws_secret_access_key, s3_bucket_name) or Artifactory credentials (artifactory_username, artifactory_token, artifactory_url) must be provided" + when: not download_from_s3 and not download_from_artifactory + +- name: "Display download source" + ansible.builtin.debug: + msg: "Will download from: {{ 'S3' if download_from_s3 else 'Artifactory' }}" + +# Create restore directory +- name: "Create restore directory" + ansible.builtin.file: + path: "{{ mas_restore_dir }}" + state: directory + mode: '0755' + +# Create temporary directory for download +- name: "Create temporary directory for download" + ansible.builtin.file: + path: "{{ backup_temp_dir }}" + state: directory + mode: '0755' + +# Download from S3 or Artifactory +- name: "Download from S3" + ansible.builtin.include_tasks: download_from_s3.yml + when: download_from_s3 + +- name: "Download from Artifactory" + ansible.builtin.include_tasks: download_from_artifactory.yml + when: download_from_artifactory and not download_from_s3 + +# Extract archive +- name: "Extract backup archive" + ansible.builtin.include_tasks: extract_archive.yml + when: extract_archive | bool + +# Cleanup +- name: "Remove temporary archive file" + ansible.builtin.file: + path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + state: absent + when: + - cleanup_archive | bool + - extract_archive | bool + +- name: "Remove temporary directory" + ansible.builtin.file: + path: "{{ backup_temp_dir }}" + state: absent + when: + - cleanup_archive | bool + - extract_archive | bool From a8db2d1d83bde739649f67623214e25f1315ecb8 Mon Sep 17 00:00:00 2001 From: whitfiea Date: Thu, 29 Jan 2026 16:01:50 +0000 Subject: [PATCH 33/61] Fix how kube context is retrieved --- .../plugins/action/backup_resource.py | 21 ++----------------- .../plugins/action/restore_resource.py | 21 ++----------------- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/ibm/mas_devops/plugins/action/backup_resource.py b/ibm/mas_devops/plugins/action/backup_resource.py index c09a1d3e79..c2154e70f8 100644 --- a/ibm/mas_devops/plugins/action/backup_resource.py +++ b/ibm/mas_devops/plugins/action/backup_resource.py @@ -3,11 +3,10 @@ import logging import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client from ansible.plugins.action import ActionBase from ansible.errors import AnsibleError from ansible.utils.display import Display -from kubernetes import client, config -from kubernetes.dynamic import DynamicClient from mas.devops.backup import backupResources @@ -44,24 +43,8 @@ def run(self, tmp=None, task_vars=None): # Initialize DynamicClient and grab the task args host = self._task.args.get('host', None) api_key = self._task.args.get('api_key', None) - - # Load kubernetes configuration - try: - if host and api_key: - # Use provided host and api_key - configuration = client.Configuration() - configuration.host = host - configuration.api_key = {'authorization': f'Bearer {api_key}'} - configuration.verify_ssl = False - api_client = client.ApiClient(configuration) - else: - # Load from kubeconfig - config.load_kube_config() - api_client = client.ApiClient() - except Exception as e: - raise AnsibleError(f"Failed to initialize Kubernetes client: {e}") - dynClient = DynamicClient(api_client) + dynClient = get_api_client(api_key=api_key, host=host) backup_resources = self._task.args.get('backup_resources') backup_path = self._task.args.get('backup_path') diff --git a/ibm/mas_devops/plugins/action/restore_resource.py b/ibm/mas_devops/plugins/action/restore_resource.py index 08caadd778..ebe71c2be7 100644 --- a/ibm/mas_devops/plugins/action/restore_resource.py +++ b/ibm/mas_devops/plugins/action/restore_resource.py @@ -4,11 +4,10 @@ import os import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client from ansible.plugins.action import ActionBase from ansible.errors import AnsibleError from ansible.utils.display import Display -from kubernetes import client, config -from kubernetes.dynamic import DynamicClient from mas.devops.restore import loadYamlFile, restoreResource @@ -222,23 +221,7 @@ def run(self, tmp=None, task_vars=None): host = self._task.args.get('host', None) api_key = self._task.args.get('api_key', None) - # Load kubernetes configuration - try: - if host and api_key: - # Use provided host and api_key - configuration = client.Configuration() - configuration.host = host - configuration.api_key = {'authorization': f'Bearer {api_key}'} - configuration.verify_ssl = False - api_client = client.ApiClient(configuration) - else: - # Load from kubeconfig - config.load_kube_config() - api_client = client.ApiClient() - except Exception as e: - raise AnsibleError(f"Failed to initialize Kubernetes client: {e}") - - dynClient = DynamicClient(api_client) + dynClient = get_api_client(api_key=api_key, host=host) backup_path = self._task.args.get('backup_path') replace_resource = self._task.args.get('replace_resource', True) From bf0017cb0ee8fd58bb67e434c5c49b88f02add0e Mon Sep 17 00:00:00 2001 From: whitfiea Date: Fri, 30 Jan 2026 09:20:44 +0000 Subject: [PATCH 34/61] set ibm_catalogs_backup_version --- ibm/mas_devops/playbooks/br_core.yml | 2 +- .../plugins/action/verify_backup_restore_vars.py | 2 +- ibm/mas_devops/roles/ibm_catalogs/README.md | 6 +++--- ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml | 2 +- ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml | 8 ++++---- ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml | 8 ++++---- ibm/mas_devops/roles/upload_backup_archive/README.md | 4 ++-- .../roles/upload_backup_archive/defaults/main.yml | 2 +- .../roles/upload_backup_archive/tasks/create_archive.yml | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml index b2b4b4e70f..50e52766ab 100644 --- a/ibm/mas_devops/playbooks/br_core.yml +++ b/ibm/mas_devops/playbooks/br_core.yml @@ -100,7 +100,7 @@ - role: ibm.mas_devops.ibm_catalogs vars: ibm_catalogs_action: "{{ br_action }}" - catalog_backup_version: "{{ br_action_version }}" + ibm_catalogs_backup_version: "{{ br_action_version }}" - role: ibm.mas_devops.cert_manager vars: diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index 7b85e8a90d..12e7dab745 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -9,7 +9,7 @@ class ActionModule(ActionBase): REQUIRED = { "catalog": { "backup": ["mas_backup_dir"], - "restore": ["mas_backup_dir", "catalog_backup_version"] + "restore": ["mas_backup_dir", "ibm_catalogs_backup_version"] }, "certmanager": { "backup": ["mas_backup_dir"], diff --git a/ibm/mas_devops/roles/ibm_catalogs/README.md b/ibm/mas_devops/roles/ibm_catalogs/README.md index 7ff93b106b..c835a3e47c 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/README.md +++ b/ibm/mas_devops/roles/ibm_catalogs/README.md @@ -103,16 +103,16 @@ Directory path where IBM Operator Catalog backup files will be stored. **Impact**: All backup files and metadata will be stored in subdirectories under this path. The backup creates a timestamped directory structure: `{mas_backup_dir}/backup-{version}-catalog/` -**Related variables**: Works with `catalog_backup_version` to create unique backup directories. +**Related variables**: Works with `ibm_catalogs_backup_version` to create unique backup directories. **Note**: Ensure this directory has sufficient space for backup data and is regularly backed up to external storage for disaster recovery. -#### catalog_backup_version +#### ibm_catalogs_backup_version Version identifier for the backup, used to create unique backup directories. - **Optional** for backup (auto-generated if not provided) - **Required** for restore -- Environment Variable: `CATALOG_BACKUP_VERSION` +- Environment Variable: `IBM_CATALOGS_BACKUP_VERSION` - Default: Auto-generated timestamp in format `YYMMDD-HHMMSS` **Purpose**: Provides a unique identifier for each backup, allowing multiple backups to coexist and enabling point-in-time restore operations. diff --git a/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml b/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml index 5afe63ec22..807bce437a 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/defaults/main.yml @@ -12,5 +12,5 @@ mas_catalog_digest: "{{ lookup('env', 'MAS_CATALOG_DIGEST') }}" mas_catalog_version: "{{ lookup('env', 'MAS_CATALOG_VERSION') | default ('@@MAS_LATEST_CATALOG@@', True) }}" # Backup and restore variables -catalog_backup_version: "{{ lookup('env', 'CATALOG_BACKUP_VERSION') }}" +ibm_catalogs_backup_version: "{{ lookup('env', 'IBM_CATALOGS_BACKUP_VERSION') }}" mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml index 39423dc402..db8945f95d 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml @@ -5,14 +5,14 @@ action: "backup" component: "catalog" -- name: "Check if CATALOG_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if IBM_CATALOGS_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - catalog_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" - when: catalog_backup_version is not defined or catalog_backup_version == "" or catalog_backup_version == "None" + ibm_catalogs_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + when: ibm_catalogs_backup_version is not defined or ibm_catalogs_backup_version == "" or ibm_catalogs_backup_version == "None" - name: "Set fact: Catalog backup base directory path" set_fact: - catalog_backup_path: "{{ mas_backup_dir }}/backup-{{ catalog_backup_version }}-catalog" + catalog_backup_path: "{{ mas_backup_dir }}/backup-{{ ibm_catalogs_backup_version }}-catalog" - name: "Set fact: IBM operator catalog backup resources" set_fact: diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml index f2e37e3913..e022df7e63 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml @@ -2,14 +2,14 @@ - name: "Fail if require variables for IBM operator catalog backup are not provided" ibm.mas_devops.verify_backup_restore_vars: mas_backup_dir: "{{ mas_backup_dir }}" - catalog_backup_version: "{{ catalog_backup_version }}" + ibm_catalogs_backup_version: "{{ ibm_catalogs_backup_version }}" action: "restore" component: "catalog" - name: "Set fact: Catalog backup base directory path" set_fact: - catalog_backup_path: "{{ mas_backup_dir }}/backup-{{ catalog_backup_version }}-catalog" - catalog_resources_path: "{{ mas_backup_dir }}/backup-{{ catalog_backup_version }}-catalog/resources" + catalog_backup_path: "{{ mas_backup_dir }}/backup-{{ ibm_catalogs_backup_version }}-catalog" + catalog_resources_path: "{{ mas_backup_dir }}/backup-{{ ibm_catalogs_backup_version }}-catalog/resources" - name: "Check catalog backup resource path exist" stat: @@ -24,7 +24,7 @@ - name: "MAS Catalog restore information" debug: msg: - - "Backup Version ................. {{ catalog_backup_version }}" + - "Backup Version ................. {{ ibm_catalogs_backup_version }}" - "Backup Path .................... {{ catalog_backup_path }}" # 1. Restore Secrets (required for dev catalog) diff --git a/ibm/mas_devops/roles/upload_backup_archive/README.md b/ibm/mas_devops/roles/upload_backup_archive/README.md index 64b82c6537..84438df5ec 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/README.md +++ b/ibm/mas_devops/roles/upload_backup_archive/README.md @@ -46,11 +46,11 @@ Version identifier for the backup. This is used as the default version for all c These variables allow you to specify different backup versions for individual components. If not provided, they default to the value of `backup_version`. -#### catalog_backup_version +#### ibm_catalogs_backup_version Backup version for the catalog component. - **Optional** -- Environment Variable: `CATALOG_BACKUP_VERSION` +- Environment Variable: `IBM_CATALOGS_BACKUP_VERSION` - Default Value: Value of `backup_version` #### certmanager_backup_version diff --git a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml index 75ca7235f3..4f0053d121 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml @@ -8,7 +8,7 @@ mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" # Specific backup versions for each component (optional - override backup_version) -catalog_backup_version: "{{ lookup('env', 'CATALOG_BACKUP_VERSION') | default (backup_version, true) }}" +ibm_catalogs_backup_version: "{{ lookup('env', 'IBM_CATALOGS_BACKUP_VERSION') | default (backup_version, true) }}" certmanager_backup_version: "{{ lookup('env', 'CERTMANAGER_BACKUP_VERSION') | default (backup_version, true) }}" mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') | default (backup_version, true) }}" sls_backup_version: "{{ lookup('env', 'SLS_BACKUP_VERSION') | default (backup_version, true) }}" diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml index 29980f222f..adc91b8002 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml @@ -3,7 +3,7 @@ - name: "Set backup directories to archive" ansible.builtin.set_fact: backup_directories: - - "backup-{{ catalog_backup_version }}-catalog" + - "backup-{{ ibm_catalogs_backup_version }}-catalog" - "backup-{{ certmanager_backup_version }}-certmanager" - "backup-{{ sls_backup_version }}-sls" - "backup-{{ mongodb_backup_version }}-mongoce" From a8eafa769bbca39b907d216846503c8a11ca65a4 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Sun, 1 Feb 2026 21:15:54 +0000 Subject: [PATCH 35/61] [patch] fix missing char --- ibm/mas_devops/roles/download_backup_archive/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml index fd04438120..33984c6a2c 100644 --- a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml @@ -22,7 +22,7 @@ artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" # General settings -backup_temp_dir: "{{ lookup('env', 'BACKUP_TEMP_DIR') | default('/tmp/mas-restore, true) }}" +backup_temp_dir: "{{ lookup('env', 'BACKUP_TEMP_DIR') | default('/tmp/mas-restore', true) }}" download_timeout: "{{ lookup('env', 'DOWNLOAD_TIMEOUT_SECS') | default(3600, true) }}" # Download timeout in seconds (1 hour) extract_archive: "{{ lookup('env', 'EXTRACT_ARCHIVE') | default(true, true) }}" # Whether to extract the archive after download cleanup_archive: "{{ lookup('env', 'CLEANUP_ARCHIVE') | default(true, true) }}" # Whether to remove the archive file after extraction From 9cafc0ed3896d5a8b017e0cc0880cbeab9758c0a Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Sun, 1 Feb 2026 22:38:39 +0000 Subject: [PATCH 36/61] fix condition in ibm_catalogs restore --- ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml index e022df7e63..5ecb849aba 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/restore/main.yml @@ -42,8 +42,8 @@ - Secret register: secret_result when: - - artifactory_username is not defined - - artifactory_token is not defined + - artifactory_username is not defined or artifactory_username == "" + - artifactory_token is not defined or artifactory_token == "" # 2. Restore Service Accounts (required for dev catalog) # ----------------------------------------------------------------------------- From 3ee7b92322cd439646a1517e702c53b4715fc19b Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 2 Feb 2026 09:44:05 +0000 Subject: [PATCH 37/61] Update verify_subscriptions.py --- .../plugins/action/verify_subscriptions.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ibm/mas_devops/plugins/action/verify_subscriptions.py b/ibm/mas_devops/plugins/action/verify_subscriptions.py index dfb5461c2b..0707db7040 100644 --- a/ibm/mas_devops/plugins/action/verify_subscriptions.py +++ b/ibm/mas_devops/plugins/action/verify_subscriptions.py @@ -34,12 +34,15 @@ def run(self, tmp=None, task_vars=None): notAtLatest = [] for subscription in subs.items: - display.v(f"* {subscription.metadata.namespace}/{subscription.metadata.name} = {subscription.status.state}") - if subscription.status.state != "AtLatestKnown": - allSubscriptionsAtLatestThisLoop = False - notAtLatest.append(f"{subscription.metadata.namespace}/{subscription.metadata.name} = {subscription.status.state}") + if hasattr(subscription, "status") and hasattr(subscription.status, "state"): + display.v(f"* {subscription.metadata.namespace}/{subscription.metadata.name} = {subscription.status.state}") + if subscription.status.state != "AtLatestKnown": + allSubscriptionsAtLatestThisLoop = False + notAtLatest.append(f"{subscription.metadata.namespace}/{subscription.metadata.name} = {subscription.status.state}") + else: + atLatest.append(f"{subscription.metadata.namespace}/{subscription.metadata.name} = {subscription.status.state}") else: - atLatest.append(f"{subscription.metadata.namespace}/{subscription.metadata.name} = {subscription.status.state}") + allSubscriptionsAtLatestThisLoop = False if allSubscriptionsAtLatestThisLoop: allSubscriptionsAtLatest = True From 1be6ee83ad534900fb3e862ad4369c3489418b17 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Feb 2026 17:07:28 +0000 Subject: [PATCH 38/61] remove extra condition to check slscfg or drocfg file --- ibm/mas_devops/roles/suite_restore/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/roles/suite_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_restore/tasks/main.yml index 46ee241428..8697c7999a 100644 --- a/ibm/mas_devops/roles/suite_restore/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_restore/tasks/main.yml @@ -216,7 +216,7 @@ - name: "Fail if dro_cfg_file does not exist" fail: msg: "DRO config file not found at: {{ dro_cfg_file }}" - when: not dro_cfg_file_stat.stat.exists or not dro_cfg_file_stat.stat.isreg + when: not dro_cfg_file_stat.stat.exists - name: "Apply DRO config file" kubernetes.core.k8s: @@ -248,7 +248,7 @@ - name: "Fail if sls_cfg_file does not exist" fail: msg: "SLS config file not found at: {{ sls_cfg_file }}" - when: not sls_cfg_file_stat.stat.exists or not sls_cfg_file_stat.stat.isreg + when: not sls_cfg_file_stat.stat.exists - name: "Apply SLS config file" kubernetes.core.k8s: From 55d40f38d8304e22dacbd8b4a5fa29d010ab01c6 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 6 Feb 2026 14:36:35 +0000 Subject: [PATCH 39/61] [patch] change temp dir to use backup_dir/restore_dir instead of /tmp (#2106) --- ibm/mas_devops/roles/download_backup_archive/README.md | 4 ++-- .../roles/download_backup_archive/defaults/main.yml | 2 +- ibm/mas_devops/roles/download_backup_archive/tasks/main.yml | 6 ++++++ ibm/mas_devops/roles/upload_backup_archive/README.md | 2 +- .../roles/upload_backup_archive/defaults/main.yml | 2 +- .../roles/upload_backup_archive/tasks/create_archive.yml | 5 +++++ 6 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ibm/mas_devops/roles/download_backup_archive/README.md b/ibm/mas_devops/roles/download_backup_archive/README.md index f59b6be64e..98c4a6793e 100644 --- a/ibm/mas_devops/roles/download_backup_archive/README.md +++ b/ibm/mas_devops/roles/download_backup_archive/README.md @@ -130,8 +130,8 @@ Name of the Artifactory repository where the archive is stored. Temporary directory where the archive will be downloaded before extraction. The directory is created if it doesn't exist and can be cleaned up after extraction. - **Optional** -- Environment Variable: `BACKUP_TEMP_DIR` -- Default Value: `/tmp/mas-restore` +- Environment Variable: None +- Default Value: `/mas-restore-` #### download_timeout Maximum time in seconds to wait for the download to complete. Useful for large archives or slow network connections. diff --git a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml index 33984c6a2c..fabd107d2e 100644 --- a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml @@ -22,7 +22,7 @@ artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" # General settings -backup_temp_dir: "{{ lookup('env', 'BACKUP_TEMP_DIR') | default('/tmp/mas-restore', true) }}" +backup_temp_dir: "{{ mas_restore_dir }}/mas-restore-{{ backup_version }}" download_timeout: "{{ lookup('env', 'DOWNLOAD_TIMEOUT_SECS') | default(3600, true) }}" # Download timeout in seconds (1 hour) extract_archive: "{{ lookup('env', 'EXTRACT_ARCHIVE') | default(true, true) }}" # Whether to extract the archive after download cleanup_archive: "{{ lookup('env', 'CLEANUP_ARCHIVE') | default(true, true) }}" # Whether to remove the archive file after extraction diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml index f74aae1eac..8f3e8fd378 100644 --- a/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml @@ -40,6 +40,12 @@ state: directory mode: '0755' +# Delete temporary directory to make sure its clean +- name: "Delete temporary directory before creating to make sure it is clean" + ansible.builtin.file: + path: "{{ backup_temp_dir }}" + state: absent + # Create temporary directory for download - name: "Create temporary directory for download" ansible.builtin.file: diff --git a/ibm/mas_devops/roles/upload_backup_archive/README.md b/ibm/mas_devops/roles/upload_backup_archive/README.md index 84438df5ec..b8a5f6328f 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/README.md +++ b/ibm/mas_devops/roles/upload_backup_archive/README.md @@ -173,7 +173,7 @@ Temporary directory where the archive will be created before upload. The directo - **Optional** - Environment Variable: None -- Default Value: `/tmp/mas-backup-{{ backup_version }}` +- Default Value: `/mas-backup-{{ backup_version }}` #### upload_timeout Maximum time in seconds to wait for the upload to complete. Useful for large archives or slow network connections. diff --git a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml index 4f0053d121..31a0c2c0db 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml @@ -30,5 +30,5 @@ artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" # General settings backup_archive_name: "mas-backup-{{ backup_version }}.tar.gz" -backup_temp_dir: "/tmp/mas-backup-{{ backup_version }}" +backup_temp_dir: "{{ mas_backup_dir }}/mas-backup-{{ backup_version }}" upload_timeout: 3600 # Upload timeout in seconds (1 hour) diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml index adc91b8002..44ee94e829 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml @@ -39,6 +39,11 @@ msg: "No backup directories found in {{ mas_backup_dir }} for version {{ backup_version }}" when: existing_backup_dirs | length == 0 +- name: "Remove temporary dir if it exists to avoid conflicts" + ansible.builtin.file: + path: "{{ backup_temp_dir }}" + state: absent + - name: "Create temporary directory for archive" ansible.builtin.file: path: "{{ backup_temp_dir }}" From b15cb3ff6fe9616d5663f8491db878f3e72ff266 Mon Sep 17 00:00:00 2001 From: Andrew Whitfield Date: Fri, 6 Feb 2026 15:03:22 +0000 Subject: [PATCH 40/61] Backup for manage (#2104) --- docs/playbooks/backup-restore.md | 8 +- ibm/mas_devops/playbooks/br_core.yml | 2 +- ibm/mas_devops/playbooks/br_manage.yml | 48 +++++ .../action/verify_backup_restore_vars.py | 3 + .../plugins/module_utils/backuprestore.py | 2 - ibm/mas_devops/roles/cert_manager/README.md | 2 +- .../tasks/provider/redhat/backup.yml | 2 +- .../roles/db2/tasks/backup/main.yml | 2 +- .../roles/db2/tasks/backup_database/main.yml | 2 +- .../roles/download_backup_archive/README.md | 24 +-- ibm/mas_devops/roles/ibm_catalogs/README.md | 2 +- .../roles/ibm_catalogs/tasks/backup/main.yml | 2 +- .../tasks/providers/community/backup.yml | 2 +- .../providers/community/backup_database.yml | 2 +- ibm/mas_devops/roles/sls/README.md | 6 +- .../roles/sls/tasks/backup/main.yml | 2 +- .../roles/suite_app_backup/README.md | 181 +++++++++++++----- .../roles/suite_app_backup/defaults/main.yml | 14 ++ .../roles/suite_app_backup/tasks/main.yml | 60 +++++- .../tasks/manage/backup-namespace.yml | 151 +++++++++++++++ .../tasks/manage/backup-pv.yml | 126 ++++++++++++ .../tasks/manage/backup-single-pv.yml | 93 +++++++++ .../tasks/manage/backup-vars.yml | 1 - .../suite_app_backup/tasks/manage/pv-info.yml | 98 ---------- .../tasks/manage/restore-namespace.yml | 35 ---- .../roles/suite_backup/tasks/main.yml | 2 +- ibm/mas_devops/roles/suite_restore/README.md | 2 +- .../roles/upload_backup_archive/README.md | 23 ++- .../upload_backup_archive/defaults/main.yml | 1 + .../tasks/create_archive.yml | 1 + 30 files changed, 677 insertions(+), 222 deletions(-) create mode 100644 ibm/mas_devops/playbooks/br_manage.yml create mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml create mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-pv.yml create mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-single-pv.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/manage/pv-info.yml delete mode 100644 ibm/mas_devops/roles/suite_app_backup/tasks/manage/restore-namespace.yml diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 48dff3dd63..1fc7d051d4 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -144,7 +144,7 @@ Restore MAS Core from a backup to the same cluster: export MAS_INSTANCE_ID=inst1 export MAS_BACKUP_DIR=/backup/mas export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=260122-131500 +export BACKUP_VERSION_TO_RESTORE=20260122-131500 export IBM_ENTITLEMENT_KEY=xxx export DRO_CONTACT_EMAIL=user@example.com @@ -163,7 +163,7 @@ Restore MAS Core from a backup without grafana: export MAS_INSTANCE_ID=inst1 export MAS_BACKUP_DIR=/backup/mas export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=260122-131500 +export BACKUP_VERSION_TO_RESTORE=20260122-131500 export INCLUDE_GRAFANA=false @@ -185,7 +185,7 @@ Restore MAS Core to a different cluster with a different domain: export MAS_INSTANCE_ID=inst1 export MAS_BACKUP_DIR=/backup/mas export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=260122-131500 +export BACKUP_VERSION_TO_RESTORE=20260122-131500 export IBM_ENTITLEMENT_KEY=xxx export DRO_CONTACT_EMAIL=user@example.com @@ -207,7 +207,7 @@ Restore MAS Core using external SLS and DRO services: export MAS_INSTANCE_ID=inst1 export MAS_BACKUP_DIR=/backup/mas export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=260122-131500 +export BACKUP_VERSION_TO_RESTORE=20260122-131500 # Skip SLS and DRO installation export INCLUDE_SLS=false diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml index 50e52766ab..2eeb9fd59c 100644 --- a/ibm/mas_devops/playbooks/br_core.yml +++ b/ibm/mas_devops/playbooks/br_core.yml @@ -54,7 +54,7 @@ pre_tasks: - name: "Set backup_version with timestamp if not provided" ansible.builtin.set_fact: - backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: backup_version is not defined or backup_version == '' - name: "Fail if mas_instance_id is not set to" diff --git a/ibm/mas_devops/playbooks/br_manage.yml b/ibm/mas_devops/playbooks/br_manage.yml new file mode 100644 index 0000000000..c5326edbea --- /dev/null +++ b/ibm/mas_devops/playbooks/br_manage.yml @@ -0,0 +1,48 @@ +--- +- name: "MAS Manage Application Backup" + hosts: localhost + any_errors_fatal: true + vars: + # Define the target for backup + mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" + mas_app_id: "manage" + + # The top level location to save the archive files + mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups + + # Backup version to use during backup action + # Note: The default timestamp will be set in pre_tasks + mas_app_backup_version: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION') }}" + + pre_tasks: + - name: "Set mas_app_backup_version with timestamp if not provided" + ansible.builtin.set_fact: + mas_app_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" + when: mas_app_backup_version is not defined or mas_app_backup_version == '' + + - name: "Fail if mas_instance_id is not set" + ansible.builtin.assert: + that: + - mas_instance_id is defined + - mas_instance_id != '' + fail_msg: "mas_instance_id is required and must be set" + + - name: "Fail if mas_workspace_id is not set" + ansible.builtin.assert: + that: + - mas_workspace_id is defined + - mas_workspace_id != '' + fail_msg: "mas_workspace_id is required and must be set" + + - name: "Fail if mas_backup_dir is not set" + ansible.builtin.assert: + that: + - mas_backup_dir is defined + - mas_backup_dir != '' + fail_msg: "mas_backup_dir is required and must be set" + + roles: + - role: ibm.mas_devops.suite_app_backup + vars: + mas_app_id: "manage" diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index 12e7dab745..e178694777 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -35,6 +35,9 @@ class ActionModule(ActionBase): "suite": { "backup": ["mas_instance_id", "mas_backup_dir"], "restore": ["mas_instance_id", "mas_backup_dir", "suite_backup_version"] + }, + "manage": { + "backup": ["mas_instance_id", "mas_workspace_id", "mas_backup_dir"], } } diff --git a/ibm/mas_devops/plugins/module_utils/backuprestore.py b/ibm/mas_devops/plugins/module_utils/backuprestore.py index 1b8fc65330..5e1f5f5983 100644 --- a/ibm/mas_devops/plugins/module_utils/backuprestore.py +++ b/ibm/mas_devops/plugins/module_utils/backuprestore.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -import os -import yaml from mas.devops.ocp import getSecret from kubernetes.dynamic import DynamicClient from kubernetes.dynamic.exceptions import NotFoundError diff --git a/ibm/mas_devops/roles/cert_manager/README.md b/ibm/mas_devops/roles/cert_manager/README.md index 1db614f013..e4cab517de 100644 --- a/ibm/mas_devops/roles/cert_manager/README.md +++ b/ibm/mas_devops/roles/cert_manager/README.md @@ -78,7 +78,7 @@ Version identifier for the backup, used to create unique backup directories. - For backup: Leave unset to auto-generate a timestamp-based version, or provide a custom identifier - For restore: Must specify the exact version identifier of the backup to restore -**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYMMDD-HHMMSS` (e.g., `260122-131500`) +**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYYYMMDD-HHMMSS` (e.g., `20260122-131500`) **Impact**: - For backup: Creates directory `{mas_backup_dir}/backup-{version}-certmanager/` diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml index 495fa54e19..5d865538ff 100644 --- a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml @@ -7,7 +7,7 @@ - name: "Check if CERTMANAGER_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - certmanager_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + certmanager_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: certmanager_backup_version is not defined or certmanager_backup_version == "" or certmanager_backup_version == "None" - name: "Set fact: cert-manager backup base directory path" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index c7bbab3aac..653e52d996 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -14,7 +14,7 @@ # ----------------------------------------------------------------------------- - name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - db2_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + db2_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: db2_backup_version is not defined or db2_backup_version == "" or db2_backup_version == "None" - name: "Set fact: DB2 backup base directory path" diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml index 91be4ffb56..b29b2c8b20 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml @@ -14,7 +14,7 @@ # ----------------------------------------------------------------------------- - name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - db2_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + db2_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: db2_backup_version is not defined or db2_backup_version == "" or db2_backup_version == "None" - name: "Set fact: DB2 backup base directory path" diff --git a/ibm/mas_devops/roles/download_backup_archive/README.md b/ibm/mas_devops/roles/download_backup_archive/README.md index 98c4a6793e..cb7d173f8d 100644 --- a/ibm/mas_devops/roles/download_backup_archive/README.md +++ b/ibm/mas_devops/roles/download_backup_archive/README.md @@ -163,7 +163,7 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_restore_dir: /restore/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" s3_bucket_name: my-mas-backups @@ -178,7 +178,7 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_restore_dir: /restore/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" artifactory_url: https://artifactory.example.com/artifactory @@ -193,7 +193,7 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_restore_dir: /restore/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY') }}" aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_KEY') }}" s3_bucket_name: mas-backups @@ -209,7 +209,7 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_restore_dir: /restore/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" extract_archive: false cleanup_archive: false aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" @@ -226,7 +226,7 @@ After installing the Ansible Collection you can easily run the role standalone u ```bash export MAS_RESTORE_DIR=/restore/mas -export BACKUP_VERSION=260117-191500 +export BACKUP_VERSION=20260117-191500 export S3_ACCESS_KEY_ID=your_access_key export S3_SECRET_ACCESS_KEY=your_secret_key export S3_BUCKET_NAME=my-mas-backups @@ -238,7 +238,7 @@ ROLE_NAME=download_backup_archive ansible-playbook ibm.mas_devops.run_role ```bash export MAS_RESTORE_DIR=/restore/mas -export BACKUP_VERSION=260117-191500 +export BACKUP_VERSION=20260117-191500 export ARTIFACTORY_USERNAME=your_username export ARTIFACTORY_TOKEN=your_token export ARTIFACTORY_URL=https://artifactory.example.com/artifactory @@ -252,12 +252,12 @@ After successful execution, the role will extract the backup archive to the rest ``` /restore/mas/ -├── backup-260117-191500-catalog/ -├── backup-260117-191500-certmanager/ -├── backup-260117-191500-sls/ -├── backup-260117-191500-mongoce/ -├── backup-260117-191500-db2u/ -└── backup-260117-191500-suite/ +├── backup-20260117-191500-catalog/ +├── backup-20260117-191500-certmanager/ +├── backup-20260117-191500-sls/ +├── backup-20260117-191500-mongoce/ +├── backup-20260117-191500-db2u/ +└── backup-20260117-191500-suite/ ``` The exact directories present will depend on which components were included in the original backup. diff --git a/ibm/mas_devops/roles/ibm_catalogs/README.md b/ibm/mas_devops/roles/ibm_catalogs/README.md index c835a3e47c..b68d016d86 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/README.md +++ b/ibm/mas_devops/roles/ibm_catalogs/README.md @@ -121,7 +121,7 @@ Version identifier for the backup, used to create unique backup directories. - For backup: Leave unset to auto-generate a timestamp-based version, or provide a custom identifier - For restore: Must specify the exact version identifier of the backup to restore -**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYMMDD-HHMMSS` (e.g., `260122-131500`) +**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYYYMMDD-HHMMSS` (e.g., `20260122-131500`) **Impact**: - For backup: Creates directory `{mas_backup_dir}/backup-{version}-catalog/` diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml index db8945f95d..9e6a922b60 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml @@ -7,7 +7,7 @@ - name: "Check if IBM_CATALOGS_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - ibm_catalogs_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + ibm_catalogs_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: ibm_catalogs_backup_version is not defined or ibm_catalogs_backup_version == "" or ibm_catalogs_backup_version == "None" - name: "Set fact: Catalog backup base directory path" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index 7414258ef4..7105e5a91b 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -12,7 +12,7 @@ - name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - mongodb_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + mongodb_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: mongodb_backup_version is not defined or mongodb_backup_version == "" or mongodb_backup_version == "None" - name: "Set fact: Create Mongodb backup base directory path" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml index 4efaff9102..8ed57c9467 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml @@ -12,7 +12,7 @@ - name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - mongodb_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + mongodb_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: mongodb_backup_version is not defined or mongodb_backup_version == "" or mongodb_backup_version == "None" - name: "Set fact: Create Mongodb backup base directory path" diff --git a/ibm/mas_devops/roles/sls/README.md b/ibm/mas_devops/roles/sls/README.md index f93931f0d5..eb2ca8151e 100644 --- a/ibm/mas_devops/roles/sls/README.md +++ b/ibm/mas_devops/roles/sls/README.md @@ -734,7 +734,7 @@ Version identifier for the backup, used to create unique backup directories. - For backup: Leave unset to auto-generate a timestamp-based version, or provide a custom identifier - For restore: Must specify the exact version identifier of the backup to restore -**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYMMDD-HHMMSS` (e.g., `260122-131500`) +**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYYYMMDD-HHMMSS` (e.g., `2020260122-131500`) **Impact**: - For backup: Creates directory `{mas_backup_dir}/backup-{version}-sls/` @@ -862,7 +862,7 @@ ansible-playbook ibm.mas_devops.run_role vars: sls_action: restore mas_backup_dir: /backup/mas - sls_backup_version: "260122-131500" + sls_backup_version: "20260122-131500" # Optional: override domain for different cluster # sls_domain: sls.newcluster.example.com @@ -874,7 +874,7 @@ ansible-playbook ibm.mas_devops.run_role ```bash export SLS_ACTION=restore export MAS_BACKUP_DIR=/backup/mas -export SLS_BACKUP_VERSION=260122-131500 +export SLS_BACKUP_VERSION=20260122-131500 # Optional: export SLS_DOMAIN=sls.newcluster.example.com ansible-playbook ibm.mas_devops.run_role diff --git a/ibm/mas_devops/roles/sls/tasks/backup/main.yml b/ibm/mas_devops/roles/sls/tasks/backup/main.yml index 8ac84fc3d0..dc814bc952 100644 --- a/ibm/mas_devops/roles/sls/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/backup/main.yml @@ -9,7 +9,7 @@ - name: "Check if SLS_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - sls_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + sls_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: sls_backup_version is not defined or sls_backup_version == "" or sls_backup_version == "None" - name: "Set fact: SLS backup base directory path" diff --git a/ibm/mas_devops/roles/suite_app_backup/README.md b/ibm/mas_devops/roles/suite_app_backup/README.md index 9c729300cd..31e8934aa2 100644 --- a/ibm/mas_devops/roles/suite_app_backup/README.md +++ b/ibm/mas_devops/roles/suite_app_backup/README.md @@ -1,98 +1,187 @@ -Backup and Restore MAS Applications +Backup MAS Applications =============================================================================== Overview ------------------------------------------------------------------------------- -This role supports backing up the data for below MAS applications: +This role supports backing up MAS application resources and data. Currently supported applications: -- `manage`: Manage namespace resources, persistent volume data (e.g. attachments) - +- **`manage`**: Backs up Manage namespace resources (CRs, secrets, subscriptions) and persistent volume data +Future support planned for: `iot`, `monitor`, `health`, `optimizer`, `visualinspection` -Supports creating on-demand full backups. +The backup process creates a timestamped backup directory containing: +1. **Namespace Resources**: Kubernetes resources including ManageApp, ManageWorkspace, secrets, and subscriptions +2. **Persistent Volume Data**: Application data stored in PVCs (automatically detected from ManageWorkspace CR) !!! important - An application backup can only be restored to an instance with the same MAS instance ID. + - An application backup can only be restored to an instance with the same MAS instance ID + - This role backs up application resources and PV data only. Database backups must be performed separately using the appropriate database backup role + - For Manage, see the [db2](db2.md) role for database backup Role Variables - General ------------------------------------------------------------------------------- ### mas_app_id -Defines the MAS application ID (`manage`, `iot`, `monitor`, `health`, `optimizer`, or `visualinspection`) for the backup or restore action. +Defines the MAS application ID for the backup action. - **Required** - Environment Variable: `MAS_APP_ID` - Default: None +- Valid Values: `manage` (currently supported) ### mas_instance_id -Defines the MAS instance ID for the backup or restore action. +Defines the MAS instance ID for the backup action. - **Required** - Environment Variable: `MAS_INSTANCE_ID` - Default: None ### mas_workspace_id -Defines the MAS workspace ID for the backup or restore action. +Defines the MAS workspace ID for the backup action. - **Required** - Environment Variable: `MAS_WORKSPACE_ID` - Default: None +### mas_backup_dir +Defines the directory where backups will be stored. The role will create a timestamped subdirectory within this location. -Role Variables - Manage +- **Required** +- Environment Variable: `MAS_BACKUP_DIR` +- Default: None +- Example: `/backup/mas` + +### mas_app_backup_version +Optional custom version identifier for the backup. If not specified, defaults to timestamp format `YYYYMMDD-HHMMSS`. + +- Optional +- Environment Variable: `MAS_APP_BACKUP_VERSION` +- Default: Auto-generated timestamp +- Example: `20240315-143022` or `v1.0-prod` + + +What Gets Backed Up +------------------------------------------------------------------------------- +### Manage Application +When backing up the Manage application, the following resources are included: + +**Namespace Resources** (automatically backed up): +- `ManageApp` CR +- `ManageWorkspace` CR +- Encryption secrets (dynamically determined from ManageWorkspace CR) +- Certificates with `mas.ibm.com/instanceId` label +- Subscription and OperatorGroup +- IBM entitlement secret +- All referenced secrets (auto-discovered) + +**Persistent Volume Data** (automatically backed up if configured in ManageWorkspace CR): +- All persistent volumes defined in `spec.settings.deployment.persistentVolumes` +- Data is backed up as compressed tar.gz archives +- Each PVC's mount path is archived separately +- Archives are stored in the `data` subdirectory + +**NOT Included** (must be backed up separately): +- Manage database (Db2) - use the [db2](db2.md) role +- Suite-level resources - use the [suite_backup](suite_backup.md) role + + +How Persistent Volume Backup Works ------------------------------------------------------------------------------- -### masbr_manage_pvc_paths -Set the Manage PVC paths to use in backup and restore. The PVC path is in the format of `:/`. Multiple PVC paths are separated by commas (e.g. `manage-doclinks1-pvc:/mnt/doclinks1/attachments,manage-doclinks2-pvc:/mnt/doclinks2`). +The role automatically detects and backs up persistent volumes configured in the ManageWorkspace CR: -The `` and `` are defined in the `ManageWorkspace` CRD instance `spec.settings.deployment.persistentVolumes`: -``` -persistentVolumes: - - accessModes: - - ReadWriteMany - mountPath: /mnt/doclinks1 - pvcName: manage-doclinks1-pvc - size: '20' - storageClassName: ocs-storagecluster-cephfs - volumeName: '' - - accessModes: - - ReadWriteMany - mountPath: /mnt/doclinks2 - pvcName: manage-doclinks2-pvc - size: '20' - storageClassName: ocs-storagecluster-cephfs - volumeName: '' +1. **Detection**: Reads `spec.settings.deployment.persistentVolumes` from ManageWorkspace CR +2. **Pod Selection**: Finds the UI or ALL server bundle pod for the workspace +3. **Archive Creation**: Creates tar.gz archives of each mount path inside the pod +4. **Transfer**: Copies archives from pod to local backup directory +5. **Cleanup**: Removes temporary archives from the pod + +Example ManageWorkspace CR configuration: +```yaml +spec: + settings: + deployment: + persistentVolumes: + - accessModes: + - ReadWriteMany + mountPath: /jmsstore + pvcName: mas-inst1-ws1-jmsserver-pvc + size: 25Gi + storageClassName: efs-csi + - accessModes: + - ReadWriteMany + mountPath: /usr/share/fonts/truetype/Free3of9Extended + pvcName: masms-inst1-ws1-fonts-pvc + size: 8Gi + storageClassName: efs-csi ``` -If not set a value for this variable, this role will not backup and restore persistent valumne data for Manage. +This configuration will result in two tar.gz archives: +- `mas-inst1-ws1-jmsserver-pvc.tar.gz` +- `masms-inst1-ws1-fonts-pvc.tar.gz` -- Optional -- Environment Variable: `MASBR_MANAGE_PVC_PATHS` -- Default: None +Backup Directory Structure +------------------------------------------------------------------------------- +The role creates a backup directory with the following structure: + +``` +/ +└── backup--manage-/ + ├── namespace/ + │ ├── ManageApp-.yaml + │ ├── ManageWorkspace--.yaml + │ ├── Secret--manage-encryptionsecret.yaml + │ ├── Secret--manage-encryptionsecret-operator.yaml + │ ├── Subscription-ibm-mas-manage.yaml + │ └── ... (other resources) + └── data/ + ├── .tar.gz + ├── .tar.gz + └── ... (one archive per PVC) +``` -Example Playbook +Example Playbooks ------------------------------------------------------------------------------- -### Backup -Backup Manage attachments, note that this does not include backup of any data in Db2, see the `backup` action in the [db2](db2.md) role. +### Basic Backup +Backup Manage namespace resources and any configured persistent volumes: + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: inst1 + mas_workspace_id: ws1 + mas_app_id: manage + mas_backup_dir: /backup/mas + roles: + - ibm.mas_devops.suite_app_backup +``` + +### Backup with Custom Version +Backup with a custom version identifier: ```yaml - hosts: localhost any_errors_fatal: true vars: - masbr_action: backup - mas_instance_id: main + mas_instance_id: inst1 mas_workspace_id: ws1 mas_app_id: manage - masbr_backup_data: pv - masbr_manage_pvc_paths: "manage-doclinks1-pvc:/mnt/doclinks1" - masbr_storage_local_folder: /tmp/masbr + mas_backup_dir: /backup/mas + mas_app_backup_version: "prod-backup-20240315" roles: - - ibm.mas_devops.suite_app_backup_restore + - ibm.mas_devops.suite_app_backup ``` + +Notes +------------------------------------------------------------------------------- +- **Database Backup**: This role does NOT backup the Manage database. Use the [db2](db2.md) role to backup Db2 databases separately +- **Suite Resources**: This role backs up application-specific resources only. For suite-level resources (Suite CR, workspace CRs, etc.), use the [suite_backup](suite_backup.md) role +- **Storage Requirements**: Ensure sufficient storage space in `mas_backup_dir` for both namespace resources and PV data +- **Pod Access**: The role uses the UI or ALL server bundle pod to access PVC data. Ensure at least one of these pods is running and healthy +- **Backup Time**: PV backup duration depends on the amount of data in the persistent volumes +- **Automatic Detection**: Persistent volumes are automatically detected from the ManageWorkspace CR - no manual configuration needed +- **Compression**: All PV data is compressed using gzip to minimize storage requirements diff --git a/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml b/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml index 77e18c291e..2ca7ac13a8 100644 --- a/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml @@ -1,3 +1,17 @@ +--- +# MAS Application Backup - Default Variables +# ============================================================================= + +# General Configuration +# ----------------------------------------------------------------------------- mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" + +# Backup Configuration +# ----------------------------------------------------------------------------- +# Directory where backups will be stored +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" + +# Optional: Specify a custom backup version (defaults to timestamp YYMMDD-HHMMSS) +mas_app_backup_version: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION') }}" diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml index b060674ce2..8aa6571f69 100644 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/main.yml @@ -1,5 +1,14 @@ --- -# Check mas app backup/restore required variables +# MAS Application Backup Role +# ============================================================================= +# This role backs up MAS application resources and data. +# Currently supports: manage +# +# The backup includes: +# - Application namespace resources (CRs, secrets, subscriptions) +# - Persistent volume data (if configured) + +# 1. Validate required variables # ----------------------------------------------------------------------------- - name: "Fail if mas_instance_id is not provided" assert: @@ -15,3 +24,52 @@ assert: that: mas_app_id is defined and mas_app_id != "" fail_msg: "mas_app_id is required" + +- name: "Fail if mas_backup_dir is not provided" + assert: + that: mas_backup_dir is defined and mas_backup_dir != "" + fail_msg: "mas_backup_dir is required" + +# 2. Display backup configuration +# ----------------------------------------------------------------------------- +- name: "Display backup configuration" + debug: + msg: + - "MAS Instance ID: {{ mas_instance_id }}" + - "MAS Workspace ID: {{ mas_workspace_id }}" + - "MAS App ID: {{ mas_app_id }}" + - "Backup Directory: {{ mas_backup_dir }}" + - "Backup Version: {{ mas_app_backup_version | default('auto-generated') }}" + +# 3. Route to app-specific backup tasks +# ----------------------------------------------------------------------------- +- name: "Execute backup for {{ mas_app_id }}" + block: + # Manage backup + # --------------------------------------------------------------------------- + - name: "Backup Manage application" + when: mas_app_id == "manage" + block: + - name: "Backup Manage namespace resources" + include_tasks: "{{ role_path }}/tasks/manage/backup-namespace.yml" + + - name: "Backup Manage persistent volumes" + include_tasks: "{{ role_path }}/tasks/manage/backup-pv.yml" + + - name: "Manage backup completed successfully" + debug: + msg: + - "==========================================" + - "Manage Backup Completed Successfully" + - "==========================================" + - "Instance ID: {{ mas_instance_id }}" + - "Workspace ID: {{ mas_workspace_id }}" + - "Backup Location: {{ manage_backup_path }}" + - "==========================================" + + # Unsupported app + # --------------------------------------------------------------------------- + - name: "Fail if app is not supported" + fail: + msg: "Application '{{ mas_app_id }}' is not yet supported for backup. Currently supported: manage" + when: mas_app_id not in ['manage'] diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml new file mode 100644 index 0000000000..7a9eafe2dd --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml @@ -0,0 +1,151 @@ +--- +# Backup Manage Namespace Resources +# ============================================================================= +# This task backs up all Manage namespace resources using the backup_resource +# plugin, following the pattern used in suite_backup role. + +# 1. Verify required variables +# ----------------------------------------------------------------------------- +- name: "Verify required variables for Manage backup" + ibm.mas_devops.verify_backup_restore_vars: + component: manage + action: backup + mas_instance_id: "{{ mas_instance_id }}" + mas_workspace_id: "{{ mas_workspace_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + +# 2. Set backup version if not provided +# ----------------------------------------------------------------------------- +- name: "Check if MAS_APP_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" + set_fact: + mas_app_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" + when: mas_app_backup_version is not defined or mas_app_backup_version == "" or mas_app_backup_version == "None" + +# 3. Set backup path +# ----------------------------------------------------------------------------- +- name: "Set fact: Manage backup base directory path" + set_fact: + manage_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-manage" + +# 4. Set Manage namespace and workspace CR name +# ----------------------------------------------------------------------------- +- name: "Set fact: Manage namespace" + set_fact: + mas_app_namespace: "mas-{{ mas_instance_id }}-manage" + +- name: "Set fact: Manage workspace CR name" + set_fact: + manage_workspace_cr_name: "{{ mas_instance_id }}-{{ mas_workspace_id }}" + +# 5. Get ManageWorkspace CR to determine encryption secret name +# ----------------------------------------------------------------------------- +- name: "Get ManageWorkspace CR" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + kind: ManageWorkspace + name: "{{ manage_workspace_cr_name }}" + namespace: "{{ mas_app_namespace }}" + register: manage_workspace_cr + +- name: "Fail if ManageWorkspace CR not found" + fail: + msg: "ManageWorkspace CR '{{ manage_workspace_cr_name }}' not found in namespace '{{ mas_app_namespace }}'" + when: + - manage_workspace_cr.resources is not defined or manage_workspace_cr.resources | length == 0 + +- name: "Set fact: Manage encryption secret name" + set_fact: + manage_encryptionsecret_name: "{{ manage_workspace_cr.resources[0].spec.settings.db.encryptionSecret | default(mas_workspace_id + '-manage-encryptionsecret', true) }}" + when: + - manage_workspace_cr.resources[0].spec is defined + - manage_workspace_cr.resources[0].spec.settings is defined + - manage_workspace_cr.resources[0].spec.settings.db is defined + +- name: "Debug: Manage encryption secret name" + debug: + msg: "Manage encryption secret name: {{ manage_encryptionsecret_name }}" + +# 6. Build the Manage namespace resources list. +# Note: the ManageWorkspace CR contains entries with the key secretName and these +# will be automatically picked up. We only need to define secrets in the fact below +# that are not referenced by a resource or not having the key secretName +# ----------------------------------------------------------------------------- +- name: "Set fact: Manage namespace resources" + set_fact: + manage_namespace_resources: + # Core Manage CRs + - kind: ManageApp + api_version: apps.mas.ibm.com/v1 + name: "{{ mas_instance_id }}" + - kind: ManageWorkspace + api_version: apps.mas.ibm.com/v1 + name: "{{ manage_workspace_cr_name }}" + # Encryption secrets + - kind: Secret + api_version: v1 + name: "{{ manage_encryptionsecret_name }}" + - kind: Secret + api_version: v1 + name: "{{ manage_encryptionsecret_name }}-operator" + # Certificates + - kind: Certificate + api_version: cert-manager.io/v1 + labels: + - "mas.ibm.com/instanceId={{ mas_instance_id }}" + # Subscription and OperatorGroup + - kind: Subscription + api_version: operators.coreos.com/v1alpha1 + name: ibm-mas-manage + - kind: OperatorGroup + api_version: operators.coreos.com/v1 + name: operatorgroup + # IBM entitlement secret + - kind: Secret + api_version: v1 + name: ibm-entitlement + +- name: "Set fact: Manage backup resources" + set_fact: + manage_backup_resources: + - namespace: "{{ mas_app_namespace }}" + resources: "{{ manage_namespace_resources }}" + +# 7. Backup Manage namespace resources +# ----------------------------------------------------------------------------- +- name: "Backup Manage namespace resources (referenced secrets are auto-discovered)" + ibm.mas_devops.backup_resource: + backup_resources: "{{ manage_backup_resources }}" + backup_path: "{{ manage_backup_path }}" + register: manage_backup_result + +# 8. Display backup results +# ----------------------------------------------------------------------------- +- name: "Display Manage backup results" + debug: + msg: + - "Backup completed{{ ' with failures' if manage_backup_result.failed_count > 0 else ' successfully' }}" + - "Total resources backed up: {{ manage_backup_result.backed_up_count }}" + - "Total resources failed: {{ manage_backup_result.failed_count }}" + - "Resources not found: {{ manage_backup_result.not_found_count }}" + - "Secrets auto-discovered: {{ manage_backup_result.discovered_secrets_count }}" + - "Backup location: {{ manage_backup_path }}" + +# 9. Display failed resources if any +# ----------------------------------------------------------------------------- +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ manage_backup_result.failed_resources | to_nice_yaml }}" + when: manage_backup_result.failed_count > 0 + +# 10. Fail if backup had errors +# ----------------------------------------------------------------------------- +- name: "Fail if backup had errors" + fail: + msg: | + Backup failed for {{ manage_backup_result.failed_count }} resource(s): + {% for resource in manage_backup_result.failed_resources %} + - {{ resource.description }} in {{ resource.scope }} + {% endfor %} + when: manage_backup_result.failed_count > 0 diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-pv.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-pv.yml new file mode 100644 index 0000000000..de3ba19606 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-pv.yml @@ -0,0 +1,126 @@ +--- +# Backup Manage Persistent Volumes +# ============================================================================= +# This task backs up Manage persistent volume data by: +# 1. Reading persistentVolumes from ManageWorkspace CR +# 2. Finding the UI or ALL server bundle pod +# 3. Creating tar.gz archives of each mount path +# 4. Storing them in the backup data folder + +# 1. Check if ManageWorkspace has persistent volumes configured +# ----------------------------------------------------------------------------- +- name: "Check if ManageWorkspace has persistent volumes" + set_fact: + manage_persistent_volumes: "{{ manage_workspace_cr.resources[0].spec.settings.deployment.persistentVolumes | default([]) }}" + +- name: "Display persistent volumes configuration" + debug: + msg: + - "Persistent volumes found: {{ manage_persistent_volumes | length }}" + - "{{ manage_persistent_volumes | to_nice_yaml }}" + +# 2. Only proceed if persistent volumes are configured +# ----------------------------------------------------------------------------- +- name: "Backup Manage persistent volumes" + when: manage_persistent_volumes | length > 0 + block: + # Find the server bundle pod (UI, ALL, or maxinst as fallback) + # ------------------------------------------------------------------------- + - name: "Find UI server bundle pod" + kubernetes.core.k8s_info: + kind: Pod + namespace: "{{ mas_app_namespace }}" + label_selectors: + - mas.ibm.com/appType=serverBundle + - mas.ibm.com/appTypeName=ui + - mas.ibm.com/workspaceId={{ mas_workspace_id }} + register: ui_pod_output + + - name: "Find ALL server bundle pod" + kubernetes.core.k8s_info: + kind: Pod + namespace: "{{ mas_app_namespace }}" + label_selectors: + - mas.ibm.com/appType=serverBundle + - mas.ibm.com/appTypeName=all + - mas.ibm.com/workspaceId={{ mas_workspace_id }} + register: all_pod_output + when: ui_pod_output.resources | length == 0 + + - name: "Find maxinst pod as fallback" + kubernetes.core.k8s_info: + kind: Pod + namespace: "{{ mas_app_namespace }}" + label_selectors: + - mas.ibm.com/appType=maxinstudb + - mas.ibm.com/workspaceId={{ mas_workspace_id }} + register: maxinst_pod_output + when: + - ui_pod_output.resources | length == 0 + - (all_pod_output.resources is not defined or all_pod_output.resources | length == 0) + + # Determine which pod to use + # ------------------------------------------------------------------------- + - name: "Set fact: server bundle pod to use" + set_fact: + server_bundle_pod: >- + {{ + ui_pod_output.resources[0] if ui_pod_output.resources | length > 0 + else (all_pod_output.resources[0] if (all_pod_output.resources is defined and all_pod_output.resources | length > 0) + else (maxinst_pod_output.resources[0] if (maxinst_pod_output is defined and maxinst_pod_output.resources is defined and maxinst_pod_output.resources | length > 0) + else None)) + }} + server_bundle_type: >- + {{ + 'ui' if ui_pod_output.resources | length > 0 + else ('all' if (all_pod_output.resources is defined and all_pod_output.resources | length > 0) + else ('maxinst' if (maxinst_pod_output is defined and maxinst_pod_output.resources is defined and maxinst_pod_output.resources | length > 0) + else 'none')) + }} + + - name: "Fail if no suitable pod found" + fail: + msg: "No UI, ALL server bundle, or maxinst pod found for workspace {{ mas_workspace_id }}" + when: server_bundle_pod is none or server_bundle_pod == None + + - name: "Display server bundle pod information" + debug: + msg: + - "Server bundle type: {{ server_bundle_type }}" + - "Pod name: {{ server_bundle_pod.metadata.name }}" + - "Container: {{ server_bundle_pod.spec.containers[0].name }}" + + # Create data backup directory + # ------------------------------------------------------------------------- + - name: "Set fact: PV backup data path" + set_fact: + manage_pv_data_path: "{{ manage_backup_path }}/data" + + - name: "Create PV backup data directory" + file: + path: "{{ manage_pv_data_path }}" + state: directory + mode: '0755' + + # Backup each persistent volume + # ------------------------------------------------------------------------- + - name: "Backup each persistent volume" + include_tasks: "{{ role_path }}/tasks/manage/backup-single-pv.yml" + loop: "{{ manage_persistent_volumes }}" + loop_control: + loop_var: pv_item + index_var: pv_index + + - name: "Display PV backup completion" + debug: + msg: + - "Manage PV backup completed" + - "Persistent volumes backed up: {{ manage_persistent_volumes | length }}" + - "Backup location: {{ manage_pv_data_path }}" + +# 3. Skip message if no persistent volumes configured +# ----------------------------------------------------------------------------- +- name: "Skip PV backup message" + debug: + msg: "Skipping Manage PV backup - no persistent volumes configured in ManageWorkspace CR" + when: manage_persistent_volumes | length == 0 diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-single-pv.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-single-pv.yml new file mode 100644 index 0000000000..d4066bbc41 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-single-pv.yml @@ -0,0 +1,93 @@ +--- +# Backup Single Persistent Volume +# ============================================================================= +# This task creates a tar.gz archive of a single persistent volume's mount path +# from the server bundle pod. + +- name: "Set fact: PV backup details" + set_fact: + pv_mount_path: "{{ pv_item.mountPath }}" + pv_pvc_name: "{{ pv_item.pvcName }}" + pv_archive_name: "{{ pv_item.pvcName }}.tar.gz" + pv_archive_path: "{{ manage_pv_data_path }}/{{ pv_item.pvcName }}.tar.gz" + +- name: "Display PV backup information" + debug: + msg: + - "Backing up PV {{ pv_index + 1 }}/{{ manage_persistent_volumes | length }}" + - "PVC Name: {{ pv_pvc_name }}" + - "Mount Path: {{ pv_mount_path }}" + - "Archive: {{ pv_archive_name }}" + +# Create tar.gz archive in the pod +# ----------------------------------------------------------------------------- +- name: "Create tar.gz archive of {{ pv_mount_path }} in pod" + kubernetes.core.k8s_exec: + namespace: "{{ mas_app_namespace }}" + pod: "{{ server_bundle_pod.metadata.name }}" + container: "{{ server_bundle_pod.spec.containers[0].name }}" + command: > + tar -czf /tmp/{{ pv_archive_name }} -C {{ pv_mount_path }} . + register: tar_result + failed_when: false + +- name: "Check tar creation result" + debug: + msg: + - "Tar creation return code: {{ tar_result.rc }}" + - "{{ 'Success' if tar_result.rc == 0 else 'Failed' }}" + +- name: "Fail if tar creation failed" + fail: + msg: "Failed to create tar archive for {{ pv_pvc_name }}: {{ tar_result.stderr | default('Unknown error') }}" + when: tar_result.rc != 0 + +# Copy tar.gz archive from pod to local backup directory +# ----------------------------------------------------------------------------- +- name: "Copy tar.gz archive from pod to backup location" + shell: | + oc cp {{ mas_app_namespace }}/{{ server_bundle_pod.metadata.name }}:/tmp/{{ pv_archive_name }} {{ pv_archive_path }} -c {{ server_bundle_pod.spec.containers[0].name }} + register: copy_result + failed_when: false + +- name: "Check copy result" + debug: + msg: + - "Copy return code: {{ copy_result.rc }}" + - "{{ 'Success' if copy_result.rc == 0 else 'Failed' }}" + +- name: "Fail if copy failed" + fail: + msg: "Failed to copy tar archive for {{ pv_pvc_name }}: {{ copy_result.stderr | default('Unknown error') }}" + when: copy_result.rc != 0 + +# Clean up tar.gz archive from pod +# ----------------------------------------------------------------------------- +- name: "Remove tar.gz archive from pod" + kubernetes.core.k8s_exec: + namespace: "{{ mas_app_namespace }}" + pod: "{{ server_bundle_pod.metadata.name }}" + container: "{{ server_bundle_pod.spec.containers[0].name }}" + command: rm -f /tmp/{{ pv_archive_name }} + register: cleanup_result + failed_when: false + +# Verify backup was created +# ----------------------------------------------------------------------------- +- name: "Verify backup archive exists and get size" + stat: + path: "{{ pv_archive_path }}" + register: archive_stat + +- name: "Display backup statistics" + debug: + msg: + - "Archive created: {{ archive_stat.stat.exists }}" + - "Archive size: {{ (archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + - "Archive location: {{ pv_archive_path }}" + when: archive_stat.stat.exists + +- name: "Warning if archive not found" + debug: + msg: "WARNING: Archive file not found at {{ pv_archive_path }}" + when: not archive_stat.stat.exists diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml deleted file mode 100644 index ed97d539c0..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-vars.yml +++ /dev/null @@ -1 +0,0 @@ ---- diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/pv-info.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/pv-info.yml deleted file mode 100644 index cb5971a048..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/pv-info.yml +++ /dev/null @@ -1,98 +0,0 @@ ---- -# Get workspace pv spec -# ----------------------------------------------------------------------------- - -# ~~ Sample workspace pv spec ~~ -# persistentVolumes: -# - accessModes: -# - ReadWriteMany -# mountPath: /mnt/doclinks1 -# pvcName: manage-doclinks1-pvc -# size: '20' -# storageClassName: ocs-storagecluster-cephfs -# volumeName: '' -# - accessModes: -# - ReadWriteMany -# mountPath: /mnt/doclinks2 -# pvcName: manage-doclinks2-pvc -# size: '20' -# storageClassName: ocs-storagecluster-cephfs -# volumeName: '' - -- name: "Set fact: workspace pv spec" - set_fact: - manage_ws_pvs: "{{ _ws_output.resources[0].spec.settings.deployment.persistentVolumes }}" - when: masbr_manage_pvc_paths is defined and masbr_manage_pvc_paths | length > 0 - -- name: "Debug: workspace pv spec" - debug: - msg: "{{ manage_ws_pvs }}" - when: manage_ws_pvs is defined - - -# Only go on processing when manage has pv defined -# ----------------------------------------------------------------------------- -- name: "Only go on processing when manage has pv defined" - when: manage_ws_pvs is defined and manage_ws_pvs | length > 0 - block: - # Get maxinst pod information - - name: "Get maxinst pod information" - kubernetes.core.k8s_info: - kind: Pod - namespace: "{{ mas_app_namespace }}" - label_selectors: - - mas.ibm.com/appType=maxinstudb - - mas.ibm.com/workspaceId={{ mas_workspace_id }} - register: _maxinst_pod_output - failed_when: - - _maxinst_pod_output.resources is not defined - - _maxinst_pod_output.resources | length == 0 - - - name: "Set fact: copy pvc file variables" - set_fact: - masbr_cf_pod_name: "{{ _maxinst_pod_output.resources[0].metadata.name }}" - masbr_cf_container_name: "{{ _maxinst_pod_output.resources[0].spec.containers[0].name }}" - masbr_cf_affinity: false - - - name: "Debug: maxinst pod information" - debug: - msg: - - "maxinst pod name ................... {{ masbr_cf_pod_name }}" - - "maxinst container name ............. {{ masbr_cf_container_name }}" - - - name: "Set fact: reset mas_app_pv_list" - set_fact: - mas_app_pv_list: [] - _manage_pvc_paths: [] - - # Set '_manage_pvc_paths' based on 'masbr_manage_pvc_paths' - - name: "Get specified pvc backup/restore paths" - when: masbr_manage_pvc_paths is defined and masbr_manage_pvc_paths | length > 0 - block: - - name: "Set fact: _manage_pvc_paths" - set_fact: - _manage_pvc_paths: >- - {{ _manage_pvc_paths + [{ - 'pvcName': item | split(':') | first | trim, - 'pvcPath': item | split(':') | last | trim - }] }} - loop: "{{ masbr_manage_pvc_paths | split(',') }}" - - - name: "Set fact: mas_app_pv_list" - set_fact: - mas_app_pv_list: >- - {{ mas_app_pv_list + [{ - 'mount_path': manage_ws_pvs | json_query( - '[?pvcName==`' + item.pvcName + '`].mountPath') | first, - 'pvc_name': _maxinst_pod_output.resources[0].spec.volumes | json_query( - '[?name==`' + item.pvcName + '`].persistentVolumeClaim.claimName') | first, - 'backup_paths': [{ - 'src_folder': item.pvcPath, - 'dest_folder': 'pv/' + item.pvcName + item.pvcPath - }], - 'restore_paths': [{ - 'src_folder': 'pv/' + item.pvcName + item.pvcPath, - 'dest_folder': item.pvcPath - }], - }] }} - loop: "{{ _manage_pvc_paths }}" diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/restore-namespace.yml deleted file mode 100644 index baadcbc74c..0000000000 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/restore-namespace.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -- name: "Set fact: workspace id in the resource file name" - set_fact: - masbr_restore_from_workspace: "{{ masbr_restore_from_yaml.component.workspace }}" - -- name: "Replace mas instance in the resource files" - when: masbr_restore_to_diff_instance - changed_when: true - shell: > - yq -i 'with(.metadata; - .namespace="{{ mas_app_namespace }}" | - .name="{{ mas_workspace_id }}-manage-encryptionsecret" - )' {{ masbr_ns_restore_folder }}/Secret-{{ masbr_restore_from_workspace }}-manage-encryptionsecret.yaml; - - yq -i 'with(.metadata; - .namespace="{{ mas_app_namespace }}" | - .name="{{ mas_workspace_id }}-manage-encryptionsecret-operator" - )' {{ masbr_ns_restore_folder }}/Secret-{{ masbr_restore_from_workspace }}-manage-encryptionsecret-operator.yaml; - -# Restore namespace resoruces -# ------------------------------------------------------------------------- -# Loop through the folder -- name: "Get the list of files from restore directory" - find: - paths: "{{ masbr_ns_restore_folder }}" - patterns: '*.yml,*.yaml' - recurse: no - register: find_result - -- name: "Apply configs" - kubernetes.core.k8s: - state: present - template: "{{ item.path }}" - with_items: "{{ find_result.files }}" - when: find_result is defined diff --git a/ibm/mas_devops/roles/suite_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_backup/tasks/main.yml index f079d9b5cb..e6e67e270e 100644 --- a/ibm/mas_devops/roles/suite_backup/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_backup/tasks/main.yml @@ -10,7 +10,7 @@ - name: "Check if SUITE_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" set_fact: - suite_backup_version: "{{ lookup('pipe', 'date +%y%m%d-%H%M%S') }}" + suite_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: suite_backup_version is not defined or suite_backup_version == "" or suite_backup_version == "None" - name: "Set fact: mas core namespace name" diff --git a/ibm/mas_devops/roles/suite_restore/README.md b/ibm/mas_devops/roles/suite_restore/README.md index 3cdc710d5b..3fd3b7177a 100644 --- a/ibm/mas_devops/roles/suite_restore/README.md +++ b/ibm/mas_devops/roles/suite_restore/README.md @@ -37,7 +37,7 @@ in the restore. - **Required** - Default: None - Environment Variable: `SUITE_BACKUP_VERSION` -- Example: `260116-130937` +- Example: `20260116-130937` ### mas_domain The domain to use for the MAS Suite instance. If not provided, the domain from the backup will be used. diff --git a/ibm/mas_devops/roles/upload_backup_archive/README.md b/ibm/mas_devops/roles/upload_backup_archive/README.md index b8a5f6328f..e0a6036cc9 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/README.md +++ b/ibm/mas_devops/roles/upload_backup_archive/README.md @@ -88,6 +88,13 @@ Backup version for the MAS Suite component. - Environment Variable: `SUITE_BACKUP_VERSION` - Default Value: Value of `backup_version` +#### manage_backup_version +Backup version for the MAS Manage app. + +- **Optional** +- Environment Variable: `MANAGE_BACKUP_VERSION` +- Default Value: Value of `backup_version` + ### S3 Upload Variables Provide these variables to upload the backup archive to AWS S3 or S3-compatible storage. If S3 credentials are provided, S3 upload takes precedence over Artifactory. @@ -191,7 +198,7 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_backup_dir: /backup/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" s3_bucket_name: my-mas-backups @@ -206,7 +213,7 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_backup_dir: /backup/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" artifactory_url: https://artifactory.example.com/artifactory @@ -221,7 +228,7 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_backup_dir: /backup/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY') }}" aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_KEY') }}" s3_bucket_name: mas-backups @@ -237,10 +244,10 @@ After installing the Ansible Collection you can include this role in your own cu - hosts: localhost vars: mas_backup_dir: /backup/mas - backup_version: "260117-191500" + backup_version: "20260117-191500" # Override specific component versions - mongodb_backup_version: "260116-120000" - db2_backup_version: "260115-180000" + mongodb_backup_version: "20260116-120000" + db2_backup_version: "20260115-180000" aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" s3_bucket_name: my-mas-backups @@ -255,7 +262,7 @@ After installing the Ansible Collection you can easily run the role standalone u ```bash export MAS_BACKUP_DIR=/backup/mas -export BACKUP_VERSION=260117-191500 +export BACKUP_VERSION=20260117-191500 export S3_ACCESS_KEY_ID=your_access_key export S3_SECRET_ACCESS_KEY=your_secret_key export S3_BUCKET_NAME=my-mas-backups @@ -267,7 +274,7 @@ ROLE_NAME=upload_backup_archive ansible-playbook ibm.mas_devops.run_role ```bash export MAS_BACKUP_DIR=/backup/mas -export BACKUP_VERSION=260117-191500 +export BACKUP_VERSION=20260117-191500 export ARTIFACTORY_USERNAME=your_username export ARTIFACTORY_TOKEN=your_token export ARTIFACTORY_URL=https://artifactory.example.com/artifactory diff --git a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml index 31a0c2c0db..4ad659674f 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml @@ -14,6 +14,7 @@ mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') | default (b sls_backup_version: "{{ lookup('env', 'SLS_BACKUP_VERSION') | default (backup_version, true) }}" db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') | default (backup_version, true) }}" suite_backup_version: "{{ lookup('env', 'SUITE_BACKUP_VERSION') | default (backup_version, true) }}" +manage_backup_version: "{{ lookup('env', 'MANAGE_BACKUP_VERSION') | default (backup_version, true) }}" # S3 Configuration (provide these to upload to S3) aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY_ID') }}" diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml index 44ee94e829..30b23fb028 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml @@ -9,6 +9,7 @@ - "backup-{{ mongodb_backup_version }}-mongoce" - "backup-{{ db2_backup_version }}-db2u" - "backup-{{ suite_backup_version }}-suite" + - "backup-{{ manage_backup_version }}-manage" - name: "Verify backup directory exists" ansible.builtin.stat: From e134e4539fc0d9f1709a2c82969728449f97334b Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 6 Feb 2026 16:15:18 +0000 Subject: [PATCH 41/61] fix default version format in missed places. --- docs/playbooks/backup-restore.md | 2 +- ibm/mas_devops/roles/cert_manager/README.md | 2 +- .../tasks/provider/redhat/backup.yml | 2 +- ibm/mas_devops/roles/db2/README.md | 28 +++++++++---------- .../roles/db2/tasks/backup/main.yml | 2 +- .../roles/db2/tasks/backup_database/main.yml | 2 +- ibm/mas_devops/roles/ibm_catalogs/README.md | 2 +- .../roles/ibm_catalogs/tasks/backup/main.yml | 2 +- ibm/mas_devops/roles/mongodb/README.md | 18 ++++++------ .../tasks/providers/community/backup.yml | 2 +- .../providers/community/backup_database.yml | 2 +- ibm/mas_devops/roles/sls/README.md | 4 +-- .../roles/sls/tasks/backup/main.yml | 2 +- .../roles/suite_app_backup/defaults/main.yml | 2 +- .../tasks/manage/backup-namespace.yml | 2 +- ibm/mas_devops/roles/suite_backup/README.md | 4 +-- .../roles/suite_backup/tasks/main.yml | 2 +- 17 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 1fc7d051d4..573cfb036d 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -65,7 +65,7 @@ All timings are estimates. See the individual role documentation for more inform - `BR_ACTION` - Set to `backup` or `restore` ### Backup-Specific Variables -- `BACKUP_VERSION` - (Optional) Custom version identifier for the backup. If not provided, defaults to timestamp format `YYMMDD-HHMMSS` +- `BACKUP_VERSION` - (Optional) Custom version identifier for the backup. If not provided, defaults to timestamp format `YYYYMMDD-HHMMSS` ### Restore-Specific Variables - `BACKUP_VERSION_TO_RESTORE` - (Required) The backup version identifier to restore diff --git a/ibm/mas_devops/roles/cert_manager/README.md b/ibm/mas_devops/roles/cert_manager/README.md index e4cab517de..068291dac8 100644 --- a/ibm/mas_devops/roles/cert_manager/README.md +++ b/ibm/mas_devops/roles/cert_manager/README.md @@ -70,7 +70,7 @@ Version identifier for the backup, used to create unique backup directories. - **Optional** for backup (auto-generated if not provided) - **Required** for restore - Environment Variable: `CERTMANAGER_BACKUP_VERSION` -- Default: Auto-generated timestamp in format `YYMMDD-HHMMSS` +- Default: Auto-generated timestamp in format `YYYYMMDD-HHMMSS` **Purpose**: Provides a unique identifier for each backup, allowing multiple backups to coexist and enabling point-in-time restore operations. diff --git a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml index 5d865538ff..1ac51474fc 100644 --- a/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml +++ b/ibm/mas_devops/roles/cert_manager/tasks/provider/redhat/backup.yml @@ -5,7 +5,7 @@ action: "backup" component: "certmanager" -- name: "Check if CERTMANAGER_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if CERTMANAGER_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: certmanager_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: certmanager_backup_version is not defined or certmanager_backup_version == "" or certmanager_backup_version == "None" diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index bb596f7fab..85cc8c83cc 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -72,7 +72,7 @@ Specifies which operation to perform on the Db2 database. **Related variables**: - `db2_version`: Required for upgrade action to specify target version - `db2_namespace`: All instances in this namespace are affected by upgrade -- `db2_backup_version`: Required for restore/restore_database action; optional for backup, defaults to YYMMDD-HHMMSS +- `db2_backup_version`: Required for restore/restore_database action; optional for backup, defaults to YYYYMMDD-HHMMSS - `override_storageclass`: In Restore, controls whether storage classes are overridden **Note**: **WARNING** - When using `upgrade`, ALL Db2 instances in the specified namespace will be upgraded. Plan accordingly and ensure `db2_version` matches the operator channel. @@ -894,7 +894,7 @@ The backup version timestamp identifier for backup and restore operations. - **Required** for `restore` and `restore_database` actions - **Auto-generated** for backup operations - Environment Variable: `DB2_BACKUP_VERSION` -- Default: Auto-generated in format `YYMMDD-HHMMSS` +- Default: Auto-generated in format `YYYYMMDD-HHMMSS` **Purpose**: Uniquely identifies a specific backup version using a timestamp. This allows multiple backups to coexist and enables point-in-time restore operations. @@ -903,7 +903,7 @@ The backup version timestamp identifier for backup and restore operations. - Must be specified when restoring to identify which backup to use - Must be specified when installing Db2 from an existing backup -**Valid values**: Timestamp string in format `YYMMDD-HHMMSS` (e.g., `251212-021316` for December 12, 2025 at 02:13:16) +**Valid values**: Timestamp string in format `YYYYMMDD-HHMMSS` (e.g., `20251212-021316` for December 12, 2025 at 02:13:16) **Impact**: - Determines the backup directory name: `backup--db2u` @@ -914,7 +914,7 @@ The backup version timestamp identifier for backup and restore operations. - `mas_backup_dir`: Parent directory containing versioned backups - `db2_action`: Required when action is `restore_database` or `restore`(instance & database) -**Example**: `251212-021316` +**Example**: `20251212-021316` ### override_storageclass Controls whether to override storage classes during Db2 installation from backup. @@ -1188,7 +1188,7 @@ Example Usage - Backup and Restore vars: db2_action: restore_database mas_instance_id: masinst1 - db2_backup_version: 251212-021316 + db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr db2_instance_name: db2u-manage db2_namespace: db2u @@ -1204,7 +1204,7 @@ Example Usage - Backup and Restore vars: db2_action: restore_database mas_instance_id: masinst1 - db2_backup_version: 251212-021316 + db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr db2_instance_name: db2u-manage backup_vendor: s3 @@ -1223,7 +1223,7 @@ Example Usage - Backup and Restore vars: db2_action: restore mas_instance_id: masinst1 - db2_backup_version: 251212-021316 + db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr backup_vendor: disk roles: @@ -1241,7 +1241,7 @@ Example Usage - Backup and Restore vars: db2_action: restore mas_instance_id: masinst1 - db2_backup_version: 251212-021316 + db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr backup_vendor: disk override_storageclass: true @@ -1261,7 +1261,7 @@ Example Usage - Backup and Restore vars: db2_action: restore mas_instance_id: masinst1 - db2_backup_version: 251212-021316 + db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr backup_vendor: s3 backup_s3_endpoint: https://s3.us-east.cloud-object-storage.appdomain.cloud @@ -1275,9 +1275,9 @@ Example Usage - Backup and Restore ### Backup Directory Structure (Disk) ``` /tmp/masbr/ -└── backup--db2u/ +└── backup--db2u/ ├── data/ - │ ├── db2-BLUDB-backup-.tar.gz + │ ├── db2-BLUDB-backup-.tar.gz │ └── db2-backup-info.yaml └── resources/ ├── db2uclusters/ @@ -1289,14 +1289,14 @@ Example Usage - Backup and Restore ### Database backup Metadata (db2-backup-info.yaml) ```yaml -source_db2_backup_version: "251212-021316" +source_db2_backup_version: "20251212-021316" source_db2_backup_timestamp: "20251212021316" source_db2_instance_name: "db2u-manage" source_db2_instance_version: "11.5.8.0-cn7" database: "BLUDB" backup_vendor: "disk" -vendor_backup_path: "/mnt/backup/251212-021316/data" -local_backup_path: "/tmp/masbr/backup-251212-021316-db2u/data/db2-BLUDB-backup-251212-021316.tar.gz" +vendor_backup_path: "/mnt/backup/20251212-021316/data" +local_backup_path: "/tmp/masbr/backup-20251212-021316-db2u/data/db2-BLUDB-backup-20251212-021316.tar.gz" status: "SUCCESS" ``` diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 653e52d996..70c9b00bf6 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -12,7 +12,7 @@ # Set DB2 backup version if not provided # ----------------------------------------------------------------------------- -- name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: db2_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: db2_backup_version is not defined or db2_backup_version == "" or db2_backup_version == "None" diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml index b29b2c8b20..3f5fcf2dbd 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml @@ -12,7 +12,7 @@ # Set DB2 backup version if not provided # ----------------------------------------------------------------------------- -- name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if DB2_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: db2_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: db2_backup_version is not defined or db2_backup_version == "" or db2_backup_version == "None" diff --git a/ibm/mas_devops/roles/ibm_catalogs/README.md b/ibm/mas_devops/roles/ibm_catalogs/README.md index b68d016d86..31100e7941 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/README.md +++ b/ibm/mas_devops/roles/ibm_catalogs/README.md @@ -113,7 +113,7 @@ Version identifier for the backup, used to create unique backup directories. - **Optional** for backup (auto-generated if not provided) - **Required** for restore - Environment Variable: `IBM_CATALOGS_BACKUP_VERSION` -- Default: Auto-generated timestamp in format `YYMMDD-HHMMSS` +- Default: Auto-generated timestamp in format `YYYYMMDD-HHMMSS` **Purpose**: Provides a unique identifier for each backup, allowing multiple backups to coexist and enabling point-in-time restore operations. diff --git a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml index 9e6a922b60..264aa066d5 100644 --- a/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/ibm_catalogs/tasks/backup/main.yml @@ -5,7 +5,7 @@ action: "backup" component: "catalog" -- name: "Check if IBM_CATALOGS_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if IBM_CATALOGS_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: ibm_catalogs_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: ibm_catalogs_backup_version is not defined or ibm_catalogs_backup_version == "" or ibm_catalogs_backup_version == "None" diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index d29d144df8..c92d614648 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -577,11 +577,11 @@ For backup and restore operations, set `mongodb_action` to one of the following: - Example: `/tmp/masbr` ### mongodb_backup_version -**Required for restore operations**. The backup version timestamp to restore from. This is automatically generated during backup in the format `YYMMDD-HHMMSS`. +**Required for restore operations**. The backup version timestamp to restore from. This is automatically generated during backup in the format `YYYYMMDD-HHMMSS`. - Environment Variable: `MONGODB_BACKUP_VERSION` -- Default Value: YYMMDD-HHMMSS -- Example: `251212-021316` +- Default Value: YYYYMMDD-HHMMSS +- Example: `20251212-021316` ### mongodb_instance_name The name of the MongoDB instance to backup. @@ -635,9 +635,9 @@ The MongoDB backup operation creates a backup of your MongoDB instance and datab **Backup Directory Structure:** ``` /tmp/masbr/ -└── backup--mongoce/ +└── backup--mongoce/ ├── data/ - │ ├── mongodump-.tar.gz + │ ├── mongodump-.tar.gz │ └── mongodb-info.yaml └── resources/ ├── mongodbcommunitys/ @@ -715,7 +715,7 @@ Restores only the database data to an existing MongoDB instance: **Storage Requirements:** - Ensure sufficient storage in the backup directory - Plan for at least 2x the database size for backup storage -- Backup directory structure: `/tmp/masbr/backup--mongoce/` +- Backup directory structure: `/tmp/masbr/backup--mongoce/` - Monitor disk space during backup operations **Security:** @@ -836,7 +836,7 @@ Restore MongoDB databases from a backup to an existing MongoDB instance without vars: mongodb_action: restore_database mas_instance_id: masinst1 - mongodb_backup_version: 251212-021316 + mongodb_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr mongodb_namespace: mongoce mongodb_instance_name: mas-mongo-ce @@ -870,7 +870,7 @@ Perform a complete restoration of MongoDB instance including all Kubernetes reso vars: mongodb_action: restore mas_instance_id: masinst1 - mongodb_backup_version: 251212-021316 + mongodb_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr roles: - ibm.mas_devops.mongodb @@ -897,7 +897,7 @@ Deploy a new MongoDB instance using configuration from a backup and restore data vars: mongodb_action: install mas_instance_id: masinst1 - mongodb_backup_version: 251212-021316 + mongodb_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr roles: - ibm.mas_devops.mongodb diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml index 7105e5a91b..37ce2dc4b9 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup.yml @@ -10,7 +10,7 @@ action: "backup" component: "mongodb" -- name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: mongodb_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: mongodb_backup_version is not defined or mongodb_backup_version == "" or mongodb_backup_version == "None" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml index 8ed57c9467..ddc43b08fb 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml @@ -10,7 +10,7 @@ action: "backup" component: "mongodb" -- name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if MONGODB_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: mongodb_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: mongodb_backup_version is not defined or mongodb_backup_version == "" or mongodb_backup_version == "None" diff --git a/ibm/mas_devops/roles/sls/README.md b/ibm/mas_devops/roles/sls/README.md index eb2ca8151e..01252b3641 100644 --- a/ibm/mas_devops/roles/sls/README.md +++ b/ibm/mas_devops/roles/sls/README.md @@ -726,7 +726,7 @@ Version identifier for the backup, used to create unique backup directories. - **Optional** for backup (auto-generated if not provided) - **Required** for restore - Environment Variable: `SLS_BACKUP_VERSION` -- Default: Auto-generated timestamp in format `YYMMDD-HHMMSS` +- Default: Auto-generated timestamp in format `YYYYMMDD-HHMMSS` **Purpose**: Provides a unique identifier for each backup, allowing multiple backups to coexist and enabling point-in-time restore operations. @@ -734,7 +734,7 @@ Version identifier for the backup, used to create unique backup directories. - For backup: Leave unset to auto-generate a timestamp-based version, or provide a custom identifier - For restore: Must specify the exact version identifier of the backup to restore -**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYYYMMDD-HHMMSS` (e.g., `2020260122-131500`) +**Valid values**: Any string suitable for directory names (alphanumeric, hyphens, underscores). Auto-generated format: `YYYYMMDD-HHMMSS` (e.g., `20260122-131500`) **Impact**: - For backup: Creates directory `{mas_backup_dir}/backup-{version}-sls/` diff --git a/ibm/mas_devops/roles/sls/tasks/backup/main.yml b/ibm/mas_devops/roles/sls/tasks/backup/main.yml index dc814bc952..ee671046e7 100644 --- a/ibm/mas_devops/roles/sls/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/sls/tasks/backup/main.yml @@ -7,7 +7,7 @@ action: "backup" component: "sls" -- name: "Check if SLS_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if SLS_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: sls_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: sls_backup_version is not defined or sls_backup_version == "" or sls_backup_version == "None" diff --git a/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml b/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml index 2ca7ac13a8..0afe09899a 100644 --- a/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_app_backup/defaults/main.yml @@ -13,5 +13,5 @@ mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Directory where backups will be stored mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" -# Optional: Specify a custom backup version (defaults to timestamp YYMMDD-HHMMSS) +# Optional: Specify a custom backup version (defaults to timestamp YYYYMMDD-HHMMSS) mas_app_backup_version: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION') }}" diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml index 7a9eafe2dd..57c60b7180 100644 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml @@ -16,7 +16,7 @@ # 2. Set backup version if not provided # ----------------------------------------------------------------------------- -- name: "Check if MAS_APP_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if MAS_APP_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: mas_app_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: mas_app_backup_version is not defined or mas_app_backup_version == "" or mas_app_backup_version == "None" diff --git a/ibm/mas_devops/roles/suite_backup/README.md b/ibm/mas_devops/roles/suite_backup/README.md index 2c7a7aef6a..d47636e513 100644 --- a/ibm/mas_devops/roles/suite_backup/README.md +++ b/ibm/mas_devops/roles/suite_backup/README.md @@ -29,10 +29,10 @@ The local directory path where backup files will be stored (for backup). - Example: `/tmp/mas_backups` ### suite_backup_version -Set version to override the default `YYMMDD-HHMMSS` timestamp version used in the name of the backup file. +Set version to override the default `YYYYMMDD-HHMMSS` timestamp version used in the name of the backup file. - **Optional** -- Default: `YYMMDD-HHMMSS` timestamp. +- Default: `YYYYMMDD-HHMMSS` timestamp. - Environment Variable: `SUITE_BACKUP_VERSION` ### include_sls diff --git a/ibm/mas_devops/roles/suite_backup/tasks/main.yml b/ibm/mas_devops/roles/suite_backup/tasks/main.yml index e6e67e270e..1e68357f27 100644 --- a/ibm/mas_devops/roles/suite_backup/tasks/main.yml +++ b/ibm/mas_devops/roles/suite_backup/tasks/main.yml @@ -8,7 +8,7 @@ mas_instance_id: "{{ mas_instance_id }}" mas_backup_dir: "{{ mas_backup_dir }}" -- name: "Check if SUITE_BACKUP_VERSION is provided, if not set to default 'YYMMDD-HHMMSS' format" +- name: "Check if SUITE_BACKUP_VERSION is provided, if not set to default 'YYYYMMDD-HHMMSS' format" set_fact: suite_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" when: suite_backup_version is not defined or suite_backup_version == "" or suite_backup_version == "None" From 473398e774b14cc7ac4e62c5c07e2877398c21de Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 12 Feb 2026 09:39:05 +0000 Subject: [PATCH 42/61] manage app restore (#2118) --- .../action/verify_backup_restore_vars.py | 1 + ibm/mas_devops/plugins/filter/filters.py | 28 +- .../db2/tasks/restore/restore-instance.yml | 3 + .../download_backup_archive/defaults/main.yml | 3 +- .../download_backup_archive/tasks/main.yml | 7 +- .../tasks/manage/backup-namespace.yml | 3 + .../roles/suite_app_restore/README.md | 373 +++++++++++++++++ .../roles/suite_app_restore/defaults/main.yml | 30 ++ .../roles/suite_app_restore/meta/main.yml | 19 + .../roles/suite_app_restore/tasks/main.yml | 79 ++++ .../tasks/manage/restore-namespace.yml | 375 ++++++++++++++++++ .../tasks/manage/restore-pv.yml | 316 +++++++++++++++ .../tasks/manage/restore-single-pv.yml | 132 ++++++ .../roles/suite_app_restore/vars/manage.yml | 8 + .../upload_backup_archive/defaults/main.yml | 6 +- .../upload_backup_archive/tasks/main.yml | 5 + 16 files changed, 1383 insertions(+), 5 deletions(-) create mode 100644 ibm/mas_devops/roles/suite_app_restore/README.md create mode 100644 ibm/mas_devops/roles/suite_app_restore/defaults/main.yml create mode 100644 ibm/mas_devops/roles/suite_app_restore/meta/main.yml create mode 100644 ibm/mas_devops/roles/suite_app_restore/tasks/main.yml create mode 100644 ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml create mode 100644 ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml create mode 100644 ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-single-pv.yml create mode 100644 ibm/mas_devops/roles/suite_app_restore/vars/manage.yml diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index e178694777..e0aa78658f 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -38,6 +38,7 @@ class ActionModule(ActionBase): }, "manage": { "backup": ["mas_instance_id", "mas_workspace_id", "mas_backup_dir"], + "restore": ["mas_instance_id", "mas_backup_dir", "mas_app_backup_version"] } } diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index 26b72fb32e..2e16c7bedd 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -545,9 +545,34 @@ def override_db2_storage_classes_names(storage_list: list, storage_class_name_me storage_item['spec']['storageClassName'] = storage_class_name_logs else: print(f'WARNING: Unhandled db2 storage name for {storage_item["name"]}') - + return storage_list +def override_manage_persistent_volumes(volumes_list: list, storage_class_name_rwo: str, storage_class_name_rwx: str): + """ + Iterate through the volumes_list list and set the storage_class_name for each storage item based on the access mode. + Expects data to be + storage: + - pvcName: manage-imagestitching + accessModes: + - ReadWriteMany + size: 20Gi + storageClassName: nfs-client + - pvcName: manage-2 + accessModes: + - ReadWriteMany + size: 20Gi + storageClassName: nfs-client + """ + + for volume_item in volumes_list: + if 'accessModes' in volume_item and 'storageClassName' in volume_item: + if volume_item['accessModes'][0] == 'ReadWriteMany': + volume_item['storageClassName'] = storage_class_name_rwx + else: + volume_item['storageClassName'] = storage_class_name_rwo + return volumes_list + class FilterModule(object): def filters(self): @@ -574,5 +599,6 @@ def filters(self): 'is_channel_upgrade_path_valid': is_channel_upgrade_path_valid, 'get_default_upgrade_channel': get_default_upgrade_channel, 'set_storage_classes_names': set_storage_classes_names, + 'override_manage_persistent_volumes': override_manage_persistent_volumes, 'override_db2_storage_classes_names': override_db2_storage_classes_names } diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml index e335608a71..7f41ba483b 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml @@ -86,6 +86,9 @@ - Secret - ConfigMap replace_resource: false + skip_files: #skip applying these files as its from core namespace, will be taken care from suite restore + Secret: + - jdbc-{{ db2_instance_name }}-credentials.yaml register: secrets_configmaps_result when: - namespace_result.success diff --git a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml index fabd107d2e..22cd07e294 100644 --- a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml @@ -3,6 +3,7 @@ # Directory where the backup archive will be downloaded and extracted mas_restore_dir: "{{ lookup('env', 'MAS_RESTORE_DIR') }}" +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" # Backup version to download backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" @@ -22,7 +23,7 @@ artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" # General settings -backup_temp_dir: "{{ mas_restore_dir }}/mas-restore-{{ backup_version }}" +backup_temp_dir: "{{ mas_restore_dir }}/mas-{{ mas_instance_id }}-restore-{{ backup_version }}" download_timeout: "{{ lookup('env', 'DOWNLOAD_TIMEOUT_SECS') | default(3600, true) }}" # Download timeout in seconds (1 hour) extract_archive: "{{ lookup('env', 'EXTRACT_ARCHIVE') | default(true, true) }}" # Whether to extract the archive after download cleanup_archive: "{{ lookup('env', 'CLEANUP_ARCHIVE') | default(true, true) }}" # Whether to remove the archive file after extraction diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml index 8f3e8fd378..2950aef3a7 100644 --- a/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml @@ -5,6 +5,11 @@ msg: "mas_restore_dir is required but not defined" when: mas_restore_dir is not defined or mas_restore_dir == '' +- name: "Fail if mas_instance_id is not defined" + ansible.builtin.fail: + msg: "mas_instance_id is required but not defined" + when: mas_instance_id is not defined or mas_instance_id == '' + - name: "Fail if either backup_version or backup_archive_name is not defined" ansible.builtin.fail: msg: "Either backup_version or backup_archive_name is required but not defined" @@ -12,7 +17,7 @@ - name: Set backup_archive_name if backup_version is provided ansible.builtin.set_fact: - backup_archive_name: "mas-backup-{{ backup_version }}.tar.gz" # Default name if backup_version is provided + backup_archive_name: "mas-{{ mas_instance_id }}-backup-{{ backup_version }}.tar.gz" # Default name if backup_version is provided when: backup_version is defined and backup_version != '' and (backup_archive_name is not defined or backup_archive_name == '') # Determine download source diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml index 57c60b7180..9df3ce2626 100644 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml @@ -74,6 +74,9 @@ set_fact: manage_namespace_resources: # Core Manage CRs + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ mas_app_namespace }}" - kind: ManageApp api_version: apps.mas.ibm.com/v1 name: "{{ mas_instance_id }}" diff --git a/ibm/mas_devops/roles/suite_app_restore/README.md b/ibm/mas_devops/roles/suite_app_restore/README.md new file mode 100644 index 0000000000..76540aff76 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/README.md @@ -0,0 +1,373 @@ +Restore MAS Applications +=============================================================================== + +Overview +------------------------------------------------------------------------------- +This role supports restoring MAS application resources and data from backups created by the `suite_app_backup` role. Currently supported applications: + +- **`manage`**: Restores Manage namespace resources (CRs, secrets, subscriptions) and persistent volume data + +Future support planned for: `iot`, `monitor`, `health`, `optimizer`, `visualinspection` + +The restore process follows these phases: +1. **Phase 1**: Restore Kubernetes resources like Project, Secrets, Configmaps, Subscription, Certificates and ManageApp CR (not ManageWorkspace CR) +2. **Phase 2**: Check if ManageWorkspace CR is already available +3. **Phase 3**: Restore persistent volume data + - If ManageWorkspace exists: Scale it down, restore data, then continue + - If ManageWorkspace doesn't exist: Create PVCs, create dummy pod, restore data, delete dummy pod +4. **Phase 4**: Restore ManageWorkspace CR +5. **Phase 5**: Wait for Manage deployment to be activated + +!!! important + - An application backup can only be restored to an instance with the same MAS instance ID + - This role restores application resources and PV data only. Database restores must be performed separately using the appropriate database restore role + - For Manage, see the [db2](db2.md) role for database restore + - The restore process is designed to be idempotent and can handle both fresh installations and existing deployments + + +Role Variables - General +------------------------------------------------------------------------------- +### mas_app_id +Defines the MAS application ID for the restore action. + +- **Required** +- Environment Variable: `MAS_APP_ID` +- Default: None +- Valid Values: `manage` (currently supported) + +### mas_instance_id +Defines the MAS instance ID for the restore action. Must match the instance ID from the backup. + +- **Required** +- Environment Variable: `MAS_INSTANCE_ID` +- Default: None + +### mas_workspace_id +Defines the MAS workspace ID for the restore action. Must match the workspace ID from the backup. + +- **Required** +- Environment Variable: `MAS_WORKSPACE_ID` +- Default: None + +### mas_backup_dir +Defines the directory where backups are stored. The role will look for the backup version subdirectory within this location. + +- **Required** +- Environment Variable: `MAS_BACKUP_DIR` +- Default: None +- Example: `/backup/mas` + +### mas_app_backup_version +Specifies which backup version to restore. This should match the version identifier used during backup. + +- **Required** +- Environment Variable: `MAS_APP_BACKUP_VERSION` +- Default: None +- Example: `20240315-143022` or `v1.0-prod` + +### mas_app_restore_wait_retries +Maximum time in seconds to wait for ManageWorkspace to become ready after restore. + +- Optional +- Environment Variable: `MAS_APP_RESTORE_WAIT_RETRIES` +- Default: `120` + +### mas_app_restore_wait_delay +Delay in seconds between status checks when waiting for ManageWorkspace to become ready. + +- Optional +- Environment Variable: `MAS_APP_RESTORE_WAIT_DELAY` +- Default: `360` + +### override_storageclass +Enable or disable storage class override during restore. When enabled, the restore process will use custom storage classes instead of the storage classes from the backup. + +- Optional +- Environment Variable: `OVERRIDE_STORAGECLASS` +- Default: `false` +- Valid Values: `true`, `false` + +### mas_app_custom_storage_class_rwx +Custom storage class to use for PVCs with ReadWriteMany (RWX) access mode when `override_storageclass` is enabled. If not provided and override is enabled, the default storage class will be used. + +- Optional +- Environment Variable: `MAS_APP_CUSTOM_STORAGE_CLASS_RWX` +- Default: Empty (uses default storage class) +- Example: `ocs-storagecluster-cephfs` + +### mas_app_custom_storage_class_rwo +Custom storage class to use for PVCs with ReadWriteOnce (RWO) access mode when `override_storageclass` is enabled. If not provided and override is enabled, the default storage class will be used. + +- Optional +- Environment Variable: `MAS_APP_CUSTOM_STORAGE_CLASS_RWO` +- Default: Empty (uses default storage class) +- Example: `ocs-storagecluster-ceph-rbd` + + +What Gets Restored +------------------------------------------------------------------------------- +### Manage Application +When restoring the Manage application, the following resources are restored: + +**Namespace Resources** (Phase 1 & 4): +- `Project` (namespace) +- Encryption secrets +- Certificates with `mas.ibm.com/instanceId` label +- IBM entitlement secret +- All referenced secrets +- Subscription and OperatorGroup +- `ManageApp` CR (Phase 1) +- `ManageWorkspace` CR (Phase 4) + +**Persistent Volume Data** (Phase 3): +- All persistent volumes defined in `spec.settings.deployment.persistentVolumes` +- Data is restored from compressed tar.gz archives +- Each PVC's mount path is restored separately +- Archives are read from the `data` subdirectory + +**NOT Restored** (must be restored separately): +- Manage database (Db2) - use the [db2](db2.md) role +- Suite-level resources - use the [suite_restore](suite_restore.md) role + + +How Persistent Volume Restore Works +------------------------------------------------------------------------------- +The role intelligently handles PV restoration based on whether ManageWorkspace CR is already deployed: + +### Scenario 1: ManageWorkspace CR Does Not Exist (Fresh Restore) +1. **Read Configuration**: Extracts PVC configuration from ManageWorkspace CR backup +2. **Create PVCs**: Creates all PVCs defined in the backup +3. **Create Dummy Pod**: Creates a temporary pod that mounts all PVCs +4. **Restore Data**: Copies tar.gz archives to the pod and extracts them to mount paths +5. **Cleanup**: Deletes the dummy pod +6. **Deploy CR**: Restores the ManageWorkspace CR +7. **Wait**: Waits for Manage deployment to be activated + +### Scenario 2: ManageWorkspace CR Already Exists (Re-restore/Update) +1. **Scale Down**: Sets ManageWorkspace `spec.settings.deployment.mode` to `down` +2. **Wait**: Waits for workspace to scale down +3. **Find Pod**: Locates UI, ALL, or maxinst pod for data access +4. **Restore Data**: Copies tar.gz archives to the pod and extracts them to mount paths +5. **Update CR**: Restores the ManageWorkspace CR (which will scale back up) +6. **Wait**: Waits for Manage deployment to be activated + +### Dummy Pod Specification +When a dummy pod is created for restoration, it uses: +- **Image**: `registry.redhat.io/ubi8/ubi-minimal:latest` +- **Command**: `sleep infinity` (keeps pod running) +- **Volumes**: All PVCs from ManageWorkspace CR configuration +- **Labels**: Tagged with instance ID and workspace ID for easy identification + + +Restore Process Phases +------------------------------------------------------------------------------- + +### Phase 1: Restore Resources Until ManageApp CR +- Restores all namespace resources except ManageWorkspace CR +- Includes: ManageApp, secrets, certificates, subscriptions, operator groups +- Waits for ManageApp CR to become ready +- Auto-discovers and restores referenced secrets + +### Phase 2: Check ManageWorkspace Status +- Checks if ManageWorkspace CR already exists +- If exists: Sets `spec.settings.deployment.mode` to `down` and waits for scale down +- If not exists: Proceeds to create PVCs and dummy pod + +### Phase 3: Restore Persistent Volume Data +- Reads PVC configuration from ManageWorkspace CR backup +- Creates PVCs if needed (when ManageWorkspace doesn't exist) +- Creates dummy pod or uses existing server bundle pod +- Restores data from tar.gz archives to each PVC +- Cleans up dummy pod if created + +### Phase 4: Restore ManageWorkspace CR +- Restores the ManageWorkspace CR +- This triggers the Manage deployment to start/restart + +### Phase 5: Wait for Deployment Activation +- Monitors ManageWorkspace CR status +- Waits for Ready condition to be True +- Configurable timeout and delay between checks + + +Expected Backup Directory Structure +------------------------------------------------------------------------------- +The role expects the backup directory to have the following structure: + +``` +/ +└── backup--manage/ + ├── resources/ + │ ├── projects + │ │ └── mas--manage.yaml + | ├── secrets + │ │ └── .yaml + │ │ └── .yaml + | ├── configmaps + │ │ └── .yaml + │ │ └── .yaml + | ├── subscriptions + │ │ └── .yaml + │ └── ... (other resources) + └── data/ + ├── .tar.gz + ├── .tar.gz +``` + + +Example Playbooks +------------------------------------------------------------------------------- + +### Basic Restore +Restore Manage namespace resources and persistent volumes from a backup: + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: inst1 + mas_workspace_id: ws1 + mas_app_id: manage + mas_backup_dir: /backup/mas + mas_app_backup_version: "20240315-143022" + roles: + - ibm.mas_devops.suite_app_restore +``` + +### Restore with Custom Timeout +Restore with a custom wait timeout for large deployments: + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: inst1 + mas_workspace_id: ws1 + mas_app_id: manage + mas_backup_dir: /backup/mas + mas_app_backup_version: "prod-backup-20240315" + mas_app_restore_wait_retries: 180 + mas_app_restore_wait_delay: 360 # Check 6 minutes + roles: + - ibm.mas_devops.suite_app_restore +``` + +### Complete Restore Workflow +Complete workflow including database restore: + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: inst1 + mas_workspace_id: ws1 + mas_backup_dir: /backup/mas + backup_version: "20240315-143022" + + tasks: + # 1. Restore Db2 database first + - name: "Restore Manage database" + include_role: + name: ibm.mas_devops.db2 + vars: + db2_action: restore + db2_backup_version: "{{ backup_version }}" + + # 2. Restore Manage application + - name: "Restore Manage application" + include_role: + name: ibm.mas_devops.suite_app_restore + vars: + mas_app_id: manage + mas_app_backup_version: "{{ backup_version }}" +``` + +### Restore with Storage Class Override +Restore to a different cluster with different storage classes: + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: inst1 + mas_workspace_id: ws1 + mas_app_id: manage + mas_backup_dir: /backup/mas + mas_app_backup_version: "20240315-143022" + # Enable storage class override + override_storageclass: true + # Specify custom storage classes + mas_app_custom_storage_class_rwo: "ocs-storagecluster-cephfs" + mas_app_custom_storage_class_rwx: "ocs-storagecluster-ceph-rbd" + roles: + - ibm.mas_devops.suite_app_restore +``` + +### Restore with Storage Class Override Using Default Classes +Restore with override enabled but using cluster's default storage classes: + +```yaml +- hosts: localhost + any_errors_fatal: true + vars: + mas_instance_id: inst1 + mas_workspace_id: ws1 + mas_app_id: manage + mas_backup_dir: /backup/mas + mas_app_backup_version: "20240315-143022" + # Enable storage class override without specifying custom classes + # Will automatically use the cluster's default storage class + override_storageclass: true + roles: + - ibm.mas_devops.suite_app_restore +``` + + +Troubleshooting +------------------------------------------------------------------------------- + +### Restore Fails at Phase 1 +**Issue**: Resources fail to restore in Phase 1 +**Solution**: +- Check that the backup directory exists and contains the expected files +- Verify namespace exists: `oc get namespace mas--manage` +- Check for conflicting resources: `oc get manageapp,subscription -n mas--manage` + +### Dummy Pod Fails to Start +**Issue**: Dummy pod remains in Pending state +**Solution**: +- Check PVC status: `oc get pvc -n mas--manage` +- Verify storage class exists and can provision volumes +- Check pod events: `oc describe pod -n mas--manage` + +### PV Data Restore Fails +**Issue**: Data extraction fails in the pod +**Solution**: +- Verify tar.gz archives are not corrupted +- Check pod has sufficient disk space +- Verify mount paths are accessible in the pod + +### ManageWorkspace Never Becomes Ready +**Issue**: Phase 5 times out waiting for ManageWorkspace +**Solution**: +- Check ManageWorkspace status: `oc describe manageworkspace -n mas--manage` +- Verify database is accessible and restored +- Check pod logs: `oc logs -l mas.ibm.com/appType=serverBundle -n mas--manage` +- Increase `mas_app_restore_wait_retries` and `mas_app_restore_wait_delay` values` if deployment is slow + + +Notes +------------------------------------------------------------------------------- +- **Database Restore**: This role does NOT restore the Manage database. Use the [db2](db2.md) role to restore Db2 databases separately, and do this BEFORE running the application restore +- **Suite Resources**: This role restores application-specific resources only. For suite-level resources (Suite CR, workspace CRs, etc.), use the [suite_restore](suite_restore.md) role +- **Instance ID Match**: The restore must be performed on a cluster with the same MAS instance ID as the backup +- **Idempotent**: The restore process is idempotent and can be run multiple times +- **Dummy Pod**: When ManageWorkspace doesn't exist, a temporary pod is created for data restoration and automatically cleaned up +- **Scale Down**: When ManageWorkspace exists, it's automatically scaled down before data restoration +- **Automatic Detection**: Persistent volumes are automatically detected from the ManageWorkspace CR backup +- **Compression**: All PV data is stored as compressed tar.gz archives +- **Wait Time**: The default wait timeout is 1 hour, but this can be adjusted based on deployment size +- **Prerequisites**: Ensure the MAS operator and Manage operator are installed before running restore +- **Storage Class Override**: When restoring to a different cluster with different storage classes, enable `override_storageclass` to automatically map PVCs to appropriate storage classes based on access modes (RWX/RWO) +- **Default Storage Classes**: If `override_storageclass` is enabled but custom storage classes are not specified, the cluster's default storage class will be used automatically +- **Access Mode Mapping**: The role intelligently assigns storage classes based on PVC access modes - RWX (ReadWriteMany) uses `mas_app_custom_storage_class_rwx` and RWO (ReadWriteOnce) uses `mas_app_custom_storage_class_rwo` \ No newline at end of file diff --git a/ibm/mas_devops/roles/suite_app_restore/defaults/main.yml b/ibm/mas_devops/roles/suite_app_restore/defaults/main.yml new file mode 100644 index 0000000000..c5611c4c28 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/defaults/main.yml @@ -0,0 +1,30 @@ +--- +# MAS Application Restore - Default Variables +# ============================================================================= + +# General Configuration +# ----------------------------------------------------------------------------- +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" +mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" +mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" + +# Restore Configuration +# ----------------------------------------------------------------------------- +# Directory where backups are stored +mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" + +# Backup version to restore (e.g., "20240315-143022" or "v1.0-prod") +mas_app_backup_version: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION') }}" + +# Storage Class Override Options +# ----------------------------------------------------------------------------- +# Override storage class from backup with custom storage classes +override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default('false', true) | bool }}" + +# Custom storage class for ReadWriteMany (RWX) access mode +mas_app_custom_storage_class_rwx: "{{ lookup('env', 'MAS_APP_CUSTOM_STORAGE_CLASS_RWX') | default('', true) }}" + +# Custom storage class for ReadWriteOnce (RWO) access mode +mas_app_custom_storage_class_rwo: "{{ lookup('env', 'MAS_APP_CUSTOM_STORAGE_CLASS_RWO') | default('', true) }}" + +_manage_persistent_volumes: "NO_OVERRIDE" diff --git a/ibm/mas_devops/roles/suite_app_restore/meta/main.yml b/ibm/mas_devops/roles/suite_app_restore/meta/main.yml new file mode 100644 index 0000000000..ecd01a67cc --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/meta/main.yml @@ -0,0 +1,19 @@ +--- +galaxy_info: + author: IBM + description: Restore MAS application resources and data + company: IBM + license: EPL-2.0 + min_ansible_version: "2.10" + platforms: + - name: EL + versions: + - "8" + galaxy_tags: + - ibm + - mas + - maximo + - restore + - backup + +dependencies: [] diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/main.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/main.yml new file mode 100644 index 0000000000..72e7c15d94 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/main.yml @@ -0,0 +1,79 @@ +--- +# MAS Application Restore Role +# ============================================================================= +# This role restores MAS application resources and data. +# Currently supports: manage +# +# The restore includes: +# - Application namespace resources (CRs, secrets, subscriptions) +# - Persistent volume data (if configured) + +# 1. Validate required variables +# ----------------------------------------------------------------------------- +- name: "Fail if mas_instance_id is not provided" + assert: + that: mas_instance_id is defined and mas_instance_id != "" + fail_msg: "mas_instance_id is required" + +- name: "Fail if mas_app_id is not provided" + assert: + that: mas_app_id is defined and mas_app_id != "" + fail_msg: "mas_app_id is required" + +- name: "Fail if mas_backup_dir is not provided" + assert: + that: mas_backup_dir is defined and mas_backup_dir != "" + fail_msg: "mas_backup_dir is required" + +- name: "Fail if mas_app_backup_version is not provided" + assert: + that: mas_app_backup_version is defined and mas_app_backup_version != "" + fail_msg: "mas_app_backup_version is required" + +# 2. Load var files +# ----------------------------------------------------------------------------- +- name: Load mas_appws variables + include_vars: "vars/{{ mas_app_id }}.yml" + +# 3. Display restore configuration +# ----------------------------------------------------------------------------- +- name: "Display restore configuration" + debug: + msg: + - "MAS Instance ID: {{ mas_instance_id }}" + - "MAS App ID: {{ mas_app_id }}" + - "Backup Directory: {{ mas_backup_dir }}" + - "Backup Version: {{ mas_app_backup_version }}" + +# 4. Route to app-specific restore tasks +# ----------------------------------------------------------------------------- +- name: "Execute restore for {{ mas_app_id }}" + block: + # Manage restore + # --------------------------------------------------------------------------- + - name: "Restore Manage application" + when: mas_app_id == "manage" + block: + - name: "Restore Manage namespace resources" + include_tasks: "{{ role_path }}/tasks/manage/restore-namespace.yml" + + - name: "Restore Manage persistent volumes" + include_tasks: "{{ role_path }}/tasks/manage/restore-pv.yml" + + - name: "Manage restore completed successfully" + debug: + msg: + - "==========================================" + - "Manage Restore Completed Successfully" + - "==========================================" + - "Instance ID: {{ mas_instance_id }}" + - "Backup Version: {{ mas_app_backup_version }}" + - "Restored from: {{ manage_backup_path }}" + - "==========================================" + + # Unsupported app + # --------------------------------------------------------------------------- + - name: "Fail if app is not supported" + fail: + msg: "Application '{{ mas_app_id }}' is not yet supported for restore. Currently supported: manage" + when: mas_app_id not in ['manage'] diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml new file mode 100644 index 0000000000..b8e2dc97e6 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml @@ -0,0 +1,375 @@ +--- +# Restore Manage Namespace Resources +# ============================================================================= +# This task restores Manage namespace resources following these steps: +# 1. Restore kubernetes resources until ManageApp CR +# 2. Check if ManageWorkspace CR is deployed +# 3. If deployed, change spec.mode to down +# 4. Continue with ManageWorkspace CR restore +# 5. Wait for Manage deployment to be activated + +# 1. Verify required variables +# ----------------------------------------------------------------------------- +- name: "Verify required variables for Manage restore" + ibm.mas_devops.verify_backup_restore_vars: + component: manage + action: restore + mas_instance_id: "{{ mas_instance_id }}" + mas_backup_dir: "{{ mas_backup_dir }}" + mas_app_backup_version: "{{ mas_app_backup_version }}" + +# 2. Set backup path +# ----------------------------------------------------------------------------- +- name: "Set fact: Manage backup base directory path" + set_fact: + manage_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-manage" + manage_resources_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-manage/resources" + +- name: "Verify backup directory exists" + stat: + path: "{{ manage_resources_backup_path }}" + register: backup_dir_stat + +- name: "Fail if backup directory does not exist" + fail: + msg: "Backup directory not found: {{ manage_resources_backup_path }}" + when: not backup_dir_stat.stat.exists or not backup_dir_stat.stat.isdir + +# 2.1. Verify cert-manager exists +# ----------------------------------------------------------------------------- +- name: Detect Certificate Manager installation + include_tasks: "{{ role_path }}/../../common_tasks/detect_cert_manager.yml" + +# 3. Set Manage namespace and workspace CR name +# ----------------------------------------------------------------------------- +- name: "Set fact: Manage namespace" + set_fact: + mas_app_namespace: "mas-{{ mas_instance_id }}-manage" + +- name: Get files from {{ manage_resources_backup_path }}/manageworkspaces directory + set_fact: + instance_files: "{{ lookup('fileglob', '{{ manage_resources_backup_path }}/manageworkspaces/*', wantlist=True) }}" + +- name: Assert exactly one ManageWorkspace CR exists + assert: + that: + - instance_files | length == 1 + fail_msg: "ManageWorkspace Directory must contain exactly one file" + +- name: Set fact ManageWorkspace cr + set_fact: + workspace_backup_cr: "{{ lookup('file', '{{ instance_files[0] }}') | from_yaml }}" + +- name: "Set fact: Manage workspace CR name" + set_fact: + manage_workspace_cr_name: "{{ workspace_backup_cr.metadata.name}}" + +# 4. Restore resources until ManageApp CR (excluding ManageWorkspace) +# ----------------------------------------------------------------------------- +- name: "Display restore phase 1 information" + debug: + msg: + - "==========================================" + - "Phase 1: Restore resources Project, Secrets, Configmaps, Subscription, Certificates and ManageApp CR" + - "==========================================" + - "Namespace: {{ mas_app_namespace }}" + - "Backup path: {{ manage_backup_path }}" + +# Step 1: Restore Projects first +- name: "Restore Projects" + ibm.mas_devops.restore_resource: + backup_path: "{{ manage_backup_path }}" + resource_kinds: + - Project + replace_resource: false + register: projects_result + +- name: "Display Projects restore results" + debug: + msg: >- + Projects: {{ projects_result.created_count }} created, + {{ projects_result.updated_count }} updated, + {{ projects_result.skipped_count }} skipped, + {{ projects_result.failed_count }} failed + +# Step 2: Restore Secrets and ConfigMaps +- name: "Restore Secrets and ConfigMaps" + ibm.mas_devops.restore_resource: + backup_path: "{{ manage_backup_path }}" + resource_kinds: + - Secret + - ConfigMap + register: secrets_configmaps_result + when: projects_result.success + +- name: "Display Secrets and ConfigMaps restore results" + debug: + msg: >- + Secrets and ConfigMaps: {{ secrets_configmaps_result.created_count }} created, + {{ secrets_configmaps_result.updated_count }} updated, + {{ secrets_configmaps_result.skipped_count }} skipped, + {{ secrets_configmaps_result.failed_count }} failed + when: projects_result.success + +# Step 3: Restore OperatorGroups and Subscriptions +- name: "Restore OperatorGroups" + ibm.mas_devops.restore_resource: + backup_path: "{{ manage_backup_path }}" + resource_kinds: + - OperatorGroup + register: operatorgroups_result + when: projects_result.success + +- name: "Display OperatorGroups restore results" + debug: + msg: >- + OperatorGroups: {{ operatorgroups_result.created_count }} created, + {{ operatorgroups_result.updated_count }} updated, + {{ operatorgroups_result.skipped_count }} skipped, + {{ operatorgroups_result.failed_count }} failed + when: projects_result.success + +- name: "Restore Subscriptions" + ibm.mas_devops.restore_resource: + backup_path: "{{ manage_backup_path }}" + resource_kinds: + - Subscription + register: subscriptions_result + when: projects_result.success + +- name: "Display Subscriptions restore results" + debug: + msg: >- + Subscriptions: {{ subscriptions_result.created_count }} created, + {{ subscriptions_result.updated_count }} updated, + {{ subscriptions_result.skipped_count }} skipped, + {{ subscriptions_result.failed_count }} failed + when: projects_result.success + +# Wait until the LicenseService CRD is available +# ----------------------------------------------------------------------------- +- name: "Wait until the ManageApps CRD is available" + include_tasks: "{{ role_path }}/../../common_tasks/wait_for_crd.yml" + vars: + crd_name: "manageapps.apps.mas.ibm.com" + +# Step 4: Restore Certificate resources +- name: "Restore Certificate resources" + ibm.mas_devops.restore_resource: + backup_path: "{{ manage_backup_path }}" + resource_kinds: + - Certificate + register: certmanager_result + when: projects_result.success + +- name: "Display Certificates restore results" + debug: + msg: >- + Certificate Manager resources: {{ certmanager_result.created_count }} created, + {{ certmanager_result.updated_count }} updated, + {{ certmanager_result.skipped_count }} skipped, + {{ certmanager_result.failed_count }} failed + when: projects_result.success + +- name: "Restore ManageApp CR" + ibm.mas_devops.restore_resource: + backup_path: "{{ manage_backup_path }}" + resource_kinds: + - ManageApp + register: cr_result + +- name: "Display ManageApp CR restore results" + debug: + msg: >- + Manage App resource: {{ cr_result.created_count }} created, + {{ cr_result.updated_count }} updated, + {{ cr_result.skipped_count }} skipped, + {{ cr_result.failed_count }} failed + +# Calculate Phase 1 results +# ----------------------------------------------------------------------------- +- name: "Calculate Phase 1 restore results" + set_fact: + total_created: >- + {{ + (projects_result.created_count | default(0)) + + (secrets_configmaps_result.created_count | default(0)) + + (operatorgroups_result.created_count | default(0)) + + (subscriptions_result.created_count | default(0)) + + (certmanager_result.created_count | default(0)) + + (cr_result.created_count | default(0)) + }} + total_updated: >- + {{ + (projects_result.updated_count | default(0)) + + (secrets_configmaps_result.updated_count | default(0)) + + (operatorgroups_result.updated_count | default(0)) + + (subscriptions_result.updated_count | default(0)) + + (certmanager_result.updated_count | default(0)) + + (cr_result.updated_count | default(0)) + }} + total_skipped: >- + {{ + (projects_result.skipped_count | default(0)) + + (secrets_configmaps_result.skipped_count | default(0)) + + (operatorgroups_result.skipped_count | default(0)) + + (subscriptions_result.skipped_count | default(0)) + + (certmanager_result.skipped_count | default(0)) + + (cr_result.skipped_count | default(0)) + }} + total_failed: >- + {{ + (projects_result.failed_count | default(0)) + + (secrets_configmaps_result.failed_count | default(0)) + + (operatorgroups_result.failed_count | default(0)) + + (subscriptions_result.failed_count | default(0)) + + (certmanager_result.failed_count | default(0)) + + (cr_result.failed_count | default(0)) + }} + +- name: "Display Phase 1 restore results" + debug: + msg: + - >- + Phase 1 Restore completed{{ ' with failures' if total_failed | int > 0 + else ' successfully' }} + - "Total resources created: {{ total_created }}" + - "Total resources updated: {{ total_updated }}" + - "Total resources skipped: {{ total_skipped }}" + - "Total resources failed: {{ total_failed }}" + +# Fail task if any errors occurred +# ----------------------------------------------------------------------------- +- name: "Collect Phase 1 failed resources" + set_fact: + all_failed_resources: >- + {{ + (projects_result.failed_resources | default([])) + + (secrets_configmaps_result.failed_resources | default([])) + + (operatorgroups_result.failed_resources | default([])) + + (subscriptions_result.failed_resources | default([])) + + (certmanager_result.failed_resources | default([])) + + (certmanager_result.failed_resources | default([])) + + (cr_result.failed_resources | default([])) + }} + +- name: "Display failed resources" + debug: + msg: + - "Failed resources:" + - "{{ all_failed_resources | to_nice_yaml }}" + when: total_failed | int > 0 + +- name: "Fail if phase 1 restore had errors" + fail: + msg: | + Restore failed for {{ total_failed }} resource(s): + {% for resource in all_failed_resources %} + - {{ resource.description }}: {{ resource.error }} + {% endfor %} + when: total_failed | int > 0 + +# 5. Wait for ManageApp CR to be ready +# ----------------------------------------------------------------------------- +- name: "Wait for ManageApp CR to be ready" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + kind: ManageApp + name: "{{ mas_instance_id }}" + namespace: "{{ mas_app_namespace }}" + register: manage_app_cr + retries: 15 + delay: 60 + until: + - manage_app_cr.resources is defined + - manage_app_cr.resources | length > 0 + - manage_app_cr.resources[0].status is defined + - manage_app_cr.resources[0].status.conditions is defined + - manage_app_cr.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | list | length > 0 + - (manage_app_cr.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | first).status == 'True' + +- name: "ManageApp CR is ready" + debug: + msg: "ManageApp CR {{ mas_instance_id }} is ready" + +# 6. Check if ManageWorkspace CR is already deployed +# ----------------------------------------------------------------------------- +- name: "Display restore phase 2 information" + debug: + msg: + - "==========================================" + - "Phase 2: Check if ManageWorkspace CR is already deployed and scale down" + - "==========================================" + - "Namespace: {{ mas_app_namespace }}" + - "Backup path: {{ manage_backup_path }}" + +- name: "Check if ManageWorkspace CR exists" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + kind: ManageWorkspace + name: "{{ manage_workspace_cr_name }}" + namespace: "{{ mas_app_namespace }}" + register: existing_manage_workspace_cr + +- name: "Set fact: ManageWorkspace CR exists" + set_fact: + manage_workspace_exists: "{{ existing_manage_workspace_cr.resources is defined and existing_manage_workspace_cr.resources | length > 0 }}" + +- name: "Display ManageWorkspace CR status" + debug: + msg: "ManageWorkspace CR {{ 'exists' if manage_workspace_exists else 'does not exist' }}" + +# 7. If ManageWorkspace exists, set mode to down +# ----------------------------------------------------------------------------- +- name: "Set ManageWorkspace mode to down (if exists)" + when: manage_workspace_exists + block: + - name: "Get current ManageWorkspace spec.mode" + set_fact: + current_mode: "{{ existing_manage_workspace_cr.resources[0].spec.settings.deployment.mode | default('up') }}" + + - name: "Display current mode" + debug: + msg: "Current ManageWorkspace mode: {{ current_mode }}" + + - name: "Set ManageWorkspace mode to down" + kubernetes.core.k8s: + api_version: apps.mas.ibm.com/v1 + kind: ManageWorkspace + name: "{{ manage_workspace_cr_name }}" + namespace: "{{ mas_app_namespace }}" + definition: + spec: + settings: + deployment: + mode: down + state: patched + when: current_mode != 'down' + + - name: "Wait for ManageWorkspace to scale down" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + kind: ManageWorkspace + name: "{{ manage_workspace_cr_name }}" + namespace: "{{ mas_app_namespace }}" + register: workspace_status + retries: 15 + delay: 60 + until: + - workspace_status.resources is defined + - workspace_status.resources | length > 0 + - workspace_status.resources[0].status is defined + - workspace_status.resources[0].status.conditions is defined + - workspace_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | list | length > 0 + - (workspace_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | first).status == 'False' + when: current_mode != 'down' + + - name: "ManageWorkspace scaled down successfully" + debug: + msg: "ManageWorkspace {{ manage_workspace_cr_name }} is now in down mode" + when: current_mode != 'down' + + - name: "ManageWorkspace already in down mode" + debug: + msg: "ManageWorkspace {{ manage_workspace_cr_name }} is already in down mode" + when: current_mode == 'down' diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml new file mode 100644 index 0000000000..07379d0ced --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml @@ -0,0 +1,316 @@ +--- +# Restore Manage Persistent Volumes +# ============================================================================= +# This task restores Manage persistent volume data by: +# 1. Reading the ManageWorkspace CR from backup to get PVC configuration +# 2. If ManageWorkspace CR is not deployed, create PVCs and a dummy pod +# 3. Restore PVC data using the pod +# 4. Shutdown the dummy pod +# 5. Continue with ManageWorkspace CR restore +# 6. Wait for Manage deployment to be activated + +# 1. Read ManageWorkspace CR from backup to get PVC configuration +# ----------------------------------------------------------------------------- +- name: "Display restore phase 3 information" + debug: + msg: + - "==========================================" + - "Phase 3: Restore Persistent Volume Data" + - "==========================================" + +- name: "Extract persistent volumes configuration from backup" + set_fact: + manage_persistent_volumes: "{{ workspace_backup_cr.spec.settings.deployment.persistentVolumes | default([]) }}" + +- name: "Display persistent volumes configuration" + debug: + msg: + - "Persistent volumes found in backup: {{ manage_persistent_volumes | length }}" + - "{{ manage_persistent_volumes | to_nice_yaml }}" + +# 2. Check if we need to restore PV data +# ----------------------------------------------------------------------------- +- name: "Check if PV data backup exists" + stat: + path: "{{ manage_backup_path }}/data" + register: pv_data_dir_stat + +- name: "Set fact: PV data needs restore" + set_fact: + pv_data_needs_restore: "{{ manage_persistent_volumes | length > 0 and pv_data_dir_stat.stat.exists and pv_data_dir_stat.stat.isdir }}" + +- name: "Display PV restore decision" + debug: + msg: "PV data restore {{ 'required' if pv_data_needs_restore else 'not required' }}" + +# 3. Restore PV data if needed +# ----------------------------------------------------------------------------- +- name: "Restore Manage persistent volume data" + when: pv_data_needs_restore + block: + # If ManageWorkspace doesn't exist, create PVCs and dummy pod + # ------------------------------------------------------------------------- + - name: "Display PVC creation information" + debug: + msg: + - "ManageWorkspace CR not deployed yet" + - "Creating PVCs and dummy pod for PVC data restoration" + + # Determine storage class override settings + # --------------------------------------------------------------------- + - name: "Display storage class override settings" + debug: + msg: + - "Storage class override enabled: {{ override_storageclass }}" + - "Custom RWX storage class: {{ mas_app_custom_storage_class_rwx | default('not set') }}" + - "Custom RWO storage class: {{ mas_app_custom_storage_class_rwo | default('not set') }}" + + # Get default storage classes if override is enabled but custom classes not provided + # --------------------------------------------------------------------- + - name: "Lookup default storage classes" + when: + - override_storageclass + - mas_app_custom_storage_class_rwx == '' or mas_app_custom_storage_class_rwo == '' + block: + - name: "Include default storage classes lookup" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" + + - name: "Set default RWX storage class if not provided" + set_fact: + mas_app_custom_storage_class_rwx: "{{ defaultStorageClasses.rwx }}" + when: mas_app_custom_storage_class_rwx == '' + + - name: "Set default RWO storage class if not provided" + set_fact: + mas_app_custom_storage_class_rwo: "{{ defaultStorageClasses.rwo }}" + when: mas_app_custom_storage_class_rwo == '' + + - name: "Fail storage class override if default storage classes not found" + fail: + msg: "Storage class override enabled but no custom storage classes provided and default storage classes not found" + when: mas_app_custom_storage_class_rwx == '' or mas_app_custom_storage_class_rwo == '' + + - name: Replace ManageWorkspace Persistent Volumes when override_storageclass is true + set_fact: + _manage_persistent_volumes: "{{ manage_persistent_volumes | ibm.mas_devops.override_manage_persistent_volumes(mas_app_custom_storage_class_rwo, mas_app_custom_storage_class_rwx) }}" + when: manage_persistent_volumes is defined and manage_persistent_volumes | length > 0 + + # Create PVCs + # --------------------------------------------------------------------- + - name: "Create PVCs for data restoration" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: "{{ pv_item.pvcName }}" + namespace: "{{ mas_app_namespace }}" + spec: + accessModes: "{{ pv_item.accessModes }}" + storageClassName: "{{ storage_class_to_use }}" + resources: + requests: + storage: "{{ pv_item.size }}" + vars: + storage_class_to_use: >- + {%- if override_storageclass -%} + {%- if 'ReadWriteMany' in pv_item.accessModes -%} + {{ mas_app_custom_storage_class_rwx }} + {%- else -%} + {{ mas_app_custom_storage_class_rwo }} + {%- endif -%} + {%- else -%} + {{ pv_item.storageClassName }} + {%- endif -%} + loop: "{{ manage_persistent_volumes }}" + loop_control: + loop_var: pv_item + + - name: "Wait for PVCs to be bound" + kubernetes.core.k8s_info: + api_version: v1 + kind: PersistentVolumeClaim + name: "{{ pv_item.pvcName }}" + namespace: "{{ mas_app_namespace }}" + register: pvc_status + retries: 60 + delay: 5 + until: + - pvc_status.resources is defined + - pvc_status.resources | length > 0 + - pvc_status.resources[0].status.phase == 'Bound' + loop: "{{ manage_persistent_volumes }}" + loop_control: + loop_var: pv_item + + # Create dummy pod to mount PVCs + # --------------------------------------------------------------------- + - name: "Create dummy pod for PVC data restoration" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Pod + metadata: + name: "{{ mas_instance_id }}-manage-restore-pvc-pod" + namespace: "{{ mas_app_namespace }}" + labels: + app: manage-restore + mas.ibm.com/instanceId: "{{ mas_instance_id }}" + spec: + restartPolicy: Never + containers: + - name: restore-container + image: registry.redhat.io/ubi8/ubi-minimal:latest + command: ["/bin/sh", "-c", "sleep infinity"] + volumeMounts: "{{ manage_persistent_volumes | map(attribute='mountPath') | zip(manage_persistent_volumes | map(attribute='pvcName')) | map('community.general.dict_kv', 'mountPath', 'name') | list }}" + volumes: "{{ manage_persistent_volumes | map('community.general.dict_kv', 'name', 'pvcName') | map('combine', {'persistentVolumeClaim': {'claimName': ''}}) | list }}" + vars: + volume_mounts: | + {% set mounts = [] %} + {% for pv in manage_persistent_volumes %} + {% set _ = mounts.append({'name': pv.pvcName, 'mountPath': pv.mountPath}) %} + {% endfor %} + {{ mounts }} + volumes: | + {% set vols = [] %} + {% for pv in manage_persistent_volumes %} + {% set _ = vols.append({'name': pv.pvcName, 'persistentVolumeClaim': {'claimName': pv.pvcName}}) %} + {% endfor %} + {{ vols }} + register: dummy_pod_created + + - name: "Wait for dummy pod to be running" + kubernetes.core.k8s_info: + api_version: v1 + kind: Pod + name: "{{ mas_instance_id }}-manage-restore-pvc-pod" + namespace: "{{ mas_app_namespace }}" + register: dummy_pod_status + retries: 60 + delay: 5 + until: + - dummy_pod_status.resources is defined + - dummy_pod_status.resources | length > 0 + - dummy_pod_status.resources[0].status.phase == 'Running' + + - name: "Set fact: restore pod name" + set_fact: + restore_pod_name: "{{ mas_instance_id }}-manage-restore-pvc-pod" + restore_pod_container: "restore-container" + using_dummy_pod: true + + # Restore each persistent volume + # ------------------------------------------------------------------------- + - name: "Display restore pod information" + debug: + msg: + - "Restore pod: {{ restore_pod_name }}" + - "Container: {{ restore_pod_container }}" + - "Using dummy pod: {{ using_dummy_pod }}" + + - name: "Restore each persistent volume" + include_tasks: "{{ role_path }}/tasks/manage/restore-single-pv.yml" + loop: "{{ manage_persistent_volumes }}" + loop_control: + loop_var: pv_item + index_var: pv_index + + # Shutdown dummy pod if created + # ------------------------------------------------------------------------- + - name: "Shutdown dummy pod" + when: using_dummy_pod | default(false) + block: + - name: "Delete dummy restore pod" + kubernetes.core.k8s: + api_version: v1 + kind: Pod + name: "{{ restore_pod_name }}" + namespace: "{{ mas_app_namespace }}" + state: absent + wait: yes + wait_timeout: 300 + + - name: "Dummy pod deleted successfully" + debug: + msg: "Dummy restore pod {{ restore_pod_name }} has been deleted" + + - name: "Display PV restore completion" + debug: + msg: + - "Manage PV restore completed" + - "Persistent volumes restored: {{ manage_persistent_volumes | length }}" + +# 4. Skip message if no PV data to restore +# ----------------------------------------------------------------------------- +- name: "Skip PV restore message" + debug: + msg: "Skipping Manage PV restore - no persistent volume data found in backup" + when: not pv_data_needs_restore + +# 5. Restore ManageWorkspace CR +# ----------------------------------------------------------------------------- +- name: "Display restore phase 4 information" + debug: + msg: + - "==========================================" + - "Phase 4: Restore ManageWorkspace CR" + - "==========================================" + +- name: "Restore ManageWorkspace CR" + ibm.mas_devops.restore_resource: + backup_path: "{{ manage_backup_path }}" + resource_kinds: + - ManageWorkspace + override_values: + ManageWorkspace: + - spec.settings.deployment.persistentVolumes: "{{ _manage_persistent_volumes }}" + register: manage_restore_phase4_result + +- name: "Display ManageWorkspace CR restore results" + debug: + msg: >- + Manage Worspace resource: {{ manage_restore_phase4_result.created_count }} created, + {{ manage_restore_phase4_result.updated_count }} updated, + {{ manage_restore_phase4_result.skipped_count }} skipped, + {{ manage_restore_phase4_result.failed_count }} failed + +- name: "Fail if ManageWorkspace CR restore had errors" + fail: + msg: "ManageWorkspace CR restore failed. See logs for details." + when: manage_restore_phase4_result.failed_count | int > 0 + +# 6. Wait for Manage deployment to be activated +# ----------------------------------------------------------------------------- +- name: "Display restore phase 5 information" + debug: + msg: + - "==========================================" + - "Phase 5: Wait for Manage Deployment" + - "==========================================" + +- name: "Wait for ManageWorkspace to be ready" + kubernetes.core.k8s_info: + api_version: apps.mas.ibm.com/v1 + kind: ManageWorkspace + name: "{{ manage_workspace_cr_name }}" + namespace: "{{ mas_app_namespace }}" + register: workspace_ready_status + retries: "{{ mas_app_restore_wait_retries | int }}" + delay: "{{ mas_app_restore_wait_delay | int }}" + until: + - workspace_ready_status.resources is defined + - workspace_ready_status.resources | length > 0 + - workspace_ready_status.resources[0].status is defined + - workspace_ready_status.resources[0].status.conditions is defined + - workspace_ready_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | list | length > 0 + - (workspace_ready_status.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | first).status == 'True' + +- name: "ManageWorkspace is ready" + debug: + msg: + - "==========================================" + - "ManageWorkspace {{ manage_workspace_cr_name }} is ready" + - "Manage deployment has been activated" + - "==========================================" diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-single-pv.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-single-pv.yml new file mode 100644 index 0000000000..e67373c0d1 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-single-pv.yml @@ -0,0 +1,132 @@ +--- +# Restore Single Persistent Volume +# ============================================================================= +# This task restores a single persistent volume's data from a tar.gz archive +# to the restore pod. + +- name: "Set fact: PV restore details" + set_fact: + pv_mount_path: "{{ pv_item.mountPath }}" + pv_pvc_name: "{{ pv_item.pvcName }}" + pv_archive_name: "{{ pv_item.pvcName }}.tar.gz" + pv_archive_path: "{{ manage_backup_path }}/data/{{ pv_item.pvcName }}.tar.gz" + +- name: "Display PV restore information" + debug: + msg: + - "Restoring PV {{ pv_index + 1 }}/{{ manage_persistent_volumes | length }}" + - "PVC Name: {{ pv_pvc_name }}" + - "Mount Path: {{ pv_mount_path }}" + - "Archive: {{ pv_archive_name }}" + +# Verify backup archive exists +# ----------------------------------------------------------------------------- +- name: "Verify backup archive exists" + stat: + path: "{{ pv_archive_path }}" + register: archive_stat + +- name: "Fail if backup archive not found" + fail: + msg: "Backup archive not found: {{ pv_archive_path }}" + when: not archive_stat.stat.exists + +- name: "Display archive information" + debug: + msg: + - "Archive size: {{ (archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + - "Archive location: {{ pv_archive_path }}" + +# Copy tar.gz archive from local backup to pod +# ----------------------------------------------------------------------------- +- name: "Copy tar.gz archive to restore pod" + shell: | + oc cp {{ pv_archive_path }} {{ mas_app_namespace }}/{{ restore_pod_name }}:/tmp/{{ pv_archive_name }} -c {{ restore_pod_container }} + register: copy_result + failed_when: false + +- name: "Check copy result" + debug: + msg: + - "Copy return code: {{ copy_result.rc }}" + - "{{ 'Success' if copy_result.rc == 0 else 'Failed' }}" + +- name: "Fail if copy failed" + fail: + msg: "Failed to copy tar archive to pod for {{ pv_pvc_name }}: {{ copy_result.stderr | default('Unknown error') }}" + when: copy_result.rc != 0 + +# Clear existing data in mount path (optional, be careful!) +# ----------------------------------------------------------------------------- +- name: "Clear existing data in mount path" + kubernetes.core.k8s_exec: + namespace: "{{ mas_app_namespace }}" + pod: "{{ restore_pod_name }}" + container: "{{ restore_pod_container }}" + command: > + sh -c "rm -rf {{ pv_mount_path }}/* {{ pv_mount_path }}/.[!.]* 2>/dev/null || true" + register: clear_result + failed_when: false + +- name: "Display clear result" + debug: + msg: "Cleared existing data in {{ pv_mount_path }}" + +# Extract tar.gz archive in the pod +# ----------------------------------------------------------------------------- +- name: "Extract tar.gz archive in pod at {{ pv_mount_path }}" + kubernetes.core.k8s_exec: + namespace: "{{ mas_app_namespace }}" + pod: "{{ restore_pod_name }}" + container: "{{ restore_pod_container }}" + command: > + tar -xzf /tmp/{{ pv_archive_name }} -C {{ pv_mount_path }} + register: extract_result + failed_when: false + +- name: "Check extract result" + debug: + msg: + - "Extract return code: {{ extract_result.rc }}" + - "{{ 'Success' if extract_result.rc == 0 else 'Failed' }}" + +- name: "Fail if extract failed" + fail: + msg: "Failed to extract tar archive for {{ pv_pvc_name }}: {{ extract_result.stderr | default('Unknown error') }}" + when: extract_result.rc != 0 + +# Clean up tar.gz archive from pod +# ----------------------------------------------------------------------------- +- name: "Remove tar.gz archive from pod" + kubernetes.core.k8s_exec: + namespace: "{{ mas_app_namespace }}" + pod: "{{ restore_pod_name }}" + container: "{{ restore_pod_container }}" + command: rm -f /tmp/{{ pv_archive_name }} + register: cleanup_result + failed_when: false + +# Verify restore was successful +# ----------------------------------------------------------------------------- +- name: "Verify data was restored" + kubernetes.core.k8s_exec: + namespace: "{{ mas_app_namespace }}" + pod: "{{ restore_pod_name }}" + container: "{{ restore_pod_container }}" + command: > + sh -c "ls -la {{ pv_mount_path }} | head -20" + register: verify_result + failed_when: false + +- name: "Display restored data verification" + debug: + msg: + - "Restored data in {{ pv_mount_path }}:" + - "{{ verify_result.stdout_lines | default(['No output']) }}" + when: verify_result.rc == 0 + +- name: "PV restore completed" + debug: + msg: + - "Successfully restored PVC: {{ pv_pvc_name }}" + - "Mount path: {{ pv_mount_path }}" diff --git a/ibm/mas_devops/roles/suite_app_restore/vars/manage.yml b/ibm/mas_devops/roles/suite_app_restore/vars/manage.yml new file mode 100644 index 0000000000..7566be6076 --- /dev/null +++ b/ibm/mas_devops/roles/suite_app_restore/vars/manage.yml @@ -0,0 +1,8 @@ +--- +# Restore Options +# ----------------------------------------------------------------------------- +# Wait retries for ManageWorkspace to be ready +mas_app_restore_wait_retries: "{{ lookup('env', 'MAS_APP_RESTORE_WAIT_RETRIES') | default('180', true) }}" + +# Delay between status checks (in seconds) +mas_app_restore_wait_delay: "{{ lookup('env', 'MAS_APP_RESTORE_WAIT_DELAY') | default('360', true) }}" diff --git a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml index 4ad659674f..2f3287477a 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml @@ -4,6 +4,8 @@ # Directory containing the backup folders mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" +mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + # Backup version to use for all component's backup backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" @@ -30,6 +32,6 @@ artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" # General settings -backup_archive_name: "mas-backup-{{ backup_version }}.tar.gz" -backup_temp_dir: "{{ mas_backup_dir }}/mas-backup-{{ backup_version }}" +backup_archive_name: "mas-{{ mas_instance_id }}-backup-{{ backup_version }}.tar.gz" +backup_temp_dir: "{{ mas_backup_dir }}/mas-{{ mas_instance_id }}-backup-{{ backup_version }}" upload_timeout: 3600 # Upload timeout in seconds (1 hour) diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml index 7a26701e93..9685e84f70 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml @@ -5,6 +5,11 @@ msg: "mas_backup_dir is required but not defined" when: mas_backup_dir is not defined or mas_backup_dir == '' +- name: "Fail if mas_instance_id is not defined" + ansible.builtin.fail: + msg: "mas_instance_id is required but not defined" + when: mas_instance_id is not defined or mas_instance_id == '' + - name: "Fail if backup_version is not defined" ansible.builtin.fail: msg: "backup_version is required but not defined" From 53e4c561c1cf7d21164778472e2477a66f467f55 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Wed, 18 Feb 2026 19:12:08 +0000 Subject: [PATCH 43/61] check for grafana crd exists before backing up dashboard in mongo backup --- ibm/mas_devops/plugins/action/crd_exists.py | 43 ++++++ .../backup-restore/backup-instance.yml | 143 ++++++++++-------- 2 files changed, 125 insertions(+), 61 deletions(-) create mode 100644 ibm/mas_devops/plugins/action/crd_exists.py diff --git a/ibm/mas_devops/plugins/action/crd_exists.py b/ibm/mas_devops/plugins/action/crd_exists.py new file mode 100644 index 0000000000..741433aad5 --- /dev/null +++ b/ibm/mas_devops/plugins/action/crd_exists.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import logging +import urllib3 +from ansible_collections.kubernetes.core.plugins.module_utils.k8s.client import get_api_client +from ansible.errors import AnsibleError +from ansible.plugins.action import ActionBase + +from mas.devops.ocp import crdExists + +urllib3.disable_warnings() # Disabling warnings will prevent InsecureRequestWarnings from dynClient +logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-20s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') + +class ActionModule(ActionBase): + """ + Usage Example + ------------- + tasks: + - name: "Check CRD exists" + ibm.mas_devops.crd_exists: + crdName: GrafanaDashboard + register: crd_exists + """ + def run(self, tmp=None, task_vars=None): + super(ActionModule, self).run(tmp, task_vars) + + # Target subscription + crdName = self._task.args.get('crdName', None) + + if crdName is None: + raise AnsibleError("Error: CRD Name argument was not provided") + + # Initialize DynamicClient and apply the Subscription + host = self._task.args.get('host', None) + api_key = self._task.args.get('api_key', None) + + dynClient = get_api_client(api_key=api_key, host=host) + isCRDPresent = crdExists(dynClient, crdName) + + return dict( + success=True, + exists=isCRDPresent + ) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml index 2db41832bf..818229c457 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-restore/backup-instance.yml @@ -1,71 +1,92 @@ --- # Backup Mongodb Community edition cluster Instance # ----------------------------------------------------------------------------- +# Check if GrafanaDashboard CRD exists +# ----------------------------------------------------------------------------- +- name: "Check if GrafanaDashboard CRD exists" + ibm.mas_devops.crd_exists: + crdName: grafanadashboards.grafana.integreatly.org + register: grafana_crd_check + - name: "Set fact: MongoDB CE backup resources" + set_fact: + mongoce_namespace_backup_resources: + - kind: Project + api_version: project.openshift.io/v1 + name: "{{ mongodb_namespace }}" + # CRD - mongodbcommunity.mongodbcommunity.mongodb.com + - kind: CustomResourceDefinition + api_version: apiextensions.k8s.io/v1 + name: mongodbcommunity.mongodbcommunity.mongodb.com + # mongodbcommunity.mongodb.com + - kind: MongoDBCommunity + api_version: mongodbcommunity.mongodb.com/v1 + name: "{{ mongodb_instance_name }}" + # Role + - kind: Role + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-kubernetes-operator + - kind: Role + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-database + # RoleBinding + - kind: RoleBinding + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-kubernetes-operator + - kind: RoleBinding + api_version: rbac.authorization.k8s.io/v1 + name: mongodb-database + # ServiceAccount + - kind: ServiceAccount + api_version: v1 + name: mongodb-kubernetes-operator + - kind: ServiceAccount + api_version: v1 + name: mongodb-database + # Issuers + - kind: Issuer + api_version: cert-manager.io/v1 + # Certificates + - kind: Certificate + api_version: cert-manager.io/v1 + # Deployment + - kind: Deployment + api_version: apps/v1 + name: mongodb-kubernetes-operator + # secrets + - kind: Secret + api_version: v1 + name: "{{ mongodb_instance_name }}-scram-scram-credentials" + - kind: Secret + api_version: v1 + name: "{{ mongodb_instance_name }}-admin-password" + # Configmap + - kind: ConfigMap + api_version: v1 + name: "{{ mongodb_instance_name }}-cert-map" + # Service Monitor + - kind: ServiceMonitor + api_version: monitoring.coreos.com/v1 + name: "{{ mongodb_instance_name }}-service-monitor" + +# Add GrafanaDashboard resource only if CRD exists +# ----------------------------------------------------------------------------- +- name: "Set fact: MongoDB Grafana Dashboard backup resources" + set_fact: + grafana_mongodb_backup_resources: + - kind: GrafanaDashboard + api_version: grafana.integreatly.org/v1beta1 + +- name: Add GrafanaDashboard to backup resources if CRD exists + set_fact: + mongoce_namespace_backup_resources: "{{ mongoce_namespace_backup_resources + grafana_mongodb_backup_resources }}" + when: grafana_crd_check.exists | bool + +- name: "Set fact: MongoDB backup resources" set_fact: mongoce_backup_resources: - namespace: "{{ mongodb_namespace }}" - resources: - - kind: Project - api_version: project.openshift.io/v1 - name: "{{ mongodb_namespace }}" - # CRD - mongodbcommunity.mongodbcommunity.mongodb.com - - kind: CustomResourceDefinition - api_version: apiextensions.k8s.io/v1 - name: mongodbcommunity.mongodbcommunity.mongodb.com - # mongodbcommunity.mongodb.com - - kind: MongoDBCommunity - api_version: mongodbcommunity.mongodb.com/v1 - name: "{{ mongodb_instance_name }}" - # Role - - kind: Role - api_version: rbac.authorization.k8s.io/v1 - name: mongodb-kubernetes-operator - - kind: Role - api_version: rbac.authorization.k8s.io/v1 - name: mongodb-database - # RoleBinding - - kind: RoleBinding - api_version: rbac.authorization.k8s.io/v1 - name: mongodb-kubernetes-operator - - kind: RoleBinding - api_version: rbac.authorization.k8s.io/v1 - name: mongodb-database - # ServiceAccount - - kind: ServiceAccount - api_version: v1 - name: mongodb-kubernetes-operator - - kind: ServiceAccount - api_version: v1 - name: mongodb-database - # Issuers - - kind: Issuer - api_version: cert-manager.io/v1 - # Certificates - - kind: Certificate - api_version: cert-manager.io/v1 - # Deployment - - kind: Deployment - api_version: apps/v1 - name: mongodb-kubernetes-operator - # secrets - - kind: Secret - api_version: v1 - name: "{{ mongodb_instance_name }}-scram-scram-credentials" - - kind: Secret - api_version: v1 - name: "{{ mongodb_instance_name }}-admin-password" - # Configmap - - kind: ConfigMap - api_version: v1 - name: "{{ mongodb_instance_name }}-cert-map" - # grafanaDashboard - - kind: GrafanaDashboard - api_version: grafana.integreatly.org/v1beta1 - # Service Monitor - - kind: ServiceMonitor - api_version: monitoring.coreos.com/v1 - name: "{{ mongodb_instance_name }}-service-monitor" + resources: "{{ mongoce_namespace_backup_resources }}" # Call the backup_resources plugin to execute the backup to the path provided # ----------------------------------------------------------------------------- From 259a7ecc895f66bd94996d8be8ac4536a4b73c0b Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 19 Feb 2026 09:36:56 +0000 Subject: [PATCH 44/61] tweak default region in upload and download roles --- .../roles/download_backup_archive/defaults/main.yml | 2 +- .../download_backup_archive/tasks/download_from_s3.yml | 8 ++------ .../roles/upload_backup_archive/defaults/main.yml | 2 +- .../roles/upload_backup_archive/tasks/upload_to_s3.yml | 8 ++------ 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml index 22cd07e294..bc79c665ed 100644 --- a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml @@ -13,7 +13,7 @@ backup_archive_name: "{{ lookup('env', 'BACKUP_ARCHIVE_NAME') }}" aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY_ID') }}" aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_ACCESS_KEY') }}" s3_bucket_name: "{{ lookup('env', 'S3_BUCKET_NAME') }}" -s3_region: "{{ lookup('env', 'S3_REGION') }}" +s3_region: "{{ lookup('env', 'S3_REGION') | default('us-east-1', true) }}" s3_endpoint_url: "{{ lookup('env', 'S3_ENDPOINT_URL') }}" # Artifactory Configuration (provide these to download from Artifactory) diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml index 7a88fa4177..d2fdf09015 100644 --- a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml @@ -1,14 +1,10 @@ --- # Download backup archive from S3 -- name: "Set S3 region default" - ansible.builtin.set_fact: - region: "{{ s3_region | default('us-east-1') }}" - - name: "Display S3 download information" ansible.builtin.debug: msg: - "Downloading from S3 bucket: {{ s3_bucket_name }}" - - "Region: {{ region }}" + - "Region: {{ s3_region }}" - "Archive: {{ backup_archive_name }}" - "Download to: {{ backup_temp_dir }}" @@ -19,7 +15,7 @@ bucket_name: "{{ s3_bucket_name }}" object_name: "{{ backup_archive_name }}" local_dir: "{{ backup_temp_dir }}" - region_name: "{{ region }}" + region_name: "{{ s3_region }}" endpoint_url: "{{ s3_endpoint_url | default(omit) }}" register: s3_download_result poll: 10 diff --git a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml index 2f3287477a..c7c9599bca 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml @@ -22,7 +22,7 @@ manage_backup_version: "{{ lookup('env', 'MANAGE_BACKUP_VERSION') | default (bac aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY_ID') }}" aws_secret_access_key: "{{ lookup('env', 'S3_SECRET_ACCESS_KEY') }}" s3_bucket_name: "{{ lookup('env', 'S3_BUCKET_NAME') }}" -s3_region: "{{ lookup('env', 'S3_REGION') }}" +s3_region: "{{ lookup('env', 'S3_REGION') | default('us-east-1', true) }}" s3_endpoint_url: "{{ lookup('env', 'S3_ENDPOINT_URL') }}" # Artifactory Configuration (provide these to upload to Artifactory) diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml index 59685fbb1d..d4568a0387 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml @@ -1,14 +1,10 @@ --- # Upload backup archive to S3 -- name: "Set S3 region default" - ansible.builtin.set_fact: - region: "{{ s3_region | default('us-east-1') }}" - - name: "Display S3 upload information" ansible.builtin.debug: msg: - "Uploading to S3 bucket: {{ s3_bucket_name }}" - - "Region: {{ region }}" + - "Region: {{ s3_region }}" - "Archive: {{ backup_archive_name }}" - name: "Upload archive to S3" @@ -18,7 +14,7 @@ bucket_name: "{{ s3_bucket_name }}" object_name: "{{ backup_archive_name }}" file_path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" - region_name: "{{ region }}" + region_name: "{{ s3_region }}" endpoint_url: "{{ s3_endpoint_url | default(omit) }}" register: s3_upload_result poll: 10 From ab82468faa6202df5268142002d72077729281bf Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 23 Feb 2026 12:20:53 +0000 Subject: [PATCH 45/61] add manage restore to playbook and more tweaks (#2129) Co-authored-by: Sanjay Prabhakar --- docs/playbooks/backup-restore.md | 601 ++++++++++++++++++++- ibm/mas_devops/playbooks/br_core.yml | 7 + ibm/mas_devops/playbooks/br_db2.yml | 13 +- ibm/mas_devops/playbooks/br_manage.yml | 48 +- ibm/mas_devops/playbooks/br_mongodb.yml | 12 +- ibm/mas_devops/roles/db2/defaults/main.yml | 2 +- 6 files changed, 671 insertions(+), 12 deletions(-) diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 573cfb036d..480441ad4b 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -13,10 +13,10 @@ Overview MAS Devops Collection includes playbooks for backing up and restoring MAS components and their dependencies. The backup and restore operations are designed to protect your MAS installation and enable disaster recovery, cluster migration, and testing scenarios. **Supported Components:** +- [MongoDB Community](#mongodb-community-backup-and-restore) - [MAS Core](#mas-core-backup-and-restore) -- MongoDB Community Edition -- Db2 (Coming soon) -- Manage (Coming soon) +- [Db2 Backup and Restore](#db2-backup-and-restore) +- [Manage Application](#manage-application-backup-and-restore) **Roles Supporting Backup/Restore:** - [`ibm.mas_devops.cert_manager`](../roles/cert_manager.md) @@ -26,6 +26,206 @@ MAS Devops Collection includes playbooks for backing up and restoring MAS compon - [`ibm.mas_devops.sls`](../roles/sls.md) - [`ibm.mas_devops.suite_backup`](../roles/suite_backup.md) - [`ibm.mas_devops.suite_restore`](../roles/suite_restore.md) +- [`ibm.mas_devops.suite_app_backup`](../roles/suite_app_backup.md) +- [`ibm.mas_devops.suite_app_restore`](../roles/suite_app_restore.md) + +MongoDB Community Backup and Restore +=============================================================================== + +## Overview +This playbook performs backup and restore operations for MongoDB Community Edition instances. It supports backing up both the MongoDB instance configuration and database data. + +**Important**: +- Supports MongoDB Community Edition only +- Can backup/restore entire instance or individual databases +- Backup includes both Kubernetes resources and database data + +## Playbook Content + +The playbook executes the following operations: + +### Backup Operation +1. [Backup MongoDB Instance Resources](../roles/mongodb.md) - Kubernetes resources (StatefulSet, Services, ConfigMaps, Secrets) +2. [Backup MongoDB Database Data](../roles/mongodb.md) - Database data using mongodump + +### Restore Operation +1. [Restore MongoDB Instance Resources](../roles/mongodb.md) - Recreate Kubernetes resources +2. [Restore MongoDB Database Data](../roles/mongodb.md) - Restore database data using mongorestore + +## Required Environment Variables + +### Common Variables (Backup and Restore) +- `MAS_INSTANCE_ID` - The instance ID of the MAS installation +- `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) +- `MONGODB_ACTION` - Set to `backup`, `backup_database`, `restore`, or `restore_database` +- `MONGODB_INSTANCE_NAME` - MongoDB instance name (default: `mas-mongo-ce`) +- `MONGODB_NAMESPACE` - Namespace where MongoDB is installed (default: `mongoce`) + +### Backup-Specific Variables +- `MONGODB_BACKUP_VERSION` - (Optional) Custom version identifier for the backup. If not provided, defaults to timestamp format `YYYYMMDD-HHMMSS` + +### Restore-Specific Variables +- `MONGODB_BACKUP_VERSION` - (Required) The backup version identifier to restore + +## Optional Environment Variables + +### Storage Class Override (Restore) +- `OVERRIDE_STORAGECLASS` - Set to `true` to override storage class names from backup (default: `false`) +- `MONGODB_STORAGECLASS_NAME_RWO` - Custom RWO storage class for MongoDB + +### Application-Specific +- `MAS_APP_ID` - (Optional) MAS application ID if backing up application-specific database + +## Usage Examples + +### Backup MongoDB Instance +Create a complete backup of MongoDB instance and all databases: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export MONGODB_ACTION=backup +export MONGODB_INSTANCE_NAME=mas-mongo-ce +export MONGODB_NAMESPACE=mongoce + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_mongodb +``` + +### Backup with Custom Version +Create a backup with a custom version identifier: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export MONGODB_ACTION=backup +export MONGODB_BACKUP_VERSION=pre-upgrade-mongo +export MONGODB_INSTANCE_NAME=mas-mongo-ce + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_mongodb +``` + +### Backup Individual Database +Create a backup of a specific database only: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export MONGODB_ACTION=backup_database +export MONGODB_INSTANCE_NAME=mas-mongo-ce +export MAS_APP_ID=manage + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_mongodb +``` + +### Restore MongoDB Instance +Restore MongoDB instance from a backup: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export MONGODB_ACTION=restore +export MONGODB_BACKUP_VERSION=20260122-131500 +export MONGODB_INSTANCE_NAME=mas-mongo-ce +export MONGODB_NAMESPACE=mongoce + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_mongodb +``` + +### Restore with Storage Class Override +Restore MongoDB to a different cluster with different storage class: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export MONGODB_ACTION=restore +export MONGODB_BACKUP_VERSION=20260122-131500 +export MONGODB_INSTANCE_NAME=mas-mongo-ce +export MONGODB_NAMESPACE=mongoce + +# Override storage class +export OVERRIDE_STORAGECLASS=true +export MONGODB_STORAGECLASS_NAME_RWO=ocs-storagecluster-ceph-rbd + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_mongodb +``` + +### Restore Individual Database +Restore a specific database only: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export MONGODB_ACTION=restore_database +export MONGODB_BACKUP_VERSION=20260122-131500 +export MONGODB_INSTANCE_NAME=mas-mongo-ce +export MAS_APP_ID=manage + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_mongodb +``` + +## Important Considerations + +### Backup Actions +- **backup**: Full backup of MongoDB instance (Kubernetes resources + all database data) +- **backup_database**: Backup specific database data only (requires `MAS_APP_ID`) + +### Restore Actions +- **restore**: Full restore of MongoDB instance (Kubernetes resources + all database data) +- **restore_database**: Restore specific database data only (requires `MAS_APP_ID`) + +### Prerequisites for Restore +- Target cluster must have MongoDB Community Operator installed +- Sufficient storage capacity for database restoration +- Same or compatible MongoDB version as the backup +- Target cluster must use the same MAS instance ID as the backup + +### Backup Best Practices +1. **Regular Schedule**: Perform backups regularly, especially before: + - MongoDB upgrades + - MAS upgrades + - Configuration changes +2. **Full vs Database Backups**: + - Use full backups for complete disaster recovery + - Use database backups for application-specific data protection +3. **Test Restores**: Periodically test restore procedures in non-production environments +4. **Secure Storage**: Store backups in a secure location separate from the cluster + +### Restore Best Practices +1. **Pre-Restore Validation**: + - Verify backup archive exists and is complete + - Confirm target cluster has sufficient resources + - Verify MongoDB instance name matches the backup +2. **Post-Restore Verification**: + - Verify MongoDB pods are running + - Test database connectivity + - Verify data integrity + - Check application connectivity to MongoDB + +### Storage Requirements +- Plan for sufficient storage for MongoDB backups +- Database backups can be large depending on data size +- Backup directory structure: `{mas_backup_dir}/backup-{version}-mongoce/` + +### Security Considerations +- Backup files contain sensitive data including database contents and credentials +- Secure backup directory with appropriate permissions (chmod 700 recommended) +- Consider encrypting backups for long-term storage +- Restrict access to backup files to authorized personnel only + +!!! tip + If you do not want to set up all the dependencies on your local system, you can run the playbook inside our docker image: `docker run -ti --pull always quay.io/ibmmas/cli` + +## Additional Resources + +For detailed information about MongoDB backup and restore operations, refer to the role documentation: +- [MongoDB Backup/Restore](../roles/mongodb.md) + MAS Core Backup and Restore =============================================================================== @@ -279,4 +479,399 @@ For detailed information about individual backup and restore operations, refer t - [SLS Backup/Restore](../roles/sls.md) - [MAS Core Backup](../roles/suite_backup.md) - [MAS Core Restore](../roles/suite_restore.md) +- [Db2 Backup/Restore](../roles/db2.md) + +Db2 Backup and Restore +=============================================================================== + +## Overview +This playbook performs backup and restore operations for IBM Db2 Universal Operator instances. It supports both online and offline backups, and can store backups either on disk or in S3-compatible object storage(database backups only). + +**Important**: The playbook supports multiple backup actions: +- `backup` - Full Db2 instance backup +- `backup_database` - Individual database backup +- `restore` - Full Db2 instance restore +- `restore_database` - Individual database restore + +## Required Environment Variables + +### Common Variables (Backup and Restore) +- `MAS_INSTANCE_ID` - The instance ID of the MAS installation +- `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) +- `DB2_INSTANCE_NAME` - Name of the Db2 instance +- `DB2_ACTION` - Set to `backup`, `backup_database`, `restore`, or `restore_database` + +### Backup-Specific Variables +- `DB2_BACKUP_TYPE` - Set to `online` or `offline` (default: `online`) +- `BACKUP_VENDOR` - Set to `disk` or `s3` (default: `disk`) + +### Restore-Specific Variables +- `DB2_BACKUP_VERSION` - (Required) The backup version identifier to restore + +### S3 Storage Variables (when BACKUP_VENDOR=s3) +- `BACKUP_S3_ALIAS` - S3 alias name (default: `S3DB2COS`) +- `BACKUP_S3_ENDPOINT` - S3 endpoint URL +- `BACKUP_S3_BUCKET` - S3 bucket name +- `BACKUP_S3_ACCESS_KEY` - S3 access key +- `BACKUP_S3_SECRET_KEY` - S3 secret key + +## Optional Environment Variables + +### Db2 Configuration +- `DB2_NAMESPACE` - Namespace where Db2 is installed (default: `db2u`) + +### Storage Class Override (Restore) +- `OVERRIDE_STORAGECLASS` - Set to `true` to override storage class names from backup (default: `false`) +- `DB2_META_STORAGE_CLASS` - Storage class for metadata +- `DB2_DATA_STORAGE_CLASS` - Storage class for data +- `DB2_BACKUP_STORAGE_CLASS` - Storage class for backups +- `DB2_LOGS_STORAGE_CLASS` - Storage class for logs +- `DB2_TEMP_STORAGE_CLASS` - Storage class for temporary files + +## Usage Examples + +### Backup Db2 to Disk (Online) +Create an online backup of Db2 instance to local disk: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=backup +export DB2_BACKUP_TYPE=online +export BACKUP_VENDOR=disk + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 +``` + +### Backup Db2 to S3 +Create a backup of Db2 instance to S3 storage: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=backup +export DB2_BACKUP_TYPE=online +export BACKUP_VENDOR=s3 +export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud +export BACKUP_S3_BUCKET=mas-db2-backups +export BACKUP_S3_ACCESS_KEY=your-access-key +export BACKUP_S3_SECRET_KEY=your-secret-key + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 +``` + +### Restore Db2 from Backup +Restore Db2 instance from a previous backup: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=restore +export DB2_BACKUP_VERSION=20260122-131500 +export BACKUP_VENDOR=disk + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 +``` + +### Restore with Storage Class Override +Restore Db2 to a different cluster with different storage classes: + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=restore +export DB2_BACKUP_VERSION=20260122-131500 +export BACKUP_VENDOR=disk + +# Override storage classes +export OVERRIDE_STORAGECLASS=true +export DB2_META_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_DATA_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_BACKUP_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_LOGS_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_TEMP_STORAGE_CLASS=ocs-storagecluster-ceph-rbd + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 +``` + +## Important Considerations + +### Backup Types +- **Online Backup**: Database remains available during backup (recommended for production) +- **Offline Backup**: Database is taken offline during backup (faster but causes downtime) + +### Storage Vendor Options +- **Disk**: Stores backups on local filesystem or mounted storage +- **S3**: Stores backups in S3-compatible object storage (recommended for production) + +### Prerequisites for Restore +- Target cluster must have Db2 Universal Operator installed +- Sufficient storage capacity for database restoration +- Same or compatible Db2 version as the backup + +Manage Application Backup and Restore +=============================================================================== + +## Overview +This playbook performs backup and restore operations for IBM Maximo Manage application. + +**Important**: +- Backup can only be restored to an instance with the same MAS instance ID +- **DB2 backup and restore is NOT automatically included** in the Manage backup/restore playbook +- You **MUST** run the [DB2 backup and restore playbook](#db2-backup-and-restore) as a prerequisite step before running Manage backup or restore operations + +## Playbook Content + +The playbook executes the following operations: + +### Backup Operation +1. **[Backup Db2 Database](../roles/db2.md) - PREREQUISITE STEP** (run `br_db2.yml` playbook separately first) +2. [Backup Manage Application](../roles/suite_app_backup.md) + +### Restore Operation +1. **[Restore Db2 Database](../roles/db2.md) - PREREQUISITE STEP** (run `br_db2.yml` playbook separately first) +2. [Restore Manage Application](../roles/suite_app_restore.md) + +## Required Environment Variables + +### Common Variables (Backup and Restore) +- `MAS_INSTANCE_ID` - The instance ID of the MAS installation +- `MAS_WORKSPACE_ID` - The workspace ID for Manage +- `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) +- `MAS_APP_ACTION` - Set to `backup` or `restore` (default: `backup`) + +### Backup-Specific Variables +- `MAS_APP_BACKUP_VERSION` - (Optional) Custom version identifier for the backup. If not provided, defaults to timestamp format `YYYYMMDD-HHMMSS` + +### Restore-Specific Variables +- `MAS_APP_BACKUP_VERSION_TO_RESTORE` - (Required) The backup version identifier to restore + +!!! warning "DB2 Backup/Restore Prerequisites" + Before running Manage backup or restore, you **MUST** first run the DB2 backup or restore playbook separately. See the [DB2 Backup and Restore](#db2-backup-and-restore) section for all required DB2 environment variables including: + + - `DB2_INSTANCE_NAME` + - `DB2_ACTION` (set to `backup` or `restore`) + - `DB2_BACKUP_TYPE` + - `BACKUP_VENDOR` + - `DB2_BACKUP_VERSION` (for restore operations) + - S3 variables (if using S3 storage) + +## Optional Environment Variables + +### Storage Class Override (Restore) +- `OVERRIDE_STORAGECLASS` - Set to `true` to override storage class names from backup (default: `false`) +- `MAS_APP_CUSTOM_STORAGE_CLASS_RWO` - Custom RWO storage class for Manage +- `MAS_APP_CUSTOM_STORAGE_CLASS_RWX` - Custom RWX storage class for Manage + +### Db2 Configuration +- `DB2_NAMESPACE` - Namespace where Db2 is installed (default: `db2u`) + +## Usage Examples + +### Backup Manage Application +Create a complete backup of Manage application. **Note:** You must backup DB2 first as a separate step. + +```bash +# STEP 1: Backup DB2 database (PREREQUISITE) +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=backup +export DB2_BACKUP_TYPE=online +export BACKUP_VENDOR=disk + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 + +# STEP 2: Backup Manage application +export MAS_INSTANCE_ID=inst1 +export MAS_WORKSPACE_ID=masdev +export MAS_BACKUP_DIR=/backup/mas +export MAS_APP_ACTION=backup + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_manage +``` + +### Backup with Custom Version +Create a backup with a custom version identifier: + +```bash +# STEP 1: Backup DB2 database (PREREQUISITE) +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=backup +export DB2_BACKUP_TYPE=online +export BACKUP_VENDOR=disk + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 + +# STEP 2: Backup Manage application with custom version +export MAS_INSTANCE_ID=inst1 +export MAS_WORKSPACE_ID=masdev +export MAS_BACKUP_DIR=/backup/mas +export MAS_APP_ACTION=backup +export MAS_APP_BACKUP_VERSION=pre-upgrade-manage + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_manage +``` + +### Backup to S3 Storage +Create a backup storing DB2 and Manage data to S3: + +```bash +# STEP 1: Backup DB2 database to S3 (PREREQUISITE) +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=backup +export DB2_BACKUP_TYPE=online +export BACKUP_VENDOR=s3 +export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud +export BACKUP_S3_BUCKET=mas-manage-backups +export BACKUP_S3_ACCESS_KEY=your-access-key +export BACKUP_S3_SECRET_KEY=your-secret-key + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 + +# STEP 2: Backup Manage application +export MAS_INSTANCE_ID=inst1 +export MAS_WORKSPACE_ID=masdev +export MAS_BACKUP_DIR=/backup/mas +export MAS_APP_ACTION=backup + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_manage +``` + +### Restore Manage Application +Restore Manage application. **Note:** You must restore DB2 first as a separate step. + +```bash +# STEP 1: Restore DB2 database (PREREQUISITE) +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=restore +export DB2_BACKUP_VERSION=20260122-131500 +export BACKUP_VENDOR=disk + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 + +# STEP 2: Restore Manage application +export MAS_INSTANCE_ID=inst1 +export MAS_WORKSPACE_ID=masdev +export MAS_BACKUP_DIR=/backup/mas +export MAS_APP_ACTION=restore +export MAS_APP_BACKUP_VERSION_TO_RESTORE=20260122-131500 + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_manage +``` + +### Restore with Storage Class Override +Restore Manage to a different cluster with different storage classes: + +```bash +# STEP 1: Restore DB2 database with storage override (PREREQUISITE) +export MAS_INSTANCE_ID=inst1 +export MAS_BACKUP_DIR=/backup/mas +export DB2_INSTANCE_NAME=db2w-shared +export DB2_ACTION=restore +export DB2_BACKUP_VERSION=20260122-131500 +export BACKUP_VENDOR=disk +export OVERRIDE_STORAGECLASS=true +export DB2_META_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_DATA_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_BACKUP_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_LOGS_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +export DB2_TEMP_STORAGE_CLASS=ocs-storagecluster-ceph-rbd + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_db2 + +# STEP 2: Restore Manage application with storage override +export MAS_INSTANCE_ID=inst1 +export MAS_WORKSPACE_ID=masdev +export MAS_BACKUP_DIR=/backup/mas +export MAS_APP_ACTION=restore +export MAS_APP_BACKUP_VERSION_TO_RESTORE=20260122-131500 +export OVERRIDE_STORAGECLASS=true +export MAS_APP_CUSTOM_STORAGE_CLASS_RWO=ocs-storagecluster-ceph-rbd +export MAS_APP_CUSTOM_STORAGE_CLASS_RWX=ocs-storagecluster-cephfs + +oc login --token=xxxx --server=https://myocpserver +ansible-playbook ibm.mas_devops.br_manage +``` + +## Important Considerations + +### Prerequisites for Restore +- Target cluster must have MAS Core installed and configured +- Target cluster must have Db2 Universal Operator installed +- Workspace must exist with the same workspace ID +- Sufficient resources (CPU, memory, storage) for both Db2 and Manage +- Target cluster must use the same MAS instance ID as the backup + +### Backup Best Practices +1. **Two-Step Process**: Always backup DB2 first, then Manage application + - Run `br_db2.yml` playbook before `br_manage.yml` + - DB2 backup is NOT automatically included in Manage backup +2. **Version Alignment**: Use consistent version identifiers for both DB2 and Manage backups for easier tracking +3. **Regular Schedule**: Perform backups regularly, especially before: + - Manage upgrades or updates + - Configuration changes + - Data migrations +4. **Test Restores**: Periodically test restore procedures in non-production environments +5. **Secure Storage**: Store backups in a secure location, preferably using S3 storage + +### Restore Best Practices +1. **Pre-Restore Validation**: + - Verify both DB2 and Manage backup archives exist + - Confirm target cluster has sufficient resources + - Verify MAS instance ID and workspace ID match the backup +2. **Restore Order**: **CRITICAL** - Always restore DB2 first, then Manage application + - Run `br_db2.yml` playbook before `br_manage.yml` + - DB2 restore is NOT automatically included in Manage restore +3. **Post-Restore Verification**: + - Verify DB2 instance is running and accessible + - Verify Manage workspace status is Ready + - Test Manage application functionality + - Verify data integrity + +### Storage Requirements +- Plan for sufficient storage for both Db2 and Manage backups +- Db2 backups can be large depending on database size +- Manage application configuration is relatively small +- Consider using S3 storage for production backups + +### Security Considerations +- Backup files contain sensitive data including database contents and credentials +- Secure backup directory with appropriate permissions +- Consider encrypting backups for long-term storage +- Restrict access to backup files to authorized personnel only +- Ensure secure transfer of backup files to restore environment + +!!! tip + If you do not want to set up all the dependencies on your local system, you can run the playbook inside our docker image: `docker run -ti --pull always quay.io/ibmmas/cli` + +## Additional Resources + +For detailed information about individual backup and restore operations, refer to the role documentation: +- [Db2 Backup/Restore](../roles/db2.md) +- [Manage Application Backup](../roles/suite_app_backup.md) +- [Manage Application Restore](../roles/suite_app_restore.md) diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml index 2eeb9fd59c..6a6ec01277 100644 --- a/ibm/mas_devops/playbooks/br_core.yml +++ b/ibm/mas_devops/playbooks/br_core.yml @@ -37,6 +37,11 @@ dro_contact_firstname: "{{ lookup('env', 'DRO_CONTACT_FIRSTNAME') }}" dro_contact_lastname: "{{ lookup('env', 'DRO_CONTACT_LASTNAME') }}" + # mongo restore configuration + override_mongodb_storageclass: "{{ lookup('env', 'OVERRIDE_MONGODB_STORAGECLASS') | default('false', true) | bool }}" # Set to true to override + # when OVERRIDE_MONGODB_STORAGECLASS to true, MONGODB_STORAGECLASS_NAME_RWO will be used + mongodb_storageclass_rwo: "{{ lookup('env', 'MONGODB_STORAGECLASS_NAME_RWO') | default('', true) }}" + # Set the following vars to control the domain and urls used on the restore. # This is used when you are restoring to a different cluster than the one that was backed up, # and the domain will be different. @@ -120,6 +125,8 @@ mongodb_backup_version: "{{ br_action_version }}" mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" mongodb_provider: "community" # only community is supported + override_storageclass: "{{ override_mongodb_storageclass }}" + mongodb_storage_class: "{{ mongodb_storageclass_rwo }}" - role: ibm.mas_devops.sls when: include_sls | bool diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index f393957c33..bd5a5ef384 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -10,13 +10,21 @@ db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or backup_database or restore or restore_database db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" # Required for restore action backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline and required for backup - backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) }}" + backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) | lower }}" backup_s3_alias: "{{ lookup('env', 'BACKUP_S3_ALIAS') | default('S3DB2COS', true) }}" # Required for backup_vendor=s3 backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" # Required for backup_vendor=s3 backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" # Required for backup_vendor=s3 backup_s3_access_key: "{{ lookup('env', 'BACKUP_S3_ACCESS_KEY') }}" # Required for backup_vendor=s3 backup_s3_secret_key: "{{ lookup('env', 'BACKUP_S3_SECRET_KEY') }}" # Required for backup_vendor=s3 - + # Set OVERRIDE_STORAGECLASS to true to override the storage class names in backup + override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default(false, true) }}" + # Set OVERRIDE_STORAGECLASS to true and use the below storage classes. + # when OVERRIDE_STORAGECLASS is true and below classes are not set, then the cluster's default storage classes will be used. + db2_meta_storage_class: "{{ lookup('env', 'DB2_META_STORAGE_CLASS') }}" + db2_data_storage_class: "{{ lookup('env', 'DB2_DATA_STORAGE_CLASS') }}" + db2_backup_storage_class: "{{ lookup('env', 'DB2_BACKUP_STORAGE_CLASS') }}" + db2_logs_storage_class: "{{ lookup('env', 'DB2_LOGS_STORAGE_CLASS') }}" + db2_temp_storage_class: "{{ lookup('env', 'DB2_TEMP_STORAGE_CLASS') }}" pre_tasks: - name: "Fail if DB2_ACTION is not set to backup|restore" @@ -44,6 +52,7 @@ backup_s3_secret_key: "{{ backup_s3_secret_key }}" action: "s3_setup" component: "db2" + when: backup_vendor == "s3" roles: - role: ibm.mas_devops.db2 diff --git a/ibm/mas_devops/playbooks/br_manage.yml b/ibm/mas_devops/playbooks/br_manage.yml index c5326edbea..56dc1014bb 100644 --- a/ibm/mas_devops/playbooks/br_manage.yml +++ b/ibm/mas_devops/playbooks/br_manage.yml @@ -1,20 +1,36 @@ --- -- name: "MAS Manage Application Backup" +- name: "MAS Manage Application Backup and Restore" hosts: localhost any_errors_fatal: true vars: - # Define the target for backup + # Define the target for backup/restore mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" mas_app_id: "manage" - # The top level location to save the archive files + # Set the action for this playbook of either backup or restore + mas_app_action: "{{ lookup('env', 'MAS_APP_ACTION') | default('backup', true) }}" + + # The top level location to save (on backup) or retrieve (on restore) the archive files mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups # Backup version to use during backup action # Note: The default timestamp will be set in pre_tasks mas_app_backup_version: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION') }}" + # Required for restore action, the backup version to use on restore + mas_app_backup_version_to_restore: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION_TO_RESTORE') }}" + + # Computed variable: use mas_app_backup_version for backup action, mas_app_backup_version_to_restore for restore action + mas_app_action_version: "{{ mas_app_backup_version if mas_app_action == 'backup' else mas_app_backup_version_to_restore }}" + + # Storage Class Override Options for restore + # Set OVERRIDE_STORAGECLASS to true to override the storage class names from backup + override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default('false', true) | bool }}" + # When OVERRIDE_STORAGECLASS is true, use the below storage classes + mas_app_custom_storage_class_rwo: "{{ lookup('env', 'MAS_APP_CUSTOM_STORAGE_CLASS_RWO') | default('', true) }}" + mas_app_custom_storage_class_rwx: "{{ lookup('env', 'MAS_APP_CUSTOM_STORAGE_CLASS_RWX') | default('', true) }}" + pre_tasks: - name: "Set mas_app_backup_version with timestamp if not provided" ansible.builtin.set_fact: @@ -42,7 +58,33 @@ - mas_backup_dir != '' fail_msg: "mas_backup_dir is required and must be set" + - name: "Fail if mas_app_action is not set to backup|restore" + ansible.builtin.assert: + that: + - mas_app_action is defined + - mas_app_action in ["backup", "restore"] + fail_msg: "mas_app_action is required and must be set to 'backup' or 'restore'" + + - name: "Fail if mas_app_backup_version_to_restore is not set on restore" + ansible.builtin.assert: + that: + - mas_app_backup_version_to_restore is defined + - mas_app_backup_version_to_restore != '' + fail_msg: "mas_app_backup_version_to_restore is required when running 'restore'" + when: mas_app_action == "restore" + roles: - role: ibm.mas_devops.suite_app_backup + when: mas_app_action == "backup" + vars: + mas_app_id: "manage" + mas_app_backup_version: "{{ mas_app_action_version }}" + + - role: ibm.mas_devops.suite_app_restore + when: mas_app_action == "restore" vars: mas_app_id: "manage" + mas_app_backup_version: "{{ mas_app_action_version }}" + override_storageclass: "{{ override_storageclass }}" + mas_app_custom_storage_class_rwo: "{{ mas_app_custom_storage_class_rwo }}" + mas_app_custom_storage_class_rwx: "{{ mas_app_custom_storage_class_rwx }}" diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index 490661d42b..3dab59d7e5 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -4,19 +4,25 @@ vars: # Define the target for backup/restore mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('backup', true) }}" # backup or restore_database or install + mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('backup', true) }}" # backup or backup_database or restore_database or install mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups - mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" # Required for install from backup or restore_database action + mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" # Required for restore or restore_database action mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" mongodb_namespace: "{{ lookup('env', 'MONGODB_NAMESPACE') | default('mongoce', true) }}" mongodb_provider: "community" # only community is supported currently mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Optional + # mongo restore configuration + override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default('false', true) | bool }}" # Set to true to override + # when OVERRIDE_MONGODB_STORAGECLASS to true, MONGODB_STORAGECLASS_NAME_RWO will be used + mongodb_storageclass_rwo: "{{ lookup('env', 'MONGODB_STORAGECLASS_NAME_RWO') | default('', true) }}" pre_tasks: - - name: "Fail if mongodb_action is not set to backup|install|restore_database" + - name: "Fail if mongodb_action is not set to backup|backup_database|restore|restore_database" assert: that: mongodb_action in ["backup", "backup_database", "restore_database", "restore"] fail_msg: "mongodb_action is required and must be set to 'backup' or 'backup_database' or 'restore' or 'restore_database'" roles: - role: ibm.mas_devops.mongodb + vars: + mongodb_storage_class: "{{ mongodb_storageclass_rwo }}" diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index 48ac80995e..df8108cbf9 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -133,7 +133,7 @@ db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" # Set flag to true, to use cluster's default storage classes override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default(false, true) }}" -backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) }}" # Supported values are s3 and disk +backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) | lower }}" # Supported values are s3 and disk backup_s3_alias: "{{ lookup('env', 'BACKUP_S3_ALIAS') | default('S3DB2COS', true) }}" backup_s3_endpoint: "{{ lookup('env', 'BACKUP_S3_ENDPOINT') }}" backup_s3_bucket: "{{ lookup('env', 'BACKUP_S3_BUCKET') }}" From 2b772ccf45121e44c07c679e976f9c683c278d44 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 26 Feb 2026 07:51:47 +0000 Subject: [PATCH 46/61] changes to backup and restore db2, download & upload roles (#2134) --- ibm/mas_devops/playbooks/br_db2.yml | 1 + .../plugins/action/download_from_s3.py | 4 +- .../action/verify_backup_restore_vars.py | 8 +- ibm/mas_devops/plugins/filter/filters.py | 44 +---- ibm/mas_devops/roles/db2/defaults/main.yml | 4 +- .../db2/tasks/backup/backup-instance.yml | 6 + .../roles/db2/tasks/backup/main.yml | 3 +- .../tasks/backup_database/backup-database.yml | 45 ++--- .../roles/db2/tasks/backup_database/main.yml | 3 +- .../restore/determine-storage-classes.yml | 45 ++++- .../db2/tasks/restore/restore-instance.yml | 37 ++-- .../restore_database/restore-database.yml | 14 +- .../restore_database/restore-db-from-disk.yml | 12 +- .../restore_database/restore-db-from-s3.yml | 2 +- .../db2/templates/backup/db2_backup.sh.j2 | 6 +- .../roles/download_backup_archive/README.md | 175 ++++++++++++++++-- .../download_backup_archive/defaults/main.yml | 23 ++- .../tasks/download_from_artifactory.yml | 85 ++++----- .../tasks/download_from_s3.yml | 55 ++++-- .../tasks/extract_archive.yml | 43 +++-- .../download_backup_archive/tasks/main.yml | 88 ++++++++- .../roles/suite_app_backup/README.md | 20 +- .../tasks/manage/backup-namespace.yml | 2 +- .../roles/suite_app_restore/README.md | 2 +- .../tasks/manage/restore-namespace.yml | 4 +- .../tasks/manage/restore-pv.yml | 4 +- .../roles/upload_backup_archive/README.md | 21 ++- .../upload_backup_archive/defaults/main.yml | 7 +- .../tasks/create_archive.yml | 39 ++-- .../upload_backup_archive/tasks/main.yml | 11 +- .../tasks/upload_to_artifactory.yml | 60 +++--- .../tasks/upload_to_s3.yml | 30 +-- 32 files changed, 596 insertions(+), 307 deletions(-) diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index bd5a5ef384..7c83e1e0a5 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -4,6 +4,7 @@ vars: # Define the target for backup/restore mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" + mas_application_id: "{{ lookup('env', 'MAS_APP_ID') }}" mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" diff --git a/ibm/mas_devops/plugins/action/download_from_s3.py b/ibm/mas_devops/plugins/action/download_from_s3.py index e46024ba13..df1410a9a1 100644 --- a/ibm/mas_devops/plugins/action/download_from_s3.py +++ b/ibm/mas_devops/plugins/action/download_from_s3.py @@ -51,13 +51,13 @@ def run(self, tmp=None, task_vars=None): endpoint_url = normalize_endpoint_url(endpoint=endpoint_url) - upload_status = downloadFromS3( + download_status = downloadFromS3( local_dir=local_dir, bucket_name=bucket_name, object_name=object_name, endpoint_url=endpoint_url, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region_name ) return dict( - success=upload_status + success=download_status ) diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index e0aa78658f..fd73d5c195 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -16,10 +16,10 @@ class ActionModule(ActionBase): "restore": ["mas_backup_dir", "certmanager_backup_version"] }, "db2": { - "backup": ["db2_instance_name", "db2_namespace", "mas_backup_dir", "mas_instance_id"], - "restore_instance": ["mas_backup_dir", "db2_backup_version"], - "restore_database": ["mas_backup_dir", "db2_backup_version", "db2_instance_name", "backup_vendor"], - "s3_setup": ["backup_vendor","backup_s3_alias", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] + "backup": ["db2_instance_name", "db2_namespace", "mas_backup_dir", "mas_instance_id", "mas_application_id"], + "restore_instance": ["mas_backup_dir", "db2_backup_version", "mas_application_id"], + "restore_database": ["mas_backup_dir", "db2_backup_version", "db2_instance_name", "backup_vendor", "mas_application_id"], + "s3_setup": ["backup_vendor", "backup_s3_alias", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] }, "grafana": { "backup": ["mas_backup_dir"] diff --git a/ibm/mas_devops/plugins/filter/filters.py b/ibm/mas_devops/plugins/filter/filters.py index 2e16c7bedd..e827e37b4f 100644 --- a/ibm/mas_devops/plugins/filter/filters.py +++ b/ibm/mas_devops/plugins/filter/filters.py @@ -507,47 +507,6 @@ def set_storage_classes_names(storage_list: list, storage_class_name_rwo: str, s storage_item['spec']['storageClassName'] = storage_class_name_rwo return storage_list -def override_db2_storage_classes_names(storage_list: list, storage_class_name_meta: str, storage_class_name_data: str, storage_class_name_backup: str, storage_class_name_logs: str, storage_class_name_temp: str): - """ - Iterate through the storage_list list and set the storage_class_name for each storage item based on the storage name. - Expects data to be - storage: - - name: meta - spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 20Gi - storageClassName: nfs-client - type: create - - name: backup - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi - storageClassName: nfs-client - type: create - """ - for storage_item in storage_list: - if 'name' in storage_item and 'spec' in storage_item and 'storageClassName' in storage_item['spec']: - if storage_item['name'] == 'meta': - storage_item['spec']['storageClassName'] = storage_class_name_meta - elif storage_item['name'] == 'data': - storage_item['spec']['storageClassName'] = storage_class_name_data - elif storage_item['name'] == 'backup': - storage_item['spec']['storageClassName'] = storage_class_name_backup - elif storage_item['name'] == 'tempts': - storage_item['spec']['storageClassName'] = storage_class_name_temp - elif storage_item['name'] == 'activelogs': - storage_item['spec']['storageClassName'] = storage_class_name_logs - else: - print(f'WARNING: Unhandled db2 storage name for {storage_item["name"]}') - - return storage_list - def override_manage_persistent_volumes(volumes_list: list, storage_class_name_rwo: str, storage_class_name_rwx: str): """ Iterate through the volumes_list list and set the storage_class_name for each storage item based on the access mode. @@ -599,6 +558,5 @@ def filters(self): 'is_channel_upgrade_path_valid': is_channel_upgrade_path_valid, 'get_default_upgrade_channel': get_default_upgrade_channel, 'set_storage_classes_names': set_storage_classes_names, - 'override_manage_persistent_volumes': override_manage_persistent_volumes, - 'override_db2_storage_classes_names': override_db2_storage_classes_names + 'override_manage_persistent_volumes': override_manage_persistent_volumes } diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index 492d80f939..54ac79d482 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -124,7 +124,7 @@ mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" mas_config_scope: "{{ lookup('env', 'MAS_CONFIG_SCOPE') | default('system', true) }}" # Supported values are "system", "ws", "app", or "wsapp" mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" # Necessary for ws and wsapp scopes -mas_application_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Necessary for app and wsapp scopes +mas_application_id: "{{ lookup('env', 'MAS_APP_ID') }}" # Necessary for app and wsapp scopes and backup and restore tasks # Entitlement # ----------------------------------------------------------------------------- @@ -144,6 +144,8 @@ db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" # Set flag to true, to use cluster's default storage classes override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default(false, true) }}" +custom_storage_class_rwo: "{{ lookup('env', 'CUSTOM_STORAGE_CLASS_RWO') | default('', true) }}" +custom_storage_class_rwx: "{{ lookup('env', 'CUSTOM_STORAGE_CLASS_RWX') | default('', true) }}" backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) | lower }}" # Supported values are s3 and disk backup_s3_alias: "{{ lookup('env', 'BACKUP_S3_ALIAS') | default('S3DB2COS', true) }}" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml index 1ed4710de5..d38b362a12 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/backup-instance.yml @@ -39,6 +39,12 @@ - kind: Secret api_version: v1 name: "db2u-certificate-{{ db2_instance_name }}" + - kind: Secret + api_version: v1 + name: db2u-license-keys + - kind: ConfigMap + api_version: v1 + name: db2u-release # Issuers - kind: Issuer api_version: cert-manager.io/v1 diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 70c9b00bf6..8b06ad07b2 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -9,6 +9,7 @@ mas_backup_dir: "{{ mas_backup_dir }}" mas_instance_id: "{{ mas_instance_id }}" db2_namespace: "{{ db2_namespace }}" + mas_application_id: "{{ mas_application_id }}" # Set DB2 backup version if not provided # ----------------------------------------------------------------------------- @@ -19,7 +20,7 @@ - name: "Set fact: DB2 backup base directory path" set_fact: - db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u" + db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u-{{ mas_application_id }}" - name: "Create {{ db2_backup_path }} directory for Db2 backup" file: diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml index 35f9bf6088..7f0c5517d0 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml @@ -3,15 +3,16 @@ ansible.builtin.debug: msg: - "================== BACKUP DB2 DATABASE CONFIGURATION SUMMARY ==================" - - "MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }}" - - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" - - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" - - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" - - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" - - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" - - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" - - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" - - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" + - "MAS INSTANCE ID : {{ mas_instance_id | default('UNDEFINED') }}" + - "MAS APPLICATION ID : {{ mas_application_id | default('UNDEFINED') }}" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" + - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" + - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" + - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" - "================================================================================" # Check if backup vendor is s3, then check s3 related variables @@ -40,7 +41,7 @@ - name: "Set fact backup_path for the backup script when backup vendor is s3" set_fact: - full_backup_path: "DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }}" + full_backup_path: "DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2-{{ mas_application_id }}/{{ db2_backup_version }}" when: backup_vendor == "s3" - name: "Set fact backup_path for the backup script when backup vendor is disk" @@ -203,7 +204,7 @@ # If vendor is disk, tar the backup files and cp to backup_data_path # ----------------------------------------------------------------------------- - name: "Move backup files to {{ db2_backup_data_path }} when backup vendor is disk.. This will take a while..." - shell: "oc cp --retries=50 -c db2u {{ db2_namespace }}/{{ db2_pod_name }}:{{ base_backup_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz {{ db2_backup_data_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" + shell: "oc cp --retries=50 -c db2u {{ db2_namespace }}/{{ db2_pod_name }}:{{ base_backup_path }}/db2-{{ mas_application_id }}-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz {{ db2_backup_data_path }}/db2-{{ mas_application_id }}-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" register: rsync_result when: backup_vendor == "disk" @@ -229,10 +230,11 @@ source_db2_instance_name: "{{ db2_instance_name | lower }}" source_db2_instance_version: "{{ db2_pod_name_result.db2_version }}" database: "{{ db2_dbname }}" + app_id: "{{ mas_application_id }}" backup_vendor: "{{ backup_vendor }}" vendor_backup_path: "{{ full_backup_path }}" {% if backup_vendor == 'disk' %} - local_backup_path: "{{ db2_backup_data_path }}/db2-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" + local_backup_path: "{{ db2_backup_data_path }}/db2-{{ mas_application_id }}-{{ db2_dbname }}-backup-{{ db2_backup_version }}.tar.gz" {% endif %} status: "SUCCESS" @@ -240,13 +242,14 @@ ansible.builtin.debug: msg: - "================== BACKUP DB2 DATABASE CONFIGURATION SUMMARY ==================" - - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" - - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" - - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" - - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" - - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" - - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" - - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" - - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" - - "STATUS : SUCCESS" + - "MAS APPLICATION ID : {{ mas_application_id | default('UNDEFINED') }}" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "NAMESPACE : {{ db2_namespace | default('UNDEFINED') }}" + - "DB NAME : {{ db2_dbname | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" + - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" + - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" + - "BACKUP DIR : {{ db2_backup_path | default('UNDEFINED') }}" + - "STATUS : SUCCESS" - "================================================================================" diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml index 3f5fcf2dbd..dfd3f70d32 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml @@ -9,6 +9,7 @@ mas_backup_dir: "{{ mas_backup_dir }}" mas_instance_id: "{{ mas_instance_id }}" db2_namespace: "{{ db2_namespace }}" + mas_application_id: "{{ mas_application_id }}" # Set DB2 backup version if not provided # ----------------------------------------------------------------------------- @@ -19,7 +20,7 @@ - name: "Set fact: DB2 backup base directory path" set_fact: - db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u" + db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u-{{ mas_application_id }}" - name: "Create {{ db2_backup_path }} directory for Db2 backup" file: diff --git a/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml b/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml index 8989c2d8da..2f09839b96 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/determine-storage-classes.yml @@ -1,14 +1,43 @@ --- -- name: "Determining storage class name to override" - include_tasks: "tasks/install/determine-storage-classes.yml" +# 1. Lookup storage class availabiity +# ----------------------------------------------------------------------------- +- name: "determine-storage-classes : Load default storage class information" + include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" + +# 2. RWO Storage (Required) +# ----------------------------------------------------------------------------- +- name: Default RWO Storage for ROKS if not set by user (ReadWriteOnce) + when: + - custom_storage_class_rwo is not defined or custom_storage_class_rwo == "" + - defaultStorageClasses.success + vars: + set_fact: + custom_storage_class_rwo: "{{ defaultStorageClasses.rwo }}" + +- name: Assert that a RWO storage class has been defined + assert: + that: custom_storage_class_rwo is defined and custom_storage_class_rwo != "" + fail_msg: "custom_storage_class_rwo must be defined" + +# 3. RWX Storage (Required) +# ----------------------------------------------------------------------------- +- name: Default RWX Storage for ROKS if not set by user (ReadWriteMany) + when: + - custom_storage_class_rwx is not defined or custom_storage_class_rwx == "" + - defaultStorageClasses.success + set_fact: + custom_storage_class_rwx: "{{ defaultStorageClasses.rwx }}" + +- name: Assert that a meta storage class has been defined + assert: + that: custom_storage_class_rwx is defined and custom_storage_class_rwx != "" + fail_msg: "custom_storage_class_rwx must be defined" + # Get Storage from DB2 CR - name: Override storage class names in the db2 spec.storage set_fact: - _db2_storages: "{{ db2ucluster_cr_cfg.spec.storage | ibm.mas_devops.override_db2_storage_classes_names(db2_meta_storage_class, db2_data_storage_class, db2_backup_storage_class, db2_logs_storage_class, db2_temp_storage_class) }}" + _db2_storages: "{{ db2ucluster_cr_cfg.spec.storage | ibm.mas_devops.set_storage_classes_names(custom_storage_class_rwo, custom_storage_class_rwx) }}" when: - - db2_meta_storage_class is defined - - db2_data_storage_class is defined - - db2_backup_storage_class is defined - - db2_logs_storage_class is defined - - db2_temp_storage_class is defined + - custom_storage_class_rwo is defined + - custom_storage_class_rwx is defined diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml index 7f41ba483b..bd84e595e0 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml @@ -7,12 +7,13 @@ action: "restore_instance" db2_backup_version: "{{ db2_backup_version }}" mas_backup_dir: "{{ mas_backup_dir }}" + mas_application_id: "{{ mas_application_id }}" # Set backup path facts - name: "Set fact: backup dir paths" set_fact: - db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u" - db2_resources_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u/resources" + db2_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u-{{ mas_application_id }}" + db2_resources_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u-{{ mas_application_id }}/resources" - name: "Check Db2u resource path exist" stat: @@ -79,20 +80,28 @@ # 2. Restore Secrets & configmaps # ----------------------------------------------------------------------------- -- name: Restore Secrets and configmaps +- name: Restore Secrets ibm.mas_devops.restore_resource: backup_path: "{{ db2_backup_path }}" resource_kinds: - Secret - - ConfigMap replace_resource: false skip_files: #skip applying these files as its from core namespace, will be taken care from suite restore Secret: - jdbc-{{ db2_instance_name }}-credentials.yaml - register: secrets_configmaps_result + register: secrets_result when: - namespace_result.success +- name: Restore ConfigMaps + ibm.mas_devops.restore_resource: + backup_path: "{{ db2_backup_path }}" + resource_kinds: + - ConfigMap + replace_resource: false + register: configmaps_result + when: + - namespace_result.success # 3. Restore Operatorgroups # ----------------------------------------------------------------------------- @@ -105,7 +114,8 @@ register: operatorgroups_result when: - namespace_result.success - - secrets_configmaps_result is defined and secrets_configmaps_result.success + - secrets_result is defined and secrets_result.success + - configmaps_result is defined and configmaps_result.success # 4. Restore Subscription # ----------------------------------------------------------------------------- @@ -259,7 +269,8 @@ total_created: >- {{ (namespace_result.created_count | default(0)) + - (secrets_configmaps_result.created_count | default(0)) + + (secrets_result.created_count | default(0)) + + (configmaps_result.created_count | default(0)) + (operatorgroups_result.created_count | default(0)) + (subscriptions_result.created_count | default(0)) + (certmanager_result.created_count | default(0)) + @@ -269,7 +280,8 @@ total_updated: >- {{ (namespace_result.updated_count | default(0)) + - (secrets_configmaps_result.updated_count | default(0)) + + (secrets_result.updated_count | default(0)) + + (configmaps_result.updated_count | default(0)) + (operatorgroups_result.updated_count | default(0)) + (subscriptions_result.updated_count | default(0)) + (certmanager_result.updated_count | default(0)) + @@ -279,7 +291,8 @@ total_skipped: >- {{ (namespace_result.skipped_count | default(0)) + - (secrets_configmaps_result.skipped_count | default(0)) + + (secrets_result.skipped_count | default(0)) + + (configmaps_result.skipped_count | default(0)) + (operatorgroups_result.skipped_count | default(0)) + (subscriptions_result.skipped_count | default(0)) + (certmanager_result.skipped_count | default(0)) + @@ -289,7 +302,8 @@ total_failed: >- {{ (namespace_result.failed_count | default(0)) + - (secrets_configmaps_result.failed_count | default(0)) + + (secrets_result.failed_count | default(0)) + + (configmaps_result.failed_count | default(0)) + (operatorgroups_result.failed_count | default(0)) + (subscriptions_result.failed_count | default(0)) + (certmanager_result.failed_count | default(0)) + @@ -315,7 +329,8 @@ all_failed_resources: >- {{ (namespace_result.failed_resources | default([])) + - (secrets_configmaps_result.failed_resources | default([])) + + (secrets_result.failed_resources | default([])) + + (configmaps_result.failed_resources | default([])) + (operatorgroups_result.failed_resources | default([])) + (subscriptions_result.failed_resources | default([])) + (certmanager_result.failed_resources | default([])) + diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml index d8f23aedee..63282f5076 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml @@ -9,6 +9,7 @@ mas_backup_dir: "{{ mas_backup_dir }}" db2_instance_name: "{{ db2_instance_name }}" backup_vendor: "{{ backup_vendor }}" + mas_application_id: "{{ mas_application_id }}" # Check s3 variables when backup vendor is s3 # ----------------------------------------------------------------------------- @@ -28,12 +29,13 @@ ansible.builtin.debug: msg: - "================== STARTING RESTORE DB2 DATABASE ==============================" - - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" - - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" - - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" - - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" - - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" - - "BACKUP DIR : {{ mas_backup_dir | default('UNDEFINED') }}" + - "DB2 INSTANCE NAME : {{ db2_instance_name | default('UNDEFINED') }}" + - "MAS APPLICATION ID : {{ mas_application_id | default('UNDEFINED') }}" + - "BACKUP VERSION : {{ db2_backup_version | default('UNDEFINED') }}" + - "VENDOR : {{ backup_vendor | default('UNDEFINED') }}" + - "S3 ENDPOINT : {{ backup_s3_endpoint | default('UNDEFINED') }}" + - "S3 BUCKET : {{ backup_s3_bucket | default('UNDEFINED') }}" + - "BACKUP DIR : {{ mas_backup_dir | default('UNDEFINED') }}" - "================================================================================" - name: "Run restore-db-from-disk.yml when backup vendor is disk" diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml index a83f11588f..48f9846c9e 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml @@ -1,7 +1,7 @@ - name: "Set fact backup_path for the backup script when backup vendor is disk" set_fact: pod_full_backup_path: "/mnt/backup/{{ db2_backup_version }}" - local_full_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u/data" + local_full_backup_path: "{{ mas_backup_dir }}/backup-{{ db2_backup_version }}-db2u-{{ mas_application_id }}/data" # Check backup files in local backup directory if vendor is disk # ----------------------------------------------------------------------------- @@ -15,7 +15,7 @@ assert: that: - db2_backup_info_file_stat.stat.exists == true - fail_msg: "db2-backup-info.yaml file does not exist in {{ mas_backup_dir }}/data. Cannot proceed with restore." + fail_msg: "db2-backup-info.yaml file does not exist in {{ local_full_backup_path }}. Cannot proceed with restore." - name: include vars from db2-backup-info.yaml include_vars: @@ -25,18 +25,18 @@ # check tar.gz file exists in local backup directory - name: Check for tar.gz file in backup directory stat: - path: "{{ local_full_backup_path }}/db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz" + path: "{{ local_full_backup_path }}/db2-{{ mas_application_id }}-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz" register: db2_backup_tar_stat -- name: "Fail if db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz file not found" +- name: "Fail if db2-{{ mas_application_id }}-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz file not found" assert: that: - db2_backup_tar_stat.stat.exists == true - fail_msg: "No db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz file found in {{ local_full_backup_path }}. Cannot proceed with restore." + fail_msg: "No db2-{{ mas_application_id }}-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz file found in {{ local_full_backup_path }}. Cannot proceed with restore." - name: "Set fact backup_archive" set_fact: - backup_archive_filename: "db2-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz" + backup_archive_filename: "db2-{{ mas_application_id }}-{{ db2_backup_info.database }}-backup-{{ db2_backup_info.source_db2_backup_version }}.tar.gz" # Check if Db2 instance is running and version match with backup version # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml index 5fe12c5ab3..d10a94f426 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml @@ -1,7 +1,7 @@ - name: "Set fact backup_path for the backup script when backup vendor is disk" set_fact: pod_full_backup_path: "/mnt/backup/{{ db2_backup_version }}" - s3_full_backup_path: "DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }}" + s3_full_backup_path: "DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2-{{ mas_application_id }}/{{ db2_backup_version }}" # Check if Db2 instance is running and version match with backup version # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 index 13e69479b4..9937ce8f77 100644 --- a/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 +++ b/ibm/mas_devops/roles/db2/templates/backup/db2_backup.sh.j2 @@ -124,7 +124,8 @@ function startDB() DATABASE={{ db2_dbname }} BACKUP_TYPE={{ backup_type }} VENDOR={{ backup_vendor }} -### for S3, BACKUP_PATH=DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2/{{ db2_backup_version }} +APP_ID={{ mas_application_id }} +### for S3, BACKUP_PATH=DB2REMOTE://{{ backup_s3_alias }}/{{ backup_s3_bucket }}/backups-db2-{{ mas_application_id }}/{{ db2_backup_version }} ### for disk, BACKUP_PATH=/mnt/backup/{{ db2_backup_version }}/data BACKUP_PATH={{ full_backup_path }} DB2_BACKUP_VERSION={{ db2_backup_version }} @@ -284,6 +285,7 @@ YAML_FILE=/tmp/db2-backup-info.yaml cat <> ${YAML_FILE} source_db2_backup_version: {{ db2_backup_version }} database: ${DATABASE} +app_id: ${APP_ID} backup_type: ${BACKUP_TYPE} backup_vendor: ${VENDOR} vendor_backup_path: ${BACKUP_PATH} @@ -308,7 +310,7 @@ if [ ${VENDOR} = 'disk' ] ; then cp ${YAML_FILE} ${BACKUP_PATH}/db2-backup-info.yaml echo "" echo "INFO: Creating tar archive of backup files..." - tar -czf ${DISK_BACKUP_BASE}/db2-${DATABASE}-backup-${DB2_BACKUP_VERSION}.tar.gz -C ${BACKUP_PATH} . + tar -czf ${DISK_BACKUP_BASE}/db2-${APP_ID}-${DATABASE}-backup-${DB2_BACKUP_VERSION}.tar.gz -C ${BACKUP_PATH} . echo "INFO: Tar archive created successfully" echo "" echo "Disk Usage:" diff --git a/ibm/mas_devops/roles/download_backup_archive/README.md b/ibm/mas_devops/roles/download_backup_archive/README.md index cb7d173f8d..2a95acafb3 100644 --- a/ibm/mas_devops/roles/download_backup_archive/README.md +++ b/ibm/mas_devops/roles/download_backup_archive/README.md @@ -6,6 +6,9 @@ This role automates the process of downloading MAS backup archives from remote s Key features: - Downloads compressed tar.gz archives from AWS S3 or S3-compatible storage - Downloads compressed tar.gz archives from Artifactory repositories +- **Support for downloading multiple archives in a single operation** +- **Auto-generation of archive names based on component versions** +- **Archive management option to keep downloaded archives organized** - Automatic extraction of downloaded archives - Configurable download timeouts for large archives - Optional cleanup of temporary files after extraction @@ -28,6 +31,13 @@ Key features: ### Required Variables +#### mas_instance_id +Instance ID of the MAS instance. This is used to identify the backup directories. + +- **Required** +- Environment Variable: `MAS_INSTANCE_ID` +- Default Value: None + #### mas_restore_dir Directory where the backup archive will be downloaded and extracted. This is the parent directory where all component backup directories will be restored. @@ -38,20 +48,36 @@ Directory where the backup archive will be downloaded and extracted. This is the ### Backup Archive Variables #### backup_version -Version identifier for the backup to download. This must match the version used when the backup was created. This is used to construct the value `mas-backup-.tar.gz` for `backup_archive_name`. -If you are using a custom archive name using `backup_archive_name`, you can omit this variable. +Version identifier for the backup to download. This must match the version used when the backup was created. When downloading multiple archives, this is used as the default version for all components unless specific component versions are provided. -- **optional** +- **Required** - Environment Variable: `BACKUP_VERSION` - Default Value: None -#### backup_archive_name -Custom archive name of the tar.gz archive file to download, including the `.tar.gz` extension. -If you are using `backup_version`, you can omit this variable. +#### backup_archive_names +List of archive names to download. Can be provided as a comma-separated string or as a list. If not provided, the role will auto-generate archive names based on component versions. + +Examples: +- String format: `"archive1.tar.gz,archive2.tar.gz,archive3.tar.gz"` +- List format: `["archive1.tar.gz", "archive2.tar.gz", "archive3.tar.gz"]` - **Optional** -- Environment Variable: `BACKUP_ARCHIVE_NAME` -- Default Value: None +- Environment Variable: `BACKUP_ARCHIVE_NAMES` +- Default Value: Auto-generated from component versions + +#### Component-Specific Backup Versions + +These variables allow you to specify different backup versions for each component. If not provided, they default to the value of `backup_version`. These are used to auto-generate archive names when `backup_archive_names` is not provided. + +- `ibm_catalogs_backup_version` - Environment Variable: `IBM_CATALOGS_BACKUP_VERSION` +- `certmanager_backup_version` - Environment Variable: `CERTMANAGER_BACKUP_VERSION` +- `mongodb_backup_version` - Environment Variable: `MONGODB_BACKUP_VERSION` +- `sls_backup_version` - Environment Variable: `SLS_BACKUP_VERSION` +- `db2_backup_version` - Environment Variable: `DB2_BACKUP_VERSION` +- `suite_backup_version` - Environment Variable: `SUITE_BACKUP_VERSION` +- `manage_backup_version` - Environment Variable: `MANAGE_BACKUP_VERSION` + +All are **Optional** and default to `backup_version`. ### S3 Download Variables @@ -110,13 +136,6 @@ Artifactory API token for authentication. - Environment Variable: `ARTIFACTORY_TOKEN` - Default Value: None -#### artifactory_url -Base URL of the Artifactory server (e.g., `https://artifactory.example.com/artifactory`). - -- **Required for Artifactory download** -- Environment Variable: `ARTIFACTORY_URL` -- Default Value: None - #### artifactory_repository Name of the Artifactory repository where the archive is stored. @@ -124,6 +143,13 @@ Name of the Artifactory repository where the archive is stored. - Environment Variable: `ARTIFACTORY_REPOSITORY` - Default Value: None +#### artifactory_url +Base URL of the Artifactory server (e.g., `https://artifactory.example.com/artifactory`). + +- **Optional** +- Environment Variable: `ARTIFACTORY_URL` +- Default Value: `https://na.artifactory.swg-devops.com/artifactory` + ### General Configuration #### backup_temp_dir @@ -148,22 +174,86 @@ Whether to automatically extract the downloaded archive. Set to `false` if you o - Default Value: `true` #### cleanup_archive -Whether to remove the archive file and temporary directory after successful extraction. Set to `false` to keep the downloaded archive. +Whether to remove the archive file and temporary directory after successful extraction. Set to `false` to keep the downloaded archive. - **Optional** - Environment Variable: `CLEANUP_ARCHIVE` - Default Value: `true` +#### include_sls_archive +Whether to download SLS archive (`-sls.tar.gz`) from S3 or Artifactory. + +**Important:** When set to `false`, the role will automatically skip downloading sls archive from S3 or Artifactory. This prevents unnecessary downloads when SLS component restore is not needed. + +- **Optional** +- Environment Variable: `INCLUDE_SLS_ARCHIVE` +- Default Value: `true` + +#### include_manage_app_archive +Whether to download Manage app archive (`-app-manage.tar.gz`) from S3 or Artifactory. + +**Important:** When set to `false`, the role will automatically skip downloading manage-app archive from S3 or Artifactory. This prevents unnecessary downloads when manage component restore is not needed. + +- **Optional** +- Environment Variable: `INCLUDE_MANAGE_APP_ARCHIVE` +- Default Value: `true` + +#### include_manage_db_archive +Whether to download Manage DB2 archive (`-db2u-manage.tar.gz`) from S3 or Artifactory. + +**Important:** When set to `false`, the role will automatically skip downloading manage-db2 archive (`-db2u-manage.tar.gz`) from S3 or Artifactory. This prevents unnecessary downloads when manage component restore is not needed. + +- **Optional** +- Environment Variable: `INCLUDE_MANAGE_DB_ARCHIVE` +- Default Value: `true` + ## Example Playbook -### S3 Download +### Download Multiple Archives from S3 (Auto-generated names) After installing the Ansible Collection you can include this role in your own custom playbooks. ```yaml - hosts: localhost vars: + mas_instance_id: inst1 + mas_restore_dir: /restore/mas + backup_version: "20260117-191500" + aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + s3_bucket_name: my-mas-backups + s3_region: us-west-2 + roles: + - ibm.mas_devops.download_backup_archive +``` + +### Download Specific Archives from S3 +```yaml +- hosts: localhost + vars: + mas_instance_id: inst1 mas_restore_dir: /restore/mas backup_version: "20260117-191500" + backup_archive_names: + - mas-inst1-backup-20260117-191500-catalog.tar.gz + - mas-inst1-backup-20260117-191500-suite.tar.gz + - mas-inst1-backup-20260117-191500-manage.tar.gz + aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + s3_bucket_name: my-mas-backups + s3_region: us-west-2 + roles: + - ibm.mas_devops.download_backup_archive +``` + +### Download with Different Component Versions +```yaml +- hosts: localhost + vars: + mas_instance_id: inst1 + mas_restore_dir: /restore/mas + backup_version: "20260117-191500" + mongodb_backup_version: "20260116-120000" + db2_backup_version: "20260115-100000" aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" s3_bucket_name: my-mas-backups @@ -177,6 +267,7 @@ After installing the Ansible Collection you can include this role in your own cu ```yaml - hosts: localhost vars: + mas_instance_id: inst1 mas_restore_dir: /restore/mas backup_version: "20260117-191500" artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" @@ -192,6 +283,7 @@ After installing the Ansible Collection you can include this role in your own cu ```yaml - hosts: localhost vars: + mas_instance_id: inst1 mas_restore_dir: /restore/mas backup_version: "20260117-191500" aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY') }}" @@ -208,6 +300,7 @@ After installing the Ansible Collection you can include this role in your own cu ```yaml - hosts: localhost vars: + mas_instance_id: inst1 mas_restore_dir: /restore/mas backup_version: "20260117-191500" extract_archive: false @@ -219,14 +312,59 @@ After installing the Ansible Collection you can include this role in your own cu - ibm.mas_devops.download_backup_archive ``` +### Download with Archive Management - without Manage DB2 archives + +```yaml +- hosts: localhost + vars: + mas_instance_id: inst1 + mas_restore_dir: /restore/mas + backup_version: "20260117-191500" + include_manage_db_archive: false + aws_access_key_id: "{{ lookup('env', 'AWS_ACCESS_KEY_ID') }}" + aws_secret_access_key: "{{ lookup('env', 'AWS_SECRET_ACCESS_KEY') }}" + s3_bucket_name: my-mas-backups + roles: + - ibm.mas_devops.download_backup_archive +``` + ## Run Role Playbook After installing the Ansible Collection you can easily run the role standalone using the `run_role` playbook provided. -### S3 Download +### Download Multiple Archives from S3 (Auto-generated) + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_RESTORE_DIR=/restore/mas +export BACKUP_VERSION=20260117-191500 +export S3_ACCESS_KEY_ID=your_access_key +export S3_SECRET_ACCESS_KEY=your_secret_key +export S3_BUCKET_NAME=my-mas-backups +export S3_REGION=us-west-2 +ROLE_NAME=download_backup_archive ansible-playbook ibm.mas_devops.run_role +``` + +### Download Specific Archives from S3 + +```bash +export MAS_INSTANCE_ID=inst1 +export MAS_RESTORE_DIR=/restore/mas +export BACKUP_VERSION=20260117-191500 +export BACKUP_ARCHIVE_NAMES="mas-inst1-backup-20260117-191500-catalog.tar.gz,mas-inst1-backup-20260117-191500-suite.tar.gz" +export S3_ACCESS_KEY_ID=your_access_key +export S3_SECRET_ACCESS_KEY=your_secret_key +export S3_BUCKET_NAME=my-mas-backups +export S3_REGION=us-west-2 +ROLE_NAME=download_backup_archive ansible-playbook ibm.mas_devops.run_role +``` + +### Download with Archive Management ```bash +export MAS_INSTANCE_ID=inst1 export MAS_RESTORE_DIR=/restore/mas export BACKUP_VERSION=20260117-191500 +export MANAGE_ARCHIVES=false export S3_ACCESS_KEY_ID=your_access_key export S3_SECRET_ACCESS_KEY=your_secret_key export S3_BUCKET_NAME=my-mas-backups @@ -237,6 +375,7 @@ ROLE_NAME=download_backup_archive ansible-playbook ibm.mas_devops.run_role ### Artifactory Download ```bash +export MAS_INSTANCE_ID=inst1 export MAS_RESTORE_DIR=/restore/mas export BACKUP_VERSION=20260117-191500 export ARTIFACTORY_USERNAME=your_username diff --git a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml index bc79c665ed..326e54dfaf 100644 --- a/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/defaults/main.yml @@ -7,7 +7,20 @@ mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" # Backup version to download backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" -backup_archive_name: "{{ lookup('env', 'BACKUP_ARCHIVE_NAME') }}" + +# Multiple archives download - list of archive names to download +# If not provided, will auto-generate based on backup_version and component versions +backup_archive_names: "{{ lookup('env', 'BACKUP_ARCHIVE_NAMES') }}" + +# Specific backup versions for each component (optional - override backup_version) +# These are used to auto-generate archive names when backup_archive_names is not provided +ibm_catalogs_backup_version: "{{ lookup('env', 'IBM_CATALOGS_BACKUP_VERSION') | default (backup_version, true) }}" +certmanager_backup_version: "{{ lookup('env', 'CERTMANAGER_BACKUP_VERSION') | default (backup_version, true) }}" +mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') | default (backup_version, true) }}" +sls_backup_version: "{{ lookup('env', 'SLS_BACKUP_VERSION') | default (backup_version, true) }}" +db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') | default (backup_version, true) }}" +suite_backup_version: "{{ lookup('env', 'SUITE_BACKUP_VERSION') | default (backup_version, true) }}" +manage_backup_version: "{{ lookup('env', 'MANAGE_BACKUP_VERSION') | default (backup_version, true) }}" # S3 Configuration (provide these to download from S3) aws_access_key_id: "{{ lookup('env', 'S3_ACCESS_KEY_ID') }}" @@ -19,11 +32,15 @@ s3_endpoint_url: "{{ lookup('env', 'S3_ENDPOINT_URL') }}" # Artifactory Configuration (provide these to download from Artifactory) artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" -artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" +artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') | default('https://na.artifactory.swg-devops.com/artifactory', true) }}" artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" # General settings backup_temp_dir: "{{ mas_restore_dir }}/mas-{{ mas_instance_id }}-restore-{{ backup_version }}" -download_timeout: "{{ lookup('env', 'DOWNLOAD_TIMEOUT_SECS') | default(3600, true) }}" # Download timeout in seconds (1 hour) +download_timeout: "{{ lookup('env', 'DOWNLOAD_TIMEOUT_SECS') | default(10800, true) }}" # Download timeout in seconds (3 hour) extract_archive: "{{ lookup('env', 'EXTRACT_ARCHIVE') | default(true, true) }}" # Whether to extract the archive after download cleanup_archive: "{{ lookup('env', 'CLEANUP_ARCHIVE') | default(true, true) }}" # Whether to remove the archive file after extraction + +include_sls_archive: "{{ lookup('env', 'INCLUDE_SLS_ARCHIVE') | default(true, true) }}" +include_manage_app_archive: "{{ lookup('env', 'INCLUDE_MANAGE_APP_ARCHIVE') | default(true, true) }}" +include_manage_db_archive: "{{ lookup('env', 'INCLUDE_MANAGE_DB_ARCHIVE') | default(true, true) }}" diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml index f8a25c323d..72c96e9dc1 100644 --- a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_artifactory.yml @@ -1,78 +1,79 @@ --- -# Download backup archive from Artifactory +# Download backup archives from Artifactory - name: "Validate Artifactory repository is defined" ansible.builtin.fail: msg: "artifactory_repository is required when downloading from Artifactory" when: artifactory_repository is not defined or artifactory_repository == '' -- name: "Set Artifactory download URL" - ansible.builtin.set_fact: - artifactory_download_url: "{{ artifactory_url }}/{{ artifactory_repository }}/{{ backup_archive_name }}" - - name: "Display Artifactory download information" ansible.builtin.debug: msg: - "Downloading from Artifactory: {{ artifactory_url }}" - "Repository: {{ artifactory_repository }}" - - "Archive: {{ backup_archive_name }}" - - "Full URL: {{ artifactory_download_url }}" + - "Number of archives: {{ backup_archives_to_download | length }}" + - "Archives: {{ backup_archives_to_download }}" - "Download to: {{ backup_temp_dir }}" -- name: "Download archive from Artifactory using curl" +- name: "Download archives from Artifactory using curl" ansible.builtin.command: cmd: > curl -X GET -u {{ artifactory_username }}:{{ artifactory_token }} - -o {{ backup_temp_dir }}/{{ backup_archive_name }} - {{ artifactory_download_url }} + -o {{ backup_temp_dir }}/mas-{{ mas_instance_id }}-backups/{{ item }} + {{ artifactory_url }}/{{ artifactory_repository }}/mas-{{ mas_instance_id }}-backups/{{ item }} --max-time {{ download_timeout }} --connect-timeout 60 --fail --silent --show-error --location - --write-out '%{http_code}' - register: artifactory_download_result - changed_when: artifactory_download_result.rc == 0 - failed_when: artifactory_download_result.rc != 0 - async: "{{ download_timeout }}" - poll: 10 + register: artifactory_download_results + changed_when: artifactory_download_results.rc == 0 + failed_when: false no_log: true + loop: "{{ backup_archives_to_download }}" -- name: "Check download HTTP status code" +- name: "Process download results" ansible.builtin.set_fact: - download_http_code: "{{ artifactory_download_result.stdout | regex_search('[0-9]{3}$') }}" - when: artifactory_download_result is succeeded + artifactory_download_summary: "{{ artifactory_download_summary | default([]) + [{'archive': item.item, 'success': item.rc == 0, 'http_code': item.stdout | regex_search('[0-9]{3}$') if item.rc == 0 else 'N/A'}] }}" + loop: "{{ artifactory_download_results.results }}" -- name: "Verify downloaded archive exists" +- name: "Verify downloaded archives exist" ansible.builtin.stat: - path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" - register: downloaded_archive_stat + path: "{{ backup_temp_dir }}/mas-{{ mas_instance_id }}-backups/{{ item }}" + register: downloaded_archives_stat + loop: "{{ backup_archives_to_download }}" -- name: "Display Artifactory download result" +- name: "Display Artifactory download results" ansible.builtin.debug: - msg: - - "Successfully downloaded {{ backup_archive_name }} from Artifactory" - - "HTTP Status: {{ download_http_code | default('N/A') }}" - - "Archive size: {{ (downloaded_archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + msg: "Successfully downloaded {{ item.archive }} from Artifactory - HTTP Status: {{ item.http_code }} - Size: {{ (downloaded_archives_stat.results[loop_index].stat.size / 1024 / 1024) | round(2) }} MB" when: - - artifactory_download_result is succeeded - - downloaded_archive_stat.stat.exists + - item.success + - downloaded_archives_stat.results[loop_index].stat.exists + loop: "{{ artifactory_download_summary }}" + loop_control: + index_var: loop_index -- name: "Fail if Artifactory download failed" - ansible.builtin.fail: - msg: "Failed to download archive from Artifactory. HTTP Status: {{ download_http_code | default('Unknown') }}" - when: - - artifactory_download_result is succeeded - - download_http_code is defined - - download_http_code not in ['200', '201'] +- name: "Check for failed downloads" + ansible.builtin.set_fact: + failed_artifactory_downloads: "{{ artifactory_download_summary | selectattr('success', 'equalto', false) | map(attribute='archive') | list }}" -- name: "Handle download command failure" +- name: "Check for missing archives" + ansible.builtin.set_fact: + missing_archives: "{{ downloaded_archives_stat.results | selectattr('stat.exists', 'equalto', false) | map(attribute='item') | list }}" + +- name: "Fail if any Artifactory download failed" ansible.builtin.fail: - msg: "Failed to download archive from Artifactory: {{ artifactory_download_result.stderr | default('Unknown error') }}" - when: artifactory_download_result is failed + msg: "Failed to download {{ failed_artifactory_downloads | length }} archive(s) from Artifactory: {{ failed_artifactory_downloads | join(', ') }}" + when: failed_artifactory_downloads | length > 0 -- name: "Fail if downloaded archive does not exist" +- name: "Fail if any downloaded archive does not exist" ansible.builtin.fail: - msg: "Downloaded archive {{ backup_temp_dir }}/{{ backup_archive_name }} does not exist" - when: not downloaded_archive_stat.stat.exists + msg: "{{ missing_archives | length }} downloaded archive(s) do not exist: {{ missing_archives | join(', ') }}" + when: missing_archives | length > 0 + +- name: "Display download summary" + ansible.builtin.debug: + msg: + - "Successfully downloaded {{ backup_archives_to_download | length }} archive(s) from Artifactory" + - "Total size: {{ (downloaded_archives_stat.results | map(attribute='stat.size') | sum / 1024 / 1024) | round(2) }} MB" diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml index d2fdf09015..abcbeaeac6 100644 --- a/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/download_from_s3.yml @@ -1,45 +1,60 @@ --- -# Download backup archive from S3 +# Download backup archives from S3 - name: "Display S3 download information" ansible.builtin.debug: msg: - "Downloading from S3 bucket: {{ s3_bucket_name }}" - "Region: {{ s3_region }}" - - "Archive: {{ backup_archive_name }}" + - "Number of archives: {{ backup_archives_to_download | length }}" + - "Archives: {{ backup_archives_to_download }}" - "Download to: {{ backup_temp_dir }}" -- name: "Download archive from S3" +- name: "Download archives from S3" ibm.mas_devops.download_from_s3: aws_access_key_id: "{{ aws_access_key_id }}" aws_secret_access_key: "{{ aws_secret_access_key }}" bucket_name: "{{ s3_bucket_name }}" - object_name: "{{ backup_archive_name }}" + object_name: "mas-{{ mas_instance_id }}-backups/{{ item }}" local_dir: "{{ backup_temp_dir }}" region_name: "{{ s3_region }}" endpoint_url: "{{ s3_endpoint_url | default(omit) }}" - register: s3_download_result + register: s3_download_results poll: 10 + loop: "{{ backup_archives_to_download }}" -- name: "Verify downloaded archive exists" +- name: "Verify downloaded archives exist" ansible.builtin.stat: - path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" - register: downloaded_archive_stat + path: "{{ backup_temp_dir }}/mas-{{ mas_instance_id }}-backups/{{ item }}" + register: downloaded_archives_stat + loop: "{{ backup_archives_to_download }}" -- name: "Display S3 download result" +- name: "Display S3 download results" ansible.builtin.debug: - msg: - - "Successfully downloaded {{ backup_archive_name }} from S3 bucket {{ s3_bucket_name }}" - - "Archive size: {{ (downloaded_archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + msg: "Successfully downloaded {{ item.item }} from S3 bucket {{ s3_bucket_name }} - Size: {{ (item.stat.size / 1024 / 1024) | round(2) }} MB" when: - - s3_download_result.success | bool - - downloaded_archive_stat.stat.exists + - item.stat.exists + loop: "{{ downloaded_archives_stat.results }}" + +- name: "Check for failed downloads" + ansible.builtin.set_fact: + failed_s3_downloads: "{{ s3_download_results.results | selectattr('success', 'equalto', false) | map(attribute='item') | list }}" + +- name: "Check for missing archives" + ansible.builtin.set_fact: + missing_archives: "{{ downloaded_archives_stat.results | selectattr('stat.exists', 'equalto', false) | map(attribute='item') | list }}" -- name: "Fail if S3 download failed" +- name: "Fail if any S3 download failed" ansible.builtin.fail: - msg: "Failed to download archive from S3: {{ s3_download_result.msg | default('Unknown error') }}" - when: not s3_download_result.success | bool + msg: "Failed to download {{ failed_s3_downloads | length }} archive(s) from S3: {{ failed_s3_downloads | join(', ') }}" + when: failed_s3_downloads | length > 0 -- name: "Fail if downloaded archive does not exist" +- name: "Fail if any downloaded archive does not exist" ansible.builtin.fail: - msg: "Downloaded archive {{ backup_temp_dir }}/{{ backup_archive_name }} does not exist" - when: not downloaded_archive_stat.stat.exists + msg: "{{ missing_archives | length }} downloaded archive(s) do not exist: {{ missing_archives | join(', ') }}" + when: missing_archives | length > 0 + +- name: "Display download summary" + ansible.builtin.debug: + msg: + - "Successfully downloaded {{ backup_archives_to_download | length }} archive(s) from S3" + - "Total size: {{ (downloaded_archives_stat.results | map(attribute='stat.size') | sum / 1024 / 1024) | round(2) }} MB" diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml index f98c124561..7214078a7a 100644 --- a/ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/extract_archive.yml @@ -1,27 +1,42 @@ --- -# Extract downloaded backup archive -- name: "Verify archive exists before extraction" +# Extract downloaded backup archives +- name: "Verify archives exist before extraction" ansible.builtin.stat: - path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" - register: archive_stat + path: "{{ backup_temp_dir }}/mas-{{ mas_instance_id }}-backups/{{ item }}" + register: archives_stat + loop: "{{ backup_archives_to_download }}" -- name: "Fail if archive does not exist" +- name: "Check for missing archives" + ansible.builtin.set_fact: + missing_archives_for_extraction: "{{ archives_stat.results | selectattr('stat.exists', 'equalto', false) | map(attribute='item') | list }}" + +- name: "Fail if any archive does not exist" ansible.builtin.fail: - msg: "Archive {{ backup_temp_dir }}/{{ backup_archive_name }} does not exist" - when: not archive_stat.stat.exists + msg: "{{ missing_archives_for_extraction | length }} archive(s) do not exist: {{ missing_archives_for_extraction | join(', ') }}" + when: missing_archives_for_extraction | length > 0 - name: "Display extraction information" ansible.builtin.debug: msg: - - "Extracting archive: {{ backup_archive_name }}" - - "Archive size: {{ (archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + - "Extracting {{ backup_archives_to_download | length }} archive(s)" + - "Total size: {{ (archives_stat.results | map(attribute='stat.size') | sum / 1024 / 1024) | round(2) }} MB" - "Extract to: {{ mas_restore_dir }}" -- name: "Extract tar.gz archive to restore directory" +- name: "Extract tar.gz archives to restore directory" ansible.builtin.command: - cmd: "tar -xzf {{ backup_temp_dir }}/{{ backup_archive_name }} -C {{ mas_restore_dir }}" - register: tar_extract_result - changed_when: tar_extract_result.rc == 0 + cmd: "tar -xzf {{ backup_temp_dir }}/mas-{{ mas_instance_id }}-backups/{{ item }} -C {{ mas_restore_dir }}" + register: tar_extract_results + changed_when: tar_extract_results.rc == 0 + loop: "{{ backup_archives_to_download }}" + +- name: "Check for extraction failures" + ansible.builtin.set_fact: + failed_extractions: "{{ tar_extract_results.results | selectattr('rc', 'ne', 0) | map(attribute='item') | list }}" + +- name: "Fail if any extraction failed" + ansible.builtin.fail: + msg: "Failed to extract {{ failed_extractions | length }} archive(s): {{ failed_extractions | join(', ') }}" + when: failed_extractions | length > 0 - name: "List extracted directories" ansible.builtin.find: @@ -33,7 +48,7 @@ - name: "Display extraction result" ansible.builtin.debug: msg: - - "Successfully extracted {{ backup_archive_name }}" + - "Successfully extracted {{ backup_archives_to_download | length }} archive(s)" - "Extracted {{ extracted_dirs.files | length }} backup directories to {{ mas_restore_dir }}" - "Directories: {{ extracted_dirs.files | map(attribute='path') | map('basename') | list }}" diff --git a/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml index 2950aef3a7..d559fbf3ae 100644 --- a/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml +++ b/ibm/mas_devops/roles/download_backup_archive/tasks/main.yml @@ -10,15 +10,68 @@ msg: "mas_instance_id is required but not defined" when: mas_instance_id is not defined or mas_instance_id == '' -- name: "Fail if either backup_version or backup_archive_name is not defined" +- name: "Fail if backup_version is not defined" ansible.builtin.fail: - msg: "Either backup_version or backup_archive_name is required but not defined" - when: (backup_version is not defined or backup_version == '') and (backup_archive_name is not defined or backup_archive_name == '') + msg: "backup_version is required but not defined" + when: backup_version is not defined or backup_version == '' -- name: Set backup_archive_name if backup_version is provided +# Determine which archives to download +- name: "Set backup archive names list" ansible.builtin.set_fact: - backup_archive_name: "mas-{{ mas_instance_id }}-backup-{{ backup_version }}.tar.gz" # Default name if backup_version is provided - when: backup_version is defined and backup_version != '' and (backup_archive_name is not defined or backup_archive_name == '') + backup_archives_to_download: [] + +# Handle explicit list of archives +- name: "Parse backup_archive_names if provided as string" + ansible.builtin.set_fact: + backup_archives_to_download: "{{ backup_archive_names.split(',') | map('trim') | list }}" + when: + - backup_archive_names is defined + - backup_archive_names != '' + - backup_archive_names is string + +- name: "Use backup_archive_names if provided as list" + ansible.builtin.set_fact: + backup_archives_to_download: "{{ backup_archive_names }}" + when: + - backup_archive_names is defined + - backup_archive_names != '' + - backup_archive_names is not string + +# Auto-generate archive names based on component versions +- name: "Auto-generate archive names from component versions" + ansible.builtin.set_fact: + backup_archives_to_download: + - "mas-{{ mas_instance_id }}-backup-{{ ibm_catalogs_backup_version }}-catalog.tar.gz" + - "mas-{{ mas_instance_id }}-backup-{{ certmanager_backup_version }}-certmanager.tar.gz" + - "mas-{{ mas_instance_id }}-backup-{{ mongodb_backup_version }}-mongoce.tar.gz" + - "mas-{{ mas_instance_id }}-backup-{{ sls_backup_version }}-sls.tar.gz" + - "mas-{{ mas_instance_id }}-backup-{{ suite_backup_version }}-suite.tar.gz" + - "mas-{{ mas_instance_id }}-backup-{{ manage_backup_version }}-app-manage.tar.gz" + - "mas-{{ mas_instance_id }}-backup-{{ db2_backup_version }}-db2u-manage.tar.gz" + when: backup_archives_to_download | length == 0 + + +- name: "Filter out SLS archive when include_sls_archive is false" + ansible.builtin.set_fact: + backup_archives_to_download: "{{ backup_archives_to_download | reject('search', '-sls\\.tar\\.gz$') | list }}" + when: not (include_sls_archive | bool) + +- name: "Filter out manage app archive when include_manage_app_archive is false" + ansible.builtin.set_fact: + backup_archives_to_download: "{{ backup_archives_to_download | reject('search', '-app-manage\\.tar\\.gz$') | list }}" + when: not (include_manage_app_archive | bool) + +# Filter out manage archives if include_manage_db_archive is false +- name: "Filter out manage archives when include_manage_db_archive is false" + ansible.builtin.set_fact: + backup_archives_to_download: "{{ backup_archives_to_download | reject('search', '-db2u-manage\\.tar\\.gz$') | list }}" + when: not (include_manage_db_archive | bool) + +- name: "Display archives to download" + ansible.builtin.debug: + msg: + - "Will download {{ backup_archives_to_download | length }} archive(s):" + - "{{ backup_archives_to_download }}" # Determine download source - name: "Check if S3 credentials are provided" @@ -58,6 +111,13 @@ state: directory mode: '0755' +# Download from S3, boto3 needs this base directory for bucket base dir +- name: "Create temporary base directory" + ansible.builtin.file: + path: "{{ backup_temp_dir }}/mas-{{ mas_instance_id }}-backups" + state: directory + mode: '0755' + # Download from S3 or Artifactory - name: "Download from S3" ansible.builtin.include_tasks: download_from_s3.yml @@ -67,16 +127,17 @@ ansible.builtin.include_tasks: download_from_artifactory.yml when: download_from_artifactory and not download_from_s3 -# Extract archive -- name: "Extract backup archive" +# Extract archives +- name: "Extract backup archives" ansible.builtin.include_tasks: extract_archive.yml when: extract_archive | bool # Cleanup -- name: "Remove temporary archive file" +- name: "Remove temporary archive files" ansible.builtin.file: - path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + path: "{{ backup_temp_dir }}/{{ item }}" state: absent + loop: "{{ backup_archives_to_download }}" when: - cleanup_archive | bool - extract_archive | bool @@ -88,3 +149,10 @@ when: - cleanup_archive | bool - extract_archive | bool + +- name: "Display archive management information" + ansible.builtin.debug: + msg: + - "Archives are being managed in: {{ backup_temp_dir }}" + - "Downloaded archives: {{ backup_archives_to_download }}" + - "To cleanup manually, remove the directory: {{ backup_temp_dir }}" diff --git a/ibm/mas_devops/roles/suite_app_backup/README.md b/ibm/mas_devops/roles/suite_app_backup/README.md index 31e8934aa2..d8264d7a32 100644 --- a/ibm/mas_devops/roles/suite_app_backup/README.md +++ b/ibm/mas_devops/roles/suite_app_backup/README.md @@ -126,18 +126,22 @@ The role creates a backup directory with the following structure: ``` / -└── backup--manage-/ - ├── namespace/ - │ ├── ManageApp-.yaml - │ ├── ManageWorkspace--.yaml - │ ├── Secret--manage-encryptionsecret.yaml - │ ├── Secret--manage-encryptionsecret-operator.yaml - │ ├── Subscription-ibm-mas-manage.yaml +└── backup--app-manage/ + ├── resources/ + │ ├── projects + │ │ └── mas--manage.yaml + | ├── secrets + │ │ └── .yaml + │ │ └── .yaml + | ├── configmaps + │ │ └── .yaml + │ │ └── .yaml + | ├── subscriptions + │ │ └── .yaml │ └── ... (other resources) └── data/ ├── .tar.gz ├── .tar.gz - └── ... (one archive per PVC) ``` diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml index 9df3ce2626..9096700b4b 100644 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml @@ -25,7 +25,7 @@ # ----------------------------------------------------------------------------- - name: "Set fact: Manage backup base directory path" set_fact: - manage_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-manage" + manage_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-app-manage" # 4. Set Manage namespace and workspace CR name # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/suite_app_restore/README.md b/ibm/mas_devops/roles/suite_app_restore/README.md index 76540aff76..f7176a795a 100644 --- a/ibm/mas_devops/roles/suite_app_restore/README.md +++ b/ibm/mas_devops/roles/suite_app_restore/README.md @@ -196,7 +196,7 @@ The role expects the backup directory to have the following structure: ``` / -└── backup--manage/ +└── backup--app-manage/ ├── resources/ │ ├── projects │ │ └── mas--manage.yaml diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml index b8e2dc97e6..a5d15c44de 100644 --- a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-namespace.yml @@ -22,8 +22,8 @@ # ----------------------------------------------------------------------------- - name: "Set fact: Manage backup base directory path" set_fact: - manage_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-manage" - manage_resources_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-manage/resources" + manage_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-app-manage" + manage_resources_backup_path: "{{ mas_backup_dir }}/backup-{{ mas_app_backup_version }}-app-manage/resources" - name: "Verify backup directory exists" stat: diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml index 07379d0ced..efbb71fbe5 100644 --- a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml @@ -164,8 +164,8 @@ - name: restore-container image: registry.redhat.io/ubi8/ubi-minimal:latest command: ["/bin/sh", "-c", "sleep infinity"] - volumeMounts: "{{ manage_persistent_volumes | map(attribute='mountPath') | zip(manage_persistent_volumes | map(attribute='pvcName')) | map('community.general.dict_kv', 'mountPath', 'name') | list }}" - volumes: "{{ manage_persistent_volumes | map('community.general.dict_kv', 'name', 'pvcName') | map('combine', {'persistentVolumeClaim': {'claimName': ''}}) | list }}" + volumeMounts: "{{ volume_mounts }}" + volumes: "{{ volumes }}" vars: volume_mounts: | {% set mounts = [] %} diff --git a/ibm/mas_devops/roles/upload_backup_archive/README.md b/ibm/mas_devops/roles/upload_backup_archive/README.md index e0a6036cc9..02b0dcdb31 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/README.md +++ b/ibm/mas_devops/roles/upload_backup_archive/README.md @@ -28,6 +28,13 @@ Key features: ### Required Variables +#### mas_instance_id +Instance ID of the MAS instance. This is used to identify the backup directories. + +- **Required** +- Environment Variable: `MAS_INSTANCE_ID` +- Default Value: None + #### mas_backup_dir Directory containing the MAS backup folders. This is the parent directory where all component backup directories are located. @@ -152,13 +159,6 @@ Artifactory API token for authentication. - Environment Variable: `ARTIFACTORY_TOKEN` - Default Value: None -#### artifactory_url -Base URL of the Artifactory server (e.g., `https://artifactory.example.com/artifactory`). - -- **Required for Artifactory upload** -- Environment Variable: `ARTIFACTORY_URL` -- Default Value: None - #### artifactory_repository Name of the Artifactory repository where the archive will be uploaded. @@ -166,6 +166,13 @@ Name of the Artifactory repository where the archive will be uploaded. - Environment Variable: `ARTIFACTORY_REPOSITORY` - Default Value: None +#### artifactory_url +Base URL of the Artifactory server (e.g., `https://artifactory.example.com/artifactory`). + +- **Optional** +- Environment Variable: `ARTIFACTORY_URL` +- Default Value: `https://na.artifactory.swg-devops.com/artifactory` + ### General Configuration #### backup_archive_name diff --git a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml index c7c9599bca..a39a8dd0f1 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/defaults/main.yml @@ -28,10 +28,11 @@ s3_endpoint_url: "{{ lookup('env', 'S3_ENDPOINT_URL') }}" # Artifactory Configuration (provide these to upload to Artifactory) artifactory_username: "{{ lookup('env', 'ARTIFACTORY_USERNAME') }}" artifactory_token: "{{ lookup('env', 'ARTIFACTORY_TOKEN') }}" -artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') }}" +artifactory_url: "{{ lookup('env', 'ARTIFACTORY_URL') | default('https://na.artifactory.swg-devops.com/artifactory', true) }}" artifactory_repository: "{{ lookup('env', 'ARTIFACTORY_REPOSITORY') }}" # General settings -backup_archive_name: "mas-{{ mas_instance_id }}-backup-{{ backup_version }}.tar.gz" +# Archive naming pattern: mas-{{ mas_instance_id }}-{{ backup_directory_name }}.tar.gz +# Multiple archives will be created, one for each backup directory backup_temp_dir: "{{ mas_backup_dir }}/mas-{{ mas_instance_id }}-backup-{{ backup_version }}" -upload_timeout: 3600 # Upload timeout in seconds (1 hour) +upload_timeout: 10800 # Upload timeout in seconds (3 hours) diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml index 30b23fb028..31a2af600e 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/create_archive.yml @@ -5,11 +5,11 @@ backup_directories: - "backup-{{ ibm_catalogs_backup_version }}-catalog" - "backup-{{ certmanager_backup_version }}-certmanager" - - "backup-{{ sls_backup_version }}-sls" - "backup-{{ mongodb_backup_version }}-mongoce" - - "backup-{{ db2_backup_version }}-db2u" + - "backup-{{ sls_backup_version }}-sls" - "backup-{{ suite_backup_version }}-suite" - - "backup-{{ manage_backup_version }}-manage" + - "backup-{{ manage_backup_version }}-app-manage" + - "backup-{{ db2_backup_version }}-db2u-manage" - name: "Verify backup directory exists" ansible.builtin.stat: @@ -45,30 +45,37 @@ path: "{{ backup_temp_dir }}" state: absent -- name: "Create temporary directory for archive" +- name: "Create temporary directory for archives" ansible.builtin.file: path: "{{ backup_temp_dir }}" state: directory mode: '0755' -- name: "Create tar.gz archive of backup directories" +- name: "Create tar.gz archive for each backup directory" ansible.builtin.command: - cmd: "tar -czf {{ backup_temp_dir }}/{{ backup_archive_name }} -C {{ mas_backup_dir }} {{ existing_backup_dirs | join(' ') }}" - register: tar_result - changed_when: tar_result.rc == 0 + cmd: "tar -czf {{ backup_temp_dir }}/mas-{{ mas_instance_id }}-{{ item }}.tar.gz -C {{ mas_backup_dir }} {{ item }}" + register: tar_results + changed_when: tar_results.rc == 0 + loop: "{{ existing_backup_dirs }}" -- name: "Verify archive was created" +- name: "Verify archives were created" ansible.builtin.stat: - path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" - register: archive_stat + path: "{{ backup_temp_dir }}/mas-{{ mas_instance_id }}-{{ item }}.tar.gz" + register: archive_stats + loop: "{{ existing_backup_dirs }}" -- name: "Fail if archive was not created" +- name: "Fail if any archive was not created" ansible.builtin.fail: - msg: "Failed to create archive {{ backup_temp_dir }}/{{ backup_archive_name }}" - when: not archive_stat.stat.exists + msg: "Failed to create archive {{ backup_temp_dir }}/mas-{{ mas_instance_id }}-{{ item.item }}.tar.gz" + when: not item.stat.exists + loop: "{{ archive_stats.results }}" + +- name: "Build list of created archives" + ansible.builtin.set_fact: + backup_archives: "{{ archive_stats.results | map(attribute='stat.path') | list }}" - name: "Display archive information" ansible.builtin.debug: msg: - - "Archive created successfully: {{ backup_temp_dir }}/{{ backup_archive_name }}" - - "Archive size: {{ (archive_stat.stat.size / 1024 / 1024) | round(2) }} MB" + - "Created {{ backup_archives | length }} archives successfully:" + - "{% for item in archive_stats.results %} - {{ item.stat.path | basename }}: {{ (item.stat.size / 1024 / 1024) | round(2) }} MB{% endfor %}" diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml index 9685e84f70..278cddd10a 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/main.yml @@ -47,8 +47,15 @@ when: upload_to_artifactory and not upload_to_s3 # Cleanup -- name: "Remove temporary archive file" +- name: "Remove temporary archive files" ansible.builtin.file: - path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + path: "{{ item }}" + state: absent + loop: "{{ backup_archives }}" + when: backup_archives is defined + +- name: "Remove temporary directory" + ansible.builtin.file: + path: "{{ backup_temp_dir }}" state: absent when: backup_temp_dir is defined diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml index 604d26a664..56aedb0e31 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_artifactory.yml @@ -1,69 +1,51 @@ --- -# Upload backup archive to Artifactory +# Upload backup archives to Artifactory - name: "Validate Artifactory repository is defined" ansible.builtin.fail: msg: "artifactory_repository is required when uploading to Artifactory" when: artifactory_repository is not defined or artifactory_repository == '' -- name: "Set Artifactory upload URL" - ansible.builtin.set_fact: - artifactory_upload_url: "{{ artifactory_url }}/{{ artifactory_repository }}/{{ backup_archive_name }}" - - name: "Display Artifactory upload information" ansible.builtin.debug: msg: - "Uploading to Artifactory: {{ artifactory_url }}" - "Repository: {{ artifactory_repository }}" - - "Archive: {{ backup_archive_name }}" - - "Full URL: {{ artifactory_upload_url }}" - -- name: "Get archive file stats" - ansible.builtin.stat: - path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" - register: archive_file_stat + - "Number of archives: {{ backup_archives | length }}" -- name: "Upload archive to Artifactory using curl" +- name: "Upload archives to Artifactory using curl" ansible.builtin.command: cmd: > curl -X PUT -u {{ artifactory_username }}:{{ artifactory_token }} - -T {{ backup_temp_dir }}/{{ backup_archive_name }} - {{ artifactory_upload_url }} + -T {{ item }} + {{ artifactory_url }}/{{ artifactory_repository }}/mas-{{ mas_instance_id }}-backups/{{ item | basename }} --max-time {{ upload_timeout }} --connect-timeout 60 --fail --silent --show-error - --write-out '%{http_code}' - register: artifactory_upload_result - changed_when: artifactory_upload_result.rc == 0 - failed_when: artifactory_upload_result.rc != 0 - async: "{{ upload_timeout }}" - poll: 10 + register: artifactory_upload_results + changed_when: artifactory_upload_results.rc == 0 + failed_when: false no_log: true + loop: "{{ backup_archives }}" -- name: "Check upload HTTP status code" +- name: "Process upload results" ansible.builtin.set_fact: - upload_http_code: "{{ artifactory_upload_result.stdout | regex_search('[0-9]{3}$') }}" - when: artifactory_upload_result is succeeded + artifactory_upload_summary: "{{ artifactory_upload_summary | default([]) + [{'archive': item.item | basename, 'success': item.rc == 0, 'http_code': item.stdout | regex_search('[0-9]{3}$') if item.rc == 0 else 'N/A'}] }}" + loop: "{{ artifactory_upload_results.results }}" -- name: "Display Artifactory upload result" +- name: "Display Artifactory upload results" ansible.builtin.debug: msg: - - "Successfully uploaded {{ backup_archive_name }} to Artifactory" - - "HTTP Status: {{ upload_http_code | default('N/A') }}" - - "Archive size: {{ (archive_file_stat.stat.size / 1024 / 1024) | round(2) }} MB" - when: artifactory_upload_result is succeeded + - "Upload summary:" + - "{% for result in artifactory_upload_summary %} - {{ result.archive }}: {{ 'SUCCESS' if result.success else 'FAILED' }} (HTTP {{ result.http_code }}){% endfor %}" -- name: "Fail if Artifactory upload failed" - ansible.builtin.fail: - msg: "Failed to upload archive to Artifactory. HTTP Status: {{ upload_http_code | default('Unknown') }}" - when: - - artifactory_upload_result is succeeded - - upload_http_code is defined - - upload_http_code not in ['200', '201'] +- name: "Check for failed uploads" + ansible.builtin.set_fact: + failed_artifactory_uploads: "{{ artifactory_upload_summary | selectattr('success', 'equalto', false) | map(attribute='archive') | list }}" -- name: "Handle upload command failure" +- name: "Fail if any Artifactory upload failed" ansible.builtin.fail: - msg: "Failed to upload archive to Artifactory: {{ artifactory_upload_result.stderr | default('Unknown error') }}" - when: artifactory_upload_result is failed + msg: "Failed to upload {{ failed_artifactory_uploads | length }} archive(s) to Artifactory: {{ failed_artifactory_uploads | join(', ') }}" + when: failed_artifactory_uploads | length > 0 diff --git a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml index d4568a0387..d05eb54267 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml +++ b/ibm/mas_devops/roles/upload_backup_archive/tasks/upload_to_s3.yml @@ -1,30 +1,36 @@ --- -# Upload backup archive to S3 +# Upload backup archives to S3 - name: "Display S3 upload information" ansible.builtin.debug: msg: - "Uploading to S3 bucket: {{ s3_bucket_name }}" - "Region: {{ s3_region }}" - - "Archive: {{ backup_archive_name }}" + - "Number of archives: {{ backup_archives | length }}" -- name: "Upload archive to S3" +- name: "Upload archives to S3" ibm.mas_devops.upload_to_s3: aws_access_key_id: "{{ aws_access_key_id }}" aws_secret_access_key: "{{ aws_secret_access_key }}" bucket_name: "{{ s3_bucket_name }}" - object_name: "{{ backup_archive_name }}" - file_path: "{{ backup_temp_dir }}/{{ backup_archive_name }}" + object_name: "mas-{{ mas_instance_id }}-backups/{{ item | basename }}" + file_path: "{{ item }}" region_name: "{{ s3_region }}" endpoint_url: "{{ s3_endpoint_url | default(omit) }}" - register: s3_upload_result + register: s3_upload_results poll: 10 + loop: "{{ backup_archives }}" -- name: "Display S3 upload result" +- name: "Display S3 upload results" ansible.builtin.debug: - msg: "Successfully uploaded {{ backup_archive_name }} to S3 bucket {{ s3_bucket_name }}" - when: s3_upload_result.success | bool + msg: "Successfully uploaded {{ item.item | basename }} to S3 bucket {{ s3_bucket_name }}/mas-{{ mas_instance_id }}-backups" + when: item.success | bool + loop: "{{ s3_upload_results.results }}" -- name: "Fail if S3 upload failed" +- name: "Check for failed uploads" + ansible.builtin.set_fact: + failed_uploads: "{{ s3_upload_results.results | selectattr('success', 'equalto', false) | map(attribute='item') | list }}" + +- name: "Fail if any S3 upload failed" ansible.builtin.fail: - msg: "Failed to upload archive to S3: {{ s3_upload_result.msg | default('Unknown error') }}" - when: not s3_upload_result.success | bool + msg: "Failed to upload {{ failed_uploads | length }} archive(s) to S3: {{ failed_uploads | map('basename') | join(', ') }}" + when: failed_uploads | length > 0 From c3d334fff3abff0283a497bc80693b38f8340220 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 26 Feb 2026 11:56:06 +0000 Subject: [PATCH 47/61] update image for pv restore helper pod --- .../tasks/manage/restore-pv.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml index efbb71fbe5..c3d60afd04 100644 --- a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml @@ -144,9 +144,9 @@ loop_control: loop_var: pv_item - # Create dummy pod to mount PVCs + # Create helper pod to mount PVCs # --------------------------------------------------------------------- - - name: "Create dummy pod for PVC data restoration" + - name: "Create PV restore helper pod for PVC data restoration" kubernetes.core.k8s: state: present definition: @@ -162,7 +162,7 @@ restartPolicy: Never containers: - name: restore-container - image: registry.redhat.io/ubi8/ubi-minimal:latest + image: registry.redhat.io/ubi8/ubi:latest command: ["/bin/sh", "-c", "sleep infinity"] volumeMounts: "{{ volume_mounts }}" volumes: "{{ volumes }}" @@ -181,7 +181,7 @@ {{ vols }} register: dummy_pod_created - - name: "Wait for dummy pod to be running" + - name: "Wait for helper pod to be running" kubernetes.core.k8s_info: api_version: v1 kind: Pod @@ -208,7 +208,7 @@ msg: - "Restore pod: {{ restore_pod_name }}" - "Container: {{ restore_pod_container }}" - - "Using dummy pod: {{ using_dummy_pod }}" + - "Using helper pod: {{ using_dummy_pod }}" - name: "Restore each persistent volume" include_tasks: "{{ role_path }}/tasks/manage/restore-single-pv.yml" @@ -217,7 +217,7 @@ loop_var: pv_item index_var: pv_index - # Shutdown dummy pod if created + # Shutdown helper pod if created # ------------------------------------------------------------------------- - name: "Shutdown dummy pod" when: using_dummy_pod | default(false) @@ -232,9 +232,9 @@ wait: yes wait_timeout: 300 - - name: "Dummy pod deleted successfully" + - name: "Helper pod deleted successfully" debug: - msg: "Dummy restore pod {{ restore_pod_name }} has been deleted" + msg: "Helper restore pod {{ restore_pod_name }} has been deleted" - name: "Display PV restore completion" debug: From 2308067e4fcb4f43681ff22707dc07f9ed204aad Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 26 Feb 2026 12:13:01 +0000 Subject: [PATCH 48/61] option to pass image for pvc restore helper pod --- ibm/mas_devops/roles/suite_app_restore/README.md | 8 ++++++++ ibm/mas_devops/roles/suite_app_restore/defaults/main.yml | 3 +++ .../roles/suite_app_restore/tasks/manage/restore-pv.yml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ibm/mas_devops/roles/suite_app_restore/README.md b/ibm/mas_devops/roles/suite_app_restore/README.md index f7176a795a..55af8f2fea 100644 --- a/ibm/mas_devops/roles/suite_app_restore/README.md +++ b/ibm/mas_devops/roles/suite_app_restore/README.md @@ -79,6 +79,14 @@ Delay in seconds between status checks when waiting for ManageWorkspace to becom - Environment Variable: `MAS_APP_RESTORE_WAIT_DELAY` - Default: `360` +### helper_pod_image +Name of the image to use by the PVC-restore-helper pod. This pod will be deployed on the app's namespace to mount PVC volumes and extract the pvc backup to the mounted path. +Image must have `tar` installed for `oc cp` to work. + +- Optional +- Environment Variable: `HELPER_POD_IMAGE` +- Default: `registry.redhat.io/ubi9/ubi:latest` + ### override_storageclass Enable or disable storage class override during restore. When enabled, the restore process will use custom storage classes instead of the storage classes from the backup. diff --git a/ibm/mas_devops/roles/suite_app_restore/defaults/main.yml b/ibm/mas_devops/roles/suite_app_restore/defaults/main.yml index c5611c4c28..d831e186f2 100644 --- a/ibm/mas_devops/roles/suite_app_restore/defaults/main.yml +++ b/ibm/mas_devops/roles/suite_app_restore/defaults/main.yml @@ -16,6 +16,9 @@ mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # Backup version to restore (e.g., "20240315-143022" or "v1.0-prod") mas_app_backup_version: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION') }}" +# PVC helper pod configuration +helper_pod_image: "{{ lookup('env', 'HELPER_POD_IMAGE') | default('registry.redhat.io/ubi9/ubi:latest', true) }}" + # Storage Class Override Options # ----------------------------------------------------------------------------- # Override storage class from backup with custom storage classes diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml index c3d60afd04..a0f80e4198 100644 --- a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml @@ -162,7 +162,7 @@ restartPolicy: Never containers: - name: restore-container - image: registry.redhat.io/ubi8/ubi:latest + image: "{{ helper_pod_image }}" command: ["/bin/sh", "-c", "sleep infinity"] volumeMounts: "{{ volume_mounts }}" volumes: "{{ volumes }}" From 83a3fa68d83d2742554603656ef96772d4fb3238 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 26 Feb 2026 12:20:03 +0000 Subject: [PATCH 49/61] update readme --- ibm/mas_devops/roles/download_backup_archive/README.md | 3 ++- ibm/mas_devops/roles/upload_backup_archive/README.md | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ibm/mas_devops/roles/download_backup_archive/README.md b/ibm/mas_devops/roles/download_backup_archive/README.md index 2a95acafb3..cad68f967d 100644 --- a/ibm/mas_devops/roles/download_backup_archive/README.md +++ b/ibm/mas_devops/roles/download_backup_archive/README.md @@ -395,7 +395,8 @@ After successful execution, the role will extract the backup archive to the rest ├── backup-20260117-191500-certmanager/ ├── backup-20260117-191500-sls/ ├── backup-20260117-191500-mongoce/ -├── backup-20260117-191500-db2u/ +├── backup-20260117-191500-app-/ +├── backup-20260117-191500-db2u-/ └── backup-20260117-191500-suite/ ``` diff --git a/ibm/mas_devops/roles/upload_backup_archive/README.md b/ibm/mas_devops/roles/upload_backup_archive/README.md index 02b0dcdb31..138c1b36d3 100644 --- a/ibm/mas_devops/roles/upload_backup_archive/README.md +++ b/ibm/mas_devops/roles/upload_backup_archive/README.md @@ -175,13 +175,6 @@ Base URL of the Artifactory server (e.g., `https://artifactory.example.com/artif ### General Configuration -#### backup_archive_name -Name of the tar.gz archive file that will be created and uploaded. - -- **Optional** -- Environment Variable: None -- Default Value: `mas-backup-{{ backup_version }}.tar.gz` - #### backup_temp_dir Temporary directory where the archive will be created before upload. The directory is created if it doesn't exist and cleaned up after upload. From 4ee0b02266d98d5371dbfb3bae4430c956f14314 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Thu, 26 Feb 2026 13:48:59 +0000 Subject: [PATCH 50/61] Update restore-pv.yml --- .../roles/suite_app_restore/tasks/manage/restore-pv.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml index a0f80e4198..0ad7134f75 100644 --- a/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml +++ b/ibm/mas_devops/roles/suite_app_restore/tasks/manage/restore-pv.yml @@ -70,7 +70,6 @@ - name: "Lookup default storage classes" when: - override_storageclass - - mas_app_custom_storage_class_rwx == '' or mas_app_custom_storage_class_rwo == '' block: - name: "Include default storage classes lookup" include_tasks: "{{ role_path }}/../../common_tasks/default_storage_classes.yml" From 9ef0da2a39480c3a2d38ec70836b32322ecfca0b Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 2 Mar 2026 16:08:09 +0000 Subject: [PATCH 51/61] update playbook --- ibm/mas_devops/playbooks/br_core.yml | 22 ++++++++++++++++ ibm/mas_devops/playbooks/br_db2.yml | 34 +++++++++++++++++++++++++ ibm/mas_devops/playbooks/br_manage.yml | 22 ++++++++++++++++ ibm/mas_devops/playbooks/br_mongodb.yml | 21 +++++++++++++++ 4 files changed, 99 insertions(+) diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml index 6a6ec01277..29161850c0 100644 --- a/ibm/mas_devops/playbooks/br_core.yml +++ b/ibm/mas_devops/playbooks/br_core.yml @@ -57,6 +57,28 @@ mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" pre_tasks: + - name: Important Notice + debug: + msg: | + ********************************************************************* + ************************* IMPORTANT NOTICE ************************** + ********************************************************************* + * * + * These playbooks are samples to demonstrate how to use the roles * + * in this collection. * + * * + * They are NOT INTENDED FOR PRODUCTION USE as-is, they are a * + * starting point for power users to aid in the development of * + * their own Ansible playbooks using the roles in this collection * + * * + * The recommended way to install MAS is to use the MAS CLI, which * + * uses this Ansible Collection to deliver a complete managed * + * lifecycle for your MAS instance. * + * * + * https://ibm-mas.github.io/cli/ * + * * + ********************************************************************* + - name: "Set backup_version with timestamp if not provided" ansible.builtin.set_fact: backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index 7c83e1e0a5..f2b79b9ca2 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -28,6 +28,28 @@ db2_temp_storage_class: "{{ lookup('env', 'DB2_TEMP_STORAGE_CLASS') }}" pre_tasks: + - name: Important Notice + debug: + msg: | + ********************************************************************* + ************************* IMPORTANT NOTICE ************************** + ********************************************************************* + * * + * These playbooks are samples to demonstrate how to use the roles * + * in this collection. * + * * + * They are NOT INTENDED FOR PRODUCTION USE as-is, they are a * + * starting point for power users to aid in the development of * + * their own Ansible playbooks using the roles in this collection * + * * + * The recommended way to install MAS is to use the MAS CLI, which * + * uses this Ansible Collection to deliver a complete managed * + * lifecycle for your MAS instance. * + * * + * https://ibm-mas.github.io/cli/ * + * * + ********************************************************************* + - name: "Fail if DB2_ACTION is not set to backup|restore" assert: that: db2_action in ["backup", "backup_database", "restore_database", "restore"] @@ -42,6 +64,18 @@ assert: that: backup_type in ["online", "offline"] fail_msg: "DB2_BACKUP_TYPE is required and must be set to 'online' or 'offline'" + + - name: "Fail if mas_instance_id is not set to" + ansible.builtin.assert: + that: + - mas_instance_id is defined + fail_msg: "mas_instance_id is required and must be set" + + - name: "Fail if mas_application_id is not set to" + ansible.builtin.assert: + that: + - mas_application_id is defined + fail_msg: "mas_application_id is required and must be set" - name: "Fail if S3 variables are not set when backup_vendor is s3" ibm.mas_devops.verify_backup_restore_vars: diff --git a/ibm/mas_devops/playbooks/br_manage.yml b/ibm/mas_devops/playbooks/br_manage.yml index 56dc1014bb..3e5270634f 100644 --- a/ibm/mas_devops/playbooks/br_manage.yml +++ b/ibm/mas_devops/playbooks/br_manage.yml @@ -32,6 +32,28 @@ mas_app_custom_storage_class_rwx: "{{ lookup('env', 'MAS_APP_CUSTOM_STORAGE_CLASS_RWX') | default('', true) }}" pre_tasks: + - name: Important Notice + debug: + msg: | + ********************************************************************* + ************************* IMPORTANT NOTICE ************************** + ********************************************************************* + * * + * These playbooks are samples to demonstrate how to use the roles * + * in this collection. * + * * + * They are NOT INTENDED FOR PRODUCTION USE as-is, they are a * + * starting point for power users to aid in the development of * + * their own Ansible playbooks using the roles in this collection * + * * + * The recommended way to install MAS is to use the MAS CLI, which * + * uses this Ansible Collection to deliver a complete managed * + * lifecycle for your MAS instance. * + * * + * https://ibm-mas.github.io/cli/ * + * * + ********************************************************************* + - name: "Set mas_app_backup_version with timestamp if not provided" ansible.builtin.set_fact: mas_app_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index 3dab59d7e5..eba3ea3ab2 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -17,6 +17,27 @@ mongodb_storageclass_rwo: "{{ lookup('env', 'MONGODB_STORAGECLASS_NAME_RWO') | default('', true) }}" pre_tasks: + - name: Important Notice + debug: + msg: | + ********************************************************************* + ************************* IMPORTANT NOTICE ************************** + ********************************************************************* + * * + * These playbooks are samples to demonstrate how to use the roles * + * in this collection. * + * * + * They are NOT INTENDED FOR PRODUCTION USE as-is, they are a * + * starting point for power users to aid in the development of * + * their own Ansible playbooks using the roles in this collection * + * * + * The recommended way to install MAS is to use the MAS CLI, which * + * uses this Ansible Collection to deliver a complete managed * + * lifecycle for your MAS instance. * + * * + * https://ibm-mas.github.io/cli/ * + * * + ********************************************************************* - name: "Fail if mongodb_action is not set to backup|backup_database|restore|restore_database" assert: that: mongodb_action in ["backup", "backup_database", "restore_database", "restore"] From 26995a95c8a69db71be377910fff9988ee9bdcfe Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Mon, 2 Mar 2026 16:10:51 +0000 Subject: [PATCH 52/61] Update br_db2.yml --- ibm/mas_devops/playbooks/br_db2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index f2b79b9ca2..c34827905d 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -64,13 +64,13 @@ assert: that: backup_type in ["online", "offline"] fail_msg: "DB2_BACKUP_TYPE is required and must be set to 'online' or 'offline'" - + - name: "Fail if mas_instance_id is not set to" ansible.builtin.assert: that: - mas_instance_id is defined fail_msg: "mas_instance_id is required and must be set" - + - name: "Fail if mas_application_id is not set to" ansible.builtin.assert: that: From 68a7e062fc433f28d41cf276696b4d49fb8dabe4 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Mar 2026 09:30:09 +0000 Subject: [PATCH 53/61] fix logs --- ibm/mas_devops/plugins/action/backup_resource.py | 4 ++-- ibm/mas_devops/plugins/action/restore_resource.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ibm/mas_devops/plugins/action/backup_resource.py b/ibm/mas_devops/plugins/action/backup_resource.py index c2154e70f8..805436f0de 100644 --- a/ibm/mas_devops/plugins/action/backup_resource.py +++ b/ibm/mas_devops/plugins/action/backup_resource.py @@ -54,7 +54,7 @@ def run(self, tmp=None, task_vars=None): if backup_path is None or backup_path == "": raise AnsibleError(f"Error: backup_path argument was not provided") - display.v(f"Starting backup of MAS Suite resources to '{backup_path}'") + display.v(f"Starting backup of MAS resources to '{backup_path}'") total_backed_up = 0 total_failed = 0 @@ -155,7 +155,7 @@ def run(self, tmp=None, task_vars=None): has_failures = total_failed > 0 return dict( - message=f"Backed up {total_backed_up} MAS Suite resources" + (f" with {total_failed} failures" if has_failures else ""), + message=f"Backed up {total_backed_up} MAS resources" + (f" with {total_failed} failures" if has_failures else ""), failed=has_failures, changed=False, success=not has_failures, diff --git a/ibm/mas_devops/plugins/action/restore_resource.py b/ibm/mas_devops/plugins/action/restore_resource.py index ebe71c2be7..66597f6fca 100644 --- a/ibm/mas_devops/plugins/action/restore_resource.py +++ b/ibm/mas_devops/plugins/action/restore_resource.py @@ -181,7 +181,7 @@ class ActionModule(ActionBase): tasks: - name: "Restore and replace specific MAS Suite resources" ibm.mas_devops.restore_resource: - backup_path: "/backup/backup-250115-120000-suite" + backup_path: "/backup/backup-20250115-120000-suite" resource_kinds: - Secret - ConfigMap @@ -189,12 +189,12 @@ class ActionModule(ActionBase): - name: "Restore all resources (skip existing)" ibm.mas_devops.restore_resource: - backup_path: "/backup/backup-250115-120000-suite" + backup_path: "/backup/backup-20250115-120000-suite" replace_resource: false - name: "Restore resources with overrides, filter_values and skip_files" ibm.mas_devops.restore_resource: - backup_path: "/backup/backup-250115-120000-suite" + backup_path: "/backup/backup-20250115-120000-suite" resource_kinds: - Suite - Secret @@ -242,7 +242,7 @@ def run(self, tmp=None, task_vars=None): if not os.path.exists(resources_path): raise AnsibleError(f"Error: resources directory not found in backup: {resources_path}") - display.v(f"Starting restore of MAS Suite resources from '{backup_path}'") + display.v(f"Starting restore of MAS resources from '{backup_path}'") display.v(f"Replace existing resources: {'enabled' if replace_resource else 'disabled'}") if override_values: override_kinds = ', '.join(override_values.keys()) @@ -403,7 +403,7 @@ def run(self, tmp=None, task_vars=None): has_failures = total_failed > 0 return dict( - message=f"Restored {total_created + total_updated} MAS Suite resources ({total_created} created, {total_updated} updated, {total_skipped} skipped)" + (f" with {total_failed} failures" if has_failures else ""), + message=f"Restored {total_created + total_updated} MAS resources ({total_created} created, {total_updated} updated, {total_skipped} skipped)" + (f" with {total_failed} failures" if has_failures else ""), failed=has_failures, changed=(total_created + total_updated) > 0, success=not has_failures, From 0f7455815c3943b6c619b95254746389f217bed5 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Mar 2026 17:24:08 +0000 Subject: [PATCH 54/61] Update backup-namespace.yml --- .../roles/suite_app_backup/tasks/manage/backup-namespace.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml index 9096700b4b..24b518c2a6 100644 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml @@ -61,6 +61,11 @@ - manage_workspace_cr.resources[0].spec.settings is defined - manage_workspace_cr.resources[0].spec.settings.db is defined +- name: "Set Manage default encryption secret name" + set_fact: + manage_encryptionsecret_name: "{{ mas_workspace_id }}-manage-encryptionsecret" + when: manage_encryptionsecret_name is not defined + - name: "Debug: Manage encryption secret name" debug: msg: "Manage encryption secret name: {{ manage_encryptionsecret_name }}" From 6f2fadd6e1a347001e153d9ae08981b91ea5c013 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Mar 2026 17:28:47 +0000 Subject: [PATCH 55/61] Update backup-namespace.yml --- .../suite_app_backup/tasks/manage/backup-namespace.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml index 24b518c2a6..a81050b913 100644 --- a/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml +++ b/ibm/mas_devops/roles/suite_app_backup/tasks/manage/backup-namespace.yml @@ -59,12 +59,6 @@ when: - manage_workspace_cr.resources[0].spec is defined - manage_workspace_cr.resources[0].spec.settings is defined - - manage_workspace_cr.resources[0].spec.settings.db is defined - -- name: "Set Manage default encryption secret name" - set_fact: - manage_encryptionsecret_name: "{{ mas_workspace_id }}-manage-encryptionsecret" - when: manage_encryptionsecret_name is not defined - name: "Debug: Manage encryption secret name" debug: From 934476ce1f404ec471333430ce20f98291be5b82 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Mar 2026 18:48:41 +0000 Subject: [PATCH 56/61] mongo cleanup --- .../tasks/providers/community/install-mongo.yml | 4 ---- .../mongodb/templates/community/0.12.0/cr.yml.j2 | 16 ---------------- .../mongodb/templates/community/0.8.3/cr.yml.j2 | 16 ---------------- .../mongodb/templates/community/0.9.0/cr.yml.j2 | 16 ---------------- 4 files changed, 52 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml index 646b11496e..7cddbf26c4 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml @@ -146,7 +146,6 @@ wait_condition: status: "True" type: Ready - when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources - name: "Create a ca certificate in '{{ mongodb_namespace }}' namespace" kubernetes.core.k8s: @@ -156,7 +155,6 @@ wait_condition: status: "True" type: Ready - when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources - name: "Create a Issuer for server certificate in '{{ mongodb_namespace }}' namespace" kubernetes.core.k8s: @@ -166,7 +164,6 @@ wait_condition: status: "True" type: Ready - when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources - name: "Create a server certificate in '{{ mongodb_namespace }}' namespace" kubernetes.core.k8s: @@ -176,7 +173,6 @@ wait_condition: status: "True" type: Ready - when: mongodb_action == "install" # restore action uses these tasks but should not recreate these resources # Taking ca.crt value in a variable # Mongo needs a configmap having ca.crt value diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 index a08fc54ee6..f2967f623f 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 @@ -29,21 +29,6 @@ spec: username: metrics-endpoint-user passwordSecretRef: name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" -{% if mongodb_users is defined %} - users: -{% for user in mongodb_users %} - - name: {{ user.name }} - db: {{ user.db }} - passwordSecretRef: - name: {{ user.passwordSecretRef.name }} - roles: -{% for role in user.roles %} - - name: {{ role.name }} - db: {{ role.db }} -{% endfor %} - scramCredentialsSecretName: "{{ user.scramCredentialsSecretName }}" -{% endfor %} -{% else %} users: - name: admin db: admin @@ -59,7 +44,6 @@ spec: - name: readWriteAnyDatabase db: admin scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" -{% endif %} additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 index c912aff927..8febb09904 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 @@ -29,21 +29,6 @@ spec: username: metrics-endpoint-user passwordSecretRef: name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" -{% if mongodb_users is defined %} - users: -{% for user in mongodb_users %} - - name: {{ user.name }} - db: {{ user.db }} - passwordSecretRef: - name: {{ user.passwordSecretRef.name }} - roles: -{% for role in user.roles %} - - name: {{ role.name }} - db: {{ role.db }} -{% endfor %} - scramCredentialsSecretName: "{{ user.scramCredentialsSecretName }}" -{% endfor %} -{% else %} users: - name: admin db: admin @@ -61,7 +46,6 @@ spec: - name: dbAdminAnyDatabase db: admin scramCredentialsSecretName: ""{{ mongodb_instance_name }}-scram" -{% endif %} additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 index a08fc54ee6..f2967f623f 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 @@ -29,21 +29,6 @@ spec: username: metrics-endpoint-user passwordSecretRef: name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" -{% if mongodb_users is defined %} - users: -{% for user in mongodb_users %} - - name: {{ user.name }} - db: {{ user.db }} - passwordSecretRef: - name: {{ user.passwordSecretRef.name }} - roles: -{% for role in user.roles %} - - name: {{ role.name }} - db: {{ role.db }} -{% endfor %} - scramCredentialsSecretName: "{{ user.scramCredentialsSecretName }}" -{% endfor %} -{% else %} users: - name: admin db: admin @@ -59,7 +44,6 @@ spec: - name: readWriteAnyDatabase db: admin scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" -{% endif %} additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true From 363f3b7b3298e8fd685d62effb50b6fd40395d55 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Mar 2026 19:15:19 +0000 Subject: [PATCH 57/61] revert mongo install changes --- .../roles/mongodb/defaults/main.yml | 4 +-- .../community/check-mongo-exists.yml | 2 +- .../community/controlled-upgrade.yml | 2 +- .../providers/community/install-mongo.yml | 32 +++++++++---------- .../tasks/providers/community/uninstall.yml | 6 ++-- .../templates/community/0.12.0/cr.yml.j2 | 14 ++++---- .../templates/community/0.7.9/cr.yml.j2 | 14 ++++---- .../templates/community/0.8.3/cr.yml.j2 | 14 ++++---- .../templates/community/0.9.0/cr.yml.j2 | 14 ++++---- .../templates/community/admin-password.yml | 2 +- .../mongodb/templates/community/ca-cert.yml | 2 +- .../community/metrics-endpoint-secret.yml.j2 | 4 +-- .../templates/community/mongo-hosts.yml.j2 | 4 +-- .../templates/community/server-cert.yml | 2 +- .../templates/community/servicemonitor.yml.j2 | 8 ++--- .../roles/mongodb/templates/community/tls.yml | 2 +- 16 files changed, 63 insertions(+), 63 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index 68b3807f9d..cbd5b61078 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -6,9 +6,8 @@ mongodb_provider: "{{ lookup('env','MONGODB_PROVIDER') | default('community', Tr mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" -# Supported actions: install, uninstall, backup, restore, restore_database +# Supported actions: install, uninstall, backup, backup_database, restore, restore_database mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('install', True) }}" -mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" # Backup mongodb databases for specified MAS application. Backup all mongodb databases if not specify this value. mas_app_id: "{{ lookup('env', 'MAS_APP_ID') }}" @@ -122,6 +121,7 @@ custom_labels: "{{ lookup('env', 'CUSTOM_LABELS') | default(None, true) | string # MongoCE backup and restore vars # ----------------------------------------------------------------------------- +mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml index 14606bed8f..6b14c39aac 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/check-mongo-exists.yml @@ -3,7 +3,7 @@ kubernetes.core.k8s_info: api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" register: existing_mongodb diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml index 88cd591821..2737278a1b 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/controlled-upgrade.yml @@ -24,7 +24,7 @@ kubernetes.core.k8s_info: api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" register: existing_mongodb until: diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml index 7cddbf26c4..55a4c5f870 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/install-mongo.yml @@ -189,7 +189,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: ConfigMap - name: "{{ mongodb_instance_name }}-cert-map" + name: mas-mongo-ce-cert-map namespace: "{{ mongodb_namespace }}" register: ca_cfgmap_info @@ -210,7 +210,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: Secret - name: "{{ mongodb_instance_name }}-admin-password" + name: mas-mongo-ce-admin-password namespace: "{{ mongodb_namespace }}" register: admin_password_info @@ -228,7 +228,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: Secret - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret namespace: "{{ mongodb_namespace }}" register: metrics_endpoint_secret_info when: @@ -311,15 +311,15 @@ - debug: var: scale_down_mongo_deployment_output.stdout - - name: "community : install : Delete {{ mongodb_instance_name }} statefulset so next time it recreates with right image" - shell: oc delete statefulset {{ mongodb_instance_name }} -n {{ mongodb_namespace }} + - name: "community : install : Delete mas-mongo-ce statefulset so next time it recreates with right image" + shell: oc delete statefulset mas-mongo-ce -n {{ mongodb_namespace }} register: delete_mongo_ss_output - debug: var: delete_mongo_ss_output.stdout - - name: "community : install : Delete {{ mongodb_instance_name }}-arb statefulset so next time it recreates with right image" - shell: oc delete statefulset {{ mongodb_instance_name }}-arb -n {{ mongodb_namespace }} + - name: "community : install : Delete mas-mongo-ce-arb statefulset so next time it recreates with right image" + shell: oc delete statefulset mas-mongo-ce-arb -n {{ mongodb_namespace }} register: delete_mongo_ss_arb_output - debug: @@ -332,11 +332,11 @@ - debug: var: scale_up_mongo_deployment_output.stdout -- name: "community : install : Wait for {{ mongodb_instance_name }} stateful set to be ready" +- name: "community : install : Wait for mas-mongo-ce stateful set to be ready" kubernetes.core.k8s_info: api_version: apps/v1 kind: StatefulSet - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" vars: mongodb_replicas_check: "{{ existing_mongodb.resources[0].spec.members | default(mongodb_replicas|int) }}" @@ -349,12 +349,12 @@ - mongodb_statefulset.resources[0].status.readyReplicas is defined - mongodb_statefulset.resources[0].status.readyReplicas == (mongodb_replicas_check|int) -- name: "community : install : Wait for {{ mongodb_instance_name }}-arb stateful set to be ready" +- name: "community : install : Wait for mas-mongo-ce-arb stateful set to be ready" when: target_mongodb_version is version('4.4.0','>=') # this statefulset will only exist in Mongo v4.4+ kubernetes.core.k8s_info: api_version: apps/v1 kind: StatefulSet - name: "{{ mongodb_instance_name }}-arb" + name: mas-mongo-ce-arb namespace: "{{ mongodb_namespace }}" register: mongodb_arb_statefulset retries: 45 # Approx 90 minutes @@ -369,7 +369,7 @@ kubernetes.core.k8s_info: api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" register: mongodb_cr retries: 45 # Approx 45 minutes @@ -384,7 +384,7 @@ kubernetes.core.k8s_info: api_version: monitoring.coreos.com/v1 kind: ServiceMonitor - name: "{{ mongodb_instance_name }}-service-monitor" + name: mas-mongo-ce-service-monitor namespace: "{{ mongodb_namespace }}" register: mongoce_servicemonitor_info when: @@ -477,7 +477,7 @@ kind: Pod namespace: "{{ mongodb_namespace }}" label_selectors: - - "app={{ mongodb_instance_name }}-svc" + - app=mas-mongo-ce-svc register: mongo_pods_output # List all mongo replicas @@ -498,7 +498,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: Secret - name: "{{ mongodb_instance_name }}-admin-admin" + name: mas-mongo-ce-admin-admin namespace: "{{ mongodb_namespace }}" register: admin_password_lookup retries: 30 # 30 * 30 seconds = 30 minutes @@ -512,7 +512,7 @@ kubernetes.core.k8s_info: api_version: v1 kind: ConfigMap - name: "{{ mongodb_instance_name }}-cert-map" + name: mas-mongo-ce-cert-map namespace: "{{ mongodb_namespace }}" register: mongodb_ca_lookup diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml index d412b42526..e71fcd4082 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/uninstall.yml @@ -6,7 +6,7 @@ msg: - "MongoDb namespace ............ {{ mongodb_namespace }}" -# 2. Delete MongoDBCommunity CR +# 2. Delete MongoDBCommunity (mas-mongo-ce) # ----------------------------------------------------------------------------- - name: "community : uninstall : Delete the MongoDBCommunity CR" kubernetes.core.k8s: @@ -14,7 +14,7 @@ api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity namespace: "{{ mongodb_namespace }}" - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce # 3. Wait for StatefulSet to shut down # ----------------------------------------------------------------------------- @@ -37,7 +37,7 @@ api_version: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity namespace: "{{ mongodb_namespace }}" - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce register: verify_mongo_delete - name: "uninstall : Verify the MongoDBCommunity CR was deleted" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 index f2967f623f..93becce073 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -19,7 +19,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: "{{ mongodb_instance_name }}-cert-map" + name: mas-mongo-ce-cert-map authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -28,12 +28,12 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret users: - name: admin db: admin passwordSecretRef: - name: "{{ mongodb_instance_name }}-admin-password" + name: mas-mongo-ce-admin-password roles: - name: clusterAdmin db: admin @@ -43,14 +43,14 @@ spec: db: admin - name: readWriteAnyDatabase db: admin - scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" + scramCredentialsSecretName: mas-mongo-ce-scram additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: "{{ mongodb_instance_name }}-svc" + serviceName: mas-mongo-ce-svc selector: {} template: spec: @@ -84,4 +84,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" + storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 index 2a9f190c90..a2efee5149 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: "{{ mongodb_instance_name }} + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -18,7 +18,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: "{{ mongodb_instance_name }}-cert-map" + name: mas-mongo-ce-cert-map authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -27,12 +27,12 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret users: - name: admin db: admin passwordSecretRef: - name: "{{ mongodb_instance_name }}-admin-password" + name: mas-mongo-ce-admin-password roles: - name: clusterAdmin db: admin @@ -44,14 +44,14 @@ spec: db: admin - name: dbAdminAnyDatabase db: admin - scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" + scramCredentialsSecretName: mas-mongo-ce-scram additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: "{{ mongodb_instance_name }}-svc" + serviceName: mas-mongo-ce-svc selector: {} template: spec: @@ -85,4 +85,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" + storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 index 8febb09904..b5f40f95f3 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -19,7 +19,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: "{{ mongodb_instance_name }}-cert-map" + name: mas-mongo-ce-cert-map authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -28,12 +28,12 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret users: - name: admin db: admin passwordSecretRef: - name: "{{ mongodb_instance_name }}-admin-password" + name: mas-mongo-ce-admin-password roles: - name: clusterAdmin db: admin @@ -45,14 +45,14 @@ spec: db: admin - name: dbAdminAnyDatabase db: admin - scramCredentialsSecretName: ""{{ mongodb_instance_name }}-scram" + scramCredentialsSecretName: mas-mongo-ce-scram additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: "{{ mongodb_instance_name }}-svc" + serviceName: mas-mongo-ce-svc selector: {} template: spec: @@ -86,4 +86,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" + storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 index f2967f623f..93becce073 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 @@ -2,7 +2,7 @@ apiVersion: mongodbcommunity.mongodb.com/v1 kind: MongoDBCommunity metadata: - name: "{{ mongodb_instance_name }}" + name: mas-mongo-ce namespace: "{{ mongodb_namespace }}" spec: members: {{ mongodb_replicas }} @@ -19,7 +19,7 @@ spec: certificateKeySecretRef: name: mongo-server-cert caConfigMapRef: - name: "{{ mongodb_instance_name }}-cert-map" + name: mas-mongo-ce-cert-map authentication: modes: # Note: MAS 8.5 and earlier only support SCRAM-SHA-1 @@ -28,12 +28,12 @@ spec: prometheus: username: metrics-endpoint-user passwordSecretRef: - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret users: - name: admin db: admin passwordSecretRef: - name: "{{ mongodb_instance_name }}-admin-password" + name: mas-mongo-ce-admin-password roles: - name: clusterAdmin db: admin @@ -43,14 +43,14 @@ spec: db: admin - name: readWriteAnyDatabase db: admin - scramCredentialsSecretName: "{{ mongodb_instance_name }}-scram" + scramCredentialsSecretName: mas-mongo-ce-scram additionalMongodConfig: storage.wiredTiger.engineConfig.journalCompressor: snappy net.tls.allowInvalidCertificates: true net.tls.allowInvalidHostnames: true statefulSet: spec: - serviceName: "{{ mongodb_instance_name }}-svc" + serviceName: mas-mongo-ce-svc selector: {} template: spec: @@ -84,4 +84,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" + storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml b/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml index 8e96577723..b6eece0373 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/admin-password.yml @@ -4,7 +4,7 @@ apiVersion: v1 kind: Secret metadata: - name: "{{ mongodb_instance_name }}-admin-password" + name: mas-mongo-ce-admin-password namespace: "{{ mongodb_namespace }}" type: Opaque stringData: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml b/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml index e8174be116..9637715c32 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/ca-cert.yml @@ -13,7 +13,7 @@ spec: algorithm: ECDSA size: 256 dnsNames: - - "*.{{mongodb_instance_name}}-svc.{{mongodb_namespace}}.svc.cluster.local" + - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" - "127.0.0.1" - "localhost" issuerRef: diff --git a/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 index 212a04c75f..5f44023dec 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 @@ -4,9 +4,9 @@ apiVersion: v1 kind: Secret metadata: - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret namespace: "{{ mongodb_namespace }}" type: Opaque stringData: username: metrics-endpoint-user - password: "{{ lookup('password', '/tmp/mongoce-password.txt chars=ascii_letters,digits length=16') }}" + password: "{{ lookup('password', '/tmp/mongoce-password.txt chars=ascii_letters,digits length=16') }}" \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 index 0238035588..d5c042531e 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 @@ -1,4 +1,4 @@ {% for host in mongo_replicas %} -- host: "{{ host }}.{{ mongodb_instance_name }}-svc.{{ mongodb_namespace }}.svc.cluster.local" +- host: "{{ host }}.mas-mongo-ce-svc.{{ mongodb_namespace }}.svc.cluster.local" port: 27017 -{% endfor %} +{% endfor %} \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml b/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml index 9feaeddcd8..3c6ca1e53f 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/server-cert.yml @@ -7,7 +7,7 @@ spec: duration: 8760h # 365d renewBefore: 360h # 15d dnsNames: - - "*.{{mongodb_instance_name}}-svc.{{mongodb_namespace}}.svc.cluster.local" + - "*.mas-mongo-ce-svc.{{mongodb_namespace}}.svc.cluster.local" - "127.0.0.1" - "localhost" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 index f35f80d64f..3bf6ec62f8 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/servicemonitor.yml.j2 @@ -2,17 +2,17 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: "{{ mongodb_instance_name }}-service-monitor" + name: mas-mongo-ce-service-monitor namespace: "{{ mongodb_namespace }}" spec: endpoints: - basicAuth: password: key: password - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret username: key: username - name: "{{ mongodb_instance_name }}-metrics-endpoint-secret" + name: mas-mongo-ce-metrics-endpoint-secret # This port matches what we created in our MongoDB Service. port: prometheus @@ -33,4 +33,4 @@ spec: # Service labels to match selector: matchLabels: - app: "{{ mongodb_instance_name }}-svc" \ No newline at end of file + app: mas-mongo-ce-svc \ No newline at end of file diff --git a/ibm/mas_devops/roles/mongodb/templates/community/tls.yml b/ibm/mas_devops/roles/mongodb/templates/community/tls.yml index 030ef9f4f8..9e9b832dbf 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/tls.yml +++ b/ibm/mas_devops/roles/mongodb/templates/community/tls.yml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: "{{ mongodb_instance_name }}-cert-map" + name: mas-mongo-ce-cert-map namespace: "{{ mongodb_namespace }}" type: Opaque data: From 0b536dd58fb78dafeb1f8efac33930b44d75a4af Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Mar 2026 19:20:37 +0000 Subject: [PATCH 58/61] extra newline --- ibm/mas_devops/roles/mongodb/README.md | 2 +- .../roles/mongodb/templates/community/0.12.0/cr.yml.j2 | 2 +- .../roles/mongodb/templates/community/0.7.9/cr.yml.j2 | 2 +- .../roles/mongodb/templates/community/0.8.3/cr.yml.j2 | 2 +- .../roles/mongodb/templates/community/0.9.0/cr.yml.j2 | 2 +- .../mongodb/templates/community/metrics-endpoint-secret.yml.j2 | 2 +- .../roles/mongodb/templates/community/mongo-hosts.yml.j2 | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index c001fe66b1..fad478546d 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -121,7 +121,7 @@ Specifies which operation to perform on the MongoDB instance. - Use `create-mongo-service-credentials` to generate service credentials (ibm only) **Valid values** (provider-specific): -- **community**: `install`, `uninstall`, `backup`, `restore`, `restore_database` +- **community**: `install`, `uninstall`, `backup`, `backup_database`, `restore`, `restore_database` - **aws**: `install`, `uninstall`, `docdb_secret_rotate`, `destroy-data` - **ibm**: `install`, `uninstall`, `backup`, `restore`, `create-mongo-service-credentials` diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 index 93becce073..51b8a460e7 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.12.0/cr.yml.j2 @@ -84,4 +84,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file + storage: "{{ mongodb_storage_capacity_logs }}" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 index a2efee5149..e44b3468a6 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.7.9/cr.yml.j2 @@ -85,4 +85,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file + storage: "{{ mongodb_storage_capacity_logs }}" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 index b5f40f95f3..8cb97b18ea 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.8.3/cr.yml.j2 @@ -86,4 +86,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file + storage: "{{ mongodb_storage_capacity_logs }}" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 index 93becce073..51b8a460e7 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/0.9.0/cr.yml.j2 @@ -84,4 +84,4 @@ spec: storageClassName: "{{ mongodb_storage_class }}" resources: requests: - storage: "{{ mongodb_storage_capacity_logs }}" \ No newline at end of file + storage: "{{ mongodb_storage_capacity_logs }}" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 index 5f44023dec..f5081d0825 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/metrics-endpoint-secret.yml.j2 @@ -9,4 +9,4 @@ metadata: type: Opaque stringData: username: metrics-endpoint-user - password: "{{ lookup('password', '/tmp/mongoce-password.txt chars=ascii_letters,digits length=16') }}" \ No newline at end of file + password: "{{ lookup('password', '/tmp/mongoce-password.txt chars=ascii_letters,digits length=16') }}" diff --git a/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 b/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 index d5c042531e..8a7c795c2c 100644 --- a/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 +++ b/ibm/mas_devops/roles/mongodb/templates/community/mongo-hosts.yml.j2 @@ -1,4 +1,4 @@ {% for host in mongo_replicas %} - host: "{{ host }}.mas-mongo-ce-svc.{{ mongodb_namespace }}.svc.cluster.local" port: 27017 -{% endfor %} \ No newline at end of file +{% endfor %} From c7ddb1daaf4e778ec13b70905e741d1fffd8ecca Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Tue, 3 Mar 2026 19:25:01 +0000 Subject: [PATCH 59/61] cleanup grafana role --- ibm/mas_devops/roles/grafana/defaults/main.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ibm/mas_devops/roles/grafana/defaults/main.yml b/ibm/mas_devops/roles/grafana/defaults/main.yml index a19f16c31d..3a12f9d38c 100644 --- a/ibm/mas_devops/roles/grafana/defaults/main.yml +++ b/ibm/mas_devops/roles/grafana/defaults/main.yml @@ -11,7 +11,3 @@ grafana_v5_namespace: "{{ lookup('env', 'GRAFANA_V5_NAMESPACE') | default('grafa # Settings to set grafana to define a specific storage class and size grafana_instance_storage_class: "{{ lookup('env', 'GRAFANA_INSTANCE_STORAGE_CLASS') }}" grafana_instance_storage_size: "{{ lookup('env', 'GRAFANA_INSTANCE_STORAGE_SIZE') | default('10Gi', true) }}" - -# Backup and restore variables -grafana_backup_version: "{{ lookup('env', 'GRAFANA_BACKUP_VERSION') }}" -mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" From 6e3386ccb2652cd5d5cf76c709d1cdb84e39c8e2 Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Wed, 11 Mar 2026 08:18:52 +0000 Subject: [PATCH 60/61] Change backup and restore database actions to have - separator in mongodb and db2 (#2154) --- docs/playbooks/backup-restore.md | 16 +- ibm/mas_devops/playbooks/br_core.yml | 183 ------------------ ibm/mas_devops/playbooks/br_db2.yml | 6 +- ibm/mas_devops/playbooks/br_manage.yml | 112 ----------- ibm/mas_devops/playbooks/br_mongodb.yml | 10 +- .../action/verify_backup_restore_vars.py | 4 +- ibm/mas_devops/roles/db2/README.md | 20 +- ibm/mas_devops/roles/db2/defaults/main.yml | 2 +- .../backup-database.yml | 0 .../main.yml | 2 +- .../roles/db2/tasks/backup/main.yml | 2 +- ibm/mas_devops/roles/db2/tasks/main.yml | 6 +- .../main.yml | 2 +- .../restore-database.yml | 4 +- .../restore-db-from-disk.yml | 0 .../restore-db-from-s3.yml | 0 .../roles/db2/tasks/restore/main.yml | 2 +- .../db2/tasks/restore/restore-instance.yml | 2 +- ibm/mas_devops/roles/mongodb/README.md | 26 +-- .../roles/mongodb/defaults/main.yml | 2 +- ibm/mas_devops/roles/mongodb/tasks/main.yml | 4 +- ...ackup_database.yml => backup-database.yml} | 0 ...tore_database.yml => restore-database.yml} | 0 ...tore_instance.yml => restore-instance.yml} | 0 .../tasks/providers/community/restore.yml | 4 +- 25 files changed, 57 insertions(+), 352 deletions(-) delete mode 100644 ibm/mas_devops/playbooks/br_core.yml delete mode 100644 ibm/mas_devops/playbooks/br_manage.yml rename ibm/mas_devops/roles/db2/tasks/{backup_database => backup-database}/backup-database.yml (100%) rename ibm/mas_devops/roles/db2/tasks/{backup_database => backup-database}/main.yml (95%) rename ibm/mas_devops/roles/db2/tasks/{restore_database => restore-database}/main.yml (61%) rename ibm/mas_devops/roles/db2/tasks/{restore_database => restore-database}/restore-database.yml (95%) rename ibm/mas_devops/roles/db2/tasks/{restore_database => restore-database}/restore-db-from-disk.yml (100%) rename ibm/mas_devops/roles/db2/tasks/{restore_database => restore-database}/restore-db-from-s3.yml (100%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{backup_database.yml => backup-database.yml} (100%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{restore_database.yml => restore-database.yml} (100%) rename ibm/mas_devops/roles/mongodb/tasks/providers/community/{restore_instance.yml => restore-instance.yml} (100%) diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index 480441ad4b..dbecbff6a6 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -57,7 +57,7 @@ The playbook executes the following operations: ### Common Variables (Backup and Restore) - `MAS_INSTANCE_ID` - The instance ID of the MAS installation - `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) -- `MONGODB_ACTION` - Set to `backup`, `backup_database`, `restore`, or `restore_database` +- `MONGODB_ACTION` - Set to `backup`, `backup-database`, `restore`, or `restore-database` - `MONGODB_INSTANCE_NAME` - MongoDB instance name (default: `mas-mongo-ce`) - `MONGODB_NAMESPACE` - Namespace where MongoDB is installed (default: `mongoce`) @@ -112,7 +112,7 @@ Create a backup of a specific database only: ```bash export MAS_INSTANCE_ID=inst1 export MAS_BACKUP_DIR=/backup/mas -export MONGODB_ACTION=backup_database +export MONGODB_ACTION=backup-database export MONGODB_INSTANCE_NAME=mas-mongo-ce export MAS_APP_ID=manage @@ -160,7 +160,7 @@ Restore a specific database only: ```bash export MAS_INSTANCE_ID=inst1 export MAS_BACKUP_DIR=/backup/mas -export MONGODB_ACTION=restore_database +export MONGODB_ACTION=restore-database export MONGODB_BACKUP_VERSION=20260122-131500 export MONGODB_INSTANCE_NAME=mas-mongo-ce export MAS_APP_ID=manage @@ -173,11 +173,11 @@ ansible-playbook ibm.mas_devops.br_mongodb ### Backup Actions - **backup**: Full backup of MongoDB instance (Kubernetes resources + all database data) -- **backup_database**: Backup specific database data only (requires `MAS_APP_ID`) +- **backup-database**: Backup specific database data only (requires `MAS_APP_ID`) ### Restore Actions - **restore**: Full restore of MongoDB instance (Kubernetes resources + all database data) -- **restore_database**: Restore specific database data only (requires `MAS_APP_ID`) +- **restore-database**: Restore specific database data only (requires `MAS_APP_ID`) ### Prerequisites for Restore - Target cluster must have MongoDB Community Operator installed @@ -489,9 +489,9 @@ This playbook performs backup and restore operations for IBM Db2 Universal Opera **Important**: The playbook supports multiple backup actions: - `backup` - Full Db2 instance backup -- `backup_database` - Individual database backup +- `backup-database` - Individual database backup - `restore` - Full Db2 instance restore -- `restore_database` - Individual database restore +- `restore-database` - Individual database restore ## Required Environment Variables @@ -499,7 +499,7 @@ This playbook performs backup and restore operations for IBM Db2 Universal Opera - `MAS_INSTANCE_ID` - The instance ID of the MAS installation - `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) - `DB2_INSTANCE_NAME` - Name of the Db2 instance -- `DB2_ACTION` - Set to `backup`, `backup_database`, `restore`, or `restore_database` +- `DB2_ACTION` - Set to `backup`, `backup-database`, `restore`, or `restore-database` ### Backup-Specific Variables - `DB2_BACKUP_TYPE` - Set to `online` or `offline` (default: `online`) diff --git a/ibm/mas_devops/playbooks/br_core.yml b/ibm/mas_devops/playbooks/br_core.yml deleted file mode 100644 index 29161850c0..0000000000 --- a/ibm/mas_devops/playbooks/br_core.yml +++ /dev/null @@ -1,183 +0,0 @@ ---- -- name: "MAS Core Backup and Restore" - hosts: localhost - any_errors_fatal: true - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - - # Set the action for this playbook of either backup or restore - br_action: "{{ lookup('env', 'BR_ACTION') | default('backup', true) }}" - - # The top level location to save (on backup) or retrieve (on restore) the archive files - mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups - - # Backup version to use for all during backup action - # Note: The default timestamp will be set in pre_tasks to ensure consistency across all roles - backup_version: "{{ lookup('env', 'BACKUP_VERSION') }}" - - # Required for restore action, the backup version to use on restore - backup_version_to_restore: "{{ lookup('env', 'BACKUP_VERSION_TO_RESTORE') }}" - - # Computed variable: use backup_version for backup action, backup_version_to_restore for restore action - br_action_version: "{{ backup_version if br_action == 'backup' else backup_version_to_restore }}" - - # Configurable flags for Grafana. Set to false to skip Grafana - include_grafana: "{{ lookup('env', 'INCLUDE_GRAFANA') | default('true', true) | bool }}" - - # Configurable flags for SLS and DRO. You should not include SLS or DRO if theses are configured externally - # the the cluster running the Suite. When these values are false then the slscfg and bascfg (for DRO) will - # be restored from what is defined in the backup config for the suite. - include_sls: "{{ lookup('env', 'INCLUDE_SLS') | default('true', true) | bool }}" # Set to false to skip SLS - include_dro: "{{ lookup('env', 'INCLUDE_DRO') | default('true', true) | bool }}" # Set to false to skip DRO - - # Required for DRO install - ibm_entitlement_key: "{{ lookup('env', 'IBM_ENTITLEMENT_KEY') }}" - dro_contact_email: "{{ lookup('env', 'DRO_CONTACT_EMAIL') }}" - dro_contact_firstname: "{{ lookup('env', 'DRO_CONTACT_FIRSTNAME') }}" - dro_contact_lastname: "{{ lookup('env', 'DRO_CONTACT_LASTNAME') }}" - - # mongo restore configuration - override_mongodb_storageclass: "{{ lookup('env', 'OVERRIDE_MONGODB_STORAGECLASS') | default('false', true) | bool }}" # Set to true to override - # when OVERRIDE_MONGODB_STORAGECLASS to true, MONGODB_STORAGECLASS_NAME_RWO will be used - mongodb_storageclass_rwo: "{{ lookup('env', 'MONGODB_STORAGECLASS_NAME_RWO') | default('', true) }}" - - # Set the following vars to control the domain and urls used on the restore. - # This is used when you are restoring to a different cluster than the one that was backed up, - # and the domain will be different. - mas_domain_on_restore: "{{ lookup('env', 'MAS_DOMAIN_ON_RESTORE') }}" - # If `include_sls` is false. The URL for the Suite License Service (SLS). If not provided, the - # URL from the backup will be used. This is used when the domain has changed for SLS but SLS was restored - # from a backup and so the regristration key is the same. - sls_url_on_restore: "{{ lookup('env', 'SLS_URL_ON_RESTORE') }}" - # Same setting as above but for dro when `include_dro` is false. - dro_url_on_restore: "{{ lookup('env', 'MAS_DOMAIN_ON_RESTORE') }}" - - # For DRO and SLS set the mas_config_dir for when these are either restored or installed new. - mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" - - pre_tasks: - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * These playbooks are samples to demonstrate how to use the roles * - * in this collection. * - * * - * They are NOT INTENDED FOR PRODUCTION USE as-is, they are a * - * starting point for power users to aid in the development of * - * their own Ansible playbooks using the roles in this collection * - * * - * The recommended way to install MAS is to use the MAS CLI, which * - * uses this Ansible Collection to deliver a complete managed * - * lifecycle for your MAS instance. * - * * - * https://ibm-mas.github.io/cli/ * - * * - ********************************************************************* - - - name: "Set backup_version with timestamp if not provided" - ansible.builtin.set_fact: - backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" - when: backup_version is not defined or backup_version == '' - - - name: "Fail if mas_instance_id is not set to" - ansible.builtin.assert: - that: - - mas_instance_id is defined - fail_msg: "mas_instance_id is required and must be set" - - - name: "Fail if mas_backup_dir is not set to" - ansible.builtin.assert: - that: - - mas_backup_dir is defined - fail_msg: "mas_backup_dir is required and must be set" - - - name: "Fail if br_action is not set to backup|restore" - ansible.builtin.assert: - that: - - br_action is defined - - br_action in ["backup", "restore"] - fail_msg: "br_action is required and must be set to 'backup' or 'restore'" - - - name: "Fail if backup_version_to_restore is not set on restore" - ansible.builtin.assert: - that: - - backup_version_to_restore is defined - fail_msg: "backup_version_to_restore is required when running 'restore'" - when: br_action == "restore" - - - name: Check for required DRO environment variables - assert: - that: - # DRO. - - lookup('env', 'IBM_ENTITLEMENT_KEY') != "" - - lookup('env', 'DRO_CONTACT_EMAIL') != "" - - lookup('env', 'DRO_CONTACT_FIRSTNAME') != "" - - lookup('env', 'DRO_CONTACT_LASTNAME') != "" - fail_msg: "One or more required environment variables are not defined for DRO" - when: - - include_dro | bool - - br_action == "restore" - - roles: - - role: ibm.mas_devops.ibm_catalogs - vars: - ibm_catalogs_action: "{{ br_action }}" - ibm_catalogs_backup_version: "{{ br_action_version }}" - - - role: ibm.mas_devops.cert_manager - vars: - cert_manager_action: "{{ br_action }}" - certmanager_backup_version: "{{ br_action_version }}" - - - role: ibm.mas_devops.grafana - when: - - include_grafana | bool - - br_action == "restore" - vars: - grafana_action: "install" - - - role: ibm.mas_devops.mongodb - vars: - mongodb_action: "{{ br_action }}" - mongodb_backup_version: "{{ br_action_version }}" - mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" - mongodb_provider: "community" # only community is supported - override_storageclass: "{{ override_mongodb_storageclass }}" - mongodb_storage_class: "{{ mongodb_storageclass_rwo }}" - - - role: ibm.mas_devops.sls - when: include_sls | bool - vars: - sls_action: "{{ br_action }}" - sls_backup_version: "{{ br_action_version }}" - - - role: ibm.mas_devops.suite_backup - when: br_action == "backup" - vars: - suite_backup_version: "{{ br_action_version }}" - - - role: ibm.mas_devops.dro - when: - - include_dro | bool - - br_action == "restore" - vars: - dro_action: "install" - - - role: ibm.mas_devops.suite_restore - when: br_action == "restore" - vars: - # SLS skiped in this play so include the config in the restore from the backup - include_sls_from_backup: "{{ not (include_sls | bool) }}" - # DRO skiped in this play so include the config in the restore from the backup - include_dro_from_backup: "{{ not (include_dro | bool) }}" - suite_backup_version: "{{ br_action_version }}" - sls_cfg_file: "{{ (mas_config_dir + '/sls.yml') if (include_sls | bool) else omit }}" - dro_cfg_file: "{{ (mas_config_dir + '/dro.yml') if (include_dro | bool) else omit }}" - mas_domain: "{{ mas_domain_on_restore if (mas_domain_on_restore is defined and mas_domain_on_restore != '') else omit }}" - sls_url: "{{ sls_url_on_restore if (sls_url_on_restore is defined and sls_url_on_restore != '') else omit }}" - dro_url: "{{ dro_url_on_restore if (dro_url_on_restore is defined and dro_url_on_restore != '') else omit }}" diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index c34827905d..28b65df3bd 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -8,7 +8,7 @@ mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups db2_instance_name: "{{ lookup('env', 'DB2_INSTANCE_NAME') }}" db2_namespace: "{{ lookup('env', 'DB2_NAMESPACE') | default('db2u', true) }}" - db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or backup_database or restore or restore_database + db2_action: "{{ lookup('env', 'DB2_ACTION') }}" # backup or backup-database or restore or restore-database db2_backup_version: "{{ lookup('env', 'DB2_BACKUP_VERSION') }}" # Required for restore action backup_type: "{{ lookup('env', 'DB2_BACKUP_TYPE') | default('online', true) }}" # Supported values are online, offline and required for backup backup_vendor: "{{ lookup('env', 'BACKUP_VENDOR') | default('disk', true) | lower }}" @@ -52,8 +52,8 @@ - name: "Fail if DB2_ACTION is not set to backup|restore" assert: - that: db2_action in ["backup", "backup_database", "restore_database", "restore"] - fail_msg: "DB2_ACTION is required and must be set to 'backup' or 'backup_database' or 'restore_database' or 'restore'" + that: db2_action in ["backup", "backup-database", "restore-database", "restore"] + fail_msg: "DB2_ACTION is required and must be set to 'backup' or 'backup-database' or 'restore-database' or 'restore'" - name: "Fail if BACKUP_VENDOR is not set to s3|disk" assert: diff --git a/ibm/mas_devops/playbooks/br_manage.yml b/ibm/mas_devops/playbooks/br_manage.yml deleted file mode 100644 index 3e5270634f..0000000000 --- a/ibm/mas_devops/playbooks/br_manage.yml +++ /dev/null @@ -1,112 +0,0 @@ ---- -- name: "MAS Manage Application Backup and Restore" - hosts: localhost - any_errors_fatal: true - vars: - # Define the target for backup/restore - mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mas_workspace_id: "{{ lookup('env', 'MAS_WORKSPACE_ID') }}" - mas_app_id: "manage" - - # Set the action for this playbook of either backup or restore - mas_app_action: "{{ lookup('env', 'MAS_APP_ACTION') | default('backup', true) }}" - - # The top level location to save (on backup) or retrieve (on restore) the archive files - mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups - - # Backup version to use during backup action - # Note: The default timestamp will be set in pre_tasks - mas_app_backup_version: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION') }}" - - # Required for restore action, the backup version to use on restore - mas_app_backup_version_to_restore: "{{ lookup('env', 'MAS_APP_BACKUP_VERSION_TO_RESTORE') }}" - - # Computed variable: use mas_app_backup_version for backup action, mas_app_backup_version_to_restore for restore action - mas_app_action_version: "{{ mas_app_backup_version if mas_app_action == 'backup' else mas_app_backup_version_to_restore }}" - - # Storage Class Override Options for restore - # Set OVERRIDE_STORAGECLASS to true to override the storage class names from backup - override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default('false', true) | bool }}" - # When OVERRIDE_STORAGECLASS is true, use the below storage classes - mas_app_custom_storage_class_rwo: "{{ lookup('env', 'MAS_APP_CUSTOM_STORAGE_CLASS_RWO') | default('', true) }}" - mas_app_custom_storage_class_rwx: "{{ lookup('env', 'MAS_APP_CUSTOM_STORAGE_CLASS_RWX') | default('', true) }}" - - pre_tasks: - - name: Important Notice - debug: - msg: | - ********************************************************************* - ************************* IMPORTANT NOTICE ************************** - ********************************************************************* - * * - * These playbooks are samples to demonstrate how to use the roles * - * in this collection. * - * * - * They are NOT INTENDED FOR PRODUCTION USE as-is, they are a * - * starting point for power users to aid in the development of * - * their own Ansible playbooks using the roles in this collection * - * * - * The recommended way to install MAS is to use the MAS CLI, which * - * uses this Ansible Collection to deliver a complete managed * - * lifecycle for your MAS instance. * - * * - * https://ibm-mas.github.io/cli/ * - * * - ********************************************************************* - - - name: "Set mas_app_backup_version with timestamp if not provided" - ansible.builtin.set_fact: - mas_app_backup_version: "{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}" - when: mas_app_backup_version is not defined or mas_app_backup_version == '' - - - name: "Fail if mas_instance_id is not set" - ansible.builtin.assert: - that: - - mas_instance_id is defined - - mas_instance_id != '' - fail_msg: "mas_instance_id is required and must be set" - - - name: "Fail if mas_workspace_id is not set" - ansible.builtin.assert: - that: - - mas_workspace_id is defined - - mas_workspace_id != '' - fail_msg: "mas_workspace_id is required and must be set" - - - name: "Fail if mas_backup_dir is not set" - ansible.builtin.assert: - that: - - mas_backup_dir is defined - - mas_backup_dir != '' - fail_msg: "mas_backup_dir is required and must be set" - - - name: "Fail if mas_app_action is not set to backup|restore" - ansible.builtin.assert: - that: - - mas_app_action is defined - - mas_app_action in ["backup", "restore"] - fail_msg: "mas_app_action is required and must be set to 'backup' or 'restore'" - - - name: "Fail if mas_app_backup_version_to_restore is not set on restore" - ansible.builtin.assert: - that: - - mas_app_backup_version_to_restore is defined - - mas_app_backup_version_to_restore != '' - fail_msg: "mas_app_backup_version_to_restore is required when running 'restore'" - when: mas_app_action == "restore" - - roles: - - role: ibm.mas_devops.suite_app_backup - when: mas_app_action == "backup" - vars: - mas_app_id: "manage" - mas_app_backup_version: "{{ mas_app_action_version }}" - - - role: ibm.mas_devops.suite_app_restore - when: mas_app_action == "restore" - vars: - mas_app_id: "manage" - mas_app_backup_version: "{{ mas_app_action_version }}" - override_storageclass: "{{ override_storageclass }}" - mas_app_custom_storage_class_rwo: "{{ mas_app_custom_storage_class_rwo }}" - mas_app_custom_storage_class_rwx: "{{ mas_app_custom_storage_class_rwx }}" diff --git a/ibm/mas_devops/playbooks/br_mongodb.yml b/ibm/mas_devops/playbooks/br_mongodb.yml index eba3ea3ab2..1c2b64dfd7 100644 --- a/ibm/mas_devops/playbooks/br_mongodb.yml +++ b/ibm/mas_devops/playbooks/br_mongodb.yml @@ -4,9 +4,9 @@ vars: # Define the target for backup/restore mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" - mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('backup', true) }}" # backup or backup_database or restore_database or install + mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('backup', true) }}" # backup or backup-database or restore-database or install mas_backup_dir: "{{ lookup('env', 'MAS_BACKUP_DIR') }}" # eg: /tmp/mas_backups - mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" # Required for restore or restore_database action + mongodb_backup_version: "{{ lookup('env', 'MONGODB_BACKUP_VERSION') }}" # Required for restore or restore-database action mongodb_instance_name: "{{ lookup('env', 'MONGODB_INSTANCE_NAME') | default('mas-mongo-ce', true) }}" mongodb_namespace: "{{ lookup('env', 'MONGODB_NAMESPACE') | default('mongoce', true) }}" mongodb_provider: "community" # only community is supported currently @@ -38,10 +38,10 @@ * https://ibm-mas.github.io/cli/ * * * ********************************************************************* - - name: "Fail if mongodb_action is not set to backup|backup_database|restore|restore_database" + - name: "Fail if mongodb_action is not set to backup|backup-database|restore|restore-database" assert: - that: mongodb_action in ["backup", "backup_database", "restore_database", "restore"] - fail_msg: "mongodb_action is required and must be set to 'backup' or 'backup_database' or 'restore' or 'restore_database'" + that: mongodb_action in ["backup", "backup-database", "restore-database", "restore"] + fail_msg: "mongodb_action is required and must be set to 'backup' or 'backup-database' or 'restore' or 'restore-database'" roles: - role: ibm.mas_devops.mongodb diff --git a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py index fd73d5c195..03e7f06dac 100644 --- a/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py +++ b/ibm/mas_devops/plugins/action/verify_backup_restore_vars.py @@ -17,8 +17,8 @@ class ActionModule(ActionBase): }, "db2": { "backup": ["db2_instance_name", "db2_namespace", "mas_backup_dir", "mas_instance_id", "mas_application_id"], - "restore_instance": ["mas_backup_dir", "db2_backup_version", "mas_application_id"], - "restore_database": ["mas_backup_dir", "db2_backup_version", "db2_instance_name", "backup_vendor", "mas_application_id"], + "restore-instance": ["mas_backup_dir", "db2_backup_version", "mas_application_id"], + "restore-database": ["mas_backup_dir", "db2_backup_version", "db2_instance_name", "backup_vendor", "mas_application_id"], "s3_setup": ["backup_vendor", "backup_s3_alias", "backup_s3_endpoint", "backup_s3_bucket", "backup_s3_access_key", "backup_s3_secret_key"] }, "grafana": { diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index 30bd61771e..68ad569926 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -103,21 +103,21 @@ Specifies which operation to perform on the Db2 database. - Use `upgrade` to upgrade all Db2 instances in the namespace to a new version - Use `backup` to create a backup of Db2 instance and/or database - Use `restore` to restore a backup of db2 instance and database -- Use `restore_database` to restore only the database to an existing Db2 instance +- Use `restore-database` to restore only the database to an existing Db2 instance -**Valid values**: `install`, `upgrade`, `backup`, `restore`, `restore_database` +**Valid values**: `install`, `upgrade`, `backup`, `restore`, `restore-database` **Impact**: - `install`: Creates new Db2 operator and instance. When `db2_backup_version` is provided, installs from backup (instance + database) - `upgrade`: Upgrades ALL instances in `db2_namespace` to `db2_version` (affects all instances in namespace) - `backup`: Creates backup of Db2 instance resources and/or database data - `restore`: Creates new Db2 operator and instance from the backup of Db2 instance resources and restores database data to the created instance -- `restore_database`: Restores database to an existing running Db2 instance (does not restore instance resources) +- `restore-database`: Restores database to an existing running Db2 instance (does not restore instance resources) **Related variables**: - `db2_version`: Required for upgrade action to specify target version - `db2_namespace`: All instances in this namespace are affected by upgrade -- `db2_backup_version`: Required for restore/restore_database action; optional for backup, defaults to YYYYMMDD-HHMMSS +- `db2_backup_version`: Required for restore/restore-database action; optional for backup, defaults to YYYYMMDD-HHMMSS - `override_storageclass`: In Restore, controls whether storage classes are overridden **Note**: **WARNING** - When using `upgrade`, ALL Db2 instances in the specified namespace will be upgraded. Plan accordingly and ensure `db2_version` matches the operator channel. @@ -1028,7 +1028,7 @@ Local directory path where backups will be stored or restored from. ### db2_backup_version The backup version timestamp identifier for backup and restore operations. -- **Required** for `restore` and `restore_database` actions +- **Required** for `restore` and `restore-database` actions - **Auto-generated** for backup operations - Environment Variable: `DB2_BACKUP_VERSION` - Default: Auto-generated in format `YYYYMMDD-HHMMSS` @@ -1049,7 +1049,7 @@ The backup version timestamp identifier for backup and restore operations. **Related variables**: - `mas_backup_dir`: Parent directory containing versioned backups -- `db2_action`: Required when action is `restore_database` or `restore`(instance & database) +- `db2_action`: Required when action is `restore-database` or `restore`(instance & database) **Example**: `20251212-021316` @@ -1275,7 +1275,7 @@ Example Usage - Backup and Restore vars: mas_instance_id: masinst1 mas_backup_dir: /tmp/masbr - db2_action: backup_database + db2_action: backup-database db2_instance_name: db2u-manage db2_namespace: db2u backup_type: online @@ -1291,7 +1291,7 @@ Example Usage - Backup and Restore vars: mas_instance_id: masinst1 mas_backup_dir: /tmp/masbr - db2_action: backup_database + db2_action: backup-database db2_instance_name: db2u-manage backup_type: online backup_vendor: s3 @@ -1323,7 +1323,7 @@ Example Usage - Backup and Restore - hosts: localhost any_errors_fatal: true vars: - db2_action: restore_database + db2_action: restore-database mas_instance_id: masinst1 db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr @@ -1339,7 +1339,7 @@ Example Usage - Backup and Restore - hosts: localhost any_errors_fatal: true vars: - db2_action: restore_database + db2_action: restore-database mas_instance_id: masinst1 db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr diff --git a/ibm/mas_devops/roles/db2/defaults/main.yml b/ibm/mas_devops/roles/db2/defaults/main.yml index 54ac79d482..8fe702e383 100644 --- a/ibm/mas_devops/roles/db2/defaults/main.yml +++ b/ibm/mas_devops/roles/db2/defaults/main.yml @@ -1,7 +1,7 @@ --- # db2 action # --------------------------------------------------------------------------- -# Supported actions: install, uninstall, backup, backup_database, restore, restore_database +# Supported actions: install, uninstall, backup, backup-database, restore, restore-database db2_action: "{{ lookup('env', 'DB2_ACTION') | default('install', true) }}" # Configure Db2 instance diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml b/ibm/mas_devops/roles/db2/tasks/backup-database/backup-database.yml similarity index 100% rename from ibm/mas_devops/roles/db2/tasks/backup_database/backup-database.yml rename to ibm/mas_devops/roles/db2/tasks/backup-database/backup-database.yml diff --git a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml b/ibm/mas_devops/roles/db2/tasks/backup-database/main.yml similarity index 95% rename from ibm/mas_devops/roles/db2/tasks/backup_database/main.yml rename to ibm/mas_devops/roles/db2/tasks/backup-database/main.yml index dfd3f70d32..d425340330 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup_database/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup-database/main.yml @@ -31,4 +31,4 @@ # Backup Db2 database Data using Db2Backup CR # ------------------------------------------------------------------------- - name: "Start Database backup process." - include_tasks: "tasks/backup_database/backup-database.yml" + include_tasks: "tasks/backup-database/backup-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/backup/main.yml b/ibm/mas_devops/roles/db2/tasks/backup/main.yml index 8b06ad07b2..fe7c8897c8 100644 --- a/ibm/mas_devops/roles/db2/tasks/backup/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/backup/main.yml @@ -36,4 +36,4 @@ # Backup Db2 database Data using Db2Backup CR # ------------------------------------------------------------------------- - name: "Start Database backup process." - include_tasks: "{{ role_path }}/tasks/backup_database/backup-database.yml" + include_tasks: "{{ role_path }}/tasks/backup-database/backup-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/main.yml b/ibm/mas_devops/roles/db2/tasks/main.yml index 5b025b1c13..20b54999ce 100644 --- a/ibm/mas_devops/roles/db2/tasks/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/main.yml @@ -3,10 +3,10 @@ include_tasks: "tasks/{{ db2_action }}/main.yml" when: - db2_action != "none" - - db2_action in ["install", "upgrade", "backup", "restore_database", "backup_database", "restore"] + - db2_action in ["install", "upgrade", "backup", "restore-database", "backup-database", "restore"] - name: "Fail if db2_action is invalid" fail: - msg: "db2_action must be one of ['install', 'upgrade', 'backup', 'restore_database', 'backup_database', 'restore']" + msg: "db2_action must be one of ['install', 'upgrade', 'backup', 'restore-database', 'backup-database', 'restore']" when: - - db2_action not in ["install", "upgrade", "backup", "restore_database", "backup_database", "restore"] + - db2_action not in ["install", "upgrade", "backup", "restore-database", "backup-database", "restore"] diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/main.yml b/ibm/mas_devops/roles/db2/tasks/restore-database/main.yml similarity index 61% rename from ibm/mas_devops/roles/db2/tasks/restore_database/main.yml rename to ibm/mas_devops/roles/db2/tasks/restore-database/main.yml index 1862e5e2e0..41bca3b031 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore-database/main.yml @@ -1,4 +1,4 @@ --- # Restore Db2 Universal operator database - name: "Start Database restore process." - include_tasks: "{{ role_path }}/tasks/restore_database/restore-database.yml" + include_tasks: "{{ role_path }}/tasks/restore-database/restore-database.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml b/ibm/mas_devops/roles/db2/tasks/restore-database/restore-database.yml similarity index 95% rename from ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml rename to ibm/mas_devops/roles/db2/tasks/restore-database/restore-database.yml index 63282f5076..7e650ee0f3 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-database.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore-database/restore-database.yml @@ -4,7 +4,7 @@ - name: Verify DB2 restore variables ibm.mas_devops.verify_backup_restore_vars: component: "db2" - action: "restore_database" + action: "restore-database" db2_backup_version: "{{ db2_backup_version }}" mas_backup_dir: "{{ mas_backup_dir }}" db2_instance_name: "{{ db2_instance_name }}" @@ -39,4 +39,4 @@ - "================================================================================" - name: "Run restore-db-from-disk.yml when backup vendor is disk" - include_tasks: "{{ role_path }}/tasks/restore_database/restore-db-from-{{ backup_vendor }}.yml" + include_tasks: "{{ role_path }}/tasks/restore-database/restore-db-from-{{ backup_vendor }}.yml" diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml b/ibm/mas_devops/roles/db2/tasks/restore-database/restore-db-from-disk.yml similarity index 100% rename from ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-disk.yml rename to ibm/mas_devops/roles/db2/tasks/restore-database/restore-db-from-disk.yml diff --git a/ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml b/ibm/mas_devops/roles/db2/tasks/restore-database/restore-db-from-s3.yml similarity index 100% rename from ibm/mas_devops/roles/db2/tasks/restore_database/restore-db-from-s3.yml rename to ibm/mas_devops/roles/db2/tasks/restore-database/restore-db-from-s3.yml diff --git a/ibm/mas_devops/roles/db2/tasks/restore/main.yml b/ibm/mas_devops/roles/db2/tasks/restore/main.yml index 4389c5809a..07a68076fe 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/main.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/main.yml @@ -4,4 +4,4 @@ include_tasks: "{{ role_path }}/tasks/restore/restore-instance.yml" # Restore Db2 database -- include_tasks: tasks/restore_database/main.yml +- include_tasks: tasks/restore-database/main.yml diff --git a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml index bd84e595e0..edbafb8f61 100644 --- a/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml +++ b/ibm/mas_devops/roles/db2/tasks/restore/restore-instance.yml @@ -4,7 +4,7 @@ - name: Verify DB2 restore variables ibm.mas_devops.verify_backup_restore_vars: component: "db2" - action: "restore_instance" + action: "restore-instance" db2_backup_version: "{{ db2_backup_version }}" mas_backup_dir: "{{ mas_backup_dir }}" mas_application_id: "{{ mas_application_id }}" diff --git a/ibm/mas_devops/roles/mongodb/README.md b/ibm/mas_devops/roles/mongodb/README.md index 219590889a..8c7f1fd1ad 100644 --- a/ibm/mas_devops/roles/mongodb/README.md +++ b/ibm/mas_devops/roles/mongodb/README.md @@ -128,7 +128,7 @@ Specifies which operation to perform on the MongoDB instance. - Use `create-mongo-service-credentials` to generate service credentials (ibm only) **Valid values** (provider-specific): -- **community**: `install`, `uninstall`, `backup`, `backup_database`, `restore`, `restore_database` +- **community**: `install`, `uninstall`, `backup`, `backup-database`, `restore`, `restore-database` - **aws**: `install`, `uninstall`, `docdb_secret_rotate`, `destroy-data` - **ibm**: `install`, `uninstall`, `backup`, `restore`, `create-mongo-service-credentials` - **atlas**: `install`, `uninstall`, `restore` @@ -564,18 +564,18 @@ Role Variables - Backup and Restore (CE Operator) ### mongodb_action For backup and restore operations, set `mongodb_action` to one of the following: - `backup`: Create a backup of MongoDB databases and instance resources -- `backup_database`: Create a backup of MongoDB databases only +- `backup-database`: Create a backup of MongoDB databases only - `restore`: Restore both MongoDB instance resources and databases from a backup (full restore) -- `restore_database`: Restore only MongoDB databases from a backup to an existing instance (database-only restore) +- `restore-database`: Restore only MongoDB databases from a backup to an existing instance (database-only restore) - Environment Variable: `MONGODB_ACTION` - Default Value: `install` **Action Details:** - **backup**: Creates a complete backup including database data and Kubernetes resources (secrets, certificates, CRs) -- **backup_database**: Creates a complete backup including database data only +- **backup-database**: Creates a complete backup including database data only - **restore**: Performs a full restore by recreating the MongoDB instance from backup resources and then restoring database data -- **restore_database**: Restores only the database data to an already running MongoDB instance without touching instance resources +- **restore-database**: Restores only the database data to an already running MongoDB instance without touching instance resources ### mas_backup_dir **Required for backup/restore operations**. Local directory path where backup files will be stored or read from. @@ -621,9 +621,9 @@ This section provides comprehensive information about MongoDB backup and restore | Action | Purpose | Instance Resources | Database Data | Prerequisites | Use Case | |--------|---------|-------------------|---------------|---------------|----------| | `backup` | Create backup | Yes | Yes | Running MongoDB instance | Regular backups, disaster recovery preparation | -| `backup_database` | Database-only backup | No | Yes | Running MongoDB instance | Regular backups, disaster recovery preparation | +| `backup-database` | Database-only backup | No | Yes | Running MongoDB instance | Regular backups, disaster recovery preparation | | `restore` | Full restore | Yes (recreates instance) | Yes | Backup with instance resources | Disaster recovery, cluster migration, complete restoration | -| `restore_database` | Database-only restore | No (preserves existing) | Yes | Running MongoDB instance with matching version | Data recovery, selective restore, testing | +| `restore-database` | Database-only restore | No (preserves existing) | Yes | Running MongoDB instance with matching version | Data recovery, selective restore, testing | ### Backup Process @@ -689,7 +689,7 @@ Performs a complete restoration of both the MongoDB instance and its databases: - `mongodb_backup_version`: Timestamp of the backup to restore - `override_storageclass`: Set to `true` to override storage class during instance restore -#### 2. Database-Only Restore (`restore_database` action) +#### 2. Database-Only Restore (`restore-database` action) Restores only the database data to an existing MongoDB instance: **Steps:** @@ -718,7 +718,7 @@ Restores only the database data to an existing MongoDB instance: - Target MongoDB version must match the backup version exactly - Version upgrades should be performed separately, not during restore - The restore process validates version compatibility before proceeding -- For `restore_database` action, the existing MongoDB instance must be running the same version as the backup +- For `restore-database` action, the existing MongoDB instance must be running the same version as the backup **Storage Requirements:** - Ensure sufficient storage in the backup directory @@ -742,7 +742,7 @@ Restores only the database data to an existing MongoDB instance: **Restore Action Differences:** - **`restore` action**: Recreates the entire MongoDB instance from scratch, including all Kubernetes resources and restores database -- **`restore_database` action**: Only restores database data to an existing instance, preserving current configuration +- **`restore-database` action**: Only restores database data to an existing instance, preserving current configuration ### Backup and Restore Best Practices @@ -823,7 +823,7 @@ Create a backup of database data. vars: mas_instance_id: masinst1 mas_backup_dir: /tmp/masbr - mongodb_action: backup_database + mongodb_action: backup-database roles: - ibm.mas_devops.mongodb ``` @@ -842,7 +842,7 @@ Restore MongoDB databases from a backup to an existing MongoDB instance without - hosts: localhost any_errors_fatal: true vars: - mongodb_action: restore_database + mongodb_action: restore-database mas_instance_id: masinst1 mongodb_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr @@ -894,7 +894,7 @@ Perform a complete restoration of MongoDB instance including all Kubernetes reso 7. Waits for MongoDB instance to be fully operational 8. Restores database data using `mongorestore` -**Note**: This is a complete restoration that recreates the MongoDB instance from scratch. Use `restore_database` action if you only need to restore data to an existing instance. +**Note**: This is a complete restoration that recreates the MongoDB instance from scratch. Use `restore-database` action if you only need to restore data to an existing instance. ### Install from Backup (CE Operator) Deploy a new MongoDB instance using configuration from a backup and restore data from a backup. This is useful for disaster recovery or migrating MongoDB to a new cluster. diff --git a/ibm/mas_devops/roles/mongodb/defaults/main.yml b/ibm/mas_devops/roles/mongodb/defaults/main.yml index c358071c6f..020746e3c8 100644 --- a/ibm/mas_devops/roles/mongodb/defaults/main.yml +++ b/ibm/mas_devops/roles/mongodb/defaults/main.yml @@ -6,7 +6,7 @@ mongodb_provider: "{{ lookup('env','MONGODB_PROVIDER') | default('community', Tr mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}" mas_config_dir: "{{ lookup('env', 'MAS_CONFIG_DIR') }}" -# Supported actions: install, uninstall, backup, backup_database, restore, restore_database +# Supported actions: install, uninstall, backup, backup-database, restore, restore-database mongodb_action: "{{ lookup('env', 'MONGODB_ACTION') | default('install', True) }}" # Backup mongodb databases for specified MAS application. Backup all mongodb databases if not specify this value. diff --git a/ibm/mas_devops/roles/mongodb/tasks/main.yml b/ibm/mas_devops/roles/mongodb/tasks/main.yml index df9d250833..8e2daa9236 100755 --- a/ibm/mas_devops/roles/mongodb/tasks/main.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/main.yml @@ -12,8 +12,8 @@ assert: that: - mongodb_action is defined - - mongodb_action in ["install", "uninstall", "backup", "backup_database", "restore_database", "restore"] - fail_msg: "mongodb_action property is required and must be one of 'install', 'uninstall', 'backup', 'backup_database','restore_database', 'restore'" + - mongodb_action in ["install", "uninstall", "backup", "backup-database", "restore-database", "restore"] + fail_msg: "mongodb_action property is required and must be one of 'install', 'uninstall', 'backup', 'backup-database','restore-database', 'restore'" # 2. Run the install / uninstall for specified provider # ----------------------------------------------------------------------------- diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-database.yml similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/backup_database.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/backup-database.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_database.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database.yml similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_database.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-database.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-instance.yml similarity index 100% rename from ibm/mas_devops/roles/mongodb/tasks/providers/community/restore_instance.yml rename to ibm/mas_devops/roles/mongodb/tasks/providers/community/restore-instance.yml diff --git a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml index df8ca2aebb..7f83ac8df1 100644 --- a/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml +++ b/ibm/mas_devops/roles/mongodb/tasks/providers/community/restore.yml @@ -1,8 +1,8 @@ --- # Task to restore instance # ----------------------------------------------------------------------------- -- include_tasks: "tasks/providers/{{ mongodb_provider }}/restore_instance.yml" +- include_tasks: "tasks/providers/{{ mongodb_provider }}/restore-instance.yml" # Task to restore database # ----------------------------------------------------------------------------- -- include_tasks: "tasks/providers/{{ mongodb_provider }}/restore_database.yml" +- include_tasks: "tasks/providers/{{ mongodb_provider }}/restore-database.yml" From fe790df0b7c6df45e2e9f63642c77f7295d26afa Mon Sep 17 00:00:00 2001 From: Sanjay Prabhakar Date: Fri, 13 Mar 2026 10:33:19 +0000 Subject: [PATCH 61/61] updates to b/r docs (#2159) Co-authored-by: Sanjay Prabhakar --- docs/playbooks/backup-restore.md | 528 ++++----------------- docs/playbooks/legacy-backup-restore.md | 604 ++++++++++++++++++++++++ ibm/mas_devops/playbooks/br_db2.yml | 7 +- ibm/mas_devops/roles/db2/README.md | 110 ++++- mkdocs.yml | 2 + 5 files changed, 790 insertions(+), 461 deletions(-) create mode 100644 docs/playbooks/legacy-backup-restore.md diff --git a/docs/playbooks/backup-restore.md b/docs/playbooks/backup-restore.md index dbecbff6a6..988eb4995e 100644 --- a/docs/playbooks/backup-restore.md +++ b/docs/playbooks/backup-restore.md @@ -10,7 +10,7 @@ Backup and Restore Overview ------------------------------------------------------------------------------- -MAS Devops Collection includes playbooks for backing up and restoring MAS components and their dependencies. The backup and restore operations are designed to protect your MAS installation and enable disaster recovery, cluster migration, and testing scenarios. +MAS Devops Collection includes playbooks/guidance for backing up and restoring MAS components and their dependencies. The backup and restore operations are designed to protect your MAS installation and enable disaster recovery, cluster migration, and testing scenarios. **Supported Components:** - [MongoDB Community](#mongodb-community-backup-and-restore) @@ -45,7 +45,7 @@ This playbook performs backup and restore operations for MongoDB Community Editi The playbook executes the following operations: ### Backup Operation -1. [Backup MongoDB Instance Resources](../roles/mongodb.md) - Kubernetes resources (StatefulSet, Services, ConfigMaps, Secrets) +1. [Backup MongoDB Instance Resources](../roles/mongodb.md) - Kubernetes resources (Deployment, Custom resources, ConfigMaps, Secrets) 2. [Backup MongoDB Database Data](../roles/mongodb.md) - Database data using mongodump ### Restore Operation @@ -226,261 +226,6 @@ ansible-playbook ibm.mas_devops.br_mongodb For detailed information about MongoDB backup and restore operations, refer to the role documentation: - [MongoDB Backup/Restore](../roles/mongodb.md) - -MAS Core Backup and Restore -=============================================================================== - -## Overview -This playbook performs comprehensive backup and restore operations for IBM Maximo Application Suite Core and its dependencies. The playbook can be run against any OCP cluster regardless of its type; whether it's running in IBM Cloud, Azure, AWS, or your local datacenter. - -**Important**: Backup can only be restored to an instance with the same MAS instance ID. - -## Playbook Content - -The playbook executes the following roles in sequence: - -### Backup Operation -1. [Backup IBM Operator Catalogs](../roles/ibm_catalogs.md) (~1 minute) -2. [Backup Certificate Manager](../roles/cert_manager.md) (~1 minute) -3. [Backup MongoDB Community Edition](../roles/mongodb.md) (~5-30 minutes depending on database size) -4. [Backup Suite License Service](../roles/sls.md) (~2 minutes, optional) -5. [Backup MAS Core](../roles/suite_backup.md) (~5 minutes) - -### Restore Operation -1. [Restore IBM Operator Catalogs](../roles/ibm_catalogs.md) (~2 minutes) -2. [Restore Certificate Manager](../roles/cert_manager.md) (~5 minutes) -3. [Install Grafana](../roles/grafana.md) (~10 minutes, optional) -4. [Restore MongoDB Community Edition](../roles/mongodb.md) (~10-60 minutes depending on database size) -5. [Restore Suite License Service](../roles/sls.md) (~10 minutes, optional) -6. [Install Data Reporter Operator](../roles/dro.md) (~10 minutes, optional) -7. [Restore MAS Core](../roles/suite_restore.md) (~30 minutes) - -All timings are estimates. See the individual role documentation for more information and full details of all configuration options. - -## Required Environment Variables - -### Common Variables (Backup and Restore) -- `MAS_INSTANCE_ID` - The instance ID of the MAS installation -- `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) -- `BR_ACTION` - Set to `backup` or `restore` - -### Backup-Specific Variables -- `BACKUP_VERSION` - (Optional) Custom version identifier for the backup. If not provided, defaults to timestamp format `YYYYMMDD-HHMMSS` - -### Restore-Specific Variables -- `BACKUP_VERSION_TO_RESTORE` - (Required) The backup version identifier to restore -- `IBM_ENTITLEMENT_KEY` - Your IBM Entitlement key (required if `INCLUDE_DRO=true`) -- `DRO_CONTACT_EMAIL` - Primary contact email (required if `INCLUDE_DRO=true`) -- `DRO_CONTACT_FIRSTNAME` - Primary contact first name (required if `INCLUDE_DRO=true`) -- `DRO_CONTACT_LASTNAME` - Primary contact last name (required if `INCLUDE_DRO=true`) - -## Optional Environment Variables - -### SLS and DRO Configuration -- `INCLUDE_SLS` - Set to `false` to skip SLS backup/restore (default: `true`) - - Use `false` if SLS is configured externally to the cluster -- `INCLUDE_DRO` - Set to `false` to skip DRO installation on restore (default: `true`) - - Use `false` if DRO is configured externally to the cluster - -### GRAFANA Configuration -- `INCLUDE_GRAFANA` - Set to ``false`` to skip Grafana install (default: `true`) - -### Restore to Different Cluster -When restoring to a different cluster with a different domain: -- `MAS_DOMAIN_ON_RESTORE` - Override the MAS domain for the restored instance -- `SLS_URL_ON_RESTORE` - Override the SLS URL in the SLS Config (when `INCLUDE_SLS=false`) -- `DRO_URL_ON_RESTORE` - Override the DRO URL in the DRO Config (when `INCLUDE_DRO=false`) - -### MongoDB Configuration -- `MONGODB_INSTANCE_NAME` - MongoDB instance name (default: `mas-mongo-ce`) - -### Configuration Directory -- `MAS_CONFIG_DIR` - Directory for SLS/DRO configuration files (required when `INCLUDE_SLS=true` or `INCLUDE_DRO=true` on restore) - -## Usage Examples - -### Backup MAS Core -Create a complete backup of MAS Core and all dependencies: - -```bash -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export BR_ACTION=backup - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_core -``` - -### Backup with Custom Version -Create a backup with a custom version identifier: - -```bash -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export BR_ACTION=backup -export BACKUP_VERSION=pre-upgrade-backup - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_core -``` - -### Backup Without SLS -Create a backup excluding SLS (when using external SLS): - -```bash -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export BR_ACTION=backup -export INCLUDE_SLS=false - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_core -``` - -### Restore MAS Core -Restore MAS Core from a backup to the same cluster: - -```bash -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=20260122-131500 - -export IBM_ENTITLEMENT_KEY=xxx -export DRO_CONTACT_EMAIL=user@example.com -export DRO_CONTACT_FIRSTNAME=John -export DRO_CONTACT_LASTNAME=Doe -export MAS_CONFIG_DIR=~/masconfig - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_core -``` - -### Restore MAS Core without Grafana -Restore MAS Core from a backup without grafana: - -```bash -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=20260122-131500 - -export INCLUDE_GRAFANA=false - -export IBM_ENTITLEMENT_KEY=xxx -export DRO_CONTACT_EMAIL=user@example.com -export DRO_CONTACT_FIRSTNAME=John -export DRO_CONTACT_LASTNAME=Doe -export MAS_CONFIG_DIR=~/masconfig - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_core -``` - - -### Restore to Different Cluster -Restore MAS Core to a different cluster with a different domain: - -```bash -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=20260122-131500 - -export IBM_ENTITLEMENT_KEY=xxx -export DRO_CONTACT_EMAIL=user@example.com -export DRO_CONTACT_FIRSTNAME=John -export DRO_CONTACT_LASTNAME=Doe -export MAS_CONFIG_DIR=~/masconfig - -# Override domain for new cluster -export MAS_DOMAIN_ON_RESTORE=mas.newcluster.example.com - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_core -``` - -### Restore with External SLS and DRO -Restore MAS Core using external SLS and DRO services: - -```bash -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export BR_ACTION=restore -export BACKUP_VERSION_TO_RESTORE=20260122-131500 - -# Skip SLS and DRO installation -export INCLUDE_SLS=false -export INCLUDE_DRO=false - -# Provide external service URLs -export SLS_URL_ON_RESTORE=https://sls.external.example.com -export DRO_URL_ON_RESTORE=https://dro.external.example.com - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_core -``` - -## Important Considerations - -### Prerequisites for Restore -- Target cluster must have sufficient resources (CPU, memory, storage) -- Certificate Manager must be installed (handled by playbook) -- Target cluster must use the same MAS instance ID as the backup -- Backup files must be accessible from the restore environment - -### Backup Best Practices -1. **Regular Schedule**: Perform backups regularly, especially before: - - MAS upgrades - - Configuration changes - - Application installations - - Cluster maintenance -2. **Test Restores**: Periodically test restore procedures in non-production environments -3. **Secure Storage**: Store backups in a secure location separate from the cluster -4. **Retention Policy**: Implement and document backup retention policies -5. **Verify Integrity**: Verify backup integrity after completion - -### Restore Best Practices -1. **Pre-Restore Validation**: - - Verify backup archive exists and is complete - - Confirm target cluster has sufficient resources - - Verify MAS instance ID matches the backup -2. **Dependency Coordination**: - - Ensure all external services (SLS, DRO, databases) are accessible - - Verify network connectivity to external services -3. **Post-Restore Verification**: - - Verify Suite status is Ready - - Verify all Workspaces are Ready - - Test application connectivity - - Test user authentication - -### Storage Requirements -- Ensure sufficient storage in the backup directory -- Plan for at least 2x the database size for MongoDB backups -- Monitor disk space during backup operations -- Backup directory structure: `{mas_backup_dir}/backup-{version}-{component}/` - -### Security Considerations -- Backup files contain sensitive data including credentials and certificates -- Secure backup directory with appropriate permissions (chmod 700 recommended) -- Consider encrypting backups for long-term storage -- Restrict access to backup files to authorized personnel only -- Ensure secure transfer of backup files to restore environment - -!!! tip - If you do not want to set up all the dependencies on your local system, you can run the playbook inside our docker image: `docker run -ti --pull always quay.io/ibmmas/cli` - -## Additional Resources - -For detailed information about individual backup and restore operations, refer to the role documentation: -- [IBM Operator Catalogs Backup/Restore](../roles/ibm_catalogs.md) -- [Certificate Manager Backup/Restore](../roles/cert_manager.md) -- [MongoDB Backup/Restore](../roles/mongodb.md) -- [SLS Backup/Restore](../roles/sls.md) -- [MAS Core Backup](../roles/suite_backup.md) -- [MAS Core Restore](../roles/suite_restore.md) -- [Db2 Backup/Restore](../roles/db2.md) - Db2 Backup and Restore =============================================================================== @@ -522,11 +267,9 @@ This playbook performs backup and restore operations for IBM Db2 Universal Opera ### Storage Class Override (Restore) - `OVERRIDE_STORAGECLASS` - Set to `true` to override storage class names from backup (default: `false`) -- `DB2_META_STORAGE_CLASS` - Storage class for metadata -- `DB2_DATA_STORAGE_CLASS` - Storage class for data -- `DB2_BACKUP_STORAGE_CLASS` - Storage class for backups -- `DB2_LOGS_STORAGE_CLASS` - Storage class for logs -- `DB2_TEMP_STORAGE_CLASS` - Storage class for temporary files +- `CUSTOM_STORAGE_CLASS_RWO` - Storage class for Read-write-only +- `CUSTOM_STORAGE_CLASS_RWX` - Storage class for Read-write-many + ## Usage Examples @@ -617,205 +360,118 @@ ansible-playbook ibm.mas_devops.br_db2 - Sufficient storage capacity for database restoration - Same or compatible Db2 version as the backup -Manage Application Backup and Restore + +MAS Core Backup and Restore =============================================================================== ## Overview -This playbook performs backup and restore operations for IBM Maximo Manage application. +This guide shows backup and restore operations for IBM Maximo Application Suite Core and its dependencies. This guidance can be used to build your own playbooks to run against any OCP cluster regardless of its type; whether it's running in IBM Cloud, Azure, AWS, or your local datacenter. -**Important**: -- Backup can only be restored to an instance with the same MAS instance ID -- **DB2 backup and restore is NOT automatically included** in the Manage backup/restore playbook -- You **MUST** run the [DB2 backup and restore playbook](#db2-backup-and-restore) as a prerequisite step before running Manage backup or restore operations +**Important**: Backup can only be restored to an instance with the same MAS instance ID. -## Playbook Content +## Guidance Content -The playbook executes the following operations: +Sequence of roles: ### Backup Operation -1. **[Backup Db2 Database](../roles/db2.md) - PREREQUISITE STEP** (run `br_db2.yml` playbook separately first) -2. [Backup Manage Application](../roles/suite_app_backup.md) +1. [Backup IBM Operator Catalogs](../roles/ibm_catalogs.md) (~1 minute) +2. [Backup Certificate Manager](../roles/cert_manager.md) (~1 minute) +3. [Backup MongoDB Community Edition](../roles/mongodb.md) (~5-30 minutes depending on database size) +4. [Backup Suite License Service](../roles/sls.md) (~2 minutes, optional) +5. [Backup MAS Core](../roles/suite_backup.md) (~5 minutes) ### Restore Operation -1. **[Restore Db2 Database](../roles/db2.md) - PREREQUISITE STEP** (run `br_db2.yml` playbook separately first) -2. [Restore Manage Application](../roles/suite_app_restore.md) - -## Required Environment Variables - -### Common Variables (Backup and Restore) -- `MAS_INSTANCE_ID` - The instance ID of the MAS installation -- `MAS_WORKSPACE_ID` - The workspace ID for Manage -- `MAS_BACKUP_DIR` - Directory where backup files will be stored/retrieved (e.g., `/tmp/mas_backups`) -- `MAS_APP_ACTION` - Set to `backup` or `restore` (default: `backup`) - -### Backup-Specific Variables -- `MAS_APP_BACKUP_VERSION` - (Optional) Custom version identifier for the backup. If not provided, defaults to timestamp format `YYYYMMDD-HHMMSS` - -### Restore-Specific Variables -- `MAS_APP_BACKUP_VERSION_TO_RESTORE` - (Required) The backup version identifier to restore - -!!! warning "DB2 Backup/Restore Prerequisites" - Before running Manage backup or restore, you **MUST** first run the DB2 backup or restore playbook separately. See the [DB2 Backup and Restore](#db2-backup-and-restore) section for all required DB2 environment variables including: - - - `DB2_INSTANCE_NAME` - - `DB2_ACTION` (set to `backup` or `restore`) - - `DB2_BACKUP_TYPE` - - `BACKUP_VENDOR` - - `DB2_BACKUP_VERSION` (for restore operations) - - S3 variables (if using S3 storage) - -## Optional Environment Variables - -### Storage Class Override (Restore) -- `OVERRIDE_STORAGECLASS` - Set to `true` to override storage class names from backup (default: `false`) -- `MAS_APP_CUSTOM_STORAGE_CLASS_RWO` - Custom RWO storage class for Manage -- `MAS_APP_CUSTOM_STORAGE_CLASS_RWX` - Custom RWX storage class for Manage - -### Db2 Configuration -- `DB2_NAMESPACE` - Namespace where Db2 is installed (default: `db2u`) - -## Usage Examples - -### Backup Manage Application -Create a complete backup of Manage application. **Note:** You must backup DB2 first as a separate step. - -```bash -# STEP 1: Backup DB2 database (PREREQUISITE) -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export DB2_INSTANCE_NAME=db2w-shared -export DB2_ACTION=backup -export DB2_BACKUP_TYPE=online -export BACKUP_VENDOR=disk - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_db2 - -# STEP 2: Backup Manage application -export MAS_INSTANCE_ID=inst1 -export MAS_WORKSPACE_ID=masdev -export MAS_BACKUP_DIR=/backup/mas -export MAS_APP_ACTION=backup - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_manage -``` - -### Backup with Custom Version -Create a backup with a custom version identifier: - -```bash -# STEP 1: Backup DB2 database (PREREQUISITE) -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export DB2_INSTANCE_NAME=db2w-shared -export DB2_ACTION=backup -export DB2_BACKUP_TYPE=online -export BACKUP_VENDOR=disk - -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_db2 +1. [Restore IBM Operator Catalogs](../roles/ibm_catalogs.md) (~2 minutes) +2. [Restore Certificate Manager](../roles/cert_manager.md) (~5 minutes) +3. [Install Grafana](../roles/grafana.md) (~10 minutes, optional) +4. [Restore MongoDB Community Edition](../roles/mongodb.md) (~10-60 minutes depending on database size) +5. [Restore Suite License Service](../roles/sls.md) (~10 minutes, optional) +6. [Install Data Reporter Operator](../roles/dro.md) (~10 minutes, optional) +7. [Restore MAS Core](../roles/suite_restore.md) (~30 minutes) -# STEP 2: Backup Manage application with custom version -export MAS_INSTANCE_ID=inst1 -export MAS_WORKSPACE_ID=masdev -export MAS_BACKUP_DIR=/backup/mas -export MAS_APP_ACTION=backup -export MAS_APP_BACKUP_VERSION=pre-upgrade-manage +All timings are estimates. See the individual role documentation for more information and full details of all configuration options. -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_manage -``` +## Important Considerations -### Backup to S3 Storage -Create a backup storing DB2 and Manage data to S3: +### Prerequisites for Restore +- Target cluster must have sufficient resources (CPU, memory, storage) +- Certificate Manager must be installed (handled by playbook) +- Target cluster must use the same MAS instance ID as the backup +- Backup files must be accessible from the restore environment -```bash -# STEP 1: Backup DB2 database to S3 (PREREQUISITE) -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export DB2_INSTANCE_NAME=db2w-shared -export DB2_ACTION=backup -export DB2_BACKUP_TYPE=online -export BACKUP_VENDOR=s3 -export BACKUP_S3_ENDPOINT=https://s3.us-east.cloud-object-storage.appdomain.cloud -export BACKUP_S3_BUCKET=mas-manage-backups -export BACKUP_S3_ACCESS_KEY=your-access-key -export BACKUP_S3_SECRET_KEY=your-secret-key +### Backup Best Practices +1. **Regular Schedule**: Perform backups regularly, especially before: + - MAS upgrades + - Configuration changes + - Application installations + - Cluster maintenance +2. **Test Restores**: Periodically test restore procedures in non-production environments +3. **Secure Storage**: Store backups in a secure location separate from the cluster +4. **Retention Policy**: Implement and document backup retention policies +5. **Verify Integrity**: Verify backup integrity after completion -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_db2 +### Restore Best Practices +1. **Pre-Restore Validation**: + - Verify backup archive exists and is complete + - Confirm target cluster has sufficient resources + - Verify MAS instance ID matches the backup +2. **Dependency Coordination**: + - Ensure all external services (SLS, DRO, databases) are accessible + - Verify network connectivity to external services +3. **Post-Restore Verification**: + - Verify Suite status is Ready + - Verify all Workspaces are Ready + - Test application connectivity + - Test user authentication -# STEP 2: Backup Manage application -export MAS_INSTANCE_ID=inst1 -export MAS_WORKSPACE_ID=masdev -export MAS_BACKUP_DIR=/backup/mas -export MAS_APP_ACTION=backup +### Storage Requirements +- Ensure sufficient storage in the backup directory +- Plan for at least 2x the database size for MongoDB backups +- Monitor disk space during backup operations +- Backup directory structure: `{mas_backup_dir}/backup-{version}-{component}/` -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_manage -``` +### Security Considerations +- Backup files contain sensitive data including credentials and certificates +- Secure backup directory with appropriate permissions (chmod 700 recommended) +- Consider encrypting backups for long-term storage +- Restrict access to backup files to authorized personnel only +- Ensure secure transfer of backup files to restore environment -### Restore Manage Application -Restore Manage application. **Note:** You must restore DB2 first as a separate step. +!!! tip + If you do not want to set up all the dependencies on your local system, you can run the playbook inside our docker image: `docker run -ti --pull always quay.io/ibmmas/cli` -```bash -# STEP 1: Restore DB2 database (PREREQUISITE) -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export DB2_INSTANCE_NAME=db2w-shared -export DB2_ACTION=restore -export DB2_BACKUP_VERSION=20260122-131500 -export BACKUP_VENDOR=disk +## Additional Resources -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_db2 +For detailed information about individual backup and restore operations, refer to the role documentation: +- [IBM Operator Catalogs Backup/Restore](../roles/ibm_catalogs.md) +- [Certificate Manager Backup/Restore](../roles/cert_manager.md) +- [MongoDB Backup/Restore](../roles/mongodb.md) +- [SLS Backup/Restore](../roles/sls.md) +- [MAS Core Backup](../roles/suite_backup.md) +- [MAS Core Restore](../roles/suite_restore.md) +- [Db2 Backup/Restore](../roles/db2.md) -# STEP 2: Restore Manage application -export MAS_INSTANCE_ID=inst1 -export MAS_WORKSPACE_ID=masdev -export MAS_BACKUP_DIR=/backup/mas -export MAS_APP_ACTION=restore -export MAS_APP_BACKUP_VERSION_TO_RESTORE=20260122-131500 +Manage Application Backup and Restore +=============================================================================== -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_manage -``` +## Overview +This guide shows backup and restore operations for IBM Maximo Manage application. -### Restore with Storage Class Override -Restore Manage to a different cluster with different storage classes: +**Important**: +- Backup can only be restored to an instance with the same MAS instance ID +- You **MUST** run the [DB2 backup and restore playbook](#db2-backup-and-restore) as a prerequisite step before running Manage backup or restore operations -```bash -# STEP 1: Restore DB2 database with storage override (PREREQUISITE) -export MAS_INSTANCE_ID=inst1 -export MAS_BACKUP_DIR=/backup/mas -export DB2_INSTANCE_NAME=db2w-shared -export DB2_ACTION=restore -export DB2_BACKUP_VERSION=20260122-131500 -export BACKUP_VENDOR=disk -export OVERRIDE_STORAGECLASS=true -export DB2_META_STORAGE_CLASS=ocs-storagecluster-ceph-rbd -export DB2_DATA_STORAGE_CLASS=ocs-storagecluster-ceph-rbd -export DB2_BACKUP_STORAGE_CLASS=ocs-storagecluster-ceph-rbd -export DB2_LOGS_STORAGE_CLASS=ocs-storagecluster-ceph-rbd -export DB2_TEMP_STORAGE_CLASS=ocs-storagecluster-ceph-rbd +## Content -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_db2 +Executes the following operations: -# STEP 2: Restore Manage application with storage override -export MAS_INSTANCE_ID=inst1 -export MAS_WORKSPACE_ID=masdev -export MAS_BACKUP_DIR=/backup/mas -export MAS_APP_ACTION=restore -export MAS_APP_BACKUP_VERSION_TO_RESTORE=20260122-131500 -export OVERRIDE_STORAGECLASS=true -export MAS_APP_CUSTOM_STORAGE_CLASS_RWO=ocs-storagecluster-ceph-rbd -export MAS_APP_CUSTOM_STORAGE_CLASS_RWX=ocs-storagecluster-cephfs +### Backup Operation +1. **[Backup Db2 Database](../roles/db2.md) - PREREQUISITE STEP** (run `br_db2.yml` playbook separately first) +2. [Backup Manage Application](../roles/suite_app_backup.md) -oc login --token=xxxx --server=https://myocpserver -ansible-playbook ibm.mas_devops.br_manage -``` +### Restore Operation +1. **[Restore Db2 Database](../roles/db2.md) - PREREQUISITE STEP** (run `br_db2.yml` playbook separately first) +2. [Restore Manage Application](../roles/suite_app_restore.md) ## Important Considerations @@ -828,7 +484,7 @@ ansible-playbook ibm.mas_devops.br_manage ### Backup Best Practices 1. **Two-Step Process**: Always backup DB2 first, then Manage application - - Run `br_db2.yml` playbook before `br_manage.yml` + - Run `br_db2.yml` playbook before running Manage application backup - DB2 backup is NOT automatically included in Manage backup 2. **Version Alignment**: Use consistent version identifiers for both DB2 and Manage backups for easier tracking 3. **Regular Schedule**: Perform backups regularly, especially before: @@ -844,7 +500,7 @@ ansible-playbook ibm.mas_devops.br_manage - Confirm target cluster has sufficient resources - Verify MAS instance ID and workspace ID match the backup 2. **Restore Order**: **CRITICAL** - Always restore DB2 first, then Manage application - - Run `br_db2.yml` playbook before `br_manage.yml` + - Run `br_db2.yml` playbook before running Manage application restore - DB2 restore is NOT automatically included in Manage restore 3. **Post-Restore Verification**: - Verify DB2 instance is running and accessible diff --git a/docs/playbooks/legacy-backup-restore.md b/docs/playbooks/legacy-backup-restore.md new file mode 100644 index 0000000000..51ce4ed12d --- /dev/null +++ b/docs/playbooks/legacy-backup-restore.md @@ -0,0 +1,604 @@ +Legacy Backup and Restore +=============================================================================== + +!!! important + The Legacy backup and restore playbooks are removed in CLI v19.0 and later. + Use older CLI versions to continue using the legacy playbooks. + Please refer to the [backup and restore documentation](../backup-and-restore.md) for the latest information. + +Overview +------------------------------------------------------------------------------- +MAS Devops Collection includes playbooks for backing up and restoring of the following MAS components and their dependencies: + +- [MongoDB](#backuprestore-for-mongodb) +- [Db2](#backuprestore-for-db2) +- [MAS Core](#backuprestore-for-mas-core) +- [Manage](#backuprestore-for-manage) +- [IoT](#backuprestore-for-iot) +- [Monitor](#backuprestore-for-monitor) +- [Health](#backuprestore-for-health) +- [Optimizer](#backuprestore-for-optimizer) +- [Visual Inspection](#backuprestore-for-visual-inspection) + + +Creation of both **full** and **incremental** backups are supported. The backup and restore Ansible roles can also be used individually, allowing you to build your own customized backup and restore playbook covering exactly what you need. For example, you can only [backup/restore Manage attachments](../roles/suite_app_backup_restore.md). + +!!! important + The backup and restore playbooks in this collection are still work in progress, they are not suitable for production use at this time. You may track development progress using the [Backup & Restore](https://github.com/ibm-mas/ansible-devops/issues?q=label%3A%22Backup+%26+Restore%22+) label in the Github repository. + + Production-ready backup and restore options are detailed in the [Backup and restore](https://www.ibm.com/docs/en/mas-cd/continuous-delivery?topic=administering-backing-up-restoring-maximo-application-suite) topic in the product documentation. + +Configuration - Storage +------------------------------------------------------------------------------- +You can save the backup files to a folder on your local file system by setting the following environment variables: + +| Envrionment variable | Required (Default Value) | Description | +| ------------------------------------ | -------------------------- | ----------- | +| MASBR_STORAGE_LOCAL_FOLDER | **Yes** | The local path to save the backup files | +| MASBR_LOCAL_TEMP_FOLDER | No (`/tmp/masbr`) | Local folder for saving the temporary backup/restore data, the data in this folder will be deleted after the backup/restore job completed. | + + +Configuration - Backup +------------------------------------------------------------------------------- + +| Envrionment variable | Required (Default Value) | Description | +| ------------------------------------ | ------------------------ | ----------- | +| MASBR_ACTION | **Yes** | Whether to run the playbook to perform a `backup` or a `restore` | +| MASBR_BACKUP_TYPE | No (`full`) | Set `full` or `incr` to indicate the playbook to create a **full** backup or **incremental** backup. | +| MASBR_BACKUP_FROM_VERSION | No | Set the full backup version to use in the incremental backup, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`). | + +The playbooks are switched to backup mode by setting `MASBR_ACTION` to `backup`. + +### Full Backups +If you set environment variable `MASBR_BACKUP_TYPE=full` or do not specify a value for this variable, the playbook will take a full backup. + +### Incremental Backups +You can set environment variable `MASBR_BACKUP_TYPE=incr` to indicate the playbook to take an incremental backup. + +!!! important + Only supports creating incremental backup for MonogDB, Db2 and persistent volume data. The playbook will always create a full backup for other type of data regardless of whether this variable be set to `incr`. + +The environment variable `MASBR_BACKUP_FROM_VERSION` is only valid if `MASBR_BACKUP_TYPE=incr`. It indicates which backup version that the incremental backup to based on. If you do not set a value for this variable, the playbook will try to find the latest Completed Full backup from the specified storage location, and then take an incremental backup based on it. + +!!! important + The backup files you specified by `MASBR_BACKUP_FROM_VERSION` must be a Full backup. And the component name and data types in the specified Full backup file must be same as the current incremental backup job. + + +Configuration - Restore +------------------------------------------------------------------------------- + +| Envrionment variable | Required (Default Value) | Description | +| ------------------------------------ | ------------------------ | ----------- | +| MASBR_ACTION | **Yes** | Whether to run the playbook to perform a `backup` or a `restore` | +| MASBR_RESTORE_FROM_VERSION | **Yes** | Set the backup version to use in the restore, this will be in the format of a `YYYMMDDHHMMSS` timestamp (e.g. `20240621021316`) | +| MASBR_RESTORE_OVERWRITE | **Yes** | Set whether the restore should **overwrite** any existing data or if we should stop and **FAIL** if there is data detected in the directory. **WARNING:** This will overwrite all data when restoring! | + +The playbooks are switched to restore mode by setting `MASBR_ACTION` to `restore`. You **must** specify the `MASBR_RESTORE_FROM_VERSION` environment variable to indicate which version of the backup files to use. + +In the case of restoring from an incremental backup, the corresponding full backup will be restored first before continuing to restore the incremental backup. + + +Backup/Restore for MongoDB +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_mongodb` will invoke the role [mongodb](../roles/mongodb.md) to backup/restore the MongoDB databases. + +This playbook supports backing up and restoring databases for an in-cluster MongoDB CE instance. If you are using other MongoDB venders, such as IBM Cloud Databases for MongoDB, Amazon DocumentDB or MongoDB Altas Database, please refer to the corresponding vender's documentation for more information about their provided backup/restore service. + +### Environment Variables +- `MONGODB_NAMESPACE`: By default the backup and restore processes will use a namespace of `mongoce`, if you have customized the install of MongoDb CE you must set this environment variable to the appropriate namespace you wish to backup from/restore to. +- `MAS_INSTANCE_ID`: **Required**. This playbook supports backup/restore MongoDB databases that belong to a specific MAS instance, call the playbook multiple times with different values for `MAS_INSTANCE_ID` if you wish to back up multiple MAS instances that use the same MongoDB CE instance. +- `MAS_APP_ID`: **Optional**. By default, this playbook will backup all databases belonging to the specified MAS instance. You can backup the databases only belong to a specific MAS application by setting this environment variable to a supported MAS application id `core`, `manage`, `iot`, `monitor`, `health`, `optimizer` or `visualinspection`. + +### Examples +```bash +# Full backup all MongoDB data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_mongodb + +# Incremental backup all MongoDB data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_mongodb + +# Restore all MongoDB data for the dev1 instance +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_mongodb + +# Backup just the IoT MongoDB data for the dev2 instance +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev2 +export MAS_APP_ID=iot +ansible-playbook ibm.mas_devops.br_mongodb +``` + + +Backup/Restore for Db2 +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_db2` will invoke the role [db2](../roles/db2.md) to backup/restore a single Db2 instance. + +### Environment Variables + +- `DB2_INSTANCE_NAME`: **Required** This playbook only supports backing up specific Db2 instance at a time. If you want to backup all Db2 instances in the Db2 cluster, you need to run this playbook multiple times with different value of this environment variable. +- `MAS_INSTANCE_ID`: **Required** Set the instance ID for the MAS install. +- `MASBR_ACTION`: **Required** Set the action to be performed, `backup` or `restore`. +- `MASBR_STORAGE_LOCAL_FOLDER`: **Required** Set the local path to the directory to be used for backup and restore. +- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` + +### Examples +```bash +# Full backup for the db2w-shared Db2 instance +export MAS_INSTANCE_ID=dev +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_db2 + +# Incremental backup for the db2w-shared Db2 instance +export MAS_INSTANCE_ID=dev +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_db2 + +# Restore for the db2w-shared Db2 instance +export MAS_INSTANCE_ID=dev +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_db2 +``` + +Backup/Restore for MAS Core +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_core` will backup the following components that MAS Core depends on in order: + +| Component | Ansible Role | Data included | +| --------- | -------------------------------------------------------- | ---------------------------------- | +| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core | +| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | + + +### Environment Variables + +- `MAS_INSTANCE_ID` **Required**. The MAS instance ID to perform a backup for. + +### Examples +```bash +# Full backup all core data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_core + +# Incremental backup all core data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_core + +# Restore all core data for the dev1 instance +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +ansible-playbook ibm.mas_devops.br_core +``` + + +Backup/Restore for Manage +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_manage` will backup the following components that Manage depends on in order: + +| Component | Role | Data included | +| --------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core | +| db2 | [db2](../roles/db2.md) | Db2 instance used by Manage | +| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | +| manage | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Manage namespace resources
Persistent volume data, such as attachments | + + +### Environment Variables + +- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `DB2_INSTANCE_NAME` **Optional**. When defined, this playbook will backup the Db2 instance used by Manage. DB2 role is skipped when environment variable is not defined.. +- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` + +### Examples + +```bash +# Full backup all manage data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role +ansible-playbook ibm.mas_devops.br_manage + +# Incremental backup all manage data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role +ansible-playbook ibm.mas_devops.br_manage + +# Restore all manage data for the dev1 instance and ws1 workspace +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage # set this to execute db2 backup role +ansible-playbook ibm.mas_devops.br_manage +``` + + +Backup/Restore for IoT +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_iot` will backup the following components that IoT depends on in order: + +| Component | Ansible Role | Data included | +| --------- | ---------------------------------------------------------------- | ------------------------------------------ | +| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core and IoT | +| db2 | [db2](../roles/db2.md) | Db2 instance used by IoT | +| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | +| iot | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | IoT namespace resources | + +### Environment Variables + +- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by IoT, you need to set the correct Db2 instance name for this environment variable. +- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` + +### Examples + +```bash +# Full backup all iot data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_iot + +# Incremental backup all iot data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_iot + +# Restore all iot data for the dev1 instance +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_iot + +``` + + +Backup/Restore for Monitor +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_monitor` will backup the following components that Monitor depends on in order: + +| Component | Ansible Role | Data included | +| --------- | ---------------------------------------------------------------- | --------------------------------------------------- | +| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core, IoT and Monitor | +| db2 | [db2](../roles/db2.md) | Db2 instance used by IoT and Monitor | +| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | +| iot | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | IoT namespace resources | +| monitor | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Monitor namespace resources | + + +### Environment Variables + +- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by IoT and Monitor, you need to set the correct Db2 instance name for this environment variable. +- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` + +### Examples + +```bash +# Full backup all monitor data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_monitor + +# Incremental backup all monitor data for the dev1 instance +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_monitor + +# Restore all monitor data for the dev1 instance +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=db2w-shared +ansible-playbook ibm.mas_devops.br_monitor +``` + + +Backup/Restore for Health +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_health` will backup the following components that Health depends on in order: + +| Component | Ansible Role | Data included | +| --------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core | +| db2 | [db2](../roles/db2.md) | Db2 instance used by Manage and Health | +| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | +| manage | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Manage namespace resources
Persistent volume data, such as attachments | +| health | [suite_backup_restore](../roles/suite_backup_restore.md) | Health namespace resources
Watson Studio project assets | + +### Environment Variables + +- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by Manage and Health, you need to set the correct Db2 instance name for this environment variable. + +### Examples + +```bash +# Full backup all health data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +ansible-playbook ibm.mas_devops.br_health + +# Incremental backup all health data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +ansible-playbook ibm.mas_devops.br_health + +# Restore all health data for the dev1 instance and ws1 workspace +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +ansible-playbook ibm.mas_devops.br_health +``` + + +Backup/Restore for Optimizer +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_optimizer` will backup the following components that Optimizer depends on in order: + +| Component | Ansible Role | Data included | +| --------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core and Optimizer | +| db2 | [db2](../roles/db2.md) | Db2 instance used by Manage | +| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | +| manage | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Manage namespace resources
Persistent volume data, such as attachments | +| optimizer | [suite_backup_restore](../roles/suite_backup_restore.md) | Optimizer namespace resources | + +### Environment Variables + +- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `DB2_INSTANCE_NAME` **Required**. This playbook will backup the the Db2 instance used by Manage, you need to set the correct Db2 instance name for this environment variable. +- `DB2_NAMESPACE`: **Optional** Set the DB2 namespace, defaults to `db2u` + +### Examples + +```bash +# Full backup all optimizer data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +ansible-playbook ibm.mas_devops.br_optimizer + +# Incremental backup all optimizer data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +ansible-playbook ibm.mas_devops.br_optimizer + +# Restore all optimizer data for the dev1 instance and ws1 workspace +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +export DB2_INSTANCE_NAME=mas-dev1-ws1-manage +ansible-playbook ibm.mas_devops.br_optimizer +``` + + +Backup/Restore for Visual Inspection +------------------------------------------------------------------------------- +This playbook `ibm.mas_devops.br_visualinspection` will backup the following components that Visual Inspection depends on in order: + +| Component | Ansible Role | Data included | +| ---------------- | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| mongodb | [mongodb](../roles/mongodb.md) | MongoDB databases used by MAS Core and Visual Inspection | +| core | [suite_backup_restore](../roles/suite_backup_restore.md) | MAS Core namespace resources | +| visualinspection | [suite_app_backup_restore](../roles/suite_app_backup_restore.md) | Visual Inspection namespace resources
Persistent volume data, such as images and models | + +### Environment Variables + +- `MAS_INSTANCE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS instance at a time. If you have multiple MAS instances in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. +- `MAS_WORKSPACE_ID` **Required**. This playbook only supports backing up components belong to a specific MAS workspace at a time. If you have multiple MAS workspaces in the cluster to be backed up, you need to run this playbook multiple times with different value of this environment variable. + +### Examples + +```bash +# Full backup all visual inspection data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +ansible-playbook ibm.mas_devops.br_visualinspection + +# Incremental backup all visual inspection data for the dev1 instance and ws1 workspace +export MASBR_ACTION=backup +export MASBR_BACKUP_TYPE=incr +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +ansible-playbook ibm.mas_devops.br_visualinspection + +# Restore all visual inspection data for the dev1 instance and ws1 workspace +export MASBR_ACTION=restore +export MASBR_STORAGE_LOCAL_FOLDER=/tmp/backup +export MASBR_RESTORE_FROM_VERSION=20240630132439 +export MAS_INSTANCE_ID=dev +export MAS_WORKSPACE_ID=ws1 +ansible-playbook ibm.mas_devops.br_visualinspection +``` + + +Reference +------------------------------------------------------------------------------- +### Directory Structure +No matter what kind of storage systems you choose, the folder structure created in the storage system is same. + +Below is the sample folder structure for saving backup jobs: + +``` +/backups/mongodb-main-full-20240621122530 +├── backup.yml +├── database +│ ├── mongodb-main-full-20240621122530.tar.gz +│ └── query.json +└── log + ├── mongodb-main-full-20240621122530-backup-log.tar.gz + └── mongodb-main-full-20240621122530-ansible-log.tar.gz + +/backups/core-main-full-20240621122530 +├── backup.yml +├── log +│ ├── core-main-full-20240621122530-ansible-log.tar.gz +│ └── core-main-full-20240621122530-namespace-log.tar.gz +└── namespace + └── core-main-full-20240621122530-namespace.tar.gz +``` + +- ``: the root folder is specified by `MASBR_STORAGE_LOCAL_FOLDER` or `MASBR_STORAGE_CLOUD_BUCKET` +- The backup playbooks will create a seperated backup job folder under the `backups` folder for each component. The backup job folder is named by following this format: `{BACKUP COMPONENT}-{INSTANCE ID}-{BACKUP TYPE}-{BACKUP VERSION}`. +- When using playbook to backup multiple components at once, all backup job folders will be assigned to the same backup version. In above example, the same backup version `20240621122530` for backing up `mongodb` and `core` components. +- `backup.yml`: keep the backup job information +- `database`: data type for database. This folder save the backup files of MongoDB database, Db2 database. +- `namespace`: data type for namespace resources. This folder save the exported namespace resources. +- `pv`: data type for persistent volume. This folder save the persistent volume data, e.g. the Manage attachments, VI images and models. +- `log`: this folder save all job running log files + +In addition to the backup jobs, we also save restore jobs in the specified storage location. For example: + +``` +/restores/mongodb-main-incr-20240622040201-20240622075501 +├── log +│ ├── mongodb-main-incr-20240622040201-20240622075501-ansible-log.tar.gz +│ └── mongodb-main-incr-20240622040201-20240622075501-restore-log.tar.gz +└── restore.yml + +/restores/core-main-incr-20240622040201-20240622075501 +├── log +│ ├── core-main-incr-20240622040201-20240622075501-ansible-log.tar.gz +│ └── core-main-incr-20240622040201-20240622075501-namespace-log.tar.gz +└── restore.yml +``` + +The restore playbooks will create a seperated restore job folder under the `restores` folder for each component. The restore job folder is named by following this format: `{BACKUP JOB NAME}-{RESTORE VERSION}`. + +- `restore.yml`: keep the restore job information +- `log`: this folder save all job running log files + +### Data Model +#### backup.yml +```yaml +kind: Backup +name: "core-main-incr-20240622040201" +version: "20240622040201" +type: "incr" +from: "core-main-full-20240621122530" +source: + domain: "source-cluster.mydomain.com" + suite: "8.11.11" + instance: "main" + workspace: "" +component: + name: "core" + instance: "main" + namespace: "mas-main-core" +data: + - seq: "1" + type: "namespace" + phase: "Completed" +status: + phase: "Completed" + startTimestamp: "2024-06-22T04:05:22" + completionTimestamp: "2024-06-22T04:06:04" + sentNotifications: + - type: "Slack" + channel: "#ansible-slack-dev" + timestamp: "2024-06-22T04:05:34" + phase: "InProgress" + - type: "Slack" + channel: "#ansible-slack-dev" + timestamp: "2024-06-22T04:06:10" + phase: "Completed" +``` + +#### restore.yml +```yaml +kind: Restore +name: "core-main-incr-20240622040201-20240622075501" +version: "20240622075501" +from: "core-main-incr-20240622040201" +target: + domain: "target-cluster.mydomain.com" +component: + name: "core" + instance: "main" + namespace: "mas-main-core" +data: + - seq: 1 + type: "namespace" + phase: "Completed" +status: + phase: "Completed" + startTimestamp: "2024-06-22T08:04:19" + completionTimestamp: "2024-06-22T08:04:33" +``` diff --git a/ibm/mas_devops/playbooks/br_db2.yml b/ibm/mas_devops/playbooks/br_db2.yml index 28b65df3bd..5ac82d18e0 100644 --- a/ibm/mas_devops/playbooks/br_db2.yml +++ b/ibm/mas_devops/playbooks/br_db2.yml @@ -21,11 +21,8 @@ override_storageclass: "{{ lookup('env', 'OVERRIDE_STORAGECLASS') | default(false, true) }}" # Set OVERRIDE_STORAGECLASS to true and use the below storage classes. # when OVERRIDE_STORAGECLASS is true and below classes are not set, then the cluster's default storage classes will be used. - db2_meta_storage_class: "{{ lookup('env', 'DB2_META_STORAGE_CLASS') }}" - db2_data_storage_class: "{{ lookup('env', 'DB2_DATA_STORAGE_CLASS') }}" - db2_backup_storage_class: "{{ lookup('env', 'DB2_BACKUP_STORAGE_CLASS') }}" - db2_logs_storage_class: "{{ lookup('env', 'DB2_LOGS_STORAGE_CLASS') }}" - db2_temp_storage_class: "{{ lookup('env', 'DB2_TEMP_STORAGE_CLASS') }}" + custom_storage_class_rwo: "{{ lookup('env', 'CUSTOM_STORAGE_CLASS_RWO') }}" + custom_storage_class_rwx: "{{ lookup('env', 'CUSTOM_STORAGE_CLASS_RWX') }}" pre_tasks: - name: Important Notice diff --git a/ibm/mas_devops/roles/db2/README.md b/ibm/mas_devops/roles/db2/README.md index 68ad569926..531b85ea5e 100644 --- a/ibm/mas_devops/roles/db2/README.md +++ b/ibm/mas_devops/roles/db2/README.md @@ -998,6 +998,40 @@ This is only used when both `mas_config_dir` and `mas_instance_id` are set, and Role Variables - Backup and Restore ------------------------------------------------------------------------------- +### mas_instance_id +MAS instance identifier for the backup/restore operation. + +- **Required** for backup and restore operations +- Environment Variable: `MAS_INSTANCE_ID` +- Default: None + +**Purpose**: Identifies the MAS instance associated with the Db2 backup. Used for organizing backups and ensuring restore operations target the correct instance. + +**When to use**: +- Always required when performing backup or restore operations +- Must match the MAS instance ID that uses this Db2 instance + +**Valid values**: Valid MAS instance ID (e.g., `inst1`, `masinst1`) + +**Example**: `masinst1` + +### mas_application_id +MAS application identifier for the backup/restore operation. + +- **Required** for backup and restore operations +- Environment Variable: `MAS_APP_ID` +- Default: None + +**Purpose**: Identifies the MAS application (e.g., manage, iot) that uses this Db2 database. Used for organizing backups and database-specific operations. + +**When to use**: +- Always required when performing backup or restore operations +- Must match the MAS application that uses this Db2 database + +**Valid values**: Valid MAS application ID (e.g., `manage`, `iot`, `monitor`) + +**Example**: `manage` + ### mas_backup_dir Local directory path where backups will be stored or restored from. @@ -1016,7 +1050,7 @@ Local directory path where backups will be stored or restored from. **Impact**: - Backup files and metadata are stored in subdirectories under this path -- Directory structure: `/backup--db2u/` +- Directory structure: `/backup--db2u-/` - Insufficient space will cause backup failures **Related variables**: @@ -1043,7 +1077,7 @@ The backup version timestamp identifier for backup and restore operations. **Valid values**: Timestamp string in format `YYYYMMDD-HHMMSS` (e.g., `20251212-021316` for December 12, 2025 at 02:13:16) **Impact**: -- Determines the backup directory name: `backup--db2u` +- Determines the backup directory name: `backup--db2u-` - Used to locate backup files during restore operations - Recorded in backup metadata file for verification @@ -1071,15 +1105,50 @@ Only used in Db2 instance restore. **Valid values**: `true`, `false` **Impact**: -- When `true`: Uses `DB2_META_STORAGE_CLASS`, `DB2_DATA_STORAGE_CLASS`, `DB2_BACKUP_STORAGE_CLASS`, `DB2_LOGS_STORAGE_CLASS`, `DB2_TEMP_STORAGE_CLASS` if set, otherwise uses cluster default storage classes +- When `true`: Uses `CUSTOM_STORAGE_CLASS_RWO` and `CUSTOM_STORAGE_CLASS_RWX` if set, otherwise uses cluster default storage classes - When `false`: Uses storage classes from backup metadata (original instance configuration) **Related variables**: -- `db2_meta_storage_class`: Override for metadata storage -- `db2_data_storage_class`: Override for data storage -- `db2_backup_storage_class`: Override for backup storage -- `db2_logs_storage_class`: Override for logs storage -- `db2_temp_storage_class`: Override for temp storage +- `custom_storage_class_rwo`: Override for ReadWriteOnce storage (applies to data, logs, temp, archivelogs, audit_logs) +- `custom_storage_class_rwx`: Override for ReadWriteMany storage (applies to meta, backup) + +### custom_storage_class_rwo +Custom ReadWriteOnce storage class for Db2 restore operations. + +- **Optional** +- Environment Variable: `CUSTOM_STORAGE_CLASS_RWO` +- Default: None + +**Purpose**: Provides a single storage class override for all ReadWriteOnce (RWO) PVCs during restore. This simplifies storage class configuration when all RWO volumes can use the same storage class. + +**When to use**: +- Set when `override_storageclass` is `true` and you want to use the same storage class for all RWO volumes +- Applies to: data, logs, temp, archivelogs, and audit_logs storage + +**Valid values**: Valid storage class name available in the target cluster + +**Impact**: When set, overrides the storage class for all RWO PVCs unless specific DB2 storage class variables are also set (which take precedence) + +**Example**: `ocs-storagecluster-ceph-rbd` + +### custom_storage_class_rwx +Custom ReadWriteMany storage class for Db2 restore operations. + +- **Optional** +- Environment Variable: `CUSTOM_STORAGE_CLASS_RWX` +- Default: None + +**Purpose**: Provides a single storage class override for all ReadWriteMany (RWX) PVCs during restore. This simplifies storage class configuration when all RWX volumes can use the same storage class. + +**When to use**: +- Set when `override_storageclass` is `true` and you want to use the same storage class for all RWX volumes +- Applies to: meta and backup storage + +**Valid values**: Valid storage class name available in the target cluster + +**Impact**: When set, overrides the storage class for all RWX PVCs unless specific DB2 storage class variables are also set (which take precedence) + +**Example**: `ocs-storagecluster-cephfs` ### backup_type Type of backup operation to perform on the Db2 database. @@ -1274,6 +1343,7 @@ Example Usage - Backup and Restore any_errors_fatal: true vars: mas_instance_id: masinst1 + mas_application_id: manage mas_backup_dir: /tmp/masbr db2_action: backup-database db2_instance_name: db2u-manage @@ -1290,6 +1360,7 @@ Example Usage - Backup and Restore any_errors_fatal: true vars: mas_instance_id: masinst1 + mas_application_id: manage mas_backup_dir: /tmp/masbr db2_action: backup-database db2_instance_name: db2u-manage @@ -1309,6 +1380,7 @@ Example Usage - Backup and Restore any_errors_fatal: true vars: mas_instance_id: masinst1 + mas_application_id: manage mas_backup_dir: /tmp/masbr db2_action: backup db2_instance_name: db2u-manage @@ -1325,6 +1397,7 @@ Example Usage - Backup and Restore vars: db2_action: restore-database mas_instance_id: masinst1 + mas_application_id: manage db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr db2_instance_name: db2u-manage @@ -1341,6 +1414,7 @@ Example Usage - Backup and Restore vars: db2_action: restore-database mas_instance_id: masinst1 + mas_application_id: manage db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr db2_instance_name: db2u-manage @@ -1360,6 +1434,7 @@ Example Usage - Backup and Restore vars: db2_action: restore mas_instance_id: masinst1 + mas_application_id: manage db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr backup_vendor: disk @@ -1367,26 +1442,20 @@ Example Usage - Backup and Restore - ibm.mas_devops.db2 ``` -### Restore Db2 from Backup (Instance + Database) w/ storage class override -# This will override the storage class for all Db2 PVCs -# If you want to override specific PVCs, use the following variables: -# db2_meta_storage_class, db2_data_storage_class, db2_backup_storage_class, db2_logs_storage_class, db2_temp_storage_class -# or cluster's default storage class will be used to override. +### Restore Db2 from Backup (Instance + Database) w/ storage class override using custom storage classes ```yaml - hosts: localhost any_errors_fatal: true vars: db2_action: restore mas_instance_id: masinst1 + mas_application_id: manage db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr backup_vendor: disk override_storageclass: true - db2_meta_storage_class: nfs-client # optional - db2_data_storage_class: nfs-client # optional - db2_backup_storage_class: nfs-client # optional - db2_logs_storage_class: nfs-client # optional - db2_temp_storage_class: nfs-client # optional + custom_storage_class_rwo: ocs-storagecluster-ceph-rbd # For data, logs, temp, archivelogs, audit_logs + custom_storage_class_rwx: ocs-storagecluster-cephfs # For meta, backup roles: - ibm.mas_devops.db2 ``` @@ -1398,6 +1467,7 @@ Example Usage - Backup and Restore vars: db2_action: restore mas_instance_id: masinst1 + mas_application_id: manage db2_backup_version: 20251212-021316 mas_backup_dir: /tmp/masbr backup_vendor: s3 @@ -1412,9 +1482,9 @@ Example Usage - Backup and Restore ### Backup Directory Structure (Disk) ``` /tmp/masbr/ -└── backup--db2u/ +└── backup--db2u-/ ├── data/ - │ ├── db2-BLUDB-backup-.tar.gz + │ ├── db2--BLUDB-backup-.tar.gz │ └── db2-backup-info.yaml └── resources/ ├── db2uclusters/ diff --git a/mkdocs.yml b/mkdocs.yml index b26e6f6efd..44ad9c2c05 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -25,6 +25,8 @@ nav: - "Add Real Estate and Facilities": playbooks/mas-facilities.md - "Update": playbooks/mas-update.md - "Upgrade": playbooks/mas-upgrade.md + - "Backup and Restore": playbooks/backup_restore.md + - "Legacy Backup and Restore": playbooks/legacy_backup_restore.md - "Roles: OCP Mgmt": - "ocp_cluster_monitoring": roles/ocp_cluster_monitoring.md - "ocp_config": roles/ocp_config.md