diff --git a/docs/commands/backup.md b/docs/commands/backup.md index 21cba28762e..d93891c4f38 100644 --- a/docs/commands/backup.md +++ b/docs/commands/backup.md @@ -6,13 +6,15 @@ Usage Usage information can be obtained using `mas backup --help` ``` -usage: mas backup [-i MAS_INSTANCE_ID] [--backup-version BACKUP_VERSION] [--backup-storage-size BACKUP_STORAGE_SIZE] +usage: mas backup [-i MAS_INSTANCE_ID] [--backup-version BACKUP_VERSION] + [--backup-storage-size BACKUP_STORAGE_SIZE] [--backup-storage-class BACKUP_STORAGE_CLASS] + [--backup-storage-access-mode BACKUP_STORAGE_ACCESS_MODE] [--clean-backup] [--no-clean-backup] [--upload-backup] [--aws-access-key-id AWS_ACCESS_KEY_ID] [--aws-secret-access-key AWS_SECRET_ACCESS_KEY] [--s3-bucket-name S3_BUCKET_NAME] [--s3-region S3_REGION] [--artifactory-url ARTIFACTORY_URL] [--artifactory-repository ARTIFACTORY_REPOSITORY] [--backup-manage-app] [--manage-workspace-id MANAGE_WORKSPACE_ID] [--backup-manage-db] [--manage-db2-namespace MANAGE_DB2_NAMESPACE] [--manage-db2-instance-name MANAGE_DB2_INSTANCE_NAME] - [--manage-db2-backup-type {offline,online}] [--include-sls] [--exclude-sls] + [--manage-db2-backup-type {offline,online}] [--include-sls] [--exclude-sls] [--include-mongo] [--exclude-mongo] [--mongodb-namespace MONGODB_NAMESPACE] [--mongodb-instance-name MONGODB_INSTANCE_NAME] [--mongodb-provider {community}] [--sls-namespace SLS_NAMESPACE] [--cert-manager-provider {redhat,ibm}] [--artifactory-username ARTIFACTORY_USERNAME] [--artifactory-token ARTIFACTORY_TOKEN] [--dev-mode] [--no-confirm] @@ -31,8 +33,12 @@ MAS Instance: Backup Configuration: --backup-version BACKUP_VERSION Version/timestamp for the backup (auto-generated if not provided) + --backup-storage-class BACKUP_STORAGE_CLASS + Storage class for backup-pvc PVC storage --backup-storage-size BACKUP_STORAGE_SIZE Size of the backup PVC storage (default: 20Gi) + --backup-storage-access-mode BACKUP_STORAGE_ACCESS_MODE + Access mode for backup PVC storage (ReadWriteOnce or ReadWriteMany) --clean-backup Clean backup and config workspaces after completion (default: true) --no-clean-backup Do not clean backup and config workspaces after completion @@ -64,6 +70,8 @@ Manage Application Backup: Manage Db2 backup type: offline (database unavailable) or online (database remains available) Components: + --include-mongo Include Mongo in backup (default: true) + --exclude-mongo Exclude Mongo from backup (use if Mongo is external) --include-sls Include SLS in backup (default: true) --exclude-sls Exclude SLS from backup (use if SLS is external) @@ -142,6 +150,16 @@ Create a backup without including Suite License Service (useful when SLS is exte mas backup --instance-id inst1 --exclude-sls --no-confirm ``` +### Backup Excluding MongoDB +Create a backup without including MongoDB (useful when MongoDB is externally hosted): + +```bash +mas backup --instance-id inst1 --exclude-mongo --no-confirm +``` + +!!! note + Use `--exclude-mongo` when using external MongoDB providers such as IBM Cloud Databases for MongoDB, MongoDB Atlas, or other managed MongoDB services. You must back up your MongoDB database separately using your provider's native backup tools. + ### Backup with Custom MongoDB Configuration Specify custom MongoDB settings: @@ -254,6 +272,7 @@ If not specified, the following defaults are used: - **MongoDB Provider**: `community` - **SLS Namespace**: `ibm-sls` - **Certificate Manager Provider**: `redhat` +- **Include MongoDB**: `true` - **Include SLS**: `true` ### Storage Requirements diff --git a/docs/commands/restore.md b/docs/commands/restore.md index 2115a86cdee..ef451455112 100644 --- a/docs/commands/restore.md +++ b/docs/commands/restore.md @@ -6,7 +6,9 @@ Usage Usage information can be obtained using `mas restore --help` ``` -usage: mas restore [-i MAS_INSTANCE_ID] [--restore-version RESTORE_VERSION] [--backup-storage-size BACKUP_STORAGE_SIZE] +usage: mas restore [-i MAS_INSTANCE_ID] [--restore-version RESTORE_VERSION] + [--backup-storage-size BACKUP_STORAGE_SIZE] [--backup-storage-class BACKUP_STORAGE_CLASS] + [--backup-storage-access-mode BACKUP_STORAGE_ACCESS_MODE] [--mas-domain-restore MAS_DOMAIN_ON_RESTORE] [--sls-url-restore SLS_URL_ON_RESTORE] [--dro-url-restore DRO_URL_ON_RESTORE] [--include-slscfg-from-backup] [--exclude-slscfg-from-backup] [--sls-cfg-file SLS_CFG_FILE] [--dro-cfg-file DRO_CFG_FILE] [--include-drocfg-from-backup] @@ -15,6 +17,7 @@ usage: mas restore [-i MAS_INSTANCE_ID] [--restore-version RESTORE_VERSION] [--b [--s3-bucket-name S3_BUCKET_NAME] [--s3-region S3_REGION] [--artifactory-url ARTIFACTORY_URL] [--artifactory-repository ARTIFACTORY_REPOSITORY] [--custom-backup-archive-name BACKUP_ARCHIVE_NAME] [--include-grafana] [--exclude-grafana] [--include-dro] [--exclude-dro] [--include-sls] [--exclude-sls] + [--include-mongo] [--exclude-mongo] [--sls-domain SLS_DOMAIN] [--ibm-entitlement-key IBM_ENTITLEMENT_KEY] [--contact-email DRO_CONTACT_EMAIL] [--contact-firstname DRO_CONTACT_FIRSTNAME] [--contact-lastname DRO_CONTACT_LASTNAME] [--dro-namespace DRO_NAMESPACE] [--override-mongodb-storageclass] [--mongodb-storageclass-name MONGODB_STORAGECLASS_NAME] @@ -55,8 +58,12 @@ MAS Instance: Restore Configuration: --restore-version RESTORE_VERSION Version/timestamp used in backup. Example: YYYYMMDD-HHMMSS + --backup-storage-class BACKUP_STORAGE_CLASS + Storage class for backup-pvc workspace. --backup-storage-size BACKUP_STORAGE_SIZE Size of the PVC storage, must be bigger than backup archive size. (default: 20Gi) + --backup-storage-access-mode BACKUP_STORAGE_ACCESS_MODE + Access mode for backup PVC storage (ReadWriteOnce or ReadWriteMany) --clean-backup Clean backup and config workspaces after completion (default: true) --no-clean-backup Do not clean backup and config workspaces after completion @@ -78,6 +85,8 @@ Download Configuration: Custom backup archive name to download from S3 or Artifactory Components: + --include-mongo Include Mongo in restore (default: true) + --exclude-mongo Exclude Mongo from restore (use if Mongo is external) --include-grafana Include Grafana in restore (default: true) --exclude-grafana Skip installing Grafana. --include-dro Include DRO in restore, this will install new DRO instance (default: true) @@ -211,6 +220,20 @@ mas restore \ --no-confirm ``` +### Restore Excluding MongoDB +Restore a backup without restoring MongoDB (useful when using external MongoDB): + +```bash +mas restore \ + --instance-id inst1 \ + --restore-version 20260117-191701 \ + --exclude-mongo \ + --no-confirm +``` + +!!! note + Use `--exclude-mongo` when your MongoDB is externally hosted and was not included in the backup. Ensure your external MongoDB database is properly configured and accessible before restoring MAS. + ### Restore with Custom SLS Configuration File Restore using a custom SLS configuration file instead of the one from backup: @@ -403,6 +426,7 @@ If not specified, the following defaults are used: - **Backup Storage Size**: `20Gi` - **Clean Workspaces**: `true` (workspaces are cleaned after completion) +- **Include MongoDB**: `true` - **Include SLS**: `true` - **Include Grafana**: `true` - **Include DRO**: `true` @@ -440,9 +464,10 @@ The restore command provides flexibility in how configurations are restored: - By default, the MAS domain is restored from the backup - Use `--mas-domain-restore` to change the domain during restore -### Component Installation +### Component Selection The restore process can optionally install components that are not part of the backup: +- **Mongo**: MongoDb Community edition (when backed up, can be restored or skipped if using external Mongo). Use `--include-mongo` to restore Mongo from backup or `--exclude-mongo` to skip Mongo restoration. - **Grafana**: Monitoring and visualization (not backed up, can be installed during restore). Use `--include-grafana` to install grafana during restore or `--exclude-grafana` to skip grafana installation. - **DRO**: Data Reporting Operator (not backed up, can be installed during restore). Use `--include-dro` to install DRO during restore or `--exclude-dro` to skip DRO installation. - **SLS**: Suite License Service (backed up, can be restored or skipped if using external SLS). Use `--include-sls` to restore SLS from backup or `--exclude-sls` to skip SLS installation. diff --git a/docs/guides/backup-restore.md b/docs/guides/backup-restore.md index 9ec2bd7e5e1..98a02d57130 100644 --- a/docs/guides/backup-restore.md +++ b/docs/guides/backup-restore.md @@ -176,14 +176,28 @@ The Data Reporter Operator (DRO) is **not included in backup operations** as it ### MongoDB Configuration -The backup process supports **MongoDB Community Edition only**. Ensure you specify the correct MongoDB configuration: +The backup process supports **MongoDB Community Edition only**. By default, MongoDB is included in the backup. You can configure MongoDB settings or exclude it if using an external MongoDB provider. + +**Including MongoDB in Backup (Default)** + +When MongoDB is included, ensure you specify the correct MongoDB configuration: - **Namespace** - Where MongoDB is deployed (default: `mongoce`) - **Instance Name** - MongoDB instance identifier (default: `mas-mongo-ce`) - **Provider** - Must be `community` (only supported provider for backup) +**Excluding MongoDB from Backup** + +If you are using an external MongoDB provider (such as IBM Cloud Databases for MongoDB or other hosted MongoDB services), you should exclude MongoDB from the backup: + +- Use `--exclude-mongo` flag in non-interactive mode +- In interactive mode, answer "No" when prompted to include MongoDB in backup + !!! warning - IBM Cloud Databases for MongoDB and other external MongoDB providers are not supported by the backup process. You must use their native backup mechanisms. + IBM Cloud Databases for MongoDB and other external MongoDB providers are not supported by the backup process. You must use their native backup mechanisms. When using external MongoDB, always use `--exclude-mongo` to skip MongoDB backup. + +!!! tip + When excluding MongoDB from backup, you are responsible for backing up your MongoDB database using your provider's native backup tools. Ensure MongoDB backups are coordinated with MAS backups for consistency. ### Certificate Manager @@ -338,7 +352,29 @@ mas backup \ --no-confirm ``` -### Scenario 4: Backup with S3 Upload +### Scenario 4: External MongoDB Deployment + +**Environment:** +- MAS in OpenShift cluster +- MongoDB hosted externally (e.g., IBM Cloud Databases for MongoDB, MongoDB Atlas, or other managed service) +- In-cluster SLS +- Red Hat Certificate Manager + +**Backup Command:** +```bash +mas backup \ + --instance-id inst1 \ + --backup-storage-size 30Gi \ + --exclude-mongo \ + --no-confirm +``` + +Use `--exclude-mongo` to skip backing up MongoDB when it's managed externally. You must use your MongoDB provider's native backup mechanisms to back up the database separately. + +!!! important + When using external MongoDB, coordinate your MongoDB backups with MAS backups to ensure data consistency. Back up MongoDB before or immediately after the MAS backup completes. + +### Scenario 5: Backup with S3 Upload **Environment:** - Standard MAS deployment @@ -362,7 +398,7 @@ mas backup \ !!! tip Store AWS credentials securely using environment variables or secrets management systems rather than hardcoding them in scripts. -### Scenario 5: Backup with Manage Application and Db2 Database +### Scenario 6: Backup with Manage Application and Db2 Database **Environment:** - Standard MAS deployment with Manage application @@ -387,7 +423,7 @@ mas backup \ !!! tip When backing up Manage with Db2, ensure sufficient backup storage (100Gi+ recommended) to accommodate application PV data and database backups. Use offline backup type if your Db2 instance uses the default circular logging configuration. -### Scenario 6: Backup with Manage Application Only (External Db2) +### Scenario 7: Backup with Manage Application Only (External Db2) **Environment:** - MAS deployment with Manage application @@ -407,7 +443,7 @@ mas backup \ !!! note When using an external Db2 database, omit the `--backup-manage-db` flag. The database should be backed up separately using your organization's database backup procedures. -### Scenario 7: Backup for Troubleshooting (No Cleanup) +### Scenario 8: Backup for Troubleshooting (No Cleanup) **Environment:** - Backup for troubleshooting purposes @@ -428,7 +464,7 @@ mas backup \ !!! note Use `--no-clean-backup` when you need to inspect the backup workspace contents for troubleshooting. Remember to manually clean up the workspaces later to free up storage. -### Scenario 8: Minimal Backup (Skip Pre-Check) +### Scenario 9: Minimal Backup (Skip Pre-Check) **Environment:** - Emergency backup scenario @@ -479,7 +515,12 @@ The backup process automatically selects appropriate storage: - **Multi-node clusters**: Prefers ReadWriteMany (RWX) storage when available - Falls back to RWO if RWX is not available -The storage class is determined from your cluster's default storage classes. +The storage class is determined from your cluster's storage classes. + +You can override the default storage configuration using: + +- `--backup-storage-class`: Specify a custom storage class for the backup PVC +- `--backup-storage-access-mode`: Specify the access mode (e.g., ReadWriteOnce, ReadWriteMany) for the backup PVC Backup Process Details @@ -792,6 +833,7 @@ When downloading from S3 or Artifactory, the `download_backup_archive` role sele | Parameter | Default | Description | |-----------|---------|-------------| +| `include_mongo_archive` | `false` | Download the Mongo backup archive | | `include_sls_archive` | `false` | Download the SLS backup archive | | `include_manage_db_archive` | `false` | Download the Manage Db2 database backup archive | | `include_manage_app_archive` | `false` | Download the Manage application backup archive | diff --git a/python/src/mas/cli/backup/app.py b/python/src/mas/cli/backup/app.py index 3b718583cdb..1d0a0fafbd9 100644 --- a/python/src/mas/cli/backup/app.py +++ b/python/src/mas/cli/backup/app.py @@ -12,15 +12,15 @@ import logging from datetime import datetime from halo import Halo -from prompt_toolkit import print_formatted_text, HTML +from prompt_toolkit import prompt, print_formatted_text, HTML from prompt_toolkit.completion import WordCompleter from openshift.dynamic.exceptions import ResourceNotFoundError from ..cli import BaseApp -from ..validators import InstanceIDValidator +from ..validators import InstanceIDValidator, StorageClassValidator from .argParser import backupArgParser -from mas.devops.ocp import createNamespace, getConsoleURL +from mas.devops.ocp import createNamespace, getConsoleURL, getStorageClasses from mas.devops.mas import listMasInstances, getDefaultStorageClasses, getWorkspaceId from mas.devops.tekton import preparePipelinesNamespace, installOpenShiftPipelines, updateTektonDefinitions, launchBackupPipeline @@ -49,9 +49,12 @@ def backup(self, argv): requiredParams = ["mas_instance_id"] optionalParams = [ "backup_version", + "backup_storage_class", + "backup_storage_access_mode", "backup_storage_size", "clean_backup", "include_sls", + "include_mongo", "mongodb_namespace", "mongodb_instance_name", "mongodb_provider", @@ -125,19 +128,19 @@ def backup(self, argv): self.promptForInstanceId() # Prompt for backup storage size if not provided - if self.args.backup_storage_size is None: - self.promptForBackupStorageSize() + if self.args.backup_storage_class is None or self.args.backup_storage_access_mode is None or self.args.backup_storage_size is None: + self.promptForBackupStorage() # Prompt for backup version if not provided if self.args.backup_version is None: self.promptForBackupVersion() - # Prompt for SLS configuration - self.promptForSLSConfiguration() - # Prompt for MongoDB configuration self.promptForMongoDBConfiguration() + # Prompt for SLS configuration + self.promptForSLSConfiguration() + # Prompt for Manage app backup self.promptForManageAppBackup() @@ -168,7 +171,9 @@ def backup(self, argv): self.printH2("Components") self.printSummary("Include SLS", self.getParam("include_sls") if self.getParam("include_sls") else "true") - self.printSummary("MongoDB Namespace", self.getParam("mongodb_namespace") if self.getParam("mongodb_namespace") else "mongoce") + self.printSummary("Include MongoDB", self.getParam("include_mongo") if self.getParam("include_mongo") else "true") + if self.getParam("include_mongo") != "false": + self.printSummary("MongoDB Namespace", self.getParam("mongodb_namespace") if self.getParam("mongodb_namespace") else "mongoce") self.printSummary("SLS Namespace", self.getParam("sls_namespace") if self.getParam("sls_namespace") else "ibm-sls") if self.getParam("backup_manage_app") == "true": @@ -197,14 +202,10 @@ def backup(self, argv): instanceId = self.getParam("mas_instance_id") pipelinesNamespace = f"mas-{instanceId}-pipelines" - # Determine storage class and access mode for pipeline PVCs - defaultStorageClasses = getDefaultStorageClasses(self.dynamicClient) - if self.isSNO() or defaultStorageClasses.rwx == "none": - self.pipelineStorageClass = defaultStorageClasses.rwo - self.pipelineStorageAccessMode = "ReadWriteOnce" - else: - self.pipelineStorageClass = defaultStorageClasses.rwx - self.pipelineStorageAccessMode = "ReadWriteMany" + if self.getParam("backup_storage_class") is None or self.getParam("backup_storage_class") == "": + self.fatalError("No storage class specified for 'backup-pvc' pvc, please specify a storage class for the backup storage using --backup-storage-class") + if self.getParam("backup_storage_access_mode") is None or self.getParam("backup_storage_access_mode") == "": + self.fatalError("No storage access mode specified for 'backup-pvc' pvc, please specify a storage access mode for the backup storage using --backup-storage-access-mode") with Halo(text='Validating OpenShift Pipelines installation', spinner=self.spinner) as h: if installOpenShiftPipelines(self.dynamicClient): @@ -219,8 +220,8 @@ def backup(self, argv): preparePipelinesNamespace( dynClient=self.dynamicClient, instanceId=instanceId, - storageClass=self.pipelineStorageClass, - accessMode=self.pipelineStorageAccessMode, + storageClass=self.getParam("backup_storage_class"), + accessMode=self.getParam("backup_storage_access_mode"), createConfigPVC=False, createBackupPVC=True, backupStorageSize=backupStorageSize @@ -276,8 +277,80 @@ def promptForInstanceId(self) -> None: except ResourceNotFoundError: self.fatalError("Unable to list MAS instances") - def promptForBackupStorageSize(self) -> None: + def promptForBackupStorage(self) -> None: self.printH1("Backup Storage Configuration") + + # Check if this is a Single Node OpenShift cluster + isSNOCluster = self.isSNO() + + if isSNOCluster: + self.printDescription([ + " - Single Node OpenShift detected", + " - You can use ReadWriteOnce storage class to create backup-pvc pvc for backup pipeline.", + ]) + else: + self.printDescription([ + " - Select a storage class to use to create backup-pvc pvc for backup pipeline(Recommended access mode ReadWriteMany).", + ]) + self.printDescription([ + " - Make sure to have enough storage accomodate archives and tar the contents.", + " - Example, if your accumulated size of backup archives is 8Gi, choose 20Gi.", + " - Note: There's option to clean up the archives in the end." + ]) + + defaultStorageClasses = getDefaultStorageClasses(self.dynamicClient) + pipelineStorageClass = None + pipelineStorageAccessMode = "ReadWriteMany" + + # For SNO, use ReadWriteOnce access mode and RWO storage class + if isSNOCluster or defaultStorageClasses.rwx is None: + if defaultStorageClasses.provider is not None: + pipelineStorageAccessMode = "ReadWriteOnce" + print_formatted_text(HTML(f"Storage provider auto-detected: {defaultStorageClasses.providerName}")) + print_formatted_text(HTML(f" - Storage class (ReadWriteOnce): {defaultStorageClasses.rwo}")) + pipelineStorageClass = defaultStorageClasses.rwo + else: + if defaultStorageClasses.provider is not None: + pipelineStorageAccessMode = "ReadWriteMany" + print_formatted_text(HTML(f"Storage provider auto-detected: {defaultStorageClasses.providerName}")) + print_formatted_text(HTML(f" - Storage class (ReadWriteMany): {defaultStorageClasses.rwx}")) + pipelineStorageClass = defaultStorageClasses.rwx + + customSC = False + if pipelineStorageClass is not None and pipelineStorageClass != "": + customSC = not self.yesOrNo("Use the auto-detected storage classes") + + if pipelineStorageClass is None or pipelineStorageClass == "" or customSC: + if isSNOCluster: + self.printDescription([ + "Select ReadWriteOnce storage class to use from the list below:" + ]) + for storageClass in getStorageClasses(self.dynamicClient): + print_formatted_text(HTML(f" - {storageClass.metadata.name}")) + pipelineStorageAccessMode = "ReadWriteOnce" + pipelineStorageClass = prompt(HTML('ReadWriteOnce (RWO) storage class '), validator=StorageClassValidator(), validate_while_typing=False) + else: + self.printDescription([ + "Choose Storage class access mode:", + " 1. ReadWriteOnce (RWO)", + " 2. ReadWriteMany (RWX)", + ]) + pipelineStorageAccessMode = self.promptForListSelect( + "Select Storage class access mode", + ["ReadWriteOnce", "ReadWriteMany"], + "backup_storage_access_mode", + default=1 + ) + self.printDescription([ + "Select storage class to use from the list below:" + ]) + for storageClass in getStorageClasses(self.dynamicClient): + print_formatted_text(HTML(f" - {storageClass.metadata.name}")) + pipelineStorageClass = prompt(HTML(f'Enter {pipelineStorageAccessMode} storage class '), validator=StorageClassValidator(), validate_while_typing=False) + + self.setParam("backup_storage_class", pipelineStorageClass) + self.setParam("backup_storage_access_mode", pipelineStorageAccessMode) + # Get pvc size storageSize = self.promptForString("Enter backup PVC storage size", default="20Gi") self.setParam("backup_storage_size", storageSize) @@ -336,20 +409,29 @@ def promptForMongoDBConfiguration(self) -> None: """Prompt user for MongoDB configuration""" self.printH1("MongoDB Configuration") self.printDescription([ - "Configure MongoDB settings for the backup.", - "These settings specify where MongoDB is deployed and how to access it." + " - You can skip Mongo backup if you have external MongoDB.", + " - If included, you will need to specify the MongoDB namespace and instance name used in cluster" ]) - # Prompt for MongoDB namespace - mongoNamespace = self.promptForString("MongoDB Namespace", default="mongoce") - self.setParam("mongodb_namespace", mongoNamespace) + includeMongo = self.yesOrNo("Include MongoDB in backup") + + if includeMongo: + self.setParam("include_mongo", "true") - # Prompt for MongoDB instance name - mongoInstanceName = self.promptForString("MongoDB Instance Name", default="mas-mongo-ce") - self.setParam("mongodb_instance_name", mongoInstanceName) + # Prompt for MongoDB namespace + mongoNamespace = self.promptForString("MongoDB Namespace", default="mongoce") + self.setParam("mongodb_namespace", mongoNamespace) + + # Prompt for MongoDB instance name + mongoInstanceName = self.promptForString("MongoDB Instance Name", default="mas-mongo-ce") + self.setParam("mongodb_instance_name", mongoInstanceName) + else: + self.setParam("include_mongo", "false") def setDefaultParams(self) -> None: """Set default values for optional parameters if not already set""" + if not self.getParam("include_mongo"): + self.setParam("include_mongo", "true") if not self.getParam("mongodb_namespace"): self.setParam("mongodb_namespace", "mongoce") if not self.getParam("mongodb_instance_name"): @@ -382,7 +464,7 @@ def promptForUploadConfiguration(self) -> None: self.setParam("upload_backup", "true") self.printDescription([ - "Development mode is enabled. Choose upload destination:", + "Choose upload destination:", " 1. S3", " 2. Artifactory", ]) diff --git a/python/src/mas/cli/backup/argParser.py b/python/src/mas/cli/backup/argParser.py index f1400608603..2f6418391a5 100644 --- a/python/src/mas/cli/backup/argParser.py +++ b/python/src/mas/cli/backup/argParser.py @@ -46,11 +46,23 @@ required=False, help="Version/timestamp for the backup (auto-generated if not provided)" ) +backupArgGroup.add_argument( + '--backup-storage-class', + dest='backup_storage_class', + required=False, + help="Storage class for backup-pvc PVC storage" +) backupArgGroup.add_argument( '--backup-storage-size', required=False, help="Size of the backup PVC storage (default: 20Gi)" ) +backupArgGroup.add_argument( + '--backup-storage-access-mode', + dest='backup_storage_access_mode', + required=False, + help="Access mode for backup PVC storage" +) backupArgGroup.add_argument( '--clean-backup', dest='clean_backup', @@ -182,6 +194,23 @@ const="false", help="Exclude SLS from backup (use if SLS is external)" ) +componentsArgGroup.add_argument( + '--include-mongo', + dest='include_mongo', + required=False, + action="store_const", + const="true", + default="true", + help="Include MongoDB in backup (default: true)" +) +componentsArgGroup.add_argument( + '--exclude-mongo', + dest='include_mongo', + required=False, + action="store_const", + const="false", + help="Exclude MongoDB from backup (use if MongoDB is external)" +) depsArgGroup = backupArgParser.add_argument_group( 'Dependencies Configuration', diff --git a/python/src/mas/cli/restore/app.py b/python/src/mas/cli/restore/app.py index fc2e6929559..07f43d1642a 100644 --- a/python/src/mas/cli/restore/app.py +++ b/python/src/mas/cli/restore/app.py @@ -53,8 +53,11 @@ def restore(self, argv): "sls_cfg_file", "dro_url_on_restore", "dro_cfg_file", + "backup_storage_class", + "backup_storage_access_mode", "backup_storage_size", "clean_backup", + "include_mongo", "include_sls", "include_grafana", "include_dro", @@ -138,9 +141,15 @@ def restore(self, argv): if self.args.restore_version is None: self.promptForBackupVersion() + # Prompt for backup storage size if not provided + if self.args.backup_storage_class is None or self.args.backup_storage_access_mode is None or self.args.backup_storage_size is None: + self.promptForBackupStorage() + # Prompt for backup class override self.configStorageClasses() + self.promptForMongo() + # Prompt for Grafana install self.promptForIncludeGrafana() @@ -160,10 +169,6 @@ def restore(self, argv): # Prompt for Manage app restore self.promptForManageAppRestore() - # Prompt for backup storage size if not provided - if self.args.backup_storage_size is None: - self.promptForBackupStorageSize() - self.promptForDownloadConfiguration() # Set default values for optional parameters if not provided @@ -244,6 +249,11 @@ def restore(self, argv): instanceId = self.getParam("mas_instance_id") pipelinesNamespace = f"mas-{instanceId}-pipelines" + if self.getParam("backup_storage_class") is None or self.getParam("backup_storage_class") == "": + self.fatalError("No storage class specified for 'backup-pvc' pvc, please specify a storage class for the backup storage using --backup-storage-class") + if self.getParam("backup_storage_access_mode") is None or self.getParam("backup_storage_access_mode") == "": + self.fatalError("No storage access mode specified for 'backup-pvc' pvc, please specify a storage access mode for the backup storage using --backup-storage-access-mode") + # Determine storage class and access mode for pipeline PVCs defaultStorageClasses = getDefaultStorageClasses(self.dynamicClient) if self.isSNO() or defaultStorageClasses.rwx == "none": @@ -266,8 +276,8 @@ def restore(self, argv): preparePipelinesNamespace( dynClient=self.dynamicClient, instanceId=instanceId, - storageClass=self.pipelineStorageClass, - accessMode=self.pipelineStorageAccessMode, + storageClass=self.getParam("backup_storage_class"), + accessMode=self.getParam("backup_storage_access_mode"), createConfigPVC=False, createBackupPVC=True, backupStorageSize=backupStorageSize @@ -381,6 +391,20 @@ def promptForIncludeDRO(self) -> None: else: self.setParam("include_dro", "false") + def promptForMongo(self) -> None: + """Prompt user for MongoDB configuration""" + self.printH1("MongoDB Configuration") + self.printDescription([ + " - You can skip Mongo restore if you have external MongoDB.", + ]) + + includeMongo = self.yesOrNo("Include MongoDB in restore") + + if includeMongo: + self.setParam("include_mongo", "true") + else: + self.setParam("include_mongo", "false") + def promptForIncludeGrafana(self) -> None: self.printH1("Grafana Configuration") self.printDescription([" - Grafana is not part of backup/restore. You can install Grafana instance or skip it."]) @@ -390,13 +414,80 @@ def promptForIncludeGrafana(self) -> None: else: self.setParam("include_grafana", "false") - def promptForBackupStorageSize(self) -> None: + def promptForBackupStorage(self) -> None: self.printH1("Backup Storage Configuration") + + # Check if this is a Single Node OpenShift cluster + isSNOCluster = self.isSNO() + + if isSNOCluster: + self.printDescription([ + " - Single Node OpenShift detected", + " - You can use ReadWriteOnce storage class to create backup-pvc pvc for restore pipeline.", + ]) + else: + self.printDescription([ + " - Select a storage class to use to create backup-pvc pvc for restore pipeline(Recommended access mode ReadWriteMany).", + ]) self.printDescription([ " - Make sure to have enough storage to download the archive(s) and extract the contents.", " - Example, if your accumulated size of backup archives is 8Gi, choose 20Gi.", " - Note: The downloaded archive will be deleted after the contents are extracted." ]) + + defaultStorageClasses = getDefaultStorageClasses(self.dynamicClient) + pipelineStorageClass = None + pipelineStorageAccessMode = "ReadWriteMany" + + # For SNO, use ReadWriteOnce access mode and RWO storage class + if isSNOCluster or defaultStorageClasses.rwx is None: + if defaultStorageClasses.provider is not None: + pipelineStorageAccessMode = "ReadWriteOnce" + print_formatted_text(HTML(f"Storage provider auto-detected: {defaultStorageClasses.providerName}")) + print_formatted_text(HTML(f" - Storage class (ReadWriteOnce): {defaultStorageClasses.rwo}")) + pipelineStorageClass = defaultStorageClasses.rwo + else: + if defaultStorageClasses.provider is not None: + pipelineStorageAccessMode = "ReadWriteMany" + print_formatted_text(HTML(f"Storage provider auto-detected: {defaultStorageClasses.providerName}")) + print_formatted_text(HTML(f" - Storage class (ReadWriteMany): {defaultStorageClasses.rwx}")) + pipelineStorageClass = defaultStorageClasses.rwx + + customSC = False + if pipelineStorageClass is not None and pipelineStorageClass != "": + customSC = not self.yesOrNo("Use the auto-detected storage classes") + + if pipelineStorageClass is None or pipelineStorageClass == "" or customSC: + if isSNOCluster: + self.printDescription([ + "Select ReadWriteOnce storage class to use from the list below:" + ]) + for storageClass in getStorageClasses(self.dynamicClient): + print_formatted_text(HTML(f" - {storageClass.metadata.name}")) + pipelineStorageAccessMode = "ReadWriteOnce" + pipelineStorageClass = prompt(HTML('ReadWriteOnce (RWO) storage class '), validator=StorageClassValidator(), validate_while_typing=False) + else: + self.printDescription([ + "Choose Storage class access mode:", + " 1. ReadWriteOnce (RWO)", + " 2. ReadWriteMany (RWX)", + ]) + pipelineStorageAccessMode = self.promptForListSelect( + "Select Storage class access mode", + ["ReadWriteOnce", "ReadWriteMany"], + "backup_storage_access_mode", + default=1 + ) + self.printDescription([ + "Select storage class to use from the list below:" + ]) + for storageClass in getStorageClasses(self.dynamicClient): + print_formatted_text(HTML(f" - {storageClass.metadata.name}")) + pipelineStorageClass = prompt(HTML(f'Enter {pipelineStorageAccessMode} storage class '), validator=StorageClassValidator(), validate_while_typing=False) + + self.setParam("backup_storage_class", pipelineStorageClass) + self.setParam("backup_storage_access_mode", pipelineStorageAccessMode) + # Get pvc size storageSize = self.promptForString("Enter PVC storage size, must be bigger than backup archive size.", default="20Gi") self.setParam("backup_storage_size", storageSize) @@ -408,6 +499,8 @@ def promptForBackupVersion(self) -> None: def setDefaultParams(self) -> None: """Set default values for optional parameters if not already set""" + if not self.getParam("include_mongo"): + self.setParam("include_mongo", "true") if not self.getParam("include_sls"): self.setParam("include_sls", "true") if not self.getParam("include_grafana"): @@ -434,8 +527,11 @@ def promptForDownloadConfiguration(self) -> None: self.setParam("download_backup", "true") self.printDescription([ - "Development mode is enabled. Choose download location:" + "Choose download destination:", + " 1. S3", + " 2. Artifactory", ]) + downloadDestination = self.promptForListSelect( "Select download location", ["S3", "Artifactory"], @@ -521,9 +617,9 @@ def promptForManageAppRestore(self) -> None: self.setParam("restore_manage_db", "false") def configStorageClasses(self): - self.printH1("Configure Storage Class during Restore") + self.printH1("Configure MAS Component's Storage Class during Restore") self.printDescription([ - " - You can override the storage class for components during restore.", + " - You can override the storage class for MAS components during restore.", " - This is useful when restoring to a cluster with different storage classes." ]) overrideStorageClasses = not self.yesOrNo("Do you want to use the storage classes from backup") diff --git a/python/src/mas/cli/restore/argParser.py b/python/src/mas/cli/restore/argParser.py index ca45d72616e..9abfb596a40 100644 --- a/python/src/mas/cli/restore/argParser.py +++ b/python/src/mas/cli/restore/argParser.py @@ -119,11 +119,23 @@ required=False, help="Version/timestamp used in backup. Example: YYYYMMDD-HHMMSS" ) +restoreArgGroup.add_argument( + '--backup-storage-class', + dest='backup_storage_class', + required=False, + help="Storage class for backup-pvc PVC storage" +) restoreArgGroup.add_argument( '--backup-storage-size', required=False, help="Size of the PVC storage, must be bigger than backup archive size. (default: 20Gi)" ) +restoreArgGroup.add_argument( + '--backup-storage-access-mode', + dest='backup_storage_access_mode', + required=False, + help="Access mode for backup PVC storage" +) restoreArgGroup.add_argument( '--clean-backup', dest='clean_backup', @@ -288,6 +300,24 @@ 'Configure which components to include in the restore.' ) +componentsArgGroup.add_argument( + '--include-mongo', + dest='include_mongo', + required=False, + action="store_const", + const="true", + default="true", + help="Include MongoDB in backup (default: true)" +) +componentsArgGroup.add_argument( + '--exclude-mongo', + dest='include_mongo', + required=False, + action="store_const", + const="false", + help="Exclude MongoDB from backup (use if MongoDB is external)" +) + componentsArgGroup.add_argument( '--include-grafana', required=False, diff --git a/tekton/src/params/backup.yml.j2 b/tekton/src/params/backup.yml.j2 index d461392f8e0..ebccfb3f74a 100644 --- a/tekton/src/params/backup.yml.j2 +++ b/tekton/src/params/backup.yml.j2 @@ -99,6 +99,11 @@ description: Set to false to skip Grafana install default: "true" +- name: include_mongo + type: string + description: Set to false to skip Mongo backup/restore (if Mongo is external) + default: "true" + - name: include_sls type: string description: Set to false to skip SLS backup/restore (if SLS is external) diff --git a/tekton/src/pipelines/mas-backup.yml.j2 b/tekton/src/pipelines/mas-backup.yml.j2 index 9d6ce4610dd..44f57b4a2a5 100644 --- a/tekton/src/pipelines/mas-backup.yml.j2 +++ b/tekton/src/pipelines/mas-backup.yml.j2 @@ -153,6 +153,11 @@ spec: - name: mongodb_provider value: $(params.mongodb_provider) + when: + - input: "$(params.include_mongo)" + operator: in + values: ["true", "True"] + taskRef: kind: Task name: mas-devops-mongodb diff --git a/tekton/src/pipelines/mas-restore.yml.j2 b/tekton/src/pipelines/mas-restore.yml.j2 index d974bca6680..ab7495de7bc 100644 --- a/tekton/src/pipelines/mas-restore.yml.j2 +++ b/tekton/src/pipelines/mas-restore.yml.j2 @@ -95,6 +95,8 @@ spec: value: $(params.mas_instance_id) # Archive specific + - name: include_mongo_archive + value: $(params.include_mongo) - name: include_sls_archive value: $(params.include_sls) - name: include_manage_db_archive @@ -244,6 +246,11 @@ spec: value: $(params.mongodb_storageclass_name) - name: mas_config_dir value: /workspace/backups/configs + + when: + - input: "$(params.include_mongo)" + operator: in + values: ["true", "True"] taskRef: kind: Task diff --git a/tekton/src/tasks/download-backup-archive.yml.j2 b/tekton/src/tasks/download-backup-archive.yml.j2 index e3ee3256ab7..45ca152b70f 100644 --- a/tekton/src/tasks/download-backup-archive.yml.j2 +++ b/tekton/src/tasks/download-backup-archive.yml.j2 @@ -21,6 +21,10 @@ spec: description: Instance ID # Archives specific parameters + - name: include_mongo_archive + type: string + description: Download Mongo archive + default: "false" - name: include_sls_archive type: string description: Download SLS archive @@ -89,6 +93,8 @@ spec: value: $(params.backup_archive_name) # Archive specific + - name: INCLUDE_MONGO_ARCHIVE + value: $(params.include_mongo_archive) - name: INCLUDE_SLS_ARCHIVE value: $(params.include_sls_archive) - name: INCLUDE_MANAGE_DB_ARCHIVE