diff --git a/docs/BACKUP.md b/docs/BACKUP.md index 4421394..1fffe10 100644 --- a/docs/BACKUP.md +++ b/docs/BACKUP.md @@ -94,11 +94,17 @@ storage: bucket: some_bucket backup_path: some/path/to/backups/ profile: services + aws_access_key_id: your-access-key-id + aws_secret_access_key: your-secret-access-key + aws_default_region: us-east-1 ``` Here, the meaning of different `storage` fields is as follows: - bucket - S3 storage bucket containing the backups - backup_path - absolute path within the S3 bucket which leads to the root directory where the backups should be saved - profile (optional) - AWS profile to be used +- aws_access_key_id (optional) - AWS access key ID to be used +- aws_secret_access_key (optional) - AWS secret access key to be used +- aws_default_region (optional) - AWS region to be used ### Local Storage diff --git a/docs/RESTORE.md b/docs/RESTORE.md index 32fab50..60b224f 100644 --- a/docs/RESTORE.md +++ b/docs/RESTORE.md @@ -50,12 +50,18 @@ storage_type: s3 storage: bucket: some_bucket backup_path: some/path/to/backups/gd_org_id/ - profile: services + profile: services + aws_access_key_id: your-access-key-id + aws_secret_access_key: your-secret-access-key + aws_default_region: us-east-1 ``` Here, the meaning of different `storage` fields is as follows: - bucket - S3 storage bucket containing the backups - backup_path - absolute path within the S3 bucket which leads to the root directory of the backups (the input csv file defines sources from here) - profile (optional) - AWS profile to be used +- aws_access_key_id (optional) - AWS access key ID to be used +- aws_secret_access_key (optional) - AWS secret access key to be used +- aws_default_region (optional) - AWS region to be used ## Input CSV file (ws_csv) diff --git a/scripts/backup.py b/scripts/backup.py index bd9b618..35ded04 100644 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -63,8 +63,7 @@ def export(self, folder, org_id): class S3Storage(BackupStorage): def __init__(self, conf: BackupRestoreConfig): self._config = conf.storage - self._profile = self._config.get("profile", "default") - self._session = self._create_boto_session(self._profile) + self._session = self._create_boto_session(self._config) self._resource = self._session.resource("s3") self._bucket = self._resource.Bucket(self._config["bucket"]) # type: ignore [missing library stubs] suffix = "/" if not self._config["backup_path"].endswith("/") else "" @@ -73,12 +72,25 @@ def __init__(self, conf: BackupRestoreConfig): self._verify_connection() @staticmethod - def _create_boto_session(profile: str) -> boto3.Session: + def _create_boto_session(config: dict[str, str]) -> boto3.Session: + if config.get("aws_access_key_id") and config.get("aws_secret_access_key"): + if not config.get("aws_default_region"): + logger.warning("No AWS region specified. Defaulting to us-east-1.") + try: + return boto3.Session( + aws_access_key_id=config["aws_access_key_id"], + aws_secret_access_key=config["aws_secret_access_key"], + region_name=config["aws_default_region"], + ) + except Exception: + logger.warning( + "Failed to create boto3 session with supplied credentials. Falling back to profile..." + ) try: - return boto3.Session(profile_name=profile) + return boto3.Session(profile_name=config.get("profile")) except Exception: logger.warning( - 'AWS profile "[default]" not found. Trying other fallback methods...' + f'AWS profile "{config.get("profile")}" not found. Trying other fallback methods...' ) return boto3.Session() diff --git a/scripts/restore.py b/scripts/restore.py index f1e774c..73e6db1 100644 --- a/scripts/restore.py +++ b/scripts/restore.py @@ -82,6 +82,9 @@ def __init__(self, storconf: dict[str, Any]): suffix = "/" if not storconf["backup_path"].endswith("/") else "" self.backup_path: str = storconf["backup_path"] + suffix self.profile = storconf.get("profile", "default") + self.aws_access_key_id = storconf.get("aws_access_key_id") + self.aws_secret_access_key = storconf.get("aws_secret_access_key") + self.aws_default_region = storconf.get("aws_default_region") class S3Storage(BackupStorage): @@ -91,18 +94,31 @@ class S3Storage(BackupStorage): def __init__(self, conf: BackupRestoreConfig): self._config = S3StorageConfig(conf.storage) - self._session = self._create_boto_session(self._config.profile) + self._session = self._create_boto_session(self._config) self._api = self._session.resource("s3") self._bucket = self._api.Bucket(self._config.bucket) self._validate_backup_path() @staticmethod - def _create_boto_session(profile: str) -> boto3.Session: + def _create_boto_session(config: S3StorageConfig) -> boto3.Session: + if config.aws_access_key_id and config.aws_secret_access_key: + if not config.aws_default_region: + logger.warning("No AWS region specified. Defaulting to us-east-1.") + try: + return boto3.Session( + aws_access_key_id=config.aws_access_key_id, + aws_secret_access_key=config.aws_secret_access_key, + region_name=config.aws_default_region, + ) + except Exception: + logger.warning( + "Failed to create boto3 session with supplied credentials. Falling back to profile..." + ) try: - return boto3.Session(profile_name=profile) + return boto3.Session(profile_name=config.profile) except Exception: logger.warning( - 'AWS profile "[default]" not found. Trying other fallback methods...' + f'AWS profile "{config.profile}" not found. Trying other fallback methods...' ) return boto3.Session()