From 519280ea8a3f6ecf774a03f62e6020401c9a6c6e Mon Sep 17 00:00:00 2001 From: ntkathole Date: Fri, 25 Nov 2022 14:34:53 +0530 Subject: [PATCH] Added support to read multiple accounts --- .gitignore | 1 + cloudwash/client.py | 37 +---- cloudwash/config.py | 26 ++-- cloudwash/providers/aws.py | 211 ++++++++++++++------------- cloudwash/providers/azure.py | 275 ++++++++++++++++++++--------------- cloudwash/providers/gce.py | 89 +++++++----- cloudwash/utils.py | 15 +- settings.yaml.template | 3 + 8 files changed, 360 insertions(+), 297 deletions(-) diff --git a/.gitignore b/.gitignore index 06eb78e06..64a5dc26c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ settings.yaml .idea .cache/ .eggs +.env/ *.egg-info **/cleanup*.log *.pyc diff --git a/cloudwash/client.py b/cloudwash/client.py index 3db7f46d9..f396ee70a 100644 --- a/cloudwash/client.py +++ b/cloudwash/client.py @@ -1,44 +1,19 @@ -import json from contextlib import contextmanager -import wrapanapi - -from cloudwash.config import settings - @contextmanager -def compute_client(compute_resource, **kwargs): +def compute_client(compute_resource, client=None): """The context manager for compute resource client to initiate and disconnect :param str compute_resource: The compute resource name + :param obj client: Client object """ - if compute_resource == "azure": - client = wrapanapi.AzureSystem( - username=settings.azure.auth.client_id, - password=settings.azure.auth.secret_id, - tenant_id=settings.azure.auth.tenant_id, - subscription_id=settings.azure.auth.subscription_id, - provisioning={ - "resource_group": kwargs['resource_group'], - "template_container": None, - "region_api": kwargs['azure_region'], - }, - ) - elif compute_resource == "gce": - client = wrapanapi.GoogleCloudSystem( - project=settings.gce.auth.project_id, - service_account=json.loads(settings.gce.auth.service_account), - ) - elif compute_resource == "aws": - client = wrapanapi.EC2System( - username=settings.aws.auth.access_key, - password=settings.aws.auth.secret_key, - region=kwargs['aws_region'], - ) + supported_providers = ["azure", "gce", "aws"] + if compute_resource in supported_providers: + client = client else: raise ValueError( - f"{compute_resource} is an incorrect value. It should be one of azure or gce or ec2" + f"{compute_resource} is an incorrect value. It should be one of {supported_providers}" ) - try: yield client finally: diff --git a/cloudwash/config.py b/cloudwash/config.py index aa6ad0e2d..b8aef07e1 100644 --- a/cloudwash/config.py +++ b/cloudwash/config.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from pathlib import PurePath @@ -21,12 +22,19 @@ def validate_provider(provider_name): provider = provider_name.upper() - provider_settings = [ - f"{provider}.{setting_key}" for setting_key in settings.to_dict().get(provider) - ] - settings.validators.register(Validator(*provider_settings, ne=None)) - try: - settings.validators.validate() - logger.info(f"The {provider} providers settings are initialized and validated !") - except Exception: - raise + provider_settings = settings.to_dict().get(provider) + if not provider_settings: + logging.error("No provider settings found! Please check settings file!") + exit() + for account in provider_settings: + account_settings = [f"{account}.{account_key}" for account_key in provider_settings] + settings.validators.register(Validator(*account_settings, ne=None)) + try: + settings.validators.validate() + logger.info( + f"The {provider} provider settings for account {account['NAME']}" + f" are initialized and validated!" + ) + except Exception as e: + logging.error(f"Settings validation failed for {provider} account {account['NAME']}") + raise e diff --git a/cloudwash/providers/aws.py b/cloudwash/providers/aws.py index ef7f0b557..9be1b0eb4 100644 --- a/cloudwash/providers/aws.py +++ b/cloudwash/providers/aws.py @@ -1,9 +1,12 @@ """ec2 CR Cleanup Utilities""" +import wrapanapi + from cloudwash.client import compute_client from cloudwash.config import settings from cloudwash.logger import logger from cloudwash.utils import dry_data from cloudwash.utils import echo_dry +from cloudwash.utils import pprint from cloudwash.utils import total_running_time @@ -11,110 +14,122 @@ def cleanup(**kwargs): is_dry_run = kwargs["dry_run"] data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS'] - regions = settings.aws.auth.regions - if "all" in regions: - with compute_client("aws", aws_region="us-west-2") as client: - regions = client.list_regions() - for region in regions: - dry_data['VMS']['stop'] = [] - dry_data['VMS']['skip'] = [] - for items in data: - dry_data[items]['delete'] = [] - with compute_client("aws", aws_region=region) as aws_client: - # Dry Data Collection Defs - def dry_vms(): - all_vms = aws_client.list_vms() - for vm in all_vms: - if vm.name in settings.aws.exceptions.vm.vm_list: - dry_data["VMS"]["skip"].append(vm.name) - continue - elif total_running_time(vm).minutes >= settings.aws.criteria.vm.sla_minutes: - if vm.name in settings.aws.exceptions.vm.stop_list: - dry_data["VMS"]["stop"].append(vm.name) + accounts = settings.aws + for account in accounts: + pprint("green", f"\nCloudwash execution for account: {account['NAME']}") + regions = account.auth.regions + if "all" in regions: + client = wrapanapi.EC2System( + username=account.auth.access_key, + password=account.auth.secret_key, + region="us-west-2", + ) + with compute_client("aws", client=client) as client: + regions = client.list_regions() + for region in regions: + client = wrapanapi.EC2System( + username=account.auth.access_key, + password=account.auth.secret_key, + region=region, + ) + dry_data['VMS']['stop'] = [] + dry_data['VMS']['skip'] = [] + for items in data: + dry_data[items]['delete'] = [] + with compute_client("aws", client=client) as aws_client: + # Dry Data Collection Defs + def dry_vms(): + all_vms = aws_client.list_vms() + for vm in all_vms: + if vm.name in account.exceptions.vm.vm_list: + dry_data["VMS"]["skip"].append(vm.name) continue - elif vm.name.startswith(settings.aws.criteria.vm.delete_vm): - dry_data["VMS"]["delete"].append(vm.name) - return dry_data["VMS"] + elif total_running_time(vm).minutes >= account.criteria.vm.sla_minutes: + if vm.name in account.exceptions.vm.stop_list: + dry_data["VMS"]["stop"].append(vm.name) + continue + elif vm.name.startswith(account.criteria.vm.delete_vm): + dry_data["VMS"]["delete"].append(vm.name) + return dry_data["VMS"] - def dry_nics(): - rnics = [] - if settings.aws.criteria.nic.unassigned: - rnics = aws_client.get_all_unused_network_interfaces() - [ - dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"]) - for dnic in rnics - ] - return rnics + def dry_nics(): + rnics = [] + if account.criteria.nic.unassigned: + rnics = aws_client.get_all_unused_network_interfaces() + [ + dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"]) + for dnic in rnics + ] + return rnics - def dry_discs(): - rdiscs = [] - if settings.aws.criteria.disc.unassigned: - rdiscs = aws_client.get_all_unattached_volumes() - [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs] - return rdiscs + def dry_discs(): + rdiscs = [] + if account.criteria.disc.unassigned: + rdiscs = aws_client.get_all_unattached_volumes() + [dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs] + return rdiscs - def dry_pips(): - rpips = [] - if settings.aws.criteria.public_ip.unassigned: - rpips = aws_client.get_all_disassociated_addresses() - [dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips] - return rpips + def dry_pips(): + rpips = [] + if account.criteria.public_ip.unassigned: + rpips = aws_client.get_all_disassociated_addresses() + [dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips] + return rpips - def dry_stacks(): - rstacks = [] - [ - rstacks.append(stack.name) - for stack in aws_client.list_stacks() - if ( - total_running_time(stack).minutes - >= settings.aws.criteria.stacks.sla_minutes - and stack.name.startswith(settings.aws.criteria.stacks.delete_stack) - ) - and stack.name not in settings.aws.exceptions.stacks.stack_list - ] - [dry_data['STACKS']['delete'].append(stack) for stack in rstacks] + def dry_stacks(): + rstacks = [] + [ + rstacks.append(stack.name) + for stack in aws_client.list_stacks() + if ( + total_running_time(stack).minutes >= account.criteria.stacks.sla_minutes + and stack.name.startswith(account.criteria.stacks.delete_stack) + ) + and stack.name not in account.exceptions.stacks.stack_list + ] + [dry_data['STACKS']['delete'].append(stack) for stack in rstacks] - return rstacks + return rstacks - # Remove / Stop VMs - def remove_vms(avms): - # Remove VMs - [aws_client.get_vm(vm_name).delete() for vm_name in avms["delete"]] - # Stop VMs - [aws_client.get_vm(vm_name).stop() for vm_name in avms["stop"]] + # Remove / Stop VMs + def remove_vms(avms): + # Remove VMs + [aws_client.get_vm(vm_name).delete() for vm_name in avms["delete"]] + # Stop VMs + [aws_client.get_vm(vm_name).stop() for vm_name in avms["stop"]] - # Delete CloudFormations - def remove_stacks(stacks): - [aws_client.get_stack(stack_name).delete() for stack_name in stacks] + # Delete CloudFormations + def remove_stacks(stacks): + [aws_client.get_stack(stack_name).delete() for stack_name in stacks] - # Actual Cleaning and dry execution - logger.info(f"\nResources from the region: {region}") - if kwargs["vms"] or kwargs["_all"]: - avms = dry_vms() - if not is_dry_run: - remove_vms(avms=avms) - logger.info(f"Stopped VMs: \n{avms['stop']}") - logger.info(f"Removed VMs: \n{avms['delete']}") - logger.info(f"Skipped VMs: \n{avms['skip']}") - if kwargs["nics"] or kwargs["_all"]: - rnics = dry_nics() - if not is_dry_run and rnics: - aws_client.remove_all_unused_nics() - logger.info(f"Removed NICs: \n{rnics}") - if kwargs["discs"] or kwargs["_all"]: - rdiscs = dry_discs() - if not is_dry_run and rdiscs: - aws_client.remove_all_unused_volumes() - logger.info(f"Removed Discs: \n{rdiscs}") - if kwargs["pips"] or kwargs["_all"]: - rpips = dry_pips() - if not is_dry_run and rpips: - aws_client.remove_all_unused_ips() - logger.info(f"Removed PIPs: \n{rpips}") - if kwargs["stacks"] or kwargs["_all"]: - rstacks = dry_stacks() - if not is_dry_run: - remove_stacks(stacks=rstacks) - logger.info(f"Removed Stacks: \n{rstacks}") - if is_dry_run: - echo_dry(dry_data) + # Actual Cleaning and dry execution + logger.info(f"\nResources from the region: {region}") + if kwargs["vms"] or kwargs["_all"]: + avms = dry_vms() + if not is_dry_run: + remove_vms(avms=avms) + logger.info(f"Stopped VMs: \n{avms['stop']}") + logger.info(f"Removed VMs: \n{avms['delete']}") + logger.info(f"Skipped VMs: \n{avms['skip']}") + if kwargs["nics"] or kwargs["_all"]: + rnics = dry_nics() + if not is_dry_run and rnics: + aws_client.remove_all_unused_nics() + logger.info(f"Removed NICs: \n{rnics}") + if kwargs["discs"] or kwargs["_all"]: + rdiscs = dry_discs() + if not is_dry_run and rdiscs: + aws_client.remove_all_unused_volumes() + logger.info(f"Removed Discs: \n{rdiscs}") + if kwargs["pips"] or kwargs["_all"]: + rpips = dry_pips() + if not is_dry_run and rpips: + aws_client.remove_all_unused_ips() + logger.info(f"Removed PIPs: \n{rpips}") + if kwargs["stacks"] or kwargs["_all"]: + rstacks = dry_stacks() + if not is_dry_run: + remove_stacks(stacks=rstacks) + logger.info(f"Removed Stacks: \n{rstacks}") + if is_dry_run: + echo_dry(dry_data) diff --git a/cloudwash/providers/azure.py b/cloudwash/providers/azure.py index 6bb5b1db3..f90fa959d 100644 --- a/cloudwash/providers/azure.py +++ b/cloudwash/providers/azure.py @@ -1,4 +1,6 @@ """Azure CR Cleanup Utilities""" +import wrapanapi + from cloudwash.client import compute_client from cloudwash.config import settings from cloudwash.logger import logger @@ -7,7 +9,7 @@ from cloudwash.utils import total_running_time -def _dry_vms(all_vms): +def _dry_vms(all_vms, account): """Filters and returns running VMs to be deleted from all VMs""" _vms = {"stop": [], "delete": [], "skip": []} for vm in all_vms: @@ -19,19 +21,17 @@ def _dry_vms(all_vms): if vm.state.lower() == "vmstate.creating": continue # Match the user defined criteria in settings to delete the VM - if vm.name in settings.azure.exceptions.vm.vm_list: + if vm.name in account.exceptions.vm.vm_list: _vms["skip"].append(vm.name) continue elif ( - getattr( - total_running_time(vm), "minutes", int(settings.azure.criteria.vm.sla_minutes) + 1 - ) - >= settings.azure.criteria.vm.sla_minutes + getattr(total_running_time(vm), "minutes", int(account.criteria.vm.sla_minutes) + 1) + >= account.criteria.vm.sla_minutes ): - if vm.name in settings.azure.exceptions.vm.stop_list: + if vm.name in account.exceptions.vm.stop_list: _vms["stop"].append(vm.name) continue - elif vm.name.startswith(settings.azure.criteria.vm.delete_vm): + elif vm.name.startswith(account.criteria.vm.delete_vm): _vms["delete"].append(vm.name) return _vms @@ -40,117 +40,156 @@ def cleanup(**kwargs): is_dry_run = kwargs["dry_run"] data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES'] - regions = settings.azure.auth.regions - groups = settings.azure.auth.resource_groups - - if "all" in regions: - # non-existent RG can be chosen for query - # as it's never accessed and is only stored within wrapper - with compute_client("azure", azure_region="us-west", resource_group="foo") as azure_client: - regions = list(zip(*azure_client.list_region()))[0] + accounts = settings.azure + for account in accounts: + regions = account.auth.regions + groups = account.auth.resource_groups - for region in regions: - if "all" in groups: + if "all" in regions: # non-existent RG can be chosen for query # as it's never accessed and is only stored within wrapper - with compute_client("azure", azure_region=region, resource_group="foo") as azure_client: - groups = azure_client.list_resource_groups() - - for group in groups: - dry_data['VMS']['stop'] = [] - dry_data['VMS']['skip'] = [] - for items in data: - dry_data[items]['delete'] = [] - - with compute_client("azure", azure_region=region, resource_group=group) as azure_client: - # Dry Data Collection Defs - def dry_vms(): - all_vms = azure_client.list_vms() - dry_data["VMS"] = _dry_vms(all_vms) - return dry_data["VMS"] - - def dry_nics(): - rnics = [] - if settings.azure.criteria.nic.unassigned: - rnics = azure_client.list_free_nics() - [dry_data["NICS"]["delete"].append(dnic) for dnic in rnics] - return rnics - - def dry_discs(): - rdiscs = [] - if settings.azure.criteria.disc.unassigned: - rdiscs = azure_client.list_free_discs() - [dry_data["DISCS"]["delete"].append(ddisc) for ddisc in rdiscs] - return rdiscs - - def dry_pips(): - rpips = [] - if settings.azure.criteria.public_ip.unassigned: - rpips = azure_client.list_free_pip() - [dry_data["PIPS"]["delete"].append(dpip) for dpip in rpips] - return rpips - - def dry_resources(hours_old=None): - dry_data["RESOURCES"]["delete"] = azure_client.list_resources_from_hours_old( - hours_old=hours_old - or (settings.azure.criteria.resource_group.resources_sla_minutes / 60) - ) - return dry_data["RESOURCES"]["delete"] - - # Remove / Stop VMs - def remove_vms(avms): - # Remove VMs - [azure_client.get_vm(vm_name).delete() for vm_name in avms["delete"]] - # Stop VMs - [azure_client.get_vm(vm_name).stop() for vm_name in avms["stop"]] - - # Actual Cleaning and dry execution - logger.info(f"\nResources from the region and resource group: {region}/{group}") - - if kwargs["vms"] or kwargs["_all"]: - avms = dry_vms() - if not is_dry_run: - remove_vms(avms=avms) - logger.info(f"Stopped VMs: \n{avms['stop']}") - logger.info(f"Removed VMs: \n{avms['delete']}") - logger.info(f"Skipped VMs: \n{avms['skip']}") - if kwargs["nics"] or kwargs["_all"]: - rnics = dry_nics() - if not is_dry_run and rnics: - azure_client.remove_nics_by_search() - logger.info(f"Removed NICs: \n{rnics}") - if kwargs["discs"] or kwargs["_all"]: - rdiscs = dry_discs() - if not is_dry_run and rdiscs: - azure_client.remove_discs_by_search() - logger.info(f"Removed Discs: \n{rdiscs}") - if kwargs["pips"] or kwargs["_all"]: - rpips = dry_pips() - if not is_dry_run and rpips: - azure_client.remove_pips_by_search() - logger.info(f"Removed PIPs: \n{rpips}") - if kwargs["_all_rg"]: - sla_time = settings.azure.criteria.resource_group.resources_sla_minutes - - # Exception list has priority - if group in settings.azure.exceptions.group.rg_list: - continue - - if settings.azure.criteria.resource_group.logic == "AND": - if group.startswith(settings.azure.criteria.resource_group.delete_group): - rres = dry_resources(sla_time) - elif settings.azure.criteria.resource_group.logic == "OR": - if group.startswith(settings.azure.criteria.resource_group.delete_group): - sla_time = 0 - - rres = dry_resources(sla_time) - else: - raise Exception("Invalid logic for resource group cleanup") - - if not is_dry_run and rres: - azure_client.remove_resource_group_of_old_resources( - hours_old=(sla_time / 60) + client = wrapanapi.AzureSystem( + username=account.auth.client_id, + password=account.auth.secret_id, + tenant_id=account.auth.tenant_id, + subscription_id=account.auth.subscription_id, + provisioning={ + "resource_group": "foo", + "template_container": None, + "region_api": "us-west", + }, + ) + with compute_client("azure", client=client) as azure_client: + regions = list(zip(*azure_client.list_region()))[0] + + for region in regions: + if "all" in groups: + # non-existent RG can be chosen for query + # as it's never accessed and is only stored within wrapper + client = wrapanapi.AzureSystem( + username=account.auth.client_id, + password=account.auth.secret_id, + tenant_id=account.auth.tenant_id, + subscription_id=account.auth.subscription_id, + provisioning={ + "resource_group": "foo", + "template_container": None, + "region_api": region, + }, + ) + with compute_client("azure", client=client) as azure_client: + groups = azure_client.list_resource_groups() + + for group in groups: + dry_data['VMS']['stop'] = [] + dry_data['VMS']['skip'] = [] + for items in data: + dry_data[items]['delete'] = [] + + client = wrapanapi.AzureSystem( + username=account.auth.client_id, + password=account.auth.secret_id, + tenant_id=account.auth.tenant_id, + subscription_id=account.auth.subscription_id, + provisioning={ + "resource_group": group, + "template_container": None, + "region_api": region, + }, + ) + + with compute_client("azure", client=client) as azure_client: + # Dry Data Collection Defs + def dry_vms(): + all_vms = azure_client.list_vms() + dry_data["VMS"] = _dry_vms(all_vms, account=account) + return dry_data["VMS"] + + def dry_nics(): + rnics = [] + if account.criteria.nic.unassigned: + rnics = azure_client.list_free_nics() + [dry_data["NICS"]["delete"].append(dnic) for dnic in rnics] + return rnics + + def dry_discs(): + rdiscs = [] + if account.criteria.disc.unassigned: + rdiscs = azure_client.list_free_discs() + [dry_data["DISCS"]["delete"].append(ddisc) for ddisc in rdiscs] + return rdiscs + + def dry_pips(): + rpips = [] + if account.criteria.public_ip.unassigned: + rpips = azure_client.list_free_pip() + [dry_data["PIPS"]["delete"].append(dpip) for dpip in rpips] + return rpips + + def dry_resources(hours_old=None): + dry_data["RESOURCES"][ + "delete" + ] = azure_client.list_resources_from_hours_old( + hours_old=hours_old + or (account.criteria.resource_group.resources_sla_minutes / 60) ) - logger.info(f"Removed Resources: \n{rres}") - if is_dry_run: - echo_dry(dry_data) + return dry_data["RESOURCES"]["delete"] + + # Remove / Stop VMs + def remove_vms(avms): + # Remove VMs + [azure_client.get_vm(vm_name).delete() for vm_name in avms["delete"]] + # Stop VMs + [azure_client.get_vm(vm_name).stop() for vm_name in avms["stop"]] + + # Actual Cleaning and dry execution + logger.info(f"\nExecution for account: {account['NAME']}") + logger.info(f"\nResources from the region and resource group: {region}/{group}") + + if kwargs["vms"] or kwargs["_all"]: + avms = dry_vms() + if not is_dry_run: + remove_vms(avms=avms) + logger.info(f"Stopped VMs: \n{avms['stop']}") + logger.info(f"Removed VMs: \n{avms['delete']}") + logger.info(f"Skipped VMs: \n{avms['skip']}") + if kwargs["nics"] or kwargs["_all"]: + rnics = dry_nics() + if not is_dry_run and rnics: + azure_client.remove_nics_by_search() + logger.info(f"Removed NICs: \n{rnics}") + if kwargs["discs"] or kwargs["_all"]: + rdiscs = dry_discs() + if not is_dry_run and rdiscs: + azure_client.remove_discs_by_search() + logger.info(f"Removed Discs: \n{rdiscs}") + if kwargs["pips"] or kwargs["_all"]: + rpips = dry_pips() + if not is_dry_run and rpips: + azure_client.remove_pips_by_search() + logger.info(f"Removed PIPs: \n{rpips}") + if kwargs["_all_rg"]: + sla_time = account.criteria.resource_group.resources_sla_minutes + + # Exception list has priority + if group in account.exceptions.group.rg_list: + continue + + if account.criteria.resource_group.logic == "AND": + if group.startswith(account.criteria.resource_group.delete_group): + rres = dry_resources(sla_time) + elif account.criteria.resource_group.logic == "OR": + if group.startswith(account.criteria.resource_group.delete_group): + sla_time = 0 + + rres = dry_resources(sla_time) + else: + raise Exception("Invalid logic for resource group cleanup") + + if not is_dry_run and rres: + azure_client.remove_resource_group_of_old_resources( + hours_old=(sla_time / 60) + ) + logger.info(f"Removed Resources: \n{rres}") + if is_dry_run: + echo_dry(dry_data) diff --git a/cloudwash/providers/gce.py b/cloudwash/providers/gce.py index 7e3f257f3..9a2dc9571 100644 --- a/cloudwash/providers/gce.py +++ b/cloudwash/providers/gce.py @@ -1,4 +1,8 @@ """GCE CR Cleanup Utilities""" +import json + +import wrapanapi + from cloudwash.client import compute_client from cloudwash.config import settings from cloudwash.logger import logger @@ -11,43 +15,50 @@ def cleanup(**kwargs): is_dry_run = kwargs["dry_run"] - with compute_client("gce") as gce_client: - if kwargs["vms"] or kwargs["_all"]: - allvms = gce_client.list_vms(zones=gce_zones()) - for vm in allvms: - if vm.name in settings.gce.exceptions.vm.vm_list: - dry_data["VMS"]["skip"].append(vm.name) - continue - elif total_running_time(vm).minutes >= settings.gce.criteria.vm.sla_minutes: - if vm.name in settings.gce.exceptions.vm.stop_list: - dry_data["VMS"]["stop"].append(vm.name) - if not is_dry_run: - try: - vm.stop() - except TypeError: - logger.info(f"Stopped VM: {vm.name}") - except Exception: - logger.exception(f"Error stopping VM {vm.name}") + accounts = settings.gce + for account in accounts: + client = wrapanapi.GoogleCloudSystem( + project=account.auth.project_id, + service_account=json.loads(account.auth.service_account), + ) + logger.info(f"\nExecution for account: {account['NAME']}") + with compute_client("gce", client=client) as gce_client: + if kwargs["vms"] or kwargs["_all"]: + allvms = gce_client.list_vms(zones=gce_zones()) + for vm in allvms: + if vm.name in settings.gce.exceptions.vm.vm_list: + dry_data["VMS"]["skip"].append(vm.name) continue - elif vm.name.startswith(settings.gce.criteria.vm.delete_vm): - dry_data["VMS"]["delete"].append(vm.name) - if not is_dry_run: - try: - vm.delete() - # There as an issue with GCE API while deleting/stopping the VM - # That it throws TypeError, hence catching that error here - # Remove it once its fixed - except TypeError: - logger.info(f"Deleted VM: {vm.name}") - except Exception: - logger.exception(f"Error deleting VM {vm.name}") - if kwargs["nics"] or kwargs["_all"]: - logger.warning( - "Cloudwash dependency 'WrapanAPI' does not supports NICs operation for GCE yet!" - ) - if kwargs["discs"] or kwargs["_all"]: - logger.warning( - "Cloudwash dependency 'WrapanAPI' does not supports DISCs operation for GCE yet!" - ) - if is_dry_run: - echo_dry(dry_data) + elif total_running_time(vm).minutes >= settings.gce.criteria.vm.sla_minutes: + if vm.name in settings.gce.exceptions.vm.stop_list: + dry_data["VMS"]["stop"].append(vm.name) + if not is_dry_run: + try: + vm.stop() + except TypeError: + logger.info(f"Stopped VM: {vm.name}") + except Exception: + logger.exception(f"Error stopping VM {vm.name}") + continue + elif vm.name.startswith(settings.gce.criteria.vm.delete_vm): + dry_data["VMS"]["delete"].append(vm.name) + if not is_dry_run: + try: + vm.delete() + # There as an issue with GCE API while deleting/stopping the VM + # That it throws TypeError, hence catching that error here + # Remove it once its fixed + except TypeError: + logger.info(f"Deleted VM: {vm.name}") + except Exception: + logger.exception(f"Error deleting VM {vm.name}") + if kwargs["nics"] or kwargs["_all"]: + logger.warning( + "Cloudwash dependency 'WrapanAPI' does not supports NICs operation for GCE yet!" + ) + if kwargs["discs"] or kwargs["_all"]: + logger.warning( + "Cloudwash dependency 'WrapanAPI' does not supports DISCs operation for GCE yet!" # noqa + ) + if is_dry_run: + echo_dry(dry_data) diff --git a/cloudwash/utils.py b/cloudwash/utils.py index 30b2277d9..6c6c2725c 100644 --- a/cloudwash/utils.py +++ b/cloudwash/utils.py @@ -23,7 +23,7 @@ def echo_dry(dry_data=None) -> None: :param dict dry_data: The deletable resources dry data of a Compute Resource, it follows the format of module scoped `dry_data` variable in this module """ - logger.info("\n=========== DRY SUMMARY ============\n") + pprint("blue", "\n=========== DRY SUMMARY ============\n") deletable_vms = dry_data["VMS"]["delete"] stopable_vms = dry_data["VMS"]["stop"] skipped_vms = dry_data["VMS"]["skip"] @@ -58,7 +58,7 @@ def echo_dry(dry_data=None) -> None: deletable_stacks, ] ): - logger.info("\nNo resources are eligible for cleanup!") + pprint("yellow", "\nNo resources are eligible for cleanup!") logger.info("\n====================================\n") @@ -107,3 +107,14 @@ def gce_zones() -> list: _zones_combo = {**_bcds, **_abcfs, **_abcs} zones = [f"{loc}-{zone}" for loc, zones in _zones_combo.items() for zone in zones] return zones + + +def pprint(color, data): + colors = { + "blue": "\033[94m", + "green": "\033[92m", + "yellow": "\033[93m", + "red": "\033[91m", + "ENDC": "\033[0m", + } + print(f"{colors[color]}{data}{colors['ENDC']}") diff --git a/settings.yaml.template b/settings.yaml.template index 56a785089..785bd9c85 100644 --- a/settings.yaml.template +++ b/settings.yaml.template @@ -1,6 +1,7 @@ --- # Compute Resources and there credentials to connect to them for cleanup GCE: + - NAME: gce-account-1 AUTH: PROJECT_ID: # Multiline json contents from Service Account key @@ -24,6 +25,7 @@ GCE: STOP_LIST: [] AZURE: + - NAME: azure-account-1 AUTH: CLIENT_ID: SECRET_ID: @@ -63,6 +65,7 @@ AZURE: RG_LIST: [] AWS: + - NAME: aws-account-1 AUTH: ACCESS_KEY: SECRET_KEY: