From 48ee6b8337485ccb17bb4598b37acc1c6b13e08a Mon Sep 17 00:00:00 2001 From: Victor Shih Date: Thu, 15 Jun 2023 18:33:11 -0700 Subject: [PATCH 1/5] Drop Python 2.7+ requirement. --- setup.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.py b/setup.py index de76382..562550a 100644 --- a/setup.py +++ b/setup.py @@ -17,14 +17,8 @@ setuptools install script for Graffiti Monkey """ -import sys -major, minor = sys.version_info[0:2] -if major != 2 or minor < 7: - print 'Graffiti Monkey requires Python 2.7.x' - sys.exit(1) - from setuptools import setup, find_packages - + import graffiti_monkey with open('requirements.txt') as fh: From eca18d65d762a5293b4a929429e15d219b59ce3a Mon Sep 17 00:00:00 2001 From: Victor Shih Date: Thu, 15 Jun 2023 22:46:38 -0700 Subject: [PATCH 2/5] Python 3 compatibility. --- graffiti_monkey/cli.py | 6 +++--- graffiti_monkey/core.py | 26 ++++++++++++++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/graffiti_monkey/cli.py b/graffiti_monkey/cli.py index 2d51186..1198635 100644 --- a/graffiti_monkey/cli.py +++ b/graffiti_monkey/cli.py @@ -65,7 +65,7 @@ def set_cli_args(self): help='the region to tag things in (default is current region of EC2 instance this is running on). E.g. us-east-1') parser.add_argument('--profile', metavar='PROFILE', help='the profile (credentials) to use to connect to EC2') - parser.add_argument('--verbose', '-v', action='count', + parser.add_argument('--verbose', '-v', action='count', default=0, help='enable verbose output (-vvv for more)') parser.add_argument('--version', action='version', version='%(prog)s ' + __version__, help='display version number and exit') @@ -95,14 +95,14 @@ def set_config(self): if self.args.config: try: import yaml - except: + from yaml import CLoader log.error("When the config parameter is used, you need to have the python PyYAML library.") log.error("It can be installed with pip `pip install PyYAML`.") sys.exit(5) try: #TODO: take default values and these can be overwritten by config - self.config = yaml.load(self.args.config) + self.config = yaml.load(self.args.config, Loader=CLoader) if self.config is None: self.fail_due_to_bad_config_file() except: diff --git a/graffiti_monkey/core.py b/graffiti_monkey/core.py index 6d562f4..eac7ed0 100644 --- a/graffiti_monkey/core.py +++ b/graffiti_monkey/core.py @@ -161,19 +161,22 @@ def tag_volumes(self): log.debug('Skipping %s as it is not attached to an EC2 instance, so there is nothing to propagate', volume.id) continue + err = None for attempt in range(5): try: self.tag_volume(volume, instances) - except boto.exception.EC2ResponseError, e: + except boto.exception.EC2ResponseError as e: log.error("Encountered Error %s on volume %s", e.error_code, volume.id) + err = e break - except boto.exception.BotoServerError, e: + except boto.exception.BotoServerError as e: log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume.id, attempt) + err = e time.sleep(attempt) else: break else: - log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume.id, attempt) + log.error("Encountered Error %s on volume %s, %d retries failed, continuing", err.error_code, volume.id, attempt) continue log.info('Processed a total of {0} GB of AWS Volumes'.format(storage_counter)) @@ -240,7 +243,7 @@ def tag_snapshots(self, volumes): for snapshot_id in self._snapshots_to_tag: if snapshot_id not in snapshot_ids: log.info('Snapshot %s does not exist and will not be tagged', snapshot_id) - self._snapshots_to_tag.remove(snapshot) + self._snapshots_to_tag.remove(snapshot_id) else: log.info('Getting list of all snapshots') snapshots = self._conn.get_all_snapshots(owner='self') @@ -268,19 +271,22 @@ def tag_snapshots(self, volumes): for snapshot in snapshots: this_snap += 1 log.info ('Processing snapshot %d of %d total snapshots', this_snap, total_snaps) + err = None for attempt in range(5): try: self.tag_snapshot(snapshot, volumes) - except boto.exception.EC2ResponseError, e: + except boto.exception.EC2ResponseError as e: log.error("Encountered Error %s on snapshot %s", e.error_code, snapshot.id) + err = e break - except boto.exception.BotoServerError, e: + except boto.exception.BotoServerError as e: log.error("Encountered Error %s on snapshot %s, waiting %d seconds then retrying", e.error_code, snapshot.id, attempt) + err = e time.sleep(attempt) else: break else: - log.error("Encountered Error %s on snapshot %s, %d retries failed, continuing", e.error_code, snapshot.id, attempt) + log.error("Encountered Error %s on snapshot %s, %d retries failed, continuing", err.error_code, snapshot.id, attempt) continue log.info('Completed processing all snapshots') @@ -324,8 +330,8 @@ def _set_resource_tags(self, resource, tags): delta_tags = {} - for tag_key, tag_value in tags.iteritems(): - if not tag_key in resource.tags or resource.tags[tag_key] != tag_value: + for tag_key, tag_value in tags.items(): + if tag_key not in resource.tags or resource.tags[tag_key] != tag_value: delta_tags[tag_key] = tag_value if len(delta_tags) == 0: @@ -341,7 +347,7 @@ class Logging(object): _log_simple_format = '%(asctime)s [%(levelname)s] %(message)s' _log_detailed_format = '%(asctime)s [%(levelname)s] [%(name)s(%(lineno)s):%(funcName)s] %(message)s' - def configure(self, verbosity = None): + def configure(self, verbosity): ''' Configure the logging format and verbosity ''' # Configure our logging output From b9378e648c5d836031769ad56a134070fdbc7049 Mon Sep 17 00:00:00 2001 From: Victor Shih Date: Thu, 15 Jun 2023 22:46:48 -0700 Subject: [PATCH 3/5] PEP8 adherence, allowances. --- graffiti_monkey/cli.py | 11 ++++++----- graffiti_monkey/core.py | 32 ++++++++++++++++---------------- tox.ini | 8 ++++++++ 3 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 tox.ini diff --git a/graffiti_monkey/cli.py b/graffiti_monkey/cli.py index 1198635..7c924b3 100644 --- a/graffiti_monkey/cli.py +++ b/graffiti_monkey/cli.py @@ -70,7 +70,7 @@ def set_cli_args(self): parser.add_argument('--version', action='version', version='%(prog)s ' + __version__, help='display version number and exit') parser.add_argument('--config', '-c', nargs="?", type=argparse.FileType('r'), - default=None, help="Give a yaml configuration file") + default=None, help="Give a yaml configuration file") parser.add_argument('--dryrun', action='store_true', help='dryrun only, display tagging actions but do not perform them') parser.add_argument('--append', action='store_true', @@ -88,24 +88,25 @@ def set_cli_args(self): @staticmethod def fail_due_to_bad_config_file(self): self._fail("Something went wrong reading the passed yaml config file. " - "Make sure to use valid yaml syntax. " - "Also the start of the file should not be marked with '---'.", 6) + "Make sure to use valid yaml syntax. " + "Also the start of the file should not be marked with '---'.", 6) def set_config(self): if self.args.config: try: import yaml from yaml import CLoader + except ImportError: log.error("When the config parameter is used, you need to have the python PyYAML library.") log.error("It can be installed with pip `pip install PyYAML`.") sys.exit(5) try: - #TODO: take default values and these can be overwritten by config + # TODO: take default values and these can be overwritten by config self.config = yaml.load(self.args.config, Loader=CLoader) if self.config is None: self.fail_due_to_bad_config_file() - except: + except yaml.YAMLError: self.fail_due_to_bad_config_file() diff --git a/graffiti_monkey/core.py b/graffiti_monkey/core.py index eac7ed0..d6eaf51 100644 --- a/graffiti_monkey/core.py +++ b/graffiti_monkey/core.py @@ -14,7 +14,7 @@ import logging -from exceptions import * +from .exceptions import GraffitiMonkeyException import boto from boto import ec2 @@ -91,7 +91,7 @@ def propagate_tags(self): if not self._novolumes: volumes = self.tag_volumes() - volumes = { v.id: v for v in volumes } + volumes = {v.id: v for v in volumes} if not self._nosnapshots: self.tag_snapshots(volumes) @@ -101,23 +101,23 @@ def tag_volumes(self): them ''' storage_counter = 0 - volumes = [] + volumes = [] instances = {} if self._volumes_to_tag: log.info('Using volume list from cli/config file') # Max of 200 filters in a request - for chunk in (self._volumes_to_tag[n:n+200] for n in xrange(0, len(self._volumes_to_tag), 200)): + for chunk in (self._volumes_to_tag[n:n + 200] for n in range(0, len(self._volumes_to_tag), 200)): chunk_volumes = self._conn.get_all_volumes( - filters = { 'volume-id': chunk } - ) + filters={'volume-id': chunk} + ) volumes += chunk_volumes chunk_instance_ids = set(v.attach_data.instance_id for v in chunk_volumes) reservations = self._conn.get_all_instances( - filters = {'instance-id': [id for id in chunk_instance_ids]} - ) + filters={'instance-id': chunk_instance_ids} + ) for reservation in reservations: for instance in reservation.instances: instances[instance.id] = instance @@ -155,7 +155,7 @@ def tag_volumes(self): for volume in volumes: this_vol += 1 storage_counter += volume.size - log.info ('Processing volume %d of %d total volumes', this_vol, total_vols) + log.info('Processing volume %d of %d total volumes', this_vol, total_vols) if volume.status != 'in-use': log.debug('Skipping %s as it is not attached to an EC2 instance, so there is nothing to propagate', volume.id) @@ -231,10 +231,10 @@ def tag_snapshots(self, volumes): log.info('Using snapshot list from cli/config file') # Max of 200 filters in a request - for chunk in (self._snapshots_to_tag[n:n+200] for n in xrange(0, len(self._snapshots_to_tag), 200)): + for chunk in (self._snapshots_to_tag[n:n + 200] for n in range(0, len(self._snapshots_to_tag), 200)): chunk_snapshots = self._conn.get_all_snapshots( - filters = { 'snapshot-id': chunk } - ) + filters={'snapshot-id': chunk} + ) snapshots += chunk_snapshots snapshot_ids = [s.id for s in snapshots] @@ -256,10 +256,10 @@ def tag_snapshots(self, volumes): extra_volume_ids = [id for id in all_volume_ids if id not in volumes] ''' Fetch any extra volumes that weren't carried over from tag_volumes() (if any) ''' - for chunk in (extra_volume_ids[n:n+200] for n in xrange(0, len(extra_volume_ids), 200)): + for chunk in (extra_volume_ids[n:n + 200] for n in range(0, len(extra_volume_ids), 200)): extra_volumes = self._conn.get_all_volumes( - filters = { 'volume-id': chunk } - ) + filters={'volume-id': chunk} + ) for vol in extra_volumes: volumes[vol.id] = vol @@ -270,7 +270,7 @@ def tag_snapshots(self, volumes): for snapshot in snapshots: this_snap += 1 - log.info ('Processing snapshot %d of %d total snapshots', this_snap, total_snaps) + log.info('Processing snapshot %d of %d total snapshots', this_snap, total_snaps) err = None for attempt in range(5): try: diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..44e5f12 --- /dev/null +++ b/tox.ini @@ -0,0 +1,8 @@ + +[flake8] +ignore = + # Line too long. + E501, + # Too many blank lines. + E303, + From 1fa513ae583ee9b010224ca5e5d0790e12e5f9bf Mon Sep 17 00:00:00 2001 From: Victor Shih Date: Fri, 16 Jun 2023 01:18:54 -0700 Subject: [PATCH 4/5] Drop incorrect @staticmethod decoration. --- graffiti_monkey/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/graffiti_monkey/cli.py b/graffiti_monkey/cli.py index 7c924b3..31adb9d 100644 --- a/graffiti_monkey/cli.py +++ b/graffiti_monkey/cli.py @@ -85,7 +85,6 @@ def set_cli_args(self): help='do not perform snapshot tagging') self.args = parser.parse_args(self.get_argv()) - @staticmethod def fail_due_to_bad_config_file(self): self._fail("Something went wrong reading the passed yaml config file. " "Make sure to use valid yaml syntax. " From dd907bb521ce4e062f968ed916804341812c6b5c Mon Sep 17 00:00:00 2001 From: Victor Shih Date: Sun, 18 Jun 2023 01:02:13 -0700 Subject: [PATCH 5/5] End gracefully if no volumes found. --- graffiti_monkey/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graffiti_monkey/core.py b/graffiti_monkey/core.py index d6eaf51..67169a0 100644 --- a/graffiti_monkey/core.py +++ b/graffiti_monkey/core.py @@ -146,7 +146,7 @@ def tag_volumes(self): if not volumes: log.info('No volumes found') - return True + return volumes log.debug('Volume list >%s<', volumes) total_vols = len(volumes)