Skip to content

Upgrade to Python 3. #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vshih wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Upgrade to Python 3. #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
48ee6b8
Drop Python 2.7+ requirement.
vshih Jun 16, 2023
eca18d6
Python 3 compatibility.
vshih Jun 16, 2023
b9378e6
PEP8 adherence, allowances.
vshih Jun 16, 2023
1fa513a
Drop incorrect @staticmethod decoration.
vshih Jun 16, 2023
dd907bb
End gracefully if no volumes found.
vshih Jun 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions graffiti_monkey/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,12 @@ def set_cli_args(self):
help='the region to tag things in (default is current region of EC2 instance this is running on). E.g. us-east-1')
parser.add_argument('--profile', metavar='PROFILE',
help='the profile (credentials) to use to connect to EC2')
parser.add_argument('--verbose', '-v', action='count',
parser.add_argument('--verbose', '-v', action='count', default=0,
help='enable verbose output (-vvv for more)')
parser.add_argument('--version', action='version', version='%(prog)s ' + __version__,
help='display version number and exit')
parser.add_argument('--config', '-c', nargs="?", type=argparse.FileType('r'),
default=None, help="Give a yaml configuration file")
default=None, help="Give a yaml configuration file")
parser.add_argument('--dryrun', action='store_true',
help='dryrun only, display tagging actions but do not perform them')
parser.add_argument('--append', action='store_true',
Expand All @@ -85,27 +85,27 @@ def set_cli_args(self):
help='do not perform snapshot tagging')
self.args = parser.parse_args(self.get_argv())

@staticmethod
def fail_due_to_bad_config_file(self):
self._fail("Something went wrong reading the passed yaml config file. "
"Make sure to use valid yaml syntax. "
"Also the start of the file should not be marked with '---'.", 6)
"Make sure to use valid yaml syntax. "
"Also the start of the file should not be marked with '---'.", 6)

def set_config(self):
if self.args.config:
try:
import yaml
except:
from yaml import CLoader
except ImportError:
log.error("When the config parameter is used, you need to have the python PyYAML library.")
log.error("It can be installed with pip `pip install PyYAML`.")
sys.exit(5)

try:
#TODO: take default values and these can be overwritten by config
self.config = yaml.load(self.args.config)
# TODO: take default values and these can be overwritten by config
self.config = yaml.load(self.args.config, Loader=CLoader)
if self.config is None:
self.fail_due_to_bad_config_file()
except:
except yaml.YAMLError:
self.fail_due_to_bad_config_file()


Expand Down
60 changes: 33 additions & 27 deletions graffiti_monkey/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

import logging

from exceptions import *
from .exceptions import GraffitiMonkeyException

import boto
from boto import ec2
Expand Down Expand Up @@ -91,7 +91,7 @@ def propagate_tags(self):
if not self._novolumes:
volumes = self.tag_volumes()

volumes = { v.id: v for v in volumes }
volumes = {v.id: v for v in volumes}

if not self._nosnapshots:
self.tag_snapshots(volumes)
Expand All @@ -101,23 +101,23 @@ def tag_volumes(self):
them '''

storage_counter = 0
volumes = []
volumes = []
instances = {}

if self._volumes_to_tag:
log.info('Using volume list from cli/config file')

# Max of 200 filters in a request
for chunk in (self._volumes_to_tag[n:n+200] for n in xrange(0, len(self._volumes_to_tag), 200)):
for chunk in (self._volumes_to_tag[n:n + 200] for n in range(0, len(self._volumes_to_tag), 200)):
chunk_volumes = self._conn.get_all_volumes(
filters = { 'volume-id': chunk }
)
filters={'volume-id': chunk}
)
volumes += chunk_volumes

chunk_instance_ids = set(v.attach_data.instance_id for v in chunk_volumes)
reservations = self._conn.get_all_instances(
filters = {'instance-id': [id for id in chunk_instance_ids]}
)
filters={'instance-id': chunk_instance_ids}
)
for reservation in reservations:
for instance in reservation.instances:
instances[instance.id] = instance
Expand Down Expand Up @@ -146,7 +146,7 @@ def tag_volumes(self):

if not volumes:
log.info('No volumes found')
return True
return volumes

log.debug('Volume list >%s<', volumes)
total_vols = len(volumes)
Expand All @@ -155,25 +155,28 @@ def tag_volumes(self):
for volume in volumes:
this_vol += 1
storage_counter += volume.size
log.info ('Processing volume %d of %d total volumes', this_vol, total_vols)
log.info('Processing volume %d of %d total volumes', this_vol, total_vols)

if volume.status != 'in-use':
log.debug('Skipping %s as it is not attached to an EC2 instance, so there is nothing to propagate', volume.id)
continue

err = None
for attempt in range(5):
try:
self.tag_volume(volume, instances)
except boto.exception.EC2ResponseError, e:
except boto.exception.EC2ResponseError as e:
log.error("Encountered Error %s on volume %s", e.error_code, volume.id)
err = e
break
except boto.exception.BotoServerError, e:
except boto.exception.BotoServerError as e:
log.error("Encountered Error %s on volume %s, waiting %d seconds then retrying", e.error_code, volume.id, attempt)
err = e
time.sleep(attempt)
else:
break
else:
log.error("Encountered Error %s on volume %s, %d retries failed, continuing", e.error_code, volume.id, attempt)
log.error("Encountered Error %s on volume %s, %d retries failed, continuing", err.error_code, volume.id, attempt)
continue

log.info('Processed a total of {0} GB of AWS Volumes'.format(storage_counter))
Expand Down Expand Up @@ -228,10 +231,10 @@ def tag_snapshots(self, volumes):
log.info('Using snapshot list from cli/config file')

# Max of 200 filters in a request
for chunk in (self._snapshots_to_tag[n:n+200] for n in xrange(0, len(self._snapshots_to_tag), 200)):
for chunk in (self._snapshots_to_tag[n:n + 200] for n in range(0, len(self._snapshots_to_tag), 200)):
chunk_snapshots = self._conn.get_all_snapshots(
filters = { 'snapshot-id': chunk }
)
filters={'snapshot-id': chunk}
)
snapshots += chunk_snapshots
snapshot_ids = [s.id for s in snapshots]

Expand All @@ -240,7 +243,7 @@ def tag_snapshots(self, volumes):
for snapshot_id in self._snapshots_to_tag:
if snapshot_id not in snapshot_ids:
log.info('Snapshot %s does not exist and will not be tagged', snapshot_id)
self._snapshots_to_tag.remove(snapshot)
self._snapshots_to_tag.remove(snapshot_id)
else:
log.info('Getting list of all snapshots')
snapshots = self._conn.get_all_snapshots(owner='self')
Expand All @@ -253,10 +256,10 @@ def tag_snapshots(self, volumes):
extra_volume_ids = [id for id in all_volume_ids if id not in volumes]

''' Fetch any extra volumes that weren't carried over from tag_volumes() (if any) '''
for chunk in (extra_volume_ids[n:n+200] for n in xrange(0, len(extra_volume_ids), 200)):
for chunk in (extra_volume_ids[n:n + 200] for n in range(0, len(extra_volume_ids), 200)):
extra_volumes = self._conn.get_all_volumes(
filters = { 'volume-id': chunk }
)
filters={'volume-id': chunk}
)
for vol in extra_volumes:
volumes[vol.id] = vol

Expand All @@ -267,20 +270,23 @@ def tag_snapshots(self, volumes):

for snapshot in snapshots:
this_snap += 1
log.info ('Processing snapshot %d of %d total snapshots', this_snap, total_snaps)
log.info('Processing snapshot %d of %d total snapshots', this_snap, total_snaps)
err = None
for attempt in range(5):
try:
self.tag_snapshot(snapshot, volumes)
except boto.exception.EC2ResponseError, e:
except boto.exception.EC2ResponseError as e:
log.error("Encountered Error %s on snapshot %s", e.error_code, snapshot.id)
err = e
break
except boto.exception.BotoServerError, e:
except boto.exception.BotoServerError as e:
log.error("Encountered Error %s on snapshot %s, waiting %d seconds then retrying", e.error_code, snapshot.id, attempt)
err = e
time.sleep(attempt)
else:
break
else:
log.error("Encountered Error %s on snapshot %s, %d retries failed, continuing", e.error_code, snapshot.id, attempt)
log.error("Encountered Error %s on snapshot %s, %d retries failed, continuing", err.error_code, snapshot.id, attempt)
continue
log.info('Completed processing all snapshots')

Expand Down Expand Up @@ -324,8 +330,8 @@ def _set_resource_tags(self, resource, tags):

delta_tags = {}

for tag_key, tag_value in tags.iteritems():
if not tag_key in resource.tags or resource.tags[tag_key] != tag_value:
for tag_key, tag_value in tags.items():
if tag_key not in resource.tags or resource.tags[tag_key] != tag_value:
delta_tags[tag_key] = tag_value

if len(delta_tags) == 0:
Expand All @@ -341,7 +347,7 @@ class Logging(object):
_log_simple_format = '%(asctime)s [%(levelname)s] %(message)s'
_log_detailed_format = '%(asctime)s [%(levelname)s] [%(name)s(%(lineno)s):%(funcName)s] %(message)s'

def configure(self, verbosity = None):
def configure(self, verbosity):
''' Configure the logging format and verbosity '''

# Configure our logging output
Expand Down
8 changes: 1 addition & 7 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,8 @@
setuptools install script for Graffiti Monkey
"""

import sys
major, minor = sys.version_info[0:2]
if major != 2 or minor < 7:
print 'Graffiti Monkey requires Python 2.7.x'
sys.exit(1)

from setuptools import setup, find_packages

import graffiti_monkey

with open('requirements.txt') as fh:
Expand Down
8 changes: 8 additions & 0 deletions tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@

[flake8]
ignore =
# Line too long.
E501,
# Too many blank lines.
E303,