From d6bd9056236fbab9f7cb11061f4c330ee8573812 Mon Sep 17 00:00:00 2001 From: David Hamm Date: Tue, 17 Mar 2020 09:50:08 -0400 Subject: [PATCH 1/2] resource pagination --- cftdeploy/manifest.py | 34 +++++++++++++++++++++++++++------- cftdeploy/stack.py | 4 ++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/cftdeploy/manifest.py b/cftdeploy/manifest.py index 31dc135..2c6e8d8 100644 --- a/cftdeploy/manifest.py +++ b/cftdeploy/manifest.py @@ -79,6 +79,26 @@ def validate(self, override=None): payload = self.build_cft_payload() return(payload) + def shellout(self, cmd): + """Simply shell out and run a command. Handy for injecting dynamic data into stack.""" + stream = os.popen('echo Returned output') + output = stream.read() + return output + + def prepvar(self, var): + if var.startswith('!Sys '): + return self.shellout( var[len('!Sys '):] ) + + return var + + def load_stack(self, stack_name): + """Retrieves the elemnts of a previously deployed stact for use in the current running deployment.""" + my_stack = CFStack( stack_name, self.region, self.session) + if my_stack is None: + logger.error(f"Creating stack object for {stack_name} returned None") + raise CFStackDoesNotExistError(stack_name) + return my_stack + def fetch_parameters(self, override=None): """Based on the manifest's Sourced Parameters, find all the parameters and populate them.""" @@ -100,19 +120,19 @@ def fetch_parameters(self, override=None): if 'DependentStacks' in self.document and self.document['DependentStacks'] is not None: # The new way for source_key, source_stack_name in self.document['DependentStacks'].items(): - my_stack = CFStack(source_stack_name, self.region, self.session) - if my_stack is None: - logger.error(f"Creating stack object for {source_stack_name} returned None") - raise CFStackDoesNotExistError(source_stack_name) + my_stack = self.load_stack(source_stack_name) stack_map[source_key] = my_stack if 'SourcedParameters' in self.document and self.document['SourcedParameters'] is not None: for k, v in self.document['SourcedParameters'].items(): (stack_map_key, section, resource_id) = v.split('.') if stack_map_key not in stack_map: - logger.error(f"DependentStack {stack_map_key} was required by {k} but was not found or referenced.") - continue + my_stack = self.load_stack( stack_map_key ) + if my_stack: + stack_map[stack_map_key] = my_stack + source_stack = stack_map[stack_map_key] + if section == "Parameters": params = source_stack.get_parameters() if resource_id in params: @@ -177,7 +197,7 @@ def build_cft_payload(self): # format and add the tags if 'Tags' in self.document: for k, v in self.document['Tags'].items(): - payload['Tags'].append({'Key': k, 'Value': v}) + payload['Tags'].append({'Key': k, 'Value': self.prepvar( v )}) return(payload) diff --git a/cftdeploy/stack.py b/cftdeploy/stack.py index bd5c840..92a91f9 100644 --- a/cftdeploy/stack.py +++ b/cftdeploy/stack.py @@ -113,6 +113,10 @@ def get_resources(self): """ Return all the PhysicalResourceIds for each LogicalId in the template""" response = self.cf_client.list_stack_resources(StackName=self.StackId) self.resources = response['StackResourceSummaries'] + while "NextToken" in response: + response = self.cf_client.list_stack_resources(StackName=self.StackId, NextToken=response[ "NextToken"]) + self.resources.extend( response['StackResourceSummaries'] ) + output = {} for o in self.resources: if 'PhysicalResourceId' not in o: From 3f7d9df9eddc29f5831037bfa08555e514a468c5 Mon Sep 17 00:00:00 2001 From: David Hamm Date: Tue, 17 Mar 2020 10:11:40 -0400 Subject: [PATCH 2/2] invalid refereence to source_stack.name should be source_stack.stack_name --- cftdeploy/manifest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cftdeploy/manifest.py b/cftdeploy/manifest.py index 2c6e8d8..9fced58 100644 --- a/cftdeploy/manifest.py +++ b/cftdeploy/manifest.py @@ -138,19 +138,19 @@ def fetch_parameters(self, override=None): if resource_id in params: param_dict[k] = {'ParameterKey': k, 'ParameterValue': params[resource_id], 'UsePreviousValue': False} else: - logger.error(f"Unable to find {resource_id} in {source_stack.name} (aliased as {stack_map_key}) Parameters") + logger.error(f"Unable to find {resource_id} in {source_stack.stack_name} (aliased as {stack_map_key}) Parameters") elif section == "Outputs": outputs = source_stack.get_outputs() if resource_id in outputs: param_dict[k] = {'ParameterKey': k, 'ParameterValue': outputs[resource_id], 'UsePreviousValue': False} else: - logger.error(f"Unable to find {resource_id} in {source_stack.name} (aliased as {stack_map_key}) Outputs") + logger.error(f"Unable to find {resource_id} in {source_stack.stack_name} (aliased as {stack_map_key}) Outputs") elif section == "Resources": resources = source_stack.get_resources() if resource_id in resources: param_dict[k] = {'ParameterKey': k, 'ParameterValue': resources[resource_id], 'UsePreviousValue': False} else: - logger.error(f"Unable to find {resource_id} in {source_stack.name} (aliased as {stack_map_key}) Resources") + logger.error(f"Unable to find {resource_id} in {source_stack.stack_name} (aliased as {stack_map_key}) Resources") else: logger.error(f"Invaluid SourcedParameters section type: {section}")