From 958032be722ddd49a8e15b1b940b096dcd77ae6d Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Fri, 21 Jan 2022 14:15:29 -0500 Subject: [PATCH 1/9] Fix use of global variables as function default argument values --- cob.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/cob.py b/cob.py index 0510731..571a8df 100644 --- a/cob.py +++ b/cob.py @@ -274,7 +274,7 @@ def get_region_from_s3url(url): return "us-east-1" -def retry_url(url, retry_on_404=False, num_retries=retries, timeout=timeout): +def retry_url(url, retry_on_404=False): """ Retry a url. This is specifically used for accessing the metadata service on an instance. Since this address should never be proxied @@ -285,7 +285,7 @@ def retry_url(url, retry_on_404=False, num_retries=retries, timeout=timeout): original = socket.getdefaulttimeout() socket.setdefaulttimeout(timeout) - for i in range(0, num_retries): + for i in range(0, retries): try: proxy_handler = urllib2.ProxyHandler({}) opener = urllib2.build_opener(proxy_handler) @@ -305,28 +305,28 @@ def retry_url(url, retry_on_404=False, num_retries=retries, timeout=timeout): pass print '[ERROR] Caught exception reading instance data' # If not on the last iteration of the loop then sleep. - if i + 1 != num_retries: + if i + 1 != retries: time.sleep(2 ** i) print '[ERROR] Unable to read instance data, giving up' return None -def get_region(url=metadata_server, version="latest", +def get_region(version="latest", params="meta-data/placement/availability-zone/"): """ Fetch the region from AWS metadata store. """ - url = urlparse.urljoin(url, "/".join([version, params])) + url = urlparse.urljoin(metadata_server, "/".join([version, params])) result = retry_url(url) return result[:-1].strip() -def get_iam_role(url=metadata_server, version="latest", +def get_iam_role(version="latest", params="meta-data/iam/security-credentials/"): """ Read IAM role from AWS metadata store. """ - url = urlparse.urljoin(url, "/".join([version, params])) + url = urlparse.urljoin(metadata_server, "/".join([version, params])) result = retry_url(url) if result is None: # print "No IAM role found in the machine" @@ -335,14 +335,13 @@ def get_iam_role(url=metadata_server, version="latest", return result -def get_credentials_from_iam_role(url=metadata_server, +def get_credentials_from_iam_role(iam_role, version="latest", - params="meta-data/iam/security-credentials", - iam_role=None): + params="meta-data/iam/security-credentials"): """ Read IAM credentials from AWS metadata store. """ - url = urlparse.urljoin(url, "/".join([version, params, iam_role])) + url = urlparse.urljoin(metadata_server, "/".join([version, params, iam_role])) result = retry_url(url) if result is None: # print "No IAM credentials found in the machine" @@ -512,7 +511,7 @@ def set_credentials(self): "for the repo '%s'" % self.repoid) raise IncorrectCredentialsError - credentials = get_credentials_from_iam_role(iam_role=iam_role) + credentials = get_credentials_from_iam_role(iam_role) if credentials is None: self.conduit.info(3, "[ERROR] Fail to get IAM credentials" "for the repo '%s'" % self.repoid) From 27230d0139936d721e1adb3292b223953d30c2ff Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Fri, 21 Jan 2022 14:16:02 -0500 Subject: [PATCH 2/9] Remove extraneous settings from config file --- cob.conf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cob.conf b/cob.conf index badb134..d0c23bc 100644 --- a/cob.conf +++ b/cob.conf @@ -16,15 +16,6 @@ ; limitations under the License. ; [main] -cachedir=/var/cache/yum/$basearch/$releasever -keepcache=1 -debuglevel=4 -logfile=/var/log/yum.log -exactarch=1 -obsoletes=0 -gpgcheck=0 -plugins=1 -distroverpkg=centos-release enabled=1 [aws] From 30748725dc634aba5288ff148baf3ff819c05d89 Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Fri, 21 Jan 2022 14:45:36 -0500 Subject: [PATCH 3/9] Close HTTP response and error objects after reading --- cob.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cob.py b/cob.py index 571a8df..6805079 100644 --- a/cob.py +++ b/cob.py @@ -292,6 +292,7 @@ def retry_url(url, retry_on_404=False): req = urllib2.Request(url) r = opener.open(req) result = r.read() + r.close() return result except urllib2.HTTPError as e: # in 2.6 you use getcode(), in 2.5 and earlier you use code @@ -299,6 +300,7 @@ def retry_url(url, retry_on_404=False): code = e.getcode() else: code = e.code + e.close() if code == 404 and not retry_on_404: return None except Exception as e: From d7ba17ad95001105e00e1d7132eb0f2c96f924c9 Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Fri, 21 Jan 2022 15:07:37 -0500 Subject: [PATCH 4/9] Enable IMDSv2 --- cob.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/cob.py b/cob.py index 6805079..84d28f5 100644 --- a/cob.py +++ b/cob.py @@ -43,6 +43,7 @@ timeout = 60 retries = 5 metadata_server = "http://169.254.169.254" +imds_token = None EMPTY_SHA256_HASH = ( 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') @@ -274,7 +275,7 @@ def get_region_from_s3url(url): return "us-east-1" -def retry_url(url, retry_on_404=False): +def retry_url(url, retry_on_404=False, method=None, add_headers=[]): """ Retry a url. This is specifically used for accessing the metadata service on an instance. Since this address should never be proxied @@ -285,11 +286,19 @@ def retry_url(url, retry_on_404=False): original = socket.getdefaulttimeout() socket.setdefaulttimeout(timeout) + add_headers = list(add_headers) + if imds_token: + add_headers.append(('X-aws-ec2-metadata-token', imds_token)) + for i in range(0, retries): try: proxy_handler = urllib2.ProxyHandler({}) opener = urllib2.build_opener(proxy_handler) + if add_headers: + opener.addheaders = add_headers req = urllib2.Request(url) + if method: + req.get_method = lambda: method r = opener.open(req) result = r.read() r.close() @@ -313,6 +322,21 @@ def retry_url(url, retry_on_404=False): return None +def get_imds_token(version="latest", + params="api/token", + ttl=21600): + """ + Get an IMDSv2 token. + """ + url = urlparse.urljoin(metadata_server, "/".join([version, params])) + result = retry_url(url, method="PUT", add_headers=[('X-aws-ec2-metadata-token-ttl-seconds', str(ttl))]) + if result is None: + #print "Could not get IMDSv2 token; is IMDSv2 enabled?" + return None + else: + return result + + def get_region(version="latest", params="meta-data/placement/availability-zone/"): """ @@ -460,7 +484,7 @@ def _getFile(self, url=None, relative=None, local=None, def set_region(self): # Fetch params from local config file - global timeout, retries, metadata_server + global timeout, retries, metadata_server, imds_token timeout = self.conduit.confInt('aws', 'timeout', default=timeout) retries = self.conduit.confInt('aws', 'retries', default=retries) metadata_server = self.conduit.confString('aws', @@ -475,6 +499,9 @@ def set_region(self): if self.region: return True + # Try to get IMDSv2 token + imds_token = imds_token or get_imds_token() + # Fetch region from meta data region = get_region() if region is None: @@ -488,7 +515,7 @@ def set_region(self): def set_credentials(self): # Fetch params from local config file - global timeout, retries, metadata_server + global timeout, retries, metadata_server, imds_token timeout = self.conduit.confInt('aws', 'timeout', default=timeout) retries = self.conduit.confInt('aws', 'retries', default=retries) metadata_server = self.conduit.confString('aws', @@ -506,6 +533,9 @@ def set_credentials(self): if self.access_key and self.secret_key: return True + # Try to get IMDSv2 token + imds_token = imds_token or get_imds_token() + # Fetch credentials from iam role meta data iam_role = get_iam_role() if iam_role is None: From c417342acd278f1eb8ce12b88b7e3c37dfeb7523 Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Fri, 21 Jan 2022 16:46:57 -0500 Subject: [PATCH 5/9] Option to get credentials from ECS metadata service This can be set in the config file or on the command line, e.g. `--setopt=aws.container_credentials_path=$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` --- cob.py | 65 +++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/cob.py b/cob.py index 84d28f5..d240ed3 100644 --- a/cob.py +++ b/cob.py @@ -361,16 +361,14 @@ def get_iam_role(version="latest", return result -def get_credentials_from_iam_role(iam_role, - version="latest", - params="meta-data/iam/security-credentials"): +def get_credentials_from_path(path): """ - Read IAM credentials from AWS metadata store. + Read IAM credentials from a given path in the AWS metadata store. """ - url = urlparse.urljoin(metadata_server, "/".join([version, params, iam_role])) + url = urlparse.urljoin(metadata_server, path) result = retry_url(url) if result is None: - # print "No IAM credentials found in the machine" + # print "No credentials found at URL", repr(url) return None try: data = json.loads(result) @@ -390,6 +388,15 @@ def get_credentials_from_iam_role(iam_role, token.encode("utf-8")) +def get_credentials_for_iam_role(iam_role, + version="latest", + params="meta-data/iam/security-credentials"): + """ + Read IAM role credentials from AWS metadata store. + """ + return get_credentials_from_path("/".join([version, params, iam_role])) + + def init_hook(conduit): """ Setup the S3 repositories @@ -533,21 +540,37 @@ def set_credentials(self): if self.access_key and self.secret_key: return True - # Try to get IMDSv2 token - imds_token = imds_token or get_imds_token() - - # Fetch credentials from iam role meta data - iam_role = get_iam_role() - if iam_role is None: - self.conduit.info(3, "[ERROR] No credentials in the plugin conf " - "for the repo '%s'" % self.repoid) - raise IncorrectCredentialsError - - credentials = get_credentials_from_iam_role(iam_role) - if credentials is None: - self.conduit.info(3, "[ERROR] Fail to get IAM credentials" - "for the repo '%s'" % self.repoid) - raise IncorrectCredentialsError + container_credentials_path = self.conduit.confString('aws', + 'container_credentials_path', + default=None) + if container_credentials_path: + # Reload metadata server address, default to ECS metadata service + metadata_server = self.conduit.confString('aws', + 'metadata_server', + default="http://169.254.170.2") + + # Fetch credentials from given path + credentials = get_credentials_from_path(container_credentials_path) + if credentials is None: + self.conduit.info(3, "[ERROR] Fail to get container credentials" + "for the repo '%s'" % self.repoid) + raise IncorrectCredentialsError + else: + # Try to get IMDSv2 token + imds_token = imds_token or get_imds_token() + + # Fetch credentials from iam role meta data + iam_role = get_iam_role() + if iam_role is None: + self.conduit.info(3, "[ERROR] No credentials in the plugin conf " + "for the repo '%s'" % self.repoid) + raise IncorrectCredentialsError + + credentials = get_credentials_for_iam_role(iam_role) + if credentials is None: + self.conduit.info(3, "[ERROR] Fail to get IAM credentials" + "for the repo '%s'" % self.repoid) + raise IncorrectCredentialsError self.access_key, self.secret_key, self.token = credentials return True From 5509aa1c9c2fbcff1185513616323c09be85a098 Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Wed, 26 Jan 2022 13:44:39 -0500 Subject: [PATCH 6/9] Default plugin configuration should be empty The container metadata server cannot be used if the default metadata server http://169.254.169.254 is specified in the config file. --- cob.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cob.conf b/cob.conf index d0c23bc..6030c04 100644 --- a/cob.conf +++ b/cob.conf @@ -19,8 +19,8 @@ enabled=1 [aws] -# access_key = -# secret_key = -timeout = 60 -retries = 5 -metadata_server = http://169.254.169.254 +# metadata_server = http://192.0.2.169 ; alternate URL for metadata server +# timeout = 60 +# retries = 5 +# access_key = ; AWS credentials may be configured here, rather than +# secret_key = ; retrieved from metadata server From 35b8aab85403d7ad0279af16a3038b2a4710100a Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Wed, 26 Jan 2022 14:37:10 -0500 Subject: [PATCH 7/9] Add command-line option for relative path in container metadata service The use of `--setopt=...` described in commit c417342 unfortunately has no effect. --- cob.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/cob.py b/cob.py index d240ed3..2fb8232 100644 --- a/cob.py +++ b/cob.py @@ -35,7 +35,8 @@ __all__ = ['requires_api_version', 'plugin_type', - 'init_hook'] + 'init_hook', + 'prereposetup_hook'] requires_api_version = '2.5' plugin_type = yum.plugins.TYPE_CORE @@ -398,6 +399,15 @@ def get_credentials_for_iam_role(iam_role, def init_hook(conduit): + """ + Add argument for relative path in container credentials metadata service + """ + parser = conduit.getOptParser() + parser.add_option("--aws-container-credentials-relative-uri", + dest='aws_container_credentials_relative_uri') + + +def prereposetup_hook(conduit): """ Setup the S3 repositories """ @@ -540,17 +550,15 @@ def set_credentials(self): if self.access_key and self.secret_key: return True - container_credentials_path = self.conduit.confString('aws', - 'container_credentials_path', - default=None) - if container_credentials_path: + opts, cmd = self.conduit.getCmdLine() + if opts.aws_container_credentials_relative_uri: # Reload metadata server address, default to ECS metadata service metadata_server = self.conduit.confString('aws', 'metadata_server', default="http://169.254.170.2") # Fetch credentials from given path - credentials = get_credentials_from_path(container_credentials_path) + credentials = get_credentials_from_path(opts.aws_container_credentials_relative_uri) if credentials is None: self.conduit.info(3, "[ERROR] Fail to get container credentials" "for the repo '%s'" % self.repoid) From 22218a25540e25a315f2a01968ce17929c5aac39 Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Thu, 10 Feb 2022 13:30:09 -0500 Subject: [PATCH 8/9] Expect that option parsing might be unavailable The PLUGINS doc doesn't mention this possibility, but the pattern in other Yum plugins is clear. --- cob.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cob.py b/cob.py index 2fb8232..f10fd70 100644 --- a/cob.py +++ b/cob.py @@ -403,8 +403,9 @@ def init_hook(conduit): Add argument for relative path in container credentials metadata service """ parser = conduit.getOptParser() - parser.add_option("--aws-container-credentials-relative-uri", - dest='aws_container_credentials_relative_uri') + if parser: + parser.add_option("--aws-container-credentials-relative-uri", + dest='aws_container_credentials_relative_uri') def prereposetup_hook(conduit): @@ -551,7 +552,7 @@ def set_credentials(self): return True opts, cmd = self.conduit.getCmdLine() - if opts.aws_container_credentials_relative_uri: + if opts and opts.aws_container_credentials_relative_uri: # Reload metadata server address, default to ECS metadata service metadata_server = self.conduit.confString('aws', 'metadata_server', From 795714825cd1015a48c63265908d32d31053ccf4 Mon Sep 17 00:00:00 2001 From: Ben Slusky Date: Thu, 9 Jun 2022 13:42:51 -0400 Subject: [PATCH 9/9] Append newline to URL during request canonicaliztion The latest Python packages for Amazon Linux 2 incorporate a fix for CVE-2022-0391, which makes the `urlsplit` function remove newlines and tabs from the URL. --- cob.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cob.py b/cob.py index f10fd70..eb1282c 100644 --- a/cob.py +++ b/cob.py @@ -162,7 +162,7 @@ def signed_headers(self, headers_to_sign): def canonical_request(self, request): cr = [request.method.upper()] path = self._normalize_url_path(urlsplit(request.url).path) - cr.append(path) + cr.append(path + '\n') headers_to_sign = self.headers_to_sign(request) cr.append(self.canonical_headers(headers_to_sign) + '\n') cr.append(self.signed_headers(headers_to_sign)) @@ -587,8 +587,7 @@ def set_credentials(self): def fetch_headers(self, url, path): headers = {} - # "\n" in the url, required by AWS S3 Auth v4 - url = urlparse.urljoin(url, urllib2.quote(path)) + "\n" + url = urlparse.urljoin(url, urllib2.quote(path)) credentials = Credentials(self.access_key, self.secret_key, self.token) request = HTTPRequest("GET", url) signer = S3SigV4Auth(credentials, "s3", self.region, self.conduit)