From ed4e761dfd5f222d71a0b194e5d8dd6fd5323662 Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 19 Nov 2020 23:08:05 +0000 Subject: [PATCH 01/38] Workaround missing ssl.PROTOCOL_TLS Some older python versions (e.g. 3.5.2) don't have it. --- ftw/http.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ftw/http.py b/ftw/http.py index f951e9d..aaa9119 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -21,6 +21,10 @@ from . import errors +# Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. +PROTOCOL_TLS = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23) + + if PY2: reload(sys) # pragma: no flakes sys.setdefaultencoding('utf8') @@ -305,7 +309,7 @@ def build_socket(self): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Check if TLS if self.request_object.protocol == 'https': - context = ssl.SSLContext(ssl.PROTOCOL_TLS) + context = ssl.SSLContext(PROTOCOL_TLS) context.set_ciphers(self.CIPHERS) self.sock = context.wrap_socket( self.sock, server_hostname=self.request_object.dest_addr) From 43f663fa9478e0e41af86b02b497514766e2873f Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 19 Nov 2020 23:09:44 +0000 Subject: [PATCH 02/38] Bump version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8c9ee2c..70cd745 100644 --- a/setup.py +++ b/setup.py @@ -3,12 +3,12 @@ from setuptools import setup setup(name='ftw', - version='1.2.2', + version='1.2.3', description='Framework for Testing WAFs', author='Chaim Sanders, Zack Allen', author_email='zma4580@gmail.com, chaim.sanders@gmail.com', url='https://www.github.com/coreruleset/ftw', - download_url='https://github.com/coreruleset/ftw/tarball/1.2.2', + download_url='https://github.com/coreruleset/ftw/tarball/1.2.3', include_package_data=True, package_data={ 'ftw': ['util/public_suffix_list.dat'] From ca4d8b7d93aa5222ff8b410c752749cbd9de3495 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Tue, 24 Nov 2020 13:28:46 -0300 Subject: [PATCH 03/38] feat(py3): apply 2to3 to move away from py2 syntax where needed Signed-off-by: Felipe Zipitria --- ftw/http.py | 19 ++++++++++--------- ftw/pytest_plugin.py | 6 +++--- ftw/ruleset.py | 16 +++++----------- ftw/testrunner.py | 2 +- ftw/util.py | 2 +- ftw/util/ironbee.py | 2 +- test/integration/test_http.py | 4 ++-- test/unit/test_ruleset.py | 2 +- 8 files changed, 24 insertions(+), 29 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index aaa9119..bea9acd 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -1,4 +1,4 @@ -from __future__ import print_function + import brotli import io import socket @@ -19,6 +19,7 @@ from six.moves import http_cookies from . import errors +import importlib # Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. @@ -26,7 +27,7 @@ if PY2: - reload(sys) # pragma: no flakes + importlib.reload(sys) # pragma: no flakes sys.setdefaultencoding('utf8') escape_codec = 'string_escape' else: @@ -106,7 +107,7 @@ def check_for_cookie(self, cookie): IP(self.dest_addr) except ValueError: origin_is_ip = False - for cookie_morsals in cookie.values(): + for cookie_morsals in list(cookie.values()): # If the coverdomain is blank or the domain is an IP # set the domain to be the origin if cookie_morsals['domain'] == '' or origin_is_ip: @@ -206,7 +207,7 @@ def process_response(self): }) header = ensure_str(header[0]), ensure_str(header[1]) response_headers[header[0].lower()] = header[1].lstrip() - if 'set-cookie' in response_headers.keys(): + if 'set-cookie' in list(response_headers.keys()): try: cookie = http_cookies.SimpleCookie() cookie.load(response_headers['set-cookie']) @@ -232,7 +233,7 @@ def process_response(self): response_data = self.CRLF.join(split_response[data_line:]) # if the output headers say there is encoding - if 'content-encoding' in response_headers.keys(): + if 'content-encoding' in list(response_headers.keys()): response_data = self.parse_content_encoding( response_headers, response_data) if len(response_line.split(' ', 2)) != 3: @@ -333,7 +334,7 @@ def find_cookie(self): return_cookies = [] origin_domain = self.request_object.dest_addr for cookie in self.cookiejar: - for cookie_morsals in cookie[0].values(): + for cookie_morsals in list(cookie[0].values()): cover_domain = cookie_morsals['domain'] if cover_domain == '': if origin_domain == cookie[1]: @@ -361,7 +362,7 @@ def build_request(self): # If the user has requested a tracked cookie and we have one set it if available_cookies: cookie_value = '' - if 'cookie' in self.request_object.headers.keys(): + if 'cookie' in list(self.request_object.headers.keys()): # Create a SimpleCookie out of our provided cookie try: provided_cookie = http_cookies.SimpleCookie() @@ -382,7 +383,7 @@ def build_request(self): provided_cookie[cookie_key].value for cookie in available_cookies: for cookie_key, cookie_morsal in iteritems(cookie): - if cookie_key in result_cookie.keys(): + if cookie_key in list(result_cookie.keys()): # we don't overwrite a user specified # cookie with a saved one pass @@ -419,7 +420,7 @@ def build_request(self): encoding = "utf-8" # Check to see if we have a content type and magic is # off (otherwise UTF-8) - if 'Content-Type' in self.request_object.headers.keys() and \ + if 'Content-Type' in list(self.request_object.headers.keys()) and \ self.request_object.stop_magic is False: pattern = re.compile(r'\;\s{0,1}?charset\=(.*?)(?:$|\;|\s)') m = re.search(pattern, diff --git a/ftw/pytest_plugin.py b/ftw/pytest_plugin.py index 79cf0c8..710e81f 100644 --- a/ftw/pytest_plugin.py +++ b/ftw/pytest_plugin.py @@ -27,10 +27,10 @@ def test_id(val): Dynamically names tests, useful for when we are running dozens to hundreds of tests """ - if isinstance(val, (dict, Test,)): + if isinstance(val, (dict, Test)): # We must be carful here because errors are swallowed and # defaults returned - if 'name' in val.ruleset_meta.keys(): + if 'name' in list(val.ruleset_meta.keys()): return '%s -- %s' % (val.ruleset_meta['name'], val.test_title) else: return '%s -- %s' % ("Unnamed_Test", val.test_title) @@ -104,7 +104,7 @@ def pytest_addoption(parser): help='pass in a tablename to parse journal results') parser.addoption('--port', action='store', default=None, help='destination port to direct tests towards', - choices=range(1, 65536), + choices=list(range(1, 65536)), type=int) parser.addoption('--protocol', action='store', default=None, help='destination protocol to direct tests towards', diff --git a/ftw/ruleset.py b/ftw/ruleset.py index 83d6721..640f8f9 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -115,10 +115,10 @@ def __init__(self, raw_request=None, # Check if there is any data and do defaults if self.data != '': # Default values for content length and header - if 'Content-Type' not in headers.keys() and stop_magic is False: + if 'Content-Type' not in list(headers.keys()) and stop_magic is False: headers['Content-Type'] = 'application/x-www-form-urlencoded' # check if encoded and encode if it should be - if 'Content-Type' in headers.keys(): + if 'Content-Type' in list(headers.keys()): if headers['Content-Type'] == \ 'application/x-www-form-urlencoded' and stop_magic is False: if ensure_str(unquote(self.data)) == self.data: @@ -126,7 +126,7 @@ def __init__(self, raw_request=None, if len(query_string) != 0: encoded_args = urlencode(query_string) self.data = encoded_args - if 'Content-Length' not in headers.keys() and stop_magic is False: + if 'Content-Length' not in list(headers.keys()) and stop_magic is False: # The two is for the trailing CRLF and the one after headers['Content-Length'] = len(self.data) @@ -159,10 +159,7 @@ def build_stages(self): """ Processes and loads an array of stages from the test dictionary """ - return map( - lambda stage_dict: Stage(stage_dict['stage']), - self.test_dict['stages'] - ) + return [Stage(stage_dict['stage']) for stage_dict in self.test_dict['stages']] class Ruleset(object): @@ -184,10 +181,7 @@ def extract_tests(self): creates test objects based on input """ try: - return map( - lambda test_dict: Test(test_dict, self.meta), - self.yaml_file['tests'] - ) + return [Test(test_dict, self.meta) for test_dict in self.yaml_file['tests']] except errors.TestError as e: e.args[1]['meta'] = self.meta raise e diff --git a/ftw/testrunner.py b/ftw/testrunner.py index fb0d0ba..12fbda0 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -1,4 +1,4 @@ -from __future__ import print_function + import datetime from dateutil import parser import pytest diff --git a/ftw/util.py b/ftw/util.py index b5f8f0b..ce7fdea 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,4 +1,4 @@ -from __future__ import print_function + import io import yaml import os diff --git a/ftw/util/ironbee.py b/ftw/util/ironbee.py index cb36089..59de95b 100644 --- a/ftw/util/ironbee.py +++ b/ftw/util/ironbee.py @@ -1,4 +1,4 @@ -from __future__ import print_function + import os import request_to_yaml diff --git a/test/integration/test_http.py b/test/integration/test_http.py index 54e41f1..d5d439e 100644 --- a/test/integration/test_http.py +++ b/test/integration/test_http.py @@ -1,4 +1,4 @@ -from __future__ import print_function + from ftw import ruleset, http, errors import pytest @@ -13,7 +13,7 @@ def test_cookies1(): http_ua.send_request(x) with pytest.raises(KeyError): print(http_ua.request_object.headers["cookie"]) - assert("set-cookie" in http_ua.response_object.headers.keys()) + assert("set-cookie" in list(http_ua.response_object.headers.keys())) cookie_data = http_ua.response_object.headers["set-cookie"] cookie_var = cookie_data.split("=")[0] x = ruleset.Input(protocol="https", port=443, dest_addr="www.ieee.org", diff --git a/test/unit/test_ruleset.py b/test/unit/test_ruleset.py index d2d5035..ac15c97 100644 --- a/test/unit/test_ruleset.py +++ b/test/unit/test_ruleset.py @@ -25,7 +25,7 @@ def test_input(): dictionary = {} dictionary['headers'] = headers input_2 = ruleset.Input(**dictionary) - assert(len(input_2.headers.keys()) == 2) + assert(len(list(input_2.headers.keys())) == 2) dictionary_2 = {'random_key': 'bar'} with pytest.raises(TypeError): ruleset.Input(**dictionary_2) From 1a7da1af7be052eb92cc91a075d1edee282d5de0 Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 12:00:57 +0000 Subject: [PATCH 04/38] Test against python versions 3.6 to 3.9 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08edf34..084b24f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [ '2.x', '3.x' ] + python-version: [ '2.x', '3.6', '3.7', '3.8', '3.9' ] steps: - name: Checkout repo From 308a2bd040387373f3647cb262a53081e37e5e07 Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 12:01:24 +0000 Subject: [PATCH 05/38] Fail hard continue-on-error will not bubble up the error, so fail for now. --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 084b24f..b54a6ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,4 +34,3 @@ jobs: py.test test/integration/test_runner.py --rule=test/integration/BASICFIXTURE.yaml py.test --pycodestyle ftw || true py.test --flakes ftw - continue-on-error: true From 753121d3ddbe4eec5ecb3c21bac4e337a9532aee Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 13:21:07 +0000 Subject: [PATCH 06/38] Revert import introduced in 2to3 This is only needed in the python 2 case so we don't need to do anything here. --- ftw/http.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index bea9acd..a4b91d8 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -19,7 +19,6 @@ from six.moves import http_cookies from . import errors -import importlib # Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. @@ -27,7 +26,7 @@ if PY2: - importlib.reload(sys) # pragma: no flakes + reload(sys) # pragma: no flakes sys.setdefaultencoding('utf8') escape_codec = 'string_escape' else: From dee41d02d776754e945090584ce8a6f4e6e814b5 Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 13:30:04 +0000 Subject: [PATCH 07/38] Fix code style introduced during 2to3 --- ftw/ruleset.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ftw/ruleset.py b/ftw/ruleset.py index 640f8f9..1a865f6 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -115,7 +115,8 @@ def __init__(self, raw_request=None, # Check if there is any data and do defaults if self.data != '': # Default values for content length and header - if 'Content-Type' not in list(headers.keys()) and stop_magic is False: + if 'Content-Type' not in list(headers.keys()) and \ + stop_magic is False: headers['Content-Type'] = 'application/x-www-form-urlencoded' # check if encoded and encode if it should be if 'Content-Type' in list(headers.keys()): @@ -126,7 +127,8 @@ def __init__(self, raw_request=None, if len(query_string) != 0: encoded_args = urlencode(query_string) self.data = encoded_args - if 'Content-Length' not in list(headers.keys()) and stop_magic is False: + if 'Content-Length' not in list(headers.keys()) and \ + stop_magic is False: # The two is for the trailing CRLF and the one after headers['Content-Length'] = len(self.data) @@ -159,7 +161,8 @@ def build_stages(self): """ Processes and loads an array of stages from the test dictionary """ - return [Stage(stage_dict['stage']) for stage_dict in self.test_dict['stages']] + return [Stage(stage_dict['stage']) + for stage_dict in self.test_dict['stages']] class Ruleset(object): @@ -181,7 +184,8 @@ def extract_tests(self): creates test objects based on input """ try: - return [Test(test_dict, self.meta) for test_dict in self.yaml_file['tests']] + return [Test(test_dict, self.meta) + for test_dict in self.yaml_file['tests']] except errors.TestError as e: e.args[1]['meta'] = self.meta raise e From 4e3f47d4e6ac6700d2edd6b324d3aa4477eaf910 Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 13:34:11 +0000 Subject: [PATCH 08/38] Update codestyle package --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b54a6ca..e169b69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: - name: Install requirements run: | python -m pip install -r requirements.txt - python -m pip install pytest-codestyle || true + python -m pip install pytest-pycodestyle || true python -m pip install pytest-flakes python setup.py install - name: Run tests From 0c95befa92787035011ee059084028594e8c0141 Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 13:43:20 +0000 Subject: [PATCH 09/38] Limit flakes and codestyle to Python 3.9 There are various errors that require a newer version of pytest, which is incompatible with python2. While we migrate away, limit these tests to Python 3.9. --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e169b69..7e18cc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,11 @@ jobs: python -m pip install pytest-pycodestyle || true python -m pip install pytest-flakes python setup.py install + - name: Check source files + if: matrix.python-version == '3.9' + run: | + py.test --pycodestyle ftw + py.test --flakes ftw - name: Run tests run: | py.test test/unit @@ -32,5 +37,3 @@ jobs: py.test test/integration/test_http.py py.test test/integration/test_cookie.py --rule=test/integration/COOKIEFIXTURE.yaml py.test test/integration/test_runner.py --rule=test/integration/BASICFIXTURE.yaml - py.test --pycodestyle ftw || true - py.test --flakes ftw From 1f0445da0aa7cc4142296618ab5dd4f3becb421e Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 13:55:51 +0000 Subject: [PATCH 10/38] Move this temporarily --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e18cc0..56796e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,12 +21,12 @@ jobs: - name: Install requirements run: | python -m pip install -r requirements.txt - python -m pip install pytest-pycodestyle || true - python -m pip install pytest-flakes python setup.py install - name: Check source files if: matrix.python-version == '3.9' run: | + python -m pip install pytest-pycodestyle + python -m pip install pytest-flakes py.test --pycodestyle ftw py.test --flakes ftw - name: Run tests From 376fbda5da0d13721f51a0323de4e66204f745e4 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:01:27 -0300 Subject: [PATCH 11/38] feat(six): remove six Signed-off-by: Felipe Zipitria --- ftw/http.py | 61 ++++++++++++++++++-------------------------- ftw/logchecker.py | 8 +++--- ftw/pytest_plugin.py | 4 +-- ftw/ruleset.py | 5 ++-- ftw/testrunner.py | 5 ++-- 5 files changed, 34 insertions(+), 49 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index a4b91d8..578467b 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -1,6 +1,5 @@ - import brotli -import io +from io import BytesIO import socket import ssl import errno @@ -13,29 +12,19 @@ import zlib import encodings from IPy import IP - -from six import BytesIO, PY2, ensure_binary, ensure_str, iteritems, \ - text_type -from six.moves import http_cookies +from http import cookies from . import errors +from . import util # Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. PROTOCOL_TLS = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23) -if PY2: - reload(sys) # pragma: no flakes - sys.setdefaultencoding('utf8') - escape_codec = 'string_escape' -else: - escape_codec = 'unicode_escape' - - class HttpResponse(object): def __init__(self, http_response, user_agent): - self.response = ensure_binary(http_response) + self.response = util.ensure_binary(http_response) # For testing purposes HTTPResponse might be called OOL try: self.dest_addr = user_agent.request_object.dest_addr @@ -139,7 +128,7 @@ def check_for_cookie(self, cookie): 'function': 'http.HttpResponse.check_for_cookie' }) try: - with io.open(psl_path, 'r', encoding='utf-8') as fo: + with open(psl_path, 'r', encoding='utf-8') as fo: for line in fo: if line[:2] == '//' or line[0] == ' ' or \ line[0].strip() == '': @@ -185,7 +174,7 @@ def process_response(self): Parses an HTTP response after an HTTP request is sent """ split_response = self.response.split(self.CRLF) - response_line = ensure_str(split_response[0]) + response_line = util.ensure_str(split_response[0]) response_headers = {} response_data = None data_line = None @@ -204,13 +193,13 @@ def process_response(self): 'header_rcvd': str(header), 'function': 'http.HttpResponse.process_response' }) - header = ensure_str(header[0]), ensure_str(header[1]) + header = util.ensure_str(header[0]), util.ensure_str(header[1]) response_headers[header[0].lower()] = header[1].lstrip() if 'set-cookie' in list(response_headers.keys()): try: - cookie = http_cookies.SimpleCookie() + cookie = cookies.SimpleCookie() cookie.load(response_headers['set-cookie']) - except http_cookies.CookieError as err: + except cookies.CookieError as err: raise errors.TestError( 'Error processing the cookie content into a SimpleCookie', { @@ -364,9 +353,9 @@ def build_request(self): if 'cookie' in list(self.request_object.headers.keys()): # Create a SimpleCookie out of our provided cookie try: - provided_cookie = http_cookies.SimpleCookie() + provided_cookie = cookies.SimpleCookie() provided_cookie.load(self.request_object.headers['cookie']) - except http_cookies.CookieError as err: + except cookies.CookieError as err: raise errors.TestError( 'Error processing the existing cookie into a ' 'SimpleCookie', @@ -390,16 +379,16 @@ def build_request(self): result_cookie[cookie_key] = \ cookie[cookie_key].value for key, value in iteritems(result_cookie): - cookie_value += (text_type(key) + '=' + - text_type(value) + '; ') + cookie_value += (str(key) + '=' + + str(value) + '; ') # Remove the trailing semicolon cookie_value = cookie_value[:-2] self.request_object.headers['cookie'] = cookie_value else: for cookie in available_cookies: for cookie_key, cookie_morsal in iteritems(cookie): - cookie_value += (text_type(cookie_key) + '=' + - text_type(cookie_morsal.coded_value) + + cookie_value += (str(cookie_key) + '=' + + str(cookie_morsal.coded_value) + '; ') # Remove the trailing semicolon cookie_value = cookie_value[:-2] @@ -410,7 +399,7 @@ def build_request(self): if self.request_object.headers != {}: for hname, hvalue in iteritems(self.request_object.headers): headers += text_type(hname) + ': ' + \ - text_type(hvalue) + self.CRLF + str(hvalue) + self.CRLF request = request.replace('#headers#', headers) # If we have data append it @@ -435,18 +424,18 @@ def build_request(self): if choice in possible_choices: encoding = choice try: - data = self.request_object.data.encode(encoding) - except UnicodeEncodeError as err: + data_bytes = self.request_object.data.encode(encoding, 'strict') + except UnicodeError as err: raise errors.TestError( 'Error encoding the data with the charset specified', { 'msg': str(err), 'Content-Type': str(self.request_object.headers['Content-Type']), - 'data': text_type(self.request_object.data), + 'data': str(self.request_object.data), 'function': 'http.HttpResponse.build_request' }) - request = request.replace('#data#', ensure_str(data)) + request = request.replace('#data#', util.ensure_str(data_bytes)) else: request = request.replace('#data#', '') # If we have a Raw Request we should use that instead @@ -457,15 +446,15 @@ def build_request(self): { 'function': 'http.HttpUA.build_request' }) - request = ensure_binary(self.request_object.raw_request) + request = self.request_object.raw_request.encode('utf-8', 'strict') # We do this regardless of magic if you want to send a literal # '\' 'r' or 'n' use encoded request. - request = request.decode(escape_codec) + request = request.decode('unicode_escape') if self.request_object.encoded_request is not None: request = base64.b64decode(self.request_object.encoded_request) - request = request.decode(escape_codec) + request = request.decode('unicode_escape') # if we have an Encoded request we should use that - self.request = ensure_binary(request) + self.request = request.encode('utf-8', 'strict') def get_response(self): """ @@ -486,7 +475,7 @@ def get_response(self): try: data = self.sock.recv(self.RECEIVE_BYTES) if data: - our_data.append(ensure_binary(data)) + our_data.append(util.ensure_binary(data)) begin = time.time() else: # Sleep for sometime to indicate a gap diff --git a/ftw/logchecker.py b/ftw/logchecker.py index c99ad83..7a0a990 100644 --- a/ftw/logchecker.py +++ b/ftw/logchecker.py @@ -1,9 +1,7 @@ -import abc -import six +from abc improt ABC, abstractmethod -@six.add_metaclass(abc.ABCMeta) -class LogChecker(): +class LogChecker(ABC): """ LogChecker is an abstract class that integrations with WAFs MUST implement. This class is used by the testrunner to test log lines against an expected @@ -17,7 +15,7 @@ def set_times(self, start, end): self.start = start self.end = end - @abc.abstractmethod + @abstractmethod def get_logs(self): """ MUST be implemented, MUST return an array of strings diff --git a/ftw/pytest_plugin.py b/ftw/pytest_plugin.py index 710e81f..e141d59 100644 --- a/ftw/pytest_plugin.py +++ b/ftw/pytest_plugin.py @@ -3,8 +3,8 @@ from . import util from .ruleset import Test -from six.moves.BaseHTTPServer import HTTPServer -from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler +from http.server import HTTPServer +from http.server import SimpleHTTPRequestHandler def get_testdata(rulesets): diff --git a/ftw/ruleset.py b/ftw/ruleset.py index 1a865f6..aefdfea 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -1,7 +1,6 @@ import re -from six import ensure_str -from six.moves.urllib.parse import parse_qsl, unquote, urlencode +from urllib.parse import parse_qsl, unquote, urlencode from . import errors @@ -122,7 +121,7 @@ def __init__(self, raw_request=None, if 'Content-Type' in list(headers.keys()): if headers['Content-Type'] == \ 'application/x-www-form-urlencoded' and stop_magic is False: - if ensure_str(unquote(self.data)) == self.data: + if unquote(self.data) == self.data: query_string = parse_qsl(self.data) if len(query_string) != 0: encoded_args = urlencode(query_string) diff --git a/ftw/testrunner.py b/ftw/testrunner.py index 12fbda0..217e6b5 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -4,7 +4,6 @@ import pytest import sqlite3 -from six import ensure_str from . import errors from . import http @@ -57,7 +56,7 @@ def test_response(self, response_object, regex): 'response_object': response_object, 'function': 'testrunner.TestRunner.test_response' }) - if regex.search(ensure_str(response_object.response)): + if regex.search(util.ensure_str(response_object.response)): assert True else: assert False @@ -67,7 +66,7 @@ def test_response_str(self, response, regex): Checks if the response response contains a regex specified in the output stage. It will assert that the regex is present. """ - if regex.search(ensure_str(response)): + if regex.search(util.ensure_str(response)): assert True else: assert False From 270a9c7af9384aa97dd8b2a80f848f4e1ebf82cb Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:02:16 -0300 Subject: [PATCH 12/38] feat(util): add ensure helper functions Signed-off-by: Felipe Zipitria --- ftw/util.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/ftw/util.py b/ftw/util.py index ce7fdea..b7ed703 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,5 +1,4 @@ -import io import yaml import os import sqlite3 @@ -86,7 +85,7 @@ def extract_yaml(yaml_files): loaded_yaml = [] for yaml_file in yaml_files: try: - with io.open(yaml_file, encoding='utf-8') as fd: + with open(yaml_file, encoding='utf-8') as fd: loaded_yaml.append(yaml.safe_load(fd)) except IOError as e: print('Error reading file', yaml_file) @@ -98,3 +97,22 @@ def extract_yaml(yaml_files): print('General error') raise e return loaded_yaml + + +def ensure_str(s, encoding='utf-8', errors='strict'): + # Optimization: Fast return for the common case. + if type(s) is str: + return s + if isinstance(s, bytes): + return s.decode(encoding, errors) + elif not isinstance(s, (str, bytes)): + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_binary(s, encoding='utf-8', errors='strict'): + if isinstance(s, bytes): + return s + if isinstance(s, str): + return s.encode(encoding, errors) + raise TypeError("not expecting type '%s'" % type(s)) + From cee9c0b1af326d018bfdd34f178e520f47a88dfe Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:04:40 -0300 Subject: [PATCH 13/38] fix(build): remove six dependency Signed-off-by: Felipe Zipitria --- requirements.txt | 1 - setup.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6f40c81..a5d2a37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ IPy==0.83 PyYAML==4.2b1 pytest==4.6 python-dateutil==2.6.0 -six==1.14.0 diff --git a/setup.py b/setup.py index 70cd745..9df9859 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,5 @@ 'IPy==0.83', 'PyYAML==4.2b1', 'pytest==4.6', - 'python-dateutil==2.6.0', - 'six==1.14.0' + 'python-dateutil==2.6.0' ]) From aed586467d6c63b06b4fceff60d95bff688e8ad5 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:08:39 -0300 Subject: [PATCH 14/38] fix(typo): fixes typo in class Signed-off-by: Felipe Zipitria --- ftw/logchecker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/logchecker.py b/ftw/logchecker.py index 7a0a990..3f3d557 100644 --- a/ftw/logchecker.py +++ b/ftw/logchecker.py @@ -1,4 +1,4 @@ -from abc improt ABC, abstractmethod +from abc import ABC, abstractmethod class LogChecker(ABC): From fb59dadbdf2d049a13c9ff34801b80aa11f6cf16 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:16:43 -0300 Subject: [PATCH 15/38] fix(util): use isinstance instead of type Signed-off-by: Felipe Zipitria --- ftw/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index b7ed703..b4687d5 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -101,7 +101,7 @@ def extract_yaml(yaml_files): def ensure_str(s, encoding='utf-8', errors='strict'): # Optimization: Fast return for the common case. - if type(s) is str: + if isinstance(s) is str: return s if isinstance(s, bytes): return s.decode(encoding, errors) From c19fad11a3d02c3488291e8a59f17701f79d11d4 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:17:14 -0300 Subject: [PATCH 16/38] fix(util): remove empty line Signed-off-by: Felipe Zipitria --- ftw/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index b4687d5..0560039 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,4 +1,3 @@ - import yaml import os import sqlite3 From ee2421ce7132dd1bd2152222cba87878763b85c4 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:27:00 -0300 Subject: [PATCH 17/38] fix(str): ensure we are using string Signed-off-by: Felipe Zipitria --- ftw/ruleset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ftw/ruleset.py b/ftw/ruleset.py index aefdfea..7b250e1 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -3,6 +3,7 @@ from urllib.parse import parse_qsl, unquote, urlencode from . import errors +from . import util class Output(object): @@ -121,7 +122,7 @@ def __init__(self, raw_request=None, if 'Content-Type' in list(headers.keys()): if headers['Content-Type'] == \ 'application/x-www-form-urlencoded' and stop_magic is False: - if unquote(self.data) == self.data: + if util.ensure_str(unquote(self.data)) == self.data: query_string = parse_qsl(self.data) if len(query_string) != 0: encoded_args = urlencode(query_string) From f0380f763a8b39bf47b9ae16df412ebe4618b002 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:31:05 -0300 Subject: [PATCH 18/38] fix(util): fix introduced typo Signed-off-by: Felipe Zipitria --- ftw/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index 0560039..b58382e 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -100,7 +100,7 @@ def extract_yaml(yaml_files): def ensure_str(s, encoding='utf-8', errors='strict'): # Optimization: Fast return for the common case. - if isinstance(s) is str: + if isinstance(s, str): return s if isinstance(s, bytes): return s.decode(encoding, errors) From 678daf0b2ca529e4679ba8a0e1886420810c3532 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:34:54 -0300 Subject: [PATCH 19/38] feat(ci): test with multiple python versions Signed-off-by: Felipe Zipitria --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56796e6..5ede2e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [ '2.x', '3.6', '3.7', '3.8', '3.9' ] + python-version: [ '3.6', '3.7', '3.8', '3.9' ] steps: - name: Checkout repo From f33677eb0a441899b174ff2325ba3b50df1d1f28 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 12:41:28 -0300 Subject: [PATCH 20/38] fix(http): remove pending iteritems Signed-off-by: Felipe Zipitria --- ftw/http.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 578467b..4e8e354 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -1,4 +1,3 @@ -import brotli from io import BytesIO import socket import ssl @@ -11,6 +10,7 @@ import base64 import zlib import encodings +import brotli from IPy import IP from http import cookies @@ -366,11 +366,11 @@ def build_request(self): 'function': 'http.HttpResponse.build_request' }) result_cookie = {} - for cookie_key, cookie_morsal in iteritems(provided_cookie): + for cookie_key, cookie_morsal in provided_cookie: result_cookie[cookie_key] = \ provided_cookie[cookie_key].value for cookie in available_cookies: - for cookie_key, cookie_morsal in iteritems(cookie): + for cookie_key, cookie_morsal in cookie: if cookie_key in list(result_cookie.keys()): # we don't overwrite a user specified # cookie with a saved one @@ -378,7 +378,7 @@ def build_request(self): else: result_cookie[cookie_key] = \ cookie[cookie_key].value - for key, value in iteritems(result_cookie): + for key, value in result_cookie: cookie_value += (str(key) + '=' + str(value) + '; ') # Remove the trailing semicolon @@ -386,7 +386,7 @@ def build_request(self): self.request_object.headers['cookie'] = cookie_value else: for cookie in available_cookies: - for cookie_key, cookie_morsal in iteritems(cookie): + for cookie_key, cookie_morsal in cookie: cookie_value += (str(cookie_key) + '=' + str(cookie_morsal.coded_value) + '; ') @@ -397,8 +397,8 @@ def build_request(self): # Expand out our headers into a string headers = '' if self.request_object.headers != {}: - for hname, hvalue in iteritems(self.request_object.headers): - headers += text_type(hname) + ': ' + \ + for hname, hvalue in self.request_object.headers.items(): + headers += str(hname) + ': ' + \ str(hvalue) + self.CRLF request = request.replace('#headers#', headers) From b0b455c5b0ecf2228271b6658089ab40a4c2c8ff Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 13:52:07 -0300 Subject: [PATCH 21/38] fix(http): add additional items from cookies Signed-off-by: Felipe Zipitria --- ftw/http.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 4e8e354..55b8f57 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -366,7 +366,7 @@ def build_request(self): 'function': 'http.HttpResponse.build_request' }) result_cookie = {} - for cookie_key, cookie_morsal in provided_cookie: + for cookie_key, cookie_morsal in list(provided_cookie.items()): result_cookie[cookie_key] = \ provided_cookie[cookie_key].value for cookie in available_cookies: @@ -378,7 +378,7 @@ def build_request(self): else: result_cookie[cookie_key] = \ cookie[cookie_key].value - for key, value in result_cookie: + for key, value in list(result_cookie.items()): cookie_value += (str(key) + '=' + str(value) + '; ') # Remove the trailing semicolon @@ -386,7 +386,7 @@ def build_request(self): self.request_object.headers['cookie'] = cookie_value else: for cookie in available_cookies: - for cookie_key, cookie_morsal in cookie: + for cookie_key, cookie_morsal in list(cookie.items()): cookie_value += (str(cookie_key) + '=' + str(cookie_morsal.coded_value) + '; ') From c8a12126f5602721e2dc3ba0ccb93efee78a7bab Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 25 Nov 2020 18:40:35 -0300 Subject: [PATCH 22/38] fix(style): split lines Signed-off-by: Felipe Zipitria --- ftw/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ftw/util.py b/ftw/util.py index b58382e..176c447 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,7 +1,7 @@ -import yaml import os import sqlite3 from glob import glob +import yaml from . import ruleset From 96f0b38d669f304313193b1722a10da053823465 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Thu, 26 Nov 2020 13:35:27 -0300 Subject: [PATCH 23/38] fix(flake8): remove spurious empty spaces Signed-off-by: Felipe Zipitria --- ftw/http.py | 3 ++- ftw/util.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 55b8f57..31e6f27 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -424,7 +424,8 @@ def build_request(self): if choice in possible_choices: encoding = choice try: - data_bytes = self.request_object.data.encode(encoding, 'strict') + data_bytes = \ + self.request_object.data.encode(encoding, 'strict') except UnicodeError as err: raise errors.TestError( 'Error encoding the data with the charset specified', diff --git a/ftw/util.py b/ftw/util.py index 176c447..7c4ef09 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -96,8 +96,8 @@ def extract_yaml(yaml_files): print('General error') raise e return loaded_yaml - - + + def ensure_str(s, encoding='utf-8', errors='strict'): # Optimization: Fast return for the common case. if isinstance(s, str): @@ -114,4 +114,3 @@ def ensure_binary(s, encoding='utf-8', errors='strict'): if isinstance(s, str): return s.encode(encoding, errors) raise TypeError("not expecting type '%s'" % type(s)) - From 2fc2fa55ff3bcca54f47c19877db0846adbb9bbb Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 17:19:35 +0000 Subject: [PATCH 24/38] Rearrange imports and minor polish --- ftw/http.py | 19 ++++++++++--------- ftw/pytest_plugin.py | 6 +++--- ftw/ruleset.py | 1 - ftw/testrunner.py | 5 ++--- ftw/util.py | 3 ++- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 31e6f27..e375d84 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -1,18 +1,19 @@ +from http import cookies from io import BytesIO -import socket -import ssl +import base64 +import encodings import errno -import time import gzip import os -import sys import re -import base64 +import socket +import ssl +import sys +import time import zlib -import encodings + import brotli from IPy import IP -from http import cookies from . import errors from . import util @@ -115,8 +116,8 @@ def check_for_cookie(self, cookie): cover_domain = cover_domain[first_non_dot:] # We must parse the coverDomain to make sure its # not in the suffix list - psl_path = os.path.dirname(__file__) + os.path.sep + \ - 'util' + os.path.sep + 'public_suffix_list.dat' + psl_path = os.path.join(os.path.dirname(__file__), + 'util', 'public_suffix_list.dat') # Check if the public suffix list is present in the ftw dir if os.path.exists(psl_path): pass diff --git a/ftw/pytest_plugin.py b/ftw/pytest_plugin.py index e141d59..078cc16 100644 --- a/ftw/pytest_plugin.py +++ b/ftw/pytest_plugin.py @@ -1,11 +1,11 @@ +from http.server import HTTPServer +from http.server import SimpleHTTPRequestHandler + import pytest from . import util from .ruleset import Test -from http.server import HTTPServer -from http.server import SimpleHTTPRequestHandler - def get_testdata(rulesets): """ diff --git a/ftw/ruleset.py b/ftw/ruleset.py index 7b250e1..04f951c 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -1,5 +1,4 @@ import re - from urllib.parse import parse_qsl, unquote, urlencode from . import errors diff --git a/ftw/testrunner.py b/ftw/testrunner.py index 217e6b5..d11cfdc 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -1,9 +1,8 @@ - import datetime -from dateutil import parser -import pytest import sqlite3 +from dateutil import parser +import pytest from . import errors from . import http diff --git a/ftw/util.py b/ftw/util.py index 7c4ef09..6573423 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -1,6 +1,7 @@ import os -import sqlite3 from glob import glob +import sqlite3 + import yaml from . import ruleset From a2b79da120c1503e6a1cdd5fd9720196b4b3fc5a Mon Sep 17 00:00:00 2001 From: "Federico G. Schwindt" Date: Thu, 26 Nov 2020 21:20:32 +0000 Subject: [PATCH 25/38] Use single quotes consistently --- ftw/errors.py | 2 +- ftw/http.py | 4 +- ftw/pytest_plugin.py | 2 +- ftw/testrunner.py | 4 +- ftw/util.py | 4 +- ftw/util/ironbee.py | 6 +- ftw/util/request_to_yaml.py | 24 ++-- test/integration/test_http.py | 228 +++++++++++++++++----------------- test/unit/test_response.py | 4 +- 9 files changed, 139 insertions(+), 139 deletions(-) diff --git a/ftw/errors.py b/ftw/errors.py index d129cab..7d1ee9c 100644 --- a/ftw/errors.py +++ b/ftw/errors.py @@ -1,3 +1,3 @@ class TestError(Exception): def __init___(self, msg, context_args): - Exception.__init__(self, "{0} {1}".format(msg, context_args)) + Exception.__init__(self, '{0} {1}'.format(msg, context_args)) diff --git a/ftw/http.py b/ftw/http.py index e375d84..463b98f 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -20,7 +20,7 @@ # Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. -PROTOCOL_TLS = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23) +PROTOCOL_TLS = getattr(ssl, 'PROTOCOL_TLS', ssl.PROTOCOL_SSLv23) class HttpResponse(object): @@ -406,7 +406,7 @@ def build_request(self): # If we have data append it if self.request_object.data != '': # Before we do that see if that is a charset - encoding = "utf-8" + encoding = 'utf-8' # Check to see if we have a content type and magic is # off (otherwise UTF-8) if 'Content-Type' in list(self.request_object.headers.keys()) and \ diff --git a/ftw/pytest_plugin.py b/ftw/pytest_plugin.py index 078cc16..d6ebf16 100644 --- a/ftw/pytest_plugin.py +++ b/ftw/pytest_plugin.py @@ -33,7 +33,7 @@ def test_id(val): if 'name' in list(val.ruleset_meta.keys()): return '%s -- %s' % (val.ruleset_meta['name'], val.test_title) else: - return '%s -- %s' % ("Unnamed_Test", val.test_title) + return '%s -- %s' % ('Unnamed_Test', val.test_title) @pytest.fixture diff --git a/ftw/testrunner.py b/ftw/testrunner.py index d11cfdc..5f1c907 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -97,10 +97,10 @@ def run_stage_with_journal(self, rule_id, test, journal_file, conn.text_factory = str cur = conn.cursor() for i, stage in enumerate(test.stages): - ''' + """ Query DB here for rule_id & test_title Compare against logger_obj - ''' + """ q = self.query_for_stage_results(tablename) results = cur.execute(q, [i, test.test_title]).fetchall() if len(results) == 0: diff --git a/ftw/util.py b/ftw/util.py index 6573423..d15f48c 100644 --- a/ftw/util.py +++ b/ftw/util.py @@ -106,7 +106,7 @@ def ensure_str(s, encoding='utf-8', errors='strict'): if isinstance(s, bytes): return s.decode(encoding, errors) elif not isinstance(s, (str, bytes)): - raise TypeError("not expecting type '%s'" % type(s)) + raise TypeError('not expecting type "%s"' % type(s)) def ensure_binary(s, encoding='utf-8', errors='strict'): @@ -114,4 +114,4 @@ def ensure_binary(s, encoding='utf-8', errors='strict'): return s if isinstance(s, str): return s.encode(encoding, errors) - raise TypeError("not expecting type '%s'" % type(s)) + raise TypeError('not expecting type "%s"' % type(s)) diff --git a/ftw/util/ironbee.py b/ftw/util/ironbee.py index 59de95b..6f40bd1 100644 --- a/ftw/util/ironbee.py +++ b/ftw/util/ironbee.py @@ -4,15 +4,15 @@ filelist = [] -for root, dirs, files in os.walk("waf-research", topdown=False): +for root, dirs, files in os.walk('waf-research', topdown=False): for name in files: extension = name[-5:] - if extension == ".test": + if extension == '.test': filelist.append(os.path.join(root, name)) for fname in filelist: f = open(fname, 'r') - request = "" + request = '' for line in f.readlines(): if line[0] != '#': request += line diff --git a/ftw/util/request_to_yaml.py b/ftw/util/request_to_yaml.py index 24edd25..7ba43d3 100644 --- a/ftw/util/request_to_yaml.py +++ b/ftw/util/request_to_yaml.py @@ -9,15 +9,15 @@ def __init__(self): def double_quote(self, mystr): return mystr - return "\"" + str(mystr) + "\"" + return '"' + str(mystr) + '"' def generate_yaml(self): data = dict( meta=dict( - author="Zack", + author='Zack', enabled=True, - name="EXAMPLE.yaml", - description="Description" + name='EXAMPLE.yaml', + description='Description' ), tests=[dict( rule_id=1234, @@ -36,9 +36,9 @@ def generate_yaml(self): def get_request_line(self, req): req = req.split('\r\n')[0] req = req.split(' ', 2) - self.input["method"] = self.double_quote(req[0]) - self.input["uri"] = self.double_quote(req[1]) - self.input["version"] = self.double_quote(req[2]) + self.input['method'] = self.double_quote(req[0]) + self.input['uri'] = self.double_quote(req[1]) + self.input['version'] = self.double_quote(req[2]) def get_headers(self, req): req = req.split('\r\n')[1:] @@ -49,12 +49,12 @@ def get_headers(self, req): break head = req[num].split(':') header[head[0]] = head[1].strip() - self.input["headers"] = self.double_quote(header) + self.input['headers'] = self.double_quote(header) def get_data(self, req): req = req.split('\r\n')[1:] - self.input["data"] = self.double_quote( - "\r\n".join(req[self.data_start + 1:])) + self.input['data'] = self.double_quote( + '\r\n'.join(req[self.data_start + 1:])) def write_yaml(self, fname, yaml_out): f = open(fname, 'w') @@ -63,12 +63,12 @@ def write_yaml(self, fname, yaml_out): # Example Usage # req = Request() # -# request = """GET / HTTP/1.1 +# request = '''GET / HTTP/1.1 # User-Agent: test:/data # # xyz # -# """ +# ''' # request = request.replace('\n', '\r\n') # req.get_request_line(request) # req.get_headers(request) diff --git a/test/integration/test_http.py b/test/integration/test_http.py index d5d439e..d545d92 100644 --- a/test/integration/test_http.py +++ b/test/integration/test_http.py @@ -8,53 +8,53 @@ def test_cookies1(): """Tests accessing a site that sets a cookie and then wants to resend the cookie""" http_ua = http.HttpUA() - x = ruleset.Input(protocol="https", port=443, dest_addr="www.ieee.org", - headers={"Host": "www.ieee.org"}) + x = ruleset.Input(protocol='https', port=443, dest_addr='www.ieee.org', + headers={'Host': 'www.ieee.org'}) http_ua.send_request(x) with pytest.raises(KeyError): - print(http_ua.request_object.headers["cookie"]) - assert("set-cookie" in list(http_ua.response_object.headers.keys())) - cookie_data = http_ua.response_object.headers["set-cookie"] - cookie_var = cookie_data.split("=")[0] - x = ruleset.Input(protocol="https", port=443, dest_addr="www.ieee.org", - headers={"Host": "www.ieee.org"}) - http_ua.send_request(x) - assert(http_ua.request_object.headers["cookie"].split('=')[0] == + print(http_ua.request_object.headers['cookie']) + assert('set-cookie' in list(http_ua.response_object.headers.keys())) + cookie_data = http_ua.response_object.headers['set-cookie'] + cookie_var = cookie_data.split('=')[0] + x = ruleset.Input(protocol='https', port=443, dest_addr='www.ieee.org', + headers={'Host': 'www.ieee.org'}) + http_ua.send_request(x) + assert(http_ua.request_object.headers['cookie'].split('=')[0] == cookie_var) def test_cookies2(): """Test to make sure that we don't override user specified cookies""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr="ieee.org", headers={"Host": "ieee.org"}) + x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr="ieee.org", + x = ruleset.Input(dest_addr='ieee.org', headers={ - "Host": "ieee.org", - "cookie": "TS01247332=012f3506234413e6c5cb14e8c0" - "d5bf890fdd02481614b01cd6cd30911c6733e" - "3e6f79e72aa"}) + 'Host': 'ieee.org', + 'cookie': 'TS01247332=012f3506234413e6c5cb14e8c0' + 'd5bf890fdd02481614b01cd6cd30911c6733e' + '3e6f79e72aa'}) http_ua.send_request(x) assert('TS01247332=012f3506234413e6c5cb14e8c0d5bf890fdd02481614b01cd6c' 'd30911c6733e3e6f79e72aa' in - http_ua.request_object.headers["cookie"]) + http_ua.request_object.headers['cookie']) def test_cookies3(): """Test to make sure we retain cookies when user specified values are provided""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr="ieee.org", headers={"Host": "ieee.org"}) + x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr="ieee.org", + x = ruleset.Input(dest_addr='ieee.org', headers={ - "Host": "ieee.org", - "cookie": "TS01247332=012f3506234413e6c5cb14e8c0d" - "5bf890fdd02481614b01cd6cd30911c6733e3e" - "6f79e72aa; XYZ=123"}) + 'Host': 'ieee.org', + 'cookie': 'TS01247332=012f3506234413e6c5cb14e8c0d' + '5bf890fdd02481614b01cd6cd30911c6733e3e' + '6f79e72aa; XYZ=123'}) http_ua.send_request(x) assert(set([chunk.split('=')[0].strip() for chunk in - http_ua.request_object.headers["cookie"].split(';')]) == + http_ua.request_object.headers['cookie'].split(';')]) == set(['XYZ', 'TS01247332'])) @@ -62,19 +62,19 @@ def test_cookies4(): """Test to make sure cookies are saved when user-specified cookie is added""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr="ieee.org", headers={"Host": "ieee.org"}) + x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr="ieee.org", headers={"Host": "ieee.org", - "cookie": "XYZ=123"}) + x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org', + 'cookie': 'XYZ=123'}) http_ua.send_request(x) - assert('XYZ' in http_ua.request_object.headers["cookie"]) + assert('XYZ' in http_ua.request_object.headers['cookie']) def test_raw1(): """Test to make sure a raw request will work with \r\n replacement""" - x = ruleset.Input(dest_addr="example.com", - raw_request="""GET / HTTP/1.1\r\n""" - """Host: example.com\r\n\r\n""") + x = ruleset.Input(dest_addr='example.com', + raw_request='GET / HTTP/1.1\r\n' \ + 'Host: example.com\r\n\r\n') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -83,11 +83,11 @@ def test_raw1(): @pytest.mark.skip(reason='Integration failure, @chaimsanders for more info') def test_raw2(): """Test to make sure a raw request will work with actual seperators""" - x = ruleset.Input(dest_addr="example.com", raw_request="""GET / HTTP/1.1 + x = ruleset.Input(dest_addr='example.com', raw_request='''GET / HTTP/1.1 Host: example.com -""") +''') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -96,10 +96,10 @@ def test_raw2(): def test_both1(): """Test to make sure that if both encoded and raw are provided there is an error""" - x = ruleset.Input(dest_addr="example.com", - raw_request="""GET / HTTP/1.1\r\n""" - """Host: example.com\r\n\r\n""", - encoded_request="abc123==") + x = ruleset.Input(dest_addr='example.com', + raw_request='GET / HTTP/1.1\r\n' \ + 'Host: example.com\r\n\r\n', + encoded_request='abc123==') http_ua = http.HttpUA() with pytest.raises(errors.TestError): http_ua.send_request(x) @@ -107,9 +107,9 @@ def test_both1(): def test_encoded1(): """Test to make sure a encode request works""" - x = ruleset.Input(dest_addr="example.com", - encoded_request="R0VUIC8gSFRUUC8xLjFcclxuSG9zdDogZXhh" - "bXBsZS5jb21cclxuXHJcbg==") + x = ruleset.Input(dest_addr='example.com', + encoded_request='R0VUIC8gSFRUUC8xLjFcclxuSG9zdDogZXhh' + 'bXBsZS5jb21cclxuXHJcbg==') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -117,8 +117,8 @@ def test_encoded1(): def test_error1(): """Will return mail -- not header should cause error""" - x = ruleset.Input(dest_addr="Smtp.aol.com", port=25, - headers={"Host": "example.com"}) + x = ruleset.Input(dest_addr='Smtp.aol.com', port=25, + headers={'Host': 'example.com'}) http_ua = http.HttpUA() with pytest.raises(errors.TestError): http_ua.send_request(x) @@ -128,27 +128,27 @@ def test_error5(): """Invalid Header should cause error""" http_ua = http.HttpUA() with pytest.raises(errors.TestError): - http.HttpResponse("HTTP/1.1 200 OK\r\ntest\r\n", http_ua) + http.HttpResponse('HTTP/1.1 200 OK\r\ntest\r\n', http_ua) def test_error6(): """Valid HTTP response should process fine""" http_ua = http.HttpUA() - http.HttpResponse("HTTP/1.1 200 OK\r\ntest: hello\r\n", http_ua) + http.HttpResponse('HTTP/1.1 200 OK\r\ntest: hello\r\n', http_ua) def test_error7(): """Invalid content-type should fail""" http_ua = http.HttpUA() with pytest.raises(errors.TestError): - http.HttpResponse("HTTP/1.1 200 OK\r\nContent-Encoding: XYZ\r\n", + http.HttpResponse('HTTP/1.1 200 OK\r\nContent-Encoding: XYZ\r\n', http_ua) def test_error2(): """Invalid request should cause timeout""" - x = ruleset.Input(dest_addr="example.com", port=123, - headers={"Host": "example.com"}) + x = ruleset.Input(dest_addr='example.com', port=123, + headers={'Host': 'example.com'}) http_ua = http.HttpUA() with pytest.raises(errors.TestError): http_ua.send_request(x) @@ -158,22 +158,22 @@ def test_error3(): """Invalid status returned in response line""" http_ua = http.HttpUA() with pytest.raises(errors.TestError): - http.HttpResponse("HTTP1.1 test OK\r\n", http_ua) + http.HttpResponse('HTTP1.1 test OK\r\n', http_ua) def test_error4(): """Wrong number of elements returned in response line""" with pytest.raises(errors.TestError): http_ua = http.HttpUA() - http.HttpResponse("HTTP1.1 OK\r\n", http_ua) + http.HttpResponse('HTTP1.1 OK\r\n', http_ua) def test_invalid_gzip(): """Invalid gzip content""" http_ua = http.HttpUA() with pytest.raises(errors.TestError): - http.HttpResponse("HTTP1.1 200 OK\r\n" - "Content-Encoding: gzip\r\n\r\ninvalid data", + http.HttpResponse('HTTP1.1 200 OK\r\n' + 'Content-Encoding: gzip\r\n\r\ninvalid data', http_ua) @@ -181,8 +181,8 @@ def test_invalid_deflate(): """Invalid deflate content""" http_ua = http.HttpUA() with pytest.raises(errors.TestError): - http.HttpResponse("HTTP1.1 200 OK\r\n" - "Content-Encoding: deflate\r\n\r\ninvalid data", + http.HttpResponse('HTTP1.1 200 OK\r\n' + 'Content-Encoding: deflate\r\n\r\ninvalid data', http_ua) @@ -190,14 +190,14 @@ def test_invalid_brotli(): """Invalid brotli content""" http_ua = http.HttpUA() with pytest.raises(errors.TestError): - http.HttpResponse("HTTP1.1 200 OK\r\n" - "Content-Encoding: br\r\n\r\ninvalid data", + http.HttpResponse('HTTP1.1 200 OK\r\n' + 'Content-Encoding: br\r\n\r\ninvalid data', http_ua) def test1(): """Typical request specified should be valid""" - x = ruleset.Input(dest_addr="example.com", headers={"Host": "example.com"}) + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com'}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -205,7 +205,7 @@ def test1(): def test2(): """Basic GET without Host on 1.1 - Expect 400""" - x = ruleset.Input(dest_addr="example.com", headers={}) + x = ruleset.Input(dest_addr='example.com', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 400 @@ -213,7 +213,7 @@ def test2(): def test3(): """Basic GET without Host on 1.0 - Expect 404 (server is VHosted)""" - x = ruleset.Input(dest_addr="example.com", version="HTTP/1.0", headers={}) + x = ruleset.Input(dest_addr='example.com', version='HTTP/1.0', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 404 @@ -221,8 +221,8 @@ def test3(): def test4(): """Basic GET wit Host on 1.0 - Expect 200""" - x = ruleset.Input(dest_addr="example.com", version="HTTP/1.0", - headers={"Host": "example.com"}) + x = ruleset.Input(dest_addr='example.com', version='HTTP/1.0', + headers={'Host': 'example.com'}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -230,7 +230,7 @@ def test4(): def test5(): """Basic GET without Host on 0.9 - Expect 505 version not supported""" - x = ruleset.Input(dest_addr="example.com", version="HTTP/0.9", headers={}) + x = ruleset.Input(dest_addr='example.com', version='HTTP/0.9', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 505 @@ -239,7 +239,7 @@ def test5(): def test6(): """Basic GET without Host with invalid version (request line) - Expect 505 not supported""" - x = ruleset.Input(dest_addr="example.com", version="HTTP/1.0 x", + x = ruleset.Input(dest_addr='example.com', version='HTTP/1.0 x', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) @@ -248,8 +248,8 @@ def test6(): def test7(): """TEST method which doesn't exist - Expect 501""" - x = ruleset.Input(method="TEST", dest_addr="example.com", - version="HTTP/1.0", headers={}) + x = ruleset.Input(method='TEST', dest_addr='example.com', + version='HTTP/1.0', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 501 @@ -257,8 +257,8 @@ def test7(): def test8(): """PROPFIND method which isn't allowed - Expect 405""" - x = ruleset.Input(method="PROPFIND", dest_addr="example.com", - version="HTTP/1.0", headers={}) + x = ruleset.Input(method='PROPFIND', dest_addr='example.com', + version='HTTP/1.0', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 405 @@ -266,8 +266,8 @@ def test8(): def test9(): """OPTIONS method - Expect 200""" - x = ruleset.Input(method="OPTIONS", dest_addr="example.com", - version="HTTP/1.0", headers={}) + x = ruleset.Input(method='OPTIONS', dest_addr='example.com', + version='HTTP/1.0', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -275,8 +275,8 @@ def test9(): def test10(): """HEAD method - Expect 200""" - x = ruleset.Input(method="HEAD", dest_addr="example.com", - version="HTTP/1.0", headers={"Host": "example.com"}) + x = ruleset.Input(method='HEAD', dest_addr='example.com', + version='HTTP/1.0', headers={'Host': 'example.com'}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -284,8 +284,8 @@ def test10(): def test11(): """POST method no data - Expect 411""" - x = ruleset.Input(method="POST", dest_addr="example.com", - version="HTTP/1.0", headers={}) + x = ruleset.Input(method='POST', dest_addr='example.com', + version='HTTP/1.0', headers={}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 411 @@ -293,10 +293,10 @@ def test11(): def test12(): """POST method no data with content length header - Expect 200""" - x = ruleset.Input(method="POST", dest_addr="example.com", - version="HTTP/1.0", - headers={"Content-Length": "0", "Host": "example.com"}, - data="") + x = ruleset.Input(method='POST', dest_addr='example.com', + version='HTTP/1.0', + headers={'Content-Length': '0', 'Host': 'example.com'}, + data='') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -304,8 +304,8 @@ def test12(): def test13(): """Request https on port 80 (default)""" - x = ruleset.Input(protocol="https", dest_addr="example.com", - headers={"Host": "example.com"}) + x = ruleset.Input(protocol='https', dest_addr='example.com', + headers={'Host': 'example.com'}) http_ua = http.HttpUA() with pytest.raises(errors.TestError): http_ua.send_request(x) @@ -313,8 +313,8 @@ def test13(): def test14(): """Request https on port 443 should work""" - x = ruleset.Input(protocol="https", port=443, dest_addr="example.com", - headers={"Host": "example.com"}) + x = ruleset.Input(protocol='https', port=443, dest_addr='example.com', + headers={'Host': 'example.com'}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -322,12 +322,12 @@ def test14(): def test15(): """Request with content-type and content-length specified""" - x = ruleset.Input(method="POST", protocol="http", port=80, - dest_addr="example.com", + x = ruleset.Input(method='POST', protocol='http', port=80, + dest_addr='example.com', headers={ - "Content-Type": "application/x-www-form-urlencoded", - "Host": "example.com", "Content-Length": "7"}, - data="test=hi") + 'Content-Type': 'application/x-www-form-urlencoded', + 'Host': 'example.com', 'Content-Length': '7'}, + data='test=hi') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -335,12 +335,12 @@ def test15(): def test16(): """Post request with content-type but not content-length""" - x = ruleset.Input(method="POST", protocol="http", port=80, - dest_addr="example.com", + x = ruleset.Input(method='POST', protocol='http', port=80, + dest_addr='example.com', headers={ - "Content-Type": "application/x-www-form-urlencoded", - "Host": "example.com"}, - data="test=hi") + 'Content-Type': 'application/x-www-form-urlencoded', + 'Host': 'example.com'}, + data='test=hi') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -348,9 +348,9 @@ def test16(): def test17(): """Post request with no content-type AND no content-length""" - x = ruleset.Input(method="POST", protocol="http", port=80, uri="/", - dest_addr="example.com", - headers={"Host": "example.com"}, data="test=hi") + x = ruleset.Input(method='POST', protocol='http', port=80, uri='/', + dest_addr='example.com', + headers={'Host': 'example.com'}, data='test=hi') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -358,53 +358,53 @@ def test17(): def test18(): """Send a request and check that the space is encoded automagically""" - x = ruleset.Input(method="POST", protocol="http", port=80, - uri="/", dest_addr="example.com", - headers={"Host": "example.com"}, - data="test=hit f&test2=hello") + x = ruleset.Input(method='POST', protocol='http', port=80, + uri='/', dest_addr='example.com', + headers={'Host': 'example.com'}, + data='test=hit f&test2=hello') http_ua = http.HttpUA() http_ua.send_request(x) - assert http_ua.request_object.data == "test=hit+f&test2=hello" + assert http_ua.request_object.data == 'test=hit+f&test2=hello' def test19(): """Send a raw question mark and test it is encoded automagically""" - x = ruleset.Input(method="POST", protocol="http", port=80, uri="/", - dest_addr="example.com", - headers={"Host": "example.com"}, data="test=hello?x") + x = ruleset.Input(method='POST', protocol='http', port=80, uri='/', + dest_addr='example.com', + headers={'Host': 'example.com'}, data='test=hello?x') http_ua = http.HttpUA() http_ua.send_request(x) - assert http_ua.request_object.data == "test=hello%3Fx" + assert http_ua.request_object.data == 'test=hello%3Fx' def test_brotli(): """Accept-Encoding br""" - x = ruleset.Input(dest_addr="httpbin.org", uri="/brotli", - headers={"Host": "httpbin.org", - "Accept-Encoding": "br"}) + x = ruleset.Input(dest_addr='httpbin.org', uri='/brotli', + headers={'Host': 'httpbin.org', + 'Accept-Encoding': 'br'}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 - assert http_ua.response_object.headers["content-encoding"] == "br" + assert http_ua.response_object.headers['content-encoding'] == 'br' def test_deflate(): """Accept-Encoding deflate""" - x = ruleset.Input(dest_addr="example.com", version="HTTP/1.0", - headers={"Host": "example.com", - "Accept-Encoding": "deflate"}) + x = ruleset.Input(dest_addr='example.com', version='HTTP/1.0', + headers={'Host': 'example.com', + 'Accept-Encoding': 'deflate'}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 - assert http_ua.response_object.headers["content-encoding"] == "deflate" + assert http_ua.response_object.headers['content-encoding'] == 'deflate' def test_gzip(): """Accept-Encoding gzip""" - x = ruleset.Input(dest_addr="example.com", version="HTTP/1.0", - headers={"Host": "example.com", - "Accept-Encoding": "gzip"}) + x = ruleset.Input(dest_addr='example.com', version='HTTP/1.0', + headers={'Host': 'example.com', + 'Accept-Encoding': 'gzip'}) http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 - assert http_ua.response_object.headers["content-encoding"] == "gzip" + assert http_ua.response_object.headers['content-encoding'] == 'gzip' diff --git a/test/unit/test_response.py b/test/unit/test_response.py index fc92776..e673ee6 100644 --- a/test/unit/test_response.py +++ b/test/unit/test_response.py @@ -16,7 +16,7 @@ def test_response_failure(): http_ua = http.HttpUA() with pytest.raises(AssertionError): runner.test_response(http.HttpResponse( - "HTTP/1.1 200 OK\r\n\r\ncat", http_ua), + 'HTTP/1.1 200 OK\r\n\r\ncat', http_ua), re.compile('dog')) @@ -24,5 +24,5 @@ def test_response_success(): runner = testrunner.TestRunner() http_ua = http.HttpUA() runner.test_response(http.HttpResponse( - "HTTP/1.1 200 OK\r\n\r\ncat", http_ua), + 'HTTP/1.1 200 OK\r\n\r\ncat', http_ua), re.compile('cat')) From 83b210d2e4d7c88e39ee38fa5dec6a143b50d652 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Wed, 18 Nov 2020 19:47:02 -0300 Subject: [PATCH 26/38] feat(pypi): adds publishing using twine Signed-off-by: Felipe Zipitria --- .github/workflows/publish.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..87ce4bb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,28 @@ +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* From bb57caca0f61cba8ff478986bacfc45423e00751 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Tue, 23 Mar 2021 20:28:17 -0300 Subject: [PATCH 27/38] fix(encoding): remove encoding for base64 encoded request Signed-off-by: Felipe Zipitria --- ftw/http.py | 14 +++++++++----- test/integration/test_http.py | 11 +---------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 463b98f..58e3366 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -437,9 +437,12 @@ def build_request(self): 'data': str(self.request_object.data), 'function': 'http.HttpResponse.build_request' }) - request = request.replace('#data#', util.ensure_str(data_bytes)) + data = util.ensure_str(data_bytes) else: - request = request.replace('#data#', '') + data = '' + + request = request.replace('#data#', data).encode('utf-8', 'strict') + # If we have a Raw Request we should use that instead if self.request_object.raw_request is not None: if self.request_object.encoded_request is not None: @@ -452,11 +455,12 @@ def build_request(self): # We do this regardless of magic if you want to send a literal # '\' 'r' or 'n' use encoded request. request = request.decode('unicode_escape') + if self.request_object.encoded_request is not None: request = base64.b64decode(self.request_object.encoded_request) - request = request.decode('unicode_escape') - # if we have an Encoded request we should use that - self.request = request.encode('utf-8', 'strict') + + # Use the request created + self.request = util.ensure_binary(request) def get_response(self): """ diff --git a/test/integration/test_http.py b/test/integration/test_http.py index d545d92..d2bd8d1 100644 --- a/test/integration/test_http.py +++ b/test/integration/test_http.py @@ -108,8 +108,7 @@ def test_both1(): def test_encoded1(): """Test to make sure a encode request works""" x = ruleset.Input(dest_addr='example.com', - encoded_request='R0VUIC8gSFRUUC8xLjFcclxuSG9zdDogZXhh' - 'bXBsZS5jb21cclxuXHJcbg==') + encoded_request='R0VUIC8gSFRUUC8xLjENCkhvc3Q6IGV4YW1wbGUuY29tDQoNCgo=') http_ua = http.HttpUA() http_ua.send_request(x) assert http_ua.response_object.status == 200 @@ -228,14 +227,6 @@ def test4(): assert http_ua.response_object.status == 200 -def test5(): - """Basic GET without Host on 0.9 - Expect 505 version not supported""" - x = ruleset.Input(dest_addr='example.com', version='HTTP/0.9', headers={}) - http_ua = http.HttpUA() - http_ua.send_request(x) - assert http_ua.response_object.status == 505 - - def test6(): """Basic GET without Host with invalid version (request line) - Expect 505 not supported""" From 328932d877c8e9510eb77aa17c298940d6d83966 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Tue, 23 Mar 2021 22:02:36 -0300 Subject: [PATCH 28/38] build: use git tag for version release Signed-off-by: Felipe Zipitria --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9df9859..006a82b 100644 --- a/setup.py +++ b/setup.py @@ -3,12 +3,10 @@ from setuptools import setup setup(name='ftw', - version='1.2.3', description='Framework for Testing WAFs', author='Chaim Sanders, Zack Allen', author_email='zma4580@gmail.com, chaim.sanders@gmail.com', url='https://www.github.com/coreruleset/ftw', - download_url='https://github.com/coreruleset/ftw/tarball/1.2.3', include_package_data=True, package_data={ 'ftw': ['util/public_suffix_list.dat'] @@ -20,6 +18,8 @@ }, packages=['ftw'], keywords=['waf'], + use_scm_version=True, + setup_requires=['setuptools_scm'], install_requires=[ 'Brotli==1.0.7', 'IPy==0.83', From db5c3c2e2981852b7aecbe0ccc47233c6cf52faa Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Thu, 25 Mar 2021 10:22:43 -0300 Subject: [PATCH 29/38] docs: enhance pypi documentation Signed-off-by: Felipe Zipitria --- pyproject.toml | 6 +++++ setup.py | 67 ++++++++++++++++++++++++++++++-------------------- 2 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..374b58c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 006a82b..3167678 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,44 @@ -#!/usr/bin/env python +import setuptools -from setuptools import setup +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() -setup(name='ftw', - description='Framework for Testing WAFs', - author='Chaim Sanders, Zack Allen', - author_email='zma4580@gmail.com, chaim.sanders@gmail.com', - url='https://www.github.com/coreruleset/ftw', - include_package_data=True, - package_data={ +setuptools.setup( + name='ftw', + description='Framework for Testing WAFs', + long_description=long_description, + long_description_content_type="text/markdown", + author='Chaim Sanders, Zack Allen', + author_email='zma4580@gmail.com, chaim.sanders@gmail.com', + url='https://github.com/coreruleset/ftw', + include_package_data=True, + package_data={ 'ftw': ['util/public_suffix_list.dat'] - }, - entry_points={ - 'pytest11': [ - 'ftw = ftw.pytest_plugin' - ] - }, - packages=['ftw'], - keywords=['waf'], - use_scm_version=True, - setup_requires=['setuptools_scm'], - install_requires=[ - 'Brotli==1.0.7', - 'IPy==0.83', - 'PyYAML==4.2b1', - 'pytest==4.6', - 'python-dateutil==2.6.0' - ]) + }, + entry_points={ + 'pytest11': [ + 'ftw = ftw.pytest_plugin' + ] + }, + keywords=['waf'], + project_urls={ + "Bug Tracker": 'https://github.com/coreruleset/ftw/issues', + }, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Framework :: Pytest", + ], + packages=["ftw"], + python_requires=">=3.6", + use_scm_version=True, + setup_requires=['setuptools_scm'], + install_requires=[ + 'Brotli==1.0.7', + 'IPy==0.83', + 'PyYAML==4.2b1', + 'pytest==4.6', + 'python-dateutil==2.6.0' + ], +) From cc23968c00f73ffd2fba702b7719b7e1bea5e207 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Mar 2021 21:27:06 +0000 Subject: [PATCH 30/38] Bump pyyaml from 4.2b1 to 5.4 Bumps [pyyaml](https://github.com/yaml/pyyaml) from 4.2b1 to 5.4. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/commits/5.4) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a5d2a37..1bf435b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ Brotli==1.0.7 IPy==0.83 -PyYAML==4.2b1 +PyYAML==5.4 pytest==4.6 python-dateutil==2.6.0 diff --git a/setup.py b/setup.py index 3167678..5964e47 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires=[ 'Brotli==1.0.7', 'IPy==0.83', - 'PyYAML==4.2b1', + 'PyYAML==5.4', 'pytest==4.6', 'python-dateutil==2.6.0' ], From 7f907db17baa98059ebb19cc4a8fea0344ad26cc Mon Sep 17 00:00:00 2001 From: Max Leske Date: Tue, 8 Mar 2022 22:29:41 +0100 Subject: [PATCH 31/38] Made tests more reliable, enabled and fixed some tests --- test/integration/COOKIEFIXTURE.yaml | 105 +++++++++++++++++----------- test/integration/test_cookie.py | 5 ++ test/integration/test_http.py | 26 ++++--- 3 files changed, 83 insertions(+), 53 deletions(-) diff --git a/test/integration/COOKIEFIXTURE.yaml b/test/integration/COOKIEFIXTURE.yaml index e5676e6..f700204 100644 --- a/test/integration/COOKIEFIXTURE.yaml +++ b/test/integration/COOKIEFIXTURE.yaml @@ -1,40 +1,67 @@ --- - meta: - author: "Chaim" - enabled: true - name: "COOKIEFIXTURE.yaml" - description: "Tests cookie saving functionality" - tests: - - - test_title: "Multi-Stage w\\ Cookie" - stages: - - - stage: - input: - save_cookie: true - dest_addr: "www.ieee.org" - method: "GET" - port: 443 - headers: - User-Agent: "Foo" - Host: "www.ieee.org" - protocol: "https" - uri: "/" - output: - status: 200 - response_contains: "Set-Cookie: TS01247332=" - - - stage: - input: - save_cookie: true - dest_addr: "www.ieee.org" - method: "GET" - port: 443 - headers: - User-Agent: "Foo" - Host: "www.ieee.org" - protocol: "https" - uri: "/" - output: - status: 200 - response_contains: "Set-Cookie: TS01247332=" +meta: + author: "Chaim" + enabled: true + name: "COOKIEFIXTURE.yaml" + description: "Tests cookie saving functionality" +tests: + - test_title: "Multi-Stage w\\ Cookie" + stages: + - stage: + input: + save_cookie: true + dest_addr: "www.cloudflare.com" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.cloudflare.com" + protocol: "https" + uri: "/" + output: + status: 200 + response_contains: "[Ss]et-[Cc]ookie: __cf_bm=" + - stage: + input: + save_cookie: true + dest_addr: "www.cloudflare.com" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.cloudflare.com" + protocol: "https" + uri: "/" + output: + status: 200 + no_response_contains: "[Ss]et-[Cc]ookie: __cf_bm=" + - test_title: "Multi-Stage w\\ Cookie; failure because the cookie is reset if not all cookies are present and ftw can only handle one cookie header" + stages: + - stage: + input: + save_cookie: true + dest_addr: "www.ieee.org" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.ieee.org" + protocol: "https" + uri: "/" + output: + status: 200 + response_contains: "[Ss]et-[Cc]ookie: TS01247332=" + - stage: + input: + save_cookie: true + dest_addr: "www.ieee.org" + method: "GET" + port: 443 + headers: + User-Agent: "Foo" + Host: "www.ieee.org" + protocol: "https" + uri: "/" + output: + status: 200 + no_response_contains: "[Ss]et-[Cc]ookie: TS01247332=" diff --git a/test/integration/test_cookie.py b/test/integration/test_cookie.py index 0dd9f8a..bf36434 100644 --- a/test/integration/test_cookie.py +++ b/test/integration/test_cookie.py @@ -2,6 +2,11 @@ import pytest +@pytest.mark.skip( + reason=""" + 1. ieee.org has a very bad web server, so responses fail a lot + 2. ieee.org sends multiple set-cookie headers and ftw can only handle a single header of the same name""" +) def test_default(ruleset, test, destaddr): """ Default tester with no logger obj. Useful for HTML contains and Status code diff --git a/test/integration/test_http.py b/test/integration/test_http.py index d2bd8d1..dc656aa 100644 --- a/test/integration/test_http.py +++ b/test/integration/test_http.py @@ -3,21 +3,20 @@ import pytest -@pytest.mark.skip(reason='Integration failure, @chaimsanders for more info') def test_cookies1(): """Tests accessing a site that sets a cookie and then wants to resend the cookie""" http_ua = http.HttpUA() - x = ruleset.Input(protocol='https', port=443, dest_addr='www.ieee.org', - headers={'Host': 'www.ieee.org'}) + x = ruleset.Input(protocol='https', port=443, dest_addr='www.cloudflare.com', + headers={'Host': 'www.cloudflare.com'}) http_ua.send_request(x) with pytest.raises(KeyError): print(http_ua.request_object.headers['cookie']) assert('set-cookie' in list(http_ua.response_object.headers.keys())) cookie_data = http_ua.response_object.headers['set-cookie'] cookie_var = cookie_data.split('=')[0] - x = ruleset.Input(protocol='https', port=443, dest_addr='www.ieee.org', - headers={'Host': 'www.ieee.org'}) + x = ruleset.Input(protocol='https', port=443, dest_addr='www.cloudflare.com', + headers={'Host': 'www.cloudflare.com'}) http_ua.send_request(x) assert(http_ua.request_object.headers['cookie'].split('=')[0] == cookie_var) @@ -26,11 +25,11 @@ def test_cookies1(): def test_cookies2(): """Test to make sure that we don't override user specified cookies""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr='ieee.org', + x = ruleset.Input(dest_addr='example.com', headers={ - 'Host': 'ieee.org', + 'Host': 'example.com', 'cookie': 'TS01247332=012f3506234413e6c5cb14e8c0' 'd5bf890fdd02481614b01cd6cd30911c6733e' '3e6f79e72aa'}) @@ -44,11 +43,11 @@ def test_cookies3(): """Test to make sure we retain cookies when user specified values are provided""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr='ieee.org', + x = ruleset.Input(dest_addr='example.com', headers={ - 'Host': 'ieee.org', + 'Host': 'example.com', 'cookie': 'TS01247332=012f3506234413e6c5cb14e8c0d' '5bf890fdd02481614b01cd6cd30911c6733e3e' '6f79e72aa; XYZ=123'}) @@ -62,9 +61,9 @@ def test_cookies4(): """Test to make sure cookies are saved when user-specified cookie is added""" http_ua = http.HttpUA() - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org'}) + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com'}) http_ua.send_request(x) - x = ruleset.Input(dest_addr='ieee.org', headers={'Host': 'ieee.org', + x = ruleset.Input(dest_addr='example.com', headers={'Host': 'example.com', 'cookie': 'XYZ=123'}) http_ua.send_request(x) assert('XYZ' in http_ua.request_object.headers['cookie']) @@ -80,7 +79,6 @@ def test_raw1(): assert http_ua.response_object.status == 200 -@pytest.mark.skip(reason='Integration failure, @chaimsanders for more info') def test_raw2(): """Test to make sure a raw request will work with actual seperators""" x = ruleset.Input(dest_addr='example.com', raw_request='''GET / HTTP/1.1 From 8872cab25a9db7ff0934a228669c216ba1d1bebc Mon Sep 17 00:00:00 2001 From: Max Leske Date: Fri, 4 Mar 2022 08:47:58 +0100 Subject: [PATCH 32/38] Updated dependencies Added test build for Python 3.10 Split from PR #66, part 2 --- .github/workflows/ci.yml | 4 ++-- requirements.txt | 10 +++++----- setup.cfg | 2 +- setup.py | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ede2e0..be7cfb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [ '3.6', '3.7', '3.8', '3.9' ] + python-version: [ '3.6', '3.7', '3.8', '3.9', '3.10' ] steps: - name: Checkout repo @@ -23,7 +23,7 @@ jobs: python -m pip install -r requirements.txt python setup.py install - name: Check source files - if: matrix.python-version == '3.9' + if: matrix.python-version == '3.10' run: | python -m pip install pytest-pycodestyle python -m pip install pytest-flakes diff --git a/requirements.txt b/requirements.txt index 1bf435b..b46f7dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -Brotli==1.0.7 -IPy==0.83 -PyYAML==5.4 -pytest==4.6 -python-dateutil==2.6.0 +Brotli==1.0.9 +IPy==1.01 +PyYAML==6.0 +pytest==6.2.5 +python-dateutil==2.8.2 diff --git a/setup.cfg b/setup.cfg index cc752ab..8b88600 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -description-file = README.md +description_file = README.md [tool:pytest] addopts = -s -v diff --git a/setup.py b/setup.py index 5964e47..59309d0 100644 --- a/setup.py +++ b/setup.py @@ -35,10 +35,10 @@ use_scm_version=True, setup_requires=['setuptools_scm'], install_requires=[ - 'Brotli==1.0.7', - 'IPy==0.83', - 'PyYAML==5.4', - 'pytest==4.6', - 'python-dateutil==2.6.0' + 'Brotli==1.0.9', + 'IPy==1.01', + 'PyYAML==6.0', + 'pytest==6.2.5', + 'python-dateutil==2.8.2' ], ) From ca0a902124418d232bbbda088b4ba569b5363271 Mon Sep 17 00:00:00 2001 From: Max Leske Date: Fri, 4 Mar 2022 09:04:03 +0100 Subject: [PATCH 33/38] Added mark_start() and mark_end() to give the log checker the chance to pre and post process runs --- ftw/logchecker.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ftw/logchecker.py b/ftw/logchecker.py index 3f3d557..22e0fa5 100644 --- a/ftw/logchecker.py +++ b/ftw/logchecker.py @@ -15,6 +15,20 @@ def set_times(self, start, end): self.start = start self.end = end + def mark_start(self, stage_id): + """ + May be implemented to set up the log checker before + the request is being sent + """ + pass + + def mark_end(self, stage_id): + """ + May be implemented to tell the log checker that + the response has been received + """ + pass + @abstractmethod def get_logs(self): """ From b21f687c2a495503deb13de5ddc7861a515e5ce9 Mon Sep 17 00:00:00 2001 From: Max Leske Date: Fri, 4 Mar 2022 09:05:55 +0100 Subject: [PATCH 34/38] Improved test parameterization Give log checker more information, so it can produce identifiable log markers --- ftw/pytest_plugin.py | 21 +++++++++++++++------ ftw/ruleset.py | 20 ++++++++++++++------ ftw/testrunner.py | 18 +++++++++++++----- test/integration/test_logcontains.py | 4 ++-- test/unit/test_ruleset.py | 5 +++-- 5 files changed, 47 insertions(+), 21 deletions(-) diff --git a/ftw/pytest_plugin.py b/ftw/pytest_plugin.py index d6ebf16..63d288c 100644 --- a/ftw/pytest_plugin.py +++ b/ftw/pytest_plugin.py @@ -7,7 +7,7 @@ from .ruleset import Test -def get_testdata(rulesets): +def get_testdata(rulesets, use_rulesets): """ In order to do test-level parametrization (is this a word?), we have to bundle the test data from rulesets into tuples so py.test can understand @@ -17,7 +17,10 @@ def get_testdata(rulesets): for ruleset in rulesets: for test in ruleset.tests: if test.enabled: - testdata.append((ruleset, test)) + args = [test] + if use_rulesets: + args = [rulesets] + args + testdata.append(args) return testdata @@ -127,7 +130,13 @@ def pytest_generate_tests(metafunc): metafunc.config.option.ruledir_recurse, True) if metafunc.config.option.rule: rulesets = util.get_rulesets(metafunc.config.option.rule, False) - if 'ruleset' in metafunc.fixturenames and \ - 'test' in metafunc.fixturenames: - metafunc.parametrize('ruleset, test', get_testdata(rulesets), - ids=test_id) + if 'test' in metafunc.fixturenames: + use_rulesets = False + arg_names = ['test'] + if 'ruleset' in metafunc.fixturenames: + use_rulesets = True + arg_names = ['ruleset'] + arg_names + metafunc.parametrize( + arg_names, + get_testdata(rulesets, use_rulesets), + ids=test_id) diff --git a/ftw/ruleset.py b/ftw/ruleset.py index 04f951c..13f06af 100644 --- a/ftw/ruleset.py +++ b/ftw/ruleset.py @@ -137,18 +137,26 @@ class Stage(object): This class holds information about 1 stage in a test, which contains 1 input and 1 output """ - def __init__(self, stage_dict): + def __init__(self, stage_dict, stage_index, test): self.stage_dict = stage_dict + self.stage_index = stage_index + self.test = test self.input = Input(**stage_dict['input']) self.output = Output(stage_dict['output']) + self.id = self.build_id() + + def build_id(self): + rule_name = self.test.ruleset_meta["name"].split('.')[0] + return f'{rule_name}-{self.test.test_index}-{self.stage_index}' class Test(object): """ This class holds information for 1 test and potentially many stages """ - def __init__(self, test_dict, ruleset_meta): + def __init__(self, test_dict, test_index, ruleset_meta): self.test_dict = test_dict + self.test_index = test_index self.ruleset_meta = ruleset_meta self.test_title = self.test_dict['test_title'] self.stages = self.build_stages() @@ -160,8 +168,8 @@ def build_stages(self): """ Processes and loads an array of stages from the test dictionary """ - return [Stage(stage_dict['stage']) - for stage_dict in self.test_dict['stages']] + return [Stage(stage_dict['stage'], index, self) + for index, stage_dict in enumerate(self.test_dict['stages'])] class Ruleset(object): @@ -183,8 +191,8 @@ def extract_tests(self): creates test objects based on input """ try: - return [Test(test_dict, self.meta) - for test_dict in self.yaml_file['tests']] + return [Test(test_dict, index, self.meta) + for index, test_dict in enumerate(self.yaml_file['tests'])] except errors.TestError as e: e.args[1]['meta'] = self.meta raise e diff --git a/ftw/testrunner.py b/ftw/testrunner.py index 5f1c907..21f6ade 100644 --- a/ftw/testrunner.py +++ b/ftw/testrunner.py @@ -174,7 +174,6 @@ def run_stage(self, stage, logger_obj=None, http_ua=None): input, waits for output then compares expected vs actual output http_ua can be passed in to persist cookies """ - # Send our request (exceptions caught as needed) if stage.output.expect_error: with pytest.raises(errors.TestError) as excinfo: @@ -187,11 +186,20 @@ def run_stage(self, stage, logger_obj=None, http_ua=None): else: if not http_ua: http_ua = http.HttpUA() - start = datetime.datetime.utcnow() + if ((stage.output.log_contains_str or + stage.output.no_log_contains_str) and + logger_obj is not None): + logger_obj.mark_start(stage.id) + start = datetime.datetime.utcnow() http_ua.send_request(stage.input) - end = datetime.datetime.utcnow() - if (stage.output.log_contains_str or - stage.output.no_log_contains_str) and logger_obj is not None: + if ((stage.output.log_contains_str or + stage.output.no_log_contains_str) and + logger_obj is not None): + logger_obj.mark_end(stage.id) + end = datetime.datetime.utcnow() + if ((stage.output.log_contains_str or + stage.output.no_log_contains_str) and + logger_obj is not None): logger_obj.set_times(start, end) lines = logger_obj.get_logs() if stage.output.log_contains_str: diff --git a/test/integration/test_logcontains.py b/test/integration/test_logcontains.py index b76f77b..4549396 100644 --- a/test/integration/test_logcontains.py +++ b/test/integration/test_logcontains.py @@ -27,13 +27,13 @@ def logchecker_obj(): return LoggerTestObj() -def test_logcontains_withlog(logchecker_obj, ruleset, test): +def test_logcontains_withlog(logchecker_obj, test): runner = testrunner.TestRunner() for stage in test.stages: runner.run_stage(stage, logchecker_obj) -def test_logcontains_nolog(logchecker_obj, ruleset, test): +def test_logcontains_nolog(logchecker_obj, test): logchecker_obj.do_nothing = True runner = testrunner.TestRunner() with(pytest.raises(AssertionError)): diff --git a/test/unit/test_ruleset.py b/test/unit/test_ruleset.py index ac15c97..763773a 100644 --- a/test/unit/test_ruleset.py +++ b/test/unit/test_ruleset.py @@ -33,11 +33,12 @@ def test_input(): def test_testobj(): with pytest.raises(KeyError) as excinfo: - ruleset.Test({}, {}) + ruleset.Test({}, {}, {}) assert 'test_title' in str(excinfo.value) + ruleset_meta = {'name': 'test-name.yaml'} stages_dict = {'test_title': 1, 'stages': [{'stage': {'output': {'log_contains': 'foo'}, 'input': {}}}]} - ruleset.Test(stages_dict, {}) + ruleset.Test(stages_dict, {}, ruleset_meta) def test_ruleset(): From 6ffa31210ab23471e7032cfa1d2618d3c00d2aaf Mon Sep 17 00:00:00 2001 From: Max Leske Date: Thu, 10 Mar 2022 21:02:42 +0100 Subject: [PATCH 35/38] Updated deprecated SSL configuration to remove warnings from tests with Python 3.10 --- ftw/http.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index 58e3366..dad0d09 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -19,10 +19,6 @@ from . import util -# Fallback to PROTOCOL_SSLv23 if PROTOCOL_TLS is not available. -PROTOCOL_TLS = getattr(ssl, 'PROTOCOL_TLS', ssl.PROTOCOL_SSLv23) - - class HttpResponse(object): def __init__(self, http_response, user_agent): self.response = util.ensure_binary(http_response) @@ -299,8 +295,9 @@ def build_socket(self): self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Check if TLS if self.request_object.protocol == 'https': - context = ssl.SSLContext(PROTOCOL_TLS) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.set_ciphers(self.CIPHERS) + context.load_default_certs(ssl.Purpose.SERVER_AUTH) self.sock = context.wrap_socket( self.sock, server_hostname=self.request_object.dest_addr) self.sock.connect( From 5509b0e87aac51b4c6ae339e7a6eedf16968d466 Mon Sep 17 00:00:00 2001 From: Max Leske Date: Fri, 4 Mar 2022 08:41:22 +0100 Subject: [PATCH 36/38] Rewrote http.py to be faster, mainly by using select() instead of timeouts Split from PR #66, part 1 --- ftw/http.py | 116 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/ftw/http.py b/ftw/http.py index dad0d09..f13679c 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -9,8 +9,8 @@ import socket import ssl import sys -import time import zlib +import select import brotli from IPy import IP @@ -263,7 +263,6 @@ def __init__(self): 'ADH-AES256-SHA:ECDHE-ECDSA-AES128-GCM-SHA256:' \ 'ECDHE-RSA-AES128-GCM-SHA256:AES128-GCM-SHA256:AES128-SHA256:HIGH:' self.CRLF = '\r\n' - self.HTTP_TIMEOUT = .3 self.RECEIVE_BYTES = 8192 self.SOCKET_TIMEOUT = 5 @@ -463,38 +462,77 @@ def get_response(self): """ Get the response from the socket """ - self.sock.setblocking(0) our_data = [] - # Beginning time - begin = time.time() + self.sock.setblocking(False) + try: + our_data = self.read_response_from_socket() + finally: + try: + self.sock.shutdown(socket.SHUT_WR) + self.sock.close() + except OSError as err: + raise errors.TestError( + 'We were unable to close the socket as expected.', + { + 'msg': err, + 'function': 'http.HttpUA.get_response' + }) + else: + self.response_object = HttpResponse(b''.join(our_data), self) + finally: + if not b''.join(our_data): + raise errors.TestError( + 'No response from server.' + + ' Request likely timed out.', + { + 'host': self.request_object.dest_addr, + 'port': self.request_object.port, + 'proto': self.request_object.protocol, + 'msg': 'Please send the request and check' + + ' Wireshark', + 'function': 'http.HttpUA.get_response' + }) + + def read_response_from_socket(self): + # wait for socket to become ready + ready_sock, _, _ = select.select( + [self.sock], [], [self.sock], self.SOCKET_TIMEOUT) + if not ready_sock: + raise errors.TestError( + f'No response from server within {self.SOCKET_TIMEOUT}s', + { + 'host': self.request_object.dest_addr, + 'port': self.request_object.port, + 'proto': self.request_object.protocol, + 'msg': 'Please send the request and check Wireshark', + 'function': 'http.HttpUA.get_response' + }) + + our_data = [] while True: - # If we have data then if we're passed the timeout break - if our_data and time.time() - begin > self.HTTP_TIMEOUT: - break - # If we're dataless wait just a bit - elif time.time() - begin > self.HTTP_TIMEOUT * 2: - break - # Recv data try: data = self.sock.recv(self.RECEIVE_BYTES) - if data: - our_data.append(util.ensure_binary(data)) - begin = time.time() - else: - # Sleep for sometime to indicate a gap - time.sleep(self.HTTP_TIMEOUT) - except socket.error as err: - # Check if we got a timeout - if err.errno == errno.EAGAIN: - pass + if len(data) == 0: + # we're done + break + our_data.append(util.ensure_binary(data)) + except BlockingIOError as e: + # If we can't handle the error here, pass it on + if e.errno == socket.EAGAIN or e.errno == socket.EWOULDBLOCK: + # we're done + break + except OSError as err: # SSL will return SSLWantRead instead of EAGAIN - elif sys.platform == 'win32' and \ - err.errno == errno.WSAEWOULDBLOCK: + if (sys.platform == 'win32' and + err.errno == errno.WSAEWOULDBLOCK): pass elif (self.request_object.protocol == 'https' and - err.args[0] == ssl.SSL_ERROR_WANT_READ): - continue - # If we didn't it's an error + err.args[0] == ssl.SSL_ERROR_WANT_READ): + ready_sock, _, _ = select.select( + [self.sock], [], [self.sock], .3) + if not ready_sock: + break + # It's an error else: raise errors.TestError( 'Failed to connect to server', @@ -505,26 +543,4 @@ def get_response(self): 'message': err, 'function': 'http.HttpUA.get_response' }) - try: - self.sock.shutdown(socket.SHUT_WR) - self.sock.close() - except socket.error as err: - raise errors.TestError( - 'We were unable to close the socket as expected.', - { - 'msg': err, - 'function': 'http.HttpUA.get_response' - }) - else: - self.response_object = HttpResponse(b''.join(our_data), self) - finally: - if not b''.join(our_data): - raise errors.TestError( - 'No response from server. Request likely timed out.', - { - 'host': self.request_object.dest_addr, - 'port': self.request_object.port, - 'proto': self.request_object.protocol, - 'msg': 'Please send the request and check Wireshark', - 'function': 'http.HttpUA.get_response' - }) + return our_data From 3bff7d6ccd5f8e022f5b90fd99daac6337563951 Mon Sep 17 00:00:00 2001 From: Max Leske Date: Fri, 11 Mar 2022 20:28:48 +0100 Subject: [PATCH 37/38] Use constant for socket timeout --- ftw/http.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ftw/http.py b/ftw/http.py index f13679c..68da291 100644 --- a/ftw/http.py +++ b/ftw/http.py @@ -19,6 +19,9 @@ from . import util +SOCKET_TIMEOUT = .3 + + class HttpResponse(object): def __init__(self, http_response, user_agent): self.response = util.ensure_binary(http_response) @@ -529,7 +532,7 @@ def read_response_from_socket(self): elif (self.request_object.protocol == 'https' and err.args[0] == ssl.SSL_ERROR_WANT_READ): ready_sock, _, _ = select.select( - [self.sock], [], [self.sock], .3) + [self.sock], [], [self.sock], SOCKET_TIMEOUT) if not ready_sock: break # It's an error From 4996d6d4126ca0dab1f5ca2d550345eb68c56398 Mon Sep 17 00:00:00 2001 From: Felipe Zipitria Date: Mon, 16 Mar 2026 13:06:01 -0300 Subject: [PATCH 38/38] fix: harden GitHub Actions workflows - pin all third-party actions to commit SHAs instead of mutable tags - add explicit minimal permissions blocks to all workflows --- .github/workflows/ci.yml | 7 +++++-- .github/workflows/publish.yml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be7cfb3..f9139f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,9 @@ name: ci on: [push, pull_request] +permissions: + contents: read + jobs: test: @@ -13,9 +16,9 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v2 + uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 with: python-version: ${{ matrix.python-version }} - name: Install requirements diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 87ce4bb..c8309e1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,15 +4,18 @@ on: release: types: [created] +permissions: + contents: read + jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 with: python-version: '3.x' - name: Install dependencies