From 7d5b820642580e334c3e5118c2c3992581e5aeb6 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 13 Jun 2022 04:26:01 +0200 Subject: [PATCH 001/695] [quic] first TLS changes --- mitmproxy/addons/tlsconfig.py | 93 +++++++++++++++++++++---- mitmproxy/certs.py | 6 +- mitmproxy/proxy/layers/http/__init__.py | 9 ++- mitmproxy/proxy/layers/http/_http3.py | 31 +++++++++ mitmproxy/proxy/layers/quic.py | 83 ++++++++++++++++++++++ setup.py | 1 + 6 files changed, 207 insertions(+), 16 deletions(-) create mode 100644 mitmproxy/proxy/layers/http/_http3.py create mode 100644 mitmproxy/proxy/layers/quic.py diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 0cb7492e28..61245cd8b5 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -1,15 +1,18 @@ import ipaddress import os from pathlib import Path +import ssl from typing import Any, Optional, TypedDict +from aioquic.quic.configuration import QuicConfiguration +from aioquic.tls import CipherSuite from OpenSSL import SSL from mitmproxy import certs, ctx, exceptions, connection, tls from mitmproxy.net import tls as net_tls from mitmproxy.options import CONF_BASENAME from mitmproxy.proxy import context from mitmproxy.proxy.layers import modes -from mitmproxy.proxy.layers import tls as proxy_tls +from mitmproxy.proxy.layers import tls as proxy_tls, quic # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://ssl-config.mozilla.org/#config=old @@ -196,6 +199,18 @@ def tls_start_client(self, tls_start: tls.TlsData) -> None: ) tls_start.ssl_conn.set_accept_state() + def _get_client_cert(self, server: connection.Server) -> Optional[str]: + if ctx.options.client_certs: + client_certs = os.path.expanduser(ctx.options.client_certs) + if os.path.isfile(client_certs): + return client_certs + else: + server_name: str = server.sni or server.address[0] + p = os.path.join(client_certs, f"{server_name}.pem") + if os.path.isfile(p): + return p + return None + def tls_start_server(self, tls_start: tls.TlsData) -> None: """Establish TLS between proxy and server.""" if tls_start.ssl_conn is not None: @@ -240,17 +255,6 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: # don't assign to client.cipher_list, doesn't need to be stored. cipher_list = server.cipher_list or DEFAULT_CIPHERS - client_cert: Optional[str] = None - if ctx.options.client_certs: - client_certs = os.path.expanduser(ctx.options.client_certs) - if os.path.isfile(client_certs): - client_cert = client_certs - else: - server_name: str = server.sni or server.address[0] - p = os.path.join(client_certs, f"{server_name}.pem") - if os.path.isfile(p): - client_cert = p - ssl_ctx = net_tls.create_proxy_server_context( min_version=net_tls.Version[ctx.options.tls_version_client_min], max_version=net_tls.Version[ctx.options.tls_version_client_max], @@ -258,7 +262,7 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: verify=verify, ca_path=ctx.options.ssl_verify_upstream_trusted_confdir, ca_pemfile=ctx.options.ssl_verify_upstream_trusted_ca, - client_cert=client_cert, + client_cert=self._get_client_cert(server), ) tls_start.ssl_conn = SSL.Connection(ssl_ctx) @@ -293,6 +297,69 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: tls_start.ssl_conn.set_connect_state() + def quic_tls_start_client(self, tls_start: quic.QuicTlsData) -> None: + """Establish QUIC between client and proxy.""" + if tls_start.settings is not None: + return # a user addon has already provided the settings. + tls_start.settings = quic.QuicTlsSettings() + + assert isinstance(tls_start.conn, connection.Client) + + client: connection.Client = tls_start.conn + server: connection.Server = tls_start.context.server + + entry = self.get_cert(tls_start.context) + tls_start.settings.certificate = entry.cert + tls_start.settings.certificate_private_key = entry.privatekey + tls_start.settings.certificate_chain = entry.chain_certs + + if not client.cipher_list and ctx.options.ciphers_client: + client.cipher_list = ctx.options.ciphers_client.split(":") + if client.cipher_list: + tls_start.settings.cipher_suites = [ + CipherSuite(cipher) for cipher in client.cipher_list + ] + if ctx.options.add_upstream_certs_to_client_chain: + tls_start.settings.certificate_chain.extend(server.certificate_list) + + def quic_tls_start_server(self, tls_start: quic.QuicTlsData) -> None: + """Establish QUIC between proxy and server.""" + if tls_start.settings is not None: + return # a user addon has already provided the settings. + tls_start.settings = quic.QuicTlsSettings() + + assert isinstance(tls_start.conn, connection.Server) + + client: connection.Client = tls_start.context.client + server: connection.Server = tls_start.conn + assert server.address + + if ctx.options.ssl_insecure: + tls_start.settings.verify_mode = ssl.CERT_NONE + + if server.sni is None: + server.sni = client.sni or server.address[0] + + if not server.alpn_offers: + server.alpn_offers = client.alpn_offers + + if not server.cipher_list and ctx.options.ciphers_server: + server.cipher_list = ctx.options.ciphers_server.split(":") + tls_start.settings.cipher_suites = [ + CipherSuite(cipher) for cipher in server.cipher_list + ] + + client_cert = self._get_client_cert(server) + if client_cert: + config = QuicConfiguration() + config.load_cert_chain(client_cert) + tls_start.settings.certificate = config.certificate + tls_start.settings.certificate_private_key = config.private_key + tls_start.settings.certificate_chain = config.certificate_chain + + tls_start.settings.ca_path = ctx.options.ssl_verify_upstream_trusted_confdir + tls_start.settings.ca_file = ctx.options.ssl_verify_upstream_trusted_ca + def running(self): # FIXME: We have a weird bug where the contract for configure is not followed and it is never called with # confdir or command_history as updated. diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index dc3787a8b7..ab26b4b8f5 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import NewType, Optional, Union +from aioquic.tls import load_pem_x509_certificates from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa, dsa, ec @@ -283,6 +284,7 @@ class CertStoreEntry: cert: Cert privatekey: rsa.RSAPrivateKey chain_file: Optional[Path] + chain_certs: Optional[list[Cert]] TCustomCertId = str # manually provided certs (e.g. mitmproxy's --certs) @@ -311,6 +313,7 @@ def __init__( self.default_privatekey = default_privatekey self.default_ca = default_ca self.default_chain_file = default_chain_file + self.default_chain_certs = load_pem_x509_certificates(self.default_chain_file.read_bytes()) if self.default_chain_file else None self.dhparams = dhparams self.certs = {} self.expire_queue = [] @@ -453,7 +456,7 @@ def add_cert_file( except ValueError: key = self.default_privatekey - self.add_cert(CertStoreEntry(cert, key, path), spec) + self.add_cert(CertStoreEntry(cert, key, path, [cert]), spec) def add_cert(self, entry: CertStoreEntry, *names: str) -> None: """ @@ -516,6 +519,7 @@ def get_cert( ), privatekey=self.default_privatekey, chain_file=self.default_chain_file, + chain_certs=self.default_chain_certs, ) self.certs[(commonname, tuple(sans))] = entry self.expire(entry) diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 2af8aefb76..f5327722f0 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -41,6 +41,7 @@ ) from ._http1 import Http1Client, Http1Connection, Http1Server from ._http2 import Http2Client, Http2Server +from ._http3 import Http3Client, Http3Server from ...context import Context @@ -821,7 +822,9 @@ def __init__(self, context: Context, mode: HTTPMode): self.command_sources = {} http_conn: HttpConnection - if self.context.client.alpn == b"h2": + if self.context.client.alpn == b"h3": + http_conn = Http3Server(context.fork()) + elif self.context.client.alpn == b"h2": http_conn = Http2Server(context.fork()) else: http_conn = Http1Server(context.fork()) @@ -1060,7 +1063,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: err = yield commands.OpenConnection(self.context.server) if not err: - if self.context.server.alpn == b"h2": + if self.context.server.alpn == b"h3": + self.child_layer = Http3Client(self.context) + elif self.context.server.alpn == b"h2": self.child_layer = Http2Client(self.context) else: self.child_layer = Http1Client(self.context) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py new file mode 100644 index 0000000000..1f57add6eb --- /dev/null +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -0,0 +1,31 @@ +from aioquic.h3.connection import H3Connection +from mitmproxy.connection import Connection +from ._base import HttpConnection +from ..quic import QuicLayer +from ...context import Context + + +class Http3Connection(HttpConnection): + h3_conn: H3Connection + + def __init__(self, context: Context, conn: Connection): + super().__init__(context, conn) + quic = context.layers[0] + assert isinstance(quic, QuicLayer) + self.h3_conn = H3Connection(quic.conn) + + +class Http3Server(Http3Connection): + def __init__(self, context: Context): + super().__init__(context, context.client) + + +class Http3Client(Http3Connection): + def __init__(self, context: Context): + super().__init__(context, context.server) + + +__all__ = [ + "Http3Client", + "Http3Server", +] diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py new file mode 100644 index 0000000000..b938c44916 --- /dev/null +++ b/mitmproxy/proxy/layers/quic.py @@ -0,0 +1,83 @@ +from dataclasses import dataclass +from ssl import VerifyMode +from typing import List, Optional, Union + +from aioquic.tls import CipherSuite +from cryptography import x509 +from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa +from mitmproxy.proxy import layer +from mitmproxy.proxy.commands import StartHook +from mitmproxy.tls import TlsData + + +@dataclass +class QuicTlsSettings: + """ + Settings necessary to establish QUIC's TLS context. + """ + + certificate: Optional[x509.Certificate] = None + """The certificate to use for the connection.""" + certificate_chain: List[x509.Certificate] = [] + """An optional list of additional certificates to send to the peer.""" + certificate_private_key: Optional[ + Union[dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey] + ] = None + """The certificate's private key.""" + cipher_suites: Optional[List[CipherSuite]] = None + """An optional list of allowed/advertised protocols.""" + ca_path: Optional[str] = None + """An optional path to a directory that contains the necessary information to verify the peer certificate.""" + ca_file: Optional[str] = None + """An optional path to a PEM file that will be used to verify the peer certificate.""" + verify_mode: Optional[VerifyMode] = None + """An optional flag that specifies how/if the peer's certificate should be validated.""" + + +@dataclass +class QuicTlsData(TlsData): + """ + Event data for `quic_tls_start_client` and `quic_tls_start_server` event hooks. + """ + + settings: Optional[QuicTlsSettings] = None + """ + The associated `QuicTlsSettings` object. + This will be set by an addon in the `quic_tls_start_*` event hooks. + """ + + +@dataclass +class QuicTlsStartClientHook(StartHook): + """ + TLS negotation between mitmproxy and a client over QUIC is about to start. + + An addon is expected to initialize at least data.certificate and data.certificate_private_key. + (by default, this is done by `mitmproxy.addons.tlsconfig`) + """ + + data: QuicTlsData + + +@dataclass +class QuicTlsStartServerHook(StartHook): + """ + TLS negotation between mitmproxy and a server over QUIC is about to start. + + An addon is expected to initialize at least data.certificate and data.certificate_private_key. + (by default, this is done by `mitmproxy.addons.tlsconfig`) + """ + + data: QuicTlsData + + +class QuicLayer(layer.Layer): + pass + + +class QuicServerLayer(layer.Layer): + pass + + +class QuicClientLayer(layer.Layer): + pass diff --git a/setup.py b/setup.py index 233228273f..575659a277 100644 --- a/setup.py +++ b/setup.py @@ -69,6 +69,7 @@ # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#install-requires # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ + "aioquic>=0.9.20", "asgiref>=3.2.10,<3.6", "blinker>=1.4, <1.5", "Brotli>=1.0,<1.1", From b4f0e28dc09e767d366457d4b2b4926c406cdb7a Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 13 Jun 2022 04:46:02 +0200 Subject: [PATCH 002/695] [quic] user proper cert type --- mitmproxy/addons/tlsconfig.py | 12 +++++++++--- mitmproxy/certs.py | 13 +++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 61245cd8b5..631ba6ad87 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -6,6 +6,8 @@ from aioquic.quic.configuration import QuicConfiguration from aioquic.tls import CipherSuite +from cryptography import x509 +from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from OpenSSL import SSL from mitmproxy import certs, ctx, exceptions, connection, tls from mitmproxy.net import tls as net_tls @@ -205,6 +207,7 @@ def _get_client_cert(self, server: connection.Server) -> Optional[str]: if os.path.isfile(client_certs): return client_certs else: + assert server.address server_name: str = server.sni or server.address[0] p = os.path.join(client_certs, f"{server_name}.pem") if os.path.isfile(p): @@ -309,9 +312,9 @@ def quic_tls_start_client(self, tls_start: quic.QuicTlsData) -> None: server: connection.Server = tls_start.context.server entry = self.get_cert(tls_start.context) - tls_start.settings.certificate = entry.cert + tls_start.settings.certificate = entry.cert._cert tls_start.settings.certificate_private_key = entry.privatekey - tls_start.settings.certificate_chain = entry.chain_certs + tls_start.settings.certificate_chain = [cert._cert for cert in entry.chain_certs] if not client.cipher_list and ctx.options.ciphers_client: client.cipher_list = ctx.options.ciphers_client.split(":") @@ -353,8 +356,11 @@ def quic_tls_start_server(self, tls_start: quic.QuicTlsData) -> None: if client_cert: config = QuicConfiguration() config.load_cert_chain(client_cert) + assert isinstance(config.certificate, x509.Certificate) tls_start.settings.certificate = config.certificate - tls_start.settings.certificate_private_key = config.private_key + if config.private_key: + assert isinstance(config.private_key, (dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey)) + tls_start.settings.certificate_private_key = config.private_key tls_start.settings.certificate_chain = config.certificate_chain tls_start.settings.ca_path = ctx.options.ssl_verify_upstream_trusted_confdir diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index ab26b4b8f5..0a84f88848 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -284,7 +284,7 @@ class CertStoreEntry: cert: Cert privatekey: rsa.RSAPrivateKey chain_file: Optional[Path] - chain_certs: Optional[list[Cert]] + chain_certs: list[Cert] TCustomCertId = str # manually provided certs (e.g. mitmproxy's --certs) @@ -313,7 +313,16 @@ def __init__( self.default_privatekey = default_privatekey self.default_ca = default_ca self.default_chain_file = default_chain_file - self.default_chain_certs = load_pem_x509_certificates(self.default_chain_file.read_bytes()) if self.default_chain_file else None + self.default_chain_certs = ( + [ + Cert(cert) + for cert in load_pem_x509_certificates( + self.default_chain_file.read_bytes() + ) + ] + if self.default_chain_file + else [] + ) self.dhparams = dhparams self.certs = {} self.expire_queue = [] From 568d03c600a9df429e410a6f0ab5d702304de151 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 13 Jun 2022 18:14:05 +0200 Subject: [PATCH 003/695] [quic] changes to proxyserver --- mitmproxy/addons/proxyserver.py | 142 ++++++++++++++++++++++++----- mitmproxy/addons/tlsconfig.py | 2 +- mitmproxy/proxy/layers/__init__.py | 3 + mitmproxy/proxy/layers/quic.py | 8 +- 4 files changed, 129 insertions(+), 26 deletions(-) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 70ae28855d..2d20a23d7e 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -3,8 +3,15 @@ import ipaddress import re import struct -from typing import Optional +from typing import Callable, Optional +from aioquic.buffer import Buffer as QuicBuffer +from aioquic.quic.packet import ( + PACKET_TYPE_INITIAL, + QuicProtocolVersion, + encode_quic_version_negotiation, + pull_quic_header, +) from mitmproxy import ( command, ctx, @@ -21,8 +28,8 @@ from mitmproxy.connection import Address from mitmproxy.flow import Flow from mitmproxy.net import udp -from mitmproxy.proxy import commands, events, layers, server_hooks -from mitmproxy.proxy import server +from mitmproxy.proxy import commands, events, layer, layers, server, server_hooks +from mitmproxy.proxy.context import Context from mitmproxy.proxy.layers.tcp import TcpMessageInjected from mitmproxy.proxy.layers.websocket import WebSocketMessageInjected from mitmproxy.utils import asyncio_utils, human @@ -62,6 +69,7 @@ class Proxyserver: tcp_server: Optional[base_events.Server] dns_server: Optional[udp.UdpServer] + quic_server: Optional[udp.UdpServer] connect_addr: Optional[Address] listen_port: int dns_reverse_addr: Optional[tuple[str, int]] @@ -74,6 +82,7 @@ def __init__(self): self._lock = asyncio.Lock() self.tcp_server = None self.dns_server = None + self.quic_server = None self.connect_addr = None self.dns_reverse_addr = None self.is_running = False @@ -99,6 +108,14 @@ def _server_desc(self): self.options.dns_listen_port, transparent=self.options.dns_mode == "transparent", ) + yield "QUIC", self.quic_server, lambda x: setattr( + self, "quic_server", x + ), ctx.options.quic_server, lambda: udp.start_server( + self.handle_quic_datagram, + self.options.listen_host or "127.0.0.1", + self.options.listen_port, + transparent=self.options.mode == "transparent", + ) @property def running_servers(self): @@ -196,6 +213,15 @@ def load(self, loader): transparent: transparent mode """, ) + loader.add_option( + "quic_server", bool, False, """Start a QUIC server. Disabled by default.""" + ) + loader.add_option( + "quic_connection_id_length", + int, + 8, + """The length in bytes of local QUIC connection IDs.""", + ) async def running(self): self.master = ctx.master @@ -261,6 +287,7 @@ def configure(self, updated): "dns_mode", "dns_listen_host", "dns_listen_port", + "quic_server", ] ): asyncio.create_task(self.refresh_server()) @@ -326,34 +353,26 @@ async def handle_tcp_connection( ) await self.handle_connection(connection_id) - def handle_dns_datagram( + def handle_udp_connection( self, transport: asyncio.DatagramTransport, data: bytes, remote_addr: Address, - local_addr: Address, + connection_id: tuple, + layer_cb: Callable[[Context], layer.Layer], + server_addr: Optional[Address] = None, + timeout: Optional[int] = None, ) -> None: - try: - dns_id = struct.unpack_from("!H", data, 0) - except struct.error: - ctx.log.info( - f"Invalid DNS datagram received from {human.format_address(remote_addr)}." - ) - return - connection_id = ("udp", dns_id, remote_addr, local_addr) if connection_id not in self._connections: reader = udp.DatagramReader() writer = udp.DatagramWriter(transport, remote_addr, reader) handler = ProxyConnectionHandler( - self.master, reader, writer, self.options, 20 - ) - handler.layer = layers.DNSLayer(handler.layer.context) - handler.layer.context.server.address = ( - local_addr - if self.options.dns_mode == "transparent" - else self.dns_reverse_addr + self.master, reader, writer, self.options, timeout ) - handler.layer.context.server.transport_protocol = "udp" + handler.layer = layer_cb(handler.layer.context) + if server_addr is not None: + handler.layer.context.server.address = server_addr + handler.layer.context.server.transport_protocol = "udp" self._connections[connection_id] = handler asyncio.create_task(self.handle_connection(connection_id)) else: @@ -363,6 +382,87 @@ def handle_dns_datagram( reader = client_reader reader.feed_data(data, remote_addr) + def handle_dns_datagram( + self, + transport: asyncio.DatagramTransport, + data: bytes, + remote_addr: Address, + local_addr: Address, + ) -> None: + try: + dns_id = struct.unpack_from("!H", data, 0) + except struct.error: + ctx.log.info( + f"Invalid DNS datagram received from {human.format_address(remote_addr)}." + ) + return + self.handle_udp_connection( + transport=transport, + date=data, + remote_addr=remote_addr, + server_addr=( + local_addr + if self.options.dns_mode == "transparent" + else self.dns_reverse_addr + ), + connection_id=("udp", dns_id, remote_addr, local_addr), + layer_cb=layers.DNSLayer, + timeout=20, + ) + + def handle_quic_datagram( + self, + transport: asyncio.DatagramTransport, + data: bytes, + remote_addr: Address, + local_addr: Address, + ) -> None: + # largely taken from aioquic's own asyncio server code + buffer = QuicBuffer(data=data) + try: + header = pull_quic_header( + buffer, host_cid_length=self.options.quic_connection_id_length + ) + except ValueError: + ctx.log.info( + f"Invalid QUIC datagram received from {human.format_address(remote_addr)}." + ) + return + + # negotiate version, support all versions known to aioquic + supported_versions = ( + version.value + for version in QuicProtocolVersion + if version is not QuicProtocolVersion.NEGOTIATION + ) + if header.version is not None and header.version not in supported_versions: + transport.sendto( + encode_quic_version_negotiation( + source_cid=header.destination_cid, + destination_cid=header.source_cid, + supported_versions=supported_versions, + ), + remote_addr, + ) + return + + # create or resume the connection + connection_id = ("quic", header.destination_cid) + if connection_id not in self._connections: + if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: + ctx.log.info( + f"QUIC packet received from {human.format_address(remote_addr)} with an unknown connection id." + ) + return + self.handle_udp_connection( + transport=transport, + date=data, + remote_addr=remote_addr, + server_addr=local_addr if self.options.mode == "transparent" else None, + connection_id=connection_id, + layer_cb=layers.ServerQuicLayer, + ) + def inject_event(self, event: events.MessageInjected): connection_id = ( "tcp", diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 631ba6ad87..34ca048c16 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -323,7 +323,7 @@ def quic_tls_start_client(self, tls_start: quic.QuicTlsData) -> None: CipherSuite(cipher) for cipher in client.cipher_list ] if ctx.options.add_upstream_certs_to_client_chain: - tls_start.settings.certificate_chain.extend(server.certificate_list) + tls_start.settings.certificate_chain.extend(cert._cert for cert in server.certificate_list) def quic_tls_start_server(self, tls_start: quic.QuicTlsData) -> None: """Establish QUIC between proxy and server.""" diff --git a/mitmproxy/proxy/layers/__init__.py b/mitmproxy/proxy/layers/__init__.py index 55553b258c..7746ed9657 100644 --- a/mitmproxy/proxy/layers/__init__.py +++ b/mitmproxy/proxy/layers/__init__.py @@ -1,6 +1,7 @@ from . import modes from .dns import DNSLayer from .http import HttpLayer +from .quic import ClientQuicLayer, ServerQuicLayer from .tcp import TCPLayer from .tls import ClientTLSLayer, ServerTLSLayer from .websocket import WebsocketLayer @@ -9,6 +10,8 @@ "modes", "DNSLayer", "HttpLayer", + "ClientQuicLayer", + "ServerQuicLayer", "TCPLayer", "ClientTLSLayer", "ServerTLSLayer", diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index b938c44916..829ef56929 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -52,7 +52,7 @@ class QuicTlsStartClientHook(StartHook): """ TLS negotation between mitmproxy and a client over QUIC is about to start. - An addon is expected to initialize at least data.certificate and data.certificate_private_key. + An addon is expected to initialize data.settings. (by default, this is done by `mitmproxy.addons.tlsconfig`) """ @@ -64,7 +64,7 @@ class QuicTlsStartServerHook(StartHook): """ TLS negotation between mitmproxy and a server over QUIC is about to start. - An addon is expected to initialize at least data.certificate and data.certificate_private_key. + An addon is expected to initialize data.settings. (by default, this is done by `mitmproxy.addons.tlsconfig`) """ @@ -75,9 +75,9 @@ class QuicLayer(layer.Layer): pass -class QuicServerLayer(layer.Layer): +class ServerQuicLayer(QuicLayer): pass -class QuicClientLayer(layer.Layer): +class ClientQuicLayer(QuicLayer): pass From 5926c45aa51316e0d9031eeeaaebcaa22d1055be Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 14 Jun 2022 16:16:46 +0200 Subject: [PATCH 004/695] [quic] replicate DestinationKnown in proxyserver --- mitmproxy/addons/proxyserver.py | 35 +++++++++---- mitmproxy/proxy/layers/http/__init__.py | 4 +- mitmproxy/proxy/layers/quic.py | 67 +++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 16 deletions(-) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 2d20a23d7e..2e016eb209 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -27,7 +27,7 @@ ) from mitmproxy.connection import Address from mitmproxy.flow import Flow -from mitmproxy.net import udp +from mitmproxy.net import server_spec, udp from mitmproxy.proxy import commands, events, layer, layers, server, server_hooks from mitmproxy.proxy.context import Context from mitmproxy.proxy.layers.tcp import TcpMessageInjected @@ -359,8 +359,9 @@ def handle_udp_connection( data: bytes, remote_addr: Address, connection_id: tuple, - layer_cb: Callable[[Context], layer.Layer], + layer_factory: Callable[[Context], layer.Layer], server_addr: Optional[Address] = None, + server_sni: Optional[str] = None, timeout: Optional[int] = None, ) -> None: if connection_id not in self._connections: @@ -369,10 +370,10 @@ def handle_udp_connection( handler = ProxyConnectionHandler( self.master, reader, writer, self.options, timeout ) - handler.layer = layer_cb(handler.layer.context) - if server_addr is not None: - handler.layer.context.server.address = server_addr - handler.layer.context.server.transport_protocol = "udp" + handler.layer = layer_factory(handler.layer.context) + handler.layer.context.server.transport_protocol = "udp" + handler.layer.context.server.address = server_addr + handler.layer.context.server.sni = server_sni self._connections[connection_id] = handler asyncio.create_task(self.handle_connection(connection_id)) else: @@ -406,7 +407,7 @@ def handle_dns_datagram( else self.dns_reverse_addr ), connection_id=("udp", dns_id, remote_addr, local_addr), - layer_cb=layers.DNSLayer, + layer_factory=layers.DNSLayer, timeout=20, ) @@ -446,7 +447,7 @@ def handle_quic_datagram( ) return - # create or resume the connection + # check if a new connection is possible connection_id = ("quic", header.destination_cid) if connection_id not in self._connections: if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: @@ -454,13 +455,27 @@ def handle_quic_datagram( f"QUIC packet received from {human.format_address(remote_addr)} with an unknown connection id." ) return + + # determine the server settings (similar to modes.DestinationKnown) + server_addr: Optional[Address] = None + server_sni: Optional[str] = None + if self.options.mode == "transparent": + server_addr = local_addr + elif self.options.mode.startswith("reverse:"): + spec = server_spec.parse_with_mode(self.options.mode)[1] + server_addr = spec.address + if not self.options.keep_host_header: + server_sni = spec.address[0] + + # create or resume the connection self.handle_udp_connection( transport=transport, date=data, remote_addr=remote_addr, - server_addr=local_addr if self.options.mode == "transparent" else None, + server_addr=server_addr, + server_sni=server_sni, connection_id=connection_id, - layer_cb=layers.ServerQuicLayer, + layer_factory=layers.ClientQuicLayer, ) def inject_event(self, event: events.MessageInjected): diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index f5327722f0..36e0969a9d 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -879,7 +879,9 @@ def _handle_event(self, event: events.Event): elif isinstance(event, events.DataReceived): # The peer has sent data. This can happen with HTTP/2 servers that already send a settings frame. child_layer: HttpConnection - if self.context.server.alpn == b"h2": + if self.context.server.alpn == b"h3": + child_layer = Http3Client(self.context.fork()) + elif self.context.server.alpn == b"h2": child_layer = Http2Client(self.context.fork()) else: child_layer = Http1Client(self.context.fork()) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 829ef56929..14437074f8 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,12 +1,17 @@ +from abc import abstractmethod from dataclasses import dataclass +import io from ssl import VerifyMode from typing import List, Optional, Union -from aioquic.tls import CipherSuite +from aioquic.buffer import Buffer as QuicBuffer +from aioquic.quic.connection import QuicConnection +from aioquic.tls import CipherSuite, HandshakeType from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from mitmproxy.proxy import layer from mitmproxy.proxy.commands import StartHook +from mitmproxy.proxy.context import Context from mitmproxy.tls import TlsData @@ -19,13 +24,13 @@ class QuicTlsSettings: certificate: Optional[x509.Certificate] = None """The certificate to use for the connection.""" certificate_chain: List[x509.Certificate] = [] - """An optional list of additional certificates to send to the peer.""" + """A list of additional certificates to send to the peer.""" certificate_private_key: Optional[ Union[dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey] ] = None """The certificate's private key.""" cipher_suites: Optional[List[CipherSuite]] = None - """An optional list of allowed/advertised protocols.""" + """An optional list of allowed/advertised cipher suites.""" ca_path: Optional[str] = None """An optional path to a directory that contains the necessary information to verify the peer certificate.""" ca_file: Optional[str] = None @@ -72,12 +77,64 @@ class QuicTlsStartServerHook(StartHook): class QuicLayer(layer.Layer): - pass + conn: QuicConnection + + +self._protocols[connection.host_cid] = protocol + + def _connection_id_issued(self, cid: bytes, protocol: QuicConnectionProtocol): + self._protocols[cid] = protocol + + def _connection_id_retired( + self, cid: bytes, protocol: QuicConnectionProtocol + ) -> None: + assert self._protocols[cid] == protocol + del self._protocols[cid] + + def _connection_terminated(self, protocol: QuicConnectionProtocol): + for cid, proto in list(self._protocols.items()): + if proto == protocol: + del self._protocols[cid] class ServerQuicLayer(QuicLayer): + """ + This layer establishes QUIC for a single server connection. + """ pass +@dataclass +class ClientHelloException(Exception): + data: bytes + + class ClientQuicLayer(QuicLayer): - pass + _intercept_client_hello: bool + + """ + This layer establishes QUIC on a single client connection. + """ + + def __init__(self, context: Context) -> None: + super().__init__(context) + + # patch aioquic to intercept the client hello + orig_initialize = self.conn._initialize + def initialize_replacement(peer_cid: bytes) -> None: + try: + return orig_initialize(peer_cid) + finally: + orig_server_handle_hello = self.conn.tls._server_handle_hello + def server_handle_hello_replacement( + input_buf: QuicBuffer, + initial_buf: QuicBuffer, + handshake_buf: QuicBuffer, + onertt_buf: QuicBuffer, + ) -> None: + if self._intercept_client_hello and input_buf.pull_uint8() == HandshakeType.CLIENT_HELLO: + raise ClientHelloException(input_buf.data[:input_buf.tell()]) + else: + orig_server_handle_hello(input_buf, initial_buf, handshake_buf, onertt_buf) + self.conn.tls._server_handle_hello = server_handle_hello_replacement + self.conn._initialize = initialize_replacement From ff42d291147eb444c785842b71aac07ee7bfa46e Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 16 Jun 2022 04:03:28 +0200 Subject: [PATCH 005/695] [quic] connection_id handling --- mitmproxy/addons/proxyserver.py | 40 ++++-- mitmproxy/addons/tlsconfig.py | 6 +- mitmproxy/proxy/layers/quic.py | 211 ++++++++++++++++++++++++-------- 3 files changed, 194 insertions(+), 63 deletions(-) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 2e016eb209..b856ac7daf 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -29,7 +29,6 @@ from mitmproxy.flow import Flow from mitmproxy.net import server_spec, udp from mitmproxy.proxy import commands, events, layer, layers, server, server_hooks -from mitmproxy.proxy.context import Context from mitmproxy.proxy.layers.tcp import TcpMessageInjected from mitmproxy.proxy.layers.websocket import WebSocketMessageInjected from mitmproxy.utils import asyncio_utils, human @@ -359,23 +358,26 @@ def handle_udp_connection( data: bytes, remote_addr: Address, connection_id: tuple, - layer_factory: Callable[[Context], layer.Layer], + layer_factory: Callable[[ProxyConnectionHandler], layer.Layer], server_addr: Optional[Address] = None, server_sni: Optional[str] = None, + done_callback: Optional[Callable[[ProxyConnectionHandler]]] = None, timeout: Optional[int] = None, - ) -> None: + ) -> Optional[asyncio.Task[None]]: if connection_id not in self._connections: reader = udp.DatagramReader() writer = udp.DatagramWriter(transport, remote_addr, reader) handler = ProxyConnectionHandler( self.master, reader, writer, self.options, timeout ) - handler.layer = layer_factory(handler.layer.context) + handler.layer = layer_factory(handler) handler.layer.context.server.transport_protocol = "udp" handler.layer.context.server.address = server_addr handler.layer.context.server.sni = server_sni self._connections[connection_id] = handler - asyncio.create_task(self.handle_connection(connection_id)) + task = asyncio.create_task(self.handle_connection(connection_id)) + if done_callback is not None: + task.add_done_callback(lambda _: done_callback(handler)) else: handler = self._connections[connection_id] client_reader = handler.transports[handler.client].reader @@ -407,7 +409,7 @@ def handle_dns_datagram( else self.dns_reverse_addr ), connection_id=("udp", dns_id, remote_addr, local_addr), - layer_factory=layers.DNSLayer, + layer_factory=lambda handler: layers.DNSLayer(handler.layer.context), timeout=20, ) @@ -467,6 +469,25 @@ def handle_quic_datagram( if not self.options.keep_host_header: server_sni = spec.address[0] + # define the callback functions + connection_ids = set([connection_id]) + + def cleanup_connection_ids(handler: ProxyConnectionHandler) -> None: + for connection_id in connection_ids: + if connection_id in self._connections: + del self._connections[connection_id] + + def issue_connection_id(handler: ProxyConnectionHandler, cid: bytes) -> None: + connection_id = ("quic", cid) + assert connection_id not in self._connections + self._connections[connection_id] = handler + connection_ids.add(connection_id) + + def retire_connection_id(handler: ProxyConnectionHandler, cid: bytes) -> None: + connection_id = ("quic", cid) + connection_ids.remove(connection_id) + del self._connections[connection_id] + # create or resume the connection self.handle_udp_connection( transport=transport, @@ -475,7 +496,12 @@ def handle_quic_datagram( server_addr=server_addr, server_sni=server_sni, connection_id=connection_id, - layer_factory=layers.ClientQuicLayer, + done_callback=cleanup_connection_ids, + layer_factory=lambda handler: layers.ClientQuicLayer( + context=handler.layer.context, + issue_cid=lambda cid: issue_connection_id(handler, cid), + retire_cid=lambda cid: retire_connection_id(handler, cid), + ), ) def inject_event(self, event: events.MessageInjected): diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 34ca048c16..7a7c7fbffc 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -201,7 +201,7 @@ def tls_start_client(self, tls_start: tls.TlsData) -> None: ) tls_start.ssl_conn.set_accept_state() - def _get_client_cert(self, server: connection.Server) -> Optional[str]: + def get_client_cert(self, server: connection.Server) -> Optional[str]: if ctx.options.client_certs: client_certs = os.path.expanduser(ctx.options.client_certs) if os.path.isfile(client_certs): @@ -265,7 +265,7 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: verify=verify, ca_path=ctx.options.ssl_verify_upstream_trusted_confdir, ca_pemfile=ctx.options.ssl_verify_upstream_trusted_ca, - client_cert=self._get_client_cert(server), + client_cert=self.get_client_cert(server), ) tls_start.ssl_conn = SSL.Connection(ssl_ctx) @@ -352,7 +352,7 @@ def quic_tls_start_server(self, tls_start: quic.QuicTlsData) -> None: CipherSuite(cipher) for cipher in server.cipher_list ] - client_cert = self._get_client_cert(server) + client_cert = self.get_client_cert(server) if client_cert: config = QuicConfiguration() config.load_cert_chain(client_cert) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 14437074f8..9a068a032c 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,17 +1,22 @@ -from abc import abstractmethod from dataclasses import dataclass -import io from ssl import VerifyMode -from typing import List, Optional, Union +from typing import Callable, List, Optional, TextIO, Union from aioquic.buffer import Buffer as QuicBuffer +from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import QuicConnection -from aioquic.tls import CipherSuite, HandshakeType +from aioquic.tls import ( + CipherSuite, + Context as QuicTlsContext, + HandshakeType, + ServerHello, + pull_server_hello, +) from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy.proxy import layer -from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.context import Context +from mitmproxy import connection +from mitmproxy.net import tls +from mitmproxy.proxy import commands, context, layer from mitmproxy.tls import TlsData @@ -53,9 +58,9 @@ class QuicTlsData(TlsData): @dataclass -class QuicTlsStartClientHook(StartHook): +class QuicTlsStartClientHook(connection.StartHook): """ - TLS negotation between mitmproxy and a client over QUIC is about to start. + TLS negotiation between mitmproxy and a client over QUIC is about to start. An addon is expected to initialize data.settings. (by default, this is done by `mitmproxy.addons.tlsconfig`) @@ -65,9 +70,9 @@ class QuicTlsStartClientHook(StartHook): @dataclass -class QuicTlsStartServerHook(StartHook): +class QuicTlsStartServerHook(connection.StartHook): """ - TLS negotation between mitmproxy and a server over QUIC is about to start. + TLS negotiation between mitmproxy and a server over QUIC is about to start. An addon is expected to initialize data.settings. (by default, this is done by `mitmproxy.addons.tlsconfig`) @@ -76,65 +81,165 @@ class QuicTlsStartServerHook(StartHook): data: QuicTlsData -class QuicLayer(layer.Layer): - conn: QuicConnection +class QuicSecretsLogger(TextIO): + conn: connection.Connection + logger: tls.MasterSecretLogger + + def __init__( + self, conn: connection.Connection, logger: tls.MasterSecretLogger + ) -> None: + super().__init__() + self.conn = conn + self.logger = logger + + def write(self, s: str) -> int: + self.logger(self.conn, s.encode()) + + def flush(self) -> None: + # done by the logger during write + pass + + +@dataclass +class QuicClientHelloException(Exception): + data: bytes + + +def hook_quic_tls(quic: QuicConnection, cb: Callable[[QuicTlsContext]]) -> None: + assert quic.tls is None + + # patch aioquic to intercept the client/server hello + orig_initialize = quic._initialize + + def initialize_replacement(peer_cid: bytes) -> None: + try: + return orig_initialize(peer_cid) + finally: + cb(quic.tls) + + quic._initialize = initialize_replacement + +def throw_on_client_hello(tls: QuicTlsContext) -> None: + def server_handle_hello_replacement( + input_buf: QuicBuffer, + initial_buf: QuicBuffer, + handshake_buf: QuicBuffer, + onertt_buf: QuicBuffer, + ) -> None: + assert input_buf.pull_uint8() == HandshakeType.CLIENT_HELLO + length = 0 + for b in input_buf.pull_bytes(3): + length = (length << 8) | b + offset = input_buf.tell() + raise QuicClientHelloException( + data=input_buf.data_slice(offset, offset + length) + ) + + tls._server_handle_hello = server_handle_hello_replacement -self._protocols[connection.host_cid] = protocol - def _connection_id_issued(self, cid: bytes, protocol: QuicConnectionProtocol): - self._protocols[cid] = protocol +def callback_on_server_hello(tls: QuicTlsContext, cb: Callable[[ServerHello]]) -> None: + orig_client_handle_hello = tls._client_handle_hello - def _connection_id_retired( - self, cid: bytes, protocol: QuicConnectionProtocol + def _client_handle_hello_replacement( + input_buf: QuicBuffer, + output_buf: QuicBuffer, ) -> None: - assert self._protocols[cid] == protocol - del self._protocols[cid] + offset = input_buf.tell() + cb(pull_server_hello(input_buf)) + input_buf.seek(offset) + orig_client_handle_hello(input_buf, output_buf) + + tls._client_handle_hello = _client_handle_hello_replacement + - def _connection_terminated(self, protocol: QuicConnectionProtocol): - for cid, proto in list(self._protocols.items()): - if proto == protocol: - del self._protocols[cid] +class QuicLayer(layer.Layer): + buffer: List[bytes] + quic: Optional[QuicConnection] + conn: connection.Connection + issue_cid: Callable[[bytes]] + retire_cid: Callable[[bytes]] + + def __init__( + self, + context: context.Context, + conn: connection.Connection, + issue_cid: Callable[[bytes]], + retire_cid: Callable[[bytes]], + ) -> None: + super().__init__(context) + self.buffer = [] + self.quic = None + self.conn = conn + + def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: + return QuicConfiguration( + alpn_protocols=self.conn.alpn_offers, + connection_id_length=self.context.options.quic_connection_id_length, + is_client=self.conn == self.context.server, + secrets_log_file=QuicSecretsLogger(self.conn, tls.log_master_secret) + if tls.log_master_secret is not None + else None, + server_name=self.conn.sni, + cafile=settings.ca_file, + capath=settings.ca_path, + certificate=settings.certificate, + certificate_chain=settings.certificate_chain, + cipher_suites=settings.cipher_suites, + private_key=settings.certificate_private_key, + verify_mode=settings.verify_mode, + ) + + def initialize_connection( + self, original_destination_connection_id: Union[bytes, None] + ) -> layer.CommandGenerator[None]: + assert not self.quic + + # (almost) identical to _TLSLayer.start_tls + tls_data = QuicTlsData(self.conn, self.context) + if self.conn == self.context.client: + yield QuicTlsStartClientHook(tls_data) + else: + yield QuicTlsStartServerHook(tls_data) + if not tls_data.settings: + yield commands.Log( + "No TLS settings were provided, failing connection.", "error" + ) + yield commands.CloseConnection(self.conn) + return + assert tls_data.settings + + self.quic = QuicConnection( + configuration=self.build_configuration(tls_data.settings), + original_destination_connection_id=original_destination_connection_id, + ) + self.issue_cid(self.quic.host_cid) class ServerQuicLayer(QuicLayer): """ This layer establishes QUIC for a single server connection. """ - pass - -@dataclass -class ClientHelloException(Exception): - data: bytes + def __init__( + self, + context: context.Context, + issue_cid: Callable[[bytes]], + retire_cid: Callable[[bytes]], + ) -> None: + super().__init__(context, context.server, issue_cid, retire_cid) class ClientQuicLayer(QuicLayer): - _intercept_client_hello: bool - """ This layer establishes QUIC on a single client connection. """ - def __init__(self, context: Context) -> None: - super().__init__(context) - - # patch aioquic to intercept the client hello - orig_initialize = self.conn._initialize - def initialize_replacement(peer_cid: bytes) -> None: - try: - return orig_initialize(peer_cid) - finally: - orig_server_handle_hello = self.conn.tls._server_handle_hello - def server_handle_hello_replacement( - input_buf: QuicBuffer, - initial_buf: QuicBuffer, - handshake_buf: QuicBuffer, - onertt_buf: QuicBuffer, - ) -> None: - if self._intercept_client_hello and input_buf.pull_uint8() == HandshakeType.CLIENT_HELLO: - raise ClientHelloException(input_buf.data[:input_buf.tell()]) - else: - orig_server_handle_hello(input_buf, initial_buf, handshake_buf, onertt_buf) - self.conn.tls._server_handle_hello = server_handle_hello_replacement - self.conn._initialize = initialize_replacement + def __init__( + self, + context: context.Context, + issue_cid: Callable[[bytes]], + retire_cid: Callable[[bytes]], + ) -> None: + super().__init__(context, context.client, issue_cid, retire_cid) From c271f246e247d2cf847141a80fdd31a595e99993 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 16 Jun 2022 05:54:09 +0200 Subject: [PATCH 006/695] [quic] parse client hello --- mitmproxy/proxy/layers/quic.py | 89 ++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 4 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 9a068a032c..41f69e7aa4 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,10 +1,11 @@ +import asyncio from dataclasses import dataclass from ssl import VerifyMode from typing import Callable, List, Optional, TextIO, Union from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic.configuration import QuicConfiguration -from aioquic.quic.connection import QuicConnection +from aioquic.quic.connection import QuicConnection, QuicConnectionError from aioquic.tls import ( CipherSuite, Context as QuicTlsContext, @@ -12,12 +13,14 @@ ServerHello, pull_server_hello, ) +from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from mitmproxy import connection from mitmproxy.net import tls -from mitmproxy.proxy import commands, context, layer -from mitmproxy.tls import TlsData +from mitmproxy.proxy import commands, context, events, layer, layers +from mitmproxy.proxy.utils import expect +from mitmproxy.tls import ClientHello, ClientHelloData, TlsData @dataclass @@ -120,7 +123,7 @@ def initialize_replacement(peer_cid: bytes) -> None: quic._initialize = initialize_replacement -def throw_on_client_hello(tls: QuicTlsContext) -> None: +def raise_on_client_hello(tls: QuicTlsContext) -> None: def server_handle_hello_replacement( input_buf: QuicBuffer, initial_buf: QuicBuffer, @@ -154,7 +157,31 @@ def _client_handle_hello_replacement( tls._client_handle_hello = _client_handle_hello_replacement +def read_client_hello(data: bytes, connection_id_length: int) -> ClientHello: + buffer = QuicBuffer(data=data) + header = pull_quic_header( + buffer, host_cid_length=connection_id_length + ) + assert header.packet_type == PACKET_TYPE_INITIAL + temp_quic = QuicConnection( + configuration=QuicConfiguration(connection_id_length=connection_id_length), + original_destination_connection_id=header.destination_cid, + ) + hook_quic_tls(temp_quic, raise_on_client_hello) + try: + temp_quic.receive_datagram(data, ("0.0.0.0", 0), now=0) + except QuicClientHelloException as hello: + try: + return ClientHello(hello.data) + except EOFError as e: + raise ValueError("Invalid ClientHello data.") from e + except QuicConnectionError as e: + raise ValueError(e.reason_phrase) from e + raise ValueError("No ClientHello returned.") + + class QuicLayer(layer.Layer): + loop: asyncio.AbstractEventLoop buffer: List[bytes] quic: Optional[QuicConnection] conn: connection.Connection @@ -169,9 +196,12 @@ def __init__( retire_cid: Callable[[bytes]], ) -> None: super().__init__(context) + self.loop = asyncio.get_event_loop() self.buffer = [] self.quic = None self.conn = conn + self.issue_cid = issue_cid + self.retire_cid = retire_cid def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: return QuicConfiguration( @@ -243,3 +273,54 @@ def __init__( retire_cid: Callable[[bytes]], ) -> None: super().__init__(context, context.client, issue_cid, retire_cid) + + @expect(events.Start) + def handle_start(self, _: events.Event) -> layer.CommandGenerator[None]: + self._handle_event = self.handle_client_hello + yield from () + + @expect(events.DataReceived, events.ConnectionClosed) + def handle_client_hello(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.DataReceived): + assert event.connection == self.conn + + # extract the client hello + try: + client_hello = read_client_hello(event.data, connection_id_length=self.context.options.quic_connection_id_length) + except ValueError as e: + yield commands.Log( + f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})", "warn" + ) + yield commands.CloseConnection(self.conn) + else: + self.conn.sni = client_hello.sni + self.conn.alpn_offers = client_hello.alpn_protocols + + # check with addons what we shall do + hook_data = ClientHelloData(self.context, client_hello) + yield layers.tls.TlsClienthelloHook(hook_data) + if hook_data.ignore_connection: + # simply relay everything (including the client hello) + relay_layer = layers.TCPLayer(self.context, ignore=True) + self._handle_event = relay_layer.handle_event + yield from relay_layer.handle_event(events.Start()) + yield from relay_layer.handle_event(event) + + elif hook_data.establish_server_tls_first: + pass + + else: + pass + + elif isinstance(event, events.ConnectionClosed): + assert event.connection == self.conn + self._handle_event = self.handle_done + + else: + raise AssertionError(f"Unexpected event: {event}") + + @expect(events.DataReceived, events.ConnectionClosed) + def handle_done(self, _) -> layer.CommandGenerator[None]: + yield from () + + _handle_event = handle_start From 0e49eebe62dbe3e527ba95934e05ade39fde62d8 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 16 Jun 2022 14:09:38 +0200 Subject: [PATCH 007/695] [quic] support roaming --- mitmproxy/addons/proxyserver.py | 27 +++++++++++++++---------- mitmproxy/net/udp.py | 16 +++++++-------- mitmproxy/proxy/commands.py | 8 +++++--- mitmproxy/proxy/events.py | 3 ++- mitmproxy/proxy/layers/quic.py | 36 ++++++++++++++++----------------- mitmproxy/proxy/server.py | 16 +++++++++++---- 6 files changed, 61 insertions(+), 45 deletions(-) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index b856ac7daf..d163c293d4 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -3,7 +3,7 @@ import ipaddress import re import struct -from typing import Callable, Optional +from typing import Any, Callable, Optional from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic.packet import ( @@ -361,9 +361,9 @@ def handle_udp_connection( layer_factory: Callable[[ProxyConnectionHandler], layer.Layer], server_addr: Optional[Address] = None, server_sni: Optional[str] = None, - done_callback: Optional[Callable[[ProxyConnectionHandler]]] = None, + done_callback: Optional[Callable[[ProxyConnectionHandler], Any]] = None, timeout: Optional[int] = None, - ) -> Optional[asyncio.Task[None]]: + ) -> None: if connection_id not in self._connections: reader = udp.DatagramReader() writer = udp.DatagramWriter(transport, remote_addr, reader) @@ -375,9 +375,11 @@ def handle_udp_connection( handler.layer.context.server.address = server_addr handler.layer.context.server.sni = server_sni self._connections[connection_id] = handler - task = asyncio.create_task(self.handle_connection(connection_id)) - if done_callback is not None: - task.add_done_callback(lambda _: done_callback(handler)) + asyncio.create_task( + self.handle_connection(connection_id) + ).add_done_callback( + lambda _: None if done_callback is None else done_callback(handler) + ) else: handler = self._connections[connection_id] client_reader = handler.transports[handler.client].reader @@ -401,7 +403,7 @@ def handle_dns_datagram( return self.handle_udp_connection( transport=transport, - date=data, + data=data, remote_addr=remote_addr, server_addr=( local_addr @@ -420,6 +422,9 @@ def handle_quic_datagram( remote_addr: Address, local_addr: Address, ) -> None: + def build_connection_id(cid: bytes) -> tuple: + return ("quic", cid, local_addr) + # largely taken from aioquic's own asyncio server code buffer = QuicBuffer(data=data) try: @@ -450,7 +455,7 @@ def handle_quic_datagram( return # check if a new connection is possible - connection_id = ("quic", header.destination_cid) + connection_id = build_connection_id(header.destination_cid) if connection_id not in self._connections: if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: ctx.log.info( @@ -478,20 +483,20 @@ def cleanup_connection_ids(handler: ProxyConnectionHandler) -> None: del self._connections[connection_id] def issue_connection_id(handler: ProxyConnectionHandler, cid: bytes) -> None: - connection_id = ("quic", cid) + connection_id = build_connection_id(cid) assert connection_id not in self._connections self._connections[connection_id] = handler connection_ids.add(connection_id) def retire_connection_id(handler: ProxyConnectionHandler, cid: bytes) -> None: - connection_id = ("quic", cid) + connection_id = build_connection_id(cid) connection_ids.remove(connection_id) del self._connections[connection_id] # create or resume the connection self.handle_udp_connection( transport=transport, - date=data, + data=data, remote_addr=remote_addr, server_addr=server_addr, server_sni=server_sni, diff --git a/mitmproxy/net/udp.py b/mitmproxy/net/udp.py index c70647800e..61cf7cf336 100644 --- a/mitmproxy/net/udp.py +++ b/mitmproxy/net/udp.py @@ -4,7 +4,7 @@ import ipaddress import socket import struct -from typing import Any, Callable, Optional, Union, cast +from typing import Any, Callable, Optional, Tuple, Union, cast from mitmproxy import ctx from mitmproxy.connection import Address from mitmproxy.utils import human @@ -228,7 +228,7 @@ def close(self) -> None: class DatagramReader: - _packets: asyncio.Queue + _packets: asyncio.Queue[Tuple[bytes, Address]] _eof: bool def __init__(self) -> None: @@ -243,7 +243,7 @@ def feed_data(self, data: bytes, remote_addr: Address) -> None: ) else: try: - self._packets.put_nowait(data) + self._packets.put_nowait((data, remote_addr)) except asyncio.QueueFull: ctx.log.debug( f"Dropped UDP packet from {human.format_address(remote_addr)}." @@ -252,17 +252,17 @@ def feed_data(self, data: bytes, remote_addr: Address) -> None: def feed_eof(self) -> None: self._eof = True try: - self._packets.put_nowait(b"") + self._packets.put_nowait((b"", None)) # type: ignore except asyncio.QueueFull: pass - async def read(self, n: int) -> bytes: + async def read(self, n: int) -> Tuple[bytes, Address]: assert n >= MAX_DATAGRAM_SIZE if self._eof: try: return self._packets.get_nowait() except asyncio.QueueEmpty: - return b"" + return (b"", None) # type: ignore else: return await self._packets.get() @@ -295,8 +295,8 @@ def __init__( def _protocol(self) -> DrainableDatagramProtocol: return cast(DrainableDatagramProtocol, self._transport.get_protocol()) - def write(self, data: bytes) -> None: - self._transport.sendto(data, self._remote_addr) + def write(self, data: bytes, remote_addr: Optional[Address] = None) -> None: + self._transport.sendto(data, self._remote_addr if remote_addr is None else remote_addr) def write_eof(self) -> None: raise NotImplementedError("UDP does not support half-closing.") diff --git a/mitmproxy/proxy/commands.py b/mitmproxy/proxy/commands.py index 388abf9fe8..3845a7774e 100644 --- a/mitmproxy/proxy/commands.py +++ b/mitmproxy/proxy/commands.py @@ -6,10 +6,10 @@ The counterpart to commands are events. """ -from typing import Literal, Union, TYPE_CHECKING +from typing import Literal, Optional, Union, TYPE_CHECKING import mitmproxy.hooks -from mitmproxy.connection import Connection, Server +from mitmproxy.connection import Address, Connection, Server if TYPE_CHECKING: import mitmproxy.proxy.layer @@ -67,10 +67,12 @@ class SendData(ConnectionCommand): """ data: bytes + remote_addr: Optional[Address] - def __init__(self, connection: Connection, data: bytes): + def __init__(self, connection: Connection, data: bytes, remote_addr: Optional[Address] = None): super().__init__(connection) self.data = data + self.remote_addr = remote_addr def __repr__(self): target = str(self.connection).split("(", 1)[0].lower() diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index b767483f0e..fb1e925f23 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -10,7 +10,7 @@ from mitmproxy import flow from mitmproxy.proxy import commands -from mitmproxy.connection import Connection +from mitmproxy.connection import Address, Connection class Event: @@ -45,6 +45,7 @@ class DataReceived(ConnectionEvent): """ data: bytes + remote_addr: Optional[Address] = None def __repr__(self): target = type(self.connection).__name__.lower() diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 41f69e7aa4..484013de54 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -61,7 +61,7 @@ class QuicTlsData(TlsData): @dataclass -class QuicTlsStartClientHook(connection.StartHook): +class QuicTlsStartClientHook(commands.StartHook): """ TLS negotiation between mitmproxy and a client over QUIC is about to start. @@ -73,7 +73,7 @@ class QuicTlsStartClientHook(connection.StartHook): @dataclass -class QuicTlsStartServerHook(connection.StartHook): +class QuicTlsStartServerHook(commands.StartHook): """ TLS negotiation between mitmproxy and a server over QUIC is about to start. @@ -84,19 +84,21 @@ class QuicTlsStartServerHook(connection.StartHook): data: QuicTlsData -class QuicSecretsLogger(TextIO): - conn: connection.Connection +class QuicSecretsLogger: logger: tls.MasterSecretLogger def __init__( - self, conn: connection.Connection, logger: tls.MasterSecretLogger + self, logger: tls.MasterSecretLogger ) -> None: super().__init__() - self.conn = conn self.logger = logger def write(self, s: str) -> int: - self.logger(self.conn, s.encode()) + if s.endswith("\n"): + s = s[:-1] + data = s.encode() + self.logger(None, data) # type: ignore + return len(data) + 1 def flush(self) -> None: # done by the logger during write @@ -108,7 +110,7 @@ class QuicClientHelloException(Exception): data: bytes -def hook_quic_tls(quic: QuicConnection, cb: Callable[[QuicTlsContext]]) -> None: +def hook_quic_tls(quic: QuicConnection, cb: Callable[[QuicTlsContext], None]) -> None: assert quic.tls is None # patch aioquic to intercept the client/server hello @@ -142,7 +144,7 @@ def server_handle_hello_replacement( tls._server_handle_hello = server_handle_hello_replacement -def callback_on_server_hello(tls: QuicTlsContext, cb: Callable[[ServerHello]]) -> None: +def callback_on_server_hello(tls: QuicTlsContext, cb: Callable[[ServerHello], None]) -> None: orig_client_handle_hello = tls._client_handle_hello def _client_handle_hello_replacement( @@ -185,15 +187,13 @@ class QuicLayer(layer.Layer): buffer: List[bytes] quic: Optional[QuicConnection] conn: connection.Connection - issue_cid: Callable[[bytes]] - retire_cid: Callable[[bytes]] def __init__( self, context: context.Context, conn: connection.Connection, - issue_cid: Callable[[bytes]], - retire_cid: Callable[[bytes]], + issue_cid: Callable[[bytes], None], + retire_cid: Callable[[bytes], None], ) -> None: super().__init__(context) self.loop = asyncio.get_event_loop() @@ -208,7 +208,7 @@ def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: alpn_protocols=self.conn.alpn_offers, connection_id_length=self.context.options.quic_connection_id_length, is_client=self.conn == self.context.server, - secrets_log_file=QuicSecretsLogger(self.conn, tls.log_master_secret) + secrets_log_file=QuicSecretsLogger(tls.log_master_secret) if tls.log_master_secret is not None else None, server_name=self.conn.sni, @@ -255,8 +255,8 @@ class ServerQuicLayer(QuicLayer): def __init__( self, context: context.Context, - issue_cid: Callable[[bytes]], - retire_cid: Callable[[bytes]], + issue_cid: Callable[[bytes], None], + retire_cid: Callable[[bytes], None], ) -> None: super().__init__(context, context.server, issue_cid, retire_cid) @@ -269,8 +269,8 @@ class ClientQuicLayer(QuicLayer): def __init__( self, context: context.Context, - issue_cid: Callable[[bytes]], - retire_cid: Callable[[bytes]], + issue_cid: Callable[[bytes], None], + retire_cid: Callable[[bytes], None], ) -> None: super().__init__(context, context.client, issue_cid, retire_cid) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 20fd4233bf..8f213a55bb 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -249,9 +249,13 @@ async def handle_connection(self, connection: Connection) -> None: cancelled = None reader = self.transports[connection].reader assert reader + has_remote_addr = isinstance(reader, udp.DatagramReader) while True: try: - data = await reader.read(65535) + if has_remote_addr: + data, remote_addr = await reader.read(65535) + else: + data, remote_addr = await reader.read(65535), None if not data: raise OSError("Connection closed by peer.") except OSError: @@ -260,7 +264,7 @@ async def handle_connection(self, connection: Connection) -> None: cancelled = e break - self.server_event(events.DataReceived(connection, data)) + self.server_event(events.DataReceived(connection, data, remote_addr)) try: await self.drain_writers() @@ -353,8 +357,12 @@ def server_event(self, event: events.Event) -> None: pass # The connection has already been closed. elif isinstance(command, commands.SendData): writer = self.transports[command.connection].writer - assert writer - writer.write(command.data) + if command.remote_addr is not None: + assert isinstance(writer, udp.DatagramWriter) + writer.write(command.data, command.remote_addr) + else: + assert writer + writer.write(command.data) elif isinstance(command, commands.CloseConnection): self.close_connection(command.connection, command.half_close) elif isinstance(command, commands.GetSocket): From f4cb656b431a5667dbc09f1fff4971e3fa67fafa Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sat, 18 Jun 2022 03:18:45 +0200 Subject: [PATCH 008/695] [quic] more work on TLS --- mitmproxy/proxy/layers/quic.py | 230 +++++++++++++++++++++------------ 1 file changed, 147 insertions(+), 83 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 484013de54..af30a272fa 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,26 +1,21 @@ import asyncio from dataclasses import dataclass from ssl import VerifyMode -from typing import Callable, List, Optional, TextIO, Union +from typing import Callable, List, Optional, Tuple, Union from aioquic.buffer import Buffer as QuicBuffer +from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import QuicConnection, QuicConnectionError -from aioquic.tls import ( - CipherSuite, - Context as QuicTlsContext, - HandshakeType, - ServerHello, - pull_server_hello, -) +from aioquic.tls import CipherSuite, HandshakeType from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from mitmproxy import connection from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, layers -from mitmproxy.proxy.utils import expect from mitmproxy.tls import ClientHello, ClientHelloData, TlsData +from mitmproxy.utils import human @dataclass @@ -84,17 +79,25 @@ class QuicTlsStartServerHook(commands.StartHook): data: QuicTlsData +@dataclass +class QuicStreamDataReceived(quic_events.StreamDataReceived, events.ConnectionEvent): + pass + + +@dataclass +class QuicStreamReset(quic_events.StreamReset, events.ConnectionEvent): + pass + + class QuicSecretsLogger: logger: tls.MasterSecretLogger - def __init__( - self, logger: tls.MasterSecretLogger - ) -> None: + def __init__(self, logger: tls.MasterSecretLogger) -> None: super().__init__() self.logger = logger def write(self, s: str) -> int: - if s.endswith("\n"): + if s[-1:] == "\n": s = s[:-1] data = s.encode() self.logger(None, data) # type: ignore @@ -106,26 +109,24 @@ def flush(self) -> None: @dataclass -class QuicClientHelloException(Exception): +class QuicClientHello(Exception): data: bytes -def hook_quic_tls(quic: QuicConnection, cb: Callable[[QuicTlsContext], None]) -> None: - assert quic.tls is None - - # patch aioquic to intercept the client/server hello - orig_initialize = quic._initialize - - def initialize_replacement(peer_cid: bytes) -> None: - try: - return orig_initialize(peer_cid) - finally: - cb(quic.tls) - - quic._initialize = initialize_replacement +def read_client_hello(data: bytes) -> ClientHello: + # ensure the first packet is indeed the initial one + buffer = QuicBuffer(data=data) + header = pull_quic_header(buffer) + if header.packet_type != PACKET_TYPE_INITIAL: + raise ValueError("Packet is not initial one.") + # patch aioquic to intercept the client hello + quic = QuicConnection( + configuration=QuicConfiguration(), + original_destination_connection_id=header.destination_cid, + ) + _initialize = quic._initialize -def raise_on_client_hello(tls: QuicTlsContext) -> None: def server_handle_hello_replacement( input_buf: QuicBuffer, initial_buf: QuicBuffer, @@ -137,42 +138,18 @@ def server_handle_hello_replacement( for b in input_buf.pull_bytes(3): length = (length << 8) | b offset = input_buf.tell() - raise QuicClientHelloException( - data=input_buf.data_slice(offset, offset + length) - ) - - tls._server_handle_hello = server_handle_hello_replacement - - -def callback_on_server_hello(tls: QuicTlsContext, cb: Callable[[ServerHello], None]) -> None: - orig_client_handle_hello = tls._client_handle_hello - - def _client_handle_hello_replacement( - input_buf: QuicBuffer, - output_buf: QuicBuffer, - ) -> None: - offset = input_buf.tell() - cb(pull_server_hello(input_buf)) - input_buf.seek(offset) - orig_client_handle_hello(input_buf, output_buf) - - tls._client_handle_hello = _client_handle_hello_replacement + raise QuicClientHello(data=input_buf.data_slice(offset, offset + length)) + def initialize_replacement(peer_cid: bytes) -> None: + try: + return _initialize(peer_cid) + finally: + quic.tls._server_handle_hello = server_handle_hello_replacement -def read_client_hello(data: bytes, connection_id_length: int) -> ClientHello: - buffer = QuicBuffer(data=data) - header = pull_quic_header( - buffer, host_cid_length=connection_id_length - ) - assert header.packet_type == PACKET_TYPE_INITIAL - temp_quic = QuicConnection( - configuration=QuicConfiguration(connection_id_length=connection_id_length), - original_destination_connection_id=header.destination_cid, - ) - hook_quic_tls(temp_quic, raise_on_client_hello) + quic._initialize = initialize_replacement try: - temp_quic.receive_datagram(data, ("0.0.0.0", 0), now=0) - except QuicClientHelloException as hello: + quic.receive_datagram(data, ("0.0.0.0", 0), now=0) + except QuicClientHello as hello: try: return ClientHello(hello.data) except EOFError as e: @@ -184,9 +161,9 @@ def read_client_hello(data: bytes, connection_id_length: int) -> ClientHello: class QuicLayer(layer.Layer): loop: asyncio.AbstractEventLoop - buffer: List[bytes] quic: Optional[QuicConnection] conn: connection.Connection + original_destination_connection_id: Optional[bytes] def __init__( self, @@ -207,8 +184,8 @@ def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: return QuicConfiguration( alpn_protocols=self.conn.alpn_offers, connection_id_length=self.context.options.quic_connection_id_length, - is_client=self.conn == self.context.server, - secrets_log_file=QuicSecretsLogger(tls.log_master_secret) + is_client=self.conn is self.context.server, + secrets_log_file=QuicSecretsLogger(tls.log_master_secret) # type: ignore if tls.log_master_secret is not None else None, server_name=self.conn.sni, @@ -221,14 +198,13 @@ def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: verify_mode=settings.verify_mode, ) - def initialize_connection( - self, original_destination_connection_id: Union[bytes, None] - ) -> layer.CommandGenerator[None]: + def initialize_connection(self) -> layer.CommandGenerator[None]: assert not self.quic + self._handle_event = self.handle_connected # (almost) identical to _TLSLayer.start_tls tls_data = QuicTlsData(self.conn, self.context) - if self.conn == self.context.client: + if self.conn is self.context.client: yield QuicTlsStartClientHook(tls_data) else: yield QuicTlsStartServerHook(tls_data) @@ -237,15 +213,22 @@ def initialize_connection( "No TLS settings were provided, failing connection.", "error" ) yield commands.CloseConnection(self.conn) + self._handle_event = self.handle_done return assert tls_data.settings self.quic = QuicConnection( configuration=self.build_configuration(tls_data.settings), - original_destination_connection_id=original_destination_connection_id, + original_destination_connection_id=self.original_destination_connection_id, ) self.issue_cid(self.quic.host_cid) + def process_events(self) -> layer.CommandGenerator[None]: + assert self.quic + + def handle_done(self, _) -> layer.CommandGenerator[None]: + yield from () + class ServerQuicLayer(QuicLayer): """ @@ -260,12 +243,34 @@ def __init__( ) -> None: super().__init__(context, context.server, issue_cid, retire_cid) + def handle_start(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, events.Start) + + # ensure there is an UDP connection + if not self.conn.connected: + err = yield commands.OpenConnection(self.conn) + if err is not None: + yield commands.Log( + f"Failed to establish connection to {human.format_address(self.conn)}: {err}" + ) + self._handle_event = self.handle_done + return + + # try to connect + yield from self.initialize_connection() + if self.quic is not None: + self.quic.connect(addr=self.conn.peername, now=self.loop.time()) + yield from self.process_events() + class ClientQuicLayer(QuicLayer): """ This layer establishes QUIC on a single client connection. """ + server_layer: Optional[ServerQuicLayer] + buffered_packets: Optional[List[Tuple[bytes, connection.Address, float]]] + def __init__( self, context: context.Context, @@ -273,24 +278,40 @@ def __init__( retire_cid: Callable[[bytes], None], ) -> None: super().__init__(context, context.client, issue_cid, retire_cid) + self.server_layer = None + self.buffered_packets = None + + def start_client_connection(self) -> layer.CommandGenerator[None]: + assert self.buffered_packets is not None + + yield from self.initialize_connection() + if self.quic is not None: + for data, addr, now in self.buffered_packets: + self.quic.receive_datagram( + data=data, + addr=addr, + now=now, + ) + yield from self.process_events() - @expect(events.Start) - def handle_start(self, _: events.Event) -> layer.CommandGenerator[None]: + def handle_start(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, events.Start) self._handle_event = self.handle_client_hello yield from () - @expect(events.DataReceived, events.ConnectionClosed) def handle_client_hello(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.DataReceived): - assert event.connection == self.conn + assert isinstance(event, events.ConnectionEvent) + assert event.connection is self.conn + if isinstance(event, events.DataReceived): # extract the client hello try: - client_hello = read_client_hello(event.data, connection_id_length=self.context.options.quic_connection_id_length) + client_hello = read_client_hello(event.data) except ValueError as e: yield commands.Log( f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})", "warn" ) + self._handle_event = self.handle_done yield commands.CloseConnection(self.conn) else: self.conn.sni = client_hello.sni @@ -299,6 +320,7 @@ def handle_client_hello(self, event: events.Event) -> layer.CommandGenerator[Non # check with addons what we shall do hook_data = ClientHelloData(self.context, client_hello) yield layers.tls.TlsClienthelloHook(hook_data) + if hook_data.ignore_connection: # simply relay everything (including the client hello) relay_layer = layers.TCPLayer(self.context, ignore=True) @@ -306,21 +328,63 @@ def handle_client_hello(self, event: events.Event) -> layer.CommandGenerator[Non yield from relay_layer.handle_event(events.Start()) yield from relay_layer.handle_event(event) - elif hook_data.establish_server_tls_first: - pass - else: - pass + # buffer the client hello + self.buffered_packets = [ + (event.data, event.remote_addr, self.loop.time()) + ] + + # contact the upstream server first if so desired + if hook_data.establish_server_tls_first: + self.server_layer = ServerQuicLayer( + context=self.context, + issue_cid=self.issue_cid, + retire_cid=self.retire_cid, + ) + self._handle_event = self.handle_wait_for_server + yield from self.handle_wait_for_server(events.Start()) + else: + yield from self.start_client_connection() elif isinstance(event, events.ConnectionClosed): - assert event.connection == self.conn + # this is odd since this layer should only be created if there is a packet self._handle_event = self.handle_done else: raise AssertionError(f"Unexpected event: {event}") - @expect(events.DataReceived, events.ConnectionClosed) - def handle_done(self, _) -> layer.CommandGenerator[None]: - yield from () + def handle_wait_for_server( + self, event: events.Event + ) -> layer.CommandGenerator[None]: + assert self.buffered_packets is not None + assert self.server_layer is not None + + # filter DataReceived and ConnectionClosed relating to the client connection + if isinstance(event, events.ConnectionEvent): + if event.connection is self.context.client: + if isinstance(event, events.DataReceived): + # still waiting for the server, buffer the data + self.buffered_packets.append( + (event.data, event.remote_addr, self.loop.time()) + ) + + elif isinstance(event, events.ConnectionClosed): + # close the upstream connection as well and be done + yield commands.CloseConnection(self.context.server) + self._handle_event = self.handle_done + + else: + raise AssertionError(f"Unexpected event: {event}") + + # forward the event and check it's results + yield from self.server_layer.handle_event(event) + if not self.context.server.connected: + yield commands.Log( + f"Unable to establish QUIC connection with server ({self.context.server.error or 'Connection closed.'}). " + f"Trying to establish QUIC with client anyway." + ) + yield from self.start_client_connection() + elif self.context.server.tls_established: + yield from self.start_client_connection() _handle_event = handle_start From 366e696538061d13540ced440998193e4c1e4c31 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sat, 18 Jun 2022 12:33:45 +0200 Subject: [PATCH 009/695] [quic] add child layering --- mitmproxy/proxy/layers/quic.py | 187 ++++++++++++++++++++++----------- 1 file changed, 127 insertions(+), 60 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index af30a272fa..3e671356ad 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,7 +1,9 @@ +from abc import abstractmethod import asyncio from dataclasses import dataclass from ssl import VerifyMode -from typing import Callable, List, Optional, Tuple, Union +from typing import Callable, List, Literal, Optional, Tuple, Union +from urllib.parse import non_hierarchical from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events @@ -80,13 +82,19 @@ class QuicTlsStartServerHook(commands.StartHook): @dataclass -class QuicStreamDataReceived(quic_events.StreamDataReceived, events.ConnectionEvent): - pass +class QuicConnectionEvent(events.ConnectionEvent): + event: quic_events.QuicEvent @dataclass -class QuicStreamReset(quic_events.StreamReset, events.ConnectionEvent): - pass +class QuicGetConnection(commands.ConnectionCommand): # -> QuicConnection + blocking = True + + +@dataclass(repr=False) +class OpenGetConnectionCompleted(events.CommandCompleted): + command: QuicGetConnection + connection: QuicConnection class QuicSecretsLogger: @@ -113,7 +121,7 @@ class QuicClientHello(Exception): data: bytes -def read_client_hello(data: bytes) -> ClientHello: +def pull_client_hello_and_connection_id(data: bytes) -> Tuple[ClientHello, bytes]: # ensure the first packet is indeed the initial one buffer = QuicBuffer(data=data) header = pull_quic_header(buffer) @@ -151,7 +159,7 @@ def initialize_replacement(peer_cid: bytes) -> None: quic.receive_datagram(data, ("0.0.0.0", 0), now=0) except QuicClientHello as hello: try: - return ClientHello(hello.data) + return (ClientHello(hello.data), header.destination_cid) except EOFError as e: raise ValueError("Invalid ClientHello data.") from e except QuicConnectionError as e: @@ -160,25 +168,29 @@ def initialize_replacement(peer_cid: bytes) -> None: class QuicLayer(layer.Layer): - loop: asyncio.AbstractEventLoop - quic: Optional[QuicConnection] + child_layer: Optional[layer.Layer] conn: connection.Connection + loop: asyncio.AbstractEventLoop original_destination_connection_id: Optional[bytes] + quic: Optional[QuicConnection] + waiting_get_connection_commands: List[QuicGetConnection] def __init__( self, context: context.Context, conn: connection.Connection, - issue_cid: Callable[[bytes], None], - retire_cid: Callable[[bytes], None], + issue_cid: Optional[Callable[[bytes], None]] = None, + retire_cid: Optional[Callable[[bytes], None]] = None, ) -> None: super().__init__(context) + self.child_layer = None + self.conn = conn self.loop = asyncio.get_event_loop() - self.buffer = [] + self.original_destination_connection_id = None self.quic = None - self.conn = conn - self.issue_cid = issue_cid - self.retire_cid = retire_cid + self.waiting_get_connection_commands = [] + self._issue_cid = issue_cid + self._retire_cid = retire_cid def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: return QuicConfiguration( @@ -198,9 +210,42 @@ def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: verify_mode=settings.verify_mode, ) + def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.child_layer is not None + + # answer the child layers request for the connection + for command in self.child_layer.handle_event(event): + if ( + isinstance(command, QuicGetConnection) + and command.connection is self.conn + ): + if self.quic is None: + self.waiting_get_connection_commands.append(command) + else: + yield from self.child_layer.handle_event( + OpenGetConnectionCompleted( + command=command, + connection=self.quic, + ) + ) + else: + yield command + + def fail_connection( + self, + reason: str, + level: Literal["error", "warn", "info", "alert", "debug"] = "warn", + ) -> layer.CommandGenerator[None]: + yield commands.Log( + message=f"Failing connection {self.conn}: {reason}", level=level + ) + if self.conn.connected: + yield commands.CloseConnection(self.conn) + self._handle_event = self.state_done + def initialize_connection(self) -> layer.CommandGenerator[None]: - assert not self.quic - self._handle_event = self.handle_connected + assert self.quic is None + self._handle_event = self.state_ready # (almost) identical to _TLSLayer.start_tls tls_data = QuicTlsData(self.conn, self.context) @@ -209,51 +254,71 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: else: yield QuicTlsStartServerHook(tls_data) if not tls_data.settings: - yield commands.Log( - "No TLS settings were provided, failing connection.", "error" + yield from self.fail_connection( + "No TLS settings were provided, failing connection.", level="error" ) - yield commands.CloseConnection(self.conn) - self._handle_event = self.handle_done return - assert tls_data.settings + assert tls_data.settings is not None + # create the connection and let the waiters know about it self.quic = QuicConnection( configuration=self.build_configuration(tls_data.settings), original_destination_connection_id=self.original_destination_connection_id, ) - self.issue_cid(self.quic.host_cid) + if self._issue_cid: + self._issue_cid(self.quic.host_cid) + while self.waiting_get_connection_commands: + assert self.quic is not None + assert self.child_layer is not None + yield from self.child_layer.handle_event( + OpenGetConnectionCompleted( + command=self.waiting_get_connection_commands.pop(), + connection=self.quic, + ) + ) def process_events(self) -> layer.CommandGenerator[None]: - assert self.quic + assert self.quic is not None + yield from () + + @abstractmethod + def start(self) -> layer.CommandGenerator[None]: + yield from () # pragma: no cover + + def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, events.Start) + + # start this layer and the child layer + yield from self.start() + if self.child_layer is not None: + yield from self.child_layer.handle_event(event) + + def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.quic is not None + yield from () - def handle_done(self, _) -> layer.CommandGenerator[None]: + def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: yield from () + _handle_event = state_start + class ServerQuicLayer(QuicLayer): """ This layer establishes QUIC for a single server connection. """ - def __init__( - self, - context: context.Context, - issue_cid: Callable[[bytes], None], - retire_cid: Callable[[bytes], None], - ) -> None: - super().__init__(context, context.server, issue_cid, retire_cid) - - def handle_start(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, events.Start) + def __init__(self, context: context.Context) -> None: + super().__init__(context, context.server) + def start(self) -> layer.CommandGenerator[None]: # ensure there is an UDP connection if not self.conn.connected: err = yield commands.OpenConnection(self.conn) if err is not None: - yield commands.Log( + self.fail_connection( f"Failed to establish connection to {human.format_address(self.conn)}: {err}" ) - self._handle_event = self.handle_done return # try to connect @@ -294,25 +359,29 @@ def start_client_connection(self) -> layer.CommandGenerator[None]: ) yield from self.process_events() - def handle_start(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, events.Start) - self._handle_event = self.handle_client_hello + def start(self) -> layer.CommandGenerator[None]: + self._handle_event = self.state_wait_for_client_hello yield from () - def handle_client_hello(self, event: events.Event) -> layer.CommandGenerator[None]: + def state_wait_for_client_hello( + self, event: events.Event + ) -> layer.CommandGenerator[None]: assert isinstance(event, events.ConnectionEvent) assert event.connection is self.conn if isinstance(event, events.DataReceived): + assert event.remote_addr is not None + # extract the client hello try: - client_hello = read_client_hello(event.data) + ( + client_hello, + self.original_destination_connection_id, + ) = pull_client_hello_and_connection_id(event.data) except ValueError as e: - yield commands.Log( - f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})", "warn" + yield from self.fail_connection( + f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" ) - self._handle_event = self.handle_done - yield commands.CloseConnection(self.conn) else: self.conn.sni = client_hello.sni self.conn.alpn_offers = client_hello.alpn_protocols @@ -336,24 +405,20 @@ def handle_client_hello(self, event: events.Event) -> layer.CommandGenerator[Non # contact the upstream server first if so desired if hook_data.establish_server_tls_first: - self.server_layer = ServerQuicLayer( - context=self.context, - issue_cid=self.issue_cid, - retire_cid=self.retire_cid, - ) - self._handle_event = self.handle_wait_for_server - yield from self.handle_wait_for_server(events.Start()) + self.server_layer = ServerQuicLayer(self.context) + self._handle_event = self.state_wait_for_upstream_server + yield from self.state_wait_for_upstream_server(events.Start()) else: yield from self.start_client_connection() elif isinstance(event, events.ConnectionClosed): # this is odd since this layer should only be created if there is a packet - self._handle_event = self.handle_done + self._handle_event = self.state_done else: raise AssertionError(f"Unexpected event: {event}") - def handle_wait_for_server( + def state_wait_for_upstream_server( self, event: events.Event ) -> layer.CommandGenerator[None]: assert self.buffered_packets is not None @@ -361,8 +426,10 @@ def handle_wait_for_server( # filter DataReceived and ConnectionClosed relating to the client connection if isinstance(event, events.ConnectionEvent): - if event.connection is self.context.client: + if event.connection is self.conn: if isinstance(event, events.DataReceived): + assert event.remote_addr is not None + # still waiting for the server, buffer the data self.buffered_packets.append( (event.data, event.remote_addr, self.loop.time()) @@ -370,8 +437,10 @@ def handle_wait_for_server( elif isinstance(event, events.ConnectionClosed): # close the upstream connection as well and be done - yield commands.CloseConnection(self.context.server) - self._handle_event = self.handle_done + self._handle_event = self.state_done + yield from self.server_layer.fail_connection( + "Client closed the connection." + ) else: raise AssertionError(f"Unexpected event: {event}") @@ -386,5 +455,3 @@ def handle_wait_for_server( yield from self.start_client_connection() elif self.context.server.tls_established: yield from self.start_client_connection() - - _handle_event = handle_start From 08aa838e9681746c64beb0f3f3f085863501ac49 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sat, 18 Jun 2022 23:28:00 +0200 Subject: [PATCH 010/695] [quic] handle aioquic events --- mitmproxy/proxy/layers/quic.py | 130 +++++++++++++++++++++++++++------ 1 file changed, 107 insertions(+), 23 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 3e671356ad..41f4dfaa7d 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -3,7 +3,6 @@ from dataclasses import dataclass from ssl import VerifyMode from typing import Callable, List, Literal, Optional, Tuple, Union -from urllib.parse import non_hierarchical from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events @@ -13,7 +12,7 @@ from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import connection +from mitmproxy import certs, connection from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, layers from mitmproxy.tls import ClientHello, ClientHelloData, TlsData @@ -92,7 +91,7 @@ class QuicGetConnection(commands.ConnectionCommand): # -> QuicConnection @dataclass(repr=False) -class OpenGetConnectionCompleted(events.CommandCompleted): +class QuicGetConnectionCompleted(events.CommandCompleted): command: QuicGetConnection connection: QuicConnection @@ -173,7 +172,7 @@ class QuicLayer(layer.Layer): loop: asyncio.AbstractEventLoop original_destination_connection_id: Optional[bytes] quic: Optional[QuicConnection] - waiting_get_connection_commands: List[QuicGetConnection] + tls: Optional[QuicTlsSettings] def __init__( self, @@ -188,11 +187,17 @@ def __init__( self.loop = asyncio.get_event_loop() self.original_destination_connection_id = None self.quic = None - self.waiting_get_connection_commands = [] + self.tls = None + self._get_connection_commands: List[QuicGetConnection] = [] self._issue_cid = issue_cid + self._request_wakeup_command_and_timer: Optional[ + Tuple[commands.RequestWakeup, float] + ] = None self._retire_cid = retire_cid - def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: + def build_configuration(self) -> QuicConfiguration: + assert self.tls is not None + return QuicConfiguration( alpn_protocols=self.conn.alpn_offers, connection_id_length=self.context.options.quic_connection_id_length, @@ -201,13 +206,13 @@ def build_configuration(self, settings: QuicTlsSettings) -> QuicConfiguration: if tls.log_master_secret is not None else None, server_name=self.conn.sni, - cafile=settings.ca_file, - capath=settings.ca_path, - certificate=settings.certificate, - certificate_chain=settings.certificate_chain, - cipher_suites=settings.cipher_suites, - private_key=settings.certificate_private_key, - verify_mode=settings.verify_mode, + cafile=self.tls.ca_file, + capath=self.tls.ca_path, + certificate=self.tls.certificate, + certificate_chain=self.tls.certificate_chain, + cipher_suites=self.tls.cipher_suites, + private_key=self.tls.certificate_private_key, + verify_mode=self.tls.verify_mode, ) def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -220,10 +225,10 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: and command.connection is self.conn ): if self.quic is None: - self.waiting_get_connection_commands.append(command) + self._get_connection_commands.append(command) else: yield from self.child_layer.handle_event( - OpenGetConnectionCompleted( + QuicGetConnectionCompleted( command=command, connection=self.quic, ) @@ -245,7 +250,6 @@ def fail_connection( def initialize_connection(self) -> layer.CommandGenerator[None]: assert self.quic is None - self._handle_event = self.state_ready # (almost) identical to _TLSLayer.start_tls tls_data = QuicTlsData(self.conn, self.context) @@ -253,33 +257,113 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: yield QuicTlsStartClientHook(tls_data) else: yield QuicTlsStartServerHook(tls_data) - if not tls_data.settings: + if tls_data.settings is None: yield from self.fail_connection( "No TLS settings were provided, failing connection.", level="error" ) return assert tls_data.settings is not None + self.tls = tls_data.settings - # create the connection and let the waiters know about it + # create the aioquic connection self.quic = QuicConnection( - configuration=self.build_configuration(tls_data.settings), + configuration=self.build_configuration(), original_destination_connection_id=self.original_destination_connection_id, ) if self._issue_cid: self._issue_cid(self.quic.host_cid) - while self.waiting_get_connection_commands: + self._handle_event = self.state_ready + + # let the waiters know about the available connection + while self._get_connection_commands: assert self.quic is not None assert self.child_layer is not None yield from self.child_layer.handle_event( - OpenGetConnectionCompleted( - command=self.waiting_get_connection_commands.pop(), + QuicGetConnectionCompleted( + command=self._get_connection_commands.pop(), connection=self.quic, ) ) def process_events(self) -> layer.CommandGenerator[None]: assert self.quic is not None - yield from () + assert self.tls is not None + + event = self.quic.next_event() + while event is not None: + if isinstance(event, quic_events.ConnectionIdIssued): + if self._issue_cid is not None: + self._issue_cid(event.connection_id) + + elif isinstance(event, quic_events.ConnectionIdRetired): + if self._retire_cid is not None: + self._retire_cid(event.connection_id) + + elif isinstance(event, quic_events.ConnectionTerminated): + # report as TLS failure if the termination happened before the handshake + if not self.conn.tls_established: + self.conn.error = event.reason_phrase + tls_data = QuicTlsData( + conn=self.conn, context=self.context, settings=self.tls + ) + if self.conn is self.context.client: + yield layers.tls.TlsFailedClientHook(tls_data) + else: + yield layers.tls.TlsFailedServerHook(tls_data) + + # always close the connection + yield from self.fail_connection(event.reason_phrase) + + elif isinstance(event, quic_events.HandshakeCompleted): + # concatenate all peer certificates + all_certs = [] + if self.quic.tls._peer_certificate is not None: + all_certs.append(self.quic.tls._peer_certificate) + if self.quic.tls._peer_certificate_chain is not None: + all_certs.extend(self.quic.tls._peer_certificate_chain) + + # set the connection's TLS properties + self.conn.timestamp_tls_setup = self.loop.time() + self.conn.certificate_list = [ + certs.Cert.from_pyopenssl(x) for x in all_certs + ] + self.conn.alpn = event.alpn_protocol.encode() + self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name + self.conn.tls_version = "QUIC" + + # report the success to addons + tls_data = QuicTlsData( + conn=self.conn, context=self.context, settings=self.tls + ) + if self.conn is self.context.client: + yield layers.tls.TlsEstablishedClientHook(tls_data) + else: + yield layers.tls.TlsEstablishedServerHook(tls_data) + + # forward the event as a QuicConnectionEvent to the child layer + yield from self.event_to_child( + QuicConnectionEvent(connection=self.conn, event=event) + ) + + # handle the next event + event = self.quic.next_event() + + # send all queued datagrams + for data, addr in self.quic.datagrams_to_send(now=self.loop.time()): + yield commands.SendData(connection=self.conn, data=data, remote_addr=addr) + + # ensure the wakeup is set and still correct + timer = self.quic.get_timer() + if timer is None: + self._request_wakeup_command_and_timer = None + else: + if self._request_wakeup_command_and_timer is not None: + _, existing_timer = self._request_wakeup_command_and_timer + if existing_timer == timer: + return + command = commands.RequestWakeup(timer - self.loop.time()) + self._request_wakeup_command_and_timer = (command, timer) + yield command @abstractmethod def start(self) -> layer.CommandGenerator[None]: From 34776c1298f571abc7581054bbac642a9699269b Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 19 Jun 2022 01:13:56 +0200 Subject: [PATCH 011/695] [quic] use next layer and event filtering --- mitmproxy/proxy/layers/quic.py | 70 +++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 41f4dfaa7d..619c6233d6 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -167,7 +167,7 @@ def initialize_replacement(peer_cid: bytes) -> None: class QuicLayer(layer.Layer): - child_layer: Optional[layer.Layer] + child_layer: layer.Layer conn: connection.Connection loop: asyncio.AbstractEventLoop original_destination_connection_id: Optional[bytes] @@ -182,7 +182,7 @@ def __init__( retire_cid: Optional[Callable[[bytes], None]] = None, ) -> None: super().__init__(context) - self.child_layer = None + self.child_layer = layer.NextLayer(context) self.conn = conn self.loop = asyncio.get_event_loop() self.original_destination_connection_id = None @@ -216,10 +216,10 @@ def build_configuration(self) -> QuicConfiguration: ) def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.child_layer is not None - - # answer the child layers request for the connection + # filter commands coming from the child layer for command in self.child_layer.handle_event(event): + + # answer or queue requests for the aioquic connection instanc if ( isinstance(command, QuicGetConnection) and command.connection is self.conn @@ -233,6 +233,19 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: connection=self.quic, ) ) + + # properly close QUIC connections + elif ( + isinstance(command, commands.CloseConnection) + and command.connection is self.conn + ): + if self.conn.connected and self.quic is not None: + self.quic.close() + yield from self.process_events() + self._handle_event = self.state_done + yield command + + # return other commands else: yield command @@ -251,7 +264,7 @@ def fail_connection( def initialize_connection(self) -> layer.CommandGenerator[None]: assert self.quic is None - # (almost) identical to _TLSLayer.start_tls + # query addons to provide the necessary TLS settings tls_data = QuicTlsData(self.conn, self.context) if self.conn is self.context.client: yield QuicTlsStartClientHook(tls_data) @@ -262,7 +275,6 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: "No TLS settings were provided, failing connection.", level="error" ) return - assert tls_data.settings is not None self.tls = tls_data.settings # create the aioquic connection @@ -270,14 +282,13 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: configuration=self.build_configuration(), original_destination_connection_id=self.original_destination_connection_id, ) - if self._issue_cid: + if self._issue_cid is not None: self._issue_cid(self.quic.host_cid) self._handle_event = self.state_ready # let the waiters know about the available connection while self._get_connection_commands: assert self.quic is not None - assert self.child_layer is not None yield from self.child_layer.handle_event( QuicGetConnectionCompleted( command=self._get_connection_commands.pop(), @@ -289,6 +300,7 @@ def process_events(self) -> layer.CommandGenerator[None]: assert self.quic is not None assert self.tls is not None + # handle all buffered aioquic connection events event = self.quic.next_event() while event is not None: if isinstance(event, quic_events.ConnectionIdIssued): @@ -340,6 +352,10 @@ def process_events(self) -> layer.CommandGenerator[None]: else: yield layers.tls.TlsEstablishedServerHook(tls_data) + # perform next layer decisions now + if isinstance(self.child_layer, layer.NextLayer): + yield from self.child_layer._ask() + # forward the event as a QuicConnectionEvent to the child layer yield from self.event_to_child( QuicConnectionEvent(connection=self.conn, event=event) @@ -374,15 +390,43 @@ def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: # start this layer and the child layer yield from self.start() - if self.child_layer is not None: - yield from self.child_layer.handle_event(event) + yield from self.child_layer.handle_event(event) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None - yield from () + + if isinstance(event, events.DataReceived): + # forward incoming data only to aioquic + if event.connection is self.conn: + self.quic.receive_datagram( + data=event.data, addr=event.remote_addr, now=self.loop.time() + ) + yield from self.process_events() + return + + elif isinstance(event, events.ConnectionClosed): + if event.connection is self.conn: + # connection closed unexpectedly + yield from self.fail_connection( + "Client closed UDP connection.", level="info" + ) + + elif isinstance(event, events.Wakeup): + # make sure we intercept wakeup events for aioquic + if self._request_wakeup_command_and_timer is not None: + command, timer = self._request_wakeup_command_and_timer + if event.command is command: + self._request_wakeup_command_and_timer = None + self.quic.handle_timer(now=max(timer, self.loop.time())) + yield from self.process_events() + return + + # forward other events to the child layer + yield from self.event_to_child(event) def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: - yield from () + # when done, just forward the event + yield from self.child_layer.handle_event(event) _handle_event = state_start From 3aaa2f9b9b9c519291df455e1922c614e285613c Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 19 Jun 2022 03:42:58 +0200 Subject: [PATCH 012/695] [quic] introduce entry layer --- mitmproxy/addons/proxyserver.py | 2 +- mitmproxy/proxy/layers/__init__.py | 5 +- mitmproxy/proxy/layers/quic.py | 278 +++++++++++++++-------------- 3 files changed, 151 insertions(+), 134 deletions(-) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index d163c293d4..e52390be57 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -502,7 +502,7 @@ def retire_connection_id(handler: ProxyConnectionHandler, cid: bytes) -> None: server_sni=server_sni, connection_id=connection_id, done_callback=cleanup_connection_ids, - layer_factory=lambda handler: layers.ClientQuicLayer( + layer_factory=lambda handler: layers.QuicLayer( context=handler.layer.context, issue_cid=lambda cid: issue_connection_id(handler, cid), retire_cid=lambda cid: retire_connection_id(handler, cid), diff --git a/mitmproxy/proxy/layers/__init__.py b/mitmproxy/proxy/layers/__init__.py index 7746ed9657..ae31304775 100644 --- a/mitmproxy/proxy/layers/__init__.py +++ b/mitmproxy/proxy/layers/__init__.py @@ -1,7 +1,7 @@ from . import modes from .dns import DNSLayer from .http import HttpLayer -from .quic import ClientQuicLayer, ServerQuicLayer +from .quic import QuicLayer from .tcp import TCPLayer from .tls import ClientTLSLayer, ServerTLSLayer from .websocket import WebsocketLayer @@ -10,8 +10,7 @@ "modes", "DNSLayer", "HttpLayer", - "ClientQuicLayer", - "ServerQuicLayer", + "QuicLayer", "TCPLayer", "ClientTLSLayer", "ServerTLSLayer", diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 619c6233d6..dd0f55cd1e 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -166,34 +166,28 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") -class QuicLayer(layer.Layer): +class _QuicLayer(layer.Layer): child_layer: layer.Layer conn: connection.Connection - loop: asyncio.AbstractEventLoop - original_destination_connection_id: Optional[bytes] - quic: Optional[QuicConnection] - tls: Optional[QuicTlsSettings] + issue_connection_id_callback: Optional[Callable[[bytes], None]] = None + original_destination_connection_id: Optional[bytes] = None + quic: Optional[QuicConnection] = None + retire_connection_id_callback: Optional[Callable[[bytes], None]] = None + tls: Optional[QuicTlsSettings] = None def __init__( self, context: context.Context, conn: connection.Connection, - issue_cid: Optional[Callable[[bytes], None]] = None, - retire_cid: Optional[Callable[[bytes], None]] = None, ) -> None: super().__init__(context) self.child_layer = layer.NextLayer(context) self.conn = conn - self.loop = asyncio.get_event_loop() - self.original_destination_connection_id = None - self.quic = None - self.tls = None + self._loop = asyncio.get_event_loop() self._get_connection_commands: List[QuicGetConnection] = [] - self._issue_cid = issue_cid self._request_wakeup_command_and_timer: Optional[ Tuple[commands.RequestWakeup, float] ] = None - self._retire_cid = retire_cid def build_configuration(self) -> QuicConfiguration: assert self.tls is not None @@ -282,8 +276,8 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: configuration=self.build_configuration(), original_destination_connection_id=self.original_destination_connection_id, ) - if self._issue_cid is not None: - self._issue_cid(self.quic.host_cid) + if self.issue_connection_id_callback is not None: + self.issue_connection_id_callback(self.quic.host_cid) self._handle_event = self.state_ready # let the waiters know about the available connection @@ -304,12 +298,12 @@ def process_events(self) -> layer.CommandGenerator[None]: event = self.quic.next_event() while event is not None: if isinstance(event, quic_events.ConnectionIdIssued): - if self._issue_cid is not None: - self._issue_cid(event.connection_id) + if self.issue_connection_id_callback is not None: + self.issue_connection_id_callback(event.connection_id) elif isinstance(event, quic_events.ConnectionIdRetired): - if self._retire_cid is not None: - self._retire_cid(event.connection_id) + if self.retire_connection_id_callback is not None: + self.retire_connection_id_callback(event.connection_id) elif isinstance(event, quic_events.ConnectionTerminated): # report as TLS failure if the termination happened before the handshake @@ -335,7 +329,7 @@ def process_events(self) -> layer.CommandGenerator[None]: all_certs.extend(self.quic.tls._peer_certificate_chain) # set the connection's TLS properties - self.conn.timestamp_tls_setup = self.loop.time() + self.conn.timestamp_tls_setup = self._loop.time() self.conn.certificate_list = [ certs.Cert.from_pyopenssl(x) for x in all_certs ] @@ -365,7 +359,7 @@ def process_events(self) -> layer.CommandGenerator[None]: event = self.quic.next_event() # send all queued datagrams - for data, addr in self.quic.datagrams_to_send(now=self.loop.time()): + for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): yield commands.SendData(connection=self.conn, data=data, remote_addr=addr) # ensure the wakeup is set and still correct @@ -377,7 +371,7 @@ def process_events(self) -> layer.CommandGenerator[None]: _, existing_timer = self._request_wakeup_command_and_timer if existing_timer == timer: return - command = commands.RequestWakeup(timer - self.loop.time()) + command = commands.RequestWakeup(timer - self._loop.time()) self._request_wakeup_command_and_timer = (command, timer) yield command @@ -395,31 +389,34 @@ def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None - if isinstance(event, events.DataReceived): - # forward incoming data only to aioquic - if event.connection is self.conn: - self.quic.receive_datagram( - data=event.data, addr=event.remote_addr, now=self.loop.time() - ) - yield from self.process_events() - return + # forward incoming data only to aioquic + if isinstance(event, events.DataReceived) and event.connection is self.conn: + assert event.remote_addr is not None + self.quic.receive_datagram( + data=event.data, addr=event.remote_addr, now=self._loop.time() + ) + yield from self.process_events() + return - elif isinstance(event, events.ConnectionClosed): - if event.connection is self.conn: - # connection closed unexpectedly - yield from self.fail_connection( - "Client closed UDP connection.", level="info" - ) + # check if the connection was closed by peer + elif ( + isinstance(event, events.ConnectionClosed) and event.connection is self.conn + ): + yield from self.fail_connection( + "Client closed UDP connection.", level="info" + ) - elif isinstance(event, events.Wakeup): - # make sure we intercept wakeup events for aioquic - if self._request_wakeup_command_and_timer is not None: - command, timer = self._request_wakeup_command_and_timer - if event.command is command: - self._request_wakeup_command_and_timer = None - self.quic.handle_timer(now=max(timer, self.loop.time())) - yield from self.process_events() - return + # intercept wakeup events for aioquic + elif ( + isinstance(event, events.Wakeup) + and self._request_wakeup_command_and_timer is not None + ): + command, timer = self._request_wakeup_command_and_timer + if event.command is command: + self._request_wakeup_command_and_timer = None + self.quic.handle_timer(now=max(timer, self._loop.time())) + yield from self.process_events() + return # forward other events to the child layer yield from self.event_to_child(event) @@ -431,7 +428,7 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: _handle_event = state_start -class ServerQuicLayer(QuicLayer): +class ServerQuicLayer(_QuicLayer): """ This layer establishes QUIC for a single server connection. """ @@ -452,29 +449,26 @@ def start(self) -> layer.CommandGenerator[None]: # try to connect yield from self.initialize_connection() if self.quic is not None: - self.quic.connect(addr=self.conn.peername, now=self.loop.time()) + self.quic.connect(addr=self.conn.peername, now=self._loop.time()) yield from self.process_events() -class ClientQuicLayer(QuicLayer): +class ClientQuicLayer(_QuicLayer): """ This layer establishes QUIC on a single client connection. """ - server_layer: Optional[ServerQuicLayer] buffered_packets: Optional[List[Tuple[bytes, connection.Address, float]]] def __init__( self, context: context.Context, - issue_cid: Callable[[bytes], None], - retire_cid: Callable[[bytes], None], + wait_for_upstream: bool, ) -> None: - super().__init__(context, context.client, issue_cid, retire_cid) - self.server_layer = None - self.buffered_packets = None + super().__init__(context, context.client) + self.buffered_packets = [] if wait_for_upstream else None - def start_client_connection(self) -> layer.CommandGenerator[None]: + def initialize_connection_and_flush_buffer(self) -> layer.CommandGenerator[None]: assert self.buffered_packets is not None yield from self.initialize_connection() @@ -488,98 +482,122 @@ def start_client_connection(self) -> layer.CommandGenerator[None]: yield from self.process_events() def start(self) -> layer.CommandGenerator[None]: - self._handle_event = self.state_wait_for_client_hello - yield from () + if self.buffered_packets is None: + yield from self.initialize_connection() + else: + self._handle_event = self.state_wait_for_upstream - def state_wait_for_client_hello( + def state_wait_for_upstream( self, event: events.Event ) -> layer.CommandGenerator[None]: - assert isinstance(event, events.ConnectionEvent) - assert event.connection is self.conn + assert self.buffered_packets is not None - if isinstance(event, events.DataReceived): + # buffer incoming packets until the upstream handshake completed + if isinstance(event, events.DataReceived) and event.connection is self.conn: assert event.remote_addr is not None + self.buffered_packets.append( + (event.data, event.remote_addr, self._loop.time()) + ) + return - # extract the client hello - try: - ( - client_hello, - self.original_destination_connection_id, - ) = pull_client_hello_and_connection_id(event.data) - except ValueError as e: + # watch for closed connections on both legs + elif isinstance(event, events.ConnectionClosed): + if event.connection is self.conn: yield from self.fail_connection( - f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" + "Client closed UDP connection before upstream server handshake completed.", + level="info", ) - else: - self.conn.sni = client_hello.sni - self.conn.alpn_offers = client_hello.alpn_protocols + elif event.connection is self.context.server: + yield commands.Log( + f"Unable to establish QUIC connection with server ({self.context.server.error or 'Connection closed.'}). " + f"Trying to establish QUIC with client anyway." + ) + yield from self.initialize_connection_and_flush_buffer() - # check with addons what we shall do - hook_data = ClientHelloData(self.context, client_hello) - yield layers.tls.TlsClienthelloHook(hook_data) + # continue if upstream completed the handshake + elif ( + isinstance(event, QuicConnectionEvent) + and event.connection is self.context.server + and isinstance(event.event, quic_events.HandshakeCompleted) + ): + yield from self.initialize_connection_and_flush_buffer() - if hook_data.ignore_connection: - # simply relay everything (including the client hello) - relay_layer = layers.TCPLayer(self.context, ignore=True) - self._handle_event = relay_layer.handle_event - yield from relay_layer.handle_event(events.Start()) - yield from relay_layer.handle_event(event) + # forward other events to the child layer + yield from self.event_to_child(event) - else: - # buffer the client hello - self.buffered_packets = [ - (event.data, event.remote_addr, self.loop.time()) - ] - - # contact the upstream server first if so desired - if hook_data.establish_server_tls_first: - self.server_layer = ServerQuicLayer(self.context) - self._handle_event = self.state_wait_for_upstream_server - yield from self.state_wait_for_upstream_server(events.Start()) - else: - yield from self.start_client_connection() - elif isinstance(event, events.ConnectionClosed): - # this is odd since this layer should only be created if there is a packet - self._handle_event = self.state_done +class QuicLayer(layer.Layer): + """ + Entry layer for QUIC proxy server. + """ - else: - raise AssertionError(f"Unexpected event: {event}") + def __init__( + self, + context: context.Context, + issue_cid: Callable[[bytes], None], + retire_cid: Callable[[bytes], None], + ) -> None: + super().__init__(context) + self._issue_cid = issue_cid + self._retire_cid = retire_cid - def state_wait_for_upstream_server( - self, event: events.Event - ) -> layer.CommandGenerator[None]: - assert self.buffered_packets is not None - assert self.server_layer is not None + def build_client_layer( + self, connection_id: bytes, wait_for_upstream: bool + ) -> ClientQuicLayer: + layer = ClientQuicLayer( + context=self.context, wait_for_upstream=wait_for_upstream + ) + layer.original_destination_connection_id = connection_id + layer.issue_connection_id_callback = self._issue_cid + layer.retire_connection_id_callback = self._retire_cid + return layer + + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + # only handle the first packet from the client + if ( + not isinstance(event, events.DataReceived) + or event.connection is not self.context.client + ): + return - # filter DataReceived and ConnectionClosed relating to the client connection - if isinstance(event, events.ConnectionEvent): - if event.connection is self.conn: - if isinstance(event, events.DataReceived): - assert event.remote_addr is not None + # extract the client hello + try: + client_hello, connection_id = pull_client_hello_and_connection_id( + event.data + ) + except ValueError as e: + yield commands.Log( + f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" + ) + yield commands.CloseConnection(self.context.client) + return - # still waiting for the server, buffer the data - self.buffered_packets.append( - (event.data, event.remote_addr, self.loop.time()) - ) + # copy the information + self.context.client.sni = client_hello.sni + self.context.client.alpn_offers = client_hello.alpn_protocols - elif isinstance(event, events.ConnectionClosed): - # close the upstream connection as well and be done - self._handle_event = self.state_done - yield from self.server_layer.fail_connection( - "Client closed the connection." - ) + # check with addons what we shall do + next_layer: layer.Layer + hook_data = ClientHelloData(self.context, client_hello) + yield layers.tls.TlsClienthelloHook(hook_data) - else: - raise AssertionError(f"Unexpected event: {event}") + # simply relay everything + if hook_data.ignore_connection: + next_layer = layers.TCPLayer(self.context, ignore=True) - # forward the event and check it's results - yield from self.server_layer.handle_event(event) - if not self.context.server.connected: - yield commands.Log( - f"Unable to establish QUIC connection with server ({self.context.server.error or 'Connection closed.'}). " - f"Trying to establish QUIC with client anyway." + # contact the upstream server first + elif hook_data.establish_server_tls_first: + next_layer = ServerQuicLayer(self.context) + next_layer.child_layer = self.build_client_layer( + connection_id, wait_for_upstream=True ) - yield from self.start_client_connection() - elif self.context.server.tls_established: - yield from self.start_client_connection() + + # perform the client handshake immediately + else: + next_layer = self.build_client_layer(connection_id, wait_for_upstream=False) + + # replace this layer and start the next one + self.handle_event = next_layer.handle_event + self._handle_event = next_layer._handle_event + yield from next_layer.handle_event(events.Start()) + yield from next_layer.handle_event(event) From ac21eac71ebfed537933a7842c84557ec4a84b9e Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 20 Jun 2022 04:25:42 +0200 Subject: [PATCH 013/695] [quic] expose transmit improve connection shutdown --- mitmproxy/proxy/layers/quic.py | 99 ++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 40 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index dd0f55cd1e..1fc14282aa 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -7,7 +7,7 @@ from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration -from aioquic.quic.connection import QuicConnection, QuicConnectionError +from aioquic.quic.connection import QuicConnection, QuicConnectionError, QuicErrorCode from aioquic.tls import CipherSuite, HandshakeType from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header from cryptography import x509 @@ -16,7 +16,6 @@ from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, layers from mitmproxy.tls import ClientHello, ClientHelloData, TlsData -from mitmproxy.utils import human @dataclass @@ -90,6 +89,11 @@ class QuicGetConnection(commands.ConnectionCommand): # -> QuicConnection blocking = True +@dataclass +class QuicTransmit: + connection: QuicConnection + + @dataclass(repr=False) class QuicGetConnectionCompleted(events.CommandCompleted): command: QuicGetConnection @@ -228,33 +232,26 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: ) ) + # transmit buffered data and re-arm timer + elif isinstance(command, QuicTransmit) and command.connection is self.quic: + yield from self.transmit() + # properly close QUIC connections elif ( isinstance(command, commands.CloseConnection) and command.connection is self.conn ): + reason = "CloseConnection command received." if self.conn.connected and self.quic is not None: - self.quic.close() + self.quic.close(reason_phrase=reason) yield from self.process_events() - self._handle_event = self.state_done - yield command + else: + yield from self.shutdown_connection(reason, level="info") # return other commands else: yield command - def fail_connection( - self, - reason: str, - level: Literal["error", "warn", "info", "alert", "debug"] = "warn", - ) -> layer.CommandGenerator[None]: - yield commands.Log( - message=f"Failing connection {self.conn}: {reason}", level=level - ) - if self.conn.connected: - yield commands.CloseConnection(self.conn) - self._handle_event = self.state_done - def initialize_connection(self) -> layer.CommandGenerator[None]: assert self.quic is None @@ -265,7 +262,7 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: else: yield QuicTlsStartServerHook(tls_data) if tls_data.settings is None: - yield from self.fail_connection( + yield from self.shutdown_connection( "No TLS settings were provided, failing connection.", level="error" ) return @@ -318,7 +315,12 @@ def process_events(self) -> layer.CommandGenerator[None]: yield layers.tls.TlsFailedServerHook(tls_data) # always close the connection - yield from self.fail_connection(event.reason_phrase) + yield from self.shutdown_connection( + event.reason_phrase, + level=( + "info" if event.error_code is QuicErrorCode.NO_ERROR else "warn" + ), + ) elif isinstance(event, quic_events.HandshakeCompleted): # concatenate all peer certificates @@ -358,22 +360,20 @@ def process_events(self) -> layer.CommandGenerator[None]: # handle the next event event = self.quic.next_event() - # send all queued datagrams - for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): - yield commands.SendData(connection=self.conn, data=data, remote_addr=addr) + # transmit buffered data and re-arm timer + yield from self.transmit() - # ensure the wakeup is set and still correct - timer = self.quic.get_timer() - if timer is None: - self._request_wakeup_command_and_timer = None - else: - if self._request_wakeup_command_and_timer is not None: - _, existing_timer = self._request_wakeup_command_and_timer - if existing_timer == timer: - return - command = commands.RequestWakeup(timer - self._loop.time()) - self._request_wakeup_command_and_timer = (command, timer) - yield command + def shutdown_connection( + self, + reason: str, + level: Literal["error", "warn", "info", "alert", "debug"], + ) -> layer.CommandGenerator[None]: + yield commands.Log( + message=f"Connection {self.conn} closed: {reason}", level=level + ) + if self.conn.connected: + yield commands.CloseConnection(self.conn) + self._handle_event = self.state_done @abstractmethod def start(self) -> layer.CommandGenerator[None]: @@ -402,8 +402,8 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: elif ( isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): - yield from self.fail_connection( - "Client closed UDP connection.", level="info" + yield from self.shutdown_connection( + "Peer UDP connection timed out.", level="info" ) # intercept wakeup events for aioquic @@ -425,6 +425,24 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: # when done, just forward the event yield from self.child_layer.handle_event(event) + def transmit(self) -> layer.CommandGenerator[None]: + # send all queued datagrams + for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): + yield commands.SendData(connection=self.conn, data=data, remote_addr=addr) + + # ensure the wakeup is set and still correct + timer = self.quic.get_timer() + if timer is None: + self._request_wakeup_command_and_timer = None + else: + if self._request_wakeup_command_and_timer is not None: + _, existing_timer = self._request_wakeup_command_and_timer + if existing_timer == timer: + return + command = commands.RequestWakeup(timer - self._loop.time()) + self._request_wakeup_command_and_timer = (command, timer) + yield command + _handle_event = state_start @@ -441,8 +459,9 @@ def start(self) -> layer.CommandGenerator[None]: if not self.conn.connected: err = yield commands.OpenConnection(self.conn) if err is not None: - self.fail_connection( - f"Failed to establish connection to {human.format_address(self.conn)}: {err}" + self.shutdown_connection( + f"Failed to connect: {err}", + level="warn", ) return @@ -503,8 +522,8 @@ def state_wait_for_upstream( # watch for closed connections on both legs elif isinstance(event, events.ConnectionClosed): if event.connection is self.conn: - yield from self.fail_connection( - "Client closed UDP connection before upstream server handshake completed.", + yield from self.shutdown_connection( + "Client UDP connection timeout out before upstream server handshake completed.", level="info", ) elif event.connection is self.context.server: From f129a1e5a30fa3534028b667af90c2f5ece517d1 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 20 Jun 2022 04:26:36 +0200 Subject: [PATCH 014/695] [quic] generalize h2 and h3 headers --- mitmproxy/http.py | 4 ++ mitmproxy/proxy/layers/http/_base.py | 71 ++++++++++++++++++++++++++- mitmproxy/proxy/layers/http/_http2.py | 66 ++++--------------------- 3 files changed, 84 insertions(+), 57 deletions(-) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index e88242530c..394cddbdd2 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -285,6 +285,10 @@ def is_http11(self) -> bool: def is_http2(self) -> bool: return self.data.http_version == b"HTTP/2.0" + @property + def is_http3(self) -> bool: + return self.data.http_version == b"HTTP/3" + @property def headers(self) -> Headers: """ diff --git a/mitmproxy/proxy/layers/http/_base.py b/mitmproxy/proxy/layers/http/_base.py index b5f66d46ba..a0f82a2349 100644 --- a/mitmproxy/proxy/layers/http/_base.py +++ b/mitmproxy/proxy/layers/http/_base.py @@ -2,10 +2,13 @@ import textwrap from dataclasses import dataclass -from mitmproxy import http +import h2.utilities + +from mitmproxy import ctx, http from mitmproxy.connection import Connection from mitmproxy.proxy import commands, events, layer from mitmproxy.proxy.context import Context +from mitmproxy.proxy.layers.http import RequestHeaders, ResponseHeaders StreamId = int @@ -57,3 +60,69 @@ def format_error(status_code: int, message: str) -> bytes: .strip() .encode("utf8", "replace") ) + + +def get_request_headers( + event: RequestHeaders, +) -> layer.CommandGenerator[list[tuple[bytes, bytes]]]: + pseudo_headers = [ + (b":method", event.request.data.method), + (b":scheme", event.request.data.scheme), + (b":path", event.request.data.path), + ] + if event.request.authority: + pseudo_headers.append((b":authority", event.request.data.authority)) + + if event.request.is_http2 or event.request.is_http3: + hdrs = list(event.request.headers.fields) + if ctx.options.normalize_outbound_headers: + yield from normalize_h2_or_h3_headers(hdrs) + else: + headers = event.request.headers + if not event.request.authority and "host" in headers: + headers = headers.copy() + pseudo_headers.append((b":authority", headers.pop(b"host"))) + hdrs = normalize_h1_headers(list(headers.fields), True) + + return pseudo_headers + hdrs + + +def get_response_headers( + event: ResponseHeaders, +) -> layer.CommandGenerator[list[tuple[bytes, bytes]]]: + headers = [ + (b":status", b"%d" % event.response.status_code), + *event.response.headers.fields, + ] + if event.response.is_http2 or event.request.is_http3: + if ctx.options.normalize_outbound_headers: + yield from normalize_h2_or_h3_headers(headers) + else: + headers = normalize_h1_headers(headers, False) + return headers + + +def normalize_h1_headers( + headers: list[tuple[bytes, bytes]], is_client: bool +) -> list[tuple[bytes, bytes]]: + # HTTP/1 servers commonly send capitalized headers (Content-Length vs content-length), + # which isn't valid HTTP/2 or HTTP/3. As such we normalize. + headers = h2.utilities.normalize_outbound_headers( + headers, + h2.utilities.HeaderValidationFlags(is_client, False, not is_client, False), + ) + # make sure that this is not just an iterator but an iterable, + # otherwise hyper-h2 will silently drop headers. + headers = list(headers) + return headers + + +def normalize_h2_or_h3_headers( + headers: list[tuple[bytes, bytes]] +) -> layer.CommandGenerator[None]: + for i in range(len(headers)): + if not headers[i][0].islower(): + yield commands.Log( + f"Lowercased {repr(headers[i][0]).lstrip('b')} header as uppercase is not allowed with HTTP/2 nor HTTP/3." + ) + headers[i] = (headers[i][0].lower(), headers[i][1]) diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index 3e34c325da..bbb1b19f81 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -29,7 +29,14 @@ ResponseTrailers, ResponseProtocolError, ) -from ._base import HttpConnection, HttpEvent, ReceiveHttp, format_error +from ._base import ( + HttpConnection, + HttpEvent, + ReceiveHttp, + format_error, + get_request_headers, + get_response_headers, +) from ._http_h2 import BufferedH2Connection, H2ConnectionLogger from ...commands import CloseConnection, Log, SendData, RequestWakeup from ...context import Context @@ -289,30 +296,6 @@ def done(self, _) -> CommandGenerator[None]: yield from () -def normalize_h1_headers( - headers: list[tuple[bytes, bytes]], is_client: bool -) -> list[tuple[bytes, bytes]]: - # HTTP/1 servers commonly send capitalized headers (Content-Length vs content-length), - # which isn't valid HTTP/2. As such we normalize. - headers = h2.utilities.normalize_outbound_headers( - headers, - h2.utilities.HeaderValidationFlags(is_client, False, not is_client, False), - ) - # make sure that this is not just an iterator but an iterable, - # otherwise hyper-h2 will silently drop headers. - headers = list(headers) - return headers - - -def normalize_h2_headers(headers: list[tuple[bytes, bytes]]) -> CommandGenerator[None]: - for i in range(len(headers)): - if not headers[i][0].islower(): - yield Log( - f"Lowercased {repr(headers[i][0]).lstrip('b')} header as uppercase is not allowed with HTTP/2." - ) - headers[i] = (headers[i][0].lower(), headers[i][1]) - - class Http2Server(Http2Connection): h2_conf = h2.config.H2Configuration( **Http2Connection.h2_conf_defaults, @@ -330,19 +313,9 @@ def __init__(self, context: Context): def _handle_event(self, event: Event) -> CommandGenerator[None]: if isinstance(event, ResponseHeaders): if self.is_open_for_us(event.stream_id): - headers = [ - (b":status", b"%d" % event.response.status_code), - *event.response.headers.fields, - ] - if event.response.is_http2: - if self.context.options.normalize_outbound_headers: - yield from normalize_h2_headers(headers) - else: - headers = normalize_h1_headers(headers, False) - self.h2_conn.send_headers( event.stream_id, - headers, + headers=(yield from get_response_headers(event)), end_stream=event.end_stream, ) yield SendData(self.conn, self.h2_conn.data_to_send()) @@ -485,28 +458,9 @@ def _handle_event2(self, event: Event) -> CommandGenerator[None]: yield RequestWakeup(self.context.options.http2_ping_keepalive) yield from super()._handle_event(event) elif isinstance(event, RequestHeaders): - pseudo_headers = [ - (b":method", event.request.data.method), - (b":scheme", event.request.data.scheme), - (b":path", event.request.data.path), - ] - if event.request.authority: - pseudo_headers.append((b":authority", event.request.data.authority)) - - if event.request.is_http2: - hdrs = list(event.request.headers.fields) - if self.context.options.normalize_outbound_headers: - yield from normalize_h2_headers(hdrs) - else: - headers = event.request.headers - if not event.request.authority and "host" in headers: - headers = headers.copy() - pseudo_headers.append((b":authority", headers.pop(b"host"))) - hdrs = normalize_h1_headers(list(headers.fields), True) - self.h2_conn.send_headers( event.stream_id, - pseudo_headers + hdrs, + headers=(yield from get_request_headers(event)), end_stream=event.end_stream, ) self.streams[event.stream_id] = StreamState.EXPECTING_HEADERS From 71645ddc8d4236bb23de8f35793a25929dad1b39 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 20 Jun 2022 04:27:42 +0200 Subject: [PATCH 015/695] [quic] first work on H3 connections --- mitmproxy/proxy/layers/http/_http3.py | 168 ++++++++++++++++++++++++-- 1 file changed, 155 insertions(+), 13 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 1f57add6eb..3e0aea5646 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,29 +1,171 @@ -from aioquic.h3.connection import H3Connection -from mitmproxy.connection import Connection -from ._base import HttpConnection -from ..quic import QuicLayer -from ...context import Context +from abc import abstractmethod +from typing import Optional, Union + +from aioquic.quic.connection import QuicConnection +from aioquic.h3.connection import ( + H3Connection, + FrameUnexpected, + ErrorCode as H3ErrorCode, +) +from aioquic.h3 import events as h3_events + +from mitmproxy import version +from mitmproxy.net.http import status_codes +from mitmproxy.proxy import context, events, layer +from mitmproxy.proxy.layers.quic import ( + QuicConnectionEvent, + QuicGetConnection, + QuicTransmit, +) + +from . import ( + RequestData, + RequestEndOfMessage, + RequestHeaders, + RequestProtocolError, + ResponseData, + ResponseEndOfMessage, + ResponseHeaders, + RequestTrailers, + ResponseTrailers, + ResponseProtocolError, +) +from ._base import ( + HttpConnection, + HttpEvent, + format_error, + get_request_headers, + get_response_headers, +) class Http3Connection(HttpConnection): - h3_conn: H3Connection + quic: Optional[QuicConnection] = None + h3_conn: Optional[H3Connection] = None + + EventData: type[Union[RequestData, ResponseData]] + ReceiveData: type[Union[RequestData, ResponseData]] + EventEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] + ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] + EventHeaders: type[Union[RequestHeaders, ResponseHeaders]] + ReceiveHeaders: type[Union[RequestHeaders, ResponseHeaders]] + EventProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] + ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] + EventTrailers: type[Union[RequestTrailers, ResponseTrailers]] + ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] + + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.Start): + self.quic = yield QuicGetConnection() + assert isinstance(self.quic, H3Connection) + self.h3_conn = H3Connection(self.quic, enable_webtransport=False) + + else: + assert self.quic is not None + assert self.h3_conn is not None + + if isinstance(event, HttpEvent): + try: + + if isinstance(event, self.EventData): + self.h3_conn.send_data( + stream_id=event.stream_id, data=event.data, end_stream=False + ) + elif isinstance(event, self.EventHeaders): + get_headers = ( + get_request_headers + if isinstance(event, RequestHeaders) + else get_response_headers + ) + self.h3_conn.send_headers( + stream_id=event.stream_id, + headers=(yield from get_headers(event)), + end_stream=event.end_stream, + ) + elif isinstance(event, self.EventTrailers): + trailers = [*event.trailers.fields] + self.h3_conn.send_headers( + stream_id=event.stream_id, headers=trailers, end_stream=True + ) + elif isinstance(event, self.EventEndOfMessage): + self.h3_conn.send_data( + stream_id=event.stream_id, data=b"", end_stream=True + ) + elif isinstance(event, self.EventProtocolError): + self.protocol_error(event) + else: + raise AssertionError(f"Unexpected event: {event}") + + except FrameUnexpected: + # Http2Connection also ignores events that violate the current stream state + return - def __init__(self, context: Context, conn: Connection): - super().__init__(context, conn) - quic = context.layers[0] - assert isinstance(quic, QuicLayer) - self.h3_conn = H3Connection(quic.conn) + # transmit buffered data and re-arm timer + yield QuicTransmit(self.quic) + + elif isinstance(event, QuicConnectionEvent): + for h3_event in self.h3_conn.handle_event(event.event): + if isinstance(h3_event, h3_events.DataReceived): + pass + + elif isinstance(h3_event, h3_events.HeadersReceived): + pass + + else: + pass + + @abstractmethod + def protocol_error( + self, event: Union[RequestProtocolError, ResponseProtocolError] + ) -> None: + yield from () # pragma: no cover class Http3Server(Http3Connection): - def __init__(self, context: Context): + def __init__(self, context: context.Context): super().__init__(context, context.client) + def protocol_error( + self, event: Union[RequestProtocolError, ResponseProtocolError] + ) -> None: + assert self.h3_conn is not None + assert isinstance(event, ResponseProtocolError) + + # same as HTTP/2 + code = event.code + if code != status_codes.CLIENT_CLOSED_REQUEST: + code = status_codes.INTERNAL_SERVER_ERROR + self.h3_conn.send_headers( + stream_id=event.stream_id, + headers=[ + (b":status", b"%d" % code), + (b"server", version.MITMPROXY.encode()), + (b"content-type", b"text/html"), + ], + ) + self.h3_conn.send_data( + stream_id=event.stream_id, + data=format_error(code, event.message), + end_stream=True, + ) + class Http3Client(Http3Connection): - def __init__(self, context: Context): + def __init__(self, context: context.Context): super().__init__(context, context.server) + def protocol_error( + self, event: Union[RequestProtocolError, ResponseProtocolError] + ) -> None: + assert isinstance(event, RequestProtocolError) + assert self.quic is not None + + # same as HTTP/2 + code = event.code + if code != H3ErrorCode.H3_REQUEST_CANCELLED: + code = H3ErrorCode.H3_INTERNAL_ERROR + self.quic.reset_stream(stream_id=event.stream_id, error_code=code) + __all__ = [ "Http3Client", From 5454a71bb69b0836c135074725fe60ee7cd044fc Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 20 Jun 2022 17:05:32 +0200 Subject: [PATCH 016/695] [quic] more work on H3 --- mitmproxy/proxy/layers/http/_base.py | 71 +------------ mitmproxy/proxy/layers/http/_http2.py | 83 +++++++++++++-- mitmproxy/proxy/layers/http/_http3.py | 143 ++++++++++++++++++++------ mitmproxy/proxy/layers/quic.py | 4 +- 4 files changed, 189 insertions(+), 112 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_base.py b/mitmproxy/proxy/layers/http/_base.py index a0f82a2349..b5f66d46ba 100644 --- a/mitmproxy/proxy/layers/http/_base.py +++ b/mitmproxy/proxy/layers/http/_base.py @@ -2,13 +2,10 @@ import textwrap from dataclasses import dataclass -import h2.utilities - -from mitmproxy import ctx, http +from mitmproxy import http from mitmproxy.connection import Connection from mitmproxy.proxy import commands, events, layer from mitmproxy.proxy.context import Context -from mitmproxy.proxy.layers.http import RequestHeaders, ResponseHeaders StreamId = int @@ -60,69 +57,3 @@ def format_error(status_code: int, message: str) -> bytes: .strip() .encode("utf8", "replace") ) - - -def get_request_headers( - event: RequestHeaders, -) -> layer.CommandGenerator[list[tuple[bytes, bytes]]]: - pseudo_headers = [ - (b":method", event.request.data.method), - (b":scheme", event.request.data.scheme), - (b":path", event.request.data.path), - ] - if event.request.authority: - pseudo_headers.append((b":authority", event.request.data.authority)) - - if event.request.is_http2 or event.request.is_http3: - hdrs = list(event.request.headers.fields) - if ctx.options.normalize_outbound_headers: - yield from normalize_h2_or_h3_headers(hdrs) - else: - headers = event.request.headers - if not event.request.authority and "host" in headers: - headers = headers.copy() - pseudo_headers.append((b":authority", headers.pop(b"host"))) - hdrs = normalize_h1_headers(list(headers.fields), True) - - return pseudo_headers + hdrs - - -def get_response_headers( - event: ResponseHeaders, -) -> layer.CommandGenerator[list[tuple[bytes, bytes]]]: - headers = [ - (b":status", b"%d" % event.response.status_code), - *event.response.headers.fields, - ] - if event.response.is_http2 or event.request.is_http3: - if ctx.options.normalize_outbound_headers: - yield from normalize_h2_or_h3_headers(headers) - else: - headers = normalize_h1_headers(headers, False) - return headers - - -def normalize_h1_headers( - headers: list[tuple[bytes, bytes]], is_client: bool -) -> list[tuple[bytes, bytes]]: - # HTTP/1 servers commonly send capitalized headers (Content-Length vs content-length), - # which isn't valid HTTP/2 or HTTP/3. As such we normalize. - headers = h2.utilities.normalize_outbound_headers( - headers, - h2.utilities.HeaderValidationFlags(is_client, False, not is_client, False), - ) - # make sure that this is not just an iterator but an iterable, - # otherwise hyper-h2 will silently drop headers. - headers = list(headers) - return headers - - -def normalize_h2_or_h3_headers( - headers: list[tuple[bytes, bytes]] -) -> layer.CommandGenerator[None]: - for i in range(len(headers)): - if not headers[i][0].islower(): - yield commands.Log( - f"Lowercased {repr(headers[i][0]).lstrip('b')} header as uppercase is not allowed with HTTP/2 nor HTTP/3." - ) - headers[i] = (headers[i][0].lower(), headers[i][1]) diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index bbb1b19f81..ec4aed6b72 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -13,7 +13,7 @@ import h2.stream import h2.utilities -from mitmproxy import http, version +from mitmproxy import ctx, http, version from mitmproxy.connection import Connection from mitmproxy.net.http import status_codes, url from mitmproxy.utils import human @@ -29,14 +29,7 @@ ResponseTrailers, ResponseProtocolError, ) -from ._base import ( - HttpConnection, - HttpEvent, - ReceiveHttp, - format_error, - get_request_headers, - get_response_headers, -) +from ._base import HttpConnection, HttpEvent, ReceiveHttp, format_error from ._http_h2 import BufferedH2Connection, H2ConnectionLogger from ...commands import CloseConnection, Log, SendData, RequestWakeup from ...context import Context @@ -296,6 +289,70 @@ def done(self, _) -> CommandGenerator[None]: yield from () +def normalize_h1_headers( + headers: list[tuple[bytes, bytes]], is_client: bool +) -> list[tuple[bytes, bytes]]: + # HTTP/1 servers commonly send capitalized headers (Content-Length vs content-length), + # which isn't valid HTTP/2. As such we normalize. + headers = h2.utilities.normalize_outbound_headers( + headers, + h2.utilities.HeaderValidationFlags(is_client, False, not is_client, False), + ) + # make sure that this is not just an iterator but an iterable, + # otherwise hyper-h2 will silently drop headers. + headers = list(headers) + return headers + + +def normalize_h2_headers(headers: list[tuple[bytes, bytes]]) -> CommandGenerator[None]: + for i in range(len(headers)): + if not headers[i][0].islower(): + yield Log( + f"Lowercased {repr(headers[i][0]).lstrip('b')} header as uppercase is not allowed with HTTP/2." + ) + headers[i] = (headers[i][0].lower(), headers[i][1]) + + +def format_h2_request_headers( + event: RequestHeaders, +) -> CommandGenerator[list[tuple[bytes, bytes]]]: + pseudo_headers = [ + (b":method", event.request.data.method), + (b":scheme", event.request.data.scheme), + (b":path", event.request.data.path), + ] + if event.request.authority: + pseudo_headers.append((b":authority", event.request.data.authority)) + + if event.request.is_http2 or event.request.is_http3: + hdrs = list(event.request.headers.fields) + if ctx.options.normalize_outbound_headers: + yield from normalize_h2_headers(hdrs) + else: + headers = event.request.headers + if not event.request.authority and "host" in headers: + headers = headers.copy() + pseudo_headers.append((b":authority", headers.pop(b"host"))) + hdrs = normalize_h1_headers(list(headers.fields), True) + + return pseudo_headers + hdrs + + +def format_h2_response_headers( + event: ResponseHeaders, +) -> CommandGenerator[list[tuple[bytes, bytes]]]: + headers = [ + (b":status", b"%d" % event.response.status_code), + *event.response.headers.fields, + ] + if event.response.is_http2: + if ctx.options.normalize_outbound_headers: + yield from normalize_h2_headers(headers) + else: + headers = normalize_h1_headers(headers, False) + return headers + + class Http2Server(Http2Connection): h2_conf = h2.config.H2Configuration( **Http2Connection.h2_conf_defaults, @@ -315,7 +372,7 @@ def _handle_event(self, event: Event) -> CommandGenerator[None]: if self.is_open_for_us(event.stream_id): self.h2_conn.send_headers( event.stream_id, - headers=(yield from get_response_headers(event)), + headers=(yield from format_h2_response_headers(event)), end_stream=event.end_stream, ) yield SendData(self.conn, self.h2_conn.data_to_send()) @@ -460,7 +517,7 @@ def _handle_event2(self, event: Event) -> CommandGenerator[None]: elif isinstance(event, RequestHeaders): self.h2_conn.send_headers( event.stream_id, - headers=(yield from get_request_headers(event)), + headers=(yield from format_h2_request_headers(event)), end_stream=event.end_stream, ) self.streams[event.stream_id] = StreamState.EXPECTING_HEADERS @@ -596,6 +653,10 @@ def parse_h2_response_headers( __all__ = [ + "format_h2_request_headers", + "format_h2_response_headers", + "parse_h2_request_headers", + "parse_h2_response_headers", "Http2Client", "Http2Server", ] diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 3e0aea5646..ec600296e5 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,4 +1,5 @@ from abc import abstractmethod +import time from typing import Optional, Union from aioquic.quic.connection import QuicConnection @@ -6,12 +7,13 @@ H3Connection, FrameUnexpected, ErrorCode as H3ErrorCode, + HeadersState as H3HeadersState, ) from aioquic.h3 import events as h3_events -from mitmproxy import version +from mitmproxy import http, version from mitmproxy.net.http import status_codes -from mitmproxy.proxy import context, events, layer +from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( QuicConnectionEvent, QuicGetConnection, @@ -33,9 +35,14 @@ from ._base import ( HttpConnection, HttpEvent, + ReceiveHttp, format_error, - get_request_headers, - get_response_headers, +) +from ._http2 import ( + format_h2_request_headers, + format_h2_response_headers, + parse_h2_request_headers, + parse_h2_response_headers, ) @@ -43,22 +50,14 @@ class Http3Connection(HttpConnection): quic: Optional[QuicConnection] = None h3_conn: Optional[H3Connection] = None - EventData: type[Union[RequestData, ResponseData]] - ReceiveData: type[Union[RequestData, ResponseData]] - EventEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] - ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] - EventHeaders: type[Union[RequestHeaders, ResponseHeaders]] - ReceiveHeaders: type[Union[RequestHeaders, ResponseHeaders]] - EventProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] - ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] - EventTrailers: type[Union[RequestTrailers, ResponseTrailers]] ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): - self.quic = yield QuicGetConnection() - assert isinstance(self.quic, H3Connection) - self.h3_conn = H3Connection(self.quic, enable_webtransport=False) + quic = yield QuicGetConnection(self.conn) + assert isinstance(quic, QuicConnection) + self.quic = quic + self.h3_conn = H3Connection(quic, enable_webtransport=False) else: assert self.quic is not None @@ -67,31 +66,34 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, HttpEvent): try: - if isinstance(event, self.EventData): + if isinstance(event, (RequestData, ResponseData)): self.h3_conn.send_data( stream_id=event.stream_id, data=event.data, end_stream=False ) - elif isinstance(event, self.EventHeaders): - get_headers = ( - get_request_headers - if isinstance(event, RequestHeaders) - else get_response_headers - ) + elif isinstance(event, (RequestHeaders, ResponseHeaders)): self.h3_conn.send_headers( stream_id=event.stream_id, - headers=(yield from get_headers(event)), + headers=( + yield from ( + format_h2_request_headers(event) + if isinstance(event, RequestHeaders) + else format_h2_response_headers(event) + ) + ), end_stream=event.end_stream, ) - elif isinstance(event, self.EventTrailers): + elif isinstance(event, (RequestTrailers, ResponseTrailers)): trailers = [*event.trailers.fields] self.h3_conn.send_headers( stream_id=event.stream_id, headers=trailers, end_stream=True ) - elif isinstance(event, self.EventEndOfMessage): + elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): self.h3_conn.send_data( stream_id=event.stream_id, data=b"", end_stream=True ) - elif isinstance(event, self.EventProtocolError): + elif isinstance( + event, (RequestProtocolError, ResponseProtocolError) + ): self.protocol_error(event) else: raise AssertionError(f"Unexpected event: {event}") @@ -108,20 +110,49 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(h3_event, h3_events.DataReceived): pass + # handle headers and trailers elif isinstance(h3_event, h3_events.HeadersReceived): - pass + if ( + self.h3_conn._stream[h3_event.stream_id].headers_recv_state + is H3HeadersState.AFTER_TRAILERS + ): + yield ReceiveHttp( + self.ReceiveTrailers( + stream_id=h3_event.stream_id, + trailers=http.Headers(h3_event.headers), + ) + ) + else: + try: + receive_event = self.headers_received(h3_event) + except ValueError as e: + # TODO + pass + else: + yield ReceiveHttp(receive_event) + # we don't support push, web transport, etc. else: - pass + yield commands.Log( + f"Ignored unsupported H3 event: {h3_event!r}" + ) @abstractmethod def protocol_error( self, event: Union[RequestProtocolError, ResponseProtocolError] ) -> None: - yield from () # pragma: no cover + pass # pragma: no cover + + @abstractmethod + def headers_received( + self, event: h3_events.HeadersReceived + ) -> Union[RequestHeaders, ResponseHeaders]: + pass # pragma: no cover class Http3Server(Http3Connection): + ReceiveTrailers = RequestTrailers + def __init__(self, context: context.Context): super().__init__(context, context.client) @@ -149,8 +180,41 @@ def protocol_error( end_stream=True, ) + def headers_received( + self, event: h3_events.HeadersReceived + ) -> Union[RequestHeaders, ResponseHeaders]: + # same as HTTP/2 + ( + host, + port, + method, + scheme, + authority, + path, + headers, + ) = parse_h2_request_headers(event) + request = http.Request( + host=host, + port=port, + method=method, + scheme=scheme, + authority=authority, + path=path, + http_version=b"HTTP/3", + headers=headers, + content=None, + trailers=None, + timestamp_start=time.time(), + timestamp_end=None, + ) + return RequestHeaders( + stream_id=event.stream_id, request=request, end_stream=event.stream_ended + ) + class Http3Client(Http3Connection): + ReceiveTrailers = ResponseTrailers + def __init__(self, context: context.Context): super().__init__(context, context.server) @@ -166,6 +230,25 @@ def protocol_error( code = H3ErrorCode.H3_INTERNAL_ERROR self.quic.reset_stream(stream_id=event.stream_id, error_code=code) + def headers_received( + self, event: h3_events.HeadersReceived + ) -> Union[RequestHeaders, ResponseHeaders]: + # same as HTTP/2 + status_code, headers = parse_h2_response_headers(event.headers) + response = http.Response( + http_version=b"HTTP/3", + status_code=status_code, + reason=b"", + headers=headers, + content=None, + trailers=None, + timestamp_start=time.time(), + timestamp_end=None, + ) + return ResponseHeaders( + stream_id=event.stream_id, response=response, end_stream=event.stream_ended + ) + __all__ = [ "Http3Client", diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 1fc14282aa..ee656686f5 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -90,7 +90,7 @@ class QuicGetConnection(commands.ConnectionCommand): # -> QuicConnection @dataclass -class QuicTransmit: +class QuicTransmit(commands.Command): connection: QuicConnection @@ -426,6 +426,8 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.child_layer.handle_event(event) def transmit(self) -> layer.CommandGenerator[None]: + assert self.quic + # send all queued datagrams for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): yield commands.SendData(connection=self.conn, data=data, remote_addr=addr) From 9a9c962caa0ededd0f39b84bd7a25bc8633ae577 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 20 Jun 2022 22:09:22 +0200 Subject: [PATCH 017/695] [quic] improve close connection handling --- mitmproxy/proxy/layers/quic.py | 74 +++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index ee656686f5..b53f163d7a 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -7,7 +7,12 @@ from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration -from aioquic.quic.connection import QuicConnection, QuicConnectionError, QuicErrorCode +from aioquic.quic.connection import ( + QuicConnection, + QuicConnectionError, + QuicConnectionState, + QuicErrorCode, +) from aioquic.tls import CipherSuite, HandshakeType from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header from cryptography import x509 @@ -97,7 +102,7 @@ class QuicTransmit(commands.Command): @dataclass(repr=False) class QuicGetConnectionCompleted(events.CommandCompleted): command: QuicGetConnection - connection: QuicConnection + reply: QuicConnection class QuicSecretsLogger: @@ -217,7 +222,7 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: # filter commands coming from the child layer for command in self.child_layer.handle_event(event): - # answer or queue requests for the aioquic connection instanc + # answer or queue requests for the aioquic connection instance if ( isinstance(command, QuicGetConnection) and command.connection is self.conn @@ -228,7 +233,7 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.child_layer.handle_event( QuicGetConnectionCompleted( command=command, - connection=self.quic, + reply=self.quic, ) ) @@ -242,11 +247,11 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: and command.connection is self.conn ): reason = "CloseConnection command received." - if self.conn.connected and self.quic is not None: + if self.quic is None: + yield from self.shutdown_connection(reason=reason, level="info") + else: self.quic.close(reason_phrase=reason) yield from self.process_events() - else: - yield from self.shutdown_connection(reason, level="info") # return other commands else: @@ -263,7 +268,8 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: yield QuicTlsStartServerHook(tls_data) if tls_data.settings is None: yield from self.shutdown_connection( - "No TLS settings were provided, failing connection.", level="error" + reason="No TLS settings were provided, failing connection.", + level="error", ) return self.tls = tls_data.settings @@ -283,7 +289,7 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: yield from self.child_layer.handle_event( QuicGetConnectionCompleted( command=self._get_connection_commands.pop(), - connection=self.quic, + reply=self.quic, ) ) @@ -303,20 +309,8 @@ def process_events(self) -> layer.CommandGenerator[None]: self.retire_connection_id_callback(event.connection_id) elif isinstance(event, quic_events.ConnectionTerminated): - # report as TLS failure if the termination happened before the handshake - if not self.conn.tls_established: - self.conn.error = event.reason_phrase - tls_data = QuicTlsData( - conn=self.conn, context=self.context, settings=self.tls - ) - if self.conn is self.context.client: - yield layers.tls.TlsFailedClientHook(tls_data) - else: - yield layers.tls.TlsFailedServerHook(tls_data) - - # always close the connection yield from self.shutdown_connection( - event.reason_phrase, + reason=event.reason_phrase or str(event.error_code), level=( "info" if event.error_code is QuicErrorCode.NO_ERROR else "warn" ), @@ -368,6 +362,21 @@ def shutdown_connection( reason: str, level: Literal["error", "warn", "info", "alert", "debug"], ) -> layer.CommandGenerator[None]: + # ensure QUIC has been properly shut down + assert self.quic is None or self.quic._state is QuicConnectionState.TERMINATED + + # report as TLS failure if the termination happened before the handshake + if not self.conn.tls_established and self.tls is not None: + self.conn.error = reason + tls_data = QuicTlsData( + conn=self.conn, context=self.context, settings=self.tls + ) + if self.conn is self.context.client: + yield layers.tls.TlsFailedClientHook(tls_data) + else: + yield layers.tls.TlsFailedServerHook(tls_data) + + # log the reason, ensure the connection is closed and no longer handle events yield commands.Log( message=f"Connection {self.conn} closed: {reason}", level=level ) @@ -398,13 +407,22 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.process_events() return - # check if the connection was closed by peer + # handle connections closed by peer elif ( isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): - yield from self.shutdown_connection( - "Peer UDP connection timed out.", level="info" - ) + reason = "Peer UDP connection timed out." + if self.quic is not None: + # there is no point in calling quic.close, as it cannot send packets anymore + # so we simply set the state and simulate a ConnectionTerminated event + self.quic._set_state(QuicConnectionState.TERMINATED) + yield from self.event_to_child( + QuicConnectionEvent( + connection=self.conn, + event=quic_events.ConnectionTerminated(reason_phrase=reason), + ) + ) + yield from self.shutdown_connection(reason=reason, level="info") # intercept wakeup events for aioquic elif ( @@ -462,7 +480,7 @@ def start(self) -> layer.CommandGenerator[None]: err = yield commands.OpenConnection(self.conn) if err is not None: self.shutdown_connection( - f"Failed to connect: {err}", + reason=f"Failed to connect: {err}", level="warn", ) return @@ -525,7 +543,7 @@ def state_wait_for_upstream( elif isinstance(event, events.ConnectionClosed): if event.connection is self.conn: yield from self.shutdown_connection( - "Client UDP connection timeout out before upstream server handshake completed.", + reason="Client UDP connection timeout out before upstream server handshake completed.", level="info", ) elif event.connection is self.context.server: From 5608b0629a3a82493f8ecad7d41200f02115b863 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 20 Jun 2022 22:27:42 +0200 Subject: [PATCH 018/695] [quic] H3 stream ID translation on error handling --- mitmproxy/proxy/layers/http/_http3.py | 231 ++++++++++++++++++-------- 1 file changed, 164 insertions(+), 67 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index ec600296e5..32a75d0e91 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,8 +1,7 @@ from abc import abstractmethod import time -from typing import Optional, Union +from typing import Dict, Optional, Union -from aioquic.quic.connection import QuicConnection from aioquic.h3.connection import ( H3Connection, FrameUnexpected, @@ -10,6 +9,9 @@ HeadersState as H3HeadersState, ) from aioquic.h3 import events as h3_events +from aioquic.quic import events as quic_events +from aioquic.quic.connection import QuicConnection +from aioquic.quic.packet import QuicErrorCode from mitmproxy import http, version from mitmproxy.net.http import status_codes @@ -50,6 +52,9 @@ class Http3Connection(HttpConnection): quic: Optional[QuicConnection] = None h3_conn: Optional[H3Connection] = None + ReceiveData: type[Union[RequestData, ResponseData]] + ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] + ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -59,83 +64,149 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: self.quic = quic self.h3_conn = H3Connection(quic, enable_webtransport=False) - else: + if isinstance(event, events.ConnectionClosed): + self._handle_event = self.done + + # send mitmproxy HTTP events over the H3 connection + elif isinstance(event, HttpEvent): assert self.quic is not None assert self.h3_conn is not None + try: - if isinstance(event, HttpEvent): - try: + if isinstance(event, (RequestData, ResponseData)): + self.h3_conn.send_data( + stream_id=event.stream_id, data=event.data, end_stream=False + ) + elif isinstance(event, (RequestHeaders, ResponseHeaders)): + self.h3_conn.send_headers( + stream_id=event.stream_id, + headers=( + yield from ( + format_h2_request_headers(event) + if isinstance(event, RequestHeaders) + else format_h2_response_headers(event) + ) + ), + end_stream=event.end_stream, + ) + elif isinstance(event, (RequestTrailers, ResponseTrailers)): + self.h3_conn.send_headers( + stream_id=event.stream_id, + headers=[*event.trailers.fields], + end_stream=True, + ) + elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): + self.h3_conn.send_data( + stream_id=event.stream_id, data=b"", end_stream=True + ) + elif isinstance( + event, (RequestProtocolError, ResponseProtocolError) + ): + self.protocol_error(event) + else: + raise AssertionError(f"Unexpected event: {event}") - if isinstance(event, (RequestData, ResponseData)): - self.h3_conn.send_data( - stream_id=event.stream_id, data=event.data, end_stream=False - ) - elif isinstance(event, (RequestHeaders, ResponseHeaders)): - self.h3_conn.send_headers( + except FrameUnexpected: + # Http2Connection also ignores HttpEvents that violate the current stream state + return + + # transmit buffered data and re-arm timer + yield QuicTransmit(self.quic) + + # handle events from the underlying QUIC connection + elif isinstance(event, QuicConnectionEvent): + assert self.quic is not None + assert self.h3_conn is not None + + # report abrupt stream resets + if isinstance(event, quic_events.StreamReset): + if event.stream_id in self.h3_conn._stream: + try: + reason = H3ErrorCode(event.error_code).name + except ValueError: + try: + reason = QuicErrorCode(event.error_code).name + except ValueError: + reason = str(event.error_code) + code = ( + status_codes.CLIENT_CLOSED_REQUEST + if event.error_code == H3ErrorCode.H3_REQUEST_CANCELLED + else self.ReceiveProtocolError.code + ) + yield ReceiveHttp( + self.ReceiveProtocolError( stream_id=event.stream_id, - headers=( - yield from ( - format_h2_request_headers(event) - if isinstance(event, RequestHeaders) - else format_h2_response_headers(event) - ) - ), - end_stream=event.end_stream, + message=f"stream reset by client ({reason})", + code=code, + ) + ) + + # report a protocol error for all remaining open streams when a connection is terminated + elif isinstance(event, quic_events.ConnectionTerminated): + for stream in self.h3_conn._stream.values(): + if not stream.ended: + yield ReceiveHttp( + self.ReceiveProtocolError( + stream_id=stream.stream_id, + message=event.reason_phrase, + code=event.error_code, + ) ) - elif isinstance(event, (RequestTrailers, ResponseTrailers)): - trailers = [*event.trailers.fields] - self.h3_conn.send_headers( - stream_id=event.stream_id, headers=trailers, end_stream=True + + # forward QUIC events to the H3 connection + for h3_event in self.h3_conn.handle_event(event.event): + + # report received data + if isinstance(h3_event, h3_events.DataReceived): + yield ReceiveHttp( + self.ReceiveData( + stream_id=h3_event.stream_id, data=h3_event.data ) - elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): - self.h3_conn.send_data( - stream_id=event.stream_id, data=b"", end_stream=True + ) + if h3_event.stream_ended: + yield ReceiveHttp( + self.ReceiveEndOfMessage(stream_id=event.stream_id) ) - elif isinstance( - event, (RequestProtocolError, ResponseProtocolError) + + # report headers and trailers + elif isinstance(h3_event, h3_events.HeadersReceived): + if ( + self.h3_conn._stream[h3_event.stream_id].headers_recv_state + is H3HeadersState.AFTER_TRAILERS ): - self.protocol_error(event) + yield ReceiveHttp( + self.ReceiveTrailers( + stream_id=h3_event.stream_id, + trailers=http.Headers(h3_event.headers), + ) + ) else: - raise AssertionError(f"Unexpected event: {event}") - - except FrameUnexpected: - # Http2Connection also ignores events that violate the current stream state - return - - # transmit buffered data and re-arm timer - yield QuicTransmit(self.quic) - - elif isinstance(event, QuicConnectionEvent): - for h3_event in self.h3_conn.handle_event(event.event): - if isinstance(h3_event, h3_events.DataReceived): - pass - - # handle headers and trailers - elif isinstance(h3_event, h3_events.HeadersReceived): - if ( - self.h3_conn._stream[h3_event.stream_id].headers_recv_state - is H3HeadersState.AFTER_TRAILERS - ): - yield ReceiveHttp( - self.ReceiveTrailers( - stream_id=h3_event.stream_id, - trailers=http.Headers(h3_event.headers), - ) + try: + receive_event = self.headers_received(h3_event) + except ValueError as e: + # this will result in a ConnectionTerminated event + self.quic.close( + error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, + reason_phrase=f"Invalid HTTP/3 request headers: {e}", ) else: - try: - receive_event = self.headers_received(h3_event) - except ValueError as e: - # TODO - pass - else: - yield ReceiveHttp(receive_event) - - # we don't support push, web transport, etc. - else: - yield commands.Log( - f"Ignored unsupported H3 event: {h3_event!r}" - ) + yield ReceiveHttp(receive_event) + if h3_event.stream_ended: + yield ReceiveHttp( + self.ReceiveEndOfMessage(stream_id=event.stream_id) + ) + + # we don't support push, web transport, etc. + else: + yield commands.Log( + f"Ignored unsupported H3 event: {h3_event!r}" + ) + + else: + raise AssertionError(f"Unexpected event: {event!r}") + + def done(self, event: events.Event) -> layer.CommandGenerator[None]: + yield from () @abstractmethod def protocol_error( @@ -151,6 +222,9 @@ def headers_received( class Http3Server(Http3Connection): + ReceiveData = RequestData + ReceiveEndOfMessage = RequestEndOfMessage + ReceiveProtocolError = RequestProtocolError ReceiveTrailers = RequestTrailers def __init__(self, context: context.Context): @@ -213,10 +287,18 @@ def headers_received( class Http3Client(Http3Connection): + ReceiveData = ResponseData + ReceiveEndOfMessage = ResponseEndOfMessage + ReceiveProtocolError = ResponseProtocolError ReceiveTrailers = ResponseTrailers + our_stream_id: Dict[int, int] + their_stream_id: Dict[int, int] + def __init__(self, context: context.Context): super().__init__(context, context.server) + self.our_stream_id = {} + self.their_stream_id = {} def protocol_error( self, event: Union[RequestProtocolError, ResponseProtocolError] @@ -249,6 +331,21 @@ def headers_received( stream_id=event.stream_id, response=response, end_stream=event.stream_ended ) + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + # translate stream IDs just like HTTP/2 client + if isinstance(event, HttpEvent): + assert self.quic + ours = self.our_stream_id.get(event.stream_id, None) + if ours is None: + ours = self.quic.get_next_available_stream_id() + self.our_stream_id[event.stream_id] = ours + self.their_stream_id[ours] = event.stream_id + event.stream_id = ours + for cmd in super()._handle_event(event): + if isinstance(cmd, ReceiveHttp): + cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id] + yield cmd + __all__ = [ "Http3Client", From 6e66875e73ad1cb3dd2b93b69fd6079ea348da87 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 21 Jun 2022 00:10:39 +0200 Subject: [PATCH 019/695] [quic] first connectable version --- mitmproxy/proxy/layers/quic.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index b53f163d7a..78b7b18378 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,6 +1,6 @@ from abc import abstractmethod import asyncio -from dataclasses import dataclass +from dataclasses import dataclass, field from ssl import VerifyMode from typing import Callable, List, Literal, Optional, Tuple, Union @@ -31,7 +31,7 @@ class QuicTlsSettings: certificate: Optional[x509.Certificate] = None """The certificate to use for the connection.""" - certificate_chain: List[x509.Certificate] = [] + certificate_chain: List[x509.Certificate] = field(default_factory=list) """A list of additional certificates to send to the peer.""" certificate_private_key: Optional[ Union[dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey] @@ -115,7 +115,7 @@ def __init__(self, logger: tls.MasterSecretLogger) -> None: def write(self, s: str) -> int: if s[-1:] == "\n": s = s[:-1] - data = s.encode() + data = s.encode("ascii") self.logger(None, data) # type: ignore return len(data) + 1 @@ -138,7 +138,11 @@ def pull_client_hello_and_connection_id(data: bytes) -> Tuple[ClientHello, bytes # patch aioquic to intercept the client hello quic = QuicConnection( - configuration=QuicConfiguration(), + configuration=QuicConfiguration( + is_client=False, + certificate="", + private_key="", + ), original_destination_connection_id=header.destination_cid, ) _initialize = quic._initialize @@ -202,7 +206,7 @@ def build_configuration(self) -> QuicConfiguration: assert self.tls is not None return QuicConfiguration( - alpn_protocols=self.conn.alpn_offers, + alpn_protocols=[offer.decode("ascii") for offer in self.conn.alpn_offers], connection_id_length=self.context.options.quic_connection_id_length, is_client=self.conn is self.context.server, secrets_log_file=QuicSecretsLogger(tls.log_master_secret) # type: ignore @@ -329,7 +333,7 @@ def process_events(self) -> layer.CommandGenerator[None]: self.conn.certificate_list = [ certs.Cert.from_pyopenssl(x) for x in all_certs ] - self.conn.alpn = event.alpn_protocol.encode() + self.conn.alpn = event.alpn_protocol.encode("ascii") self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name self.conn.tls_version = "QUIC" From 97e482998b76e8a3f18821a409b6eaa5c52d3754 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 22 Jun 2022 00:45:51 +0200 Subject: [PATCH 020/695] [quic] implement relay layer --- mitmproxy/proxy/layers/http/_http3.py | 2 +- mitmproxy/proxy/layers/quic.py | 147 +++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 4 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 32a75d0e91..dfca3658ff 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -334,7 +334,7 @@ def headers_received( def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # translate stream IDs just like HTTP/2 client if isinstance(event, HttpEvent): - assert self.quic + assert self.quic is not None ours = self.our_stream_id.get(event.stream_id, None) if ours is None: ours = self.quic.get_next_available_stream_id() diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 78b7b18378..3ada9da3be 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -2,7 +2,7 @@ import asyncio from dataclasses import dataclass, field from ssl import VerifyMode -from typing import Callable, List, Literal, Optional, Tuple, Union +from typing import Callable, Dict, List, Literal, Optional, Tuple, Union from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events @@ -17,9 +17,10 @@ from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection +from mitmproxy import certs, connection, flow as mitm_flow, tcp from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, layers +from mitmproxy.proxy.layers import tcp as tcp_layer from mitmproxy.tls import ClientHello, ClientHelloData, TlsData @@ -179,6 +180,146 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") +class QuicRelayLayer(layer.Layer): + # for now we're (ab)using the TCPFlow until https://github.com/mitmproxy/mitmproxy/pull/5414 is resolved + datagram_flow: Optional[tcp.TCPFlow] = None + lookup_server: Dict[int, Tuple[int, tcp.TCPFlow]] + lookup_client: Dict[int, Tuple[int, tcp.TCPFlow]] + quic_server: Optional[QuicConnection] = None + quic_client: Optional[QuicConnection] = None + + def __init__(self, context: context.Context) -> None: + super().__init__(context) + self.lookup_server = {} + self.lookup_client = {} + + def end_flow(self, flow: tcp.TCPFlow, event: quic_events.ConnectionTerminated) -> layer.CommandGenerator[None]: + if event.error_code == QuicErrorCode.NO_ERROR: + yield tcp_layer.TcpEndHook(flow) + else: + flow.error = mitm_flow.Error(event.reason_phrase) + yield tcp_layer.TcpErrorHook(flow) + flow.live = False + + def get_quic( + self, conn: connection.Connection + ) -> layer.CommandGenerator[QuicConnection]: + quic = yield QuicGetConnection(conn) + assert isinstance(quic, QuicConnection) + return quic + + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.Start): + self.quic_server = yield from self.get_quic(self.context.server) + self.quic_client = yield from self.get_quic(self.context.client) + + elif isinstance(event, QuicConnectionEvent): + assert self.quic_server is not None + assert self.quic_client is not None + + quic_event = event.event + from_client = event.connection is self.context.client + lookup_in = self.lookup_client if from_client else self.lookup_server + lookup_out = self.lookup_server if from_client else self.lookup_client + # quic_in = self.quic_client if from_client else self.quic_server + quic_out = self.quic_server if from_client else self.quic_client + + # forward close and end all flows + if isinstance(quic_event, quic_events.ConnectionTerminated): + quic_out.close( + error_code=quic_event.error_code, + frame_type=quic_event.frame_type, + reason_phrase=quic_event.reason_phrase, + ) + while lookup_in: + stream_id_in = next(iter(lookup_in)) + stream_id_out, flow = lookup_in[stream_id_in] + yield from self.end_flow(flow=flow, event=quic_event) + del lookup_in[stream_id_in] + del lookup_out[stream_id_out] + + if self.datagram_flow is not None: + yield from self.end_flow(flow=flow, event=quic_event) + self.datagram_flow = None + + # forward datagrams (that are not stream-bound) + elif isinstance(quic_event, quic_events.DatagramFrameReceived): + if self.datagram_flow is None: + self.datagram_flow = tcp.TCPFlow( + client_conn=self.context.client, + server_conn=self.context.server, + live=True, + ) + yield tcp_layer.TcpStartHook(self.datagram_flow) + message = tcp.TCPMessage( + from_client=from_client, content=quic_event.data + ) + self.datagram_flow.messages.append(message) + yield tcp_layer.TcpMessageHook(self.datagram_flow) + quic_out.send_datagram_frame(data=message.content) + + # forward stream data + elif isinstance(quic_event, quic_events.StreamDataReceived): + # get or create the stream on the other side (and flow) + stream_id_in = quic_event.stream_id + if stream_id_in in lookup_in: + stream_id_out, flow = lookup_in[stream_id_in] + else: + stream_id_out = quic_out.get_next_available_stream_id() + flow = tcp.TCPFlow( + client_conn=self.context.client, + server_conn=self.context.server, + live=True, + ) + lookup_in[stream_id_in] = (stream_id_out, flow) + lookup_out[stream_id_out] = (stream_id_in, flow) + yield tcp_layer.TcpStartHook(flow) + + # forward the message allowing addons to change it + message = tcp.TCPMessage( + from_client=from_client, content=quic_event.data + ) + flow.messages.append(message) + yield tcp_layer.TcpMessageHook(flow) + quic_out.send_stream_data( + stream_id=stream_id_out, + data=message.content, + end_stream=quic_event.end_stream, + ) + + # end the flow and remove the lookup if the stream ended + if quic_event.end_stream: + yield tcp_layer.TcpEndHook(flow) + flow.live = False + del lookup_in[stream_id_in] + del lookup_out[stream_id_out] + + # forward resets to peer streams + elif isinstance(quic_event, quic_events.StreamReset): + stream_id_in = quic_event.stream_id + if stream_id_in in lookup_in: + stream_id_out, flow = lookup_in[stream_id_in] + quic_out.stop_stream( + stream_id=stream_id_out, error_code=quic_event.error_code + ) + + # try to get a name describing the reset reason + try: + err = QuicErrorCode(quic_event.error_code).name + except ValueError: + err = str(quic_event.error_code) + + # report the error to addons and delete the stream + flow.error = mitm_flow.Error(str(err)) + yield tcp_layer.TcpErrorHook(flow) + flow.live = False + del lookup_in[stream_id_in] + del lookup_out[stream_id_out] + + def done(self, _) -> layer.CommandGenerator[None]: + yield from () + + class _QuicLayer(layer.Layer): child_layer: layer.Layer conn: connection.Connection @@ -316,7 +457,7 @@ def process_events(self) -> layer.CommandGenerator[None]: yield from self.shutdown_connection( reason=event.reason_phrase or str(event.error_code), level=( - "info" if event.error_code is QuicErrorCode.NO_ERROR else "warn" + "info" if event.error_code == QuicErrorCode.NO_ERROR else "warn" ), ) From 0b5afe54c46eca45ef70bb1ac82828735fe8d7ef Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 22 Jun 2022 03:45:39 +0200 Subject: [PATCH 021/695] [quic] bugfixes and improvements - next_layer decisions - don't forward obsolete wakeups - remove excessive named arguments --- mitmproxy/addons/next_layer.py | 20 +- mitmproxy/proxy/layers/http/__init__.py | 19 +- mitmproxy/proxy/layers/http/_http3.py | 8 +- mitmproxy/proxy/layers/quic.py | 275 ++++++++++++------------ 4 files changed, 177 insertions(+), 145 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index e2e57e96b9..b9908b0a24 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -22,7 +22,7 @@ from mitmproxy.net.tls import is_tls_record_magic from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy import context, layer, layers -from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import modes, quic from mitmproxy.proxy.layers.tls import HTTP_ALPNS, parse_client_hello LayerCls = type[layer.Layer] @@ -117,6 +117,24 @@ def next_layer(self, nextlayer: layer.NextLayer): def _next_layer( self, context: context.Context, data_client: bytes, data_server: bytes ) -> Optional[layer.Layer]: + if isinstance(context.layers[0], quic.QuicLayer): + if context.client.alpn is None: + return None # should never happen, as ask is called after handshake + if context.client.alpn == b"h3" or context.client.alpn.startswith(b"h3-"): + if ctx.options.mode == "regular": + mode = HTTPMode.regular + elif ctx.options.mode == "transparent" or ctx.options.mode.startswith("reverse:"): + mode = HTTPMode.transparent + elif ctx.options.mode.startswith("upstream:"): + mode = HTTPMode.upstream + else: + return None + return layers.HttpLayer(context=context, mode=mode) + else: + if context.server.address is None: + return None + return quic.QuicRelayLayer(context) + if len(context.layers) == 0: return self.make_top_layer(context) diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 36e0969a9d..81ac1c9189 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -11,7 +11,7 @@ from mitmproxy.net.http import status_codes, url from mitmproxy.net.http.http1 import expected_http_body_size from mitmproxy.proxy import commands, events, layer, tunnel -from mitmproxy.proxy.layers import tcp, tls, websocket +from mitmproxy.proxy.layers import quic, tcp, tls, websocket from mitmproxy.proxy.layers.http import _upstream_proxy from mitmproxy.proxy.utils import expect from mitmproxy.utils import human @@ -62,6 +62,10 @@ def validate_request(mode: HTTPMode, request: http.Request) -> Optional[str]: return None +def is_h3_alpn(alpn: Optional[bytes]) -> bool: + return alpn == b"h3" or (alpn is not None and alpn.startswith(b"h3-")) + + @dataclass class GetHttpConnection(HttpCommand): """ @@ -822,7 +826,7 @@ def __init__(self, context: Context, mode: HTTPMode): self.command_sources = {} http_conn: HttpConnection - if self.context.client.alpn == b"h3": + if is_h3_alpn(self.context.client.alpn): http_conn = Http3Server(context.fork()) elif self.context.client.alpn == b"h2": http_conn = Http2Server(context.fork()) @@ -879,7 +883,7 @@ def _handle_event(self, event: events.Event): elif isinstance(event, events.DataReceived): # The peer has sent data. This can happen with HTTP/2 servers that already send a settings frame. child_layer: HttpConnection - if self.context.server.alpn == b"h3": + if is_h3_alpn(self.context.server.alpn): child_layer = Http3Client(self.context.fork()) elif self.context.server.alpn == b"h2": child_layer = Http2Client(self.context.fork()) @@ -997,7 +1001,7 @@ def get_connection( if not can_use_context_connection: - context.server = Server(event.address) + context.server = Server(event.address, transport_protocol=context.client.transport_protocol) if event.via: context.server.via = event.via @@ -1015,7 +1019,10 @@ def get_connection( context.server.sni = self.context.client.sni or event.address[0] else: context.server.sni = event.address[0] - stack /= tls.ServerTLSLayer(context) + if context.server.transport_protocol == "udp": + stack /= quic.ServerQuicLayer(context) + else: + stack /= tls.ServerTLSLayer(context) stack /= HttpClient(context) @@ -1065,7 +1072,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: err = yield commands.OpenConnection(self.context.server) if not err: - if self.context.server.alpn == b"h3": + if is_h3_alpn(self.context.server.alpn): self.child_layer = Http3Client(self.context) elif self.context.server.alpn == b"h2": self.child_layer = Http2Client(self.context) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index dfca3658ff..2c8e0b0b16 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -64,7 +64,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: self.quic = quic self.h3_conn = H3Connection(quic, enable_webtransport=False) - if isinstance(event, events.ConnectionClosed): + elif isinstance(event, events.ConnectionClosed): self._handle_event = self.done # send mitmproxy HTTP events over the H3 connection @@ -165,7 +165,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) if h3_event.stream_ended: yield ReceiveHttp( - self.ReceiveEndOfMessage(stream_id=event.stream_id) + self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) ) # report headers and trailers @@ -193,7 +193,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield ReceiveHttp(receive_event) if h3_event.stream_ended: yield ReceiveHttp( - self.ReceiveEndOfMessage(stream_id=event.stream_id) + self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) ) # we don't support push, web transport, etc. @@ -266,7 +266,7 @@ def headers_received( authority, path, headers, - ) = parse_h2_request_headers(event) + ) = parse_h2_request_headers(event.headers) request = http.Request( host=host, port=port, diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 3ada9da3be..28db9c1a40 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -2,7 +2,7 @@ import asyncio from dataclasses import dataclass, field from ssl import VerifyMode -from typing import Callable, Dict, List, Literal, Optional, Tuple, Union +from typing import Callable, Dict, List, Literal, Optional, Set, Tuple, Union from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events @@ -90,15 +90,17 @@ class QuicConnectionEvent(events.ConnectionEvent): event: quic_events.QuicEvent -@dataclass class QuicGetConnection(commands.ConnectionCommand): # -> QuicConnection blocking = True -@dataclass class QuicTransmit(commands.Command): connection: QuicConnection + def __init__(self, connection: QuicConnection) -> None: + super().__init__() + self.connection = connection + @dataclass(repr=False) class QuicGetConnectionCompleted(events.CommandCompleted): @@ -159,7 +161,7 @@ def server_handle_hello_replacement( for b in input_buf.pull_bytes(3): length = (length << 8) | b offset = input_buf.tell() - raise QuicClientHello(data=input_buf.data_slice(offset, offset + length)) + raise QuicClientHello(input_buf.data_slice(offset, offset + length)) def initialize_replacement(peer_cid: bytes) -> None: try: @@ -193,7 +195,9 @@ def __init__(self, context: context.Context) -> None: self.lookup_server = {} self.lookup_client = {} - def end_flow(self, flow: tcp.TCPFlow, event: quic_events.ConnectionTerminated) -> layer.CommandGenerator[None]: + def end_flow( + self, flow: tcp.TCPFlow, event: quic_events.ConnectionTerminated + ) -> layer.CommandGenerator[None]: if event.error_code == QuicErrorCode.NO_ERROR: yield tcp_layer.TcpEndHook(flow) else: @@ -227,36 +231,34 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # forward close and end all flows if isinstance(quic_event, quic_events.ConnectionTerminated): quic_out.close( - error_code=quic_event.error_code, - frame_type=quic_event.frame_type, - reason_phrase=quic_event.reason_phrase, + quic_event.error_code, + quic_event.frame_type, + quic_event.reason_phrase, ) while lookup_in: stream_id_in = next(iter(lookup_in)) stream_id_out, flow = lookup_in[stream_id_in] - yield from self.end_flow(flow=flow, event=quic_event) + yield from self.end_flow(flow, quic_event) del lookup_in[stream_id_in] del lookup_out[stream_id_out] if self.datagram_flow is not None: - yield from self.end_flow(flow=flow, event=quic_event) + yield from self.end_flow(flow, quic_event) self.datagram_flow = None # forward datagrams (that are not stream-bound) elif isinstance(quic_event, quic_events.DatagramFrameReceived): if self.datagram_flow is None: self.datagram_flow = tcp.TCPFlow( - client_conn=self.context.client, - server_conn=self.context.server, + self.context.client, + self.context.server, live=True, ) yield tcp_layer.TcpStartHook(self.datagram_flow) - message = tcp.TCPMessage( - from_client=from_client, content=quic_event.data - ) + message = tcp.TCPMessage(from_client, quic_event.data) self.datagram_flow.messages.append(message) yield tcp_layer.TcpMessageHook(self.datagram_flow) - quic_out.send_datagram_frame(data=message.content) + quic_out.send_datagram_frame(message.content) # forward stream data elif isinstance(quic_event, quic_events.StreamDataReceived): @@ -267,8 +269,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: stream_id_out = quic_out.get_next_available_stream_id() flow = tcp.TCPFlow( - client_conn=self.context.client, - server_conn=self.context.server, + self.context.client, + self.context.server, live=True, ) lookup_in[stream_id_in] = (stream_id_out, flow) @@ -276,15 +278,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield tcp_layer.TcpStartHook(flow) # forward the message allowing addons to change it - message = tcp.TCPMessage( - from_client=from_client, content=quic_event.data - ) + message = tcp.TCPMessage(from_client, quic_event.data) flow.messages.append(message) yield tcp_layer.TcpMessageHook(flow) quic_out.send_stream_data( - stream_id=stream_id_out, - data=message.content, - end_stream=quic_event.end_stream, + stream_id_out, + message.content, + quic_event.end_stream, ) # end the flow and remove the lookup if the stream ended @@ -299,9 +299,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: stream_id_in = quic_event.stream_id if stream_id_in in lookup_in: stream_id_out, flow = lookup_in[stream_id_in] - quic_out.stop_stream( - stream_id=stream_id_out, error_code=quic_event.error_code - ) + quic_out.stop_stream(stream_id_out, quic_event.error_code) # try to get a name describing the reset reason try: @@ -316,9 +314,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: del lookup_in[stream_id_in] del lookup_out[stream_id_out] - def done(self, _) -> layer.CommandGenerator[None]: - yield from () - class _QuicLayer(layer.Layer): child_layer: layer.Layer @@ -338,10 +333,11 @@ def __init__( self.child_layer = layer.NextLayer(context) self.conn = conn self._loop = asyncio.get_event_loop() - self._get_connection_commands: List[QuicGetConnection] = [] + self._get_connection_commands: List[QuicGetConnection] = list() self._request_wakeup_command_and_timer: Optional[ Tuple[commands.RequestWakeup, float] ] = None + self._obsolete_wakeup_commands: Set[commands.RequestWakeup] = set() def build_configuration(self) -> QuicConfiguration: assert self.tls is not None @@ -364,8 +360,13 @@ def build_configuration(self) -> QuicConfiguration: ) def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: + yield from self.handle_child_commands(self.child_layer.handle_event(event)) + + def handle_child_commands( + self, child_commands: layer.CommandGenerator[None] + ) -> layer.CommandGenerator[None]: # filter commands coming from the child layer - for command in self.child_layer.handle_event(event): + for command in child_commands: # answer or queue requests for the aioquic connection instance if ( @@ -375,11 +376,8 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: if self.quic is None: self._get_connection_commands.append(command) else: - yield from self.child_layer.handle_event( - QuicGetConnectionCompleted( - command=command, - reply=self.quic, - ) + yield from self.event_to_child( + QuicGetConnectionCompleted(command, self.quic) ) # transmit buffered data and re-arm timer @@ -393,7 +391,7 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: ): reason = "CloseConnection command received." if self.quic is None: - yield from self.shutdown_connection(reason=reason, level="info") + yield from self.shutdown_connection(reason, level="info") else: self.quic.close(reason_phrase=reason) yield from self.process_events() @@ -413,7 +411,7 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: yield QuicTlsStartServerHook(tls_data) if tls_data.settings is None: yield from self.shutdown_connection( - reason="No TLS settings were provided, failing connection.", + "No TLS settings were provided, failing connection.", level="error", ) return @@ -431,10 +429,9 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: # let the waiters know about the available connection while self._get_connection_commands: assert self.quic is not None - yield from self.child_layer.handle_event( + yield from self.event_to_child( QuicGetConnectionCompleted( - command=self._get_connection_commands.pop(), - reply=self.quic, + self._get_connection_commands.pop(), self.quic ) ) @@ -455,7 +452,7 @@ def process_events(self) -> layer.CommandGenerator[None]: elif isinstance(event, quic_events.ConnectionTerminated): yield from self.shutdown_connection( - reason=event.reason_phrase or str(event.error_code), + event.reason_phrase or str(event.error_code), level=( "info" if event.error_code == QuicErrorCode.NO_ERROR else "warn" ), @@ -479,9 +476,7 @@ def process_events(self) -> layer.CommandGenerator[None]: self.conn.tls_version = "QUIC" # report the success to addons - tls_data = QuicTlsData( - conn=self.conn, context=self.context, settings=self.tls - ) + tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) if self.conn is self.context.client: yield layers.tls.TlsEstablishedClientHook(tls_data) else: @@ -489,12 +484,10 @@ def process_events(self) -> layer.CommandGenerator[None]: # perform next layer decisions now if isinstance(self.child_layer, layer.NextLayer): - yield from self.child_layer._ask() + yield from self.handle_child_commands(self.child_layer._ask()) # forward the event as a QuicConnectionEvent to the child layer - yield from self.event_to_child( - QuicConnectionEvent(connection=self.conn, event=event) - ) + yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) # handle the next event event = self.quic.next_event() @@ -513,18 +506,14 @@ def shutdown_connection( # report as TLS failure if the termination happened before the handshake if not self.conn.tls_established and self.tls is not None: self.conn.error = reason - tls_data = QuicTlsData( - conn=self.conn, context=self.context, settings=self.tls - ) + tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) if self.conn is self.context.client: yield layers.tls.TlsFailedClientHook(tls_data) else: yield layers.tls.TlsFailedServerHook(tls_data) # log the reason, ensure the connection is closed and no longer handle events - yield commands.Log( - message=f"Connection {self.conn} closed: {reason}", level=level - ) + yield commands.Log(f"Connection {self.conn} closed: {reason}", level=level) if self.conn.connected: yield commands.CloseConnection(self.conn) self._handle_event = self.state_done @@ -538,7 +527,7 @@ def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: # start this layer and the child layer yield from self.start() - yield from self.child_layer.handle_event(event) + yield from self.event_to_child(event) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None @@ -547,7 +536,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.DataReceived) and event.connection is self.conn: assert event.remote_addr is not None self.quic.receive_datagram( - data=event.data, addr=event.remote_addr, now=self._loop.time() + event.data, event.remote_addr, now=self._loop.time() ) yield from self.process_events() return @@ -563,24 +552,32 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: self.quic._set_state(QuicConnectionState.TERMINATED) yield from self.event_to_child( QuicConnectionEvent( - connection=self.conn, - event=quic_events.ConnectionTerminated(reason_phrase=reason), + self.conn, + quic_events.ConnectionTerminated( + error_code=QuicErrorCode.APPLICATION_ERROR, + frame_type=None, + reason_phrase=reason, + ), ) ) - yield from self.shutdown_connection(reason=reason, level="info") + yield from self.shutdown_connection(reason, level="info") # intercept wakeup events for aioquic - elif ( - isinstance(event, events.Wakeup) - and self._request_wakeup_command_and_timer is not None - ): - command, timer = self._request_wakeup_command_and_timer - if event.command is command: - self._request_wakeup_command_and_timer = None - self.quic.handle_timer(now=max(timer, self._loop.time())) - yield from self.process_events() + elif isinstance(event, events.Wakeup): + # swallow obsolete wakeups + if event.command in self._obsolete_wakeup_commands: + self._obsolete_wakeup_commands.remove(event.command) return + # handle active wakeup + elif self._request_wakeup_command_and_timer is not None: + command, timer = self._request_wakeup_command_and_timer + if event.command is command: + self._request_wakeup_command_and_timer = None + self.quic.handle_timer(now=max(timer, self._loop.time())) + yield from self.process_events() + return + # forward other events to the child layer yield from self.event_to_child(event) @@ -593,17 +590,21 @@ def transmit(self) -> layer.CommandGenerator[None]: # send all queued datagrams for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): - yield commands.SendData(connection=self.conn, data=data, remote_addr=addr) + yield commands.SendData(self.conn, data, addr) # ensure the wakeup is set and still correct timer = self.quic.get_timer() if timer is None: - self._request_wakeup_command_and_timer = None + if self._request_wakeup_command_and_timer is not None: + command, _ = self._request_wakeup_command_and_timer + self._obsolete_wakeup_commands.add(command) + self._request_wakeup_command_and_timer = None else: if self._request_wakeup_command_and_timer is not None: - _, existing_timer = self._request_wakeup_command_and_timer + command, existing_timer = self._request_wakeup_command_and_timer if existing_timer == timer: return + self._obsolete_wakeup_commands.add(command) command = commands.RequestWakeup(timer - self._loop.time()) self._request_wakeup_command_and_timer = (command, timer) yield command @@ -620,20 +621,10 @@ def __init__(self, context: context.Context) -> None: super().__init__(context, context.server) def start(self) -> layer.CommandGenerator[None]: - # ensure there is an UDP connection - if not self.conn.connected: - err = yield commands.OpenConnection(self.conn) - if err is not None: - self.shutdown_connection( - reason=f"Failed to connect: {err}", - level="warn", - ) - return - # try to connect yield from self.initialize_connection() if self.quic is not None: - self.quic.connect(addr=self.conn.peername, now=self._loop.time()) + self.quic.connect(self.conn.peername, now=self._loop.time()) yield from self.process_events() @@ -658,11 +649,7 @@ def initialize_connection_and_flush_buffer(self) -> layer.CommandGenerator[None] yield from self.initialize_connection() if self.quic is not None: for data, addr, now in self.buffered_packets: - self.quic.receive_datagram( - data=data, - addr=addr, - now=now, - ) + self.quic.receive_datagram(data, addr, now) yield from self.process_events() def start(self) -> layer.CommandGenerator[None]: @@ -688,7 +675,7 @@ def state_wait_for_upstream( elif isinstance(event, events.ConnectionClosed): if event.connection is self.conn: yield from self.shutdown_connection( - reason="Client UDP connection timeout out before upstream server handshake completed.", + "Client UDP connection timeout out before upstream server handshake completed.", level="info", ) elif event.connection is self.context.server: @@ -728,60 +715,80 @@ def __init__( def build_client_layer( self, connection_id: bytes, wait_for_upstream: bool ) -> ClientQuicLayer: - layer = ClientQuicLayer( - context=self.context, wait_for_upstream=wait_for_upstream - ) + layer = ClientQuicLayer(self.context, wait_for_upstream) layer.original_destination_connection_id = connection_id layer.issue_connection_id_callback = self._issue_cid layer.retire_connection_id_callback = self._retire_cid return layer + def done(self, event: events.Event) -> layer.CommandGenerator[None]: + yield from () + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.Start): + pass + # only handle the first packet from the client - if ( - not isinstance(event, events.DataReceived) - or event.connection is not self.context.client + elif ( + isinstance(event, events.DataReceived) + and event.connection is self.context.client ): - return - - # extract the client hello - try: - client_hello, connection_id = pull_client_hello_and_connection_id( - event.data - ) - except ValueError as e: - yield commands.Log( - f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" - ) - yield commands.CloseConnection(self.context.client) - return - - # copy the information - self.context.client.sni = client_hello.sni - self.context.client.alpn_offers = client_hello.alpn_protocols + # extract the client hello + try: + client_hello, connection_id = pull_client_hello_and_connection_id( + event.data + ) + except ValueError as e: + yield commands.Log( + f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" + ) + yield commands.CloseConnection(self.context.client) + self._handle_event = self.done + else: - # check with addons what we shall do - next_layer: layer.Layer - hook_data = ClientHelloData(self.context, client_hello) - yield layers.tls.TlsClienthelloHook(hook_data) + # copy the information + self.context.client.sni = client_hello.sni + self.context.client.alpn_offers = client_hello.alpn_protocols + + # check with addons what we shall do + next_layer: layer.Layer + hook_data = ClientHelloData(self.context, client_hello) + yield layers.tls.TlsClienthelloHook(hook_data) + + # simply relay everything + if hook_data.ignore_connection: + next_layer = layers.TCPLayer(self.context, ignore=True) + + # contact the upstream server first + elif hook_data.establish_server_tls_first: + err = yield commands.OpenConnection(self.context.server) + if err is None: + next_layer = ServerQuicLayer(self.context) + next_layer.child_layer = self.build_client_layer( + connection_id, + wait_for_upstream=True, + ) + else: + yield commands.Log( + f"Failed to connect to upstream first (will continue with client anyway): {err}" + ) + next_layer = self.build_client_layer( + connection_id, + wait_for_upstream=False, + ) - # simply relay everything - if hook_data.ignore_connection: - next_layer = layers.TCPLayer(self.context, ignore=True) + # perform the client handshake immediately + else: + next_layer = self.build_client_layer( + connection_id, + wait_for_upstream=False, + ) - # contact the upstream server first - elif hook_data.establish_server_tls_first: - next_layer = ServerQuicLayer(self.context) - next_layer.child_layer = self.build_client_layer( - connection_id, wait_for_upstream=True - ) + # replace this layer and start the next one + self.handle_event = next_layer.handle_event + self._handle_event = next_layer._handle_event + yield from next_layer.handle_event(events.Start()) + yield from next_layer.handle_event(event) - # perform the client handshake immediately else: - next_layer = self.build_client_layer(connection_id, wait_for_upstream=False) - - # replace this layer and start the next one - self.handle_event = next_layer.handle_event - self._handle_event = next_layer._handle_event - yield from next_layer.handle_event(events.Start()) - yield from next_layer.handle_event(event) + raise AssertionError(f"Unexpected event: {event}") From 2426d3d03847e0273707436268d79c24616b3e74 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 22 Jun 2022 06:16:10 +0200 Subject: [PATCH 022/695] [quic] bugfixes and simplified connection opening --- mitmproxy/proxy/layers/http/__init__.py | 10 +- mitmproxy/proxy/layers/quic.py | 231 +++++++++++------------- 2 files changed, 108 insertions(+), 133 deletions(-) diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 81ac1c9189..6b887269f1 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -1001,7 +1001,10 @@ def get_connection( if not can_use_context_connection: - context.server = Server(event.address, transport_protocol=context.client.transport_protocol) + context.server = Server(event.address) + if isinstance(context.layers[0], quic.QuicLayer): + context.server.transport_protocol = "udp" + stack /= quic.ServerQuicLayer(context) if event.via: context.server.via = event.via @@ -1019,10 +1022,7 @@ def get_connection( context.server.sni = self.context.client.sni or event.address[0] else: context.server.sni = event.address[0] - if context.server.transport_protocol == "udp": - stack /= quic.ServerQuicLayer(context) - else: - stack /= tls.ServerTLSLayer(context) + stack /= tls.ServerTLSLayer(context) stack /= HttpClient(context) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 28db9c1a40..7922d8ca6c 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,4 +1,3 @@ -from abc import abstractmethod import asyncio from dataclasses import dataclass, field from ssl import VerifyMode @@ -21,6 +20,7 @@ from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, layers from mitmproxy.proxy.layers import tcp as tcp_layer +from mitmproxy.proxy.utils import expect from mitmproxy.tls import ClientHello, ClientHelloData, TlsData @@ -90,10 +90,16 @@ class QuicConnectionEvent(events.ConnectionEvent): event: quic_events.QuicEvent -class QuicGetConnection(commands.ConnectionCommand): # -> QuicConnection +class QuicGetConnection(commands.ConnectionCommand): # -> Optional[QuicConnection] blocking = True +@dataclass(repr=False) +class QuicGetConnectionCompleted(events.CommandCompleted): + command: QuicGetConnection + reply: Optional[QuicConnection] + + class QuicTransmit(commands.Command): connection: QuicConnection @@ -102,12 +108,6 @@ def __init__(self, connection: QuicConnection) -> None: self.connection = connection -@dataclass(repr=False) -class QuicGetConnectionCompleted(events.CommandCompleted): - command: QuicGetConnection - reply: QuicConnection - - class QuicSecretsLogger: logger: tls.MasterSecretLogger @@ -333,7 +333,7 @@ def __init__( self.child_layer = layer.NextLayer(context) self.conn = conn self._loop = asyncio.get_event_loop() - self._get_connection_commands: List[QuicGetConnection] = list() + self._pending_open_command: Optional[commands.OpenConnection] = None self._request_wakeup_command_and_timer: Optional[ Tuple[commands.RequestWakeup, float] ] = None @@ -368,22 +368,35 @@ def handle_child_commands( # filter commands coming from the child layer for command in child_commands: - # answer or queue requests for the aioquic connection instance + # answer with the aioquic connection instance if ( isinstance(command, QuicGetConnection) and command.connection is self.conn ): - if self.quic is None: - self._get_connection_commands.append(command) - else: - yield from self.event_to_child( - QuicGetConnectionCompleted(command, self.quic) - ) + yield from self.event_to_child( + QuicGetConnectionCompleted(command, self.quic) + ) # transmit buffered data and re-arm timer elif isinstance(command, QuicTransmit) and command.connection is self.quic: yield from self.transmit() + # open the QUIC connection + elif ( + isinstance(command, commands.OpenConnection) + and command.connection is self.conn + ): + assert self._pending_open_command is None + self._pending_open_command = command + err = yield commands.OpenConnection(self.conn) + if err is None: + yield from self.initialize_connection() + if self.quic is not None: + self.quic.connect(self.conn.peername, now=self._loop.time()) + yield from self.process_events() + else: + yield from self.shutdown_connection(err, level="warn") + # properly close QUIC connections elif ( isinstance(command, commands.CloseConnection) @@ -426,15 +439,6 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: self.issue_connection_id_callback(self.quic.host_cid) self._handle_event = self.state_ready - # let the waiters know about the available connection - while self._get_connection_commands: - assert self.quic is not None - yield from self.event_to_child( - QuicGetConnectionCompleted( - self._get_connection_commands.pop(), self.quic - ) - ) - def process_events(self) -> layer.CommandGenerator[None]: assert self.quic is not None assert self.tls is not None @@ -482,6 +486,15 @@ def process_events(self) -> layer.CommandGenerator[None]: else: yield layers.tls.TlsEstablishedServerHook(tls_data) + # let the child layer know + if self._pending_open_command is not None: + yield from self.event_to_child( + events.OpenConnectionCompleted( + self._pending_open_command, reply=None + ) + ) + self._pending_open_command = None + # perform next layer decisions now if isinstance(self.child_layer, layer.NextLayer): yield from self.handle_child_commands(self.child_layer._ask()) @@ -512,22 +525,28 @@ def shutdown_connection( else: yield layers.tls.TlsFailedServerHook(tls_data) - # log the reason, ensure the connection is closed and no longer handle events + # log the reason and ensure the connection gets closed yield commands.Log(f"Connection {self.conn} closed: {reason}", level=level) if self.conn.connected: yield commands.CloseConnection(self.conn) - self._handle_event = self.state_done - - @abstractmethod - def start(self) -> layer.CommandGenerator[None]: - yield from () # pragma: no cover - def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, events.Start) + # let the child layer know and stop handling events + if self._pending_open_command is not None: + yield from self.event_to_child( + events.OpenConnectionCompleted(self._pending_open_command, reason) + ) + self._pending_open_command = None + self._handle_event = self.state_done - # start this layer and the child layer - yield from self.start() - yield from self.event_to_child(event) + def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: + # when done, just forward the event to the child layer (except for obsolete wakeups) + if ( + isinstance(event, events.Wakeup) + and event.command in self._obsolete_wakeup_commands + ): + self._obsolete_wakeup_commands.remove(event.command) + else: + yield from self.child_layer.handle_event(event) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None @@ -570,7 +589,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: return # handle active wakeup - elif self._request_wakeup_command_and_timer is not None: + if self._request_wakeup_command_and_timer is not None: command, timer = self._request_wakeup_command_and_timer if event.command is command: self._request_wakeup_command_and_timer = None @@ -581,30 +600,27 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # forward other events to the child layer yield from self.event_to_child(event) - def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: - # when done, just forward the event - yield from self.child_layer.handle_event(event) + def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: + # wait for the state to change and inspect the child layer's commands + yield from self.event_to_child(event) def transmit(self) -> layer.CommandGenerator[None]: - assert self.quic + assert self.quic is not None # send all queued datagrams for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): yield commands.SendData(self.conn, data, addr) - # ensure the wakeup is set and still correct + # mark an existing wakeup command as obsolete if it now longer matches the time timer = self.quic.get_timer() - if timer is None: - if self._request_wakeup_command_and_timer is not None: - command, _ = self._request_wakeup_command_and_timer + if self._request_wakeup_command_and_timer is not None: + command, existing_timer = self._request_wakeup_command_and_timer + if existing_timer != timer: self._obsolete_wakeup_commands.add(command) self._request_wakeup_command_and_timer = None - else: - if self._request_wakeup_command_and_timer is not None: - command, existing_timer = self._request_wakeup_command_and_timer - if existing_timer == timer: - return - self._obsolete_wakeup_commands.add(command) + + # request a new wakeup if necessary + if timer is not None and self._request_wakeup_command_and_timer is None: command = commands.RequestWakeup(timer - self._loop.time()) self._request_wakeup_command_and_timer = (command, timer) yield command @@ -620,20 +636,13 @@ class ServerQuicLayer(_QuicLayer): def __init__(self, context: context.Context) -> None: super().__init__(context, context.server) - def start(self) -> layer.CommandGenerator[None]: - # try to connect - yield from self.initialize_connection() - if self.quic is not None: - self.quic.connect(self.conn.peername, now=self._loop.time()) - yield from self.process_events() - class ClientQuicLayer(_QuicLayer): """ This layer establishes QUIC on a single client connection. """ - buffered_packets: Optional[List[Tuple[bytes, connection.Address, float]]] + wait_for_upstream: bool def __init__( self, @@ -641,60 +650,26 @@ def __init__( wait_for_upstream: bool, ) -> None: super().__init__(context, context.client) - self.buffered_packets = [] if wait_for_upstream else None - - def initialize_connection_and_flush_buffer(self) -> layer.CommandGenerator[None]: - assert self.buffered_packets is not None - - yield from self.initialize_connection() - if self.quic is not None: - for data, addr, now in self.buffered_packets: - self.quic.receive_datagram(data, addr, now) - yield from self.process_events() - - def start(self) -> layer.CommandGenerator[None]: - if self.buffered_packets is None: - yield from self.initialize_connection() - else: - self._handle_event = self.state_wait_for_upstream + self.wait_for_upstream = wait_for_upstream + @expect(events.Start) def state_wait_for_upstream( self, event: events.Event ) -> layer.CommandGenerator[None]: - assert self.buffered_packets is not None + self._handle_event = self.state_start - # buffer incoming packets until the upstream handshake completed - if isinstance(event, events.DataReceived) and event.connection is self.conn: - assert event.remote_addr is not None - self.buffered_packets.append( - (event.data, event.remote_addr, self._loop.time()) - ) - return - - # watch for closed connections on both legs - elif isinstance(event, events.ConnectionClosed): - if event.connection is self.conn: - yield from self.shutdown_connection( - "Client UDP connection timeout out before upstream server handshake completed.", - level="info", - ) - elif event.connection is self.context.server: + # open the upstream connection if possible, but always initialize the client connection + if self.wait_for_upstream: + err = yield commands.OpenConnection(self.context.server) + if err is not None: yield commands.Log( - f"Unable to establish QUIC connection with server ({self.context.server.error or 'Connection closed.'}). " + f"Unable to establish QUIC connection with server ({err}). " f"Trying to establish QUIC with client anyway." ) - yield from self.initialize_connection_and_flush_buffer() + yield from self.initialize_connection() + yield from self.state_start(event) - # continue if upstream completed the handshake - elif ( - isinstance(event, QuicConnectionEvent) - and event.connection is self.context.server - and isinstance(event.event, quic_events.HandshakeCompleted) - ): - yield from self.initialize_connection_and_flush_buffer() - - # forward other events to the child layer - yield from self.event_to_child(event) + _handle_event = state_wait_for_upstream class QuicLayer(layer.Layer): @@ -721,18 +696,22 @@ def build_client_layer( layer.retire_connection_id_callback = self._retire_cid return layer - def done(self, event: events.Event) -> layer.CommandGenerator[None]: + @expect(events.DataReceived, events.ConnectionClosed) + def state_done(self, _) -> layer.CommandGenerator[None]: yield from () - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.Start): - pass + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + self._handle_event = self.state_wait_for_hello + yield from () + + @expect(events.DataReceived, events.ConnectionClosed) + def state_wait_for_hello(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, events.ConnectionEvent) + assert event.connection is self.context.client # only handle the first packet from the client - elif ( - isinstance(event, events.DataReceived) - and event.connection is self.context.client - ): + if isinstance(event, events.DataReceived): # extract the client hello try: client_hello, connection_id = pull_client_hello_and_connection_id( @@ -743,7 +722,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" ) yield commands.CloseConnection(self.context.client) - self._handle_event = self.done + self._handle_event = self.state_done else: # copy the information @@ -761,21 +740,11 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # contact the upstream server first elif hook_data.establish_server_tls_first: - err = yield commands.OpenConnection(self.context.server) - if err is None: - next_layer = ServerQuicLayer(self.context) - next_layer.child_layer = self.build_client_layer( - connection_id, - wait_for_upstream=True, - ) - else: - yield commands.Log( - f"Failed to connect to upstream first (will continue with client anyway): {err}" - ) - next_layer = self.build_client_layer( - connection_id, - wait_for_upstream=False, - ) + next_layer = ServerQuicLayer(self.context) + next_layer.child_layer = self.build_client_layer( + connection_id, + wait_for_upstream=True, + ) # perform the client handshake immediately else: @@ -790,5 +759,11 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield from next_layer.handle_event(events.Start()) yield from next_layer.handle_event(event) + # stop if the connection was closed (usually we will always get one packet) + elif isinstance(event, events.ConnectionClosed): + self._handle_event = self.state_done + else: raise AssertionError(f"Unexpected event: {event}") + + _handle_event = state_start From 1426dec45e360f790367a9ce851a4b75779cdee6 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 22 Jun 2022 17:34:17 +0200 Subject: [PATCH 023/695] [quic] use state machine expect --- mitmproxy/proxy/layers/quic.py | 194 +++++++++++++++++++++------------ 1 file changed, 127 insertions(+), 67 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 7922d8ca6c..daae51e05c 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -386,35 +386,47 @@ def handle_child_commands( isinstance(command, commands.OpenConnection) and command.connection is self.conn ): + # ensure only one open command at a time for uninitialized connections + assert self.quic is None assert self._pending_open_command is None self._pending_open_command = command + + # try to open the underlying UDP connection err = yield commands.OpenConnection(self.conn) if err is None: - yield from self.initialize_connection() - if self.quic is not None: + # open succeeded, now try to initialize QUIC and connect + if (yield from self.initialize_connection()): + assert self.quic is not None self.quic.connect(self.conn.peername, now=self._loop.time()) yield from self.process_events() - else: - yield from self.shutdown_connection(err, level="warn") + else: + err = "initialize QUIC failed" + yield commands.CloseConnection(self.conn) + if err is not None: + # notify the child immediately + self._pending_open_command = None + yield from self.event_to_child( + events.OpenConnectionCompleted(command, err) + ) # properly close QUIC connections elif ( isinstance(command, commands.CloseConnection) and command.connection is self.conn ): - reason = "CloseConnection command received." if self.quic is None: - yield from self.shutdown_connection(reason, level="info") + yield command else: - self.quic.close(reason_phrase=reason) + self.quic.close(reason_phrase="CloseConnection command received.") yield from self.process_events() # return other commands else: yield command - def initialize_connection(self) -> layer.CommandGenerator[None]: + def initialize_connection(self) -> layer.CommandGenerator[bool]: assert self.quic is None + assert self.tls is None # query addons to provide the necessary TLS settings tls_data = QuicTlsData(self.conn, self.context) @@ -423,21 +435,18 @@ def initialize_connection(self) -> layer.CommandGenerator[None]: else: yield QuicTlsStartServerHook(tls_data) if tls_data.settings is None: - yield from self.shutdown_connection( - "No TLS settings were provided, failing connection.", - level="error", - ) - return - self.tls = tls_data.settings + return False # create the aioquic connection + self.tls = tls_data.settings self.quic = QuicConnection( configuration=self.build_configuration(), original_destination_connection_id=self.original_destination_connection_id, ) if self.issue_connection_id_callback is not None: self.issue_connection_id_callback(self.quic.host_cid) - self._handle_event = self.state_ready + self._handle_event = self.state_connected + return True def process_events(self) -> layer.CommandGenerator[None]: assert self.quic is not None @@ -455,12 +464,24 @@ def process_events(self) -> layer.CommandGenerator[None]: self.retire_connection_id_callback(event.connection_id) elif isinstance(event, quic_events.ConnectionTerminated): + # only forward the event if the connection has been properly initialized + if self.conn.tls_established: + yield from self.event_to_child( + QuicConnectionEvent(self.conn, event) + ) + + # shutdown and close the connection yield from self.shutdown_connection( event.reason_phrase or str(event.error_code), level=( "info" if event.error_code == QuicErrorCode.NO_ERROR else "warn" ), ) + yield commands.CloseConnection(self.conn) + + elif isinstance(event, quic_events.ProtocolNegotiated): + # too early, we act on HandshakeCompleted + pass elif isinstance(event, quic_events.HandshakeCompleted): # concatenate all peer certificates @@ -488,19 +509,19 @@ def process_events(self) -> layer.CommandGenerator[None]: # let the child layer know if self._pending_open_command is not None: + command = self._pending_open_command + self._pending_open_command = None yield from self.event_to_child( - events.OpenConnectionCompleted( - self._pending_open_command, reply=None - ) + events.OpenConnectionCompleted(command, reply=None) ) - self._pending_open_command = None # perform next layer decisions now if isinstance(self.child_layer, layer.NextLayer): yield from self.handle_child_commands(self.child_layer._ask()) - # forward the event as a QuicConnectionEvent to the child layer - yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) + else: + # forward the event as a QuicConnectionEvent to the child layer + yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) # handle the next event event = self.quic.next_event() @@ -514,10 +535,18 @@ def shutdown_connection( level: Literal["error", "warn", "info", "alert", "debug"], ) -> layer.CommandGenerator[None]: # ensure QUIC has been properly shut down - assert self.quic is None or self.quic._state is QuicConnectionState.TERMINATED + assert self.quic is not None + assert self.tls is not None + assert self.quic._state is QuicConnectionState.TERMINATED + + # obsolete any current timer + if self._request_wakeup_command_and_timer is not None: + command, _ = self._request_wakeup_command_and_timer + self._obsolete_wakeup_commands.add(command) + self._request_wakeup_command_and_timer = None # report as TLS failure if the termination happened before the handshake - if not self.conn.tls_established and self.tls is not None: + if not self.conn.tls_established: self.conn.error = reason tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) if self.conn is self.context.client: @@ -525,30 +554,27 @@ def shutdown_connection( else: yield layers.tls.TlsFailedServerHook(tls_data) - # log the reason and ensure the connection gets closed - yield commands.Log(f"Connection {self.conn} closed: {reason}", level=level) - if self.conn.connected: - yield commands.CloseConnection(self.conn) + # make a log entry directly + yield commands.Log( + f"QUIC connection {self.conn} shutdown: {reason}", level=level + ) # let the child layer know and stop handling events + # we also don't handle any commands from the child at this point anymore if self._pending_open_command is not None: - yield from self.event_to_child( + yield from self.child_layer.handle_event( events.OpenConnectionCompleted(self._pending_open_command, reason) ) self._pending_open_command = None self._handle_event = self.state_done - def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: - # when done, just forward the event to the child layer (except for obsolete wakeups) - if ( - isinstance(event, events.Wakeup) - and event.command in self._obsolete_wakeup_commands - ): - self._obsolete_wakeup_commands.remove(event.command) - else: - yield from self.child_layer.handle_event(event) + def start(self) -> layer.CommandGenerator[None]: + yield from self.event_to_child(events.Start()) - def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: + @expect( + events.ConnectionClosed, events.DataReceived, events.Wakeup, QuicConnectionEvent + ) + def state_connected(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None # forward incoming data only to aioquic @@ -558,17 +584,16 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: event.data, event.remote_addr, now=self._loop.time() ) yield from self.process_events() - return # handle connections closed by peer elif ( isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): reason = "Peer UDP connection timed out." - if self.quic is not None: - # there is no point in calling quic.close, as it cannot send packets anymore - # so we simply set the state and simulate a ConnectionTerminated event - self.quic._set_state(QuicConnectionState.TERMINATED) + # there is no point in calling quic.close, as it cannot send packets anymore + # so we simply set the state and simulate a ConnectionTerminated event + self.quic._set_state(QuicConnectionState.TERMINATED) + if self.conn.tls_established: yield from self.event_to_child( QuicConnectionEvent( self.conn, @@ -579,6 +604,10 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: ), ) ) + + # forward the event only if no open command is pending and shutdown + if self._pending_open_command is None: + yield from self.event_to_child(event) yield from self.shutdown_connection(reason, level="info") # intercept wakeup events for aioquic @@ -586,23 +615,52 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # swallow obsolete wakeups if event.command in self._obsolete_wakeup_commands: self._obsolete_wakeup_commands.remove(event.command) - return - - # handle active wakeup - if self._request_wakeup_command_and_timer is not None: - command, timer = self._request_wakeup_command_and_timer - if event.command is command: - self._request_wakeup_command_and_timer = None - self.quic.handle_timer(now=max(timer, self._loop.time())) - yield from self.process_events() - return + else: + # handle active wakeup and forward others to child layer + if self._request_wakeup_command_and_timer is not None: + command, timer = self._request_wakeup_command_and_timer + if event.command is command: + self._request_wakeup_command_and_timer = None + self.quic.handle_timer(now=max(timer, self._loop.time())) + yield from self.process_events() + else: + yield from self.event_to_child(event) + else: + yield from self.event_to_child(event) + + else: + # forward other events to the child layer + yield from self.event_to_child(event) - # forward other events to the child layer - yield from self.event_to_child(event) + @expect( + events.ConnectionClosed, events.DataReceived, events.Wakeup, QuicConnectionEvent + ) + def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: + # filter out obsolete wakeups + if ( + isinstance(event, events.Wakeup) + and event.command in self._obsolete_wakeup_commands + ): + self._obsolete_wakeup_commands.remove(event.command) - def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: - # wait for the state to change and inspect the child layer's commands - yield from self.event_to_child(event) + # ignore any further received data + elif isinstance(event, events.DataReceived) and event.connection is self.conn: + pass + + # forward all other events + else: + yield from self.child_layer.handle_event(event) + + @expect( + events.ConnectionClosed, events.DataReceived, events.Wakeup, QuicConnectionEvent + ) + def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: + yield from () + + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + self._handle_event = self.state_ready + yield from self.start() def transmit(self) -> layer.CommandGenerator[None]: assert self.quic is not None @@ -652,13 +710,10 @@ def __init__( super().__init__(context, context.client) self.wait_for_upstream = wait_for_upstream - @expect(events.Start) - def state_wait_for_upstream( - self, event: events.Event - ) -> layer.CommandGenerator[None]: - self._handle_event = self.state_start + def start(self) -> layer.CommandGenerator[None]: + yield from super().start() - # open the upstream connection if possible, but always initialize the client connection + # try to open the upstream connection if self.wait_for_upstream: err = yield commands.OpenConnection(self.context.server) if err is not None: @@ -666,10 +721,15 @@ def state_wait_for_upstream( f"Unable to establish QUIC connection with server ({err}). " f"Trying to establish QUIC with client anyway." ) - yield from self.initialize_connection() - yield from self.state_start(event) - _handle_event = state_wait_for_upstream + # is still connected then initialize, close on failure + if not self.conn.connected and not (yield from self.initialize_connection()): + yield commands.CloseConnection(self.conn) + self._handle_event = self.state_failed + + @expect(events.ConnectionClosed, events.DataReceived, QuicConnectionEvent) + def state_failed(self, _) -> layer.CommandGenerator[None]: + yield from () class QuicLayer(layer.Layer): From b9afc502a699bfc66e14fe439dd1902727ebf36c Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 23 Jun 2022 01:18:06 +0200 Subject: [PATCH 024/695] [quic] rework states and add more comments --- mitmproxy/proxy/layers/quic.py | 349 +++++++++++++++++++-------------- 1 file changed, 204 insertions(+), 145 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index daae51e05c..0a6225ed13 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -359,6 +359,117 @@ def build_configuration(self) -> QuicConfiguration: verify_mode=self.tls.verify_mode, ) + def create_quic(self) -> layer.CommandGenerator[bool]: + # must only be called if QUIC is uninitialized + assert self.quic is None + assert self.tls is None + + # in case the connection is being reused, clear all handshake data + self.conn.timestamp_tls_setup = None + self.conn.certificate_list = () + self.conn.alpn = None + self.conn.cipher = None + self.conn.tls_version = None + + # query addons to provide the necessary TLS settings + tls_data = QuicTlsData(self.conn, self.context) + if self.conn is self.context.client: + yield QuicTlsStartClientHook(tls_data) + else: + yield QuicTlsStartServerHook(tls_data) + if tls_data.settings is None: + yield commands.Log( + f"{self.conn}: No QUIC TLS settings provided by addon(s).", + level="error", + ) + return False + + # create the aioquic connection + self.tls = tls_data.settings + self.quic = QuicConnection( + configuration=self.build_configuration(), + original_destination_connection_id=self.original_destination_connection_id, + ) + self._handle_event = self.state_has_quic + + # issue the host connection ID right away + if self.issue_connection_id_callback is not None: + self.issue_connection_id_callback(self.quic.host_cid) + + # record an entry in the log + yield commands.Log(f"{self.conn}: QUIC connection created.", level="info") + return True + + def destroy_quic( + self, + reason: str, + level: Literal["error", "warn", "info", "alert", "debug"], + ) -> layer.CommandGenerator[None]: + # ensure QUIC has been properly shut down + assert self.quic is not None + assert self.tls is not None + assert self.quic._state is QuicConnectionState.TERMINATED + + # report as TLS failure if the termination happened before the handshake + if not self.conn.tls_established: + self.conn.error = reason + tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) + if self.conn is self.context.client: + yield layers.tls.TlsFailedClientHook(tls_data) + else: + yield layers.tls.TlsFailedServerHook(tls_data) + + # clear the quic fields + self.quic = None + self.tls = None + self._handle_event = self.state_no_quic + + # obsolete any current timer + if self._request_wakeup_command_and_timer is not None: + command, _ = self._request_wakeup_command_and_timer + self._obsolete_wakeup_commands.add(command) + self._request_wakeup_command_and_timer = None + + # record an entry in the log + yield commands.Log( + f"{self.conn}: QUIC connection destroyed: {reason}", level=level + ) + + def establish_quic( + self, event: quic_events.HandshakeCompleted + ) -> layer.CommandGenerator[None]: + # must only be called if QUIC is initialized + assert self.quic is not None + assert self.tls is not None + + # concatenate all peer certificates + all_certs = [] + if self.quic.tls._peer_certificate is not None: + all_certs.append(self.quic.tls._peer_certificate) + if self.quic.tls._peer_certificate_chain is not None: + all_certs.extend(self.quic.tls._peer_certificate_chain) + + # set the connection's TLS properties + self.conn.timestamp_tls_setup = self._loop.time() + self.conn.certificate_list = [certs.Cert.from_pyopenssl(x) for x in all_certs] + self.conn.alpn = event.alpn_protocol.encode("ascii") + self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name + self.conn.tls_version = "QUIC" + + # report the success to addons + tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) + if self.conn is self.context.client: + yield layers.tls.TlsEstablishedClientHook(tls_data) + else: + yield layers.tls.TlsEstablishedServerHook(tls_data) + + # record an entry in the log + yield commands.Log( + f"{self.conn}: QUIC connection established. " + f"(early_data={event.early_data_accepted}, resumed={event.session_resumed})", + level="info", + ) + def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.handle_child_commands(self.child_layer.handle_event(event)) @@ -386,34 +497,17 @@ def handle_child_commands( isinstance(command, commands.OpenConnection) and command.connection is self.conn ): - # ensure only one open command at a time for uninitialized connections - assert self.quic is None - assert self._pending_open_command is None - self._pending_open_command = command - - # try to open the underlying UDP connection - err = yield commands.OpenConnection(self.conn) - if err is None: - # open succeeded, now try to initialize QUIC and connect - if (yield from self.initialize_connection()): - assert self.quic is not None - self.quic.connect(self.conn.peername, now=self._loop.time()) - yield from self.process_events() - else: - err = "initialize QUIC failed" - yield commands.CloseConnection(self.conn) - if err is not None: - # notify the child immediately - self._pending_open_command = None - yield from self.event_to_child( - events.OpenConnectionCompleted(command, err) - ) + yield from self.open_connection_begin(command) # properly close QUIC connections elif ( isinstance(command, commands.CloseConnection) and command.connection is self.conn ): + # CloseConnection during pending OpenConnection is not allowed + assert self._pending_open_command is None + + # without QUIC simply close the connection, otherwise close QUIC first if self.quic is None: yield command else: @@ -424,28 +518,38 @@ def handle_child_commands( else: yield command - def initialize_connection(self) -> layer.CommandGenerator[bool]: + def open_connection_begin( + self, command: commands.OpenConnection + ) -> layer.CommandGenerator[None]: + # ensure only one OpenConnection at a time is called for uninitialized connections assert self.quic is None - assert self.tls is None - - # query addons to provide the necessary TLS settings - tls_data = QuicTlsData(self.conn, self.context) - if self.conn is self.context.client: - yield QuicTlsStartClientHook(tls_data) + assert self._pending_open_command is None + self._pending_open_command = command + + # try to open the underlying UDP connection + err = yield commands.OpenConnection(self.conn) + if not err: + # initialize QUIC and connect (notify the child layer after handshake) + if (yield from self.create_quic()): + assert self.quic is not None + self.quic.connect(self.conn.peername, now=self._loop.time()) + yield from self.process_events() + else: + # TLS failed, close the connection (notify child layer once closed) + yield commands.CloseConnection(self.conn) else: - yield QuicTlsStartServerHook(tls_data) - if tls_data.settings is None: + # notify the child layer immediately about the error + self._pending_open_command = None + yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) + + def open_connection_end(self, reply: Optional[str]) -> layer.CommandGenerator[bool]: + if self._pending_open_command is None: return False - # create the aioquic connection - self.tls = tls_data.settings - self.quic = QuicConnection( - configuration=self.build_configuration(), - original_destination_connection_id=self.original_destination_connection_id, - ) - if self.issue_connection_id_callback is not None: - self.issue_connection_id_callback(self.quic.host_cid) - self._handle_event = self.state_connected + # let the child layer know that the connection is now open (or failed to open) + command = self._pending_open_command + self._pending_open_command = None + yield from self.event_to_child(events.OpenConnectionCompleted(command, reply)) return True def process_events(self) -> layer.CommandGenerator[None]: @@ -471,7 +575,7 @@ def process_events(self) -> layer.CommandGenerator[None]: ) # shutdown and close the connection - yield from self.shutdown_connection( + yield from self.destroy_quic( event.reason_phrase or str(event.error_code), level=( "info" if event.error_code == QuicErrorCode.NO_ERROR else "warn" @@ -479,119 +583,69 @@ def process_events(self) -> layer.CommandGenerator[None]: ) yield commands.CloseConnection(self.conn) - elif isinstance(event, quic_events.ProtocolNegotiated): - # too early, we act on HandshakeCompleted - pass + # we don't handle any further events, nor do/can we transmit data, so exit + return elif isinstance(event, quic_events.HandshakeCompleted): - # concatenate all peer certificates - all_certs = [] - if self.quic.tls._peer_certificate is not None: - all_certs.append(self.quic.tls._peer_certificate) - if self.quic.tls._peer_certificate_chain is not None: - all_certs.extend(self.quic.tls._peer_certificate_chain) - - # set the connection's TLS properties - self.conn.timestamp_tls_setup = self._loop.time() - self.conn.certificate_list = [ - certs.Cert.from_pyopenssl(x) for x in all_certs - ] - self.conn.alpn = event.alpn_protocol.encode("ascii") - self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name - self.conn.tls_version = "QUIC" - - # report the success to addons - tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) - if self.conn is self.context.client: - yield layers.tls.TlsEstablishedClientHook(tls_data) - else: - yield layers.tls.TlsEstablishedServerHook(tls_data) - - # let the child layer know - if self._pending_open_command is not None: - command = self._pending_open_command - self._pending_open_command = None - yield from self.event_to_child( - events.OpenConnectionCompleted(command, reply=None) - ) + # set all TLS fields and notify the child layer + yield from self.establish_quic(event) + yield from self.open_connection_end(None) # perform next layer decisions now if isinstance(self.child_layer, layer.NextLayer): yield from self.handle_child_commands(self.child_layer._ask()) - else: - # forward the event as a QuicConnectionEvent to the child layer + elif isinstance(event, quic_events.PingAcknowledged): + # we let aioquic do it's thing but don't really care ourselves + pass + + elif isinstance(event, quic_events.ProtocolNegotiated): + # too early, we act on HandshakeCompleted + pass + + elif isinstance( + event, + ( + quic_events.DatagramFrameReceived, + quic_events.StreamDataReceived, + quic_events.StreamReset, + ), + ): + # post-handshake event, forward as QuicConnectionEvent to the child layer + assert self.conn.tls_established yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) + else: + raise AssertionError(f"Unexpected event: {event}") + # handle the next event event = self.quic.next_event() # transmit buffered data and re-arm timer yield from self.transmit() - def shutdown_connection( - self, - reason: str, - level: Literal["error", "warn", "info", "alert", "debug"], - ) -> layer.CommandGenerator[None]: - # ensure QUIC has been properly shut down - assert self.quic is not None - assert self.tls is not None - assert self.quic._state is QuicConnectionState.TERMINATED - - # obsolete any current timer - if self._request_wakeup_command_and_timer is not None: - command, _ = self._request_wakeup_command_and_timer - self._obsolete_wakeup_commands.add(command) - self._request_wakeup_command_and_timer = None - - # report as TLS failure if the termination happened before the handshake - if not self.conn.tls_established: - self.conn.error = reason - tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) - if self.conn is self.context.client: - yield layers.tls.TlsFailedClientHook(tls_data) - else: - yield layers.tls.TlsFailedServerHook(tls_data) - - # make a log entry directly - yield commands.Log( - f"QUIC connection {self.conn} shutdown: {reason}", level=level - ) - - # let the child layer know and stop handling events - # we also don't handle any commands from the child at this point anymore - if self._pending_open_command is not None: - yield from self.child_layer.handle_event( - events.OpenConnectionCompleted(self._pending_open_command, reason) - ) - self._pending_open_command = None - self._handle_event = self.state_done - def start(self) -> layer.CommandGenerator[None]: yield from self.event_to_child(events.Start()) - @expect( - events.ConnectionClosed, events.DataReceived, events.Wakeup, QuicConnectionEvent - ) - def state_connected(self, event: events.Event) -> layer.CommandGenerator[None]: + def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None - # forward incoming data only to aioquic if isinstance(event, events.DataReceived) and event.connection is self.conn: + # forward incoming data only to aioquic assert event.remote_addr is not None self.quic.receive_datagram( event.data, event.remote_addr, now=self._loop.time() ) yield from self.process_events() - # handle connections closed by peer elif ( isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): + # handle connections closed by peer, which in UDP's case is a timeout reason = "Peer UDP connection timed out." + # there is no point in calling quic.close, as it cannot send packets anymore - # so we simply set the state and simulate a ConnectionTerminated event + # set the new connection state and simulate a ConnectionTerminated event (if established) self.quic._set_state(QuicConnectionState.TERMINATED) if self.conn.tls_established: yield from self.event_to_child( @@ -605,14 +659,14 @@ def state_connected(self, event: events.Event) -> layer.CommandGenerator[None]: ) ) - # forward the event only if no open command is pending and shutdown - if self._pending_open_command is None: + # shutdown QUIC and handle the ConnectionClosed event + yield from self.destroy_quic(reason, level="info") + if not (yield from self.open_connection_end(reason)): + # connection was opened before QUIC layer, report to the child layer yield from self.event_to_child(event) - yield from self.shutdown_connection(reason, level="info") - # intercept wakeup events for aioquic elif isinstance(event, events.Wakeup): - # swallow obsolete wakeups + # swallow obsolete wakeup events if event.command in self._obsolete_wakeup_commands: self._obsolete_wakeup_commands.remove(event.command) else: @@ -632,34 +686,37 @@ def state_connected(self, event: events.Event) -> layer.CommandGenerator[None]: # forward other events to the child layer yield from self.event_to_child(event) - @expect( - events.ConnectionClosed, events.DataReceived, events.Wakeup, QuicConnectionEvent - ) - def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: - # filter out obsolete wakeups + def state_no_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.quic is None + if ( isinstance(event, events.Wakeup) and event.command in self._obsolete_wakeup_commands ): + # filter out obsolete wakeups self._obsolete_wakeup_commands.remove(event.command) - # ignore any further received data + elif ( + isinstance(event, events.ConnectionClosed) and event.connection is self.conn + ): + # if there is was an OpenConnection command, then create_quic failed + # otherwise the connection was opened before the QUIC layer, so forward the event + if not (yield from self.open_connection_end("QUIC initialization failed")): + yield from self.event_to_child(event) + elif isinstance(event, events.DataReceived) and event.connection is self.conn: + # ignore received data events + # this either happens after QUIC is closed or if the underlying UDP connection is opened + # before the QUIC layer and missing initialization during the child layer's start event pass - # forward all other events else: - yield from self.child_layer.handle_event(event) - - @expect( - events.ConnectionClosed, events.DataReceived, events.Wakeup, QuicConnectionEvent - ) - def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: - yield from () + # forward all other events to the child layer + yield from self.event_to_child(event) @expect(events.Start) def state_start(self, _) -> layer.CommandGenerator[None]: - self._handle_event = self.state_ready + self._handle_event = self.state_no_quic yield from self.start() def transmit(self) -> layer.CommandGenerator[None]: @@ -716,15 +773,17 @@ def start(self) -> layer.CommandGenerator[None]: # try to open the upstream connection if self.wait_for_upstream: err = yield commands.OpenConnection(self.context.server) - if err is not None: + if err: yield commands.Log( f"Unable to establish QUIC connection with server ({err}). " f"Trying to establish QUIC with client anyway." ) - # is still connected then initialize, close on failure - if not self.conn.connected and not (yield from self.initialize_connection()): + # if (still) connected then initialize, close on failure + if self.conn.connected and not (yield from self.create_quic()): yield commands.CloseConnection(self.conn) + if self.wait_for_upstream and err is not None: + yield commands.CloseConnection(self.context.server) self._handle_event = self.state_failed @expect(events.ConnectionClosed, events.DataReceived, QuicConnectionEvent) From f8b7b6e173ad0167da3de9482bd04a484ce069f1 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 23 Jun 2022 01:57:01 +0200 Subject: [PATCH 025/695] [quic] fix cert issue --- mitmproxy/addons/tlsconfig.py | 8 +++++--- mitmproxy/proxy/layers/quic.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 7a7c7fbffc..3b6df62600 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -322,6 +322,7 @@ def quic_tls_start_client(self, tls_start: quic.QuicTlsData) -> None: tls_start.settings.cipher_suites = [ CipherSuite(cipher) for cipher in client.cipher_list ] + if ctx.options.add_upstream_certs_to_client_chain: tls_start.settings.certificate_chain.extend(cert._cert for cert in server.certificate_list) @@ -348,9 +349,10 @@ def quic_tls_start_server(self, tls_start: quic.QuicTlsData) -> None: if not server.cipher_list and ctx.options.ciphers_server: server.cipher_list = ctx.options.ciphers_server.split(":") - tls_start.settings.cipher_suites = [ - CipherSuite(cipher) for cipher in server.cipher_list - ] + if server.cipher_list: + tls_start.settings.cipher_suites = [ + CipherSuite(cipher) for cipher in server.cipher_list + ] client_cert = self.get_client_cert(server) if client_cert: diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 0a6225ed13..c4a57de9cd 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -443,7 +443,7 @@ def establish_quic( assert self.tls is not None # concatenate all peer certificates - all_certs = [] + all_certs: List[x509.Certificate] = [] if self.quic.tls._peer_certificate is not None: all_certs.append(self.quic.tls._peer_certificate) if self.quic.tls._peer_certificate_chain is not None: @@ -451,7 +451,7 @@ def establish_quic( # set the connection's TLS properties self.conn.timestamp_tls_setup = self._loop.time() - self.conn.certificate_list = [certs.Cert.from_pyopenssl(x) for x in all_certs] + self.conn.certificate_list = [certs.Cert(cert) for cert in all_certs] self.conn.alpn = event.alpn_protocol.encode("ascii") self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name self.conn.tls_version = "QUIC" From db9f4b5a2df6cea4d8f6f455ead3a10b55eba122 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 23 Jun 2022 02:46:52 +0200 Subject: [PATCH 026/695] [quic] add asserts --- mitmproxy/proxy/layers/http/_http3.py | 1 + mitmproxy/proxy/layers/quic.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 2c8e0b0b16..18bff41a64 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -60,6 +60,7 @@ class Http3Connection(HttpConnection): def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): quic = yield QuicGetConnection(self.conn) + assert quic is not None assert isinstance(quic, QuicConnection) self.quic = quic self.h3_conn = H3Connection(quic, enable_webtransport=False) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index c4a57de9cd..d57649ead6 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -484,6 +484,7 @@ def handle_child_commands( isinstance(command, QuicGetConnection) and command.connection is self.conn ): + assert self.quic is not None yield from self.event_to_child( QuicGetConnectionCompleted(command, self.quic) ) @@ -641,8 +642,8 @@ def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: elif ( isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): - # handle connections closed by peer, which in UDP's case is a timeout - reason = "Peer UDP connection timed out." + # handle connections closed by peer (which in UDP's case is usually a timeout) + reason = "Peer UDP connection closed or timed out." # there is no point in calling quic.close, as it cannot send packets anymore # set the new connection state and simulate a ConnectionTerminated event (if established) From 5902c7b0fa9d354172a3d438ec408d88c959857a Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 23 Jun 2022 15:13:13 +0200 Subject: [PATCH 027/695] [quic] temp workaround for QuicGetConnection issue --- mitmproxy/proxy/layers/http/_http3.py | 41 +++++++++++++++++++-------- mitmproxy/proxy/layers/quic.py | 33 ++++++++++----------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 18bff41a64..18fafb8d9f 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -17,10 +17,12 @@ from mitmproxy.net.http import status_codes from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( + _QuicLayer, QuicConnectionEvent, - QuicGetConnection, + # QuicGetConnection, QuicTransmit, ) +from mitmproxy.proxy.utils import expect from . import ( RequestData, @@ -59,14 +61,20 @@ class Http3Connection(HttpConnection): def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): - quic = yield QuicGetConnection(self.conn) - assert quic is not None - assert isinstance(quic, QuicConnection) - self.quic = quic - self.h3_conn = H3Connection(quic, enable_webtransport=False) + # this doesn't always work: + # quic = yield QuicGetConnection(self.conn) + # assert isinstance(quic, QuicConnection) + # self.quic = quic + # + # temporary workaround: + for layer_ in self.context.layers: + if isinstance(layer_, _QuicLayer) and layer_.conn is self.conn: + self.quic = layer_.quic + assert self.quic is not None + self.h3_conn = H3Connection(self.quic, enable_webtransport=False) elif isinstance(event, events.ConnectionClosed): - self._handle_event = self.done + self._handle_event = self.done # type: ignore # send mitmproxy HTTP events over the H3 connection elif isinstance(event, HttpEvent): @@ -90,6 +98,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ), end_stream=event.end_stream, ) + if event.end_stream: + # this will prevent any further headers or data from being sent + self.h3_conn._stream[event.stream_id].headers_send_state = H3HeadersState.AFTER_TRAILERS elif isinstance(event, (RequestTrailers, ResponseTrailers)): self.h3_conn.send_headers( stream_id=event.stream_id, @@ -122,6 +133,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # report abrupt stream resets if isinstance(event, quic_events.StreamReset): if event.stream_id in self.h3_conn._stream: + # try to get a name for the error from its code try: reason = H3ErrorCode(event.error_code).name except ValueError: @@ -129,6 +141,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: reason = QuicErrorCode(event.error_code).name except ValueError: reason = str(event.error_code) + + # report the protocol error (doing the same error code mingling as H2) code = ( status_codes.CLIENT_CLOSED_REQUEST if event.error_code == H3ErrorCode.H3_REQUEST_CANCELLED @@ -192,10 +206,12 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) else: yield ReceiveHttp(receive_event) - if h3_event.stream_ended: - yield ReceiveHttp( - self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) - ) + + # always report an EndOfMessage if the stream has ended + if h3_event.stream_ended: + yield ReceiveHttp( + self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) + ) # we don't support push, web transport, etc. else: @@ -206,7 +222,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unexpected event: {event!r}") - def done(self, event: events.Event) -> layer.CommandGenerator[None]: + @expect(events.DataReceived, HttpEvent, events.ConnectionClosed) + def done(self, _) -> layer.CommandGenerator[None]: yield from () @abstractmethod diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index d57649ead6..e373e40bf2 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -364,6 +364,10 @@ def create_quic(self) -> layer.CommandGenerator[bool]: assert self.quic is None assert self.tls is None + # cannot initialize QUIC on a closed connection + if not self.conn.connected: + return False + # in case the connection is being reused, clear all handshake data self.conn.timestamp_tls_setup = None self.conn.certificate_list = () @@ -419,7 +423,7 @@ def destroy_quic( else: yield layers.tls.TlsFailedServerHook(tls_data) - # clear the quic fields + # clear the QUIC fields self.quic = None self.tls = None self._handle_event = self.state_no_quic @@ -479,28 +483,27 @@ def handle_child_commands( # filter commands coming from the child layer for command in child_commands: - # answer with the aioquic connection instance if ( isinstance(command, QuicGetConnection) and command.connection is self.conn ): + # answer with the aioquic connection instance assert self.quic is not None yield from self.event_to_child( QuicGetConnectionCompleted(command, self.quic) ) - # transmit buffered data and re-arm timer elif isinstance(command, QuicTransmit) and command.connection is self.quic: + # transmit buffered data and re-arm timer yield from self.transmit() - # open the QUIC connection elif ( isinstance(command, commands.OpenConnection) and command.connection is self.conn ): + # try to open the QUIC connection and report OpenConnectionCompleted later yield from self.open_connection_begin(command) - # properly close QUIC connections elif ( isinstance(command, commands.CloseConnection) and command.connection is self.conn @@ -515,14 +518,14 @@ def handle_child_commands( self.quic.close(reason_phrase="CloseConnection command received.") yield from self.process_events() - # return other commands else: + # return other commands yield command def open_connection_begin( self, command: commands.OpenConnection ) -> layer.CommandGenerator[None]: - # ensure only one OpenConnection at a time is called for uninitialized connections + # ensure only one OpenConnection is called at a time and only for uninitialized connections assert self.quic is None assert self._pending_open_command is None self._pending_open_command = command @@ -700,15 +703,14 @@ def state_no_quic(self, event: events.Event) -> layer.CommandGenerator[None]: elif ( isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): - # if there is was an OpenConnection command, then create_quic failed + # if there was an OpenConnection command, then create_quic failed # otherwise the connection was opened before the QUIC layer, so forward the event if not (yield from self.open_connection_end("QUIC initialization failed")): yield from self.event_to_child(event) elif isinstance(event, events.DataReceived) and event.connection is self.conn: - # ignore received data events - # this either happens after QUIC is closed or if the underlying UDP connection is opened - # before the QUIC layer and missing initialization during the child layer's start event + # ignore received data, which either happens after QUIC is closed or if the underlying + # UDP connection is already opened and no QUIC initialization is being performed pass else: @@ -727,7 +729,7 @@ def transmit(self) -> layer.CommandGenerator[None]: for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): yield commands.SendData(self.conn, data, addr) - # mark an existing wakeup command as obsolete if it now longer matches the time + # mark an existing wakeup command as obsolete if it no longer matches the timer timer = self.quic.get_timer() if self._request_wakeup_command_and_timer is not None: command, existing_timer = self._request_wakeup_command_and_timer @@ -780,14 +782,13 @@ def start(self) -> layer.CommandGenerator[None]: f"Trying to establish QUIC with client anyway." ) - # if (still) connected then initialize, close on failure - if self.conn.connected and not (yield from self.create_quic()): + # initialize QUIC, shutdown on failure + if not (yield from self.create_quic()): yield commands.CloseConnection(self.conn) if self.wait_for_upstream and err is not None: yield commands.CloseConnection(self.context.server) self._handle_event = self.state_failed - @expect(events.ConnectionClosed, events.DataReceived, QuicConnectionEvent) def state_failed(self, _) -> layer.CommandGenerator[None]: yield from () @@ -874,7 +875,7 @@ def state_wait_for_hello(self, event: events.Event) -> layer.CommandGenerator[No ) # replace this layer and start the next one - self.handle_event = next_layer.handle_event + self.handle_event = next_layer.handle_event # type: ignore self._handle_event = next_layer._handle_event yield from next_layer.handle_event(events.Start()) yield from next_layer.handle_event(event) From bd213b4a25d6da2692aab6f4cf9ec6d645e22b1d Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 24 Jun 2022 15:08:45 +0200 Subject: [PATCH 028/695] [quic] unified error handling --- mitmproxy/proxy/layers/http/_http3.py | 27 ++++++++------------------ mitmproxy/proxy/layers/quic.py | 28 ++++++++++++++++++--------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 18fafb8d9f..348fae241e 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -11,7 +11,6 @@ from aioquic.h3 import events as h3_events from aioquic.quic import events as quic_events from aioquic.quic.connection import QuicConnection -from aioquic.quic.packet import QuicErrorCode from mitmproxy import http, version from mitmproxy.net.http import status_codes @@ -21,6 +20,7 @@ QuicConnectionEvent, # QuicGetConnection, QuicTransmit, + error_code_to_str, ) from mitmproxy.proxy.utils import expect @@ -100,7 +100,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) if event.end_stream: # this will prevent any further headers or data from being sent - self.h3_conn._stream[event.stream_id].headers_send_state = H3HeadersState.AFTER_TRAILERS + self.h3_conn._stream[ + event.stream_id + ].headers_send_state = H3HeadersState.AFTER_TRAILERS elif isinstance(event, (RequestTrailers, ResponseTrailers)): self.h3_conn.send_headers( stream_id=event.stream_id, @@ -111,12 +113,10 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: self.h3_conn.send_data( stream_id=event.stream_id, data=b"", end_stream=True ) - elif isinstance( - event, (RequestProtocolError, ResponseProtocolError) - ): + elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): self.protocol_error(event) else: - raise AssertionError(f"Unexpected event: {event}") + raise AssertionError(f"Unexpected event: {event!r}") except FrameUnexpected: # Http2Connection also ignores HttpEvents that violate the current stream state @@ -133,15 +133,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # report abrupt stream resets if isinstance(event, quic_events.StreamReset): if event.stream_id in self.h3_conn._stream: - # try to get a name for the error from its code - try: - reason = H3ErrorCode(event.error_code).name - except ValueError: - try: - reason = QuicErrorCode(event.error_code).name - except ValueError: - reason = str(event.error_code) - # report the protocol error (doing the same error code mingling as H2) code = ( status_codes.CLIENT_CLOSED_REQUEST @@ -151,7 +142,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield ReceiveHttp( self.ReceiveProtocolError( stream_id=event.stream_id, - message=f"stream reset by client ({reason})", + message=f"stream reset by client ({error_code_to_str(event.error_code)})", code=code, ) ) @@ -215,9 +206,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # we don't support push, web transport, etc. else: - yield commands.Log( - f"Ignored unsupported H3 event: {h3_event!r}" - ) + yield commands.Log(f"Ignored unsupported H3 event: {h3_event!r}") else: raise AssertionError(f"Unexpected event: {event!r}") diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index e373e40bf2..ac4111a7b7 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -4,6 +4,7 @@ from typing import Callable, Dict, List, Literal, Optional, Set, Tuple, Union from aioquic.buffer import Buffer as QuicBuffer +from aioquic.h3.connection import ErrorCode as H3ErrorCode from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import ( @@ -132,6 +133,16 @@ class QuicClientHello(Exception): data: bytes +def error_code_to_str(error_code: int) -> str: + try: + return H3ErrorCode(error_code).name + except ValueError: + try: + return QuicErrorCode(error_code).name + except ValueError: + return f"unknown error (0x{error_code:x})" + + def pull_client_hello_and_connection_id(data: bytes) -> Tuple[ClientHello, bytes]: # ensure the first packet is indeed the initial one buffer = QuicBuffer(data=data) @@ -301,14 +312,10 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: stream_id_out, flow = lookup_in[stream_id_in] quic_out.stop_stream(stream_id_out, quic_event.error_code) - # try to get a name describing the reset reason - try: - err = QuicErrorCode(quic_event.error_code).name - except ValueError: - err = str(quic_event.error_code) - # report the error to addons and delete the stream - flow.error = mitm_flow.Error(str(err)) + flow.error = mitm_flow.Error( + error_code_to_str(quic_event.error_code) + ) yield tcp_layer.TcpErrorHook(flow) flow.live = False del lookup_in[stream_id_in] @@ -580,9 +587,12 @@ def process_events(self) -> layer.CommandGenerator[None]: # shutdown and close the connection yield from self.destroy_quic( - event.reason_phrase or str(event.error_code), + event.reason_phrase or error_code_to_str(event.error_code), level=( - "info" if event.error_code == QuicErrorCode.NO_ERROR else "warn" + "info" + if event.error_code + in (QuicErrorCode.NO_ERROR, H3ErrorCode.H3_NO_ERROR) + else "warn" ), ) yield commands.CloseConnection(self.conn) From 46096e6af9b8431301897a4b3f411dcf401c09ea Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 26 Jun 2022 23:47:03 +0200 Subject: [PATCH 029/695] [quic] fix context and stream ended handling --- mitmproxy/proxy/layers/http/_http2.py | 10 +++--- mitmproxy/proxy/layers/http/_http3.py | 45 +++++++++++++++++---------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index ec4aed6b72..cfe2b53de9 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -314,6 +314,7 @@ def normalize_h2_headers(headers: list[tuple[bytes, bytes]]) -> CommandGenerator def format_h2_request_headers( + context: Context, event: RequestHeaders, ) -> CommandGenerator[list[tuple[bytes, bytes]]]: pseudo_headers = [ @@ -326,7 +327,7 @@ def format_h2_request_headers( if event.request.is_http2 or event.request.is_http3: hdrs = list(event.request.headers.fields) - if ctx.options.normalize_outbound_headers: + if context.options.normalize_outbound_headers: yield from normalize_h2_headers(hdrs) else: headers = event.request.headers @@ -339,6 +340,7 @@ def format_h2_request_headers( def format_h2_response_headers( + context: Context, event: ResponseHeaders, ) -> CommandGenerator[list[tuple[bytes, bytes]]]: headers = [ @@ -346,7 +348,7 @@ def format_h2_response_headers( *event.response.headers.fields, ] if event.response.is_http2: - if ctx.options.normalize_outbound_headers: + if context.options.normalize_outbound_headers: yield from normalize_h2_headers(headers) else: headers = normalize_h1_headers(headers, False) @@ -372,7 +374,7 @@ def _handle_event(self, event: Event) -> CommandGenerator[None]: if self.is_open_for_us(event.stream_id): self.h2_conn.send_headers( event.stream_id, - headers=(yield from format_h2_response_headers(event)), + headers=(yield from format_h2_response_headers(self.context, event)), end_stream=event.end_stream, ) yield SendData(self.conn, self.h2_conn.data_to_send()) @@ -517,7 +519,7 @@ def _handle_event2(self, event: Event) -> CommandGenerator[None]: elif isinstance(event, RequestHeaders): self.h2_conn.send_headers( event.stream_id, - headers=(yield from format_h2_request_headers(event)), + headers=(yield from format_h2_request_headers(self.context, event)), end_stream=event.end_stream, ) self.streams[event.stream_id] = StreamState.EXPECTING_HEADERS diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 348fae241e..26916b1800 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -10,7 +10,7 @@ ) from aioquic.h3 import events as h3_events from aioquic.quic import events as quic_events -from aioquic.quic.connection import QuicConnection +from aioquic.quic.connection import QuicConnection, stream_is_unidirectional from mitmproxy import http, version from mitmproxy.net.http import status_codes @@ -91,9 +91,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: stream_id=event.stream_id, headers=( yield from ( - format_h2_request_headers(event) + format_h2_request_headers(self.context, event) if isinstance(event, RequestHeaders) - else format_h2_response_headers(event) + else format_h2_response_headers(self.context, event) ) ), end_stream=event.end_stream, @@ -133,24 +133,33 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # report abrupt stream resets if isinstance(event, quic_events.StreamReset): if event.stream_id in self.h3_conn._stream: - # report the protocol error (doing the same error code mingling as H2) - code = ( - status_codes.CLIENT_CLOSED_REQUEST - if event.error_code == H3ErrorCode.H3_REQUEST_CANCELLED - else self.ReceiveProtocolError.code - ) - yield ReceiveHttp( - self.ReceiveProtocolError( - stream_id=event.stream_id, - message=f"stream reset by client ({error_code_to_str(event.error_code)})", - code=code, + stream = self.h3_conn._stream[event.stream_id] + if not stream.ended: + # mark the receiving part of the stream as ended + # (H3Connection alas doesn't handle StreamReset) + stream.ended = True + + # report the protocol error (doing the same error code mingling as H2) + code = ( + status_codes.CLIENT_CLOSED_REQUEST + if event.error_code == H3ErrorCode.H3_REQUEST_CANCELLED + else self.ReceiveProtocolError.code + ) + yield ReceiveHttp( + self.ReceiveProtocolError( + stream_id=event.stream_id, + message=f"stream reset by client ({error_code_to_str(event.error_code)})", + code=code, + ) ) - ) # report a protocol error for all remaining open streams when a connection is terminated elif isinstance(event, quic_events.ConnectionTerminated): for stream in self.h3_conn._stream.values(): - if not stream.ended: + if ( + self.quic._stream_can_receive(stream.stream_id) + and not stream.ended + ): yield ReceiveHttp( self.ReceiveProtocolError( stream_id=stream.stream_id, @@ -344,7 +353,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None ours = self.our_stream_id.get(event.stream_id, None) if ours is None: - ours = self.quic.get_next_available_stream_id() + ours = self.quic.get_next_available_stream_id( + is_unidirectional=stream_is_unidirectional(event.stream_id) + ) self.our_stream_id[event.stream_id] = ours self.their_stream_id[ours] = event.stream_id event.stream_id = ours From 534bc598337c29d3c004b028bd1c975c4a449715 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 27 Jun 2022 05:14:09 +0200 Subject: [PATCH 030/695] [quic] improve relay stream layer --- mitmproxy/addons/next_layer.py | 7 +- mitmproxy/proxy/layers/http/_http2.py | 2 +- mitmproxy/proxy/layers/http/_http3.py | 16 +- mitmproxy/proxy/layers/quic.py | 410 +++++++++++++++++--------- 4 files changed, 283 insertions(+), 152 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index b9908b0a24..9b78292253 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -129,11 +129,14 @@ def _next_layer( mode = HTTPMode.upstream else: return None - return layers.HttpLayer(context=context, mode=mode) + return layers.HttpLayer(context, mode) else: if context.server.address is None: return None - return quic.QuicRelayLayer(context) + if isinstance(context.layers[1], quic.ServerQuicLayer): + return quic.QuicRelayLayer(context) + else: + return quic.ServerQuicLayer(context, quic.QuicRelayLayer(context)) if len(context.layers) == 0: return self.make_top_layer(context) diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index cfe2b53de9..785b0e1e36 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -13,7 +13,7 @@ import h2.stream import h2.utilities -from mitmproxy import ctx, http, version +from mitmproxy import http, version from mitmproxy.connection import Connection from mitmproxy.net.http import status_codes, url from mitmproxy.utils import human diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 26916b1800..cb37086591 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -16,11 +16,10 @@ from mitmproxy.net.http import status_codes from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( - _QuicLayer, QuicConnectionEvent, - # QuicGetConnection, QuicTransmit, error_code_to_str, + get_quic_connection, ) from mitmproxy.proxy.utils import expect @@ -61,16 +60,7 @@ class Http3Connection(HttpConnection): def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): - # this doesn't always work: - # quic = yield QuicGetConnection(self.conn) - # assert isinstance(quic, QuicConnection) - # self.quic = quic - # - # temporary workaround: - for layer_ in self.context.layers: - if isinstance(layer_, _QuicLayer) and layer_.conn is self.conn: - self.quic = layer_.quic - assert self.quic is not None + self.quic = get_quic_connection(self.context, self.conn) self.h3_conn = H3Connection(self.quic, enable_webtransport=False) elif isinstance(event, events.ConnectionClosed): @@ -123,7 +113,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: return # transmit buffered data and re-arm timer - yield QuicTransmit(self.quic) + yield QuicTransmit(self.conn, self.quic) # handle events from the underlying QUIC connection elif isinstance(event, QuicConnectionEvent): diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index ac4111a7b7..e9bc9fe886 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -12,6 +12,7 @@ QuicConnectionError, QuicConnectionState, QuicErrorCode, + stream_is_unidirectional, ) from aioquic.tls import CipherSuite, HandshakeType from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header @@ -88,25 +89,27 @@ class QuicTlsStartServerHook(commands.StartHook): @dataclass class QuicConnectionEvent(events.ConnectionEvent): - event: quic_events.QuicEvent - + """ + Connection-based event that is triggered whenever a new event from QUIC is received. + Established connections are guaranteed to receive a ConnectionTerminated event at the end. -class QuicGetConnection(commands.ConnectionCommand): # -> Optional[QuicConnection] - blocking = True + Note: + 'Established' means that an OpenConnection command called in a child layer returned no error. + Without a predefined child layer, the QUIC layer uses NextLayer mechanics to select the child + layer. The moment is asks addons for the child layer, the connection is considered established. + """ + event: quic_events.QuicEvent -@dataclass(repr=False) -class QuicGetConnectionCompleted(events.CommandCompleted): - command: QuicGetConnection - reply: Optional[QuicConnection] +class QuicTransmit(commands.ConnectionCommand): + """Command that will transmit buffered data and re-arm the given QUIC connection's timer.""" -class QuicTransmit(commands.Command): - connection: QuicConnection + quic: QuicConnection - def __init__(self, connection: QuicConnection) -> None: - super().__init__() - self.connection = connection + def __init__(self, connection: connection.Connection, quic: QuicConnection) -> None: + super().__init__(connection) + self.quic = quic class QuicSecretsLogger: @@ -128,12 +131,9 @@ def flush(self) -> None: pass -@dataclass -class QuicClientHello(Exception): - data: bytes - - def error_code_to_str(error_code: int) -> str: + """Returns the corresponding name of the given error code or a string containing its numeric value.""" + try: return H3ErrorCode(error_code).name except ValueError: @@ -143,7 +143,37 @@ def error_code_to_str(error_code: int) -> str: return f"unknown error (0x{error_code:x})" +def get_quic_connection( + context: context.Context, connection: connection.Connection +) -> QuicConnection: + """Retrieve the QUIC connection associated with the given connection in the given context.""" + + for quic_layer in context.layers: + if isinstance(quic_layer, _QuicLayer) and quic_layer.conn is connection: + if not quic_layer.conn.tls_established: + raise ValueError( + f"QUIC on connection {connection} has not been established yet." + ) + return quic_layer.quic + raise ValueError(f"Connection {connection} has no QUIC.") + + +def is_success_error_code(error_code: int) -> bool: + """Returns whether the given error code actually indicates no error.""" + + return error_code in (QuicErrorCode.NO_ERROR, H3ErrorCode.H3_NO_ERROR) + + +@dataclass +class QuicClientHello(Exception): + """Helper error only used in `pull_client_hello_and_connection_id`.""" + + data: bytes + + def pull_client_hello_and_connection_id(data: bytes) -> Tuple[ClientHello, bytes]: + """Helper function that parses a client hello packet.""" + # ensure the first packet is indeed the initial one buffer = QuicBuffer(data=data) header = pull_quic_header(buffer) @@ -193,133 +223,249 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") +@dataclass +class QuicRelayStream: + client_ended: bool + client_id: int + flow: tcp.TCPFlow + server_ended: bool + server_id: int + + def stream_id(self, client: bool) -> int: + return self.client_id if client else self.server_id + + def has_ended(self, client: bool) -> bool: + stream_ended = self.client_ended if client else self.server_ended + return stream_ended or not self.flow.live + + class QuicRelayLayer(layer.Layer): + """ + Layer on top of `ClientQuicLayer` and `ServerQuicLayer`, that simply relays all QUIC streams and datagrams. + This layer is chosen by the default NextLayer addon if ALPN yields no known protocol. + """ + # for now we're (ab)using the TCPFlow until https://github.com/mitmproxy/mitmproxy/pull/5414 is resolved - datagram_flow: Optional[tcp.TCPFlow] = None - lookup_server: Dict[int, Tuple[int, tcp.TCPFlow]] - lookup_client: Dict[int, Tuple[int, tcp.TCPFlow]] - quic_server: Optional[QuicConnection] = None + flow: tcp.TCPFlow # used for datagrams and to signal general connection issues + streams_by_flow: Dict[tcp.TCPFlow, QuicRelayStream] + streams_by_client_id: Dict[int, QuicRelayStream] + streams_by_server_id: Dict[int, QuicRelayStream] quic_client: Optional[QuicConnection] = None + quic_server: Optional[QuicConnection] = None def __init__(self, context: context.Context) -> None: super().__init__(context) - self.lookup_server = {} - self.lookup_client = {} - - def end_flow( - self, flow: tcp.TCPFlow, event: quic_events.ConnectionTerminated - ) -> layer.CommandGenerator[None]: - if event.error_code == QuicErrorCode.NO_ERROR: - yield tcp_layer.TcpEndHook(flow) + self.flow = tcp.TCPFlow( + self.context.client, + self.context.server, + live=True, + ) + self.streams_by_flow = {} + self.streams_by_client_id = {} + self.streams_by_server_id = {} + + def get_or_create_stream( + self, stream_id: int, from_client: bool + ) -> layer.CommandGenerator[QuicRelayStream]: + streams_by_id = ( + self.streams_by_client_id if from_client else self.streams_by_server_id + ) + if stream_id in streams_by_id: + return streams_by_id[stream_id] else: - flow.error = mitm_flow.Error(event.reason_phrase) - yield tcp_layer.TcpErrorHook(flow) - flow.live = False - - def get_quic( - self, conn: connection.Connection - ) -> layer.CommandGenerator[QuicConnection]: - quic = yield QuicGetConnection(conn) - assert isinstance(quic, QuicConnection) - return quic - - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.Start): - self.quic_server = yield from self.get_quic(self.context.server) - self.quic_client = yield from self.get_quic(self.context.client) - - elif isinstance(event, QuicConnectionEvent): - assert self.quic_server is not None - assert self.quic_client is not None + # reserve the peer stream id + is_unidirectional = stream_is_unidirectional(stream_id) + peer_quic = self.quic_server if from_client else self.quic_client + assert peer_quic + peer_stream_id = peer_quic.get_next_available_stream_id(is_unidirectional) + + # create the instance and make sure unidirectional streams are marked as ended + stream = QuicRelayStream( + flow=tcp.TCPFlow( + self.context.client, + self.context.server, + live=True, + ), + client_ended=is_unidirectional and not from_client, + server_ended=is_unidirectional and from_client, + client_id=stream_id if from_client else peer_stream_id, + server_id=peer_stream_id if from_client else stream_id, + ) + + # register the stream and start the flow + self.streams_by_flow[stream.flow] = stream + self.streams_by_client_id[stream.client_id] = stream + self.streams_by_server_id[stream.server_id] = stream + yield tcp_layer.TcpStartHook(stream.flow) + return stream + + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + # retrieve the client QUIC connection and mark the main flow as started + self.quic_client = get_quic_connection(self.context, self.context.client) + yield tcp_layer.TcpStartHook(self.flow) + + # open the upstream connection if necessary + if self.context.server.timestamp_start is None: + err = yield commands.OpenConnection(self.context.server) + if err: + self.flow.error = mitm_flow.Error(str(err)) + yield tcp_layer.TcpErrorHook(self.flow) + self.flow.live = False + yield commands.CloseConnection(self.context.client) + self._handle_event = self.state_done + return + self.quic_server = get_quic_connection(self.context, self.context.server) + self._handle_event = self.state_ready + + @expect(QuicConnectionEvent, tcp_layer.TcpMessageInjected) + def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: + + if isinstance(event, tcp_layer.TcpMessageInjected): + # translate injected messages into QUIC events + flow = event.flow + assert isinstance(flow, tcp.TCPFlow) + connection = ( + self.context.client + if event.message.from_client + else self.context.server + ) + if flow is self.flow: + event = QuicConnectionEvent( + connection, quic_events.DatagramFrameReceived(event.message.content) + ) + elif flow in self.streams_by_flow: + stream = self.streams_by_flow[flow] + event = QuicConnectionEvent( + connection, + quic_events.StreamDataReceived( + stream_id=( + stream.client_id + if event.message.from_client + else stream.server_id + ), + data=event.message.content, + end_stream=False, + ), + ) + else: + # only handle messages of known flows + return + if isinstance(event, QuicConnectionEvent): + # define helper variables quic_event = event.event from_client = event.connection is self.context.client - lookup_in = self.lookup_client if from_client else self.lookup_server - lookup_out = self.lookup_server if from_client else self.lookup_client - # quic_in = self.quic_client if from_client else self.quic_server - quic_out = self.quic_server if from_client else self.quic_client + peer_connection = ( + self.context.server if from_client else self.context.client + ) + peer_quic = self.quic_server if from_client else self.quic_client + assert peer_quic is not None - # forward close and end all flows if isinstance(quic_event, quic_events.ConnectionTerminated): - quic_out.close( + # report the termination as error to all non-ended streams + for flow in self.streams_by_flow: + if flow.live: + self.flow.error = mitm_flow.Error( + "Connection terminated " + f" (code={quic_event.error_code}, reason={quic_event.reason_phrase})." + ) + yield tcp_layer.TcpErrorHook(flow) + flow.live = False + + # end the main flow + if self.flow.live: + if is_success_error_code(quic_event.error_code): + yield tcp_layer.TcpEndHook(flow) + else: + self.flow.error = mitm_flow.Error( + quic_event.reason_phrase + or error_code_to_str(quic_event.error_code) + ) + yield tcp_layer.TcpErrorHook(flow) + self.flow.live = False + + # close the peer as well and don't handle further events + peer_quic.close( quic_event.error_code, quic_event.frame_type, quic_event.reason_phrase, ) - while lookup_in: - stream_id_in = next(iter(lookup_in)) - stream_id_out, flow = lookup_in[stream_id_in] - yield from self.end_flow(flow, quic_event) - del lookup_in[stream_id_in] - del lookup_out[stream_id_out] - - if self.datagram_flow is not None: - yield from self.end_flow(flow, quic_event) - self.datagram_flow = None - - # forward datagrams (that are not stream-bound) + self._handle_event = self.state_done + elif isinstance(quic_event, quic_events.DatagramFrameReceived): - if self.datagram_flow is None: - self.datagram_flow = tcp.TCPFlow( - self.context.client, - self.context.server, - live=True, - ) - yield tcp_layer.TcpStartHook(self.datagram_flow) + # forward datagrams (that are not stream-bound) + if not self.flow.live: + return message = tcp.TCPMessage(from_client, quic_event.data) - self.datagram_flow.messages.append(message) - yield tcp_layer.TcpMessageHook(self.datagram_flow) - quic_out.send_datagram_frame(message.content) + self.flow.messages.append(message) + yield tcp_layer.TcpMessageHook(self.flow) + peer_quic.send_datagram_frame(message.content) - # forward stream data elif isinstance(quic_event, quic_events.StreamDataReceived): - # get or create the stream on the other side (and flow) - stream_id_in = quic_event.stream_id - if stream_id_in in lookup_in: - stream_id_out, flow = lookup_in[stream_id_in] - else: - stream_id_out = quic_out.get_next_available_stream_id() - flow = tcp.TCPFlow( - self.context.client, - self.context.server, - live=True, - ) - lookup_in[stream_id_in] = (stream_id_out, flow) - lookup_out[stream_id_out] = (stream_id_in, flow) - yield tcp_layer.TcpStartHook(flow) + # ignore data received from already ended streams + stream = yield from self.get_or_create_stream( + quic_event.stream_id, from_client + ) + if stream.has_ended(from_client): + return # forward the message allowing addons to change it message = tcp.TCPMessage(from_client, quic_event.data) - flow.messages.append(message) - yield tcp_layer.TcpMessageHook(flow) - quic_out.send_stream_data( - stream_id_out, - message.content, - quic_event.end_stream, + stream.flow.messages.append(message) + yield tcp_layer.TcpMessageHook(stream.flow) + peer_quic.send_stream_data( + stream_id=stream.stream_id(not from_client), + data=message.content, + end_stream=quic_event.end_stream, ) - # end the flow and remove the lookup if the stream ended + # mark the stream as ended if needed if quic_event.end_stream: - yield tcp_layer.TcpEndHook(flow) - flow.live = False - del lookup_in[stream_id_in] - del lookup_out[stream_id_out] + if from_client: + stream.client_ended = True + else: + stream.server_ended = True + + # end the flow if both legs ended + if stream.client_ended and stream.server_ended: + yield tcp_layer.TcpEndHook(stream.flow) + stream.flow.live = False - # forward resets to peer streams elif isinstance(quic_event, quic_events.StreamReset): - stream_id_in = quic_event.stream_id - if stream_id_in in lookup_in: - stream_id_out, flow = lookup_in[stream_id_in] - quic_out.stop_stream(stream_id_out, quic_event.error_code) - - # report the error to addons and delete the stream - flow.error = mitm_flow.Error( - error_code_to_str(quic_event.error_code) - ) - yield tcp_layer.TcpErrorHook(flow) - flow.live = False - del lookup_in[stream_id_in] - del lookup_out[stream_id_out] + # ignore resets from already ended streams + stream = yield from self.get_or_create_stream( + quic_event.stream_id, from_client + ) + if stream.has_ended(from_client): + return + + # forward resets to peer streams and report them to addons + peer_quic.reset_stream( + stream_id=stream.stream_id(not from_client), + error_code=quic_event.error_code, + ) + stream.flow.error = mitm_flow.Error( + error_code_to_str(quic_event.error_code) + ) + yield tcp_layer.TcpErrorHook(stream.flow) + stream.flow.live = False + + else: + # ignore other QUIC events + return + + # transmit data to the peer + yield QuicTransmit(peer_connection, peer_quic) + + else: + raise AssertionError(f"Unexpected event: {event!r}") + + @expect(QuicConnectionEvent, tcp_layer.TcpMessageInjected, events.ConnectionClosed) + def state_done(self, _) -> layer.CommandGenerator[None]: + yield from () + + _handle_event = state_start class _QuicLayer(layer.Layer): @@ -490,19 +636,10 @@ def handle_child_commands( # filter commands coming from the child layer for command in child_commands: - if ( - isinstance(command, QuicGetConnection) - and command.connection is self.conn - ): - # answer with the aioquic connection instance - assert self.quic is not None - yield from self.event_to_child( - QuicGetConnectionCompleted(command, self.quic) - ) - - elif isinstance(command, QuicTransmit) and command.connection is self.quic: + if isinstance(command, QuicTransmit) and command.connection is self.conn: # transmit buffered data and re-arm timer - yield from self.transmit() + if command.quic is self.quic: + yield from self.transmit() elif ( isinstance(command, commands.OpenConnection) @@ -589,10 +726,7 @@ def process_events(self) -> layer.CommandGenerator[None]: yield from self.destroy_quic( event.reason_phrase or error_code_to_str(event.error_code), level=( - "info" - if event.error_code - in (QuicErrorCode.NO_ERROR, H3ErrorCode.H3_NO_ERROR) - else "warn" + "info" if is_success_error_code(event.error_code) else "warn" ), ) yield commands.CloseConnection(self.conn) @@ -630,7 +764,7 @@ def process_events(self) -> layer.CommandGenerator[None]: yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) else: - raise AssertionError(f"Unexpected event: {event}") + raise AssertionError(f"Unexpected event: {event!r}") # handle the next event event = self.quic.next_event() @@ -761,8 +895,12 @@ class ServerQuicLayer(_QuicLayer): This layer establishes QUIC for a single server connection. """ - def __init__(self, context: context.Context) -> None: + def __init__( + self, context: context.Context, child_layer: Optional[layer.Layer] = None + ) -> None: super().__init__(context, context.server) + if child_layer is not None: + self.child_layer = child_layer class ClientQuicLayer(_QuicLayer): @@ -895,6 +1033,6 @@ def state_wait_for_hello(self, event: events.Event) -> layer.CommandGenerator[No self._handle_event = self.state_done else: - raise AssertionError(f"Unexpected event: {event}") + raise AssertionError(f"Unexpected event: {event!r}") _handle_event = state_start From 569faf41d0f20257a96bdf1c05d3e4b132b9e883 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 27 Jun 2022 18:09:14 +0200 Subject: [PATCH 031/695] [quic] introduce QuicStart event --- mitmproxy/proxy/layers/http/_http3.py | 7 +- mitmproxy/proxy/layers/quic.py | 376 +++++++++++++++----------- 2 files changed, 219 insertions(+), 164 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index cb37086591..86689f726d 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -17,9 +17,9 @@ from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( QuicConnectionEvent, + QuicStart, QuicTransmit, error_code_to_str, - get_quic_connection, ) from mitmproxy.proxy.utils import expect @@ -60,7 +60,10 @@ class Http3Connection(HttpConnection): def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): - self.quic = get_quic_connection(self.context, self.conn) + pass + + elif isinstance(event, QuicStart): + self.quic = event.quic self.h3_conn = H3Connection(self.quic, enable_webtransport=False) elif isinstance(event, events.ConnectionClosed): diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index e9bc9fe886..699cbbfdfe 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -102,6 +102,19 @@ class QuicConnectionEvent(events.ConnectionEvent): event: quic_events.QuicEvent +class QuicStart(events.DataReceived): + """ + Event that indicates that QUIC has been established on a given connection. + This inherits from `DataReceived` in order to trigger next layer behavior and initialize HTTP clients. + """ + + quic: QuicConnection + + def __init__(self, connection: connection.Connection, quic: QuicConnection) -> None: + super().__init__(connection, data=b"") + self.quic = quic + + class QuicTransmit(commands.ConnectionCommand): """Command that will transmit buffered data and re-arm the given QUIC connection's timer.""" @@ -143,21 +156,6 @@ def error_code_to_str(error_code: int) -> str: return f"unknown error (0x{error_code:x})" -def get_quic_connection( - context: context.Context, connection: connection.Connection -) -> QuicConnection: - """Retrieve the QUIC connection associated with the given connection in the given context.""" - - for quic_layer in context.layers: - if isinstance(quic_layer, _QuicLayer) and quic_layer.conn is connection: - if not quic_layer.conn.tls_established: - raise ValueError( - f"QUIC on connection {connection} has not been established yet." - ) - return quic_layer.quic - raise ValueError(f"Connection {connection} has no QUIC.") - - def is_success_error_code(error_code: int) -> bool: """Returns whether the given error code actually indicates no error.""" @@ -245,16 +243,21 @@ class QuicRelayLayer(layer.Layer): This layer is chosen by the default NextLayer addon if ALPN yields no known protocol. """ - # for now we're (ab)using the TCPFlow until https://github.com/mitmproxy/mitmproxy/pull/5414 is resolved + # NOTE: for now we're (ab)using the TCPFlow until https://github.com/mitmproxy/mitmproxy/pull/5414 is resolved + + buffer_from_client: List[quic_events.QuicEvent] + buffer_from_server: List[quic_events.QuicEvent] flow: tcp.TCPFlow # used for datagrams and to signal general connection issues + quic_client: Optional[QuicConnection] = None + quic_server: Optional[QuicConnection] = None streams_by_flow: Dict[tcp.TCPFlow, QuicRelayStream] streams_by_client_id: Dict[int, QuicRelayStream] streams_by_server_id: Dict[int, QuicRelayStream] - quic_client: Optional[QuicConnection] = None - quic_server: Optional[QuicConnection] = None def __init__(self, context: context.Context) -> None: super().__init__(context) + self.buffer_from_client = [] + self.buffer_from_server = [] self.flow = tcp.TCPFlow( self.context.client, self.context.server, @@ -299,10 +302,119 @@ def get_or_create_stream( yield tcp_layer.TcpStartHook(stream.flow) return stream + def handle_quic_event( + self, + event: quic_events.QuicEvent, + from_client: bool, + allow_buffering: bool, + ) -> layer.CommandGenerator[None]: + # get the peer connections + peer_connection = self.context.server if from_client else self.context.client + peer_quic = self.quic_server if from_client else self.quic_client + if peer_quic is None: + # buffer events since the peer is not ready yet + if not allow_buffering: + raise AssertionError( + f"Cannot buffer event from {'client' if from_client else 'server'}." + ) + if from_client: + self.buffer_from_client.append(event) + else: + self.buffer_from_server.append(event) + return + + if isinstance(event, quic_events.ConnectionTerminated): + # report the termination as error to all non-ended streams + for flow in self.streams_by_flow: + if flow.live: + self.flow.error = mitm_flow.Error( + "Connection terminated " + f" (code={event.error_code}, reason={event.reason_phrase})." + ) + yield tcp_layer.TcpErrorHook(flow) + flow.live = False + + # end the main flow + if self.flow.live: + if is_success_error_code(event.error_code): + yield tcp_layer.TcpEndHook(flow) + else: + self.flow.error = mitm_flow.Error( + event.reason_phrase or error_code_to_str(event.error_code) + ) + yield tcp_layer.TcpErrorHook(flow) + self.flow.live = False + + # close the peer as well and don't handle further events + peer_quic.close( + event.error_code, + event.frame_type, + event.reason_phrase, + ) + self._handle_event = self.state_done + + elif isinstance(event, quic_events.DatagramFrameReceived): + # forward datagrams (that are not stream-bound) + if not self.flow.live: + return + message = tcp.TCPMessage(from_client, event.data) + self.flow.messages.append(message) + yield tcp_layer.TcpMessageHook(self.flow) + peer_quic.send_datagram_frame(message.content) + + elif isinstance(event, quic_events.StreamDataReceived): + # ignore data received from already ended streams + stream = yield from self.get_or_create_stream(event.stream_id, from_client) + if stream.has_ended(from_client): + return + + # forward the message allowing addons to change it + message = tcp.TCPMessage(from_client, event.data) + stream.flow.messages.append(message) + yield tcp_layer.TcpMessageHook(stream.flow) + peer_quic.send_stream_data( + stream_id=stream.stream_id(not from_client), + data=message.content, + end_stream=event.end_stream, + ) + + # mark the stream as ended if needed + if event.end_stream: + if from_client: + stream.client_ended = True + else: + stream.server_ended = True + + # end the flow if both legs ended + if stream.client_ended and stream.server_ended: + yield tcp_layer.TcpEndHook(stream.flow) + stream.flow.live = False + + elif isinstance(event, quic_events.StreamReset): + # ignore resets from already ended streams + stream = yield from self.get_or_create_stream(event.stream_id, from_client) + if stream.has_ended(from_client): + return + + # forward resets to peer streams and report them to addons + peer_quic.reset_stream( + stream_id=stream.stream_id(not from_client), + error_code=event.error_code, + ) + stream.flow.error = mitm_flow.Error(error_code_to_str(event.error_code)) + yield tcp_layer.TcpErrorHook(stream.flow) + stream.flow.live = False + + else: + # ignore other QUIC events + return + + # transmit data to the peer + yield QuicTransmit(peer_connection, peer_quic) + @expect(events.Start) def state_start(self, _) -> layer.CommandGenerator[None]: - # retrieve the client QUIC connection and mark the main flow as started - self.quic_client = get_quic_connection(self.context, self.context.client) + # mark the main flow as started yield tcp_layer.TcpStartHook(self.flow) # open the upstream connection if necessary @@ -315,29 +427,49 @@ def state_start(self, _) -> layer.CommandGenerator[None]: yield commands.CloseConnection(self.context.client) self._handle_event = self.state_done return - self.quic_server = get_quic_connection(self.context, self.context.server) self._handle_event = self.state_ready - @expect(QuicConnectionEvent, tcp_layer.TcpMessageInjected) + @expect(QuicStart, QuicConnectionEvent, tcp_layer.TcpMessageInjected) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, tcp_layer.TcpMessageInjected): + if isinstance(event, QuicStart): + # QUIC connection has been established, store it and flush buffered events + if event.connection is self.context.client: + assert self.quic_client is None + self.quic_client = event.quic + for quic_event in self.buffer_from_server: + yield from self.handle_quic_event( + quic_event, + from_client=False, + allow_buffering=False, + ) + elif event.connection is self.context.server: + assert self.quic_server is None + self.quic_server = event.quic + for quic_event in self.buffer_from_client: + yield from self.handle_quic_event( + quic_event, + from_client=True, + allow_buffering=False, + ) + else: + raise AssertionError( + f"Connection {event.connection} not associated with layer." + ) + + elif isinstance(event, tcp_layer.TcpMessageInjected): # translate injected messages into QUIC events flow = event.flow assert isinstance(flow, tcp.TCPFlow) - connection = ( - self.context.client - if event.message.from_client - else self.context.server - ) if flow is self.flow: - event = QuicConnectionEvent( - connection, quic_events.DatagramFrameReceived(event.message.content) + yield from self.handle_quic_event( + quic_events.DatagramFrameReceived(event.message.content), + event.message.from_client, + allow_buffering=True, ) elif flow in self.streams_by_flow: stream = self.streams_by_flow[flow] - event = QuicConnectionEvent( - connection, + yield from self.handle_quic_event( quic_events.StreamDataReceived( stream_id=( stream.client_id @@ -347,121 +479,31 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: data=event.message.content, end_stream=False, ), + event.message.from_client, + allow_buffering=True, ) else: - # only handle messages of known flows - return - - if isinstance(event, QuicConnectionEvent): - # define helper variables - quic_event = event.event - from_client = event.connection is self.context.client - peer_connection = ( - self.context.server if from_client else self.context.client - ) - peer_quic = self.quic_server if from_client else self.quic_client - assert peer_quic is not None - - if isinstance(quic_event, quic_events.ConnectionTerminated): - # report the termination as error to all non-ended streams - for flow in self.streams_by_flow: - if flow.live: - self.flow.error = mitm_flow.Error( - "Connection terminated " - f" (code={quic_event.error_code}, reason={quic_event.reason_phrase})." - ) - yield tcp_layer.TcpErrorHook(flow) - flow.live = False - - # end the main flow - if self.flow.live: - if is_success_error_code(quic_event.error_code): - yield tcp_layer.TcpEndHook(flow) - else: - self.flow.error = mitm_flow.Error( - quic_event.reason_phrase - or error_code_to_str(quic_event.error_code) - ) - yield tcp_layer.TcpErrorHook(flow) - self.flow.live = False - - # close the peer as well and don't handle further events - peer_quic.close( - quic_event.error_code, - quic_event.frame_type, - quic_event.reason_phrase, + raise AssertionError( + f"Flow {event.flow} not associated with the current layer." ) - self._handle_event = self.state_done - elif isinstance(quic_event, quic_events.DatagramFrameReceived): - # forward datagrams (that are not stream-bound) - if not self.flow.live: - return - message = tcp.TCPMessage(from_client, quic_event.data) - self.flow.messages.append(message) - yield tcp_layer.TcpMessageHook(self.flow) - peer_quic.send_datagram_frame(message.content) - - elif isinstance(quic_event, quic_events.StreamDataReceived): - # ignore data received from already ended streams - stream = yield from self.get_or_create_stream( - quic_event.stream_id, from_client - ) - if stream.has_ended(from_client): - return - - # forward the message allowing addons to change it - message = tcp.TCPMessage(from_client, quic_event.data) - stream.flow.messages.append(message) - yield tcp_layer.TcpMessageHook(stream.flow) - peer_quic.send_stream_data( - stream_id=stream.stream_id(not from_client), - data=message.content, - end_stream=quic_event.end_stream, - ) - - # mark the stream as ended if needed - if quic_event.end_stream: - if from_client: - stream.client_ended = True - else: - stream.server_ended = True - - # end the flow if both legs ended - if stream.client_ended and stream.server_ended: - yield tcp_layer.TcpEndHook(stream.flow) - stream.flow.live = False - - elif isinstance(quic_event, quic_events.StreamReset): - # ignore resets from already ended streams - stream = yield from self.get_or_create_stream( - quic_event.stream_id, from_client - ) - if stream.has_ended(from_client): - return - - # forward resets to peer streams and report them to addons - peer_quic.reset_stream( - stream_id=stream.stream_id(not from_client), - error_code=quic_event.error_code, - ) - stream.flow.error = mitm_flow.Error( - error_code_to_str(quic_event.error_code) - ) - yield tcp_layer.TcpErrorHook(stream.flow) - stream.flow.live = False - - else: - # ignore other QUIC events - return - - # transmit data to the peer - yield QuicTransmit(peer_connection, peer_quic) + elif isinstance(event, QuicConnectionEvent): + # handle or buffer QUIC events + yield from self.handle_quic_event( + event.event, + from_client=event.connection is self.context.client, + allow_buffering=True, + ) else: raise AssertionError(f"Unexpected event: {event!r}") - @expect(QuicConnectionEvent, tcp_layer.TcpMessageInjected, events.ConnectionClosed) + @expect( + QuicStart, + QuicConnectionEvent, + tcp_layer.TcpMessageInjected, + events.ConnectionClosed, + ) def state_done(self, _) -> layer.CommandGenerator[None]: yield from () @@ -491,6 +533,7 @@ def __init__( Tuple[commands.RequestWakeup, float] ] = None self._obsolete_wakeup_commands: Set[commands.RequestWakeup] = set() + self.conn.tls = True def build_configuration(self) -> QuicConfiguration: assert self.tls is not None @@ -628,13 +671,8 @@ def establish_quic( ) def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: - yield from self.handle_child_commands(self.child_layer.handle_event(event)) - - def handle_child_commands( - self, child_commands: layer.CommandGenerator[None] - ) -> layer.CommandGenerator[None]: # filter commands coming from the child layer - for command in child_commands: + for command in self.child_layer.handle_event(event): if isinstance(command, QuicTransmit) and command.connection is self.conn: # transmit buffered data and re-arm timer @@ -729,6 +767,21 @@ def process_events(self) -> layer.CommandGenerator[None]: "info" if is_success_error_code(event.error_code) else "warn" ), ) + if self.conn is self.context.client: + # once the client connection is closed, all servers are terminated immediately + # use this opportunity to properly shutdown QUIC connections + for quic_layer in self.context.layers: + if ( + isinstance(quic_layer, _QuicLayer) + and quic_layer.conn is not self.context.client + and quic_layer.quic is not None + ): + quic_layer.quic.close( + event.error_code, + event.frame_type, + event.reason_phrase, + ) + yield from quic_layer.transmit() yield commands.CloseConnection(self.conn) # we don't handle any further events, nor do/can we transmit data, so exit @@ -738,10 +791,7 @@ def process_events(self) -> layer.CommandGenerator[None]: # set all TLS fields and notify the child layer yield from self.establish_quic(event) yield from self.open_connection_end(None) - - # perform next layer decisions now - if isinstance(self.child_layer, layer.NextLayer): - yield from self.handle_child_commands(self.child_layer._ask()) + yield from self.event_to_child(QuicStart(self.conn, self.quic)) elif isinstance(event, quic_events.PingAcknowledged): # we let aioquic do it's thing but don't really care ourselves @@ -789,27 +839,27 @@ def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: elif ( isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): - # handle connections closed by peer (which in UDP's case is usually a timeout) - reason = "Peer UDP connection closed or timed out." - # there is no point in calling quic.close, as it cannot send packets anymore # set the new connection state and simulate a ConnectionTerminated event (if established) + close_event = self.quic._close_event + if close_event is None: + close_event = quic_events.ConnectionTerminated( + error_code=QuicErrorCode.APPLICATION_ERROR, + frame_type=None, + reason_phrase="Peer UDP connection closed or timed out.", + ) self.quic._set_state(QuicConnectionState.TERMINATED) if self.conn.tls_established: yield from self.event_to_child( - QuicConnectionEvent( - self.conn, - quic_events.ConnectionTerminated( - error_code=QuicErrorCode.APPLICATION_ERROR, - frame_type=None, - reason_phrase=reason, - ), - ) + QuicConnectionEvent(self.conn, close_event) ) # shutdown QUIC and handle the ConnectionClosed event - yield from self.destroy_quic(reason, level="info") - if not (yield from self.open_connection_end(reason)): + yield from self.destroy_quic( + close_event.reason_phrase or error_code_to_str(close_event.error_code), + level="info", + ) + if not (yield from self.open_connection_end(close_event.reason_phrase)): # connection was opened before QUIC layer, report to the child layer yield from self.event_to_child(event) @@ -955,6 +1005,8 @@ def __init__( super().__init__(context) self._issue_cid = issue_cid self._retire_cid = retire_cid + self.context.client.tls = True + self.context.server.tls = True def build_client_layer( self, connection_id: bytes, wait_for_upstream: bool From 8e71b0331b8de95c4204d5cc26fb07e967883972 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 27 Jun 2022 19:30:59 +0200 Subject: [PATCH 032/695] [quic] add is_http3 where necessary --- mitmproxy/addons/dumper.py | 2 +- mitmproxy/addons/next_layer.py | 12 +++++------- mitmproxy/http.py | 8 ++++---- mitmproxy/proxy/layers/http/__init__.py | 2 +- mitmproxy/proxy/layers/http/_http1.py | 6 +++--- mitmproxy/proxy/layers/http/_http2.py | 2 +- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 7da08ecf4c..a35a350565 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -200,7 +200,7 @@ def _echo_response_line(self, flow: http.HTTPFlow) -> None: blink=(code_int == 418), ) - if not flow.response.is_http2: + if not (flow.response.is_http2 or flow.response.is_http3): reason = flow.response.reason else: reason = http.status_codes.RESPONSES.get(flow.response.status_code, "") diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 9b78292253..f067a48ec7 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -130,13 +130,11 @@ def _next_layer( else: return None return layers.HttpLayer(context, mode) - else: - if context.server.address is None: - return None - if isinstance(context.layers[1], quic.ServerQuicLayer): - return quic.QuicRelayLayer(context) - else: - return quic.ServerQuicLayer(context, quic.QuicRelayLayer(context)) + if context.server.address is None: + return None # not H3 and no predefined destination, nothing we can do + if isinstance(context.layers[1], quic.ServerQuicLayer): + return quic.QuicRelayLayer(context) # server layer already present + return quic.ServerQuicLayer(context, quic.QuicRelayLayer(context)) if len(context.layers) == 0: return self.make_top_layer(context) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 394cddbdd2..151dcdccb9 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -767,7 +767,7 @@ def host_header(self) -> Optional[str]: *See also:* `Request.authority`,`Request.host`, `Request.pretty_host` """ - if self.is_http2: + if self.is_http2 or self.is_http3: return self.authority or self.data.headers.get("Host", None) else: return self.data.headers.get("Host", None) @@ -775,13 +775,13 @@ def host_header(self) -> Optional[str]: @host_header.setter def host_header(self, val: Union[None, str, bytes]) -> None: if val is None: - if self.is_http2: + if self.is_http2 or self.is_http3: self.data.authority = b"" self.headers.pop("Host", None) else: - if self.is_http2: + if self.is_http2 or self.is_http3: self.authority = val # type: ignore - if not self.is_http2 or "Host" in self.headers: + if not (self.is_http2 or self.is_http3) or "Host" in self.headers: # For h2, we only overwrite, but not create, as :authority is the h2 host header. self.headers["Host"] = val diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 6b887269f1..69d98255a4 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -220,7 +220,7 @@ def state_wait_for_request_headers( "https" if self.context.client.tls else "http" ) - if self.mode is HTTPMode.regular and not self.flow.request.is_http2: + if self.mode is HTTPMode.regular and not (self.flow.request.is_http2 or self.flow.request.is_http3): # Set the request target to origin-form for HTTP/1, some servers don't support absolute-form requests. # see https://github.com/mitmproxy/mitmproxy/issues/1759 self.flow.request.authority = "" diff --git a/mitmproxy/proxy/layers/http/_http1.py b/mitmproxy/proxy/layers/http/_http1.py index 4cab5bd9b6..fa49fc3f23 100644 --- a/mitmproxy/proxy/layers/http/_http1.py +++ b/mitmproxy/proxy/layers/http/_http1.py @@ -189,7 +189,7 @@ def mark_done( # If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request. # This simplifies our connection management quite a bit as we can rely on # the proxyserver's max-connection-per-server throttling. - or (self.request.is_http2 and isinstance(self, Http1Client)) + or ((self.request.is_http2 or self.request.is_http3) and isinstance(self, Http1Client)) ) if connection_done: yield commands.CloseConnection(self.conn) @@ -223,7 +223,7 @@ def send(self, event: HttpEvent) -> layer.CommandGenerator[None]: if isinstance(event, ResponseHeaders): self.response = response = event.response - if response.is_http2: + if response.is_http2 or response.is_http3: response = response.copy() # Convert to an HTTP/1 response. response.http_version = "HTTP/1.1" @@ -340,7 +340,7 @@ def send(self, event: HttpEvent) -> layer.CommandGenerator[None]: if isinstance(event, RequestHeaders): request = event.request - if request.is_http2: + if request.is_http2 or request.is_http3: # Convert to an HTTP/1 request. request = ( request.copy() diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index 785b0e1e36..3307d9cfb5 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -347,7 +347,7 @@ def format_h2_response_headers( (b":status", b"%d" % event.response.status_code), *event.response.headers.fields, ] - if event.response.is_http2: + if event.response.is_http2 or event.response.is_http3: if context.options.normalize_outbound_headers: yield from normalize_h2_headers(headers) else: From 2a221ad9b7c376135386af4eecf444182ae0322f Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 27 Jun 2022 19:54:48 +0200 Subject: [PATCH 033/695] [quic] reworked close handling --- mitmproxy/proxy/layers/http/_http3.py | 66 ++++++------ mitmproxy/proxy/layers/quic.py | 138 ++++++++++++-------------- 2 files changed, 100 insertions(+), 104 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 86689f726d..14b8c0b7f4 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -58,21 +58,26 @@ class Http3Connection(HttpConnection): ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.Start): - pass + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + self._handle_event = self.state_wait_for_quic + yield from () - elif isinstance(event, QuicStart): - self.quic = event.quic - self.h3_conn = H3Connection(self.quic, enable_webtransport=False) + @expect(QuicStart) + def state_wait_for_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, QuicStart) + self.quic = event.quic + self.h3_conn = H3Connection(self.quic, enable_webtransport=False) + self._handle_event = self.state_ready + yield from () - elif isinstance(event, events.ConnectionClosed): - self._handle_event = self.done # type: ignore + @expect(HttpEvent, QuicConnectionEvent, events.ConnectionClosed) + def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.quic is not None + assert self.h3_conn is not None # send mitmproxy HTTP events over the H3 connection - elif isinstance(event, HttpEvent): - assert self.quic is not None - assert self.h3_conn is not None + if isinstance(event, HttpEvent): try: if isinstance(event, (RequestData, ResponseData)): @@ -120,8 +125,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # handle events from the underlying QUIC connection elif isinstance(event, QuicConnectionEvent): - assert self.quic is not None - assert self.h3_conn is not None # report abrupt stream resets if isinstance(event, quic_events.StreamReset): @@ -146,21 +149,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) ) - # report a protocol error for all remaining open streams when a connection is terminated - elif isinstance(event, quic_events.ConnectionTerminated): - for stream in self.h3_conn._stream.values(): - if ( - self.quic._stream_can_receive(stream.stream_id) - and not stream.ended - ): - yield ReceiveHttp( - self.ReceiveProtocolError( - stream_id=stream.stream_id, - message=event.reason_phrase, - code=event.error_code, - ) - ) - # forward QUIC events to the H3 connection for h3_event in self.h3_conn.handle_event(event.event): @@ -197,6 +185,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, reason_phrase=f"Invalid HTTP/3 request headers: {e}", ) + yield QuicTransmit(self.conn, self.quic) else: yield ReceiveHttp(receive_event) @@ -210,11 +199,26 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: yield commands.Log(f"Ignored unsupported H3 event: {h3_event!r}") + # report a protocol error for all remaining open streams when a connection is closed + elif isinstance(event, events.ConnectionClosed): + for stream in self.h3_conn._stream.values(): + if self.quic._stream_can_receive(stream.stream_id) and not stream.ended: + close_event = self.quic._close_event + assert close_event is not None + yield ReceiveHttp( + self.ReceiveProtocolError( + stream_id=stream.stream_id, + message=close_event.reason_phrase, + code=close_event.error_code, + ) + ) + self._handle_event = self.state_done + else: raise AssertionError(f"Unexpected event: {event!r}") - @expect(events.DataReceived, HttpEvent, events.ConnectionClosed) - def done(self, _) -> layer.CommandGenerator[None]: + @expect(HttpEvent, QuicConnectionEvent, events.ConnectionClosed) + def state_done(self, _) -> layer.CommandGenerator[None]: yield from () @abstractmethod @@ -229,6 +233,8 @@ def headers_received( ) -> Union[RequestHeaders, ResponseHeaders]: pass # pragma: no cover + _handle_event = state_start + class Http3Server(Http3Connection): ReceiveData = RequestData diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 699cbbfdfe..d35963896f 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -91,7 +91,6 @@ class QuicTlsStartServerHook(commands.StartHook): class QuicConnectionEvent(events.ConnectionEvent): """ Connection-based event that is triggered whenever a new event from QUIC is received. - Established connections are guaranteed to receive a ConnectionTerminated event at the end. Note: 'Established' means that an OpenConnection command called in a child layer returned no error. @@ -308,11 +307,9 @@ def handle_quic_event( from_client: bool, allow_buffering: bool, ) -> layer.CommandGenerator[None]: - # get the peer connections - peer_connection = self.context.server if from_client else self.context.client + # buffer events if the peer is not ready yet peer_quic = self.quic_server if from_client else self.quic_client if peer_quic is None: - # buffer events since the peer is not ready yet if not allow_buffering: raise AssertionError( f"Cannot buffer event from {'client' if from_client else 'server'}." @@ -322,38 +319,9 @@ def handle_quic_event( else: self.buffer_from_server.append(event) return + peer_connection = self.context.server if from_client else self.context.client - if isinstance(event, quic_events.ConnectionTerminated): - # report the termination as error to all non-ended streams - for flow in self.streams_by_flow: - if flow.live: - self.flow.error = mitm_flow.Error( - "Connection terminated " - f" (code={event.error_code}, reason={event.reason_phrase})." - ) - yield tcp_layer.TcpErrorHook(flow) - flow.live = False - - # end the main flow - if self.flow.live: - if is_success_error_code(event.error_code): - yield tcp_layer.TcpEndHook(flow) - else: - self.flow.error = mitm_flow.Error( - event.reason_phrase or error_code_to_str(event.error_code) - ) - yield tcp_layer.TcpErrorHook(flow) - self.flow.live = False - - # close the peer as well and don't handle further events - peer_quic.close( - event.error_code, - event.frame_type, - event.reason_phrase, - ) - self._handle_event = self.state_done - - elif isinstance(event, quic_events.DatagramFrameReceived): + if isinstance(event, quic_events.DatagramFrameReceived): # forward datagrams (that are not stream-bound) if not self.flow.live: return @@ -429,10 +397,56 @@ def state_start(self, _) -> layer.CommandGenerator[None]: return self._handle_event = self.state_ready - @expect(QuicStart, QuicConnectionEvent, tcp_layer.TcpMessageInjected) + @expect( + QuicStart, + QuicConnectionEvent, + tcp_layer.TcpMessageInjected, + events.ConnectionClosed, + ) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, QuicStart): + if isinstance(event, events.ConnectionClosed): + # define helper variables + from_client = event.connection is self.context.client + peer_conn = self.context.server if from_client else self.context.client + local_quic = self.quic_client if from_client else self.quic_server + peer_quic = self.quic_server if from_client else self.quic_client + assert local_quic is not None + close_event = local_quic._close_event + assert close_event is not None + + # report the termination as error to all non-ended streams + for flow in self.streams_by_flow: + if flow.live: + self.flow.error = mitm_flow.Error("Connection closed.") + yield tcp_layer.TcpErrorHook(flow) + flow.live = False + + # end the main flow + if self.flow.live: + if is_success_error_code(close_event.error_code): + yield tcp_layer.TcpEndHook(flow) + else: + self.flow.error = mitm_flow.Error( + close_event.reason_phrase + or error_code_to_str(close_event.error_code) + ) + yield tcp_layer.TcpErrorHook(flow) + self.flow.live = False + + # close the peer as well + if peer_quic is not None: + peer_quic.close( + close_event.error_code, + close_event.frame_type, + close_event.reason_phrase, + ) + yield QuicTransmit(peer_conn, peer_quic) + else: + yield commands.CloseConnection(peer_conn) + self._handle_event = self.state_done + + elif isinstance(event, QuicStart): # QUIC connection has been established, store it and flush buffered events if event.connection is self.context.client: assert self.quic_client is None @@ -459,16 +473,15 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, tcp_layer.TcpMessageInjected): # translate injected messages into QUIC events - flow = event.flow - assert isinstance(flow, tcp.TCPFlow) - if flow is self.flow: + assert isinstance(event.flow, tcp.TCPFlow) + if event.flow is self.flow: yield from self.handle_quic_event( quic_events.DatagramFrameReceived(event.message.content), event.message.from_client, allow_buffering=True, ) - elif flow in self.streams_by_flow: - stream = self.streams_by_flow[flow] + elif event.flow in self.streams_by_flow: + stream = self.streams_by_flow[event.flow] yield from self.handle_quic_event( quic_events.StreamDataReceived( stream_id=( @@ -754,12 +767,6 @@ def process_events(self) -> layer.CommandGenerator[None]: self.retire_connection_id_callback(event.connection_id) elif isinstance(event, quic_events.ConnectionTerminated): - # only forward the event if the connection has been properly initialized - if self.conn.tls_established: - yield from self.event_to_child( - QuicConnectionEvent(self.conn, event) - ) - # shutdown and close the connection yield from self.destroy_quic( event.reason_phrase or error_code_to_str(event.error_code), @@ -767,21 +774,6 @@ def process_events(self) -> layer.CommandGenerator[None]: "info" if is_success_error_code(event.error_code) else "warn" ), ) - if self.conn is self.context.client: - # once the client connection is closed, all servers are terminated immediately - # use this opportunity to properly shutdown QUIC connections - for quic_layer in self.context.layers: - if ( - isinstance(quic_layer, _QuicLayer) - and quic_layer.conn is not self.context.client - and quic_layer.quic is not None - ): - quic_layer.quic.close( - event.error_code, - event.frame_type, - event.reason_phrase, - ) - yield from quic_layer.transmit() yield commands.CloseConnection(self.conn) # we don't handle any further events, nor do/can we transmit data, so exit @@ -840,26 +832,24 @@ def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): # there is no point in calling quic.close, as it cannot send packets anymore - # set the new connection state and simulate a ConnectionTerminated event (if established) - close_event = self.quic._close_event - if close_event is None: - close_event = quic_events.ConnectionTerminated( + # just set the new connection state and ensure there is exists a close event + self.quic._set_state(QuicConnectionState.TERMINATED) + if self.quic._close_event is None: + self.quic._close_event = quic_events.ConnectionTerminated( error_code=QuicErrorCode.APPLICATION_ERROR, frame_type=None, reason_phrase="Peer UDP connection closed or timed out.", ) - self.quic._set_state(QuicConnectionState.TERMINATED) - if self.conn.tls_established: - yield from self.event_to_child( - QuicConnectionEvent(self.conn, close_event) - ) # shutdown QUIC and handle the ConnectionClosed event + reason = self.quic._close_event.reason_phrase or error_code_to_str( + self.quic._close_event.error_code + ) yield from self.destroy_quic( - close_event.reason_phrase or error_code_to_str(close_event.error_code), + reason, level="info", ) - if not (yield from self.open_connection_end(close_event.reason_phrase)): + if not (yield from self.open_connection_end(reason)): # connection was opened before QUIC layer, report to the child layer yield from self.event_to_child(event) From 0f33e70a82f2b955b7521e9aaa1c08bc401221a0 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 27 Jun 2022 20:14:41 +0200 Subject: [PATCH 034/695] [quic] fix empty layers issue --- mitmproxy/addons/next_layer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index f067a48ec7..f8a6b00324 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -117,6 +117,10 @@ def next_layer(self, nextlayer: layer.NextLayer): def _next_layer( self, context: context.Context, data_client: bytes, data_server: bytes ) -> Optional[layer.Layer]: + if len(context.layers) == 0: + return self.make_top_layer(context) + + # handle QUIC connections if isinstance(context.layers[0], quic.QuicLayer): if context.client.alpn is None: return None # should never happen, as ask is called after handshake @@ -136,9 +140,6 @@ def _next_layer( return quic.QuicRelayLayer(context) # server layer already present return quic.ServerQuicLayer(context, quic.QuicRelayLayer(context)) - if len(context.layers) == 0: - return self.make_top_layer(context) - if len(data_client) < 3 and not data_server: return None # not enough data yet to make a decision From 46195ce4a6d0e803e3a0593325647cdf8990e518 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 28 Jun 2022 04:08:30 +0200 Subject: [PATCH 035/695] [quic] remove stream ID mapping --- mitmproxy/proxy/layers/http/_http3.py | 26 ++--------------------- test/mitmproxy/addons/test_proxyserver.py | 6 +++--- test/mitmproxy/net/test_udp.py | 4 ++-- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 14b8c0b7f4..7ac4522f58 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,6 +1,6 @@ from abc import abstractmethod import time -from typing import Dict, Optional, Union +from typing import Optional, Union from aioquic.h3.connection import ( H3Connection, @@ -10,7 +10,7 @@ ) from aioquic.h3 import events as h3_events from aioquic.quic import events as quic_events -from aioquic.quic.connection import QuicConnection, stream_is_unidirectional +from aioquic.quic.connection import QuicConnection from mitmproxy import http, version from mitmproxy.net.http import status_codes @@ -307,13 +307,8 @@ class Http3Client(Http3Connection): ReceiveProtocolError = ResponseProtocolError ReceiveTrailers = ResponseTrailers - our_stream_id: Dict[int, int] - their_stream_id: Dict[int, int] - def __init__(self, context: context.Context): super().__init__(context, context.server) - self.our_stream_id = {} - self.their_stream_id = {} def protocol_error( self, event: Union[RequestProtocolError, ResponseProtocolError] @@ -346,23 +341,6 @@ def headers_received( stream_id=event.stream_id, response=response, end_stream=event.stream_ended ) - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - # translate stream IDs just like HTTP/2 client - if isinstance(event, HttpEvent): - assert self.quic is not None - ours = self.our_stream_id.get(event.stream_id, None) - if ours is None: - ours = self.quic.get_next_available_stream_id( - is_unidirectional=stream_is_unidirectional(event.stream_id) - ) - self.our_stream_id[event.stream_id] = ours - self.their_stream_id[ours] = event.stream_id - event.stream_id = ours - for cmd in super()._handle_event(event): - if isinstance(cmd, ReceiveHttp): - cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id] - yield cmd - __all__ = [ "Http3Client", diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 9abeb983ec..5edc333f56 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -266,16 +266,16 @@ async def test_dns() -> None: await tctx.master.await_log("Invalid DNS datagram received", level="info") req = tdnsreq() w.write(req.packed) - resp = dns.Message.unpack(await r.read(udp.MAX_DATAGRAM_SIZE)) + resp = dns.Message.unpack((await r.read(udp.MAX_DATAGRAM_SIZE))[0]) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps._connections) == 1 w.write(req.packed) - resp = dns.Message.unpack(await r.read(udp.MAX_DATAGRAM_SIZE)) + resp = dns.Message.unpack((await r.read(udp.MAX_DATAGRAM_SIZE))[0]) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps._connections) == 1 req.id = req.id + 1 w.write(req.packed) - resp = dns.Message.unpack(await r.read(udp.MAX_DATAGRAM_SIZE)) + resp = dns.Message.unpack((await r.read(udp.MAX_DATAGRAM_SIZE))[0]) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps._connections) == 2 await ps.shutdown_server() diff --git a/test/mitmproxy/net/test_udp.py b/test/mitmproxy/net/test_udp.py index 1db5a60997..c7f0ef7338 100644 --- a/test/mitmproxy/net/test_udp.py +++ b/test/mitmproxy/net/test_udp.py @@ -12,8 +12,8 @@ async def test_reader(): reader.feed_data(bytearray(MAX_DATAGRAM_SIZE + 1), addr) reader.feed_data(b"Second message", addr) reader.feed_eof() - assert await reader.read(65535) == b"First message" + assert await reader.read(65535) == (b"First message", addr) with pytest.raises(AssertionError): await reader.read(MAX_DATAGRAM_SIZE - 1) - assert await reader.read(65535) == b"Second message" + assert await reader.read(65535) == (b"Second message", addr) assert not await reader.read(65535) From 8a1355e4a118a672bff0681ddfe351cf9d90c7e9 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 28 Jun 2022 15:26:56 +0200 Subject: [PATCH 036/695] [quic] preserve stream IDs in relay layer --- mitmproxy/proxy/layers/quic.py | 143 +++++++++++++-------------------- test/mitmproxy/net/test_udp.py | 2 +- 2 files changed, 57 insertions(+), 88 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index d35963896f..83d32ef819 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -12,6 +12,7 @@ QuicConnectionError, QuicConnectionState, QuicErrorCode, + stream_is_client_initiated, stream_is_unidirectional, ) from aioquic.tls import CipherSuite, HandshakeType @@ -220,21 +221,32 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") -@dataclass class QuicRelayStream: - client_ended: bool - client_id: int flow: tcp.TCPFlow - server_ended: bool - server_id: int + stream_id: int - def stream_id(self, client: bool) -> int: - return self.client_id if client else self.server_id + def __init__(self, context: context.Context, stream_id: int) -> None: + self.flow = tcp.TCPFlow( + context.client, + context.server, + live=True, + ) + self.stream_id = stream_id + is_unidirectional = stream_is_unidirectional(stream_id) + from_client = stream_is_client_initiated(stream_id) + self._ended_client = is_unidirectional and not from_client + self._ended_server = is_unidirectional and from_client def has_ended(self, client: bool) -> bool: - stream_ended = self.client_ended if client else self.server_ended + stream_ended = self._ended_client if client else self._ended_server return stream_ended or not self.flow.live + def mark_ended(self, client: bool) -> None: + if client: + self._ended_client = True + else: + self._ended_server = True + class QuicRelayLayer(layer.Layer): """ @@ -244,14 +256,13 @@ class QuicRelayLayer(layer.Layer): # NOTE: for now we're (ab)using the TCPFlow until https://github.com/mitmproxy/mitmproxy/pull/5414 is resolved - buffer_from_client: List[quic_events.QuicEvent] - buffer_from_server: List[quic_events.QuicEvent] + buffer_from_client: Optional[List[quic_events.QuicEvent]] + buffer_from_server: Optional[List[quic_events.QuicEvent]] flow: tcp.TCPFlow # used for datagrams and to signal general connection issues quic_client: Optional[QuicConnection] = None quic_server: Optional[QuicConnection] = None streams_by_flow: Dict[tcp.TCPFlow, QuicRelayStream] - streams_by_client_id: Dict[int, QuicRelayStream] - streams_by_server_id: Dict[int, QuicRelayStream] + streams_by_id: Dict[int, QuicRelayStream] def __init__(self, context: context.Context) -> None: super().__init__(context) @@ -263,41 +274,18 @@ def __init__(self, context: context.Context) -> None: live=True, ) self.streams_by_flow = {} - self.streams_by_client_id = {} - self.streams_by_server_id = {} + self.streams_by_id = {} def get_or_create_stream( - self, stream_id: int, from_client: bool + self, stream_id: int ) -> layer.CommandGenerator[QuicRelayStream]: - streams_by_id = ( - self.streams_by_client_id if from_client else self.streams_by_server_id - ) - if stream_id in streams_by_id: - return streams_by_id[stream_id] + if stream_id in self.streams_by_id: + return self.streams_by_id[stream_id] else: - # reserve the peer stream id - is_unidirectional = stream_is_unidirectional(stream_id) - peer_quic = self.quic_server if from_client else self.quic_client - assert peer_quic - peer_stream_id = peer_quic.get_next_available_stream_id(is_unidirectional) - - # create the instance and make sure unidirectional streams are marked as ended - stream = QuicRelayStream( - flow=tcp.TCPFlow( - self.context.client, - self.context.server, - live=True, - ), - client_ended=is_unidirectional and not from_client, - server_ended=is_unidirectional and from_client, - client_id=stream_id if from_client else peer_stream_id, - server_id=peer_stream_id if from_client else stream_id, - ) - # register the stream and start the flow + stream = QuicRelayStream(self.context, stream_id) self.streams_by_flow[stream.flow] = stream - self.streams_by_client_id[stream.client_id] = stream - self.streams_by_server_id[stream.server_id] = stream + self.streams_by_id[stream.stream_id] = stream yield tcp_layer.TcpStartHook(stream.flow) return stream @@ -305,19 +293,13 @@ def handle_quic_event( self, event: quic_events.QuicEvent, from_client: bool, - allow_buffering: bool, ) -> layer.CommandGenerator[None]: # buffer events if the peer is not ready yet peer_quic = self.quic_server if from_client else self.quic_client if peer_quic is None: - if not allow_buffering: - raise AssertionError( - f"Cannot buffer event from {'client' if from_client else 'server'}." - ) - if from_client: - self.buffer_from_client.append(event) - else: - self.buffer_from_server.append(event) + buffer = self.buffer_from_client if from_client else self.buffer_from_server + assert buffer is not None + buffer.append(event) return peer_connection = self.context.server if from_client else self.context.client @@ -332,7 +314,7 @@ def handle_quic_event( elif isinstance(event, quic_events.StreamDataReceived): # ignore data received from already ended streams - stream = yield from self.get_or_create_stream(event.stream_id, from_client) + stream = yield from self.get_or_create_stream(event.stream_id) if stream.has_ended(from_client): return @@ -341,33 +323,30 @@ def handle_quic_event( stream.flow.messages.append(message) yield tcp_layer.TcpMessageHook(stream.flow) peer_quic.send_stream_data( - stream_id=stream.stream_id(not from_client), - data=message.content, - end_stream=event.end_stream, + stream.stream_id, + message.content, + event.end_stream, ) # mark the stream as ended if needed if event.end_stream: - if from_client: - stream.client_ended = True - else: - stream.server_ended = True + stream.mark_ended(from_client) - # end the flow if both legs ended - if stream.client_ended and stream.server_ended: + # end the flow if both sides ended + if stream.has_ended(not from_client): yield tcp_layer.TcpEndHook(stream.flow) stream.flow.live = False elif isinstance(event, quic_events.StreamReset): # ignore resets from already ended streams - stream = yield from self.get_or_create_stream(event.stream_id, from_client) + stream = yield from self.get_or_create_stream(event.stream_id) if stream.has_ended(from_client): return # forward resets to peer streams and report them to addons peer_quic.reset_stream( - stream_id=stream.stream_id(not from_client), - error_code=event.error_code, + stream.stream_id, + event.error_code, ) stream.flow.error = mitm_flow.Error(error_code_to_str(event.error_code)) yield tcp_layer.TcpErrorHook(stream.flow) @@ -447,53 +426,45 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: self._handle_event = self.state_done elif isinstance(event, QuicStart): - # QUIC connection has been established, store it and flush buffered events + # QUIC connection has been established, store it and get the peer's buffer if event.connection is self.context.client: assert self.quic_client is None self.quic_client = event.quic - for quic_event in self.buffer_from_server: - yield from self.handle_quic_event( - quic_event, - from_client=False, - allow_buffering=False, - ) + from_client = False + buffer = self.buffer_from_server + self.buffer_from_server = None elif event.connection is self.context.server: assert self.quic_server is None self.quic_server = event.quic - for quic_event in self.buffer_from_client: - yield from self.handle_quic_event( - quic_event, - from_client=True, - allow_buffering=False, - ) + from_client = True + buffer = self.buffer_from_client + self.buffer_from_client = None else: raise AssertionError( f"Connection {event.connection} not associated with layer." ) + # flush the buffer + for quic_event in buffer: + yield from self.handle_quic_event(quic_event, from_client) + elif isinstance(event, tcp_layer.TcpMessageInjected): # translate injected messages into QUIC events assert isinstance(event.flow, tcp.TCPFlow) if event.flow is self.flow: yield from self.handle_quic_event( - quic_events.DatagramFrameReceived(event.message.content), + quic_events.DatagramFrameReceived(data=event.message.content), event.message.from_client, - allow_buffering=True, ) elif event.flow in self.streams_by_flow: stream = self.streams_by_flow[event.flow] yield from self.handle_quic_event( quic_events.StreamDataReceived( - stream_id=( - stream.client_id - if event.message.from_client - else stream.server_id - ), + stream_id=stream.stream_id, data=event.message.content, end_stream=False, ), event.message.from_client, - allow_buffering=True, ) else: raise AssertionError( @@ -505,7 +476,6 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.handle_quic_event( event.event, from_client=event.connection is self.context.client, - allow_buffering=True, ) else: @@ -753,7 +723,6 @@ def open_connection_end(self, reply: Optional[str]) -> layer.CommandGenerator[bo def process_events(self) -> layer.CommandGenerator[None]: assert self.quic is not None - assert self.tls is not None # handle all buffered aioquic connection events event = self.quic.next_event() @@ -832,7 +801,7 @@ def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: isinstance(event, events.ConnectionClosed) and event.connection is self.conn ): # there is no point in calling quic.close, as it cannot send packets anymore - # just set the new connection state and ensure there is exists a close event + # just set the new connection state and ensure there exists a close event self.quic._set_state(QuicConnectionState.TERMINATED) if self.quic._close_event is None: self.quic._close_event = quic_events.ConnectionTerminated( diff --git a/test/mitmproxy/net/test_udp.py b/test/mitmproxy/net/test_udp.py index c7f0ef7338..0180550e5e 100644 --- a/test/mitmproxy/net/test_udp.py +++ b/test/mitmproxy/net/test_udp.py @@ -16,4 +16,4 @@ async def test_reader(): with pytest.raises(AssertionError): await reader.read(MAX_DATAGRAM_SIZE - 1) assert await reader.read(65535) == (b"Second message", addr) - assert not await reader.read(65535) + assert not (await reader.read(65535))[0] From f92a20af4cda9d1b01102e9177eda5b53f65e818 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 28 Jun 2022 17:16:24 +0200 Subject: [PATCH 037/695] [quic] H2<->H3 stream ID mapping --- mitmproxy/proxy/layers/http/_http3.py | 261 +++++++++++++++----------- 1 file changed, 155 insertions(+), 106 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 7ac4522f58..29cc6e3585 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,16 +1,16 @@ from abc import abstractmethod import time -from typing import Optional, Union +from typing import Dict, Optional, Union from aioquic.h3.connection import ( H3Connection, - FrameUnexpected, ErrorCode as H3ErrorCode, + FrameUnexpected as H3FrameUnexpected, HeadersState as H3HeadersState, ) from aioquic.h3 import events as h3_events from aioquic.quic import events as quic_events -from aioquic.quic.connection import QuicConnection +from aioquic.quic.connection import QuicConnection, stream_is_client_initiated from mitmproxy import http, version from mitmproxy.net.http import status_codes @@ -58,17 +58,26 @@ class Http3Connection(HttpConnection): ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - self._handle_event = self.state_wait_for_quic - yield from () + @abstractmethod + def parse_headers( + self, event: h3_events.HeadersReceived + ) -> Union[RequestHeaders, ResponseHeaders]: + pass # pragma: no cover - @expect(QuicStart) - def state_wait_for_quic(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, QuicStart) - self.quic = event.quic - self.h3_conn = H3Connection(self.quic, enable_webtransport=False) - self._handle_event = self.state_ready + def postprocess_outgoing_event(self, event: HttpEvent) -> HttpEvent: + return event + + def preprocess_incoming_event(self, event: HttpEvent) -> HttpEvent: + return event + + @abstractmethod + def send_protocol_error( + self, event: Union[RequestProtocolError, ResponseProtocolError] + ) -> None: + pass # pragma: no cover + + @expect(HttpEvent, QuicConnectionEvent, events.ConnectionClosed) + def state_done(self, _) -> layer.CommandGenerator[None]: yield from () @expect(HttpEvent, QuicConnectionEvent, events.ConnectionClosed) @@ -78,6 +87,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # send mitmproxy HTTP events over the H3 connection if isinstance(event, HttpEvent): + event = self.preprocess_incoming_event(event) try: if isinstance(event, (RequestData, ResponseData)): @@ -112,87 +122,106 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: stream_id=event.stream_id, data=b"", end_stream=True ) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): - self.protocol_error(event) + self.send_protocol_error(event) else: raise AssertionError(f"Unexpected event: {event!r}") - except FrameUnexpected: + except H3FrameUnexpected: # Http2Connection also ignores HttpEvents that violate the current stream state - return + pass - # transmit buffered data and re-arm timer - yield QuicTransmit(self.conn, self.quic) + else: + # transmit buffered data and re-arm timer + yield QuicTransmit(self.conn, self.quic) # handle events from the underlying QUIC connection elif isinstance(event, QuicConnectionEvent): # report abrupt stream resets - if isinstance(event, quic_events.StreamReset): - if event.stream_id in self.h3_conn._stream: - stream = self.h3_conn._stream[event.stream_id] - if not stream.ended: - # mark the receiving part of the stream as ended - # (H3Connection alas doesn't handle StreamReset) - stream.ended = True - - # report the protocol error (doing the same error code mingling as H2) - code = ( - status_codes.CLIENT_CLOSED_REQUEST - if event.error_code == H3ErrorCode.H3_REQUEST_CANCELLED - else self.ReceiveProtocolError.code - ) - yield ReceiveHttp( - self.ReceiveProtocolError( - stream_id=event.stream_id, - message=f"stream reset by client ({error_code_to_str(event.error_code)})", - code=code, - ) + if ( + isinstance(event, quic_events.StreamReset) + and stream_is_client_initiated(event.stream_id) + and event.stream_id in self.h3_conn._stream + and not self.h3_conn._stream[event.stream_id].ended + ): + # mark the receiving part of the stream as ended + # (H3Connection alas doesn't handle StreamReset) + self.h3_conn._stream[event.stream_id] = True + + # report the protocol error (doing the same error code mingling as H2) + code = ( + status_codes.CLIENT_CLOSED_REQUEST + if event.error_code == H3ErrorCode.H3_REQUEST_CANCELLED + else self.ReceiveProtocolError.code + ) + yield ReceiveHttp( + self.postprocess_outgoing_event( + self.ReceiveProtocolError( + stream_id=event.stream_id, + message=f"stream reset by client ({error_code_to_str(event.error_code)})", + code=code, ) + ) + ) # forward QUIC events to the H3 connection for h3_event in self.h3_conn.handle_event(event.event): # report received data - if isinstance(h3_event, h3_events.DataReceived): + if isinstance( + h3_event, h3_events.DataReceived + ) and stream_is_client_initiated(h3_event.stream_id): yield ReceiveHttp( - self.ReceiveData( - stream_id=h3_event.stream_id, data=h3_event.data + self.postprocess_outgoing_event( + self.ReceiveData( + stream_id=h3_event.stream_id, data=h3_event.data + ) ) ) if h3_event.stream_ended: yield ReceiveHttp( - self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) + self.postprocess_outgoing_event( + self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) + ) ) # report headers and trailers - elif isinstance(h3_event, h3_events.HeadersReceived): + elif isinstance( + h3_event, h3_events.HeadersReceived + ) and stream_is_client_initiated(h3_event.stream_id): if ( self.h3_conn._stream[h3_event.stream_id].headers_recv_state is H3HeadersState.AFTER_TRAILERS ): yield ReceiveHttp( - self.ReceiveTrailers( - stream_id=h3_event.stream_id, - trailers=http.Headers(h3_event.headers), + self.postprocess_outgoing_event( + self.ReceiveTrailers( + stream_id=h3_event.stream_id, + trailers=http.Headers(h3_event.headers), + ) ) ) else: try: - receive_event = self.headers_received(h3_event) + receive_event = self.parse_headers(h3_event) except ValueError as e: - # this will result in a ConnectionTerminated event + # this will result in a ConnectionClosed event self.quic.close( error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, reason_phrase=f"Invalid HTTP/3 request headers: {e}", ) yield QuicTransmit(self.conn, self.quic) else: - yield ReceiveHttp(receive_event) + yield ReceiveHttp( + self.postprocess_outgoing_event(receive_event) + ) # always report an EndOfMessage if the stream has ended if h3_event.stream_ended: yield ReceiveHttp( - self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) + self.postprocess_outgoing_event( + self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) + ) ) # we don't support push, web transport, etc. @@ -202,14 +231,16 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): for stream in self.h3_conn._stream.values(): - if self.quic._stream_can_receive(stream.stream_id) and not stream.ended: + if stream_is_client_initiated(stream.stream_id) and not stream.ended: close_event = self.quic._close_event assert close_event is not None yield ReceiveHttp( - self.ReceiveProtocolError( - stream_id=stream.stream_id, - message=close_event.reason_phrase, - code=close_event.error_code, + self.postprocess_outgoing_event( + self.ReceiveProtocolError( + stream_id=stream.stream_id, + message=close_event.reason_phrase, + code=close_event.error_code, + ) ) ) self._handle_event = self.state_done @@ -217,21 +248,19 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unexpected event: {event!r}") - @expect(HttpEvent, QuicConnectionEvent, events.ConnectionClosed) - def state_done(self, _) -> layer.CommandGenerator[None]: + @expect(events.Start) + def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, events.Start) + self._handle_event = self.state_wait_for_quic yield from () - @abstractmethod - def protocol_error( - self, event: Union[RequestProtocolError, ResponseProtocolError] - ) -> None: - pass # pragma: no cover - - @abstractmethod - def headers_received( - self, event: h3_events.HeadersReceived - ) -> Union[RequestHeaders, ResponseHeaders]: - pass # pragma: no cover + @expect(QuicStart) + def state_wait_for_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, QuicStart) + self.quic = event.quic + self.h3_conn = H3Connection(self.quic, enable_webtransport=False) + self._handle_event = self.state_ready + yield from () _handle_event = state_start @@ -245,31 +274,7 @@ class Http3Server(Http3Connection): def __init__(self, context: context.Context): super().__init__(context, context.client) - def protocol_error( - self, event: Union[RequestProtocolError, ResponseProtocolError] - ) -> None: - assert self.h3_conn is not None - assert isinstance(event, ResponseProtocolError) - - # same as HTTP/2 - code = event.code - if code != status_codes.CLIENT_CLOSED_REQUEST: - code = status_codes.INTERNAL_SERVER_ERROR - self.h3_conn.send_headers( - stream_id=event.stream_id, - headers=[ - (b":status", b"%d" % code), - (b"server", version.MITMPROXY.encode()), - (b"content-type", b"text/html"), - ], - ) - self.h3_conn.send_data( - stream_id=event.stream_id, - data=format_error(code, event.message), - end_stream=True, - ) - - def headers_received( + def parse_headers( self, event: h3_events.HeadersReceived ) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 @@ -300,6 +305,30 @@ def headers_received( stream_id=event.stream_id, request=request, end_stream=event.stream_ended ) + def send_protocol_error( + self, event: Union[RequestProtocolError, ResponseProtocolError] + ) -> None: + assert self.h3_conn is not None + assert isinstance(event, ResponseProtocolError) + + # same as HTTP/2 + code = event.code + if code != status_codes.CLIENT_CLOSED_REQUEST: + code = status_codes.INTERNAL_SERVER_ERROR + self.h3_conn.send_headers( + stream_id=event.stream_id, + headers=[ + (b":status", b"%d" % code), + (b"server", version.MITMPROXY.encode()), + (b"content-type", b"text/html"), + ], + ) + self.h3_conn.send_data( + stream_id=event.stream_id, + data=format_error(code, event.message), + end_stream=True, + ) + class Http3Client(Http3Connection): ReceiveData = ResponseData @@ -309,20 +338,10 @@ class Http3Client(Http3Connection): def __init__(self, context: context.Context): super().__init__(context, context.server) + self._event_to_quic: Dict[int, int] = {} + self._quic_to_event: Dict[int, int] = {} - def protocol_error( - self, event: Union[RequestProtocolError, ResponseProtocolError] - ) -> None: - assert isinstance(event, RequestProtocolError) - assert self.quic is not None - - # same as HTTP/2 - code = event.code - if code != H3ErrorCode.H3_REQUEST_CANCELLED: - code = H3ErrorCode.H3_INTERNAL_ERROR - self.quic.reset_stream(stream_id=event.stream_id, error_code=code) - - def headers_received( + def parse_headers( self, event: h3_events.HeadersReceived ) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 @@ -341,6 +360,36 @@ def headers_received( stream_id=event.stream_id, response=response, end_stream=event.stream_ended ) + def postprocess_outgoing_event(self, event: HttpEvent) -> HttpEvent: + event.stream_id = self._quic_to_event[event.stream_id] + return event + + def preprocess_incoming_event(self, event: HttpEvent) -> HttpEvent: + if event.stream_id in self._event_to_quic: + event.stream_id = self._event_to_quic[event.stream_id] + else: + # QUIC and HTTP/3 would actually allow for direct stream ID mapping, but since we want + # to support H2<->H3, we need to translate IDs. + # NOTE: We always create bidirectional streams, as we can't safely infer unidirectionality. + assert self.quic is not None + stream_id = self.quic.get_next_available_stream_id() + self._event_to_quic[event.stream_id] = stream_id + self._quic_to_event[stream_id] = event.stream_id + event.stream_id = stream_id + return event + + def send_protocol_error( + self, event: Union[RequestProtocolError, ResponseProtocolError] + ) -> None: + assert isinstance(event, RequestProtocolError) + assert self.quic is not None + + # same as HTTP/2 + code = event.code + if code != H3ErrorCode.H3_REQUEST_CANCELLED: + code = H3ErrorCode.H3_INTERNAL_ERROR + self.quic.reset_stream(stream_id=event.stream_id, error_code=code) + __all__ = [ "Http3Client", From d94345b2f32aa04ed57dadac99c1ce9724ec15b2 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 29 Jun 2022 11:48:24 +0200 Subject: [PATCH 038/695] [quic] properly forward disconnect reason --- mitmproxy/proxy/layers/quic.py | 62 ++++++++++++++++------------------ 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 83d32ef819..15dab790e0 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,7 +1,7 @@ import asyncio from dataclasses import dataclass, field from ssl import VerifyMode -from typing import Callable, Dict, List, Literal, Optional, Set, Tuple, Union +from typing import Callable, Dict, List, Optional, Set, Tuple, Union from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import ErrorCode as H3ErrorCode @@ -394,6 +394,17 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: close_event = local_quic._close_event assert close_event is not None + # close the peer as well (needs to be before hooks) + if peer_quic is not None: + peer_quic.close( + close_event.error_code, + close_event.frame_type, + close_event.reason_phrase, + ) + yield QuicTransmit(peer_conn, peer_quic) + else: + yield commands.CloseConnection(peer_conn) + # report the termination as error to all non-ended streams for flow in self.streams_by_flow: if flow.live: @@ -413,16 +424,6 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: yield tcp_layer.TcpErrorHook(flow) self.flow.live = False - # close the peer as well - if peer_quic is not None: - peer_quic.close( - close_event.error_code, - close_event.frame_type, - close_event.reason_phrase, - ) - yield QuicTransmit(peer_conn, peer_quic) - else: - yield commands.CloseConnection(peer_conn) self._handle_event = self.state_done elif isinstance(event, QuicStart): @@ -445,6 +446,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: ) # flush the buffer + assert buffer is not None for quic_event in buffer: yield from self.handle_quic_event(quic_event, from_client) @@ -584,9 +586,7 @@ def create_quic(self) -> layer.CommandGenerator[bool]: return True def destroy_quic( - self, - reason: str, - level: Literal["error", "warn", "info", "alert", "debug"], + self, event: quic_events.ConnectionTerminated ) -> layer.CommandGenerator[None]: # ensure QUIC has been properly shut down assert self.quic is not None @@ -594,6 +594,7 @@ def destroy_quic( assert self.quic._state is QuicConnectionState.TERMINATED # report as TLS failure if the termination happened before the handshake + reason = event.reason_phrase or error_code_to_str(event.error_code) if not self.conn.tls_established: self.conn.error = reason tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) @@ -615,15 +616,17 @@ def destroy_quic( # record an entry in the log yield commands.Log( - f"{self.conn}: QUIC connection destroyed: {reason}", level=level + f"{self.conn}: QUIC connection destroyed: {reason}", + level="info" if is_success_error_code(event.error_code) else "warn", ) def establish_quic( self, event: quic_events.HandshakeCompleted ) -> layer.CommandGenerator[None]: - # must only be called if QUIC is initialized + # must only be called if QUIC is initialized and not established assert self.quic is not None assert self.tls is not None + assert not self.conn.tls_established # concatenate all peer certificates all_certs: List[x509.Certificate] = [] @@ -737,12 +740,7 @@ def process_events(self) -> layer.CommandGenerator[None]: elif isinstance(event, quic_events.ConnectionTerminated): # shutdown and close the connection - yield from self.destroy_quic( - event.reason_phrase or error_code_to_str(event.error_code), - level=( - "info" if is_success_error_code(event.error_code) else "warn" - ), - ) + yield from self.destroy_quic(event) yield commands.CloseConnection(self.conn) # we don't handle any further events, nor do/can we transmit data, so exit @@ -803,22 +801,20 @@ def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: # there is no point in calling quic.close, as it cannot send packets anymore # just set the new connection state and ensure there exists a close event self.quic._set_state(QuicConnectionState.TERMINATED) - if self.quic._close_event is None: - self.quic._close_event = quic_events.ConnectionTerminated( + close_event = self.quic._close_event + if close_event is None: + close_event = quic_events.ConnectionTerminated( error_code=QuicErrorCode.APPLICATION_ERROR, frame_type=None, - reason_phrase="Peer UDP connection closed or timed out.", + reason_phrase="UDP connection closed or timed out.", ) + self.quic._close_event = close_event # shutdown QUIC and handle the ConnectionClosed event - reason = self.quic._close_event.reason_phrase or error_code_to_str( - self.quic._close_event.error_code - ) - yield from self.destroy_quic( - reason, - level="info", - ) - if not (yield from self.open_connection_end(reason)): + yield from self.destroy_quic(close_event) + if not ( + yield from self.open_connection_end("QUIC could not be established") + ): # connection was opened before QUIC layer, report to the child layer yield from self.event_to_child(event) From eff13002e8e11a30db2c7d763d6995f03944b00e Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 17 Aug 2022 02:42:51 +0200 Subject: [PATCH 039/695] merge multi-server --- mitmproxy/addons/next_layer.py | 24 ++++- mitmproxy/net/server_spec.py | 7 +- mitmproxy/proxy/layers/quic.py | 136 ++++++++++++++----------- mitmproxy/proxy/mode_servers.py | 82 +++++++++++++++ mitmproxy/proxy/mode_specs.py | 48 ++++++++- test/mitmproxy/net/test_server_spec.py | 3 - 6 files changed, 229 insertions(+), 71 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 4496895b7d..00831789b1 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -22,7 +22,7 @@ from mitmproxy.net.tls import is_tls_record_magic from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy import context, layer, layers -from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import modes, quic from mitmproxy.proxy.layers.tls import HTTP_ALPNS, parse_client_hello LayerCls = type[layer.Layer] @@ -118,6 +118,28 @@ def _next_layer( self, context: context.Context, data_client: bytes, data_server: bytes ) -> Optional[layer.Layer]: assert context.layers + + # handle QUIC connections + first_layer = context.layers[0] + if isinstance(first_layer, quic.QuicLayer): + if context.client.alpn is None: + return None # should never happen, as ask is called after handshake + if context.client.alpn == b"h3" or context.client.alpn.startswith(b"h3-"): + if first_layer.instance.mode.mode == "regular": + mode = HTTPMode.regular + elif first_layer.instance.mode.mode == "reverse": + mode = HTTPMode.transparent + elif first_layer.instance.mode.mode == "upstream": + mode = HTTPMode.upstream + else: + return None + return layers.HttpLayer(context, mode) + if context.server.address is None: + return None # not H3 and no predefined destination, nothing we can do + if isinstance(context.layers[1], quic.ServerQuicLayer): + return quic.QuicRelayLayer(context) # server layer already present + return quic.ServerQuicLayer(context, quic.QuicRelayLayer(context)) + if len(data_client) < 3 and not data_server: return None # not enough data yet to make a decision diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py index a53a9ba1c2..a292e519dd 100644 --- a/mitmproxy/net/server_spec.py +++ b/mitmproxy/net/server_spec.py @@ -3,12 +3,11 @@ """ import re from functools import cache -from typing import Literal from mitmproxy.net import check ServerSpec = tuple[ - Literal["http", "https", "tcp", "tls", "dns"], + str, tuple[str, int] ] @@ -45,8 +44,6 @@ def parse(server_spec: str, default_scheme: str) -> ServerSpec: scheme = m.group("scheme") else: scheme = default_scheme - if scheme not in ("tcp", "tls", "dns", "dtls", "http", "https"): - raise ValueError(f"Invalid server scheme: {scheme}") host = m.group("host") # IPv6 brackets @@ -69,4 +66,4 @@ def parse(server_spec: str, default_scheme: str) -> ServerSpec: if not check.is_valid_port(port): raise ValueError(f"Invalid port: {port}") - return scheme, (host, port) # type: ignore + return scheme, (host, port) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 15dab790e0..5f1e5d041e 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,7 +1,7 @@ +from __future__ import annotations import asyncio from dataclasses import dataclass, field from ssl import VerifyMode -from typing import Callable, Dict, List, Optional, Set, Tuple, Union from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import ErrorCode as H3ErrorCode @@ -21,7 +21,7 @@ from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from mitmproxy import certs, connection, flow as mitm_flow, tcp from mitmproxy.net import tls -from mitmproxy.proxy import commands, context, events, layer, layers +from mitmproxy.proxy import commands, context, events, layer, layers, mode_servers from mitmproxy.proxy.layers import tcp as tcp_layer from mitmproxy.proxy.utils import expect from mitmproxy.tls import ClientHello, ClientHelloData, TlsData @@ -33,21 +33,21 @@ class QuicTlsSettings: Settings necessary to establish QUIC's TLS context. """ - certificate: Optional[x509.Certificate] = None + certificate: x509.Certificate | None = None """The certificate to use for the connection.""" - certificate_chain: List[x509.Certificate] = field(default_factory=list) + certificate_chain: list[x509.Certificate] = field(default_factory=list) """A list of additional certificates to send to the peer.""" - certificate_private_key: Optional[ - Union[dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey] - ] = None + certificate_private_key: dsa.DSAPrivateKey | ec.EllipticCurvePrivateKey | rsa.RSAPrivateKey | None = ( + None + ) """The certificate's private key.""" - cipher_suites: Optional[List[CipherSuite]] = None + cipher_suites: list[CipherSuite] | None = None """An optional list of allowed/advertised cipher suites.""" - ca_path: Optional[str] = None + ca_path: str | None = None """An optional path to a directory that contains the necessary information to verify the peer certificate.""" - ca_file: Optional[str] = None + ca_file: str | None = None """An optional path to a PEM file that will be used to verify the peer certificate.""" - verify_mode: Optional[VerifyMode] = None + verify_mode: VerifyMode | None = None """An optional flag that specifies how/if the peer's certificate should be validated.""" @@ -57,7 +57,7 @@ class QuicTlsData(TlsData): Event data for `quic_tls_start_client` and `quic_tls_start_server` event hooks. """ - settings: Optional[QuicTlsSettings] = None + settings: QuicTlsSettings | None = None """ The associated `QuicTlsSettings` object. This will be set by an addon in the `quic_tls_start_*` event hooks. @@ -169,7 +169,7 @@ class QuicClientHello(Exception): data: bytes -def pull_client_hello_and_connection_id(data: bytes) -> Tuple[ClientHello, bytes]: +def pull_client_hello_and_connection_id(data: bytes) -> tuple[ClientHello, bytes]: """Helper function that parses a client hello packet.""" # ensure the first packet is indeed the initial one @@ -256,13 +256,13 @@ class QuicRelayLayer(layer.Layer): # NOTE: for now we're (ab)using the TCPFlow until https://github.com/mitmproxy/mitmproxy/pull/5414 is resolved - buffer_from_client: Optional[List[quic_events.QuicEvent]] - buffer_from_server: Optional[List[quic_events.QuicEvent]] + buffer_from_client: list[quic_events.QuicEvent] | None + buffer_from_server: list[quic_events.QuicEvent] | None flow: tcp.TCPFlow # used for datagrams and to signal general connection issues - quic_client: Optional[QuicConnection] = None - quic_server: Optional[QuicConnection] = None - streams_by_flow: Dict[tcp.TCPFlow, QuicRelayStream] - streams_by_id: Dict[int, QuicRelayStream] + quic_client: QuicConnection | None = None + quic_server: QuicConnection | None = None + streams_by_flow: dict[tcp.TCPFlow, QuicRelayStream] + streams_by_id: dict[int, QuicRelayStream] def __init__(self, context: context.Context) -> None: super().__init__(context) @@ -498,11 +498,9 @@ def state_done(self, _) -> layer.CommandGenerator[None]: class _QuicLayer(layer.Layer): child_layer: layer.Layer conn: connection.Connection - issue_connection_id_callback: Optional[Callable[[bytes], None]] = None - original_destination_connection_id: Optional[bytes] = None - quic: Optional[QuicConnection] = None - retire_connection_id_callback: Optional[Callable[[bytes], None]] = None - tls: Optional[QuicTlsSettings] = None + original_destination_connection_id: bytes | None = None + quic: QuicConnection | None = None + tls: QuicTlsSettings | None = None def __init__( self, @@ -513,11 +511,11 @@ def __init__( self.child_layer = layer.NextLayer(context) self.conn = conn self._loop = asyncio.get_event_loop() - self._pending_open_command: Optional[commands.OpenConnection] = None - self._request_wakeup_command_and_timer: Optional[ - Tuple[commands.RequestWakeup, float] - ] = None - self._obsolete_wakeup_commands: Set[commands.RequestWakeup] = set() + self._pending_open_command: commands.OpenConnection | None = None + self._request_wakeup_command_and_timer: tuple[ + commands.RequestWakeup, float + ] | None = None + self._obsolete_wakeup_commands: set[commands.RequestWakeup] = set() self.conn.tls = True def build_configuration(self) -> QuicConfiguration: @@ -578,8 +576,7 @@ def create_quic(self) -> layer.CommandGenerator[bool]: self._handle_event = self.state_has_quic # issue the host connection ID right away - if self.issue_connection_id_callback is not None: - self.issue_connection_id_callback(self.quic.host_cid) + self.issue_connection_id(self.quic.host_cid) # record an entry in the log yield commands.Log(f"{self.conn}: QUIC connection created.", level="info") @@ -629,7 +626,7 @@ def establish_quic( assert not self.conn.tls_established # concatenate all peer certificates - all_certs: List[x509.Certificate] = [] + all_certs: list[x509.Certificate] = [] if self.quic.tls._peer_certificate is not None: all_certs.append(self.quic.tls._peer_certificate) if self.quic.tls._peer_certificate_chain is not None: @@ -690,6 +687,9 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: # return other commands yield command + def issue_connection_id(self, connection_id: bytes) -> None: + pass + def open_connection_begin( self, command: commands.OpenConnection ) -> layer.CommandGenerator[None]: @@ -714,7 +714,7 @@ def open_connection_begin( self._pending_open_command = None yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) - def open_connection_end(self, reply: Optional[str]) -> layer.CommandGenerator[bool]: + def open_connection_end(self, reply: str | None) -> layer.CommandGenerator[bool]: if self._pending_open_command is None: return False @@ -731,12 +731,10 @@ def process_events(self) -> layer.CommandGenerator[None]: event = self.quic.next_event() while event is not None: if isinstance(event, quic_events.ConnectionIdIssued): - if self.issue_connection_id_callback is not None: - self.issue_connection_id_callback(event.connection_id) + self.issue_connection_id(event.connection_id) elif isinstance(event, quic_events.ConnectionIdRetired): - if self.retire_connection_id_callback is not None: - self.retire_connection_id_callback(event.connection_id) + self.retire_connection_id(event.connection_id) elif isinstance(event, quic_events.ConnectionTerminated): # shutdown and close the connection @@ -781,6 +779,9 @@ def process_events(self) -> layer.CommandGenerator[None]: # transmit buffered data and re-arm timer yield from self.transmit() + def retire_connection_id(self, connection_id: bytes) -> None: + pass + def start(self) -> layer.CommandGenerator[None]: yield from self.event_to_child(events.Start()) @@ -901,7 +902,7 @@ class ServerQuicLayer(_QuicLayer): """ def __init__( - self, context: context.Context, child_layer: Optional[layer.Layer] = None + self, context: context.Context, child_layer: layer.Layer | None = None ) -> None: super().__init__(context, context.server) if child_layer is not None: @@ -913,15 +914,38 @@ class ClientQuicLayer(_QuicLayer): This layer establishes QUIC on a single client connection. """ + parent_layer: QuicLayer wait_for_upstream: bool def __init__( self, - context: context.Context, + parent_layer: QuicLayer, + connection_id: bytes, wait_for_upstream: bool, ) -> None: - super().__init__(context, context.client) + super().__init__(parent_layer.context, parent_layer.context.client) + self.original_destination_connection_id = connection_id + self.parent_layer = parent_layer self.wait_for_upstream = wait_for_upstream + self._handler = parent_layer.instance.manager.connections[ + parent_layer.fully_qualify_connection_id(connection_id) + ] + parent_layer.connection_ids.add(connection_id) + + def issue_connection_id(self, connection_id: bytes) -> None: + # add the connection id to the manager connections + fqcid = self.parent_layer.fully_qualify_connection_id(connection_id) + if fqcid not in self.parent_layer.instance.manager.connections: + self.parent_layer.instance.manager.connections[fqcid] = self._handler + self.parent_layer.connection_ids.add(connection_id) + + def retire_connection_id(self, connection_id: bytes) -> None: + # remove the connection id from the manager connections + if connection_id in self.parent_layer.connection_ids: + del self.parent_layer.instance.manager.connections[ + self.parent_layer.fully_qualify_connection_id(connection_id) + ] + self.parent_layer.connection_ids.remove(connection_id) def start(self) -> layer.CommandGenerator[None]: yield from super().start() @@ -951,26 +975,24 @@ class QuicLayer(layer.Layer): Entry layer for QUIC proxy server. """ + instance: mode_servers.QuicInstance + connection_ids: set[bytes] + def __init__( self, context: context.Context, - issue_cid: Callable[[bytes], None], - retire_cid: Callable[[bytes], None], + instance: mode_servers.QuicInstance, ) -> None: super().__init__(context) - self._issue_cid = issue_cid - self._retire_cid = retire_cid + self.instance = instance + self.connection_ids = set() self.context.client.tls = True self.context.server.tls = True - def build_client_layer( - self, connection_id: bytes, wait_for_upstream: bool - ) -> ClientQuicLayer: - layer = ClientQuicLayer(self.context, wait_for_upstream) - layer.original_destination_connection_id = connection_id - layer.issue_connection_id_callback = self._issue_cid - layer.retire_connection_id_callback = self._retire_cid - return layer + def fully_qualify_connection_id(self, connection_id: bytes) -> tuple: + return self.instance.fully_qualify_connection_id( + connection_id, self.context.client.sockname + ) @expect(events.DataReceived, events.ConnectionClosed) def state_done(self, _) -> layer.CommandGenerator[None]: @@ -1017,16 +1039,14 @@ def state_wait_for_hello(self, event: events.Event) -> layer.CommandGenerator[No # contact the upstream server first elif hook_data.establish_server_tls_first: next_layer = ServerQuicLayer(self.context) - next_layer.child_layer = self.build_client_layer( - connection_id, - wait_for_upstream=True, + next_layer.child_layer = ClientQuicLayer( + self, connection_id, wait_for_upstream=True ) # perform the client handshake immediately else: - next_layer = self.build_client_layer( - connection_id, - wait_for_upstream=False, + next_layer = ClientQuicLayer( + self, connection_id, wait_for_upstream=False ) # replace this layer and start the next one diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 7f38084a88..858e63c9bc 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -19,6 +19,14 @@ from contextlib import contextmanager from typing import ClassVar, Generic, TypeVar, cast, get_args +from aioquic.buffer import Buffer as QuicBuffer +from aioquic.quic.packet import ( + PACKET_TYPE_INITIAL, + QuicProtocolVersion, + encode_quic_version_negotiation, + pull_quic_header, +) + from mitmproxy import ctx, flow, log from mitmproxy.connection import Address from mitmproxy.master import Master @@ -340,3 +348,77 @@ def make_connection_id( local_addr: Address, ) -> tuple | None: return ("dtls", remote_addr, local_addr) + + +class QuicInstance(UdpServerInstance[mode_specs.QuicMode]): + + def fully_qualify_connection_id(self, connection_id: bytes, local_addr: Address) -> tuple: + return ("quic", connection_id, local_addr) + + def make_connection_id( + self, + transport: asyncio.DatagramTransport, + data: bytes, + remote_addr: Address, + local_addr: Address, + ) -> tuple | None: + # largely taken from aioquic's own asyncio server code + buffer = QuicBuffer(data=data) + try: + header = pull_quic_header( + buffer, host_cid_length=ctx.options.quic_connection_id_length + ) + except ValueError: + ctx.log.info( + f"Invalid QUIC datagram received from {human.format_address(remote_addr)}." + ) + return None + + # negotiate version, support all versions known to aioquic + supported_versions = ( + version.value + for version in QuicProtocolVersion + if version is not QuicProtocolVersion.NEGOTIATION + ) + if header.version is not None and header.version not in supported_versions: + transport.sendto( + encode_quic_version_negotiation( + source_cid=header.destination_cid, + destination_cid=header.source_cid, + supported_versions=supported_versions, + ), + remote_addr, + ) + return None + + # check if a new connection is possible + connection_id = self.fully_qualify_connection_id(header.destination_cid, local_addr) + if connection_id not in self.manager.connections: + if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: + ctx.log.info( + f"QUIC packet received from {human.format_address(remote_addr)} with an unknown connection id." + ) + return None + return connection_id + + def make_top_layer(self, context: Context) -> Layer: + # determine the server settings + if self.mode.mode == "reverse": + assert self.mode.address is not None + context.server.address = self.mode.address + if not ctx.options.keep_host_header: + context.server.sni = self.mode.address[0] + context.server.transport_protocol = "udp" + return layers.QuicLayer(context, self) + + async def handle_udp_connection(self, connection_id: tuple, handler: ProxyConnectionHandler) -> None: + layer = cast(layers.QuicLayer, handler.layer) + try: + return await super().handle_udp_connection(connection_id, handler) + finally: + # clear up all additional connection ids + for cid in layer.connection_ids: + fqcid = layer.fully_qualify_connection_id(cid) + if fqcid in self.manager.connections: + assert self.manager.connections[fqcid] is handler + del self.manager.connections[fqcid] diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index 61d2fd6664..cfba0001cc 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -185,7 +185,7 @@ def __post_init__(self) -> None: scheme, self.address = server_spec.parse(self.data, default_scheme="http") if scheme != "http" and scheme != "https": raise ValueError("invalid upstream proxy scheme") - self.scheme = scheme + self.scheme = scheme # type: ignore class ReverseMode(ProxyMode): @@ -198,7 +198,7 @@ def __post_init__(self) -> None: scheme, self.address = server_spec.parse(self.data, default_scheme="https") if scheme != "http" and scheme != "https" and scheme != "tcp" and scheme != "tls": raise ValueError("invalid reverse proxy scheme") - self.scheme = scheme + self.scheme = scheme # type: ignore class Socks5Mode(ProxyMode): @@ -226,7 +226,7 @@ def __post_init__(self) -> None: scheme, self.address = server_spec.parse(server, "dns") if scheme != "dns": raise ValueError("invalid dns scheme") - self.scheme = scheme + self.scheme = scheme # type: ignore @property def resolve_local(self) -> bool: @@ -247,4 +247,44 @@ def __post_init__(self) -> None: scheme, self.address = server_spec.parse(server, "dtls") if scheme != "dtls": raise ValueError("invalid dtls scheme") - self.scheme = scheme + self.scheme = scheme # type: ignore + + +class QuicMode(ProxyMode): + """ + QUIC modes: + - regular[:version] (default) + - upstream:[version://]host:port + - reverse:[version://]host:port + + Versions: Use the given HTTP version to connect to upstream. + - h1 + - h2 + - h3 (default) + + Example: + --mode quic:upstream:h2://192.168.1.1:8080@443 + Listens on port 443 for incoming H3 connections and forwards + request to upstream proxy 192.168.1.1 on port 8080 using H2. + """ + + default_port = 8085 + transport_protocol: ClassVar[Literal["tcp", "udp"]] = "udp" + scheme: Literal["h1", "h2", "h3"] = "h3" + mode: Literal["regular", "reverse", "upstream"] = "regular" + address: tuple[str, int] | None = None + + # noinspection PyDataclass + def __post_init__(self) -> None: + if self.data: + mode, _, additional = self.data.partition(":") + if mode == "reverse" or mode == "upstream": + scheme, self.address = server_spec.parse(additional, default_mode="h3") + elif mode == "regular": + scheme = additional + else: + raise ValueError(f"Invalid QUIC mode: {mode}") + if scheme != "h3" and scheme != "h2" and scheme != "h1": + raise ValueError(f"Invalid QUIC scheme: {scheme}") + self.mode = mode # type: ignore + self.scheme = scheme # type: ignore diff --git a/test/mitmproxy/net/test_server_spec.py b/test/mitmproxy/net/test_server_spec.py index 1fe5590182..155fa7403f 100644 --- a/test/mitmproxy/net/test_server_spec.py +++ b/test/mitmproxy/net/test_server_spec.py @@ -24,9 +24,6 @@ def test_parse_err(): with pytest.raises(ValueError, match="Invalid server specification"): server_spec.parse(":", "https") - with pytest.raises(ValueError, match="Invalid server scheme"): - server_spec.parse("ftp://example.com", "https") - with pytest.raises(ValueError, match="Invalid hostname"): server_spec.parse("$$$", "https") From 0ac83aab00e653f307227405f056edffad40dbe4 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 21 Aug 2022 23:58:12 +0200 Subject: [PATCH 040/695] first steps on moving to new mode format --- mitmproxy/addons/next_layer.py | 3 +- mitmproxy/net/server_spec.py | 9 +- mitmproxy/net/udp.py | 16 +-- mitmproxy/proxy/commands.py | 17 +-- mitmproxy/proxy/events.py | 10 +- mitmproxy/proxy/layers/modes.py | 32 ++++-- mitmproxy/proxy/mode_servers.py | 134 ++++++---------------- mitmproxy/proxy/mode_specs.py | 89 +++++--------- mitmproxy/proxy/server.py | 23 +--- setup.py | 2 +- test/mitmproxy/addons/test_proxyserver.py | 6 +- test/mitmproxy/net/test_udp.py | 6 +- test/mitmproxy/proxy/test_commands.py | 1 - 13 files changed, 118 insertions(+), 230 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 00831789b1..2f96f8cc6a 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -122,8 +122,7 @@ def _next_layer( # handle QUIC connections first_layer = context.layers[0] if isinstance(first_layer, quic.QuicLayer): - if context.client.alpn is None: - return None # should never happen, as ask is called after handshake + assert context.client.alpn is not None # ask is called after handshake if context.client.alpn == b"h3" or context.client.alpn.startswith(b"h3-"): if first_layer.instance.mode.mode == "regular": mode = HTTPMode.regular diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py index a292e519dd..9f1479d28c 100644 --- a/mitmproxy/net/server_spec.py +++ b/mitmproxy/net/server_spec.py @@ -3,11 +3,12 @@ """ import re from functools import cache +from typing import Literal from mitmproxy.net import check ServerSpec = tuple[ - str, + Literal["", "http", "https", "tcp", "tls", "dtls", "quic", "dns"], tuple[str, int] ] @@ -25,7 +26,7 @@ @cache -def parse(server_spec: str, default_scheme: str) -> ServerSpec: +def parse(server_spec: str, default_scheme: str = "") -> ServerSpec: """ Parses a server mode specification, e.g.: @@ -44,6 +45,8 @@ def parse(server_spec: str, default_scheme: str) -> ServerSpec: scheme = m.group("scheme") else: scheme = default_scheme + if scheme not in ("", "http", "https", "tcp", "tls", "dtls", "quic", "dns"): + raise ValueError(f"Invalid server scheme: {scheme}") host = m.group("host") # IPv6 brackets @@ -66,4 +69,4 @@ def parse(server_spec: str, default_scheme: str) -> ServerSpec: if not check.is_valid_port(port): raise ValueError(f"Invalid port: {port}") - return scheme, (host, port) + return scheme, (host, port) # type: ignore diff --git a/mitmproxy/net/udp.py b/mitmproxy/net/udp.py index 9c341057ba..2a60260ff6 100644 --- a/mitmproxy/net/udp.py +++ b/mitmproxy/net/udp.py @@ -4,7 +4,7 @@ import ipaddress import socket import struct -from typing import Any, Callable, Optional, Tuple, Union, cast +from typing import Any, Callable, Optional, Union, cast from mitmproxy import ctx from mitmproxy.connection import Address from mitmproxy.utils import human @@ -233,7 +233,7 @@ def close(self) -> None: class DatagramReader: - _packets: asyncio.Queue[Tuple[bytes, Address]] + _packets: asyncio.Queue[bytes] _eof: bool def __init__(self) -> None: @@ -248,7 +248,7 @@ def feed_data(self, data: bytes, remote_addr: Address) -> None: ) else: try: - self._packets.put_nowait((data, remote_addr)) + self._packets.put_nowait(data) except asyncio.QueueFull: ctx.log.debug( f"Dropped UDP packet from {human.format_address(remote_addr)}." @@ -257,17 +257,17 @@ def feed_data(self, data: bytes, remote_addr: Address) -> None: def feed_eof(self) -> None: self._eof = True try: - self._packets.put_nowait((b"", None)) # type: ignore + self._packets.put_nowait(b"") except asyncio.QueueFull: pass - async def read(self, n: int) -> Tuple[bytes, Address]: + async def read(self, n: int) -> bytes: assert n >= MAX_DATAGRAM_SIZE if self._eof: try: return self._packets.get_nowait() except asyncio.QueueEmpty: - return (b"", None) # type: ignore + return b"" else: return await self._packets.get() @@ -300,8 +300,8 @@ def __init__( def _protocol(self) -> DrainableDatagramProtocol: return cast(DrainableDatagramProtocol, self._transport.get_protocol()) - def write(self, data: bytes, remote_addr: Optional[Address] = None) -> None: - self._transport.sendto(data, self._remote_addr if remote_addr is None else remote_addr) + def write(self, data: bytes) -> None: + self._transport.sendto(data, self._remote_addr) def write_eof(self) -> None: raise OSError("UDP does not support half-closing.") diff --git a/mitmproxy/proxy/commands.py b/mitmproxy/proxy/commands.py index 3845a7774e..f1cefb8440 100644 --- a/mitmproxy/proxy/commands.py +++ b/mitmproxy/proxy/commands.py @@ -6,10 +6,10 @@ The counterpart to commands are events. """ -from typing import Literal, Optional, Union, TYPE_CHECKING +from typing import Literal, Union, TYPE_CHECKING import mitmproxy.hooks -from mitmproxy.connection import Address, Connection, Server +from mitmproxy.connection import Connection, Server if TYPE_CHECKING: import mitmproxy.proxy.layer @@ -67,12 +67,10 @@ class SendData(ConnectionCommand): """ data: bytes - remote_addr: Optional[Address] - def __init__(self, connection: Connection, data: bytes, remote_addr: Optional[Address] = None): + def __init__(self, connection: Connection, data: bytes): super().__init__(connection) self.data = data - self.remote_addr = remote_addr def __repr__(self): target = str(self.connection).split("(", 1)[0].lower() @@ -121,15 +119,6 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls, *args, **kwargs) -class GetSocket(ConnectionCommand): - """ - Get the underlying socket. - This should really never be used, but is required to implement transparent mode. - """ - - blocking = True - - class Log(Command): message: str level: str diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index fb1e925f23..b571f3ad2f 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -3,14 +3,13 @@ Events represent the only way for layers to receive new data from sockets. The counterpart to events are commands. """ -import socket import warnings from dataclasses import dataclass, is_dataclass from typing import Any, Generic, Optional, TypeVar from mitmproxy import flow from mitmproxy.proxy import commands -from mitmproxy.connection import Address, Connection +from mitmproxy.connection import Connection class Event: @@ -45,7 +44,6 @@ class DataReceived(ConnectionEvent): """ data: bytes - remote_addr: Optional[Address] = None def __repr__(self): target = type(self.connection).__name__.lower() @@ -113,12 +111,6 @@ class HookCompleted(CommandCompleted): reply: None = None -@dataclass(repr=False) -class GetSocketCompleted(CommandCompleted): - command: commands.GetSocket - reply: socket.socket - - T = TypeVar("T") diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 3e4024dca6..da9dc162dd 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -4,10 +4,10 @@ from dataclasses import dataclass from typing import Optional -from mitmproxy import connection, platform +from mitmproxy import connection from mitmproxy.proxy import commands, events, layer from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import quic, tls from mitmproxy.proxy.mode_specs import ReverseMode from mitmproxy.proxy.utils import expect @@ -59,10 +59,26 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert isinstance(spec, ReverseMode) self.context.server.address = spec.address - if spec.scheme not in ("http", "tcp"): + if spec.scheme not in ("http", ""): if not self.context.options.keep_host_header: self.context.server.sni = spec.address[0] - self.child_layer = tls.ServerTLSLayer(self.context) + + # ensure proper upstream protocol and layer + if spec.scheme == "tls": + self.context.server.transport_protocol = "tcp" + self.child_layer = tls.ServerTLSLayer(self.context) + elif spec.scheme == "dtls": + self.context.server.transport_protocol = "udp" + self.child_layer = tls.ServerTLSLayer(self.context) + elif spec.scheme == "quic": + self.context.server.transport_protocol = "udp" + self.child_layer = quic.QuicLayer(self.context) + else: + self.child_layer = ( + tls.ServerTLSLayer(self.context) + if self.context.server.transport_protocol == "tcp" else + quic.QuicLayer(self.context) + ) else: self.child_layer = layer.NextLayer(self.context) @@ -74,12 +90,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: class TransparentProxy(DestinationKnown): @expect(events.Start) def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - assert platform.original_addr is not None - socket = yield commands.GetSocket(self.context.client) - try: - self.context.server.address = platform.original_addr(socket) - except Exception as e: - yield commands.Log(f"Transparent mode failure: {e!r}") + if self.context.server.address is None: + yield commands.Log("Transparent proxy layer has no server address.") self.child_layer = layer.NextLayer(self.context) diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 858e63c9bc..07af868643 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -19,15 +19,7 @@ from contextlib import contextmanager from typing import ClassVar, Generic, TypeVar, cast, get_args -from aioquic.buffer import Buffer as QuicBuffer -from aioquic.quic.packet import ( - PACKET_TYPE_INITIAL, - QuicProtocolVersion, - encode_quic_version_negotiation, - pull_quic_header, -) - -from mitmproxy import ctx, flow, log +from mitmproxy import ctx, flow, log, platform from mitmproxy.connection import Address from mitmproxy.master import Master from mitmproxy.net import udp @@ -172,6 +164,10 @@ async def listen(self, host: str, port: int) -> asyncio.Server | udp.UdpServer: def log_desc(self) -> str: pass + @property + def is_transparent(self) -> bool: + return False + @property def listen_addrs(self) -> tuple[Address, ...]: return self._listen_addrs @@ -196,6 +192,13 @@ async def handle_tcp_connection( handler = ProxyConnectionHandler( ctx.master, reader, writer, ctx.options, self.mode ) + if self.is_transparent: + assert platform.original_addr is not None + socket = writer.get_extra_info("socket") + try: + handler.layer.context.server.address = platform.original_addr(socket) + except Exception as e: + ctx.log.error(f"Transparent mode failure: {e!r}") handler.layer = self.make_top_layer(handler.layer.context) with self.manager.register_connection(connection_id, handler): await handler.handle_client() @@ -224,6 +227,7 @@ def make_top_layer(self, context: Context) -> Layer: class TransparentInstance(TcpServerInstance[mode_specs.TransparentMode]): log_desc = "Transparent proxy" + is_transparent = True def make_top_layer(self, context: Context) -> Layer: return layers.modes.TransparentProxy(context) @@ -254,7 +258,6 @@ def make_top_layer(self, context: Context) -> Layer: @abstractmethod def make_connection_id( self, - transport: asyncio.DatagramTransport, data: bytes, remote_addr: Address, local_addr: Address, @@ -266,7 +269,7 @@ async def listen(self, host: str, port: int) -> udp.UdpServer: self.handle_udp_datagram, host, port, - transparent=False + transparent=self.transparent ) def handle_udp_datagram( @@ -285,7 +288,11 @@ def handle_udp_datagram( handler = ProxyConnectionHandler( ctx.master, reader, writer, ctx.options, self.mode ) + handler.client.transport_protocol = "udp" handler.timeout_watchdog.CONNECTION_TIMEOUT = 20 + if self.is_transparent: + handler.layer.context.server.address = local_addr + handler.layer.context.server.transport_protocol = "udp" handler.layer = self.make_top_layer(handler.layer.context) # pre-register here - we may get datagrams before the task is executed. @@ -304,15 +311,15 @@ async def handle_udp_connection(self, connection_id: tuple, handler: ProxyConnec class DnsInstance(UdpServerInstance[mode_specs.DnsMode]): log_desc = "DNS server" + def is_transparent(self) -> bool: + return self.mode.data == "transparent" + def make_top_layer(self, context: Context) -> Layer: - layer = layers.DNSLayer(context) - layer.context.server.address = (self.mode.data or "resolve-local", 53) - layer.context.server.transport_protocol = "udp" - return layer + context.server.address = (self.mode.data or "resolve-local", 53) + return layers.DNSLayer(context) def make_connection_id( self, - transport: asyncio.DatagramTransport, data: bytes, remote_addr: Address, local_addr: Address, @@ -328,97 +335,26 @@ def make_connection_id( return ("udp", dns_id, remote_addr, local_addr) -class DtlsInstance(UdpServerInstance[mode_specs.DtlsMode]): - log_desc = "DTLS server" +class UdpInstance(UdpServerInstance[mode_specs.UdpMode]): + """Wrapper for a TcpServerInstance that also supports UDP.""" - def make_top_layer(self, context: Context) -> Layer: - context.client.transport_protocol = "udp" - layer = layers.ServerTLSLayer(context) - layer.child_layer = layers.ClientTLSLayer(layer.context) - layer.child_layer.child_layer = layers.UDPLayer(layer.context) - layer.context.server.address = self.mode.address - layer.context.server.transport_protocol = "udp" - return layer + def __init__(self, mode: mode_specs.UdpMode, manager: ServerManager): + super().__init__(mode.inner_mode, manager) + self.inner_server = cast(TcpServerInstance, ServerInstance.make(mode.inner_mode)) - def make_connection_id( - self, - transport: asyncio.DatagramTransport, - data: bytes, - remote_addr: Address, - local_addr: Address, - ) -> tuple | None: - return ("dtls", remote_addr, local_addr) + def is_transparent(self) -> bool: + return self.inner_server.is_transparent + def log_desc(self) -> str: + return f"{self.inner_server.log_desc} (UDP)" -class QuicInstance(UdpServerInstance[mode_specs.QuicMode]): - - def fully_qualify_connection_id(self, connection_id: bytes, local_addr: Address) -> tuple: - return ("quic", connection_id, local_addr) + def make_top_layer(self, context: Context) -> Layer: + return self.inner_server.make_top_layer(context) def make_connection_id( self, - transport: asyncio.DatagramTransport, data: bytes, remote_addr: Address, local_addr: Address, ) -> tuple | None: - # largely taken from aioquic's own asyncio server code - buffer = QuicBuffer(data=data) - try: - header = pull_quic_header( - buffer, host_cid_length=ctx.options.quic_connection_id_length - ) - except ValueError: - ctx.log.info( - f"Invalid QUIC datagram received from {human.format_address(remote_addr)}." - ) - return None - - # negotiate version, support all versions known to aioquic - supported_versions = ( - version.value - for version in QuicProtocolVersion - if version is not QuicProtocolVersion.NEGOTIATION - ) - if header.version is not None and header.version not in supported_versions: - transport.sendto( - encode_quic_version_negotiation( - source_cid=header.destination_cid, - destination_cid=header.source_cid, - supported_versions=supported_versions, - ), - remote_addr, - ) - return None - - # check if a new connection is possible - connection_id = self.fully_qualify_connection_id(header.destination_cid, local_addr) - if connection_id not in self.manager.connections: - if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: - ctx.log.info( - f"QUIC packet received from {human.format_address(remote_addr)} with an unknown connection id." - ) - return None - return connection_id - - def make_top_layer(self, context: Context) -> Layer: - # determine the server settings - if self.mode.mode == "reverse": - assert self.mode.address is not None - context.server.address = self.mode.address - if not ctx.options.keep_host_header: - context.server.sni = self.mode.address[0] - context.server.transport_protocol = "udp" - return layers.QuicLayer(context, self) - - async def handle_udp_connection(self, connection_id: tuple, handler: ProxyConnectionHandler) -> None: - layer = cast(layers.QuicLayer, handler.layer) - try: - return await super().handle_udp_connection(connection_id, handler) - finally: - # clear up all additional connection ids - for cid in layer.connection_ids: - fqcid = layer.fully_qualify_connection_id(cid) - if fqcid in self.manager.connections: - assert self.manager.connections[fqcid] is handler - del self.manager.connections[fqcid] + return ("udp", remote_addr, local_addr) diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index cfba0001cc..17847dfce3 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -60,6 +60,12 @@ class ProxyMode(Serializable, metaclass=ABCMeta): The transport protocol used by this mode's server. This information is used by the proxyserver addon to determine if two modes want to listen on the same address. """ + supports_udp: ClassVar[bool] = False + """ + Indicates that this mode can be used with UDP as well. + This flag must only be set if `transport_protocol` == "tcp" and + the corresponding `ServerInstance` inherits from `TcpServerInstance`. + """ __types: ClassVar[dict[str, Type[ProxyMode]]] = {} def __init_subclass__(cls, **kwargs): @@ -163,6 +169,7 @@ def _check_empty(data): class RegularMode(ProxyMode): """A regular HTTP(S) proxy that is interfaced with `HTTP CONNECT` calls (or absolute-form HTTP requests).""" + supports_udp = True def __post_init__(self) -> None: _check_empty(self.data) @@ -170,6 +177,7 @@ def __post_init__(self) -> None: class TransparentMode(ProxyMode): """A transparent proxy, see https://docs.mitmproxy.org/dev/howto-transparent/""" + supports_udp = True def __post_init__(self) -> None: _check_empty(self.data) @@ -177,6 +185,7 @@ def __post_init__(self) -> None: class UpstreamMode(ProxyMode): """A regular HTTP(S) proxy, but all connections are forwarded to a second upstream HTTP(S) proxy.""" + supports_udp = True scheme: Literal["http", "https"] address: tuple[str, int] @@ -185,20 +194,27 @@ def __post_init__(self) -> None: scheme, self.address = server_spec.parse(self.data, default_scheme="http") if scheme != "http" and scheme != "https": raise ValueError("invalid upstream proxy scheme") - self.scheme = scheme # type: ignore + self.scheme = scheme class ReverseMode(ProxyMode): """A reverse proxy. This acts like a normal server, but redirects all requests to a fixed target.""" - scheme: Literal["http", "https", "tcp", "tls"] + supports_udp = True + scheme: Literal["", "http", "https", "tls", "dtls", "quic"] address: tuple[str, int] # noinspection PyDataclass def __post_init__(self) -> None: - scheme, self.address = server_spec.parse(self.data, default_scheme="https") - if scheme != "http" and scheme != "https" and scheme != "tcp" and scheme != "tls": + scheme, self.address = server_spec.parse(self.data) + if ( + scheme != "" and scheme != "tcp" and + scheme != "http" and scheme != "https" and + scheme != "tls" and scheme != "dtls" and + scheme != "quic" + ): raise ValueError("invalid reverse proxy scheme") - self.scheme = scheme # type: ignore + # turn legacy TCP scheme into the agnostic scheme + self.scheme = "" if scheme == "tcp" else scheme class Socks5Mode(ProxyMode): @@ -212,7 +228,7 @@ def __post_init__(self) -> None: class DnsMode(ProxyMode): """A DNS server or proxy.""" default_port = 53 - transport_protocol: ClassVar[Literal["tcp", "udp"]] = "udp" + transport_protocol = "udp" scheme: Literal["dns"] # DoH, DoQ, ... address: tuple[str, int] | None = None @@ -226,65 +242,20 @@ def __post_init__(self) -> None: scheme, self.address = server_spec.parse(server, "dns") if scheme != "dns": raise ValueError("invalid dns scheme") - self.scheme = scheme # type: ignore + self.scheme = scheme @property def resolve_local(self) -> bool: return self.data in ["", "resolve-local"] -class DtlsMode(ProxyMode): - default_port = 8084 - transport_protocol: ClassVar[Literal["tcp", "udp"]] = "udp" - scheme: Literal["dtls"] # DoH, DoQ, ... - address: tuple[str, int] | None = None +class UdpMode(ProxyMode): + default_port = 8080 + transport_protocol = "udp" + inner_mode: ProxyMode # noinspection PyDataclass def __post_init__(self) -> None: - m, _, server = self.data.partition(":") - if m != "reverse": - raise ValueError("invalid dtls mode") - scheme, self.address = server_spec.parse(server, "dtls") - if scheme != "dtls": - raise ValueError("invalid dtls scheme") - self.scheme = scheme # type: ignore - - -class QuicMode(ProxyMode): - """ - QUIC modes: - - regular[:version] (default) - - upstream:[version://]host:port - - reverse:[version://]host:port - - Versions: Use the given HTTP version to connect to upstream. - - h1 - - h2 - - h3 (default) - - Example: - --mode quic:upstream:h2://192.168.1.1:8080@443 - Listens on port 443 for incoming H3 connections and forwards - request to upstream proxy 192.168.1.1 on port 8080 using H2. - """ - - default_port = 8085 - transport_protocol: ClassVar[Literal["tcp", "udp"]] = "udp" - scheme: Literal["h1", "h2", "h3"] = "h3" - mode: Literal["regular", "reverse", "upstream"] = "regular" - address: tuple[str, int] | None = None - - # noinspection PyDataclass - def __post_init__(self) -> None: - if self.data: - mode, _, additional = self.data.partition(":") - if mode == "reverse" or mode == "upstream": - scheme, self.address = server_spec.parse(additional, default_mode="h3") - elif mode == "regular": - scheme = additional - else: - raise ValueError(f"Invalid QUIC mode: {mode}") - if scheme != "h3" and scheme != "h2" and scheme != "h1": - raise ValueError(f"Invalid QUIC scheme: {scheme}") - self.mode = mode # type: ignore - self.scheme = scheme # type: ignore + self.inner_mode = ProxyMode.parse(self.data) + if not self.inner_mode.supports_udp: + raise ValueError(f"{self.inner_mode} doesn't support UDP") diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index a95ea68f6e..4eeab4ca10 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -82,7 +82,7 @@ class ConnectionHandler(metaclass=abc.ABCMeta): timeout_watchdog: TimeoutWatchdog client: Client max_conns: collections.defaultdict[Address, asyncio.Semaphore] - layer: layer.Layer + layer: "layer.Layer" wakeup_timer: set[asyncio.Task] def __init__(self, context: Context) -> None: @@ -249,13 +249,9 @@ async def handle_connection(self, connection: Connection) -> None: cancelled = None reader = self.transports[connection].reader assert reader - has_remote_addr = isinstance(reader, udp.DatagramReader) while True: try: - if has_remote_addr: - data, remote_addr = await reader.read(65535) - else: - data, remote_addr = await reader.read(65535), None + data = await reader.read(65535) if not data: raise OSError("Connection closed by peer.") except OSError: @@ -264,7 +260,7 @@ async def handle_connection(self, connection: Connection) -> None: cancelled = e break - self.server_event(events.DataReceived(connection, data, remote_addr)) + self.server_event(events.DataReceived(connection, data)) try: await self.drain_writers() @@ -357,19 +353,10 @@ def server_event(self, event: events.Event) -> None: pass # The connection has already been closed. elif isinstance(command, commands.SendData): writer = self.transports[command.connection].writer - if command.remote_addr is not None: - assert isinstance(writer, udp.DatagramWriter) - writer.write(command.data, command.remote_addr) - else: - assert writer - writer.write(command.data) + assert writer + writer.write(command.data) elif isinstance(command, commands.CloseConnection): self.close_connection(command.connection, command.half_close) - elif isinstance(command, commands.GetSocket): - writer = self.transports[command.connection].writer - assert writer - socket = writer.get_extra_info("socket") - self.server_event(events.GetSocketCompleted(command, socket)) elif isinstance(command, commands.StartHook): asyncio_utils.create_task( self.hook_task(command), diff --git a/setup.py b/setup.py index aff1acee41..d4ba8df77a 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#install-requires # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ - "aioquic>=0.9.20", + "aioquic>=0.9.20,<0.10", "asgiref>=3.2.10,<3.6", "Brotli>=1.0,<1.1", "certifi>=2019.9.11", # no semver here - this should always be on the last release! diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 12fa77730b..689b81a41c 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -265,16 +265,16 @@ async def test_dns() -> None: await tctx.master.await_log("Invalid DNS datagram received", level="info") req = tdnsreq() w.write(req.packed) - resp = dns.Message.unpack((await r.read(udp.MAX_DATAGRAM_SIZE))[0]) + resp = dns.Message.unpack(await r.read(udp.MAX_DATAGRAM_SIZE)) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps.connections) == 1 w.write(req.packed) - resp = dns.Message.unpack((await r.read(udp.MAX_DATAGRAM_SIZE))[0]) + resp = dns.Message.unpack(await r.read(udp.MAX_DATAGRAM_SIZE)) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps.connections) == 1 req.id = req.id + 1 w.write(req.packed) - resp = dns.Message.unpack((await r.read(udp.MAX_DATAGRAM_SIZE))[0]) + resp = dns.Message.unpack(await r.read(udp.MAX_DATAGRAM_SIZE)) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps.connections) == 2 tctx.configure(ps, server=False) diff --git a/test/mitmproxy/net/test_udp.py b/test/mitmproxy/net/test_udp.py index 0180550e5e..1db5a60997 100644 --- a/test/mitmproxy/net/test_udp.py +++ b/test/mitmproxy/net/test_udp.py @@ -12,8 +12,8 @@ async def test_reader(): reader.feed_data(bytearray(MAX_DATAGRAM_SIZE + 1), addr) reader.feed_data(b"Second message", addr) reader.feed_eof() - assert await reader.read(65535) == (b"First message", addr) + assert await reader.read(65535) == b"First message" with pytest.raises(AssertionError): await reader.read(MAX_DATAGRAM_SIZE - 1) - assert await reader.read(65535) == (b"Second message", addr) - assert not (await reader.read(65535))[0] + assert await reader.read(65535) == b"Second message" + assert not await reader.read(65535) diff --git a/test/mitmproxy/proxy/test_commands.py b/test/mitmproxy/proxy/test_commands.py index 2ca7c23f0a..1c685ef5c7 100644 --- a/test/mitmproxy/proxy/test_commands.py +++ b/test/mitmproxy/proxy/test_commands.py @@ -17,7 +17,6 @@ def test_dataclasses(tconn): assert repr(commands.SendData(tconn, b"foo")) assert repr(commands.OpenConnection(tconn)) assert repr(commands.CloseConnection(tconn)) - assert repr(commands.GetSocket(tconn)) assert repr(commands.Log("hello", "info")) From f0b87123ef2fa28b4736118608fdf9fffcc94877 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 22 Aug 2022 00:11:00 +0200 Subject: [PATCH 041/695] re-add scheme test --- mitmproxy/proxy/mode_servers.py | 2 +- test/mitmproxy/net/test_server_spec.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 07af868643..db667bcd04 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -269,7 +269,7 @@ async def listen(self, host: str, port: int) -> udp.UdpServer: self.handle_udp_datagram, host, port, - transparent=self.transparent + transparent=self.is_transparent ) def handle_udp_datagram( diff --git a/test/mitmproxy/net/test_server_spec.py b/test/mitmproxy/net/test_server_spec.py index 155fa7403f..1fe5590182 100644 --- a/test/mitmproxy/net/test_server_spec.py +++ b/test/mitmproxy/net/test_server_spec.py @@ -24,6 +24,9 @@ def test_parse_err(): with pytest.raises(ValueError, match="Invalid server specification"): server_spec.parse(":", "https") + with pytest.raises(ValueError, match="Invalid server scheme"): + server_spec.parse("ftp://example.com", "https") + with pytest.raises(ValueError, match="Invalid hostname"): server_spec.parse("$$$", "https") From 49c3fc2c637c5ea35ca1cc284a7191ea174ef54b Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 22 Aug 2022 11:29:23 +0200 Subject: [PATCH 042/695] move transparent code --- mitmproxy/proxy/mode_servers.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index e2de26cab1..d77524fd33 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -38,19 +38,6 @@ def __init__(self, master, r, w, options, mode): super().__init__(r, w, options, mode) self.log_prefix = f"{human.format_address(self.client.peername)}: " - async def handle_client(self) -> None: - if self.client.proxy_mode.type == "transparent": - writer = self.transports[self.client].writer - assert writer - socket = writer.get_extra_info("socket") - try: - assert platform.original_addr - self.layer.context.server.address = platform.original_addr(socket) - except Exception as e: - self.log(f"Transparent mode failure: {e!r}") - return - return await super().handle_client() - async def handle_hook(self, hook: commands.StartHook) -> None: with self.timeout_watchdog.disarm(): # We currently only support single-argument hooks. @@ -216,9 +203,9 @@ async def handle_tcp_connection( ctx.master, reader, writer, ctx.options, self.mode ) if self.is_transparent: - assert platform.original_addr is not None socket = writer.get_extra_info("socket") try: + assert platform.original_addr is not None handler.layer.context.server.address = platform.original_addr(socket) except Exception as e: ctx.log.error(f"Transparent mode failure: {e!r}") From 9d3380706bedbbb07960ccf43606cd8d7420002b Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 30 Aug 2022 21:57:36 +0200 Subject: [PATCH 043/695] [quic] work on layer decision --- mitmproxy/addons/next_layer.py | 85 ++++++++++++++++-------------- mitmproxy/proxy/layers/__init__.py | 6 ++- mitmproxy/proxy/layers/modes.py | 1 - mitmproxy/proxy/mode_servers.py | 15 ------ 4 files changed, 49 insertions(+), 58 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 77cdd2d802..702728e395 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -28,6 +28,8 @@ from mitmproxy.tls import ClientHello LayerCls = type[layer.Layer] +ClientSecurityLayerCls = Union[type[layers.ClientTLSLayer], type[layers.ClientQuicLayer]] +ServerSecurityLayerCls = Union[type[layers.ServerTLSLayer], type[layers.ClientQuicLayer]] def stack_match( @@ -121,8 +123,8 @@ def ignore_connection( def setup_tls_layer( self, context: context.Context, - client_layer_cls: LayerCls = layers.ClientTLSLayer, - server_layer_cls: LayerCls = layers.ServerTLSLayer, + client_layer_cls: ClientSecurityLayerCls = layers.ClientTLSLayer, + server_layer_cls: ServerSecurityLayerCls = layers.ServerTLSLayer, ) -> layer.Layer: def s(*layers): return stack_match(context, layers) @@ -135,8 +137,7 @@ def s(*layers): s(modes.HttpProxy) or s(modes.HttpUpstreamProxy) or s(modes.ReverseProxy) - or s(modes.ReverseProxy, layers.ServerQuicLayer) - or s(modes.ReverseProxy, layers.ServerTLSLayer) + or s(modes.ReverseProxy, server_layer_cls) ): return client_layer_cls(context) else: @@ -153,6 +154,28 @@ def is_destination_in_hosts(self, context: context.Context, hosts: Iterable[re.P for rex in hosts ) + def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls]]: + if len(data_client) == 0: + return None + + # first try DTLS (the parser may return None) + try: + client_hello = dtls_parse_client_hello(data_client) + if client_hello is not None: + return (client_hello, layers.ClientTLSLayer, layers.ServerTLSLayer) + except ValueError: + pass + + # next try QUIC + try: + client_hello, _ = layers.quic.pull_client_hello_and_connection_id(data_client) + return (client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer) + except ValueError: + pass + + # that's all we currently have to offer + return None + def next_layer(self, nextlayer: layer.NextLayer): if nextlayer.layer is None: nextlayer.layer = self._next_layer( @@ -221,36 +244,22 @@ def s(*layers): elif context.client.transport_protocol == "udp": # unlike TCP, we make a decision immediately - try: - client_hello = dtls_parse_client_hello(data_client) - if client_hello is None: - raise ValueError() - except ValueError: - try: - client_hello, _ = layers.quic.pull_client_hello_and_connection_id(data_client) - except ValueError: - client_hello = None - client_layer_cls = None - server_layer_cls = None - else: - client_layer_cls = layers.ClientQuicLayer - server_layer_cls = layers.ServerQuicLayer - else: - client_layer_cls = layers.ClientTLSLayer - server_layer_cls = layers.ServerTLSLayer + tls = self.detect_udp_tls(data_client) + is_quic = isinstance(context.layers[-1], layers.ClientQuicLayer) + raw_layer_cls = layers.QuicStreamLayer if is_quic else layers.UDPLayer # 1. check for --ignore/--allow if self.ignore_connection( context.server.address, data_client, - is_tls=lambda _: client_hello is not None, - client_hello=lambda _: client_hello + is_tls=lambda _: tls is not None, + client_hello=lambda _: None if tls is None else tls[0] ): return layers.UDPLayer(context, ignore=True) # 2. Check for DTLS/QUIC - if client_hello is not None: - return self.setup_tls_layer(context, client_layer_cls, server_layer_cls) + if tls is not None: + return self.setup_tls_layer(context, *tls[1:2]) # 3. Setup the HTTP layer for a regular HTTP proxy if s(modes.HttpProxy, layers.ClientQuicLayer): @@ -261,21 +270,17 @@ def s(*layers): # 4. Check for --udp if self.is_destination_in_hosts(context, self.udp_hosts): - return layers.UDPLayer(context) - - # 5. Check for raw tcp mode. + return raw_layer_cls(context) - very_likely_http = context.client.alpn and context.client.alpn in HTTP_ALPNS - probably_no_http = not very_likely_http and ( - not data_client[ - :3 - ].isalpha() # the first three bytes should be the HTTP verb, so A-Za-z is expected. - or data_server # a server greeting would be uncharacteristic. - ) - if ctx.options.rawtcp and probably_no_http: - return layers.TCPLayer(context) + # 5. Check for HTTP mode, but only on QUIC. + if ( + is_quic + and context.client.alpn + and (context.client.alpn == b"h3" or context.client.alpn.startswith(b"h3-")) + ): + return layers.HttpLayer(context, HTTPMode.transparent) - # 5. Check for DNS + # 6. Check for DNS try: dns.Message.unpack(data_client) except struct.error: @@ -283,8 +288,8 @@ def s(*layers): else: return layers.DNSLayer(context) - # 6. Use raw udp mode or ignore the connection. - return layers.UDPLayer(context, ignore=not ctx.options.rawudp) + # 7. Use raw mode or ignore the connection. + return raw_layer_cls(context, ignore=not ctx.options.rawudp) else: raise AssertionError(context.client.transport_protocol) diff --git a/mitmproxy/proxy/layers/__init__.py b/mitmproxy/proxy/layers/__init__.py index abb0193017..c4af84d66a 100644 --- a/mitmproxy/proxy/layers/__init__.py +++ b/mitmproxy/proxy/layers/__init__.py @@ -1,7 +1,7 @@ from . import modes from .dns import DNSLayer from .http import HttpLayer -from .quic import QuicLayer +from .quic import QuicStreamLayer, ClientQuicLayer, ServerQuicLayer from .tcp import TCPLayer from .udp import UDPLayer from .tls import ClientTLSLayer, ServerTLSLayer @@ -11,10 +11,12 @@ "modes", "DNSLayer", "HttpLayer", - "QuicLayer", + "QuicStreamLayer", "TCPLayer", "UDPLayer", + "ClientQuicLayer", "ClientTLSLayer", + "ServerQuicLayer", "ServerTLSLayer", "WebsocketLayer", ] diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 91bdb41f2d..4e49ef0fc2 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -79,7 +79,6 @@ class TransparentProxy(DestinationKnown): @expect(events.Start) def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.context.server.address - self.child_layer = layer.NextLayer(self.context) err = yield from self.finish_start() if err: diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index a953ae62b9..7c33c65bb4 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -200,10 +200,6 @@ async def listen(self, host: str, port: int) -> asyncio.Server | udp.UdpServer: else: raise AssertionError(self.mode.transport_protocol) - @property - def is_transparent(self) -> bool: - return False - @property def listen_addrs(self) -> tuple[Address, ...]: return self._listen_addrs @@ -221,13 +217,6 @@ async def handle_tcp_connection( handler = ProxyConnectionHandler( ctx.master, reader, writer, ctx.options, self.mode ) - if self.is_transparent: - socket = writer.get_extra_info("socket") - try: - assert platform.original_addr is not None - handler.layer.context.server.address = platform.original_addr(socket) - except Exception as e: - ctx.log.error(f"Transparent mode failure: {e!r}") handler.layer = self.make_top_layer(handler.layer.context) if isinstance(self.mode, mode_specs.TransparentMode): socket = writer.get_extra_info("socket") @@ -254,11 +243,7 @@ def handle_udp_datagram( handler = ProxyConnectionHandler( ctx.master, reader, writer, ctx.options, self.mode ) - handler.client.transport_protocol = "udp" handler.timeout_watchdog.CONNECTION_TIMEOUT = 20 - if self.is_transparent: - handler.layer.context.server.address = local_addr - handler.layer.context.server.transport_protocol = "udp" handler.layer = self.make_top_layer(handler.layer.context) handler.layer.context.client.transport_protocol = "udp" handler.layer.context.server.transport_protocol = "udp" From e45c98d19bb3dbf8227c94e53887e21626cf0a21 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 2 Sep 2022 15:50:03 +0200 Subject: [PATCH 044/695] [quic] updated QuicStreamLayer --- mitmproxy/addons/next_layer.py | 4 +- mitmproxy/addons/proxyserver.py | 15 +- mitmproxy/net/server_spec.py | 2 +- mitmproxy/proxy/layers/quic.py | 260 +++++++++++++++++--------------- mitmproxy/proxy/mode_servers.py | 5 + mitmproxy/proxy/mode_specs.py | 22 ++- 6 files changed, 179 insertions(+), 129 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 702728e395..4624e5c331 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -169,7 +169,7 @@ def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, Clie # next try QUIC try: client_hello, _ = layers.quic.pull_client_hello_and_connection_id(data_client) - return (client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer) + return (client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer) except ValueError: pass @@ -255,7 +255,7 @@ def s(*layers): is_tls=lambda _: tls is not None, client_hello=lambda _: None if tls is None else tls[0] ): - return layers.UDPLayer(context, ignore=True) + return raw_layer_cls(context, ignore=True) # 2. Check for DTLS/QUIC if tls is not None: diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 1b699bb050..b70759dfb7 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -18,12 +18,14 @@ http, platform, tcp, + udp, websocket, ) from mitmproxy.connection import Address from mitmproxy.flow import Flow from mitmproxy.proxy import events, mode_specs, server_hooks from mitmproxy.proxy.layers.tcp import TcpMessageInjected +from mitmproxy.proxy.layers.udp import UdpMessageInjected from mitmproxy.proxy.layers.websocket import WebSocketMessageInjected from mitmproxy.proxy.mode_servers import ProxyConnectionHandler, ServerInstance, ServerManager from mitmproxy.utils import human, signals @@ -267,7 +269,7 @@ def listen_addrs(self) -> list[Address]: def inject_event(self, event: events.MessageInjected): connection_id = ( - "tcp", + event.flow.client_conn.transport_protocol, event.flow.client_conn.peername, event.flow.client_conn.sockname, ) @@ -302,6 +304,17 @@ def inject_tcp(self, flow: Flow, to_client: bool, message: bytes): except ValueError as e: ctx.log.warn(str(e)) + @command.command("inject.udp") + def inject_udp(self, flow: Flow, to_client: bool, message: bytes): + if not isinstance(flow, udp.UDPFlow): + ctx.log.warn("Cannot inject UDP messages into non-UDP flows.") + + event = UdpMessageInjected(flow, udp.UDPMessage(not to_client, message)) + try: + self.inject_event(event) + except ValueError as e: + ctx.log.warn(str(e)) + def server_connect(self, data: server_hooks.ServerConnectionHookData): if data.server.sockname is None: data.server.sockname = self._connect_addr diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py index 9b71946ca7..7c741af78f 100644 --- a/mitmproxy/net/server_spec.py +++ b/mitmproxy/net/server_spec.py @@ -26,7 +26,7 @@ @cache -def parse(server_spec: str, default_scheme: str = "") -> ServerSpec: +def parse(server_spec: str, default_scheme: str) -> ServerSpec: """ Parses a server mode specification, e.g.: diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 5f1e5d041e..690185adff 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -19,10 +19,9 @@ from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection, flow as mitm_flow, tcp +from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, layers, mode_servers -from mitmproxy.proxy.layers import tcp as tcp_layer from mitmproxy.proxy.utils import expect from mitmproxy.tls import ClientHello, ClientHelloData, TlsData @@ -221,16 +220,15 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") -class QuicRelayStream: - flow: tcp.TCPFlow +class QuicStream: + flow: tcp.TCPFlow | None stream_id: int - def __init__(self, context: context.Context, stream_id: int) -> None: - self.flow = tcp.TCPFlow( - context.client, - context.server, - live=True, - ) + def __init__(self, context: context.Context, stream_id: int, ignore: bool) -> None: + if ignore: + self.flow = None + else: + self.flow = tcp.TCPFlow(context.client, context.server, live=True) self.stream_id = stream_id is_unidirectional = stream_is_unidirectional(stream_id) from_client = stream_is_client_initiated(stream_id) @@ -238,55 +236,70 @@ def __init__(self, context: context.Context, stream_id: int) -> None: self._ended_server = is_unidirectional and from_client def has_ended(self, client: bool) -> bool: - stream_ended = self._ended_client if client else self._ended_server - return stream_ended or not self.flow.live + return self._ended_client if client else self._ended_server - def mark_ended(self, client: bool) -> None: + def mark_ended(self, client: bool, err: str | None = None) -> layer.CommandGenerator[None]: + # ensure we actually change the ended state if client: + assert not self._ended_client self._ended_client = True else: + assert not self._ended_server self._ended_server = True + # we're done if the stream is ignored + if self.flow is None: + return + + # report any error if the flow hasn't one already + if err is not None and self.flow.error is None: + self.flow.error = mitm_flow.Error(str) + yield layers.tcp.TcpErrorHook(self.flow) + + # report error-free endings and always clear the live flag + if self._ended_client and self._ended_server: + if self.flow.error is None: + yield layers.tcp.TcpEndHook(self.flow) + self.flow.live = False + -class QuicRelayLayer(layer.Layer): +class QuicStreamLayer(layer.Layer): """ Layer on top of `ClientQuicLayer` and `ServerQuicLayer`, that simply relays all QUIC streams and datagrams. This layer is chosen by the default NextLayer addon if ALPN yields no known protocol. """ - # NOTE: for now we're (ab)using the TCPFlow until https://github.com/mitmproxy/mitmproxy/pull/5414 is resolved - - buffer_from_client: list[quic_events.QuicEvent] | None - buffer_from_server: list[quic_events.QuicEvent] | None - flow: tcp.TCPFlow # used for datagrams and to signal general connection issues + buffer_from_client: list[quic_events.QuicEvent] + buffer_from_server: list[quic_events.QuicEvent] + flow: udp.UDPFlow | None # used for datagrams and to signal general connection issues quic_client: QuicConnection | None = None quic_server: QuicConnection | None = None - streams_by_flow: dict[tcp.TCPFlow, QuicRelayStream] - streams_by_id: dict[int, QuicRelayStream] + streams_by_flow: dict[tcp.TCPFlow, QuicStream] + streams_by_id: dict[int, QuicStream] - def __init__(self, context: context.Context) -> None: + def __init__(self, context: context.Context, ignore: bool = False) -> None: super().__init__(context) self.buffer_from_client = [] self.buffer_from_server = [] - self.flow = tcp.TCPFlow( - self.context.client, - self.context.server, - live=True, - ) + if ignore: + self.flow = None + else: + self.flow = tcp.TCPFlow(self.context.client, self.context.server, live=True) self.streams_by_flow = {} self.streams_by_id = {} def get_or_create_stream( self, stream_id: int - ) -> layer.CommandGenerator[QuicRelayStream]: + ) -> layer.CommandGenerator[QuicStream]: if stream_id in self.streams_by_id: return self.streams_by_id[stream_id] else: # register the stream and start the flow - stream = QuicRelayStream(self.context, stream_id) - self.streams_by_flow[stream.flow] = stream + stream = QuicStream(self.context, stream_id, ignore=self.flow is None) self.streams_by_id[stream.stream_id] = stream - yield tcp_layer.TcpStartHook(stream.flow) + if stream.flow is not None: + self.streams_by_flow[stream.flow] = stream + yield layers.tcp.TcpStartHook(stream.flow) return stream def handle_quic_event( @@ -297,50 +310,51 @@ def handle_quic_event( # buffer events if the peer is not ready yet peer_quic = self.quic_server if from_client else self.quic_client if peer_quic is None: - buffer = self.buffer_from_client if from_client else self.buffer_from_server - assert buffer is not None - buffer.append(event) + (self.buffer_from_client if from_client else self.buffer_from_server).append(event) return peer_connection = self.context.server if from_client else self.context.client if isinstance(event, quic_events.DatagramFrameReceived): # forward datagrams (that are not stream-bound) - if not self.flow.live: - return - message = tcp.TCPMessage(from_client, event.data) - self.flow.messages.append(message) - yield tcp_layer.TcpMessageHook(self.flow) - peer_quic.send_datagram_frame(message.content) + if self.flow is not None: + message = udp.UDPMessage(from_client, event.data) + self.flow.messages.append(message) + yield layers.udp.UdpMessageHook(self.flow) + data = message.content + else: + data = event.data + peer_quic.send_datagram_frame(data) elif isinstance(event, quic_events.StreamDataReceived): # ignore data received from already ended streams stream = yield from self.get_or_create_stream(event.stream_id) if stream.has_ended(from_client): + yield commands.Log(f"Received {len(event.data)} byte(s) on already closed stream #{event.stream_id}.", level="debug") return # forward the message allowing addons to change it - message = tcp.TCPMessage(from_client, event.data) - stream.flow.messages.append(message) - yield tcp_layer.TcpMessageHook(stream.flow) + if stream.flow is not None: + message = tcp.TCPMessage(from_client, event.data) + stream.flow.messages.append(message) + yield layers.tcp.TcpMessageHook(stream.flow) + data = message.content + else: + data = event.data peer_quic.send_stream_data( stream.stream_id, - message.content, + data, event.end_stream, ) # mark the stream as ended if needed if event.end_stream: - stream.mark_ended(from_client) - - # end the flow if both sides ended - if stream.has_ended(not from_client): - yield tcp_layer.TcpEndHook(stream.flow) - stream.flow.live = False + yield from stream.mark_ended(from_client) elif isinstance(event, quic_events.StreamReset): # ignore resets from already ended streams stream = yield from self.get_or_create_stream(event.stream_id) if stream.has_ended(from_client): + yield commands.Log(f"Received reset for already closed stream #{event.stream_id}.", level="debug") return # forward resets to peer streams and report them to addons @@ -348,12 +362,13 @@ def handle_quic_event( stream.stream_id, event.error_code, ) - stream.flow.error = mitm_flow.Error(error_code_to_str(event.error_code)) - yield tcp_layer.TcpErrorHook(stream.flow) - stream.flow.live = False + + # mark the stream as failed + yield from stream.mark_ended(from_client, err=error_code_to_str(event.error_code)) else: # ignore other QUIC events + yield commands.Log(f"Ignored QUIC event {event!r}.", level="debug") return # transmit data to the peer @@ -362,15 +377,16 @@ def handle_quic_event( @expect(events.Start) def state_start(self, _) -> layer.CommandGenerator[None]: # mark the main flow as started - yield tcp_layer.TcpStartHook(self.flow) + if self.flow is not None: + yield layers.udp.UdpStartHook(self.flow) # open the upstream connection if necessary if self.context.server.timestamp_start is None: err = yield commands.OpenConnection(self.context.server) if err: - self.flow.error = mitm_flow.Error(str(err)) - yield tcp_layer.TcpErrorHook(self.flow) - self.flow.live = False + if self.flow is not None: + self.flow.error = mitm_flow.Error(str(err)) + yield layers.udp.UdpErrorHook(self.flow) yield commands.CloseConnection(self.context.client) self._handle_event = self.state_done return @@ -379,7 +395,8 @@ def state_start(self, _) -> layer.CommandGenerator[None]: @expect( QuicStart, QuicConnectionEvent, - tcp_layer.TcpMessageInjected, + layers.tcp.TcpMessageInjected, + layers.udp.UdpMessageInjected, events.ConnectionClosed, ) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -388,14 +405,12 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # define helper variables from_client = event.connection is self.context.client peer_conn = self.context.server if from_client else self.context.client - local_quic = self.quic_client if from_client else self.quic_server peer_quic = self.quic_server if from_client else self.quic_client - assert local_quic is not None - close_event = local_quic._close_event - assert close_event is not None + closed_quic = self.quic_client if from_client else self.quic_server + close_event = None if closed_quic is None else closed_quic._close_event # close the peer as well (needs to be before hooks) - if peer_quic is not None: + if peer_quic is not None and close_event is not None: peer_quic.close( close_event.error_code, close_event.frame_type, @@ -405,73 +420,59 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: else: yield commands.CloseConnection(peer_conn) - # report the termination as error to all non-ended streams - for flow in self.streams_by_flow: - if flow.live: - self.flow.error = mitm_flow.Error("Connection closed.") - yield tcp_layer.TcpErrorHook(flow) - flow.live = False - - # end the main flow - if self.flow.live: - if is_success_error_code(close_event.error_code): - yield tcp_layer.TcpEndHook(flow) - else: - self.flow.error = mitm_flow.Error( - close_event.reason_phrase - or error_code_to_str(close_event.error_code) - ) - yield tcp_layer.TcpErrorHook(flow) - self.flow.live = False + # report errors to the main flow + if ( + self.flow is not None + and close_event is not None + and not is_success_error_code(close_event.error_code) + ): + self.flow.error = mitm_flow.Error( + close_event.reason_phrase + or error_code_to_str(close_event.error_code) + ) + yield layers.udp.UdpErrorHook(self.flow) + # we're done handling QUIC events, pass on to generic close handling self._handle_event = self.state_done + yield from self.state_done(event) elif isinstance(event, QuicStart): # QUIC connection has been established, store it and get the peer's buffer - if event.connection is self.context.client: + from_client = event.connection is self.context.client + buffer_from_peer = self.buffer_from_server if from_client else self.buffer_from_client + if from_client: assert self.quic_client is None self.quic_client = event.quic - from_client = False - buffer = self.buffer_from_server - self.buffer_from_server = None - elif event.connection is self.context.server: + else: assert self.quic_server is None self.quic_server = event.quic - from_client = True - buffer = self.buffer_from_client - self.buffer_from_client = None - else: - raise AssertionError( - f"Connection {event.connection} not associated with layer." - ) - # flush the buffer - assert buffer is not None - for quic_event in buffer: - yield from self.handle_quic_event(quic_event, from_client) + # flush the buffer to the other side + for quic_event in buffer_from_peer: + yield from self.handle_quic_event(quic_event, not from_client) + buffer_from_peer.clear() - elif isinstance(event, tcp_layer.TcpMessageInjected): - # translate injected messages into QUIC events + elif isinstance(event, layers.tcp.TcpMessageInjected): + # translate injected TCP messages into QUIC stream events assert isinstance(event.flow, tcp.TCPFlow) - if event.flow is self.flow: - yield from self.handle_quic_event( - quic_events.DatagramFrameReceived(data=event.message.content), - event.message.from_client, - ) - elif event.flow in self.streams_by_flow: - stream = self.streams_by_flow[event.flow] - yield from self.handle_quic_event( - quic_events.StreamDataReceived( - stream_id=stream.stream_id, - data=event.message.content, - end_stream=False, - ), - event.message.from_client, - ) - else: - raise AssertionError( - f"Flow {event.flow} not associated with the current layer." - ) + stream = self.streams_by_flow[event.flow] + yield from self.handle_quic_event( + quic_events.StreamDataReceived( + stream_id=stream.stream_id, + data=event.message.content, + end_stream=False, + ), + event.message.from_client, + ) + + elif isinstance(event, layers.udp.UdpMessageInjected): + # translate injected UDP messages into QUIC datagram events + assert isinstance(event.flow, udp.UDPFlow) + assert event.flow is self.flow + yield from self.handle_quic_event( + quic_events.DatagramFrameReceived(data=event.message.content), + event.message.from_client, + ) elif isinstance(event, QuicConnectionEvent): # handle or buffer QUIC events @@ -486,11 +487,28 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: @expect( QuicStart, QuicConnectionEvent, - tcp_layer.TcpMessageInjected, + layers.tcp.TcpMessageInjected, + layers.udp.UdpMessageInjected, events.ConnectionClosed, ) - def state_done(self, _) -> layer.CommandGenerator[None]: - yield from () + def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.ConnectionClosed): + from_client = event.connection is self.context.client + + # report the termination as error to all non-ended streams + for stream in self.streams_by_id.values(): + if not stream.has_ended(from_client): + yield from stream.mark_ended(from_client, err="Connection closed.") + + # end the main flow + if ( + self.flow is not None + and not self.context.client.connected + and not self.context.server.connected + ): + if self.flow.error is None: + yield layers.udp.UdpEndHook(self.flow) + self.flow.live = False _handle_event = state_start diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 7c33c65bb4..532475bcdc 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -289,3 +289,8 @@ def make_top_layer(self, context: Context) -> Layer: class DnsInstance(AsyncioServerInstance[mode_specs.DnsMode]): def make_top_layer(self, context: Context) -> Layer: return layers.DNSLayer(context) + + +class Http3Instance(AsyncioServerInstance[mode_specs.Http3Mode]): + def make_top_layer(self, context: Context) -> Layer: + return layers.modes.HttpProxy(context) diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index ec0602129b..6f930649ff 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -192,14 +192,16 @@ class UpstreamMode(ProxyMode): """A regular HTTP(S) proxy, but all connections are forwarded to a second upstream HTTP(S) proxy.""" description = "HTTP(S) proxy (upstream mode)" transport_protocol = TCP - scheme: Literal["http", "https"] + scheme: Literal["http", "https", "http3"] address: tuple[str, int] # noinspection PyDataclass def __post_init__(self) -> None: scheme, self.address = server_spec.parse(self.data, default_scheme="http") - if scheme != "http" and scheme != "https": + if scheme != "http" and scheme != "https" and scheme != "http3": raise ValueError("invalid upstream proxy scheme") + if scheme == "http3": + self.transport_protocol = UDP self.scheme = scheme @@ -207,13 +209,13 @@ class ReverseMode(ProxyMode): """A reverse proxy. This acts like a normal server, but redirects all requests to a fixed target.""" description = "reverse proxy" transport_protocol = TCP - scheme: Literal["http", "https", "tls", "dtls", "tcp", "udp", "dns"] + scheme: Literal["http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns"] address: tuple[str, int] # noinspection PyDataclass def __post_init__(self) -> None: self.scheme, self.address = server_spec.parse(self.data, default_scheme="https") - if self.scheme in ("dns", "dtls", "udp"): + if self.scheme in ("http3", "dns", "dtls", "udp"): self.transport_protocol = UDP self.description = f"{self.description} to {self.data}" @@ -236,3 +238,15 @@ class DnsMode(ProxyMode): def __post_init__(self) -> None: _check_empty(self.data) + + +class Http3Mode(ProxyMode): + """ + A regular HTTP3 proxy that is interfaced with `HTTP CONNECT` calls (or absolute-form HTTP requests). + (This class will be merged into `RegularMode` once the UDP implementation is deemed stable enough.) + """ + description = "HTTP3 proxy" + transport_protocol = UDP + + def __post_init__(self) -> None: + _check_empty(self.data) From c4cfd06bb151da64b5619c53cb16f4fda8554847 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 5 Sep 2022 01:23:00 +0200 Subject: [PATCH 045/695] [quic] continued work on roaming --- mitmproxy/net/server_spec.py | 4 +- mitmproxy/proxy/context.py | 11 +- mitmproxy/proxy/layers/quic.py | 454 +++++++++++++++++++-------------- mitmproxy/proxy/mode_specs.py | 4 +- mitmproxy/proxy/server.py | 2 +- 5 files changed, 282 insertions(+), 193 deletions(-) diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py index 7c741af78f..c56f2bf1e0 100644 --- a/mitmproxy/net/server_spec.py +++ b/mitmproxy/net/server_spec.py @@ -8,7 +8,7 @@ from mitmproxy.net import check ServerSpec = tuple[ - Literal["http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns"], + Literal["http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns", "quic"], tuple[str, int] ] @@ -45,7 +45,7 @@ def parse(server_spec: str, default_scheme: str) -> ServerSpec: scheme = m.group("scheme") else: scheme = default_scheme - if scheme not in ("http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns"): + if scheme not in ("http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns", "quic"): raise ValueError(f"Invalid server scheme: {scheme}") host = m.group("host") diff --git a/mitmproxy/proxy/context.py b/mitmproxy/proxy/context.py index 5edb977c25..7fcb35340e 100644 --- a/mitmproxy/proxy/context.py +++ b/mitmproxy/proxy/context.py @@ -1,10 +1,11 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from mitmproxy import connection from mitmproxy.options import Options if TYPE_CHECKING: import mitmproxy.proxy.layer + from mitmproxy.proxy.server import ConnectionHandler class Context: @@ -25,6 +26,10 @@ class Context: """ Provides access to options for proxy layers. Not intended for use by addons, use `mitmproxy.ctx.options` instead. """ + handler: Optional[ConnectionHandler] + """ + The `ConnectionHandler` responsible for this context. + """ layers: list["mitmproxy.proxy.layer.Layer"] """ The protocol layer stack. @@ -34,16 +39,18 @@ def __init__( self, client: connection.Client, options: Options, + handler: Optional[ConnectionHandler] = None, ) -> None: self.client = client self.options = options + self.handler = handler self.server = connection.Server( None, transport_protocol=client.transport_protocol ) self.layers = [] def fork(self) -> "Context": - ret = Context(self.client, self.options) + ret = Context(self.client, self.options, self.handler) ret.server = self.server ret.layers = self.layers.copy() return ret diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 690185adff..2cebad42d1 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -2,6 +2,7 @@ import asyncio from dataclasses import dataclass, field from ssl import VerifyMode +from typing import ClassVar, cast from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import ErrorCode as H3ErrorCode @@ -16,14 +17,16 @@ stream_is_unidirectional, ) from aioquic.tls import CipherSuite, HandshakeType -from aioquic.quic.packet import PACKET_TYPE_INITIAL, pull_quic_header +from aioquic.quic.packet import PACKET_TYPE_INITIAL, QuicProtocolVersion, encode_quic_version_negotiation, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp +from mitmproxy import certs, connection, ctx, flow as mitm_flow, log, tcp, udp from mitmproxy.net import tls -from mitmproxy.proxy import commands, context, events, layer, layers, mode_servers +from mitmproxy.net.udp import DatagramWriter +from mitmproxy.proxy import commands, context, events, layer, layers, mode_servers, server from mitmproxy.proxy.utils import expect from mitmproxy.tls import ClientHello, ClientHelloData, TlsData +from mitmproxy.utils import asyncio_utils, human @dataclass @@ -220,6 +223,27 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") +def build_configuration(conn: connection.Connection, settings: QuicTlsSettings) -> QuicConfiguration: + """Creates a `QuicConfiguration` instance based on the given connection and TLS settings.""" + + return QuicConfiguration( + alpn_protocols=[offer.decode("ascii") for offer in conn.alpn_offers], + connection_id_length=ctx.options.quic_connection_id_length, + is_client=isinstance(conn, connection.Client), + secrets_log_file=QuicSecretsLogger(tls.log_master_secret) # type: ignore + if tls.log_master_secret is not None + else None, + server_name=conn.sni, + cafile=settings.ca_file, + capath=settings.ca_path, + certificate=settings.certificate, + certificate_chain=settings.certificate_chain, + cipher_suites=settings.cipher_suites, + private_key=settings.certificate_private_key, + verify_mode=settings.verify_mode, + ) + + class QuicStream: flow: tcp.TCPFlow | None stream_id: int @@ -267,6 +291,7 @@ class QuicStreamLayer(layer.Layer): """ Layer on top of `ClientQuicLayer` and `ServerQuicLayer`, that simply relays all QUIC streams and datagrams. This layer is chosen by the default NextLayer addon if ALPN yields no known protocol. + It uses `UDPFlow` and `TCPFlow` for datagrams and stream respectively, which makes message injection possible. """ buffer_from_client: list[quic_events.QuicEvent] @@ -513,13 +538,16 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: _handle_event = state_start -class _QuicLayer(layer.Layer): +class QuicConnectionLayer(layer.Layer): child_layer: layer.Layer conn: connection.Connection original_destination_connection_id: bytes | None = None quic: QuicConnection | None = None tls: QuicTlsSettings | None = None + writers: dict[connection.Address, DatagramWriter] + """Writers of all known endpoints that send data to this instance.""" + def __init__( self, context: context.Context, @@ -530,10 +558,8 @@ def __init__( self.conn = conn self._loop = asyncio.get_event_loop() self._pending_open_command: commands.OpenConnection | None = None - self._request_wakeup_command_and_timer: tuple[ - commands.RequestWakeup, float - ] | None = None - self._obsolete_wakeup_commands: set[commands.RequestWakeup] = set() + self._pending_wakeup_commands: dict[commands.RequestWakeup, float] = dict() + self._pending_data_received_events: list[events.DataReceived] = [] self.conn.tls = True def build_configuration(self) -> QuicConfiguration: @@ -623,12 +649,6 @@ def destroy_quic( self.tls = None self._handle_event = self.state_no_quic - # obsolete any current timer - if self._request_wakeup_command_and_timer is not None: - command, _ = self._request_wakeup_command_and_timer - self._obsolete_wakeup_commands.add(command) - self._request_wakeup_command_and_timer = None - # record an entry in the log yield commands.Log( f"{self.conn}: QUIC connection destroyed: {reason}", @@ -803,7 +823,41 @@ def retire_connection_id(self, connection_id: bytes) -> None: def start(self) -> layer.CommandGenerator[None]: yield from self.event_to_child(events.Start()) - def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + def state_after_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.quic is None + + if ( + isinstance(event, events.Wakeup) + and event.command in self._pending_wakeup_commands + ): + # filter out obsolete wakeups + del self._pending_wakeup_commands[event.command] + + else: + # forward all other events to the child layer + yield from self.event_to_child(event) + + def state_before_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.quic is None + assert len(self._pending_wakeup_commands) == 0 + + if ( + isinstance(event, events.ConnectionClosed) and event.connection is self.conn + ): + # if there was an OpenConnection command, then create_quic failed + # otherwise the connection was opened before the QUIC layer, so forward the event + if not (yield from self.open_connection_end("QUIC initialization failed")): + yield from self.event_to_child(event) + + elif isinstance(event, events.DataReceived) and event.connection is self.conn: + # buffer data until QUIC is initialized + self._pending_data_received_events.append(event) + + else: + # forward all other events to the child layer + yield from self.event_to_child(event) + + def state_quic(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None if isinstance(event, events.DataReceived) and event.connection is self.conn: @@ -838,56 +892,23 @@ def state_has_quic(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.event_to_child(event) elif isinstance(event, events.Wakeup): - # swallow obsolete wakeup events - if event.command in self._obsolete_wakeup_commands: - self._obsolete_wakeup_commands.remove(event.command) + # handle issued wakeup commands and forward others to child layer + if event.command in self._pending_wakeup_commands: + self.quic.handle_timer(now=max( + self._pending_wakeup_commands.pop(event.command), + self._loop.time() + )) + yield from self.process_events() else: - # handle active wakeup and forward others to child layer - if self._request_wakeup_command_and_timer is not None: - command, timer = self._request_wakeup_command_and_timer - if event.command is command: - self._request_wakeup_command_and_timer = None - self.quic.handle_timer(now=max(timer, self._loop.time())) - yield from self.process_events() - else: - yield from self.event_to_child(event) - else: - yield from self.event_to_child(event) - - else: - # forward other events to the child layer - yield from self.event_to_child(event) - - def state_no_quic(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.quic is None - - if ( - isinstance(event, events.Wakeup) - and event.command in self._obsolete_wakeup_commands - ): - # filter out obsolete wakeups - self._obsolete_wakeup_commands.remove(event.command) - - elif ( - isinstance(event, events.ConnectionClosed) and event.connection is self.conn - ): - # if there was an OpenConnection command, then create_quic failed - # otherwise the connection was opened before the QUIC layer, so forward the event - if not (yield from self.open_connection_end("QUIC initialization failed")): yield from self.event_to_child(event) - elif isinstance(event, events.DataReceived) and event.connection is self.conn: - # ignore received data, which either happens after QUIC is closed or if the underlying - # UDP connection is already opened and no QUIC initialization is being performed - pass - else: - # forward all other events to the child layer + # forward other events to the child layer yield from self.event_to_child(event) @expect(events.Start) def state_start(self, _) -> layer.CommandGenerator[None]: - self._handle_event = self.state_no_quic + self._handle_event = self.state_before_quic yield from self.start() def transmit(self) -> layer.CommandGenerator[None]: @@ -897,18 +918,11 @@ def transmit(self) -> layer.CommandGenerator[None]: for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): yield commands.SendData(self.conn, data, addr) - # mark an existing wakeup command as obsolete if it no longer matches the timer + # request a new wakeup if all pending requests trigger at a later time timer = self.quic.get_timer() - if self._request_wakeup_command_and_timer is not None: - command, existing_timer = self._request_wakeup_command_and_timer - if existing_timer != timer: - self._obsolete_wakeup_commands.add(command) - self._request_wakeup_command_and_timer = None - - # request a new wakeup if necessary - if timer is not None and self._request_wakeup_command_and_timer is None: + if not any(existing <= timer for existing in self._pending_wakeup_commands.values()): command = commands.RequestWakeup(timer - self._loop.time()) - self._request_wakeup_command_and_timer = (command, timer) + self._pending_wakeup_commands[command] = timer yield command _handle_event = state_start @@ -927,107 +941,122 @@ def __init__( self.child_layer = child_layer -class ClientQuicLayer(_QuicLayer): - """ - This layer establishes QUIC on a single client connection. - """ - - parent_layer: QuicLayer - wait_for_upstream: bool - - def __init__( - self, - parent_layer: QuicLayer, - connection_id: bytes, - wait_for_upstream: bool, - ) -> None: - super().__init__(parent_layer.context, parent_layer.context.client) - self.original_destination_connection_id = connection_id - self.parent_layer = parent_layer - self.wait_for_upstream = wait_for_upstream - self._handler = parent_layer.instance.manager.connections[ - parent_layer.fully_qualify_connection_id(connection_id) - ] - parent_layer.connection_ids.add(connection_id) +class QuicConnectionHandler(server.ConnectionHandler): + """Handler for QUIC connections, required for roaming.""" - def issue_connection_id(self, connection_id: bytes) -> None: - # add the connection id to the manager connections - fqcid = self.parent_layer.fully_qualify_connection_id(connection_id) - if fqcid not in self.parent_layer.instance.manager.connections: - self.parent_layer.instance.manager.connections[fqcid] = self._handler - self.parent_layer.connection_ids.add(connection_id) - - def retire_connection_id(self, connection_id: bytes) -> None: - # remove the connection id from the manager connections - if connection_id in self.parent_layer.connection_ids: - del self.parent_layer.instance.manager.connections[ - self.parent_layer.fully_qualify_connection_id(connection_id) - ] - self.parent_layer.connection_ids.remove(connection_id) - - def start(self) -> layer.CommandGenerator[None]: - yield from super().start() - - # try to open the upstream connection - if self.wait_for_upstream: - err = yield commands.OpenConnection(self.context.server) - if err: - yield commands.Log( - f"Unable to establish QUIC connection with server ({err}). " - f"Trying to establish QUIC with client anyway." - ) + def __init__(self, context: context.Context, quic: QuicConnection) -> None: + super().__init__(context) + # TODO fork context and set different client conn + handler = context.handler + assert handler is not None + self._handlers = {handler.client.peername: handler} + + def send_data(self, data: bytes, address: connection.Address) -> None: + """Sends data via a different handler. The handler for the address needs to be registered.""" + handler = self._handlers[address] + self.timeout_watchdog.register_activity() + handler.timeout_watchdog.register_activity() + handler.transports[handler.client].writer.write(data) + + def receive_data(self, data: bytes, address: connection.Address) -> None: + """Receives data from another address. The address's peer handler need to be registered.""" + assert address in self._handlers + self.client.peername = address + self.server_event(events.DataReceived(self.client, data)) + + def register_handler(self, handler: server.ConnectionHandler) -> None: + """Registers a new peer handler.""" + assert self._handlers + assert handler.client.address not in self._handlers + self._handlers[handler.client.address] = handler + + def unregister_handler(self, handler: server.ConnectionHandler) -> None: + """Removes the peer and shutdown the handler if it's the last one.""" + assert self._handlers[handler.client.address] == handler + del self._handlers[handler.client.address] + if not self._handlers: + self.server_event(events.ConnectionClosed(self.client)) + del ClientQuicLayer.connections_by_client[self.client] + + async def handle_hook(self, hook: commands.StartHook) -> None: + with self.timeout_watchdog.disarm(): + # keep in-sync with ProxyConnectionHandler + (data,) = hook.args() + await ctx.master.addons.handle_lifecycle(hook) + if isinstance(data, mitm_flow.Flow): + await data.wait_for_resume() # pragma: no cover + + def log(self, message: str, level: str = "info") -> None: + x = log.LogEntry(f"{human.format_address(self.client.address)}: {message}", level) + asyncio_utils.create_task( + ctx.master.addons.handle_lifecycle(log.AddLogHook(x)), + name="QuicConnectionHandler.log", + ) - # initialize QUIC, shutdown on failure - if not (yield from self.create_quic()): - yield commands.CloseConnection(self.conn) - if self.wait_for_upstream and err is not None: - yield commands.CloseConnection(self.context.server) - self._handle_event = self.state_failed - def state_failed(self, _) -> layer.CommandGenerator[None]: - yield from () +class ClientQuicLayer(layer.Layer): + """Client-side layer performing routing and connection initialization.""" + connections_by_client: ClassVar[dict[connection.Client, QuicConnectionHandler]] = dict() + """Mapping of client connections to quic connection handlers.""" -class QuicLayer(layer.Layer): - """ - Entry layer for QUIC proxy server. - """ + connections_by_id: ClassVar[dict[tuple[connection.Address, bytes], QuicConnectionHandler]] = dict() + """Mapping of (sockname, cid) tuples to quic connection handlers.""" - instance: mode_servers.QuicInstance - connection_ids: set[bytes] + def __init__(self, context: context.Context) -> None: + if context.client.tls: + # keep in sync with ClientTLSLayer + context.client.alpn = None + context.client.cipher = None + context.client.sni = None + context.client.timestamp_tls_setup = None + context.client.tls_version = None + context.client.certificate_list = [] + context.client.mitmcert = None + context.client.alpn_offers = [] + context.client.cipher_list = [] - def __init__( - self, - context: context.Context, - instance: mode_servers.QuicInstance, - ) -> None: super().__init__(context) - self.instance = instance - self.connection_ids = set() - self.context.client.tls = True - self.context.server.tls = True - - def fully_qualify_connection_id(self, connection_id: bytes) -> tuple: - return self.instance.fully_qualify_connection_id( - connection_id, self.context.client.sockname - ) + self._handlers: set[QuicConnectionHandler] = set() + upper_layer = isinstance(self.context.layers[-2], ServerQuicLayer) + self._server_quic_layer = upper_layer if isinstance(upper_layer, ServerQuicLayer) else None - @expect(events.DataReceived, events.ConnectionClosed) - def state_done(self, _) -> layer.CommandGenerator[None]: - yield from () + def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerator[None]: + # largely taken from aioquic's own asyncio server code and ClientTLSLayer + buffer = QuicBuffer(data=event.data) + try: + header = pull_quic_header( + buffer, host_cid_length=self.context.options.quic_connection_id_length + ) + except ValueError: + yield commands.Log("Invalid QUIC datagram received.") + return - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - self._handle_event = self.state_wait_for_hello - yield from () + # negotiate version, support all versions known to aioquic + supported_versions = ( + version.value + for version in QuicProtocolVersion + if version is not QuicProtocolVersion.NEGOTIATION + ) + if header.version is not None and header.version not in supported_versions: + yield commands.SendData( + event.connection, + encode_quic_version_negotiation( + source_cid=header.destination_cid, + destination_cid=header.source_cid, + supported_versions=supported_versions, + ), + ) + return - @expect(events.DataReceived, events.ConnectionClosed) - def state_wait_for_hello(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, events.ConnectionEvent) - assert event.connection is self.context.client + # get or create a handler for the connection + connection_id = (header.destination_cid, self.context.client.sockname) + handler = ClientQuicLayer.connections_by_id.get(connection_id, None) + if handler is None: + if len(event.data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: + yield commands.Log(f"Invalid handshake received.") + return - # only handle the first packet from the client - if isinstance(event, events.DataReceived): # extract the client hello try: client_hello, connection_id = pull_client_hello_and_connection_id( @@ -1037,47 +1066,100 @@ def state_wait_for_hello(self, event: events.Event) -> layer.CommandGenerator[No yield commands.Log( f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" ) - yield commands.CloseConnection(self.context.client) - self._handle_event = self.state_done - else: + return - # copy the information - self.context.client.sni = client_hello.sni - self.context.client.alpn_offers = client_hello.alpn_protocols + # copy the client hello information + self.context.client.sni = client_hello.sni + self.context.client.alpn_offers = client_hello.alpn_protocols - # check with addons what we shall do - next_layer: layer.Layer - hook_data = ClientHelloData(self.context, client_hello) - yield layers.tls.TlsClienthelloHook(hook_data) + # check with addons what we shall do + hook_data = ClientHelloData(self.context, client_hello) + yield layers.tls.TlsClienthelloHook(hook_data) - # simply relay everything - if hook_data.ignore_connection: - next_layer = layers.TCPLayer(self.context, ignore=True) + # ignoring a connection is only allowed if there are no existing peers + if hook_data.ignore_connection: + assert not self._handlers - # contact the upstream server first - elif hook_data.establish_server_tls_first: - next_layer = ServerQuicLayer(self.context) - next_layer.child_layer = ClientQuicLayer( - self, connection_id, wait_for_upstream=True - ) + # replace the QUIC layer with an UDP layer + next_layer = layers.UDPLayer(self.context, ignore=True) + prev_layer = ( + self + if self._server_quic_layer is None else + self._server_quic_layer + ) + prev_layer.handle_event = next_layer.handle_event + prev_layer._handle_event = next_layer._handle_event + yield from next_layer.handle_event(events.Start()) + yield from next_layer.handle_event(event) + return - # perform the client handshake immediately - else: - next_layer = ClientQuicLayer( - self, connection_id, wait_for_upstream=False + # start the server QUIC connection if demanded and available + if ( + hook_data.establish_server_tls_first + and not self.context.server.tls_established + ): + err = ( + yield commands.OpenConnection(self.context.server) + if self._server_quic_layer is not None else + "No server QUIC available." + ) + if err: + yield commands.Log( + f"Unable to establish QUIC connection with server ({err}). " + f"Trying to establish QUIC with client anyway. " + f"If you plan to redirect requests away from this server, " + f"consider setting `connection_strategy` to `lazy` to suppress early connections." ) - # replace this layer and start the next one - self.handle_event = next_layer.handle_event # type: ignore - self._handle_event = next_layer._handle_event - yield from next_layer.handle_event(events.Start()) - yield from next_layer.handle_event(event) + # query addons to provide the necessary TLS settings + tls_data = QuicTlsData(self.context.client, self.context) + yield QuicTlsStartClientHook(tls_data) + if tls_data.settings is None: + yield commands.Log("No client QUIC TLS settings provided by addon(s).", level="error") + return + + # create and register the QUIC connection and handler + handler = QuicConnectionHandler( + context=self.context, + quic=QuicConnection( + configuration=build_configuration(self.context.client, tls_data.settings), + original_destination_connection_id=header.destination_cid, + ) + ) + ClientQuicLayer.connections_by_id[connection_id] = handler + ClientQuicLayer.connections_by_client[handler.client] = handler + + else: + # ensure that the handler is registered with the peer handler + if handler not in self._handlers: + handler.register_handler(self) + self._handlers.add(handler) + + # forward the received packet + handler.receive_data(event.data, self.context.client.peername) + + @expect(events.DataReceived, events.ConnectionClosed, events.MessageInjected) + def state_route(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.MessageInjected): + # relay the injection based on the flow's client + ClientQuicLayer.connections_by_client[event.flow.client_conn].server_event(event) - # stop if the connection was closed (usually we will always get one packet) elif isinstance(event, events.ConnectionClosed): - self._handle_event = self.state_done + assert event.connection is self.context.client + # remove and unregister all peer handlers + while self._handlers: + self._handlers.pop().unregister_handler(self) + + elif isinstance(event, events.DataReceived): + assert event.connection is self.context.client + yield from self.datagram_received(event) else: raise AssertionError(f"Unexpected event: {event!r}") + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + self._handle_event = self.state_route + yield from () + _handle_event = state_start diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index 6f930649ff..e15f23bb83 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -209,13 +209,13 @@ class ReverseMode(ProxyMode): """A reverse proxy. This acts like a normal server, but redirects all requests to a fixed target.""" description = "reverse proxy" transport_protocol = TCP - scheme: Literal["http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns"] + scheme: Literal["http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns", "quic"] address: tuple[str, int] # noinspection PyDataclass def __post_init__(self) -> None: self.scheme, self.address = server_spec.parse(self.data, default_scheme="https") - if self.scheme in ("http3", "dns", "dtls", "udp"): + if self.scheme in ("http3", "dtls", "udp", "dns", "quic"): self.transport_protocol = UDP self.description = f"{self.description} to {self.data}" diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 4eeab4ca10..9cc012be40 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -409,7 +409,7 @@ def __init__( time.time(), proxy_mode=mode, ) - context = Context(client, options) + context = Context(client, options, self) super().__init__(context) self.transports[client] = ConnectionIO( handler=None, reader=reader, writer=writer From f462e0bfc922d46ea9af41a66eb5b3f604b01aff Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 5 Sep 2022 16:23:54 +0200 Subject: [PATCH 046/695] [quic] more work on roaming rewrite --- mitmproxy/proxy/layers/quic.py | 403 ++++++++++++++++----------------- 1 file changed, 190 insertions(+), 213 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 2cebad42d1..5bc430de21 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -17,7 +17,7 @@ stream_is_unidirectional, ) from aioquic.tls import CipherSuite, HandshakeType -from aioquic.quic.packet import PACKET_TYPE_INITIAL, QuicProtocolVersion, encode_quic_version_negotiation, pull_quic_header +from aioquic.quic.packet import PACKET_TYPE_INITIAL, QuicHeader, QuicProtocolVersion, encode_quic_version_negotiation, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from mitmproxy import certs, connection, ctx, flow as mitm_flow, log, tcp, udp @@ -223,27 +223,6 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") -def build_configuration(conn: connection.Connection, settings: QuicTlsSettings) -> QuicConfiguration: - """Creates a `QuicConfiguration` instance based on the given connection and TLS settings.""" - - return QuicConfiguration( - alpn_protocols=[offer.decode("ascii") for offer in conn.alpn_offers], - connection_id_length=ctx.options.quic_connection_id_length, - is_client=isinstance(conn, connection.Client), - secrets_log_file=QuicSecretsLogger(tls.log_master_secret) # type: ignore - if tls.log_master_secret is not None - else None, - server_name=conn.sni, - cafile=settings.ca_file, - capath=settings.ca_path, - certificate=settings.certificate, - certificate_chain=settings.certificate_chain, - cipher_suites=settings.cipher_suites, - private_key=settings.certificate_private_key, - verify_mode=settings.verify_mode, - ) - - class QuicStream: flow: tcp.TCPFlow | None stream_id: int @@ -541,13 +520,9 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: class QuicConnectionLayer(layer.Layer): child_layer: layer.Layer conn: connection.Connection - original_destination_connection_id: bytes | None = None quic: QuicConnection | None = None tls: QuicTlsSettings | None = None - writers: dict[connection.Address, DatagramWriter] - """Writers of all known endpoints that send data to this instance.""" - def __init__( self, context: context.Context, @@ -559,9 +534,15 @@ def __init__( self._loop = asyncio.get_event_loop() self._pending_open_command: commands.OpenConnection | None = None self._pending_wakeup_commands: dict[commands.RequestWakeup, float] = dict() - self._pending_data_received_events: list[events.DataReceived] = [] + self._pending_data_received_events: list[events.DataReceived] = list() + self._routes: dict[connection.Address, server.ConnectionHandler | None] = dict() self.conn.tls = True + def add_route(self, context: context.Context) -> None: + """Registers a new roamed context.""" + assert context.client.peername not in self._routes + self._routes[context.client.peername] = context.handler + def build_configuration(self) -> QuicConfiguration: assert self.tls is not None @@ -582,15 +563,11 @@ def build_configuration(self) -> QuicConfiguration: verify_mode=self.tls.verify_mode, ) - def create_quic(self) -> layer.CommandGenerator[bool]: + def start_tls(self, original_destination_connection_id: bytes | None) -> layer.CommandGenerator[bool]: # must only be called if QUIC is uninitialized assert self.quic is None assert self.tls is None - # cannot initialize QUIC on a closed connection - if not self.conn.connected: - return False - # in case the connection is being reused, clear all handshake data self.conn.timestamp_tls_setup = None self.conn.certificate_list = () @@ -605,17 +582,14 @@ def create_quic(self) -> layer.CommandGenerator[bool]: else: yield QuicTlsStartServerHook(tls_data) if tls_data.settings is None: - yield commands.Log( - f"{self.conn}: No QUIC TLS settings provided by addon(s).", - level="error", - ) - return False + yield commands.Log(f"{self.conn}: No QUIC TLS settings provided by addon(s).", level="error") + return False, # create the aioquic connection self.tls = tls_data.settings self.quic = QuicConnection( configuration=self.build_configuration(), - original_destination_connection_id=self.original_destination_connection_id, + original_destination_connection_id=original_destination_connection_id, ) self._handle_event = self.state_has_quic @@ -623,7 +597,7 @@ def create_quic(self) -> layer.CommandGenerator[bool]: self.issue_connection_id(self.quic.host_cid) # record an entry in the log - yield commands.Log(f"{self.conn}: QUIC connection created.", level="info") + yield commands.Log(f"{self.conn}: QUIC connection created.", level="debug") return True def destroy_quic( @@ -652,7 +626,7 @@ def destroy_quic( # record an entry in the log yield commands.Log( f"{self.conn}: QUIC connection destroyed: {reason}", - level="info" if is_success_error_code(event.error_code) else "warn", + level="debug" if is_success_error_code(event.error_code) else "info", ) def establish_quic( @@ -688,14 +662,17 @@ def establish_quic( yield commands.Log( f"{self.conn}: QUIC connection established. " f"(early_data={event.early_data_accepted}, resumed={event.session_resumed})", - level="info", + level="debug", ) def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: # filter commands coming from the child layer for command in self.child_layer.handle_event(event): - if isinstance(command, QuicTransmit) and command.connection is self.conn: + if ( + isinstance(command, QuicTransmit) + and command.connection is self.conn + ): # transmit buffered data and re-arm timer if command.quic is self.quic: yield from self.transmit() @@ -736,22 +713,23 @@ def open_connection_begin( assert self._pending_open_command is None self._pending_open_command = command - # try to open the underlying UDP connection + # try to open the underlying UDP connection and create QUIC err = yield commands.OpenConnection(self.conn) - if not err: - # initialize QUIC and connect (notify the child layer after handshake) - if (yield from self.create_quic()): - assert self.quic is not None - self.quic.connect(self.conn.peername, now=self._loop.time()) - yield from self.process_events() - else: - # TLS failed, close the connection (notify child layer once closed) - yield commands.CloseConnection(self.conn) - else: + if err: # notify the child layer immediately about the error self._pending_open_command = None yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) + elif not (yield from self.start_tls()): + # TLS failed, close the connection (notify the child layer once its closed) + yield commands.CloseConnection(self.conn) + + else: + # connect to server (notify the child layer after handshake) + assert self.quic is not None + self.quic.connect(self.conn.peername, now=self._loop.time()) + yield from self.process_events() + def open_connection_end(self, reply: str | None) -> layer.CommandGenerator[bool]: if self._pending_open_command is None: return False @@ -817,6 +795,11 @@ def process_events(self) -> layer.CommandGenerator[None]: # transmit buffered data and re-arm timer yield from self.transmit() + def remove_route(self, context: context.Context) -> None: + """Removes a registered roamed context.""" + assert self._routes[context.client.peername] == context.handler + del self._routes[context.client.peername] + def retire_connection_id(self, connection_id: bytes) -> None: pass @@ -916,7 +899,14 @@ def transmit(self) -> layer.CommandGenerator[None]: # send all queued datagrams for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): - yield commands.SendData(self.conn, data, addr) + if addr == self.conn.peername: + yield commands.SendData(self.conn, data) + else: + handler = self._routes.get(addr, None) + if handler is None: + yield commands.Log(f"{self.conn}: No route to {human.format_address(addr)}.") + else: + handler.transports[handler.client].writer(data) # request a new wakeup if all pending requests trigger at a later time timer = self.quic.get_timer() @@ -941,71 +931,64 @@ def __init__( self.child_layer = child_layer -class QuicConnectionHandler(server.ConnectionHandler): - """Handler for QUIC connections, required for roaming.""" +class QuicRoamingLayer(layer.Layer): + """Simple routing layer that replaces a `ClientQuicLayer` when a connection roams to a different `ConnectionHandler`.""" - def __init__(self, context: context.Context, quic: QuicConnection) -> None: + def __init__(self, context: context.Context, target_layer: ClientQuicLayer) -> None: super().__init__(context) - # TODO fork context and set different client conn - handler = context.handler - assert handler is not None - self._handlers = {handler.client.peername: handler} - - def send_data(self, data: bytes, address: connection.Address) -> None: - """Sends data via a different handler. The handler for the address needs to be registered.""" - handler = self._handlers[address] - self.timeout_watchdog.register_activity() - handler.timeout_watchdog.register_activity() - handler.transports[handler.client].writer.write(data) - - def receive_data(self, data: bytes, address: connection.Address) -> None: - """Receives data from another address. The address's peer handler need to be registered.""" - assert address in self._handlers - self.client.peername = address - self.server_event(events.DataReceived(self.client, data)) - - def register_handler(self, handler: server.ConnectionHandler) -> None: - """Registers a new peer handler.""" - assert self._handlers - assert handler.client.address not in self._handlers - self._handlers[handler.client.address] = handler - - def unregister_handler(self, handler: server.ConnectionHandler) -> None: - """Removes the peer and shutdown the handler if it's the last one.""" - assert self._handlers[handler.client.address] == handler - del self._handlers[handler.client.address] - if not self._handlers: - self.server_event(events.ConnectionClosed(self.client)) - del ClientQuicLayer.connections_by_client[self.client] - - async def handle_hook(self, hook: commands.StartHook) -> None: - with self.timeout_watchdog.disarm(): - # keep in-sync with ProxyConnectionHandler - (data,) = hook.args() - await ctx.master.addons.handle_lifecycle(hook) - if isinstance(data, mitm_flow.Flow): - await data.wait_for_resume() # pragma: no cover - - def log(self, message: str, level: str = "info") -> None: - x = log.LogEntry(f"{human.format_address(self.client.address)}: {message}", level) - asyncio_utils.create_task( - ctx.master.addons.handle_lifecycle(log.AddLogHook(x)), - name="QuicConnectionHandler.log", - ) + self.target_layer = target_layer + + @expect() + def state_closed(self, _) -> layer.CommandGenerator[None]: + yield from () + + @expect(events.DataReceived, events.ConnectionClosed, events.MessageInjected) + def state_relay(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.MessageInjected): + # ensure the flow matches the target and forward the event + assert event.flow.client_conn is self.target_layer.context.client + handler = self.target_layer.context.handler + assert handler + handler.server_event(event) + + elif isinstance(event, events.ConnectionClosed): + # remove the registration and stop relaying + assert event.connection is self.context.client + self.target_layer.remove_route(self.context) + self._handle_event = self.state_closed + + elif isinstance(event, events.DataReceived): + # update target's peername and forward the event + assert event.connection is self.context.client + handler = self.target_layer.context.handler + assert handler + handler.client.peername = self.context.client.peername + handler.server_event(events.DataReceived(handler.client, event.data)) + + else: + raise AssertionError(f"Unexpected event: {event!r}") + yield from () -class ClientQuicLayer(layer.Layer): - """Client-side layer performing routing and connection initialization.""" + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + # register with the target and start relaying + self.target_layer.add_route(self.context) + self._handle_event = self.state_relay + yield from () + + _handle_event = state_start - connections_by_client: ClassVar[dict[connection.Client, QuicConnectionHandler]] = dict() - """Mapping of client connections to quic connection handlers.""" - connections_by_id: ClassVar[dict[tuple[connection.Address, bytes], QuicConnectionHandler]] = dict() - """Mapping of (sockname, cid) tuples to quic connection handlers.""" +class ClientQuicLayer(QuicConnectionLayer): + """Client-side layer performing routing and connection handling.""" - def __init__(self, context: context.Context) -> None: + connections: ClassVar[dict[tuple[connection.Address, bytes], ClientQuicLayer]] = dict() + """Mapping of (sockname, cid) tuples to QUIC client layers.""" + + def __init__(self, context: context.Context, can_roam: bool = False) -> None: + # same as ClientTLSLayer, we might be nested in some other transport if context.client.tls: - # keep in sync with ClientTLSLayer context.client.alpn = None context.client.cipher = None context.client.sni = None @@ -1015,22 +998,20 @@ def __init__(self, context: context.Context) -> None: context.client.mitmcert = None context.client.alpn_offers = [] context.client.cipher_list = [] - super().__init__(context) - self._handlers: set[QuicConnectionHandler] = set() + self.can_roam = can_roam upper_layer = isinstance(self.context.layers[-2], ServerQuicLayer) - self._server_quic_layer = upper_layer if isinstance(upper_layer, ServerQuicLayer) else None + self.server_quic_layer = upper_layer if isinstance(upper_layer, ServerQuicLayer) else None - def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerator[None]: - # largely taken from aioquic's own asyncio server code and ClientTLSLayer + def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerator[str | None]: + # fail if the received data is not a QUIC packet buffer = QuicBuffer(data=event.data) try: header = pull_quic_header( buffer, host_cid_length=self.context.options.quic_connection_id_length ) except ValueError: - yield commands.Log("Invalid QUIC datagram received.") - return + return "Invalid QUIC datagram received." # negotiate version, support all versions known to aioquic supported_versions = ( @@ -1047,119 +1028,115 @@ def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerato supported_versions=supported_versions, ), ) - return + return None # get or create a handler for the connection - connection_id = (header.destination_cid, self.context.client.sockname) - handler = ClientQuicLayer.connections_by_id.get(connection_id, None) - if handler is None: - if len(event.data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: - yield commands.Log(f"Invalid handshake received.") - return + target_layer = ClientQuicLayer.connections((header.destination_cid, self.context.client.sockname), None) + if target_layer is None: + # try to start the client QUIC connection + return (yield from self.start_client_quic(event, header)) - # extract the client hello - try: - client_hello, connection_id = pull_client_hello_and_connection_id( - event.data - ) - except ValueError as e: - yield commands.Log( - f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" - ) - return + else: + # ensure that this layer can roam (usually not supported when nested) + if not self.can_roam: + return "Connection cannot roam." + + # replace the layer with a roaming layer + return (yield from self.replace_layer( + replacement_layer=QuicRoamingLayer(self.context, target_layer), + first_data_event=event + )) + + def replace_layer(self, replacement_layer: layer.Layer, first_data_event: events.DataReceived) -> layer.CommandGenerator[None]: + # we need to replace the server layer as well, if there is one + layer_to_replace = ( + self + if self.server_quic_layer is None else + self.server_quic_layer + ) + layer_to_replace.handle_event = replacement_layer.handle_event + layer_to_replace._handle_event = replacement_layer._handle_event + yield from replacement_layer.handle_event(events.Start()) + yield from replacement_layer.handle_event(first_data_event) - # copy the client hello information - self.context.client.sni = client_hello.sni - self.context.client.alpn_offers = client_hello.alpn_protocols + def start_client_quic(self, event: events.DataReceived, header: QuicHeader) -> layer.CommandGenerator[str | None]: + # ensure it's (likely) a client handshake packet + if len(event.data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: + return "Invalid handshake received." - # check with addons what we shall do - hook_data = ClientHelloData(self.context, client_hello) - yield layers.tls.TlsClienthelloHook(hook_data) + # extract the client hello + try: + client_hello, connection_id = pull_client_hello_and_connection_id( + event.data + ) + except ValueError as e: + return f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" - # ignoring a connection is only allowed if there are no existing peers - if hook_data.ignore_connection: - assert not self._handlers + # copy the client hello information + self.context.client.sni = client_hello.sni + self.context.client.alpn_offers = client_hello.alpn_protocols - # replace the QUIC layer with an UDP layer - next_layer = layers.UDPLayer(self.context, ignore=True) - prev_layer = ( - self - if self._server_quic_layer is None else - self._server_quic_layer - ) - prev_layer.handle_event = next_layer.handle_event - prev_layer._handle_event = next_layer._handle_event - yield from next_layer.handle_event(events.Start()) - yield from next_layer.handle_event(event) - return + # check with addons what we shall do + hook_data = ClientHelloData(self.context, client_hello) + yield layers.tls.TlsClienthelloHook(hook_data) - # start the server QUIC connection if demanded and available - if ( - hook_data.establish_server_tls_first - and not self.context.server.tls_established - ): - err = ( - yield commands.OpenConnection(self.context.server) - if self._server_quic_layer is not None else - "No server QUIC available." - ) - if err: - yield commands.Log( - f"Unable to establish QUIC connection with server ({err}). " - f"Trying to establish QUIC with client anyway. " - f"If you plan to redirect requests away from this server, " - f"consider setting `connection_strategy` to `lazy` to suppress early connections." - ) - - # query addons to provide the necessary TLS settings - tls_data = QuicTlsData(self.context.client, self.context) - yield QuicTlsStartClientHook(tls_data) - if tls_data.settings is None: - yield commands.Log("No client QUIC TLS settings provided by addon(s).", level="error") - return + # replace the QUIC layer with an UDP layer if requested + if hook_data.ignore_connection: + return (yield from self.replace_layer( + replacement_layer=layers.UDPLayer(self.context, ignore=True), + first_data_event=event + )) - # create and register the QUIC connection and handler - handler = QuicConnectionHandler( - context=self.context, - quic=QuicConnection( - configuration=build_configuration(self.context.client, tls_data.settings), - original_destination_connection_id=header.destination_cid, - ) + # start the server QUIC connection if demanded and available + if ( + hook_data.establish_server_tls_first + and not self.context.server.tls_established + ): + err = ( + yield commands.OpenConnection(self.context.server) + if self.server_quic_layer is not None else + "No server QUIC available." ) - ClientQuicLayer.connections_by_id[connection_id] = handler - ClientQuicLayer.connections_by_client[handler.client] = handler + if err: + return ( + f"Unable to establish QUIC connection with server ({err}). " + f"Trying to establish QUIC with client anyway. " + f"If you plan to redirect requests away from this server, " + f"consider setting `connection_strategy` to `lazy` to suppress early connections." + ) - else: - # ensure that the handler is registered with the peer handler - if handler not in self._handlers: - handler.register_handler(self) - self._handlers.add(handler) + # start the client QUIC connection + if not (yield from self.start_tls(connection_id)): + return "TLS initialization failed." - # forward the received packet - handler.receive_data(event.data, self.context.client.peername) + # success, no error + return None - @expect(events.DataReceived, events.ConnectionClosed, events.MessageInjected) - def state_route(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.MessageInjected): - # relay the injection based on the flow's client - ClientQuicLayer.connections_by_client[event.flow.client_conn].server_event(event) + @expect(events.DataReceived, events.ConnectionClosed) + def state_closed_or_failed(self, _) -> layer.CommandGenerator[None]: + yield from () - elif isinstance(event, events.ConnectionClosed): - assert event.connection is self.context.client - # remove and unregister all peer handlers - while self._handlers: - self._handlers.pop().unregister_handler(self) + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + self._handle_event = self.state_wait_for_first_packet + yield from () + + @expect(events.DataReceived, events.ConnectionClosed) + def state_wait_for_first_packet(self, event: events.Event) -> layer.CommandGenerator[None]: + assert isinstance(event, events.ConnectionEvent) + assert event.connection is self.context.client + + if isinstance(event, events.ConnectionClosed): + self._handle_event = self.state_closed_or_failed elif isinstance(event, events.DataReceived): - assert event.connection is self.context.client - yield from self.datagram_received(event) + err = yield from self.datagram_received(event) + if err: + yield commands.Log(err) + yield commands.CloseConnection(self.context.client) + self._handle_event = self.state_closed_or_failed else: raise AssertionError(f"Unexpected event: {event!r}") - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - self._handle_event = self.state_route - yield from () - _handle_event = state_start From 82397c9bbba8b1dab38ea3ca64c1361219eb1fe7 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 5 Sep 2022 22:38:12 +0200 Subject: [PATCH 047/695] [quic] dependency fixes and work on layers --- mitmproxy/addons/next_layer.py | 3 +- mitmproxy/proxy/context.py | 4 +- mitmproxy/proxy/layers/modes.py | 12 +- mitmproxy/proxy/layers/quic.py | 586 +++++++++++++++----------------- 4 files changed, 278 insertions(+), 327 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 4624e5c331..6649253648 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -24,6 +24,7 @@ from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy import context, layer, layers from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers.quic import quic_parse_client_hello from mitmproxy.proxy.layers.tls import HTTP_ALPNS, dtls_parse_client_hello, parse_client_hello from mitmproxy.tls import ClientHello @@ -168,7 +169,7 @@ def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, Clie # next try QUIC try: - client_hello, _ = layers.quic.pull_client_hello_and_connection_id(data_client) + client_hello = quic_parse_client_hello(data_client) return (client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer) except ValueError: pass diff --git a/mitmproxy/proxy/context.py b/mitmproxy/proxy/context.py index 7fcb35340e..3e492b5b23 100644 --- a/mitmproxy/proxy/context.py +++ b/mitmproxy/proxy/context.py @@ -26,7 +26,7 @@ class Context: """ Provides access to options for proxy layers. Not intended for use by addons, use `mitmproxy.ctx.options` instead. """ - handler: Optional[ConnectionHandler] + handler: Optional["ConnectionHandler"] """ The `ConnectionHandler` responsible for this context. """ @@ -39,7 +39,7 @@ def __init__( self, client: connection.Client, options: Options, - handler: Optional[ConnectionHandler] = None, + handler: Optional["ConnectionHandler"] = None, ) -> None: self.client = client self.options = options diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 4e49ef0fc2..d4afc460be 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -59,16 +59,22 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert isinstance(spec, ReverseMode) self.context.server.address = spec.address - if spec.scheme == "https" or spec.scheme == "tls" or spec.scheme == "dtls": + if ( + spec.scheme == "https" or spec.scheme == "http3" + or spec.scheme == "quic" or spec.scheme == "tls" or spec.scheme == "dtls" + ): if not self.context.options.keep_host_header: self.context.server.sni = spec.address[0] - self.child_layer = tls.ServerTLSLayer(self.context) + if (spec.scheme == "http3" or spec.scheme == "quic"): + self.child_layer = quic.ServerQuicLayer(self.context) + else: + self.child_layer = tls.ServerTLSLayer(self.context) elif spec.scheme == "http" or spec.scheme == "tcp" or spec.scheme == "udp": self.child_layer = layer.NextLayer(self.context) elif spec.scheme == "dns": self.child_layer = dns.DNSLayer(self.context) else: - raise AssertionError(self.context.client.transport_protocol) # pragma: no cover + raise AssertionError(spec.scheme) # pragma: no cover err = yield from self.finish_start() if err: diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 5bc430de21..be3f170b02 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -2,7 +2,7 @@ import asyncio from dataclasses import dataclass, field from ssl import VerifyMode -from typing import ClassVar, cast +from typing import TYPE_CHECKING, ClassVar from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import ErrorCode as H3ErrorCode @@ -20,13 +20,17 @@ from aioquic.quic.packet import PACKET_TYPE_INITIAL, QuicHeader, QuicProtocolVersion, encode_quic_version_negotiation, pull_quic_header from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection, ctx, flow as mitm_flow, log, tcp, udp +from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp from mitmproxy.net import tls -from mitmproxy.net.udp import DatagramWriter -from mitmproxy.proxy import commands, context, events, layer, layers, mode_servers, server +from mitmproxy.proxy import commands, context, events, layer +from mitmproxy.proxy.layers.tcp import TcpEndHook, TcpErrorHook, TcpMessageHook, TcpMessageInjected, TcpStartHook +from mitmproxy.proxy.layers.udp import UdpEndHook, UdpErrorHook, UdpMessageHook, UdpMessageInjected, UdpStartHook from mitmproxy.proxy.utils import expect from mitmproxy.tls import ClientHello, ClientHelloData, TlsData -from mitmproxy.utils import asyncio_utils, human +from mitmproxy.utils import human + +if TYPE_CHECKING: + from mitmproxy.proxy import server @dataclass @@ -166,12 +170,12 @@ def is_success_error_code(error_code: int) -> bool: @dataclass class QuicClientHello(Exception): - """Helper error only used in `pull_client_hello_and_connection_id`.""" + """Helper error only used in `quic_parse_client_hello`.""" data: bytes -def pull_client_hello_and_connection_id(data: bytes) -> tuple[ClientHello, bytes]: +def quic_parse_client_hello(data: bytes) -> ClientHello: """Helper function that parses a client hello packet.""" # ensure the first packet is indeed the initial one @@ -215,7 +219,7 @@ def initialize_replacement(peer_cid: bytes) -> None: quic.receive_datagram(data, ("0.0.0.0", 0), now=0) except QuicClientHello as hello: try: - return (ClientHello(hello.data), header.destination_cid) + return ClientHello(hello.data) except EOFError as e: raise ValueError("Invalid ClientHello data.") from e except QuicConnectionError as e: @@ -254,15 +258,15 @@ def mark_ended(self, client: bool, err: str | None = None) -> layer.CommandGener if self.flow is None: return - # report any error if the flow hasn't one already + # set and report the first error if err is not None and self.flow.error is None: self.flow.error = mitm_flow.Error(str) - yield layers.tcp.TcpErrorHook(self.flow) + yield TcpErrorHook(self.flow) # report error-free endings and always clear the live flag if self._ended_client and self._ended_server: if self.flow.error is None: - yield layers.tcp.TcpEndHook(self.flow) + yield TcpEndHook(self.flow) self.flow.live = False @@ -303,7 +307,7 @@ def get_or_create_stream( self.streams_by_id[stream.stream_id] = stream if stream.flow is not None: self.streams_by_flow[stream.flow] = stream - yield layers.tcp.TcpStartHook(stream.flow) + yield TcpStartHook(stream.flow) return stream def handle_quic_event( @@ -323,7 +327,7 @@ def handle_quic_event( if self.flow is not None: message = udp.UDPMessage(from_client, event.data) self.flow.messages.append(message) - yield layers.udp.UdpMessageHook(self.flow) + yield UdpMessageHook(self.flow) data = message.content else: data = event.data @@ -340,7 +344,7 @@ def handle_quic_event( if stream.flow is not None: message = tcp.TCPMessage(from_client, event.data) stream.flow.messages.append(message) - yield layers.tcp.TcpMessageHook(stream.flow) + yield TcpMessageHook(stream.flow) data = message.content else: data = event.data @@ -382,7 +386,7 @@ def handle_quic_event( def state_start(self, _) -> layer.CommandGenerator[None]: # mark the main flow as started if self.flow is not None: - yield layers.udp.UdpStartHook(self.flow) + yield UdpStartHook(self.flow) # open the upstream connection if necessary if self.context.server.timestamp_start is None: @@ -390,7 +394,7 @@ def state_start(self, _) -> layer.CommandGenerator[None]: if err: if self.flow is not None: self.flow.error = mitm_flow.Error(str(err)) - yield layers.udp.UdpErrorHook(self.flow) + yield UdpErrorHook(self.flow) yield commands.CloseConnection(self.context.client) self._handle_event = self.state_done return @@ -399,8 +403,7 @@ def state_start(self, _) -> layer.CommandGenerator[None]: @expect( QuicStart, QuicConnectionEvent, - layers.tcp.TcpMessageInjected, - layers.udp.UdpMessageInjected, + events.MessageInjected, events.ConnectionClosed, ) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -434,7 +437,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: close_event.reason_phrase or error_code_to_str(close_event.error_code) ) - yield layers.udp.UdpErrorHook(self.flow) + yield UdpErrorHook(self.flow) # we're done handling QUIC events, pass on to generic close handling self._handle_event = self.state_done @@ -456,7 +459,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.handle_quic_event(quic_event, not from_client) buffer_from_peer.clear() - elif isinstance(event, layers.tcp.TcpMessageInjected): + elif isinstance(event, TcpMessageInjected): # translate injected TCP messages into QUIC stream events assert isinstance(event.flow, tcp.TCPFlow) stream = self.streams_by_flow[event.flow] @@ -469,7 +472,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: event.message.from_client, ) - elif isinstance(event, layers.udp.UdpMessageInjected): + elif isinstance(event, UdpMessageInjected): # translate injected UDP messages into QUIC datagram events assert isinstance(event.flow, udp.UDPFlow) assert event.flow is self.flow @@ -491,8 +494,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: @expect( QuicStart, QuicConnectionEvent, - layers.tcp.TcpMessageInjected, - layers.udp.UdpMessageInjected, + events.MessageInjected, events.ConnectionClosed, ) def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -511,41 +513,82 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: and not self.context.server.connected ): if self.flow.error is None: - yield layers.udp.UdpEndHook(self.flow) + yield UdpEndHook(self.flow) self.flow.live = False _handle_event = state_start -class QuicConnectionLayer(layer.Layer): +class QuicLayer(layer.Layer): child_layer: layer.Layer conn: connection.Connection quic: QuicConnection | None = None tls: QuicTlsSettings | None = None - def __init__( - self, - context: context.Context, - conn: connection.Connection, - ) -> None: + def __init__(self, context: context.Context, conn: connection.Connection) -> None: super().__init__(context) self.child_layer = layer.NextLayer(context) self.conn = conn self._loop = asyncio.get_event_loop() - self._pending_open_command: commands.OpenConnection | None = None - self._pending_wakeup_commands: dict[commands.RequestWakeup, float] = dict() - self._pending_data_received_events: list[events.DataReceived] = list() - self._routes: dict[connection.Address, server.ConnectionHandler | None] = dict() + self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() + self._routes: dict[connection.Address, "server.ConnectionHandler" | None] = dict() self.conn.tls = True + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if ( + isinstance(event, events.DataReceived) + and event.connection is self.conn + and self.quic is not None + ): + # forward incoming data to aioquic + self.quic.receive_datagram(event.data, self.conn.peername, now=self._loop.time()) + yield from self.process_events() + + elif ( + isinstance(event, events.ConnectionClosed) + and event.connection is self.conn + and self.quic is not None + ): + # there is no point in calling quic.close, as it cannot send packets anymore + # just set the new connection state and ensure there exists a close event + self.quic._set_state(QuicConnectionState.TERMINATED) + close_event = self.quic._close_event + if close_event is None: + close_event = quic_events.ConnectionTerminated( + error_code=QuicErrorCode.APPLICATION_ERROR, + frame_type=None, + reason_phrase="UDP connection closed or timed out.", + ) + self.quic._close_event = close_event + + # simulate the termination event and forward close to the child + yield from self.handle_connection_terminated(close_event) + yield from self.event_to_child(event) + + elif ( + isinstance(event, events.Wakeup) + and event.command in self._wakeup_commands + ): + # remove the command and handle the timer if we have QUIC + timer = self._wakeup_commands.pop(event.command) + if self.quic is not None: + self.quic.handle_timer(now=max(timer, self._loop.time())) + yield from self.process_events() + + else: + # forward other events to the child layer + yield from self.event_to_child(event) + def add_route(self, context: context.Context) -> None: """Registers a new roamed context.""" + assert context.client.peername not in self._routes self._routes[context.client.peername] = context.handler def build_configuration(self) -> QuicConfiguration: - assert self.tls is not None + """Creates the aioquic configuration for the current connection.""" + assert self.tls is not None return QuicConfiguration( alpn_protocols=[offer.decode("ascii") for offer in self.conn.alpn_offers], connection_id_length=self.context.options.quic_connection_id_length, @@ -563,46 +606,48 @@ def build_configuration(self) -> QuicConfiguration: verify_mode=self.tls.verify_mode, ) - def start_tls(self, original_destination_connection_id: bytes | None) -> layer.CommandGenerator[bool]: - # must only be called if QUIC is uninitialized - assert self.quic is None - assert self.tls is None + def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: + """Forwards an event to the child layer and handles/intercepts some commands.""" + + # filter commands coming from the child layer + for command in self.child_layer.handle_event(event): - # in case the connection is being reused, clear all handshake data - self.conn.timestamp_tls_setup = None - self.conn.certificate_list = () - self.conn.alpn = None - self.conn.cipher = None - self.conn.tls_version = None + if ( + isinstance(command, QuicTransmit) + and command.connection is self.conn + ): + # transmit buffered data and re-arm timer + if command.quic is self.quic: + yield from self.transmit() - # query addons to provide the necessary TLS settings - tls_data = QuicTlsData(self.conn, self.context) - if self.conn is self.context.client: - yield QuicTlsStartClientHook(tls_data) - else: - yield QuicTlsStartServerHook(tls_data) - if tls_data.settings is None: - yield commands.Log(f"{self.conn}: No QUIC TLS settings provided by addon(s).", level="error") - return False, + elif ( + isinstance(command, commands.CloseConnection) + and command.connection is self.conn + ): + # without QUIC simply close the connection, otherwise close QUIC first + if self.quic is None: + yield command + else: + self.quic.close(reason_phrase="CloseConnection command received.") + yield from self.process_events() - # create the aioquic connection - self.tls = tls_data.settings - self.quic = QuicConnection( - configuration=self.build_configuration(), - original_destination_connection_id=original_destination_connection_id, - ) - self._handle_event = self.state_has_quic + else: + # return other commands + yield command - # issue the host connection ID right away - self.issue_connection_id(self.quic.host_cid) + def handle_connection_id_issued(self, event: quic_events.ConnectionIdIssued) -> layer.CommandGenerator[None]: + """Called when aioquic issues a new ID for a connection.""" - # record an entry in the log - yield commands.Log(f"{self.conn}: QUIC connection created.", level="debug") - return True + yield from () + + def handle_connection_id_retired(self, event: quic_events.ConnectionIdRetired) -> layer.CommandGenerator[None]: + """Called when aioquic retires an old ID for a connection.""" + + yield from () + + def handle_connection_terminated(self, event: quic_events.ConnectionTerminated) -> layer.CommandGenerator[None]: + """Called when either the aioquic or underlying connection has been closed.""" - def destroy_quic( - self, event: quic_events.ConnectionTerminated - ) -> layer.CommandGenerator[None]: # ensure QUIC has been properly shut down assert self.quic is not None assert self.tls is not None @@ -618,10 +663,8 @@ def destroy_quic( else: yield layers.tls.TlsFailedServerHook(tls_data) - # clear the QUIC fields + # clear only quic, tls will indicate we already did `start_tls` self.quic = None - self.tls = None - self._handle_event = self.state_no_quic # record an entry in the log yield commands.Log( @@ -629,9 +672,13 @@ def destroy_quic( level="debug" if is_success_error_code(event.error_code) else "info", ) - def establish_quic( - self, event: quic_events.HandshakeCompleted - ) -> layer.CommandGenerator[None]: + # also close the connection + if self.conn.connected: + yield commands.CloseConnection(self.conn) + + def handle_handshake_completed(self, event: quic_events.HandshakeCompleted) -> layer.CommandGenerator[None]: + """Called when aioquic finish the QUIC handshake.""" + # must only be called if QUIC is initialized and not established assert self.quic is not None assert self.tls is not None @@ -665,131 +712,42 @@ def establish_quic( level="debug", ) - def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: - # filter commands coming from the child layer - for command in self.child_layer.handle_event(event): - - if ( - isinstance(command, QuicTransmit) - and command.connection is self.conn - ): - # transmit buffered data and re-arm timer - if command.quic is self.quic: - yield from self.transmit() - - elif ( - isinstance(command, commands.OpenConnection) - and command.connection is self.conn - ): - # try to open the QUIC connection and report OpenConnectionCompleted later - yield from self.open_connection_begin(command) - - elif ( - isinstance(command, commands.CloseConnection) - and command.connection is self.conn - ): - # CloseConnection during pending OpenConnection is not allowed - assert self._pending_open_command is None - - # without QUIC simply close the connection, otherwise close QUIC first - if self.quic is None: - yield command - else: - self.quic.close(reason_phrase="CloseConnection command received.") - yield from self.process_events() - - else: - # return other commands - yield command - - def issue_connection_id(self, connection_id: bytes) -> None: - pass - - def open_connection_begin( - self, command: commands.OpenConnection - ) -> layer.CommandGenerator[None]: - # ensure only one OpenConnection is called at a time and only for uninitialized connections - assert self.quic is None - assert self._pending_open_command is None - self._pending_open_command = command - - # try to open the underlying UDP connection and create QUIC - err = yield commands.OpenConnection(self.conn) - if err: - # notify the child layer immediately about the error - self._pending_open_command = None - yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) - - elif not (yield from self.start_tls()): - # TLS failed, close the connection (notify the child layer once its closed) - yield commands.CloseConnection(self.conn) - - else: - # connect to server (notify the child layer after handshake) - assert self.quic is not None - self.quic.connect(self.conn.peername, now=self._loop.time()) - yield from self.process_events() - - def open_connection_end(self, reply: str | None) -> layer.CommandGenerator[bool]: - if self._pending_open_command is None: - return False - - # let the child layer know that the connection is now open (or failed to open) - command = self._pending_open_command - self._pending_open_command = None - yield from self.event_to_child(events.OpenConnectionCompleted(command, reply)) - return True + # notify the child layer about the new connection + yield from self.event_to_child(QuicStart(self.conn, self.quic)) def process_events(self) -> layer.CommandGenerator[None]: - assert self.quic is not None + """ + Retrieves and handles events generated by the aioquic connection. + This method will also call `transmit`. + """ # handle all buffered aioquic connection events + assert self.quic is not None event = self.quic.next_event() while event is not None: if isinstance(event, quic_events.ConnectionIdIssued): - self.issue_connection_id(event.connection_id) - + yield from self.handle_connection_id_issued(event) elif isinstance(event, quic_events.ConnectionIdRetired): - self.retire_connection_id(event.connection_id) - + yield from self.handle_connection_id_retired(event) elif isinstance(event, quic_events.ConnectionTerminated): - # shutdown and close the connection - yield from self.destroy_quic(event) - yield commands.CloseConnection(self.conn) - - # we don't handle any further events, nor do/can we transmit data, so exit - return - + yield from self.handle_connection_terminated(event) + return # we don't handle any further events, nor do/can we transmit data, so exit elif isinstance(event, quic_events.HandshakeCompleted): - # set all TLS fields and notify the child layer - yield from self.establish_quic(event) - yield from self.open_connection_end(None) - yield from self.event_to_child(QuicStart(self.conn, self.quic)) - + yield from self.handle_handshake_completed(event) elif isinstance(event, quic_events.PingAcknowledged): - # we let aioquic do it's thing but don't really care ourselves - pass - + pass # we let aioquic do it's thing but don't really care ourselves elif isinstance(event, quic_events.ProtocolNegotiated): - # too early, we act on HandshakeCompleted - pass - - elif isinstance( - event, - ( - quic_events.DatagramFrameReceived, - quic_events.StreamDataReceived, - quic_events.StreamReset, - ), - ): - # post-handshake event, forward as QuicConnectionEvent to the child layer - assert self.conn.tls_established + pass # too early, we act on HandshakeCompleted + elif isinstance(event, ( + quic_events.DatagramFrameReceived, + quic_events.StreamDataReceived, + quic_events.StreamReset, + )): + assert self.conn.tls_established # must be post-handshake event yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) else: raise AssertionError(f"Unexpected event: {event!r}") - - # handle the next event event = self.quic.next_event() # transmit buffered data and re-arm timer @@ -797,107 +755,46 @@ def process_events(self) -> layer.CommandGenerator[None]: def remove_route(self, context: context.Context) -> None: """Removes a registered roamed context.""" + assert self._routes[context.client.peername] == context.handler del self._routes[context.client.peername] - def retire_connection_id(self, connection_id: bytes) -> None: - pass - - def start(self) -> layer.CommandGenerator[None]: - yield from self.event_to_child(events.Start()) - - def state_after_quic(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.quic is None - - if ( - isinstance(event, events.Wakeup) - and event.command in self._pending_wakeup_commands - ): - # filter out obsolete wakeups - del self._pending_wakeup_commands[event.command] - - else: - # forward all other events to the child layer - yield from self.event_to_child(event) + def start_tls(self, original_destination_connection_id: bytes | None) -> layer.CommandGenerator[bool]: + """Initiates the aioquic connection.""" - def state_before_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + # must only be called if QUIC is uninitialized assert self.quic is None - assert len(self._pending_wakeup_commands) == 0 - - if ( - isinstance(event, events.ConnectionClosed) and event.connection is self.conn - ): - # if there was an OpenConnection command, then create_quic failed - # otherwise the connection was opened before the QUIC layer, so forward the event - if not (yield from self.open_connection_end("QUIC initialization failed")): - yield from self.event_to_child(event) - - elif isinstance(event, events.DataReceived) and event.connection is self.conn: - # buffer data until QUIC is initialized - self._pending_data_received_events.append(event) + assert self.tls is None + # query addons to provide the necessary TLS settings + tls_data = QuicTlsData(self.conn, self.context) + if self.conn is self.context.client: + yield QuicTlsStartClientHook(tls_data) else: - # forward all other events to the child layer - yield from self.event_to_child(event) - - def state_quic(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.quic is not None - - if isinstance(event, events.DataReceived) and event.connection is self.conn: - # forward incoming data only to aioquic - assert event.remote_addr is not None - self.quic.receive_datagram( - event.data, event.remote_addr, now=self._loop.time() - ) - yield from self.process_events() - - elif ( - isinstance(event, events.ConnectionClosed) and event.connection is self.conn - ): - # there is no point in calling quic.close, as it cannot send packets anymore - # just set the new connection state and ensure there exists a close event - self.quic._set_state(QuicConnectionState.TERMINATED) - close_event = self.quic._close_event - if close_event is None: - close_event = quic_events.ConnectionTerminated( - error_code=QuicErrorCode.APPLICATION_ERROR, - frame_type=None, - reason_phrase="UDP connection closed or timed out.", - ) - self.quic._close_event = close_event - - # shutdown QUIC and handle the ConnectionClosed event - yield from self.destroy_quic(close_event) - if not ( - yield from self.open_connection_end("QUIC could not be established") - ): - # connection was opened before QUIC layer, report to the child layer - yield from self.event_to_child(event) + yield QuicTlsStartServerHook(tls_data) + if tls_data.settings is None: + yield commands.Log(f"{self.conn}: No QUIC TLS settings provided by addon(s).", level="error") + return False - elif isinstance(event, events.Wakeup): - # handle issued wakeup commands and forward others to child layer - if event.command in self._pending_wakeup_commands: - self.quic.handle_timer(now=max( - self._pending_wakeup_commands.pop(event.command), - self._loop.time() - )) - yield from self.process_events() - else: - yield from self.event_to_child(event) + # create the aioquic connection + self.tls = tls_data.settings + self.quic = QuicConnection( + configuration=self.build_configuration(), + original_destination_connection_id=original_destination_connection_id, + ) - else: - # forward other events to the child layer - yield from self.event_to_child(event) + # issue the host connection ID right away + self.handle_connection_id_issued(quic_events.ConnectionIdIssued(self.quic.host_cid)) - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - self._handle_event = self.state_before_quic - yield from self.start() + # record an entry in the log + yield commands.Log(f"{self.conn}: QUIC connection created.", level="debug") + return True def transmit(self) -> layer.CommandGenerator[None]: - assert self.quic is not None + """Retrieves all pending outgoing packets from aioquic and sends the data.""" # send all queued datagrams + assert self.quic is not None for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): if addr == self.conn.peername: yield commands.SendData(self.conn, data) @@ -910,15 +807,13 @@ def transmit(self) -> layer.CommandGenerator[None]: # request a new wakeup if all pending requests trigger at a later time timer = self.quic.get_timer() - if not any(existing <= timer for existing in self._pending_wakeup_commands.values()): + if not any(existing <= timer for existing in self._wakeup_commands.values()): command = commands.RequestWakeup(timer - self._loop.time()) - self._pending_wakeup_commands[command] = timer + self._wakeup_commands[command] = timer yield command - _handle_event = state_start - -class ServerQuicLayer(_QuicLayer): +class ServerQuicLayer(QuicLayer): """ This layer establishes QUIC for a single server connection. """ @@ -930,6 +825,64 @@ def __init__( if child_layer is not None: self.child_layer = child_layer + def complete_handshake(self, event: quic_events.HandshakeCompleted) -> layer.CommandGenerator[None]: + yield from super().complete_handshake(event) + yield from self.open_connection_end(None) + + def state_before_quic(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.quic is None + + if ( + isinstance(event, events.ConnectionClosed) and event.connection is self.conn + ): + # if there was an OpenConnection command, then create_quic failed + # otherwise the connection was opened before the QUIC layer, so forward the event + if not (yield from self.open_connection_end("QUIC initialization failed")): + yield from self.event_to_child(event) + + elif isinstance(event, events.DataReceived) and event.connection is self.conn: + # buffer data until QUIC is initialized + self._pending_data_received_events.append(event) + + else: + # forward all other events to the child layer + yield from self.event_to_child(event) + + def open_connection_begin( + self, command: commands.OpenConnection + ) -> layer.CommandGenerator[None]: + # ensure only one OpenConnection is called at a time and only for uninitialized connections + assert self.quic is None + assert self._pending_open_command is None + self._pending_open_command = command + + # try to open the underlying UDP connection and create QUIC + err = yield commands.OpenConnection(self.conn) + if err: + # notify the child layer immediately about the error + self._pending_open_command = None + yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) + + elif not (yield from self.start_tls()): + # TLS failed, close the connection (notify the child layer once its closed) + yield commands.CloseConnection(self.conn) + + else: + # connect to server (notify the child layer after handshake) + assert self.quic is not None + self.quic.connect(self.conn.peername, now=self._loop.time()) + yield from self.process_events() + + def open_connection_end(self, reply: str | None) -> layer.CommandGenerator[bool]: + if self._pending_open_command is None: + return False + + # let the child layer know that the connection is now open (or failed to open) + command = self._pending_open_command + self._pending_open_command = None + yield from self.event_to_child(events.OpenConnectionCompleted(command, reply)) + return True + class QuicRoamingLayer(layer.Layer): """Simple routing layer that replaces a `ClientQuicLayer` when a connection roams to a different `ConnectionHandler`.""" @@ -980,7 +933,7 @@ def state_start(self, _) -> layer.CommandGenerator[None]: _handle_event = state_start -class ClientQuicLayer(QuicConnectionLayer): +class ClientQuicLayer(QuicLayer): """Client-side layer performing routing and connection handling.""" connections: ClassVar[dict[tuple[connection.Address, bytes], ClientQuicLayer]] = dict() @@ -998,11 +951,24 @@ def __init__(self, context: context.Context, can_roam: bool = False) -> None: context.client.mitmcert = None context.client.alpn_offers = [] context.client.cipher_list = [] - super().__init__(context) + super().__init__(context, context.client) self.can_roam = can_roam upper_layer = isinstance(self.context.layers[-2], ServerQuicLayer) self.server_quic_layer = upper_layer if isinstance(upper_layer, ServerQuicLayer) else None + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + # try to initialize TLS + if ( + isinstance(event, events.DataReceived) + and event.connection is self.conn + and self.tls is None + ): + err = yield from self.datagram_received(event) + if err: + yield commands.Log(err) + return + yield from super()._handle_event(event) + def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerator[str | None]: # fail if the received data is not a QUIC packet buffer = QuicBuffer(data=event.data) @@ -1030,14 +996,14 @@ def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerato ) return None - # get or create a handler for the connection - target_layer = ClientQuicLayer.connections((header.destination_cid, self.context.client.sockname), None) + # check if this is a new connection + target_layer = ClientQuicLayer.connections((self.context.client.sockname, header.destination_cid), None) if target_layer is None: - # try to start the client QUIC connection - return (yield from self.start_client_quic(event, header)) + # try to start QUIC + return (yield from self.start_client_tls(event, header)) else: - # ensure that this layer can roam (usually not supported when nested) + # ensure that the layer can roam (usually not supported when nested) if not self.can_roam: return "Connection cannot roam." @@ -1047,6 +1013,18 @@ def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerato first_data_event=event )) + def handle_connection_id_issued(self, event: quic_events.ConnectionIdIssued) -> layer.CommandGenerator[None]: + connection_id = (self.context.client.sockname, event.connection_id) + assert connection_id not in ClientQuicLayer.connections + ClientQuicLayer.connections[connection_id] = self + yield from super().handle_connection_id_issued(event) + + def handle_connection_id_retired(self, event: quic_events.ConnectionIdRetired) -> layer.CommandGenerator[None]: + connection_id = (self.context.client.sockname, event.connection_id) + assert ClientQuicLayer.connections[connection_id] == self + del ClientQuicLayer.connections[connection_id] + yield from super().handle_connection_id_retired(event) + def replace_layer(self, replacement_layer: layer.Layer, first_data_event: events.DataReceived) -> layer.CommandGenerator[None]: # we need to replace the server layer as well, if there is one layer_to_replace = ( @@ -1059,16 +1037,14 @@ def replace_layer(self, replacement_layer: layer.Layer, first_data_event: events yield from replacement_layer.handle_event(events.Start()) yield from replacement_layer.handle_event(first_data_event) - def start_client_quic(self, event: events.DataReceived, header: QuicHeader) -> layer.CommandGenerator[str | None]: + def start_client_tls(self, event: events.DataReceived, header: QuicHeader) -> layer.CommandGenerator[str | None]: # ensure it's (likely) a client handshake packet if len(event.data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: return "Invalid handshake received." # extract the client hello try: - client_hello, connection_id = pull_client_hello_and_connection_id( - event.data - ) + client_hello = quic_parse_client_hello(event.data) except ValueError as e: return f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" @@ -1098,7 +1074,7 @@ def start_client_quic(self, event: events.DataReceived, header: QuicHeader) -> l "No server QUIC available." ) if err: - return ( + yield commands.Log( f"Unable to establish QUIC connection with server ({err}). " f"Trying to establish QUIC with client anyway. " f"If you plan to redirect requests away from this server, " @@ -1106,37 +1082,5 @@ def start_client_quic(self, event: events.DataReceived, header: QuicHeader) -> l ) # start the client QUIC connection - if not (yield from self.start_tls(connection_id)): + if not (yield from self.start_tls(header.destination_cid)): return "TLS initialization failed." - - # success, no error - return None - - @expect(events.DataReceived, events.ConnectionClosed) - def state_closed_or_failed(self, _) -> layer.CommandGenerator[None]: - yield from () - - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - self._handle_event = self.state_wait_for_first_packet - yield from () - - @expect(events.DataReceived, events.ConnectionClosed) - def state_wait_for_first_packet(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, events.ConnectionEvent) - assert event.connection is self.context.client - - if isinstance(event, events.ConnectionClosed): - self._handle_event = self.state_closed_or_failed - - elif isinstance(event, events.DataReceived): - err = yield from self.datagram_received(event) - if err: - yield commands.Log(err) - yield commands.CloseConnection(self.context.client) - self._handle_event = self.state_closed_or_failed - - else: - raise AssertionError(f"Unexpected event: {event!r}") - - _handle_event = state_start From 52f40eb10d3c550ba5dc5676fdbf8ab8b7ee6e50 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 6 Sep 2022 15:05:30 +0200 Subject: [PATCH 048/695] [quic] complete server layer --- mitmproxy/addons/next_layer.py | 2 +- mitmproxy/proxy/layers/http/__init__.py | 17 +- mitmproxy/proxy/layers/quic.py | 214 +++++++++++++----------- 3 files changed, 131 insertions(+), 102 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 6649253648..9c007797fe 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -30,7 +30,7 @@ LayerCls = type[layer.Layer] ClientSecurityLayerCls = Union[type[layers.ClientTLSLayer], type[layers.ClientQuicLayer]] -ServerSecurityLayerCls = Union[type[layers.ServerTLSLayer], type[layers.ClientQuicLayer]] +ServerSecurityLayerCls = Union[type[layers.ServerTLSLayer], type[layers.ServerQuicLayer]] def stack_match( diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 715783ca7b..bc241fb2af 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -7,7 +7,7 @@ import wsproto.handshake from mitmproxy import flow, http -from mitmproxy.connection import Connection, Server +from mitmproxy.connection import Connection, Server, TransportProtocol from mitmproxy.net import server_spec from mitmproxy.net.http import status_codes, url from mitmproxy.net.http.http1 import expected_http_body_size @@ -78,6 +78,7 @@ class GetHttpConnection(HttpCommand): address: tuple[str, int] tls: bool via: Optional[server_spec.ServerSpec] + transport_protocol: TransportProtocol = "tcp" def __hash__(self): return id(self) @@ -88,6 +89,7 @@ def connection_spec_matches(self, connection: Connection) -> bool: and self.address == connection.address and self.tls == connection.tls and self.via == connection.via + and self.transport_protocol == connection.transport_protocol ) @@ -666,6 +668,7 @@ def make_server_connection(self) -> layer.CommandGenerator[bool]: (self.flow.request.host, self.flow.request.port), self.flow.request.scheme == "https", self.flow.server_conn.via, + self.flow.server_conn.transport_protocol, ) if err: yield from self.handle_protocol_error( @@ -1005,10 +1008,7 @@ def get_connection( if not can_use_context_connection: - context.server = Server(event.address) - if isinstance(context.layers[0], quic.QuicLayer): - context.server.transport_protocol = "udp" - stack /= quic.ServerQuicLayer(context) + context.server = Server(event.address, transport_protocol=event.transport_protocol) if event.via: context.server.via = event.via @@ -1025,7 +1025,12 @@ def get_connection( context.server.sni = self.context.client.sni or event.address[0] else: context.server.sni = event.address[0] - stack /= tls.ServerTLSLayer(context) + if context.server.transport_protocol == "tcp": + stack /= tls.ServerTLSLayer(context) + elif context.server.transport_protocol == "udp": + stack /= quic.ServerQuicLayer(context) + else: + raise AssertionError(context.server.transport_protocol) # pragma: no cover stack /= HttpClient(context) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index be3f170b02..ca3a932569 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -17,20 +17,46 @@ stream_is_unidirectional, ) from aioquic.tls import CipherSuite, HandshakeType -from aioquic.quic.packet import PACKET_TYPE_INITIAL, QuicHeader, QuicProtocolVersion, encode_quic_version_negotiation, pull_quic_header +from aioquic.quic.packet import ( + PACKET_TYPE_INITIAL, + QuicHeader, + QuicProtocolVersion, + encode_quic_version_negotiation, + pull_quic_header, +) from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer -from mitmproxy.proxy.layers.tcp import TcpEndHook, TcpErrorHook, TcpMessageHook, TcpMessageInjected, TcpStartHook -from mitmproxy.proxy.layers.udp import UdpEndHook, UdpErrorHook, UdpMessageHook, UdpMessageInjected, UdpStartHook +from mitmproxy.proxy.layers.tcp import ( + TcpEndHook, + TcpErrorHook, + TcpMessageHook, + TcpMessageInjected, + TcpStartHook, +) +from mitmproxy.proxy.layers.tls import ( + TlsClienthelloHook, + TlsEstablishedClientHook, + TlsEstablishedServerHook, + TlsFailedClientHook, + TlsFailedServerHook, +) +from mitmproxy.proxy.layers.udp import ( + UDPLayer, + UdpEndHook, + UdpErrorHook, + UdpMessageHook, + UdpMessageInjected, + UdpStartHook, +) from mitmproxy.proxy.utils import expect from mitmproxy.tls import ClientHello, ClientHelloData, TlsData from mitmproxy.utils import human if TYPE_CHECKING: - from mitmproxy.proxy import server + from mitmproxy.proxy.server import ConnectionHandler @dataclass @@ -43,9 +69,7 @@ class QuicTlsSettings: """The certificate to use for the connection.""" certificate_chain: list[x509.Certificate] = field(default_factory=list) """A list of additional certificates to send to the peer.""" - certificate_private_key: dsa.DSAPrivateKey | ec.EllipticCurvePrivateKey | rsa.RSAPrivateKey | None = ( - None - ) + certificate_private_key: dsa.DSAPrivateKey | ec.EllipticCurvePrivateKey | rsa.RSAPrivateKey | None = None """The certificate's private key.""" cipher_suites: list[CipherSuite] | None = None """An optional list of allowed/advertised cipher suites.""" @@ -260,7 +284,7 @@ def mark_ended(self, client: bool, err: str | None = None) -> layer.CommandGener # set and report the first error if err is not None and self.flow.error is None: - self.flow.error = mitm_flow.Error(str) + self.flow.error = mitm_flow.Error(err) yield TcpErrorHook(self.flow) # report error-free endings and always clear the live flag @@ -292,7 +316,7 @@ def __init__(self, context: context.Context, ignore: bool = False) -> None: if ignore: self.flow = None else: - self.flow = tcp.TCPFlow(self.context.client, self.context.server, live=True) + self.flow = udp.UDPFlow(self.context.client, self.context.server, live=True) self.streams_by_flow = {} self.streams_by_id = {} @@ -325,10 +349,10 @@ def handle_quic_event( if isinstance(event, quic_events.DatagramFrameReceived): # forward datagrams (that are not stream-bound) if self.flow is not None: - message = udp.UDPMessage(from_client, event.data) - self.flow.messages.append(message) + udp_message = udp.UDPMessage(from_client, event.data) + self.flow.messages.append(udp_message) yield UdpMessageHook(self.flow) - data = message.content + data = udp_message.content else: data = event.data peer_quic.send_datagram_frame(data) @@ -342,10 +366,10 @@ def handle_quic_event( # forward the message allowing addons to change it if stream.flow is not None: - message = tcp.TCPMessage(from_client, event.data) - stream.flow.messages.append(message) + tcp_message = tcp.TCPMessage(from_client, event.data) + stream.flow.messages.append(tcp_message) yield TcpMessageHook(stream.flow) - data = message.content + data = tcp_message.content else: data = event.data peer_quic.send_stream_data( @@ -531,7 +555,7 @@ def __init__(self, context: context.Context, conn: connection.Connection) -> Non self.conn = conn self._loop = asyncio.get_event_loop() self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() - self._routes: dict[connection.Address, "server.ConnectionHandler" | None] = dict() + self._routes: dict[connection.Address, "ConnectionHandler" | None] = dict() self.conn.tls = True def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -659,9 +683,9 @@ def handle_connection_terminated(self, event: quic_events.ConnectionTerminated) self.conn.error = reason tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) if self.conn is self.context.client: - yield layers.tls.TlsFailedClientHook(tls_data) + yield TlsFailedClientHook(tls_data) else: - yield layers.tls.TlsFailedServerHook(tls_data) + yield TlsFailedServerHook(tls_data) # clear only quic, tls will indicate we already did `start_tls` self.quic = None @@ -672,10 +696,6 @@ def handle_connection_terminated(self, event: quic_events.ConnectionTerminated) level="debug" if is_success_error_code(event.error_code) else "info", ) - # also close the connection - if self.conn.connected: - yield commands.CloseConnection(self.conn) - def handle_handshake_completed(self, event: quic_events.HandshakeCompleted) -> layer.CommandGenerator[None]: """Called when aioquic finish the QUIC handshake.""" @@ -701,9 +721,9 @@ def handle_handshake_completed(self, event: quic_events.HandshakeCompleted) -> l # report the success to addons tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) if self.conn is self.context.client: - yield layers.tls.TlsEstablishedClientHook(tls_data) + yield TlsEstablishedClientHook(tls_data) else: - yield layers.tls.TlsEstablishedServerHook(tls_data) + yield TlsEstablishedServerHook(tls_data) # record an entry in the log yield commands.Log( @@ -712,9 +732,6 @@ def handle_handshake_completed(self, event: quic_events.HandshakeCompleted) -> l level="debug", ) - # notify the child layer about the new connection - yield from self.event_to_child(QuicStart(self.conn, self.quic)) - def process_events(self) -> layer.CommandGenerator[None]: """ Retrieves and handles events generated by the aioquic connection. @@ -731,9 +748,12 @@ def process_events(self) -> layer.CommandGenerator[None]: yield from self.handle_connection_id_retired(event) elif isinstance(event, quic_events.ConnectionTerminated): yield from self.handle_connection_terminated(event) + if self.conn.connected: + yield commands.CloseConnection(self.conn) # also close the connection return # we don't handle any further events, nor do/can we transmit data, so exit elif isinstance(event, quic_events.HandshakeCompleted): yield from self.handle_handshake_completed(event) + yield from self.event_to_child(QuicStart(self.conn, self.quic)) # notify child layer elif isinstance(event, quic_events.PingAcknowledged): pass # we let aioquic do it's thing but don't really care ourselves elif isinstance(event, quic_events.ProtocolNegotiated): @@ -803,7 +823,9 @@ def transmit(self) -> layer.CommandGenerator[None]: if handler is None: yield commands.Log(f"{self.conn}: No route to {human.format_address(addr)}.") else: - handler.transports[handler.client].writer(data) + writer = handler.transports[handler.client].writer + assert writer is not None + writer.write(data) # request a new wakeup if all pending requests trigger at a later time timer = self.quic.get_timer() @@ -818,70 +840,68 @@ class ServerQuicLayer(QuicLayer): This layer establishes QUIC for a single server connection. """ - def __init__( - self, context: context.Context, child_layer: layer.Layer | None = None - ) -> None: + def __init__(self, context: context.Context) -> None: super().__init__(context, context.server) - if child_layer is not None: - self.child_layer = child_layer - - def complete_handshake(self, event: quic_events.HandshakeCompleted) -> layer.CommandGenerator[None]: - yield from super().complete_handshake(event) - yield from self.open_connection_end(None) - - def state_before_quic(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.quic is None + self._open_command: commands.OpenConnection | None = None + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if ( - isinstance(event, events.ConnectionClosed) and event.connection is self.conn + isinstance(event, events.ConnectionClosed) + and event.connection is self.conn + and self._open_command is not None ): - # if there was an OpenConnection command, then create_quic failed - # otherwise the connection was opened before the QUIC layer, so forward the event - if not (yield from self.open_connection_end("QUIC initialization failed")): - yield from self.event_to_child(event) - - elif isinstance(event, events.DataReceived) and event.connection is self.conn: - # buffer data until QUIC is initialized - self._pending_data_received_events.append(event) - + response = events.OpenConnectionCompleted( + self._open_command, + ( + "TLS initialization failed" + if self.tls is None else + "Connection closed before connect" + ) + ) + self._open_command = None + yield from self.event_to_child(response) else: - # forward all other events to the child layer - yield from self.event_to_child(event) + yield from super()._handle_event(event) - def open_connection_begin( - self, command: commands.OpenConnection - ) -> layer.CommandGenerator[None]: - # ensure only one OpenConnection is called at a time and only for uninitialized connections - assert self.quic is None - assert self._pending_open_command is None - self._pending_open_command = command + def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: + for command in super().event_to_child(event): + if ( + isinstance(command, commands.OpenConnection) + and command.connection is self.conn + and self.tls is None + ): + # store the command + assert self._open_command is None + self._open_command = command - # try to open the underlying UDP connection and create QUIC - err = yield commands.OpenConnection(self.conn) - if err: - # notify the child layer immediately about the error - self._pending_open_command = None - yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) + # try to connect with upstream + err = yield commands.OpenConnection(self.conn) + if err: + # notify the child layer immediately about the error + self._open_command = None + yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) - elif not (yield from self.start_tls()): - # TLS failed, close the connection (notify the child layer once its closed) - yield commands.CloseConnection(self.conn) + elif not (yield from self.start_tls(None)): + # TLS failed, close the connection (notify the child layer once its closed) + yield commands.CloseConnection(self.conn) - else: - # connect to server (notify the child layer after handshake) - assert self.quic is not None - self.quic.connect(self.conn.peername, now=self._loop.time()) - yield from self.process_events() + else: + # connect to server (notify the child layer after handshake) + assert self.quic is not None + self.quic.connect(self.conn.peername, now=self._loop.time()) + yield from self.process_events() - def open_connection_end(self, reply: str | None) -> layer.CommandGenerator[bool]: - if self._pending_open_command is None: - return False + else: + yield command - # let the child layer know that the connection is now open (or failed to open) - command = self._pending_open_command - self._pending_open_command = None - yield from self.event_to_child(events.OpenConnectionCompleted(command, reply)) - return True + def handle_handshake_completed(self, event: quic_events.HandshakeCompleted) -> layer.CommandGenerator[None]: + yield from super().handle_handshake_completed(event) + + # notify the child layer that the connection is now open + if self._open_command is not None: + command = self._open_command + self._open_command = None + yield from self.event_to_child(events.OpenConnectionCompleted(command, None)) class QuicRoamingLayer(layer.Layer): @@ -952,9 +972,9 @@ def __init__(self, context: context.Context, can_roam: bool = False) -> None: context.client.alpn_offers = [] context.client.cipher_list = [] super().__init__(context, context.client) - self.can_roam = can_roam - upper_layer = isinstance(self.context.layers[-2], ServerQuicLayer) - self.server_quic_layer = upper_layer if isinstance(upper_layer, ServerQuicLayer) else None + self._can_roam = can_roam + upper_layer = self.context.layers[-2] + self._server_layer = upper_layer if isinstance(upper_layer, ServerQuicLayer) else None def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # try to initialize TLS @@ -997,14 +1017,14 @@ def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerato return None # check if this is a new connection - target_layer = ClientQuicLayer.connections((self.context.client.sockname, header.destination_cid), None) + target_layer = ClientQuicLayer.connections.get((self.context.client.sockname, header.destination_cid), None) if target_layer is None: # try to start QUIC return (yield from self.start_client_tls(event, header)) else: # ensure that the layer can roam (usually not supported when nested) - if not self.can_roam: + if not self._can_roam: return "Connection cannot roam." # replace the layer with a roaming layer @@ -1025,17 +1045,18 @@ def handle_connection_id_retired(self, event: quic_events.ConnectionIdRetired) - del ClientQuicLayer.connections[connection_id] yield from super().handle_connection_id_retired(event) - def replace_layer(self, replacement_layer: layer.Layer, first_data_event: events.DataReceived) -> layer.CommandGenerator[None]: + def replace_layer(self, replacement_layer: layer.Layer, first_data_event: events.DataReceived) -> layer.CommandGenerator[str | None]: # we need to replace the server layer as well, if there is one layer_to_replace = ( self - if self.server_quic_layer is None else - self.server_quic_layer + if self._server_layer is None else + self._server_layer ) - layer_to_replace.handle_event = replacement_layer.handle_event - layer_to_replace._handle_event = replacement_layer._handle_event + layer_to_replace.handle_event = replacement_layer.handle_event # type:ignore + layer_to_replace._handle_event = replacement_layer._handle_event # type:ignore yield from replacement_layer.handle_event(events.Start()) yield from replacement_layer.handle_event(first_data_event) + return None def start_client_tls(self, event: events.DataReceived, header: QuicHeader) -> layer.CommandGenerator[str | None]: # ensure it's (likely) a client handshake packet @@ -1054,12 +1075,12 @@ def start_client_tls(self, event: events.DataReceived, header: QuicHeader) -> la # check with addons what we shall do hook_data = ClientHelloData(self.context, client_hello) - yield layers.tls.TlsClienthelloHook(hook_data) + yield TlsClienthelloHook(hook_data) # replace the QUIC layer with an UDP layer if requested if hook_data.ignore_connection: return (yield from self.replace_layer( - replacement_layer=layers.UDPLayer(self.context, ignore=True), + replacement_layer=UDPLayer(self.context, ignore=True), first_data_event=event )) @@ -1069,8 +1090,8 @@ def start_client_tls(self, event: events.DataReceived, header: QuicHeader) -> la and not self.context.server.tls_established ): err = ( - yield commands.OpenConnection(self.context.server) - if self.server_quic_layer is not None else + (yield commands.OpenConnection(self.context.server)) + if self._server_layer is not None else "No server QUIC available." ) if err: @@ -1084,3 +1105,6 @@ def start_client_tls(self, event: events.DataReceived, header: QuicHeader) -> la # start the client QUIC connection if not (yield from self.start_tls(header.destination_cid)): return "TLS initialization failed." + + # success + return None From 6b9dcc47b9a808356a70988ced9484c18fca83e0 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 7 Sep 2022 02:55:09 +0200 Subject: [PATCH 049/695] [quic] rewrite to match TunnelLayer --- mitmproxy/addons/next_layer.py | 3 +- mitmproxy/addons/proxyserver.py | 6 + mitmproxy/proxy/layers/http/_http3.py | 4 +- mitmproxy/proxy/layers/quic.py | 739 ++++++++++++-------------- 4 files changed, 344 insertions(+), 408 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 9c007797fe..a1dbf286c5 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -260,7 +260,8 @@ def s(*layers): # 2. Check for DTLS/QUIC if tls is not None: - return self.setup_tls_layer(context, *tls[1:2]) + _, client_layer_cls, server_layer_cls = tls + return self.setup_tls_layer(context, client_layer_cls, server_layer_cls) # 3. Setup the HTTP layer for a regular HTTP proxy if s(modes.HttpProxy, layers.ClientQuicLayer): diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index b70759dfb7..1bd8b1df1c 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -190,6 +190,12 @@ def load(self, loader): None, """Set the local IP address that mitmproxy should use when connecting to upstream servers.""", ) + loader.add_option( + "quic_connection_id_length", + int, + 8, + """The length in bytes of local QUIC connection IDs.""", + ) def running(self): self.is_running = True diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 29cc6e3585..d7b29c4892 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -132,7 +132,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: else: # transmit buffered data and re-arm timer - yield QuicTransmit(self.conn, self.quic) + yield QuicTransmit(self.conn) # handle events from the underlying QUIC connection elif isinstance(event, QuicConnectionEvent): @@ -210,7 +210,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, reason_phrase=f"Invalid HTTP/3 request headers: {e}", ) - yield QuicTransmit(self.conn, self.quic) + yield QuicTransmit(self.conn) else: yield ReceiveHttp( self.postprocess_outgoing_event(receive_event) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index ca3a932569..37ebac0347 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -11,7 +11,6 @@ from aioquic.quic.connection import ( QuicConnection, QuicConnectionError, - QuicConnectionState, QuicErrorCode, stream_is_client_initiated, stream_is_unidirectional, @@ -28,7 +27,7 @@ from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp from mitmproxy.net import tls -from mitmproxy.proxy import commands, context, events, layer +from mitmproxy.proxy import commands, context, events, layer, tunnel from mitmproxy.proxy.layers.tcp import ( TcpEndHook, TcpErrorHook, @@ -145,14 +144,11 @@ def __init__(self, connection: connection.Connection, quic: QuicConnection) -> N self.quic = quic -class QuicTransmit(commands.ConnectionCommand): +class QuicTransmit(commands.SendData): """Command that will transmit buffered data and re-arm the given QUIC connection's timer.""" - quic: QuicConnection - - def __init__(self, connection: connection.Connection, quic: QuicConnection) -> None: - super().__init__(connection) - self.quic = quic + def __init__(self, connection: connection.Connection) -> None: + super().__init__(connection, b"") class QuicSecretsLogger: @@ -404,7 +400,7 @@ def handle_quic_event( return # transmit data to the peer - yield QuicTransmit(peer_connection, peer_quic) + yield QuicTransmit(peer_connection) @expect(events.Start) def state_start(self, _) -> layer.CommandGenerator[None]: @@ -447,7 +443,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: close_event.frame_type, close_event.reason_phrase, ) - yield QuicTransmit(peer_conn, peer_quic) + yield QuicTransmit(peer_conn) else: yield commands.CloseConnection(peer_conn) @@ -543,65 +539,80 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: _handle_event = state_start -class QuicLayer(layer.Layer): +class QuicRoamingLayer(layer.Layer): + """Simple routing layer that replaces a `ClientQuicLayer` when a connection roams to a different `ConnectionHandler`.""" + + def __init__(self, context: context.Context, target_layer: ClientQuicLayer) -> None: + super().__init__(context) + self.target_layer = target_layer + + @expect() + def state_closed(self, _) -> layer.CommandGenerator[None]: + yield from () + + @expect(events.DataReceived, events.ConnectionClosed, events.MessageInjected) + def state_relay(self, event: events.Event) -> layer.CommandGenerator[None]: + + if isinstance(event, events.MessageInjected): + # ensure the flow matches the target and forward the event + assert event.flow.client_conn is self.target_layer.context.client + handler = self.target_layer.context.handler + assert handler + handler.server_event(event) + + elif isinstance(event, events.ConnectionClosed): + # remove the registration and stop relaying + assert event.connection is self.context.client + self.target_layer.remove_route(self.context) + self._handle_event = self.state_closed + + elif isinstance(event, events.DataReceived): + # update target's peername and forward the event + assert event.connection is self.context.client + handler = self.target_layer.context.handler + assert handler + handler.client.peername = self.context.client.peername + handler.server_event(events.DataReceived(handler.client, event.data)) + + else: + raise AssertionError(f"Unexpected event: {event!r}") + + yield from () + + @expect(events.Start) + def state_start(self, _) -> layer.CommandGenerator[None]: + # register with the target and start relaying + self.target_layer.add_route(self.context) + self._handle_event = self.state_relay + yield from () + + _handle_event = state_start + + +class QuicLayer(tunnel.TunnelLayer): child_layer: layer.Layer conn: connection.Connection quic: QuicConnection | None = None tls: QuicTlsSettings | None = None def __init__(self, context: context.Context, conn: connection.Connection) -> None: - super().__init__(context) - self.child_layer = layer.NextLayer(context) - self.conn = conn + super().__init__(context, tunnel_connection=conn, conn=conn) self._loop = asyncio.get_event_loop() self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() self._routes: dict[connection.Address, "ConnectionHandler" | None] = dict() - self.conn.tls = True + conn.tls = True def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + # turn Wakeup events into empty DataReceived events if ( - isinstance(event, events.DataReceived) - and event.connection is self.conn - and self.quic is not None - ): - # forward incoming data to aioquic - self.quic.receive_datagram(event.data, self.conn.peername, now=self._loop.time()) - yield from self.process_events() - - elif ( - isinstance(event, events.ConnectionClosed) - and event.connection is self.conn - and self.quic is not None - ): - # there is no point in calling quic.close, as it cannot send packets anymore - # just set the new connection state and ensure there exists a close event - self.quic._set_state(QuicConnectionState.TERMINATED) - close_event = self.quic._close_event - if close_event is None: - close_event = quic_events.ConnectionTerminated( - error_code=QuicErrorCode.APPLICATION_ERROR, - frame_type=None, - reason_phrase="UDP connection closed or timed out.", - ) - self.quic._close_event = close_event - - # simulate the termination event and forward close to the child - yield from self.handle_connection_terminated(close_event) - yield from self.event_to_child(event) - - elif ( isinstance(event, events.Wakeup) and event.command in self._wakeup_commands ): - # remove the command and handle the timer if we have QUIC + assert self.quic timer = self._wakeup_commands.pop(event.command) - if self.quic is not None: - self.quic.handle_timer(now=max(timer, self._loop.time())) - yield from self.process_events() - - else: - # forward other events to the child layer - yield from self.event_to_child(event) + self.quic.handle_timer(now=max(timer, self._loop.time())) + event = events.DataReceived(self.tunnel_connection, b"") + yield from super()._handle_event(event) def add_route(self, context: context.Context) -> None: """Registers a new roamed context.""" @@ -609,177 +620,23 @@ def add_route(self, context: context.Context) -> None: assert context.client.peername not in self._routes self._routes[context.client.peername] = context.handler - def build_configuration(self) -> QuicConfiguration: - """Creates the aioquic configuration for the current connection.""" - - assert self.tls is not None - return QuicConfiguration( - alpn_protocols=[offer.decode("ascii") for offer in self.conn.alpn_offers], - connection_id_length=self.context.options.quic_connection_id_length, - is_client=self.conn is self.context.server, - secrets_log_file=QuicSecretsLogger(tls.log_master_secret) # type: ignore - if tls.log_master_secret is not None - else None, - server_name=self.conn.sni, - cafile=self.tls.ca_file, - capath=self.tls.ca_path, - certificate=self.tls.certificate, - certificate_chain=self.tls.certificate_chain, - cipher_suites=self.tls.cipher_suites, - private_key=self.tls.certificate_private_key, - verify_mode=self.tls.verify_mode, - ) - - def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: - """Forwards an event to the child layer and handles/intercepts some commands.""" - - # filter commands coming from the child layer - for command in self.child_layer.handle_event(event): - - if ( - isinstance(command, QuicTransmit) - and command.connection is self.conn - ): - # transmit buffered data and re-arm timer - if command.quic is self.quic: - yield from self.transmit() - - elif ( - isinstance(command, commands.CloseConnection) - and command.connection is self.conn - ): - # without QUIC simply close the connection, otherwise close QUIC first - if self.quic is None: - yield command - else: - self.quic.close(reason_phrase="CloseConnection command received.") - yield from self.process_events() + def remove_route(self, context: context.Context) -> None: + """Removes a registered roamed context.""" - else: - # return other commands - yield command + assert self._routes[context.client.peername] == context.handler + del self._routes[context.client.peername] - def handle_connection_id_issued(self, event: quic_events.ConnectionIdIssued) -> layer.CommandGenerator[None]: + def issue_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: """Called when aioquic issues a new ID for a connection.""" yield from () - def handle_connection_id_retired(self, event: quic_events.ConnectionIdRetired) -> layer.CommandGenerator[None]: + def retire_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: """Called when aioquic retires an old ID for a connection.""" yield from () - def handle_connection_terminated(self, event: quic_events.ConnectionTerminated) -> layer.CommandGenerator[None]: - """Called when either the aioquic or underlying connection has been closed.""" - - # ensure QUIC has been properly shut down - assert self.quic is not None - assert self.tls is not None - assert self.quic._state is QuicConnectionState.TERMINATED - - # report as TLS failure if the termination happened before the handshake - reason = event.reason_phrase or error_code_to_str(event.error_code) - if not self.conn.tls_established: - self.conn.error = reason - tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) - if self.conn is self.context.client: - yield TlsFailedClientHook(tls_data) - else: - yield TlsFailedServerHook(tls_data) - - # clear only quic, tls will indicate we already did `start_tls` - self.quic = None - - # record an entry in the log - yield commands.Log( - f"{self.conn}: QUIC connection destroyed: {reason}", - level="debug" if is_success_error_code(event.error_code) else "info", - ) - - def handle_handshake_completed(self, event: quic_events.HandshakeCompleted) -> layer.CommandGenerator[None]: - """Called when aioquic finish the QUIC handshake.""" - - # must only be called if QUIC is initialized and not established - assert self.quic is not None - assert self.tls is not None - assert not self.conn.tls_established - - # concatenate all peer certificates - all_certs: list[x509.Certificate] = [] - if self.quic.tls._peer_certificate is not None: - all_certs.append(self.quic.tls._peer_certificate) - if self.quic.tls._peer_certificate_chain is not None: - all_certs.extend(self.quic.tls._peer_certificate_chain) - - # set the connection's TLS properties - self.conn.timestamp_tls_setup = self._loop.time() - self.conn.certificate_list = [certs.Cert(cert) for cert in all_certs] - self.conn.alpn = event.alpn_protocol.encode("ascii") - self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name - self.conn.tls_version = "QUIC" - - # report the success to addons - tls_data = QuicTlsData(self.conn, self.context, settings=self.tls) - if self.conn is self.context.client: - yield TlsEstablishedClientHook(tls_data) - else: - yield TlsEstablishedServerHook(tls_data) - - # record an entry in the log - yield commands.Log( - f"{self.conn}: QUIC connection established. " - f"(early_data={event.early_data_accepted}, resumed={event.session_resumed})", - level="debug", - ) - - def process_events(self) -> layer.CommandGenerator[None]: - """ - Retrieves and handles events generated by the aioquic connection. - This method will also call `transmit`. - """ - - # handle all buffered aioquic connection events - assert self.quic is not None - event = self.quic.next_event() - while event is not None: - if isinstance(event, quic_events.ConnectionIdIssued): - yield from self.handle_connection_id_issued(event) - elif isinstance(event, quic_events.ConnectionIdRetired): - yield from self.handle_connection_id_retired(event) - elif isinstance(event, quic_events.ConnectionTerminated): - yield from self.handle_connection_terminated(event) - if self.conn.connected: - yield commands.CloseConnection(self.conn) # also close the connection - return # we don't handle any further events, nor do/can we transmit data, so exit - elif isinstance(event, quic_events.HandshakeCompleted): - yield from self.handle_handshake_completed(event) - yield from self.event_to_child(QuicStart(self.conn, self.quic)) # notify child layer - elif isinstance(event, quic_events.PingAcknowledged): - pass # we let aioquic do it's thing but don't really care ourselves - elif isinstance(event, quic_events.ProtocolNegotiated): - pass # too early, we act on HandshakeCompleted - elif isinstance(event, ( - quic_events.DatagramFrameReceived, - quic_events.StreamDataReceived, - quic_events.StreamReset, - )): - assert self.conn.tls_established # must be post-handshake event - yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) - - else: - raise AssertionError(f"Unexpected event: {event!r}") - event = self.quic.next_event() - - # transmit buffered data and re-arm timer - yield from self.transmit() - - def remove_route(self, context: context.Context) -> None: - """Removes a registered roamed context.""" - - assert self._routes[context.client.peername] == context.handler - del self._routes[context.client.peername] - - def start_tls(self, original_destination_connection_id: bytes | None) -> layer.CommandGenerator[bool]: + def start_tls(self, original_destination_connection_id: bytes | None) -> layer.CommandGenerator[None]: """Initiates the aioquic connection.""" # must only be called if QUIC is uninitialized @@ -793,24 +650,42 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C else: yield QuicTlsStartServerHook(tls_data) if tls_data.settings is None: - yield commands.Log(f"{self.conn}: No QUIC TLS settings provided by addon(s).", level="error") - return False + yield commands.Log(f"No QUIC context was provided, failing connection.", level="error") + yield commands.CloseConnection(self.conn) + return - # create the aioquic connection - self.tls = tls_data.settings + # build the aioquic connection + configuration = QuicConfiguration( + alpn_protocols=[offer.decode("ascii") for offer in self.conn.alpn_offers], + connection_id_length=self.context.options.quic_connection_id_length, + is_client=self.conn is self.context.server, + secrets_log_file=QuicSecretsLogger(tls.log_master_secret) # type: ignore + if tls.log_master_secret is not None + else None, + server_name=self.conn.sni, + cafile=tls_data.settings.ca_file, + capath=tls_data.settings.ca_path, + certificate=tls_data.settings.certificate, + certificate_chain=tls_data.settings.certificate_chain, + cipher_suites=tls_data.settings.cipher_suites, + private_key=tls_data.settings.certificate_private_key, + verify_mode=tls_data.settings.verify_mode, + ) self.quic = QuicConnection( - configuration=self.build_configuration(), + configuration=configuration, original_destination_connection_id=original_destination_connection_id, ) + self.tls = tls_data.settings - # issue the host connection ID right away - self.handle_connection_id_issued(quic_events.ConnectionIdIssued(self.quic.host_cid)) + # if we act as client, connect to upstream + if original_destination_connection_id is None: + self.quic.connect(self.conn.peername, now=self._loop.time()) + yield from self.tls_interact() - # record an entry in the log - yield commands.Log(f"{self.conn}: QUIC connection created.", level="debug") - return True + # issue the host connection ID right away + self.issue_connection_id(quic_events.ConnectionIdIssued(self.quic.host_cid)) - def transmit(self) -> layer.CommandGenerator[None]: + def tls_interact(self) -> layer.CommandGenerator[None]: """Retrieves all pending outgoing packets from aioquic and sends the data.""" # send all queued datagrams @@ -829,137 +704,185 @@ def transmit(self) -> layer.CommandGenerator[None]: # request a new wakeup if all pending requests trigger at a later time timer = self.quic.get_timer() - if not any(existing <= timer for existing in self._wakeup_commands.values()): + if ( + timer is not None + and not any(existing <= timer for existing in self._wakeup_commands.values()) + ): command = commands.RequestWakeup(timer - self._loop.time()) self._wakeup_commands[command] = timer yield command + def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bool, str | None]]: + assert self.quic -class ServerQuicLayer(QuicLayer): - """ - This layer establishes QUIC for a single server connection. - """ - - def __init__(self, context: context.Context) -> None: - super().__init__(context, context.server) - self._open_command: commands.OpenConnection | None = None - - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - if ( - isinstance(event, events.ConnectionClosed) - and event.connection is self.conn - and self._open_command is not None - ): - response = events.OpenConnectionCompleted( - self._open_command, - ( - "TLS initialization failed" - if self.tls is None else - "Connection closed before connect" - ) - ) - self._open_command = None - yield from self.event_to_child(response) - else: - yield from super()._handle_event(event) - - def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: - for command in super().event_to_child(event): - if ( - isinstance(command, commands.OpenConnection) - and command.connection is self.conn - and self.tls is None - ): - # store the command - assert self._open_command is None - self._open_command = command - - # try to connect with upstream - err = yield commands.OpenConnection(self.conn) - if err: - # notify the child layer immediately about the error - self._open_command = None - yield from self.event_to_child(events.OpenConnectionCompleted(command, err)) - - elif not (yield from self.start_tls(None)): - # TLS failed, close the connection (notify the child layer once its closed) - yield commands.CloseConnection(self.conn) + # forward incoming data to aioquic + if data: + self.quic.receive_datagram(data, self.conn.peername, now=self._loop.time()) + # handle pre-handshake events + event = self.quic.next_event() + while event is not None: + if isinstance(event, quic_events.ConnectionIdIssued): + yield from self.issue_connection_id(event.connection_id) + elif isinstance(event, quic_events.ConnectionIdRetired): + yield from self.retire_connection_id(event.connection_id) + elif isinstance(event, quic_events.ConnectionTerminated): + err = event.reason_phrase or error_code_to_str(event.error_code) + return False, err + elif isinstance(event, quic_events.HandshakeCompleted): + # concatenate all peer certificates + all_certs: list[x509.Certificate] = [] + if self.quic.tls._peer_certificate is not None: + all_certs.append(self.quic.tls._peer_certificate) + if self.quic.tls._peer_certificate_chain is not None: + all_certs.extend(self.quic.tls._peer_certificate_chain) + + # set the connection's TLS properties + self.conn.timestamp_tls_setup = self._loop.time() + self.conn.alpn = event.alpn_protocol.encode("ascii") + self.conn.certificate_list = [certs.Cert(cert) for cert in all_certs] + self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name + self.conn.tls_version = "QUIC" + + # log the result and report the success to addons + if self.debug: + yield commands.Log( + f"{self.debug}[quic] established: {self.conn}", "debug" + ) + if self.conn is self.context.client: + yield TlsEstablishedClientHook(QuicTlsData(self.conn, self.context, settings=self.tls)) else: - # connect to server (notify the child layer after handshake) - assert self.quic is not None - self.quic.connect(self.conn.peername, now=self._loop.time()) - yield from self.process_events() - + yield TlsEstablishedServerHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + + # notify child layer, transmit and return success + yield from self.event_to_child(QuicStart(self.conn, self.quic)) + yield from self.tls_interact() + return True, None + elif isinstance(event, (quic_events.PingAcknowledged, quic_events.ProtocolNegotiated)): + pass else: - yield command + raise AssertionError(f"Unexpected event: {event!r}") + event = self.quic.next_event() - def handle_handshake_completed(self, event: quic_events.HandshakeCompleted) -> layer.CommandGenerator[None]: - yield from super().handle_handshake_completed(event) + # transmit buffered data and re-arm timer + yield from self.tls_interact() + return False, None - # notify the child layer that the connection is now open - if self._open_command is not None: - command = self._open_command - self._open_command = None - yield from self.event_to_child(events.OpenConnectionCompleted(command, None)) + def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: + self.conn.error = err + if self.conn is self.context.client: + yield TlsFailedClientHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + else: + yield TlsFailedServerHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + yield from super().on_handshake_error(err) + def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: + assert self.quic -class QuicRoamingLayer(layer.Layer): - """Simple routing layer that replaces a `ClientQuicLayer` when a connection roams to a different `ConnectionHandler`.""" + # forward incoming data to aioquic + if data: + self.quic.receive_datagram(data, self.conn.peername, now=self._loop.time()) - def __init__(self, context: context.Context, target_layer: ClientQuicLayer) -> None: - super().__init__(context) - self.target_layer = target_layer + # handle post-handshake events + event = self.quic.next_event() + while event is not None: + if isinstance(event, quic_events.ConnectionIdIssued): + yield from self.issue_connection_id(event.connection_id) + elif isinstance(event, quic_events.ConnectionIdRetired): + yield from self.retire_connection_id(event.connection_id) + elif isinstance(event, quic_events.ConnectionTerminated): + if self.debug: + reason = event.reason_phrase or error_code_to_str(event.error_code) + yield commands.Log( + f"{self.debug}[quic] terminated {self.conn} (reason={reason})", level="debug" + ) + yield commands.CloseConnection(self.conn) + return # we don't handle any further events, nor do/can we transmit data, so exit + elif isinstance(event, quic_events.PingAcknowledged): + pass + elif isinstance(event, ( + quic_events.DatagramFrameReceived, + quic_events.StreamDataReceived, + quic_events.StreamReset, + )): + yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) + else: + raise AssertionError(f"Unexpected event: {event!r}") + event = self.quic.next_event() - @expect() - def state_closed(self, _) -> layer.CommandGenerator[None]: - yield from () + # transmit buffered data and re-arm timer + yield from self.tls_interact() + + def receive_close(self) -> layer.CommandGenerator[None]: + # unlike TLS we haven't sent CloseConnection before + yield from super().receive_close() + + def send_data(self, data: bytes) -> layer.CommandGenerator[None]: + # no actual data is allowed, SendData (i.e. QuicTransmit) functions only as trigger + assert not data + yield from self.tls_interact() + + def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: + # properly close a QUIC connection, no half-close allowed + assert not half_close + if self.quic is not None: + self.quic.close() + yield from self.tls_interact() + else: + yield from super().send_close(half_close) - @expect(events.DataReceived, events.ConnectionClosed, events.MessageInjected) - def state_relay(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.MessageInjected): - # ensure the flow matches the target and forward the event - assert event.flow.client_conn is self.target_layer.context.client - handler = self.target_layer.context.handler - assert handler - handler.server_event(event) - elif isinstance(event, events.ConnectionClosed): - # remove the registration and stop relaying - assert event.connection is self.context.client - self.target_layer.remove_route(self.context) - self._handle_event = self.state_closed +class ServerQuicLayer(QuicLayer): + """ + This layer establishes QUIC for a single server connection. + """ - elif isinstance(event, events.DataReceived): - # update target's peername and forward the event - assert event.connection is self.context.client - handler = self.target_layer.context.handler - assert handler - handler.client.peername = self.context.client.peername - handler.server_event(events.DataReceived(handler.client, event.data)) + wait_for_clienthello: bool = False - else: - raise AssertionError(f"Unexpected event: {event!r}") + def __init__(self, context: context.Context, conn: connection.Server | None = None): + super().__init__(context, conn or context.server) - yield from () + def start_handshake(self) -> layer.CommandGenerator[None]: + wait_for_clienthello = ( + not self.command_to_reply_to + and isinstance(self.child_layer, ClientQuicLayer) + ) + if wait_for_clienthello: + self.wait_for_clienthello = True + self.tunnel_state = tunnel.TunnelState.CLOSED + else: + yield from self.start_tls(None) - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - # register with the target and start relaying - self.target_layer.add_route(self.context) - self._handle_event = self.state_relay - yield from () + def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: + if self.wait_for_clienthello: + for command in super().event_to_child(event): + if ( + isinstance(command, commands.OpenConnection) + and command.connection == self.conn + ): + self.wait_for_clienthello = False + else: + yield command + else: + yield from super().event_to_child(event) - _handle_event = state_start + def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: + yield commands.Log(f"Server QUIC handshake failed. {err}", level="warn") + yield from super().on_handshake_error(err) class ClientQuicLayer(QuicLayer): - """Client-side layer performing routing and connection handling.""" + """ + This layer establishes QUIC on a single client connection or roams to another connection. + """ connections: ClassVar[dict[tuple[connection.Address, bytes], ClientQuicLayer]] = dict() """Mapping of (sockname, cid) tuples to QUIC client layers.""" - def __init__(self, context: context.Context, can_roam: bool = False) -> None: + server_layer: ServerQuicLayer | None + is_top_level: bool + + def __init__(self, context: context.Context) -> None: # same as ClientTLSLayer, we might be nested in some other transport if context.client.tls: context.client.alpn = None @@ -971,33 +894,51 @@ def __init__(self, context: context.Context, can_roam: bool = False) -> None: context.client.mitmcert = None context.client.alpn_offers = [] context.client.cipher_list = [] + super().__init__(context, context.client) - self._can_roam = can_roam - upper_layer = self.context.layers[-2] - self._server_layer = upper_layer if isinstance(upper_layer, ServerQuicLayer) else None + parent_layer = self.context.layers[-2] + self.server_layer = parent_layer if isinstance(parent_layer, ServerQuicLayer) else None + self.is_top_level = len(context.layers) == (2 if self.server_layer is None else 3) + + def issue_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: + if self.is_top_level: + cid = (self.context.client.sockname, connection_id) + assert cid not in ClientQuicLayer.connections + ClientQuicLayer.connections[cid] = self + yield from super().issue_connection_id(connection_id) + + def retire_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: + if self.is_top_level: + cid = (self.context.client.sockname, connection_id) + assert ClientQuicLayer.connections[cid] == self + del ClientQuicLayer.connections[cid] + yield from super().retire_connection_id(connection_id) + + def replace_layer(self, initial_data: bytes, replacement_layer: layer.Layer) -> layer.CommandGenerator[tuple[bool, str | None]]: + # we need to replace the server layer as well, if there is one + layer_to_replace = self if self.server_layer is None else self.server_layer + layer_to_replace.handle_event = replacement_layer.handle_event # type: ignore + layer_to_replace._handle_event = replacement_layer._handle_event # type: ignore + yield from replacement_layer.handle_event(events.Start()) + yield from replacement_layer.handle_event(events.DataReceived(self.conn, initial_data)) + return True, None - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - # try to initialize TLS - if ( - isinstance(event, events.DataReceived) - and event.connection is self.conn - and self.tls is None - ): - err = yield from self.datagram_received(event) - if err: - yield commands.Log(err) - return - yield from super()._handle_event(event) + def start_handshake(self) -> layer.CommandGenerator[None]: + yield from () + + def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bool, str | None]]: + # if we already had a valid client hello, don't process further packets + if self.tls is not None: + return (yield from super().receive_handshake_data(data)) - def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerator[str | None]: # fail if the received data is not a QUIC packet - buffer = QuicBuffer(data=event.data) + buffer = QuicBuffer(data=data) try: header = pull_quic_header( buffer, host_cid_length=self.context.options.quic_connection_id_length ) - except ValueError: - return "Invalid QUIC datagram received." + except ValueError as e: + return False, f"Cannot parse QUIC header: {e} ({data.hex()})" # negotiate version, support all versions known to aioquic supported_versions = ( @@ -1007,93 +948,65 @@ def datagram_received(self, event: events.DataReceived) -> layer.CommandGenerato ) if header.version is not None and header.version not in supported_versions: yield commands.SendData( - event.connection, + self.conn, encode_quic_version_negotiation( source_cid=header.destination_cid, destination_cid=header.source_cid, supported_versions=supported_versions, ), ) - return None + return False, None # check if this is a new connection target_layer = ClientQuicLayer.connections.get((self.context.client.sockname, header.destination_cid), None) if target_layer is None: # try to start QUIC - return (yield from self.start_client_tls(event, header)) + return (yield from self.start_client_tls(data, header)) else: - # ensure that the layer can roam (usually not supported when nested) - if not self._can_roam: - return "Connection cannot roam." + # ensure that we can roam + if (self.is_top_level and target_layer.context.client.proxy_mode is self.context.client.proxy_mode): + return False, "Connection cannot roam." # replace the layer with a roaming layer - return (yield from self.replace_layer( - replacement_layer=QuicRoamingLayer(self.context, target_layer), - first_data_event=event - )) - - def handle_connection_id_issued(self, event: quic_events.ConnectionIdIssued) -> layer.CommandGenerator[None]: - connection_id = (self.context.client.sockname, event.connection_id) - assert connection_id not in ClientQuicLayer.connections - ClientQuicLayer.connections[connection_id] = self - yield from super().handle_connection_id_issued(event) - - def handle_connection_id_retired(self, event: quic_events.ConnectionIdRetired) -> layer.CommandGenerator[None]: - connection_id = (self.context.client.sockname, event.connection_id) - assert ClientQuicLayer.connections[connection_id] == self - del ClientQuicLayer.connections[connection_id] - yield from super().handle_connection_id_retired(event) - - def replace_layer(self, replacement_layer: layer.Layer, first_data_event: events.DataReceived) -> layer.CommandGenerator[str | None]: - # we need to replace the server layer as well, if there is one - layer_to_replace = ( - self - if self._server_layer is None else - self._server_layer - ) - layer_to_replace.handle_event = replacement_layer.handle_event # type:ignore - layer_to_replace._handle_event = replacement_layer._handle_event # type:ignore - yield from replacement_layer.handle_event(events.Start()) - yield from replacement_layer.handle_event(first_data_event) - return None + return (yield from self.replace_layer(data, QuicRoamingLayer(self.context, target_layer))) - def start_client_tls(self, event: events.DataReceived, header: QuicHeader) -> layer.CommandGenerator[str | None]: + def start_client_tls(self, data: bytes, header: QuicHeader) -> layer.CommandGenerator[tuple[bool, str | None]]: # ensure it's (likely) a client handshake packet - if len(event.data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: - return "Invalid handshake received." + if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: + return False, f"Invalid handshake received. ({data.hex()})" # extract the client hello try: - client_hello = quic_parse_client_hello(event.data) + client_hello = quic_parse_client_hello(data) except ValueError as e: - return f"Cannot parse ClientHello: {str(e)} ({event.data.hex()})" + return False, f"Cannot parse ClientHello: {str(e)} ({data.hex()})" # copy the client hello information self.context.client.sni = client_hello.sni self.context.client.alpn_offers = client_hello.alpn_protocols # check with addons what we shall do - hook_data = ClientHelloData(self.context, client_hello) - yield TlsClienthelloHook(hook_data) + tls_clienthello = ClientHelloData(self.context, client_hello) + yield TlsClienthelloHook(tls_clienthello) # replace the QUIC layer with an UDP layer if requested - if hook_data.ignore_connection: - return (yield from self.replace_layer( - replacement_layer=UDPLayer(self.context, ignore=True), - first_data_event=event - )) + if tls_clienthello.ignore_connection: + self.conn = self.tunnel_connection = connection.Client( + ("ignore-conn", 0), ("ignore-conn", 0), self._loop.time() + ) + if self.server_layer is not None: + self.server_layer.conn = self.server_layer.tunnel_connection = connection.Server( + None + ) + return (yield from self.replace_layer(data, UDPLayer(self.context, ignore=True))) # start the server QUIC connection if demanded and available if ( - hook_data.establish_server_tls_first + tls_clienthello.establish_server_tls_first and not self.context.server.tls_established ): - err = ( - (yield commands.OpenConnection(self.context.server)) - if self._server_layer is not None else - "No server QUIC available." - ) + err = yield from self.start_server_tls() if err: yield commands.Log( f"Unable to establish QUIC connection with server ({err}). " @@ -1103,8 +1016,24 @@ def start_client_tls(self, event: events.DataReceived, header: QuicHeader) -> la ) # start the client QUIC connection - if not (yield from self.start_tls(header.destination_cid)): - return "TLS initialization failed." - - # success - return None + yield from self.start_tls(header.destination_cid) + if not self.conn.connected: + return False, "connection closed early" + + # send the client hello to aioquic + return (yield from super().receive_handshake_data(data)) + + def start_server_tls(self) -> layer.CommandGenerator[str | None]: + if self.server_layer is None: + return f"No server QUIC available." + err = yield commands.OpenConnection(self.context.server) + return err + + def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: + yield commands.Log(f"Client QUIC handshake failed. {err}", level="warn") + yield from super().on_handshake_error(err) + self.event_to_child = self.errored # type: ignore + + def errored(self, event: events.Event) -> layer.CommandGenerator[None]: + if self.debug is not None: + yield commands.Log(f"Swallowing {event} as handshake failed.", "debug") From 41e56327da6dbe622e0f7394ea72f91d673c1496 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 7 Sep 2022 03:52:23 +0200 Subject: [PATCH 050/695] [quic] runnable again --- mitmproxy/proxy/layers/http/_http3.py | 66 ++++++++++++++------------- mitmproxy/proxy/layers/quic.py | 21 ++++++--- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index d7b29c4892..32dd02a4e8 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -11,6 +11,7 @@ from aioquic.h3 import events as h3_events from aioquic.quic import events as quic_events from aioquic.quic.connection import QuicConnection, stream_is_client_initiated +from aioquic.quic.packet import QuicErrorCode from mitmproxy import http, version from mitmproxy.net.http import status_codes @@ -168,9 +169,10 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: for h3_event in self.h3_conn.handle_event(event.event): # report received data - if isinstance( - h3_event, h3_events.DataReceived - ) and stream_is_client_initiated(h3_event.stream_id): + if ( + isinstance(h3_event, h3_events.DataReceived) + and stream_is_client_initiated(h3_event.stream_id) + ): yield ReceiveHttp( self.postprocess_outgoing_event( self.ReceiveData( @@ -186,13 +188,11 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: ) # report headers and trailers - elif isinstance( - h3_event, h3_events.HeadersReceived - ) and stream_is_client_initiated(h3_event.stream_id): - if ( - self.h3_conn._stream[h3_event.stream_id].headers_recv_state - is H3HeadersState.AFTER_TRAILERS - ): + elif ( + isinstance(h3_event, h3_events.HeadersReceived) + and stream_is_client_initiated(h3_event.stream_id) + ): + if self.h3_conn._stream[h3_event.stream_id].headers_recv_state is H3HeadersState.AFTER_TRAILERS: yield ReceiveHttp( self.postprocess_outgoing_event( self.ReceiveTrailers( @@ -226,20 +226,34 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # we don't support push, web transport, etc. else: - yield commands.Log(f"Ignored unsupported H3 event: {h3_event!r}") + yield commands.Log( + f"Ignored unsupported H3 event: {h3_event!r}", + level=( + "info" + if stream_is_client_initiated(h3_event.stream_id) else + "debug" + ) + ) # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): for stream in self.h3_conn._stream.values(): if stream_is_client_initiated(stream.stream_id) and not stream.ended: close_event = self.quic._close_event - assert close_event is not None yield ReceiveHttp( self.postprocess_outgoing_event( self.ReceiveProtocolError( stream_id=stream.stream_id, - message=close_event.reason_phrase, - code=close_event.error_code, + message=( + "Connection closed." + if close_event is None else + close_event.reason_phrase + ), + code=( + QuicErrorCode.APPLICATION_ERROR + if close_event is None else + close_event.error_code + ), ) ) ) @@ -274,9 +288,7 @@ class Http3Server(Http3Connection): def __init__(self, context: context.Context): super().__init__(context, context.client) - def parse_headers( - self, event: h3_events.HeadersReceived - ) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers(self, event: h3_events.HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 ( host, @@ -301,13 +313,9 @@ def parse_headers( timestamp_start=time.time(), timestamp_end=None, ) - return RequestHeaders( - stream_id=event.stream_id, request=request, end_stream=event.stream_ended - ) + return RequestHeaders(stream_id=event.stream_id, request=request, end_stream=event.stream_ended) - def send_protocol_error( - self, event: Union[RequestProtocolError, ResponseProtocolError] - ) -> None: + def send_protocol_error(self, event: Union[RequestProtocolError, ResponseProtocolError]) -> None: assert self.h3_conn is not None assert isinstance(event, ResponseProtocolError) @@ -341,9 +349,7 @@ def __init__(self, context: context.Context): self._event_to_quic: Dict[int, int] = {} self._quic_to_event: Dict[int, int] = {} - def parse_headers( - self, event: h3_events.HeadersReceived - ) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers(self, event: h3_events.HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 status_code, headers = parse_h2_response_headers(event.headers) response = http.Response( @@ -356,9 +362,7 @@ def parse_headers( timestamp_start=time.time(), timestamp_end=None, ) - return ResponseHeaders( - stream_id=event.stream_id, response=response, end_stream=event.stream_ended - ) + return ResponseHeaders(stream_id=event.stream_id, response=response, end_stream=event.stream_ended) def postprocess_outgoing_event(self, event: HttpEvent) -> HttpEvent: event.stream_id = self._quic_to_event[event.stream_id] @@ -378,9 +382,7 @@ def preprocess_incoming_event(self, event: HttpEvent) -> HttpEvent: event.stream_id = stream_id return event - def send_protocol_error( - self, event: Union[RequestProtocolError, ResponseProtocolError] - ) -> None: + def send_protocol_error(self, event: Union[RequestProtocolError, ResponseProtocolError]) -> None: assert isinstance(event, RequestProtocolError) assert self.quic is not None diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 37ebac0347..4f1f5d837d 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, ClassVar from aioquic.buffer import Buffer as QuicBuffer -from aioquic.h3.connection import ErrorCode as H3ErrorCode +from aioquic.h3.connection import H3_ALPN, ErrorCode as H3ErrorCode from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import ( @@ -656,12 +656,18 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C # build the aioquic connection configuration = QuicConfiguration( - alpn_protocols=[offer.decode("ascii") for offer in self.conn.alpn_offers], + alpn_protocols=( + [offer.decode("ascii") for offer in self.conn.alpn_offers] + if self.conn.alpn_offers else + H3_ALPN + ), connection_id_length=self.context.options.quic_connection_id_length, is_client=self.conn is self.context.server, - secrets_log_file=QuicSecretsLogger(tls.log_master_secret) # type: ignore - if tls.log_master_secret is not None - else None, + secrets_log_file=( + QuicSecretsLogger(tls.log_master_secret) # type: ignore + if tls.log_master_secret is not None + else None + ), server_name=self.conn.sni, cafile=tls_data.settings.ca_file, capath=tls_data.settings.ca_path, @@ -915,6 +921,8 @@ def retire_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[N yield from super().retire_connection_id(connection_id) def replace_layer(self, initial_data: bytes, replacement_layer: layer.Layer) -> layer.CommandGenerator[tuple[bool, str | None]]: + """Replaces the QUIC layer(s) with another layer.""" + # we need to replace the server layer as well, if there is one layer_to_replace = self if self.server_layer is None else self.server_layer layer_to_replace.handle_event = replacement_layer.handle_event # type: ignore @@ -993,7 +1001,8 @@ def start_client_tls(self, data: bytes, header: QuicHeader) -> layer.CommandGene # replace the QUIC layer with an UDP layer if requested if tls_clienthello.ignore_connection: self.conn = self.tunnel_connection = connection.Client( - ("ignore-conn", 0), ("ignore-conn", 0), self._loop.time() + ("ignore-conn", 0), ("ignore-conn", 0), self._loop.time(), + transport_protocol="udp", proxy_mode=self.context.client.proxy_mode ) if self.server_layer is not None: self.server_layer.conn = self.server_layer.tunnel_connection = connection.Server( From e4f4a393328581db0c7d6e98ff3fe3d20415e1b1 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 7 Sep 2022 13:35:00 +0200 Subject: [PATCH 051/695] [quic] minor improvements --- mitmproxy/certs.py | 2 +- mitmproxy/proxy/layers/quic.py | 16 ++++++++++------ mitmproxy/proxy/layers/tls.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 0a84f88848..f982e3cf89 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -321,7 +321,7 @@ def __init__( ) ] if self.default_chain_file - else [] + else [default_ca] ) self.dhparams = dhparams self.certs = {} diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 4f1f5d837d..7da65d7952 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -11,6 +11,7 @@ from aioquic.quic.connection import ( QuicConnection, QuicConnectionError, + QuicConnectionState, QuicErrorCode, stream_is_client_initiated, stream_is_unidirectional, @@ -610,8 +611,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ): assert self.quic timer = self._wakeup_commands.pop(event.command) - self.quic.handle_timer(now=max(timer, self._loop.time())) - event = events.DataReceived(self.tunnel_connection, b"") + if self.quic._state is not QuicConnectionState.TERMINATED: + self.quic.handle_timer(now=max(timer, self._loop.time())) + event = events.DataReceived(self.tunnel_connection, b"") yield from super()._handle_event(event) def add_route(self, context: context.Context) -> None: @@ -753,7 +755,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # log the result and report the success to addons if self.debug: yield commands.Log( - f"{self.debug}[quic] established: {self.conn}", "debug" + f"{self.debug}[quic] tls established: {self.conn}", "debug" ) if self.conn is self.context.client: yield TlsEstablishedClientHook(QuicTlsData(self.conn, self.context, settings=self.tls)) @@ -800,7 +802,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: if self.debug: reason = event.reason_phrase or error_code_to_str(event.error_code) yield commands.Log( - f"{self.debug}[quic] terminated {self.conn} (reason={reason})", level="debug" + f"{self.debug}[quic] close_notify {self.conn} (reason={reason})", level="debug" ) yield commands.CloseConnection(self.conn) return # we don't handle any further events, nor do/can we transmit data, so exit @@ -886,7 +888,9 @@ class ClientQuicLayer(QuicLayer): """Mapping of (sockname, cid) tuples to QUIC client layers.""" server_layer: ServerQuicLayer | None + """The server layer sitting on top of this layer, or `None`.""" is_top_level: bool + """Indicated whether this layer is receiving UDP packets directly.""" def __init__(self, context: context.Context) -> None: # same as ClientTLSLayer, we might be nested in some other transport @@ -956,7 +960,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo ) if header.version is not None and header.version not in supported_versions: yield commands.SendData( - self.conn, + self.tunnel_connection, encode_quic_version_negotiation( source_cid=header.destination_cid, destination_cid=header.source_cid, @@ -1045,4 +1049,4 @@ def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: def errored(self, event: events.Event) -> layer.CommandGenerator[None]: if self.debug is not None: - yield commands.Log(f"Swallowing {event} as handshake failed.", "debug") + yield commands.Log(f"{self.debug}[quic] Swallowing {event} as handshake failed.", "debug") diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index c4d92e3d26..22f45750c3 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -657,7 +657,7 @@ def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: def errored(self, event: events.Event) -> layer.CommandGenerator[None]: if self.debug is not None: - yield commands.Log(f"Swallowing {event} as handshake failed.", "debug") + yield commands.Log(f"{self.debug}[tls] Swallowing {event} as handshake failed.", "debug") class MockTLSLayer(TLSLayer): From 8770e8a9a712dbd5e1333946155b161364201cd2 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 8 Sep 2022 14:16:23 +0200 Subject: [PATCH 052/695] Update mitmproxy/proxy/layers/modes.py Co-authored-by: Maximilian Hils --- mitmproxy/proxy/layers/modes.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index d4afc460be..3b3869c8e0 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -59,10 +59,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert isinstance(spec, ReverseMode) self.context.server.address = spec.address - if ( - spec.scheme == "https" or spec.scheme == "http3" - or spec.scheme == "quic" or spec.scheme == "tls" or spec.scheme == "dtls" - ): + if spec.scheme in ("https", "http3", "quic", "tls", "dtls"): if not self.context.options.keep_host_header: self.context.server.sni = spec.address[0] if (spec.scheme == "http3" or spec.scheme == "quic"): From ce34957938e2897e3a923dc904b5123cb8c9859d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 9 Sep 2022 19:22:00 +0200 Subject: [PATCH 053/695] fix nits --- mitmproxy/addons/tlsconfig.py | 4 ++-- mitmproxy/proxy/layers/quic.py | 22 +++++++++------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 093b158302..9c32412eda 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -300,7 +300,7 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: tls_start.ssl_conn.set_connect_state() - def quic_tls_start_client(self, tls_start: quic.QuicTlsData) -> None: + def quic_start_client(self, tls_start: quic.QuicTlsData) -> None: """Establish QUIC between client and proxy.""" if tls_start.settings is not None: return # a user addon has already provided the settings. @@ -326,7 +326,7 @@ def quic_tls_start_client(self, tls_start: quic.QuicTlsData) -> None: if ctx.options.add_upstream_certs_to_client_chain: tls_start.settings.certificate_chain.extend(cert._cert for cert in server.certificate_list) - def quic_tls_start_server(self, tls_start: quic.QuicTlsData) -> None: + def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: """Establish QUIC between proxy and server.""" if tls_start.settings is not None: return # a user addon has already provided the settings. diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 7da65d7952..77ab4f40c6 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -84,18 +84,18 @@ class QuicTlsSettings: @dataclass class QuicTlsData(TlsData): """ - Event data for `quic_tls_start_client` and `quic_tls_start_server` event hooks. + Event data for `quic_start_client` and `quic_start_server` event hooks. """ settings: QuicTlsSettings | None = None """ The associated `QuicTlsSettings` object. - This will be set by an addon in the `quic_tls_start_*` event hooks. + This will be set by an addon in the `quic_start_*` event hooks. """ @dataclass -class QuicTlsStartClientHook(commands.StartHook): +class QuicStartClientHook(commands.StartHook): """ TLS negotiation between mitmproxy and a client over QUIC is about to start. @@ -107,7 +107,7 @@ class QuicTlsStartClientHook(commands.StartHook): @dataclass -class QuicTlsStartServerHook(commands.StartHook): +class QuicStartServerHook(commands.StartHook): """ TLS negotiation between mitmproxy and a server over QUIC is about to start. @@ -600,7 +600,7 @@ def __init__(self, context: context.Context, conn: connection.Connection) -> Non super().__init__(context, tunnel_connection=conn, conn=conn) self._loop = asyncio.get_event_loop() self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() - self._routes: dict[connection.Address, "ConnectionHandler" | None] = dict() + self._routes: dict[connection.Address, ConnectionHandler | None] = dict() conn.tls = True def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -648,9 +648,9 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C # query addons to provide the necessary TLS settings tls_data = QuicTlsData(self.conn, self.context) if self.conn is self.context.client: - yield QuicTlsStartClientHook(tls_data) + yield QuicStartClientHook(tls_data) else: - yield QuicTlsStartServerHook(tls_data) + yield QuicStartServerHook(tls_data) if tls_data.settings is None: yield commands.Log(f"No QUIC context was provided, failing connection.", level="error") yield commands.CloseConnection(self.conn) @@ -728,8 +728,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo self.quic.receive_datagram(data, self.conn.peername, now=self._loop.time()) # handle pre-handshake events - event = self.quic.next_event() - while event is not None: + while event := self.quic.next_event(): if isinstance(event, quic_events.ConnectionIdIssued): yield from self.issue_connection_id(event.connection_id) elif isinstance(event, quic_events.ConnectionIdRetired): @@ -770,7 +769,6 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo pass else: raise AssertionError(f"Unexpected event: {event!r}") - event = self.quic.next_event() # transmit buffered data and re-arm timer yield from self.tls_interact() @@ -792,8 +790,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: self.quic.receive_datagram(data, self.conn.peername, now=self._loop.time()) # handle post-handshake events - event = self.quic.next_event() - while event is not None: + while event := self.quic.next_event(): if isinstance(event, quic_events.ConnectionIdIssued): yield from self.issue_connection_id(event.connection_id) elif isinstance(event, quic_events.ConnectionIdRetired): @@ -816,7 +813,6 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) else: raise AssertionError(f"Unexpected event: {event!r}") - event = self.quic.next_event() # transmit buffered data and re-arm timer yield from self.tls_interact() From a164a08b412effeed6963859e1478e18c358f6bd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 9 Sep 2022 19:22:19 +0200 Subject: [PATCH 054/695] remove QuicStart event We shouldn't need this: - QuicStreamLayer can be refactored to spawn TCPLayer and UDPLayer sublayers and translate between connection and quic events. - The HTTP/3 layer ideally shouldn't have direct access to the QUIC connection objects [^1]. That's not the case for aioquic unfortunately, but we can poke through the layer stack instead. [^1]: see https://github.com/quinn-rs/quinn and https://github.com/hyperium/h3 for how the separation should be done --- mitmproxy/proxy/layers/http/_http3.py | 19 ++++---- mitmproxy/proxy/layers/quic.py | 62 ++++++++------------------- 2 files changed, 30 insertions(+), 51 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 32dd02a4e8..9bf7e13a5c 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -18,7 +18,7 @@ from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( QuicConnectionEvent, - QuicStart, + QuicLayer, QuicTransmit, error_code_to_str, ) @@ -265,14 +265,17 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: @expect(events.Start) def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: assert isinstance(event, events.Start) - self._handle_event = self.state_wait_for_quic - yield from () - @expect(QuicStart) - def state_wait_for_quic(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, QuicStart) - self.quic = event.quic - self.h3_conn = H3Connection(self.quic, enable_webtransport=False) + # aioquic does not separate QUIC and HTTP/3, poke through the layer stack to get a reference to the QUIC + # connection object. + for x in reversed(self.context.layers): + if isinstance(x, QuicLayer): + self.quic = x.quic + self.h3_conn = H3Connection(self.quic, enable_webtransport=False) + break + else: + raise AssertionError + self._handle_event = self.state_ready yield from () diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 77ab4f40c6..109969661e 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -132,22 +132,13 @@ class QuicConnectionEvent(events.ConnectionEvent): event: quic_events.QuicEvent -class QuicStart(events.DataReceived): +class QuicTransmit(commands.SendData): """ - Event that indicates that QUIC has been established on a given connection. - This inherits from `DataReceived` in order to trigger next layer behavior and initialize HTTP clients. + aioquic does not separate HTTP/3 and QUIC: H3Connection requires a QuicConnection instance to interact with. + This unfortunately breaks our abstractions. As a workaround, all "SendData" commands for HTTP/3 connections + do not carry the actual data to be sent, but just serve to notify the QUIC layer instead. """ - quic: QuicConnection - - def __init__(self, connection: connection.Connection, quic: QuicConnection) -> None: - super().__init__(connection, data=b"") - self.quic = quic - - -class QuicTransmit(commands.SendData): - """Command that will transmit buffered data and re-arm the given QUIC connection's timer.""" - def __init__(self, connection: connection.Connection) -> None: super().__init__(connection, b"") @@ -297,9 +288,6 @@ class QuicStreamLayer(layer.Layer): This layer is chosen by the default NextLayer addon if ALPN yields no known protocol. It uses `UDPFlow` and `TCPFlow` for datagrams and stream respectively, which makes message injection possible. """ - - buffer_from_client: list[quic_events.QuicEvent] - buffer_from_server: list[quic_events.QuicEvent] flow: udp.UDPFlow | None # used for datagrams and to signal general connection issues quic_client: QuicConnection | None = None quic_server: QuicConnection | None = None @@ -308,8 +296,6 @@ class QuicStreamLayer(layer.Layer): def __init__(self, context: context.Context, ignore: bool = False) -> None: super().__init__(context) - self.buffer_from_client = [] - self.buffer_from_server = [] if ignore: self.flow = None else: @@ -338,9 +324,7 @@ def handle_quic_event( ) -> layer.CommandGenerator[None]: # buffer events if the peer is not ready yet peer_quic = self.quic_server if from_client else self.quic_client - if peer_quic is None: - (self.buffer_from_client if from_client else self.buffer_from_server).append(event) - return + assert peer_quic peer_connection = self.context.server if from_client else self.context.client if isinstance(event, quic_events.DatagramFrameReceived): @@ -419,10 +403,21 @@ def state_start(self, _) -> layer.CommandGenerator[None]: yield commands.CloseConnection(self.context.client) self._handle_event = self.state_done return + + # Hack: Poke through layer stack to get reference to QUIC connection objects. + # Eventually we don't want to do this. + for x in reversed(self.context.layers): + if isinstance(x, ClientQuicLayer): + assert self.quic_client is None + self.quic_client = x.quic + if isinstance(x, ServerQuicLayer): + assert self.quic_server is None + self.quic_server = x.quic + assert self.quic_client + assert self.quic_server self._handle_event = self.state_ready @expect( - QuicStart, QuicConnectionEvent, events.MessageInjected, events.ConnectionClosed, @@ -464,22 +459,6 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: self._handle_event = self.state_done yield from self.state_done(event) - elif isinstance(event, QuicStart): - # QUIC connection has been established, store it and get the peer's buffer - from_client = event.connection is self.context.client - buffer_from_peer = self.buffer_from_server if from_client else self.buffer_from_client - if from_client: - assert self.quic_client is None - self.quic_client = event.quic - else: - assert self.quic_server is None - self.quic_server = event.quic - - # flush the buffer to the other side - for quic_event in buffer_from_peer: - yield from self.handle_quic_event(quic_event, not from_client) - buffer_from_peer.clear() - elif isinstance(event, TcpMessageInjected): # translate injected TCP messages into QUIC stream events assert isinstance(event.flow, tcp.TCPFlow) @@ -513,7 +492,6 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: raise AssertionError(f"Unexpected event: {event!r}") @expect( - QuicStart, QuicConnectionEvent, events.MessageInjected, events.ConnectionClosed, @@ -541,7 +519,8 @@ def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: class QuicRoamingLayer(layer.Layer): - """Simple routing layer that replaces a `ClientQuicLayer` when a connection roams to a different `ConnectionHandler`.""" + """Simple routing layer that replaces a `ClientQuicLayer` when a connection roams to a different + `ConnectionHandler`.""" def __init__(self, context: context.Context, target_layer: ClientQuicLayer) -> None: super().__init__(context) @@ -761,9 +740,6 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo else: yield TlsEstablishedServerHook(QuicTlsData(self.conn, self.context, settings=self.tls)) - # notify child layer, transmit and return success - yield from self.event_to_child(QuicStart(self.conn, self.quic)) - yield from self.tls_interact() return True, None elif isinstance(event, (quic_events.PingAcknowledged, quic_events.ProtocolNegotiated)): pass From cf237d69db2d4f70d5d38cf05b9849dfc556686d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 9 Sep 2022 20:38:23 +0200 Subject: [PATCH 055/695] remove `rawudp` If we want to give users the option to disable UDP, we must not silently pass traffic through but drop it and display an error. For now the easier approach is to not give such an option. --- mitmproxy/addons/next_layer.py | 4 ++-- mitmproxy/options.py | 7 ------- test/mitmproxy/addons/test_next_layer.py | 5 ----- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index a1dbf286c5..ac4d81edf7 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -290,8 +290,8 @@ def s(*layers): else: return layers.DNSLayer(context) - # 7. Use raw mode or ignore the connection. - return raw_layer_cls(context, ignore=not ctx.options.rawudp) + # 7. Use raw mode. + return raw_layer_cls(context) else: raise AssertionError(context.client.transport_protocol) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index bf21823ab7..22dc1fcc20 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -148,13 +148,6 @@ def __init__(self, **kwargs) -> None: "Enable/disable raw TCP connections. " "TCP connections are enabled by default. ", ) - self.add_option( - "rawudp", - bool, - True, - "Enable/disable raw UDP connections. " - "UDP connections are enabled by default. ", - ) self.add_option( "ssl_insecure", bool, diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index 8b592cb5b1..741f47fe15 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -161,11 +161,6 @@ def is_intercepted_udp(layer: Optional[layer.Layer]): ctx.client.transport_protocol = "udp" with taddons.context(nl) as tctx: ctx.layers = [layers.modes.HttpProxy(ctx)] - tctx.configure(nl, rawudp=False) - assert is_ignored_udp(nl._next_layer(ctx, b"", b"")) - - ctx.layers = [layers.modes.HttpProxy(ctx)] - tctx.configure(nl, rawudp=True) assert is_intercepted_udp(nl._next_layer(ctx, b"", b"")) ctx.layers = [layers.modes.HttpProxy(ctx)] From 01dba16eb1faef1d1dc502d694ebe8edb3daaf32 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 10 Sep 2022 01:20:44 +0200 Subject: [PATCH 056/695] wip: use demultiplexing quic layer for raw tcp/udp, start work on communicating message contents instead of interfacing with QuicConnection directly. --- mitmproxy/addons/next_layer.py | 4 +- mitmproxy/net/server_spec.py | 2 + mitmproxy/proxy/commands.py | 2 +- mitmproxy/proxy/events.py | 2 +- mitmproxy/proxy/layers/__init__.py | 4 +- mitmproxy/proxy/layers/http/__init__.py | 3 - mitmproxy/proxy/layers/http/_http3.py | 40 ++- mitmproxy/proxy/layers/modes.py | 8 +- mitmproxy/proxy/layers/quic.py | 393 +++++++++--------------- mitmproxy/proxy/layers/tls.py | 8 +- mitmproxy/proxy/tunnel.py | 12 +- test/mitmproxy/proxy/test_tunnel.py | 4 +- 12 files changed, 188 insertions(+), 294 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index ac4d81edf7..dd1cbf7973 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -171,7 +171,7 @@ def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, Clie try: client_hello = quic_parse_client_hello(data_client) return (client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer) - except ValueError: + except (ValueError, TypeError): pass # that's all we currently have to offer @@ -247,7 +247,7 @@ def s(*layers): # unlike TCP, we make a decision immediately tls = self.detect_udp_tls(data_client) is_quic = isinstance(context.layers[-1], layers.ClientQuicLayer) - raw_layer_cls = layers.QuicStreamLayer if is_quic else layers.UDPLayer + raw_layer_cls = layers.RawQuicLayer if is_quic else layers.UDPLayer # 1. check for --ignore/--allow if self.ignore_connection( diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py index c56f2bf1e0..945565f196 100644 --- a/mitmproxy/net/server_spec.py +++ b/mitmproxy/net/server_spec.py @@ -62,6 +62,8 @@ def parse(server_spec: str, default_scheme: str) -> ServerSpec: port = { "http": 80, "https": 443, + "quic": 443, + "http3": 443, "dns": 53, }[scheme] except KeyError: diff --git a/mitmproxy/proxy/commands.py b/mitmproxy/proxy/commands.py index f1cefb8440..366f961979 100644 --- a/mitmproxy/proxy/commands.py +++ b/mitmproxy/proxy/commands.py @@ -74,7 +74,7 @@ def __init__(self, connection: Connection, data: bytes): def __repr__(self): target = str(self.connection).split("(", 1)[0].lower() - return f"SendData({target}, {self.data})" + return f"{self.__class__.__name__}({target}, {self.data})" class OpenConnection(ConnectionCommand): diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index b571f3ad2f..6afc7841ba 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -47,7 +47,7 @@ class DataReceived(ConnectionEvent): def __repr__(self): target = type(self.connection).__name__.lower() - return f"DataReceived({target}, {self.data})" + return f"{self.__class__.__name__}({target}, {self.data})" class ConnectionClosed(ConnectionEvent): diff --git a/mitmproxy/proxy/layers/__init__.py b/mitmproxy/proxy/layers/__init__.py index c4af84d66a..97ae8fa47c 100644 --- a/mitmproxy/proxy/layers/__init__.py +++ b/mitmproxy/proxy/layers/__init__.py @@ -1,7 +1,7 @@ from . import modes from .dns import DNSLayer from .http import HttpLayer -from .quic import QuicStreamLayer, ClientQuicLayer, ServerQuicLayer +from .quic import RawQuicLayer, ClientQuicLayer, ServerQuicLayer from .tcp import TCPLayer from .udp import UDPLayer from .tls import ClientTLSLayer, ServerTLSLayer @@ -11,7 +11,7 @@ "modes", "DNSLayer", "HttpLayer", - "QuicStreamLayer", + "RawQuicLayer", "TCPLayer", "UDPLayer", "ClientQuicLayer", diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index bc241fb2af..073734cb2d 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -852,9 +852,6 @@ def _handle_event(self, event: events.Event): proxy_mode = self.context.client.proxy_mode assert isinstance(proxy_mode, UpstreamMode) self.context.server.via = (proxy_mode.scheme, proxy_mode.address) - elif isinstance(event, events.Wakeup): - stream = self.command_sources.pop(event.command) - yield from self.event_to_child(stream, event) elif isinstance(event, events.CommandCompleted): stream = self.command_sources.pop(event.command) yield from self.event_to_child(stream, event) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 9bf7e13a5c..da9affdbc9 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -17,10 +17,10 @@ from mitmproxy.net.http import status_codes from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( - QuicConnectionEvent, - QuicLayer, + ClientQuicLayer, QuicStreamDataReceived, + QuicStreamReset, QuicTransmit, - error_code_to_str, + ServerQuicLayer, error_code_to_str, ) from mitmproxy.proxy.utils import expect @@ -50,6 +50,15 @@ ) +class MockQuic: + """ + aioquic intermingles QUIC and HTTP/3. This is something we don't want to do because that makes testing much harder. + Instead, we mock our QUIC connection object here and then take out the wire data to be sent. + """ + pass + # TODO add mock for QuicConnection. + + class Http3Connection(HttpConnection): quic: Optional[QuicConnection] = None h3_conn: Optional[H3Connection] = None @@ -77,11 +86,11 @@ def send_protocol_error( ) -> None: pass # pragma: no cover - @expect(HttpEvent, QuicConnectionEvent, events.ConnectionClosed) + @expect(HttpEvent, QuicStreamDataReceived, QuicStreamReset, events.ConnectionClosed) def state_done(self, _) -> layer.CommandGenerator[None]: yield from () - @expect(HttpEvent, QuicConnectionEvent, events.ConnectionClosed) + @expect(HttpEvent, QuicStreamDataReceived, QuicStreamReset, events.ConnectionClosed) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic is not None assert self.h3_conn is not None @@ -135,13 +144,9 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # transmit buffered data and re-arm timer yield QuicTransmit(self.conn) - # handle events from the underlying QUIC connection - elif isinstance(event, QuicConnectionEvent): - - # report abrupt stream resets + elif isinstance(event, QuicStreamReset): if ( - isinstance(event, quic_events.StreamReset) - and stream_is_client_initiated(event.stream_id) + stream_is_client_initiated(event.stream_id) and event.stream_id in self.h3_conn._stream and not self.h3_conn._stream[event.stream_id].ended ): @@ -165,8 +170,12 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: ) ) - # forward QUIC events to the H3 connection - for h3_event in self.h3_conn.handle_event(event.event): + elif isinstance(event, QuicStreamDataReceived): + yield commands.Log(f"recvd data: {event=}") + # and convert back... + e = quic_events.StreamDataReceived(data=event.data, end_stream=event.end_stream, stream_id=event.stream_id) + for h3_event in self.h3_conn.handle_event(e): + yield commands.Log(f"{h3_event=}") # report received data if ( @@ -269,13 +278,16 @@ def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: # aioquic does not separate QUIC and HTTP/3, poke through the layer stack to get a reference to the QUIC # connection object. for x in reversed(self.context.layers): - if isinstance(x, QuicLayer): + if isinstance(x, ClientQuicLayer if isinstance(self, Http3Server) else ServerQuicLayer): self.quic = x.quic self.h3_conn = H3Connection(self.quic, enable_webtransport=False) break else: raise AssertionError + # self.quic = MockQuic() + # self.h3_conn = H3Connection(self.quic) + self._handle_event = self.state_ready yield from () diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 3b3869c8e0..ebcf292f48 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -7,7 +7,7 @@ from mitmproxy import connection from mitmproxy.proxy import commands, events, layer from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.layers import dns, quic, tls +from mitmproxy.proxy.layers import dns, quic, tls, udp from mitmproxy.proxy.mode_specs import ReverseMode from mitmproxy.proxy.utils import expect @@ -62,11 +62,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if spec.scheme in ("https", "http3", "quic", "tls", "dtls"): if not self.context.options.keep_host_header: self.context.server.sni = spec.address[0] - if (spec.scheme == "http3" or spec.scheme == "quic"): + if spec.scheme == "http3" or spec.scheme == "quic": self.child_layer = quic.ServerQuicLayer(self.context) else: self.child_layer = tls.ServerTLSLayer(self.context) - elif spec.scheme == "http" or spec.scheme == "tcp" or spec.scheme == "udp": + elif spec.scheme == "udp": + self.child_layer = udp.UDPLayer(self.context) + elif spec.scheme == "http" or spec.scheme == "tcp": self.child_layer = layer.NextLayer(self.context) elif spec.scheme == "dns": self.child_layer = dns.DNSLayer(self.context) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 109969661e..f3f01fd7f3 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -29,8 +29,9 @@ from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, tunnel +from mitmproxy.proxy.layer import CommandGenerator from mitmproxy.proxy.layers.tcp import ( - TcpEndHook, + TCPLayer, TcpEndHook, TcpErrorHook, TcpMessageHook, TcpMessageInjected, @@ -118,20 +119,6 @@ class QuicStartServerHook(commands.StartHook): data: QuicTlsData -@dataclass -class QuicConnectionEvent(events.ConnectionEvent): - """ - Connection-based event that is triggered whenever a new event from QUIC is received. - - Note: - 'Established' means that an OpenConnection command called in a child layer returned no error. - Without a predefined child layer, the QUIC layer uses NextLayer mechanics to select the child - layer. The moment is asks addons for the child layer, the connection is considered established. - """ - - event: quic_events.QuicEvent - - class QuicTransmit(commands.SendData): """ aioquic does not separate HTTP/3 and QUIC: H3Connection requires a QuicConnection instance to interact with. @@ -143,6 +130,52 @@ def __init__(self, connection: connection.Connection) -> None: super().__init__(connection, b"") +@dataclass +class QuicStreamDataReceived(events.DataReceived): + stream_id: int + end_stream: bool + + def __repr__(self): + target = type(self.connection).__name__.lower() + return f"{self.__class__.__name__}({target}, {self.stream_id}, {self.data}, {self.end_stream})" + + +@dataclass(repr=False) +class QuicDatagramReceived(events.DataReceived): + pass + + +@dataclass +class QuicStreamClosed(events.ConnectionClosed): + stream_id: int + + +@dataclass +class QuicStreamReset(events.ConnectionEvent): + stream_id: int + error_code: int + + +class SendQuicStreamData(commands.SendData): + stream_id: int + + def __init__(self, connection: connection.Connection, stream_id: int, data: bytes): + super().__init__(connection, data) + self.stream_id = stream_id + + +class SendQuicDatagram(commands.SendData): + pass + + +class CloseQuicStream(commands.CloseConnection): + stream_id: int + + def __init__(self, connection: connection.Connection, stream_id: int): + super().__init__(connection) + self.stream_id = stream_id + + class QuicSecretsLogger: logger: tls.MasterSecretLogger @@ -282,240 +315,74 @@ def mark_ended(self, client: bool, err: str | None = None) -> layer.CommandGener self.flow.live = False -class QuicStreamLayer(layer.Layer): +class RawQuicLayer(layer.Layer): """ - Layer on top of `ClientQuicLayer` and `ServerQuicLayer`, that simply relays all QUIC streams and datagrams. - This layer is chosen by the default NextLayer addon if ALPN yields no known protocol. - It uses `UDPFlow` and `TCPFlow` for datagrams and stream respectively, which makes message injection possible. + This layer is responsible for demultiplexing QUIC streams into an individual layer stack per stream. """ - flow: udp.UDPFlow | None # used for datagrams and to signal general connection issues - quic_client: QuicConnection | None = None - quic_server: QuicConnection | None = None - streams_by_flow: dict[tcp.TCPFlow, QuicStream] - streams_by_id: dict[int, QuicStream] + ignore: bool + substacks: dict[int, layer.Layer] + command_sources: dict[commands.Command, int | "udp"] def __init__(self, context: context.Context, ignore: bool = False) -> None: + self.ignore = ignore + self.substacks = {} + self.command_sources = {} super().__init__(context) - if ignore: - self.flow = None - else: - self.flow = udp.UDPFlow(self.context.client, self.context.server, live=True) - self.streams_by_flow = {} - self.streams_by_id = {} - - def get_or_create_stream( - self, stream_id: int - ) -> layer.CommandGenerator[QuicStream]: - if stream_id in self.streams_by_id: - return self.streams_by_id[stream_id] - else: - # register the stream and start the flow - stream = QuicStream(self.context, stream_id, ignore=self.flow is None) - self.streams_by_id[stream.stream_id] = stream - if stream.flow is not None: - self.streams_by_flow[stream.flow] = stream - yield TcpStartHook(stream.flow) - return stream - - def handle_quic_event( - self, - event: quic_events.QuicEvent, - from_client: bool, - ) -> layer.CommandGenerator[None]: - # buffer events if the peer is not ready yet - peer_quic = self.quic_server if from_client else self.quic_client - assert peer_quic - peer_connection = self.context.server if from_client else self.context.client - - if isinstance(event, quic_events.DatagramFrameReceived): - # forward datagrams (that are not stream-bound) - if self.flow is not None: - udp_message = udp.UDPMessage(from_client, event.data) - self.flow.messages.append(udp_message) - yield UdpMessageHook(self.flow) - data = udp_message.content - else: - data = event.data - peer_quic.send_datagram_frame(data) - - elif isinstance(event, quic_events.StreamDataReceived): - # ignore data received from already ended streams - stream = yield from self.get_or_create_stream(event.stream_id) - if stream.has_ended(from_client): - yield commands.Log(f"Received {len(event.data)} byte(s) on already closed stream #{event.stream_id}.", level="debug") - return - - # forward the message allowing addons to change it - if stream.flow is not None: - tcp_message = tcp.TCPMessage(from_client, event.data) - stream.flow.messages.append(tcp_message) - yield TcpMessageHook(stream.flow) - data = tcp_message.content - else: - data = event.data - peer_quic.send_stream_data( - stream.stream_id, - data, - event.end_stream, - ) - - # mark the stream as ended if needed - if event.end_stream: - yield from stream.mark_ended(from_client) - - elif isinstance(event, quic_events.StreamReset): - # ignore resets from already ended streams - stream = yield from self.get_or_create_stream(event.stream_id) - if stream.has_ended(from_client): - yield commands.Log(f"Received reset for already closed stream #{event.stream_id}.", level="debug") - return - - # forward resets to peer streams and report them to addons - peer_quic.reset_stream( - stream.stream_id, - event.error_code, - ) - - # mark the stream as failed - yield from stream.mark_ended(from_client, err=error_code_to_str(event.error_code)) - - else: - # ignore other QUIC events - yield commands.Log(f"Ignored QUIC event {event!r}.", level="debug") - return - - # transmit data to the peer - yield QuicTransmit(peer_connection) - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - # mark the main flow as started - if self.flow is not None: - yield UdpStartHook(self.flow) - - # open the upstream connection if necessary - if self.context.server.timestamp_start is None: - err = yield commands.OpenConnection(self.context.server) - if err: - if self.flow is not None: - self.flow.error = mitm_flow.Error(str(err)) - yield UdpErrorHook(self.flow) - yield commands.CloseConnection(self.context.client) - self._handle_event = self.state_done - return - - # Hack: Poke through layer stack to get reference to QUIC connection objects. - # Eventually we don't want to do this. - for x in reversed(self.context.layers): - if isinstance(x, ClientQuicLayer): - assert self.quic_client is None - self.quic_client = x.quic - if isinstance(x, ServerQuicLayer): - assert self.quic_server is None - self.quic_server = x.quic - assert self.quic_client - assert self.quic_server - self._handle_event = self.state_ready - - @expect( - QuicConnectionEvent, - events.MessageInjected, - events.ConnectionClosed, - ) - def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: - - if isinstance(event, events.ConnectionClosed): - # define helper variables - from_client = event.connection is self.context.client - peer_conn = self.context.server if from_client else self.context.client - peer_quic = self.quic_server if from_client else self.quic_client - closed_quic = self.quic_client if from_client else self.quic_server - close_event = None if closed_quic is None else closed_quic._close_event - - # close the peer as well (needs to be before hooks) - if peer_quic is not None and close_event is not None: - peer_quic.close( - close_event.error_code, - close_event.frame_type, - close_event.reason_phrase, - ) - yield QuicTransmit(peer_conn) + def get_or_create_stack(self, stream_id: int | "udp") -> CommandGenerator[layer.Layer]: + if stream_id not in self.substacks: + # v2: self.substacks[stream_id] = layer.NextLayer(self.context.fork()) + if stream_id == "udp": + self.substacks[stream_id] = UDPLayer(self.context, ignore=self.ignore) else: - yield commands.CloseConnection(peer_conn) - - # report errors to the main flow - if ( - self.flow is not None - and close_event is not None - and not is_success_error_code(close_event.error_code) - ): - self.flow.error = mitm_flow.Error( - close_event.reason_phrase - or error_code_to_str(close_event.error_code) - ) - yield UdpErrorHook(self.flow) - - # we're done handling QUIC events, pass on to generic close handling - self._handle_event = self.state_done - yield from self.state_done(event) - - elif isinstance(event, TcpMessageInjected): - # translate injected TCP messages into QUIC stream events - assert isinstance(event.flow, tcp.TCPFlow) - stream = self.streams_by_flow[event.flow] - yield from self.handle_quic_event( - quic_events.StreamDataReceived( - stream_id=stream.stream_id, - data=event.message.content, - end_stream=False, - ), - event.message.from_client, - ) - - elif isinstance(event, UdpMessageInjected): - # translate injected UDP messages into QUIC datagram events - assert isinstance(event.flow, udp.UDPFlow) - assert event.flow is self.flow - yield from self.handle_quic_event( - quic_events.DatagramFrameReceived(data=event.message.content), - event.message.from_client, - ) - - elif isinstance(event, QuicConnectionEvent): - # handle or buffer QUIC events - yield from self.handle_quic_event( - event.event, - from_client=event.connection is self.context.client, - ) - + self.substacks[stream_id] = TCPLayer(self.context, ignore=self.ignore) + + yield from self.event_to_child(stream_id, events.Start()) + + return self.substacks[stream_id] + + def _handle_event(self, e: events.Event) -> CommandGenerator[None]: + if isinstance(e, events.Start): + pass + elif isinstance(e, events.CommandCompleted): + stream_id = self.command_sources.pop(e.command) + yield from self.event_to_child(stream_id, e) + elif isinstance(e, QuicDatagramReceived): + yield from self.event_to_child("udp", events.DataReceived(e.connection, e.data)) + elif isinstance(e, QuicStreamDataReceived): + yield from self.event_to_child(e.stream_id, events.DataReceived(e.connection, e.data)) + if e.end_stream: + yield from self.event_to_child(e.stream_id, events.ConnectionClosed(e.connection)) + elif isinstance(e, QuicStreamReset): + yield from self.event_to_child(e.stream_id, events.ConnectionClosed(e.connection)) + elif isinstance(e, events.MessageInjected): + raise NotImplementedError("Unimplemented: Message injection") + elif isinstance(e, events.ConnectionClosed): + for stream_id in self.substacks: + yield from self.event_to_child(stream_id, e) else: - raise AssertionError(f"Unexpected event: {event!r}") + raise AssertionError(f"Unexpected event: {e}") - @expect( - QuicConnectionEvent, - events.MessageInjected, - events.ConnectionClosed, - ) - def state_done(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.ConnectionClosed): - from_client = event.connection is self.context.client - - # report the termination as error to all non-ended streams - for stream in self.streams_by_id.values(): - if not stream.has_ended(from_client): - yield from stream.mark_ended(from_client, err="Connection closed.") - - # end the main flow - if ( - self.flow is not None - and not self.context.client.connected - and not self.context.server.connected - ): - if self.flow.error is None: - yield UdpEndHook(self.flow) - self.flow.live = False + def event_to_child(self, stream_id: int | "udp", event: events.Event) -> CommandGenerator[None]: + stack = yield from self.get_or_create_stack(stream_id) + for command in stack.handle_event(event): + if command.blocking or isinstance(command, commands.RequestWakeup): + self.command_sources[command] = stream_id - _handle_event = state_start + if isinstance(command, commands.SendData): + if stream_id == "udp": + yield SendQuicDatagram(command.connection, command.data) + else: + yield SendQuicStreamData(command.connection, stream_id, command.data) + elif isinstance(command, commands.CloseConnection): + if stream_id == "udp": + pass + else: + yield CloseQuicStream(command.connection, stream_id) + elif isinstance(command, commands.OpenConnection): + raise NotImplementedError("Unimplemented: QUIC server change") + else: + yield command class QuicRoamingLayer(layer.Layer): @@ -670,7 +537,7 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C yield from self.tls_interact() # issue the host connection ID right away - self.issue_connection_id(quic_events.ConnectionIdIssued(self.quic.host_cid)) + self.issue_connection_id(self.quic.host_cid) def tls_interact(self) -> layer.CommandGenerator[None]: """Retrieves all pending outgoing packets from aioquic and sends the data.""" @@ -740,6 +607,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo else: yield TlsEstablishedServerHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + yield from self.tls_interact() return True, None elif isinstance(event, (quic_events.PingAcknowledged, quic_events.ProtocolNegotiated)): pass @@ -781,12 +649,16 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: return # we don't handle any further events, nor do/can we transmit data, so exit elif isinstance(event, quic_events.PingAcknowledged): pass - elif isinstance(event, ( - quic_events.DatagramFrameReceived, - quic_events.StreamDataReceived, - quic_events.StreamReset, - )): - yield from self.event_to_child(QuicConnectionEvent(self.conn, event)) + elif isinstance(event, quic_events.DatagramFrameReceived): + e = QuicDatagramReceived(self.conn, event.data) + yield from self.event_to_child(e) + elif isinstance(event, quic_events.StreamDataReceived): + e = QuicStreamDataReceived(self.conn, event.data, event.stream_id, event.end_stream) + yield commands.Log(f"{e!r}") + yield from self.event_to_child(e) + elif isinstance(event, quic_events.StreamReset): + e = QuicStreamReset(self.conn, event.stream_id, event.error_code) + yield from self.event_to_child(e) else: raise AssertionError(f"Unexpected event: {event!r}") @@ -797,19 +669,28 @@ def receive_close(self) -> layer.CommandGenerator[None]: # unlike TLS we haven't sent CloseConnection before yield from super().receive_close() - def send_data(self, data: bytes) -> layer.CommandGenerator[None]: - # no actual data is allowed, SendData (i.e. QuicTransmit) functions only as trigger - assert not data + def send_data(self, command: commands.SendData) -> layer.CommandGenerator[None]: + if isinstance(command, SendQuicDatagram): + self.quic.send_datagram_frame(command.data) + elif isinstance(command, SendQuicStreamData): + self.quic.send_stream_data(command.stream_id, command.data) + elif isinstance(command, QuicTransmit): + yield commands.Log("SendQuicTrigger") + assert not command.data + else: + raise AssertionError(f"Unexpected command: {command}") + yield from self.tls_interact() - def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: - # properly close a QUIC connection, no half-close allowed - assert not half_close - if self.quic is not None: - self.quic.close() - yield from self.tls_interact() + def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: + if isinstance(command, CloseQuicStream): + if self.quic._stream_can_send(command.stream_id): + self.quic.send_stream_data(command.stream_id, b"", end_stream=True) else: - yield from super().send_close(half_close) + if self.quic is not None: + self.quic.close() + yield from self.tls_interact() + yield from super().send_close(command) class ServerQuicLayer(QuicLayer): diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 22f45750c3..79350a12b7 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -430,17 +430,17 @@ def receive_close(self) -> layer.CommandGenerator[None]: else: yield from super().receive_close() - def send_data(self, data: bytes) -> layer.CommandGenerator[None]: + def send_data(self, command: commands.SendData) -> layer.CommandGenerator[None]: try: - self.tls.sendall(data) + self.tls.sendall(command.data) except (SSL.ZeroReturnError, SSL.SysCallError): # The other peer may still be trying to send data over, which we discard here. pass yield from self.tls_interact() - def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: + def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: # We should probably shutdown the TLS connection properly here. - yield from super().send_close(half_close) + yield from super().send_close(command) class ServerTLSLayer(TLSLayer): diff --git a/mitmproxy/proxy/tunnel.py b/mitmproxy/proxy/tunnel.py index d14023a742..1c8317f98b 100644 --- a/mitmproxy/proxy/tunnel.py +++ b/mitmproxy/proxy/tunnel.py @@ -121,14 +121,14 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: and command.connection == self.conn ): if isinstance(command, commands.SendData): - yield from self.send_data(command.data) + yield from self.send_data(command) elif isinstance(command, commands.CloseConnection): if self.conn != self.tunnel_connection: if command.half_close: self.conn.state &= ~connection.ConnectionState.CAN_WRITE else: self.conn.state = connection.ConnectionState.CLOSED - yield from self.send_close(command.half_close) + yield from self.send_close(command) elif isinstance(command, commands.OpenConnection): # create our own OpenConnection command object that blocks here. self.command_to_reply_to = command @@ -166,11 +166,11 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: def receive_close(self) -> layer.CommandGenerator[None]: yield from self.event_to_child(events.ConnectionClosed(self.conn)) - def send_data(self, data: bytes) -> layer.CommandGenerator[None]: - yield commands.SendData(self.tunnel_connection, data) + def send_data(self, command: commands.SendData) -> layer.CommandGenerator[None]: + yield commands.SendData(self.tunnel_connection, command.data) - def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: - yield commands.CloseConnection(self.tunnel_connection, half_close=half_close) + def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: + yield commands.CloseConnection(self.tunnel_connection, half_close=command.half_close) class LayerStack: diff --git a/test/mitmproxy/proxy/test_tunnel.py b/test/mitmproxy/proxy/test_tunnel.py index 24103637c6..f33f982525 100644 --- a/test/mitmproxy/proxy/test_tunnel.py +++ b/test/mitmproxy/proxy/test_tunnel.py @@ -45,8 +45,8 @@ def receive_handshake_data( else: return False, "handshake error" - def send_data(self, data: bytes) -> layer.CommandGenerator[None]: - yield SendData(self.tunnel_connection, b"tunneled-" + data) + def send_data(self, command: SendData) -> layer.CommandGenerator[None]: + yield SendData(self.tunnel_connection, b"tunneled-" + command.data) def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: yield from self.event_to_child( From 65e7aef5accde4833307be4e13d3d476f6666feb Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sat, 10 Sep 2022 06:58:54 +0200 Subject: [PATCH 057/695] [quic] added suggestions --- mitmproxy/proxy/layers/quic.py | 185 +++++++++++++++++++++------------ 1 file changed, 121 insertions(+), 64 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index f3f01fd7f3..cd4400fa09 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -2,7 +2,8 @@ import asyncio from dataclasses import dataclass, field from ssl import VerifyMode -from typing import TYPE_CHECKING, ClassVar +import time +from typing import TYPE_CHECKING, ClassVar, Literal, Union from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import H3_ALPN, ErrorCode as H3ErrorCode @@ -29,7 +30,7 @@ from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, tunnel -from mitmproxy.proxy.layer import CommandGenerator +from mitmproxy.proxy.layer import CommandGenerator, NextLayer from mitmproxy.proxy.layers.tcp import ( TCPLayer, TcpEndHook, TcpErrorHook, @@ -124,6 +125,12 @@ class QuicTransmit(commands.SendData): aioquic does not separate HTTP/3 and QUIC: H3Connection requires a QuicConnection instance to interact with. This unfortunately breaks our abstractions. As a workaround, all "SendData" commands for HTTP/3 connections do not carry the actual data to be sent, but just serve to notify the QUIC layer instead. + + + Suggestion: Introduce Send/Receive commands/events for H3 as well. + Introduce QuicUseH3Connection command, which will activate a H3 connection that lives in QuicLayer. + QuicLayer will then send H3 events and act on H3 commands. + Also no more rummaging around the context for QuicLayer to get the QuicConnection. """ def __init__(self, connection: connection.Connection) -> None: @@ -176,6 +183,18 @@ def __init__(self, connection: connection.Connection, stream_id: int): self.stream_id = stream_id +class OpenQuicStream(commands.OpenConnection): + connection: connection.Connection + blocking = True + + +@dataclass(repr=False) +class OpenQuicStreamCompleted(commands.CommandCompleted): + command: commands.OpenQuicStream + reply: int | str + """stream_id or error message""" + + class QuicSecretsLogger: logger: tls.MasterSecretLogger @@ -272,47 +291,45 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") -class QuicStream: - flow: tcp.TCPFlow | None +class QuicStreamLayer(layer.Layer): + """ + Layer for QUIC streams. + Serves as a marker for NextLayer decision and to keep track of the connection states. + """ + stream_id: int + child_layer: layer.Layer - def __init__(self, context: context.Context, stream_id: int, ignore: bool) -> None: - if ignore: - self.flow = None - else: - self.flow = tcp.TCPFlow(context.client, context.server, live=True) + def __init__(self, context: context.Context, stream_id: int) -> None: + # we shouldn't reuse the client or server from the + # QUIC connection as we have different states here + context.client = connection.Client( + peername=context.client.peername, + sockname=context.client.sockname, + timestamp_start=time.time(), + transport_protocol=context.client.transport_protocol, # would prefer TCP here, but will conflict with inject + proxy_mode=context.client.proxy_mode, + ) + + # if the server gets set in the layer again, we'll treat it as QUIC-to-somewhere-else + context.server = connection.Server( + address=context.server.address, + transport_protocol=context.server.transport_protocol, + ) + super().__init__(context) self.stream_id = stream_id - is_unidirectional = stream_is_unidirectional(stream_id) - from_client = stream_is_client_initiated(stream_id) - self._ended_client = is_unidirectional and not from_client - self._ended_server = is_unidirectional and from_client - - def has_ended(self, client: bool) -> bool: - return self._ended_client if client else self._ended_server - - def mark_ended(self, client: bool, err: str | None = None) -> layer.CommandGenerator[None]: - # ensure we actually change the ended state - if client: - assert not self._ended_client - self._ended_client = True - else: - assert not self._ended_server - self._ended_server = True + self.child_layer = NextLayer(context) + if stream_is_unidirectional(stream_id) and stream_is_client_initiated(stream_id): + context.client.state = connection.ConnectionState.CAN_READ - # we're done if the stream is ignored - if self.flow is None: - return + # TODO: track state of client and server stream - # set and report the first error - if err is not None and self.flow.error is None: - self.flow.error = mitm_flow.Error(err) - yield TcpErrorHook(self.flow) - # report error-free endings and always clear the live flag - if self._ended_client and self._ended_server: - if self.flow.error is None: - yield TcpEndHook(self.flow) - self.flow.live = False +class QuicDatagramLayer(layer.Layer): + child_layer: layer.Layer + + +StreamId = Union[int, Literal["udp"]] class RawQuicLayer(layer.Layer): @@ -320,26 +337,39 @@ class RawQuicLayer(layer.Layer): This layer is responsible for demultiplexing QUIC streams into an individual layer stack per stream. """ ignore: bool - substacks: dict[int, layer.Layer] - command_sources: dict[commands.Command, int | "udp"] + streams: dict[StreamId, layer.Layer] + """All stream layers indexed by the stream ID used in the client connection.""" + server_stream_id_to_client_stream_id: dict[StreamId, StreamId] + """We don't have a 1:1 mapping of stream ids, so keep track of it.""" + client_to_stream_id: dict[connection.Client, StreamId] + """Each stream has it's own `Client` instance, which we use as index here.""" + server_to_stream_id: dict[connection.Server, StreamId] + """Each stream has it's own `Server` instance, which we use as index here.""" + command_sources: dict[commands.Command, StreamId] def __init__(self, context: context.Context, ignore: bool = False) -> None: self.ignore = ignore - self.substacks = {} + self.streams = {} + self.server_stream_id_to_client_stream_id = {} + self.client_to_stream_id = {} + self.server_to_stream_id = {} self.command_sources = {} super().__init__(context) - def get_or_create_stack(self, stream_id: int | "udp") -> CommandGenerator[layer.Layer]: - if stream_id not in self.substacks: - # v2: self.substacks[stream_id] = layer.NextLayer(self.context.fork()) - if stream_id == "udp": - self.substacks[stream_id] = UDPLayer(self.context, ignore=self.ignore) - else: - self.substacks[stream_id] = TCPLayer(self.context, ignore=self.ignore) - + def get_or_create_stream(self, stream_id: StreamId) -> CommandGenerator[layer.Layer]: + layer = self.streams.get(stream_id, None) + if layer is None: + layer = ( + QuicDatagramLayer(self.context.fork()) + if stream_id == "udp" else + QuicStreamLayer(self.context.fork()) + ) + self.streams[stream_id] = layer + self.client_to_stream_id[layer.context.client] = stream_id + self.server_to_stream_id[layer.context.server] = stream_id yield from self.event_to_child(stream_id, events.Start()) - return self.substacks[stream_id] + return layer def _handle_event(self, e: events.Event) -> CommandGenerator[None]: if isinstance(e, events.Start): @@ -356,31 +386,58 @@ def _handle_event(self, e: events.Event) -> CommandGenerator[None]: elif isinstance(e, QuicStreamReset): yield from self.event_to_child(e.stream_id, events.ConnectionClosed(e.connection)) elif isinstance(e, events.MessageInjected): - raise NotImplementedError("Unimplemented: Message injection") + yield from self.event_to_child(self.stream_client_to_stream_id[e.flow.client_conn], e) elif isinstance(e, events.ConnectionClosed): - for stream_id in self.substacks: + for stream_id in self.streams: yield from self.event_to_child(stream_id, e) else: raise AssertionError(f"Unexpected event: {e}") - def event_to_child(self, stream_id: int | "udp", event: events.Event) -> CommandGenerator[None]: - stack = yield from self.get_or_create_stack(stream_id) - for command in stack.handle_event(event): + def event_to_child(self, stream_id: StreamId, event: events.Event) -> CommandGenerator[None]: + stream = yield from self.get_or_create_stream(stream_id) + for command in stream.handle_event(event): if command.blocking or isinstance(command, commands.RequestWakeup): self.command_sources[command] = stream_id - if isinstance(command, commands.SendData): - if stream_id == "udp": - yield SendQuicDatagram(command.connection, command.data) + if isinstance(command, commands.ConnectionCommand): + + if command.connection in self.client_to_stream_id: + conn = self.context.client + elif command.connection in self.server_to_stream_id: + conn = self.context.server else: - yield SendQuicStreamData(command.connection, stream_id, command.data) - elif isinstance(command, commands.CloseConnection): - if stream_id == "udp": - pass + # not ours, don't intercept + yield command + return + + if isinstance(command, commands.SendData): + if stream_id == "udp": + yield SendQuicDatagram(conn, command.data) + else: + yield SendQuicStreamData(conn, stream_id, command.data) + elif isinstance(command, commands.CloseConnection): + if stream_id == "udp": + pass + else: + yield CloseQuicStream(conn, stream_id) + elif isinstance(command, commands.OpenConnection): + assert conn is not self.context.client + stream_id = self.server_to_stream_id[command.connection] + if err := command.connection.error: + yield from self.event_to_child(stream_id, events.OpenConnectionCompleted(command, f"Connection killed: {err}")) + return + stream_id_or_err = yield OpenQuicStream(conn) + if isinstance(stream_id_or_err, str): + yield from self.event_to_child(stream_id, events.OpenConnectionCompleted(command, stream_id_or_err)) + return + self.server_stream_id_to_client_stream_id[stream_id_or_err] = stream_id + self.streams[stream_id].context.server.set_state(connection.ConnectionState.OPEN) + yield from self.event_to_child(stream_id, events.OpenConnectionCompleted(command, None)) + else: - yield CloseQuicStream(command.connection, stream_id) - elif isinstance(command, commands.OpenConnection): - raise NotImplementedError("Unimplemented: QUIC server change") + command.connection = conn + yield command + else: yield command From 42ccc85b6f1881d92b55e411ba9719f26459b720 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 11 Sep 2022 18:01:20 +0200 Subject: [PATCH 058/695] [quic] work on eventify layers --- mitmproxy/addons/proxyserver.py | 6 - mitmproxy/proxy/layers/quic.py | 703 +++++++++++------------ mitmproxy/proxy/layers/tls.py | 8 +- mitmproxy/proxy/tunnel.py | 69 +-- test/mitmproxy/proxy/layers/test_quic.py | 58 ++ test/mitmproxy/proxy/layers/test_tls.py | 2 +- 6 files changed, 447 insertions(+), 399 deletions(-) create mode 100644 test/mitmproxy/proxy/layers/test_quic.py diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 1bd8b1df1c..b70759dfb7 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -190,12 +190,6 @@ def load(self, loader): None, """Set the local IP address that mitmproxy should use when connecting to upstream servers.""", ) - loader.add_option( - "quic_connection_id_length", - int, - 8, - """The length in bytes of local QUIC connection IDs.""", - ) def running(self): self.is_running = True diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index cd4400fa09..c62f6f1e93 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from ssl import VerifyMode import time -from typing import TYPE_CHECKING, ClassVar, Literal, Union +from typing import TYPE_CHECKING, cast from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import H3_ALPN, ErrorCode as H3ErrorCode @@ -20,24 +20,16 @@ from aioquic.tls import CipherSuite, HandshakeType from aioquic.quic.packet import ( PACKET_TYPE_INITIAL, - QuicHeader, QuicProtocolVersion, encode_quic_version_negotiation, pull_quic_header, ) from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection, flow as mitm_flow, tcp, udp +from mitmproxy import certs, connection from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, tunnel -from mitmproxy.proxy.layer import CommandGenerator, NextLayer -from mitmproxy.proxy.layers.tcp import ( - TCPLayer, TcpEndHook, - TcpErrorHook, - TcpMessageHook, - TcpMessageInjected, - TcpStartHook, -) +from mitmproxy.proxy.layers.tcp import TCPLayer from mitmproxy.proxy.layers.tls import ( TlsClienthelloHook, TlsEstablishedClientHook, @@ -45,17 +37,8 @@ TlsFailedClientHook, TlsFailedServerHook, ) -from mitmproxy.proxy.layers.udp import ( - UDPLayer, - UdpEndHook, - UdpErrorHook, - UdpMessageHook, - UdpMessageInjected, - UdpStartHook, -) -from mitmproxy.proxy.utils import expect +from mitmproxy.proxy.layers.udp import UDPLayer from mitmproxy.tls import ClientHello, ClientHelloData, TlsData -from mitmproxy.utils import human if TYPE_CHECKING: from mitmproxy.proxy.server import ConnectionHandler @@ -120,79 +103,102 @@ class QuicStartServerHook(commands.StartHook): data: QuicTlsData -class QuicTransmit(commands.SendData): - """ - aioquic does not separate HTTP/3 and QUIC: H3Connection requires a QuicConnection instance to interact with. - This unfortunately breaks our abstractions. As a workaround, all "SendData" commands for HTTP/3 connections - do not carry the actual data to be sent, but just serve to notify the QUIC layer instead. - - - Suggestion: Introduce Send/Receive commands/events for H3 as well. - Introduce QuicUseH3Connection command, which will activate a H3 connection that lives in QuicLayer. - QuicLayer will then send H3 events and act on H3 commands. - Also no more rummaging around the context for QuicLayer to get the QuicConnection. - """ +@dataclass +class QuicStreamEvent(events.ConnectionEvent): + """Base class for all QUIC stream events.""" - def __init__(self, connection: connection.Connection) -> None: - super().__init__(connection, b"") + stream_id: int + """The ID of the stream the event was fired for.""" @dataclass -class QuicStreamDataReceived(events.DataReceived): - stream_id: int +class QuicStreamDataReceived(QuicStreamEvent): + """Event that is fired whenever data is received on a stream.""" + + data: bytes + """The data which was received.""" end_stream: bool + """Whether the STREAM frame had the FIN bit set.""" def __repr__(self): target = type(self.connection).__name__.lower() return f"{self.__class__.__name__}({target}, {self.stream_id}, {self.data}, {self.end_stream})" -@dataclass(repr=False) -class QuicDatagramReceived(events.DataReceived): - pass - - @dataclass -class QuicStreamClosed(events.ConnectionClosed): - stream_id: int +class QuicStreamReset(QuicStreamEvent): + """Event that is fired when the remote peer resets a stream.""" - -@dataclass -class QuicStreamReset(events.ConnectionEvent): - stream_id: int error_code: int + """The error code that triggered the reset.""" + +class QuicStreamCommand(commands.ConnectionCommand): + """Base class for all QUIC stream commands.""" -class SendQuicStreamData(commands.SendData): stream_id: int + """The ID of the stream the command was issued for.""" - def __init__(self, connection: connection.Connection, stream_id: int, data: bytes): - super().__init__(connection, data) + def __init__(self, connection: connection.Connection, stream_id: int): + super().__init__(connection) self.stream_id = stream_id -class SendQuicDatagram(commands.SendData): - pass +class SendQuicStreamData(QuicStreamCommand): + """Command that sends data on a stream.""" + data: bytes + """The data which should be sent.""" + end_stream: bool + """Whether the FIN bit should be set in the STREAM frame.""" -class CloseQuicStream(commands.CloseConnection): - stream_id: int + def __init__(self, connection: connection.Connection, stream_id: int, data: bytes, end_stream: bool = False): + super().__init__(connection, stream_id) + self.data = data + self.end_stream = end_stream - def __init__(self, connection: connection.Connection, stream_id: int): - super().__init__(connection) - self.stream_id = stream_id + +class ResetQuicStream(QuicStreamCommand): + """Abruptly terminate the sending part of a stream.""" + + error_code: int + """An error code indicating why the stream is being reset.""" + + def __init__(self, connection: connection.Connection, stream_id: int, error_code: int): + super().__init__(connection, stream_id) + self.error_code = error_code + + +class StopQuicStream(QuicStreamCommand): + """Request termination of the receiving part of a stream.""" + + error_code: int + """An error code indicating why the stream is being stopped.""" + + def __init__(self, connection: connection.Connection, stream_id: int, error_code: int): + super().__init__(connection, stream_id) + self.error_code = error_code -class OpenQuicStream(commands.OpenConnection): - connection: connection.Connection +class OpenQuicStream(commands.ConnectionCommand): + """Command that allocates and returns the next available stream ID.""" + + is_unidirectional: bool + """Whether the stream should be unidirectional.""" blocking = True + def __init__(self, connection: connection.Connection, is_unidirectional: bool = False): + super().__init__(connection) + self.is_unidirectional = is_unidirectional + @dataclass(repr=False) -class OpenQuicStreamCompleted(commands.CommandCompleted): - command: commands.OpenQuicStream - reply: int | str - """stream_id or error message""" +class OpenQuicStreamCompleted(events.CommandCompleted): + """Emitted when `OpenQuicStream` has been finished.""" + + command: OpenQuicStream + reply: int + """The stream ID for the next stream created by this endpoint.""" class QuicSecretsLogger: @@ -232,6 +238,18 @@ def is_success_error_code(error_code: int) -> bool: return error_code in (QuicErrorCode.NO_ERROR, H3ErrorCode.H3_NO_ERROR) +def get_stream_connection_state(stream_id: int, is_client: bool) -> connection.ConnectionState: + """Returns the initial connection state of a stream.""" + + state = connection.ConnectionState.OPEN + if stream_is_unidirectional(stream_id): + if stream_is_client_initiated(stream_id) == is_client: + state &= ~connection.ConnectionState.CAN_READ + else: + state &= ~connection.ConnectionState.CAN_WRITE + return state + + @dataclass class QuicClientHello(Exception): """Helper error only used in `quic_parse_client_hello`.""" @@ -294,213 +312,235 @@ def initialize_replacement(peer_cid: bytes) -> None: class QuicStreamLayer(layer.Layer): """ Layer for QUIC streams. - Serves as a marker for NextLayer decision and to keep track of the connection states. + Serves as a marker for NextLayer and keeps track of the connection states. """ - stream_id: int + client_stream_id: int + server_stream_id: int | None child_layer: layer.Layer - def __init__(self, context: context.Context, stream_id: int) -> None: - # we shouldn't reuse the client or server from the - # QUIC connection as we have different states here + def __init__(self, context: context.Context, ignore: bool, client_stream_id: int, server_stream_id: int | None) -> None: + # We mustn't reuse the client or server from the QUIC connection as we have different states here. context.client = connection.Client( peername=context.client.peername, sockname=context.client.sockname, timestamp_start=time.time(), - transport_protocol=context.client.transport_protocol, # would prefer TCP here, but will conflict with inject + transport_protocol=context.client.transport_protocol, proxy_mode=context.client.proxy_mode, ) - - # if the server gets set in the layer again, we'll treat it as QUIC-to-somewhere-else context.server = connection.Server( address=context.server.address, transport_protocol=context.server.transport_protocol, ) super().__init__(context) - self.stream_id = stream_id - self.child_layer = NextLayer(context) - if stream_is_unidirectional(stream_id) and stream_is_client_initiated(stream_id): - context.client.state = connection.ConnectionState.CAN_READ - - # TODO: track state of client and server stream - - -class QuicDatagramLayer(layer.Layer): - child_layer: layer.Layer - + self.client_stream_id = client_stream_id + self.server_stream_id = server_stream_id + self.child_layer = ( + TCPLayer(context, ignore=True) + if ignore else + layer.NextLayer(context) + ) -StreamId = Union[int, Literal["udp"]] + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + yield from self.child_layer.handle_event(event) class RawQuicLayer(layer.Layer): """ - This layer is responsible for demultiplexing QUIC streams into an individual layer stack per stream. + This layer is responsible for de-multiplexing QUIC streams into an individual layer stack per stream. """ + ignore: bool - streams: dict[StreamId, layer.Layer] - """All stream layers indexed by the stream ID used in the client connection.""" - server_stream_id_to_client_stream_id: dict[StreamId, StreamId] - """We don't have a 1:1 mapping of stream ids, so keep track of it.""" - client_to_stream_id: dict[connection.Client, StreamId] - """Each stream has it's own `Client` instance, which we use as index here.""" - server_to_stream_id: dict[connection.Server, StreamId] - """Each stream has it's own `Server` instance, which we use as index here.""" - command_sources: dict[commands.Command, StreamId] + """Indicates whether traffic should be routed as-is.""" + datagram_layer: layer.Layer + """ + The layer handling datagrams over QUIC. It's like a child_layer, but with a forked context. + Instead of having a datagram equivalent for all stream classes, we use traditional `SendData` and `DataReceived`. + There is also no need for another `NextLayer` marker, as a missing `QuicStreamLayer` implies UDP, + and the connection state is the same as the one of the underlying QUIC connection. + """ + client_stream_ids: dict[int, QuicStreamLayer] + """Maps stream IDs from the client connection to stream layers.""" + server_stream_ids: dict[int, QuicStreamLayer] + """Maps stream IDs from the server connection to stream layers.""" + connections: dict[connection.Connection, layer.Layer] + """Maps connections to layers.""" + command_sources: dict[commands.Command, layer.Layer] def __init__(self, context: context.Context, ignore: bool = False) -> None: - self.ignore = ignore - self.streams = {} - self.server_stream_id_to_client_stream_id = {} - self.client_to_stream_id = {} - self.server_to_stream_id = {} - self.command_sources = {} super().__init__(context) + self.datagram_layer = ( + UDPLayer(self.context.fork(), ignore=True) + if ignore else + layer.NextLayer(self.context.fork()) + ) + self.client_stream_ids = {} + self.server_stream_ids = {} + self.connections = { + context.client: self.datagram_layer, + context.server: self.datagram_layer, + } + self.command_sources = {} - def get_or_create_stream(self, stream_id: StreamId) -> CommandGenerator[layer.Layer]: - layer = self.streams.get(stream_id, None) - if layer is None: - layer = ( - QuicDatagramLayer(self.context.fork()) - if stream_id == "udp" else - QuicStreamLayer(self.context.fork()) + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + # we treat the datagram-layer as child layer, so forward Start + if isinstance(event, events.Start): + yield from self.event_to_child(self.datagram_layer, event) + + # properly forward stored completion events + elif isinstance(event, events.CommandCompleted): + yield from self.event_to_child(self.command_sources.pop(event.command), event) + + # route injection messages based on the flow's connections (prefer client, fallback to server) + elif isinstance(event, events.MessageInjected): + if event.flow.client_conn in self.connections: + yield from self.event_to_child(self.connections[event.flow.client_conn], event) + elif event.flow.server_conn in self.connections: + yield from self.event_to_child(self.connections[event.flow.server_conn], event) + else: + raise AssertionError(f"Flow not associated: {event.flow!r}") + + # handle stream events targeting this context + elif ( + isinstance(event, QuicStreamEvent) + and ( + event.connection is self.context.client + or event.connection is self.context.server ) - self.streams[stream_id] = layer - self.client_to_stream_id[layer.context.client] = stream_id - self.server_to_stream_id[layer.context.server] = stream_id - yield from self.event_to_child(stream_id, events.Start()) - - return layer - - def _handle_event(self, e: events.Event) -> CommandGenerator[None]: - if isinstance(e, events.Start): - pass - elif isinstance(e, events.CommandCompleted): - stream_id = self.command_sources.pop(e.command) - yield from self.event_to_child(stream_id, e) - elif isinstance(e, QuicDatagramReceived): - yield from self.event_to_child("udp", events.DataReceived(e.connection, e.data)) - elif isinstance(e, QuicStreamDataReceived): - yield from self.event_to_child(e.stream_id, events.DataReceived(e.connection, e.data)) - if e.end_stream: - yield from self.event_to_child(e.stream_id, events.ConnectionClosed(e.connection)) - elif isinstance(e, QuicStreamReset): - yield from self.event_to_child(e.stream_id, events.ConnectionClosed(e.connection)) - elif isinstance(e, events.MessageInjected): - yield from self.event_to_child(self.stream_client_to_stream_id[e.flow.client_conn], e) - elif isinstance(e, events.ConnectionClosed): - for stream_id in self.streams: - yield from self.event_to_child(stream_id, e) - else: - raise AssertionError(f"Unexpected event: {e}") - - def event_to_child(self, stream_id: StreamId, event: events.Event) -> CommandGenerator[None]: - stream = yield from self.get_or_create_stream(stream_id) - for command in stream.handle_event(event): - if command.blocking or isinstance(command, commands.RequestWakeup): - self.command_sources[command] = stream_id - - if isinstance(command, commands.ConnectionCommand): - - if command.connection in self.client_to_stream_id: - conn = self.context.client - elif command.connection in self.server_to_stream_id: - conn = self.context.server - else: - # not ours, don't intercept - yield command - return + ): + from_client = event.connection is self.context.client - if isinstance(command, commands.SendData): - if stream_id == "udp": - yield SendQuicDatagram(conn, command.data) - else: - yield SendQuicStreamData(conn, stream_id, command.data) - elif isinstance(command, commands.CloseConnection): - if stream_id == "udp": - pass - else: - yield CloseQuicStream(conn, stream_id) - elif isinstance(command, commands.OpenConnection): - assert conn is not self.context.client - stream_id = self.server_to_stream_id[command.connection] - if err := command.connection.error: - yield from self.event_to_child(stream_id, events.OpenConnectionCompleted(command, f"Connection killed: {err}")) - return - stream_id_or_err = yield OpenQuicStream(conn) - if isinstance(stream_id_or_err, str): - yield from self.event_to_child(stream_id, events.OpenConnectionCompleted(command, stream_id_or_err)) - return - self.server_stream_id_to_client_stream_id[stream_id_or_err] = stream_id - self.streams[stream_id].context.server.set_state(connection.ConnectionState.OPEN) - yield from self.event_to_child(stream_id, events.OpenConnectionCompleted(command, None)) + # fetch or create the layer + stream_ids = self.client_stream_ids if from_client else self.server_stream_ids + if event.stream_id in stream_ids: + stream_layer = stream_ids[event.stream_id] + else: + # ensure we haven't just forgotten to register the ID + assert stream_is_client_initiated(event.stream_id) == from_client + # for server-initiated streams we need to open the client as well + if from_client: + client_stream_id = event.stream_id + server_stream_id = None else: - command.connection = conn - yield command - + client_stream_id = cast(int, (yield OpenQuicStream( + connection=self.context.client, + is_unidirectional=stream_is_unidirectional(event.stream_id), + ))) + server_stream_id = event.stream_id + + # create, register and start the layer + stream_layer = QuicStreamLayer(self.context, self.ignore, client_stream_id, server_stream_id) + stream_layer.context.client.state = get_stream_connection_state(client_stream_id, is_client=False) + self.client_stream_ids[client_stream_id] = stream_layer + if server_stream_id is not None: + stream_layer.context.server.state = get_stream_connection_state(server_stream_id, is_client=True) + self.server_stream_ids[server_stream_id] = stream_layer + self.connections[stream_layer.context.client] = stream_layer + self.connections[stream_layer.context.server] = stream_layer + yield from self.event_to_child(stream_layer, events.Start()) + + # get the target connection and ensure it is managed + conn = stream_layer.context.client if from_client else stream_layer.context.server + assert conn in self.connections + + # forward the data and close events + if isinstance(event, QuicStreamDataReceived): + yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) + if event.end_stream: + yield from self.close_read(stream_layer, conn) + elif isinstance(event, QuicStreamReset): + if self.debug is not None: + yield commands.Log(f"{self.debug}[quic] stream_reset (stream_id={event.stream_id}, error_code={event.error_code})") + yield from self.close_read(stream_layer, conn) else: - yield command - - -class QuicRoamingLayer(layer.Layer): - """Simple routing layer that replaces a `ClientQuicLayer` when a connection roams to a different - `ConnectionHandler`.""" - - def __init__(self, context: context.Context, target_layer: ClientQuicLayer) -> None: - super().__init__(context) - self.target_layer = target_layer + raise AssertionError(f"Unexpected stream event: {event!r}") + + # handle close events that target this context + elif ( + isinstance(event, events.ConnectionClosed) + and ( + event.connection is self.context.client + or event.connection is self.context.server + ) + ): + # always for to the datagram layer + yield from self.event_to_child(self.datagram_layer, event) - @expect() - def state_closed(self, _) -> layer.CommandGenerator[None]: - yield from () + # forward to either the client or server connection of stream layers + for conn, conn_layer in self.connections.items(): + if (conn is conn_layer.context.client) == (event.connection is self.context.client): + conn.state &= ~connection.ConnectionState.CAN_WRITE + yield from self.close_read(conn_layer, conn) - @expect(events.DataReceived, events.ConnectionClosed, events.MessageInjected) - def state_relay(self, event: events.Event) -> layer.CommandGenerator[None]: - - if isinstance(event, events.MessageInjected): - # ensure the flow matches the target and forward the event - assert event.flow.client_conn is self.target_layer.context.client - handler = self.target_layer.context.handler - assert handler - handler.server_event(event) - - elif isinstance(event, events.ConnectionClosed): - # remove the registration and stop relaying - assert event.connection is self.context.client - self.target_layer.remove_route(self.context) - self._handle_event = self.state_closed - - elif isinstance(event, events.DataReceived): - # update target's peername and forward the event - assert event.connection is self.context.client - handler = self.target_layer.context.handler - assert handler - handler.client.peername = self.context.client.peername - handler.server_event(events.DataReceived(handler.client, event.data)) + # all other connection events are routed to their corresponding layer + elif isinstance(event, events.ConnectionEvent): + yield from self.event_to_child(self.connections[event.connection], event) else: raise AssertionError(f"Unexpected event: {event!r}") - yield from () + def close_read(self, layer: layer.Layer, conn: connection.Connection) -> layer.CommandGenerator[None]: + """Closes the incoming part of a connection.""" + + if conn.state & connection.ConnectionState.CAN_READ: + conn.state &= ~connection.ConnectionState.CAN_READ + yield from self.event_to_child(layer, events.ConnectionClosed(conn)) + + def event_to_child(self, layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: + for command in layer.handle_event(event): + # intercept commands for streams connections + if ( + isinstance(layer, QuicStreamLayer) + and isinstance(command, commands.ConnectionCommand) + and command.connection in self.connections + ): + # get the target connection and stream ID + from_client = command.connection is layer.context.client + conn = self.context.client if from_client else self.context.server + stream_id = layer.client_stream_id if from_client else layer.server_stream_id + assert stream_id is not None + + # write data and check CloseConnection wasn't called before + if isinstance(command, commands.SendData): + assert conn.state & connection.ConnectionState.CAN_WRITE + yield SendQuicStreamData(conn, stream_id, command.data) - @expect(events.Start) - def state_start(self, _) -> layer.CommandGenerator[None]: - # register with the target and start relaying - self.target_layer.add_route(self.context) - self._handle_event = self.state_relay - yield from () + # send a FIN and optionally also a STOP frame + elif isinstance(command, commands.CloseConnection): + if conn.state & connection.ConnectionState.CAN_WRITE: + conn.state &= ~connection.ConnectionState.CAN_WRITE + yield SendQuicStreamData(conn, stream_id, b"", end_stream=True) + if not command.half_close: + yield StopQuicStream(conn, stream_id, QuicErrorCode.NO_ERROR) + yield from self.close_read(layer, conn) + + elif isinstance(command, commands.OpenConnection): + assert not from_client + assert layer.server_stream_id is None + layer.context.server.timestamp_start = time.time() + layer.server_stream_id = cast(int, (yield OpenQuicStream(conn))) + layer.context.server.state = connection.ConnectionState.OPEN + self.server_stream_ids[layer.server_stream_id] = layer + yield from self.event_to_child(layer, events.OpenConnectionCompleted(command, None)) + + else: + raise AssertionError(f"Unexpected stream connection command: {command!r}") - _handle_event = state_start + # remember blocking and wakeup commands + else: + if command.blocking or isinstance(command, commands.RequestWakeup): + self.command_sources[command] = layer + yield command class QuicLayer(tunnel.TunnelLayer): - child_layer: layer.Layer - conn: connection.Connection quic: QuicConnection | None = None tls: QuicTlsSettings | None = None def __init__(self, context: context.Context, conn: connection.Connection) -> None: super().__init__(context, tunnel_connection=conn, conn=conn) + self.child_layer = layer.NextLayer(self.context, ask_on_start=True) self._loop = asyncio.get_event_loop() self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() self._routes: dict[connection.Address, ConnectionHandler | None] = dict() @@ -519,34 +559,52 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: event = events.DataReceived(self.tunnel_connection, b"") yield from super()._handle_event(event) - def add_route(self, context: context.Context) -> None: - """Registers a new roamed context.""" - - assert context.client.peername not in self._routes - self._routes[context.client.peername] = context.handler + def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[None]: + """Turns stream commands into aioquic connection invocations.""" - def remove_route(self, context: context.Context) -> None: - """Removes a registered roamed context.""" - - assert self._routes[context.client.peername] == context.handler - del self._routes[context.client.peername] + if ( + isinstance(command, SendQuicStreamData) + and command.connection is self.conn + ): + assert self.quic + self.quic.send_stream_data(command.stream_id, command.data, command.end_stream) + yield from self.tls_interact() - def issue_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: - """Called when aioquic issues a new ID for a connection.""" + elif ( + isinstance(command, ResetQuicStream) + and command.connection is self.conn + ): + assert self.quic + self.quic.reset_stream(command.stream_id, command.error_code) + yield from self.tls_interact() - yield from () + elif ( + isinstance(command, StopQuicStream) + and command.connection is self.conn + ): + assert self.quic + self.quic.stop_stream(command.stream_id, command.error_code) + yield from self.tls_interact() - def retire_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: - """Called when aioquic retires an old ID for a connection.""" + elif ( + isinstance(command, OpenQuicStream) + and command.connection is self.conn + ): + assert self.quic + stream_id = self.quic.get_next_available_stream_id(command.is_unidirectional) + # the next operation is a no-op, but will allocate the stream ID + self.quic.send_stream_data(stream_id, data=b"", end_stream=False) + self.event_to_child(OpenQuicStreamCompleted(command, stream_id)) - yield from () + else: + yield from super()._handle_command(command) def start_tls(self, original_destination_connection_id: bytes | None) -> layer.CommandGenerator[None]: """Initiates the aioquic connection.""" # must only be called if QUIC is uninitialized - assert self.quic is None - assert self.tls is None + assert not self.quic + assert not self.tls # query addons to provide the necessary TLS settings tls_data = QuicTlsData(self.conn, self.context) @@ -566,7 +624,6 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C if self.conn.alpn_offers else H3_ALPN ), - connection_id_length=self.context.options.quic_connection_id_length, is_client=self.conn is self.context.server, secrets_log_file=( QuicSecretsLogger(tls.log_master_secret) # type: ignore @@ -593,25 +650,14 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C self.quic.connect(self.conn.peername, now=self._loop.time()) yield from self.tls_interact() - # issue the host connection ID right away - self.issue_connection_id(self.quic.host_cid) - def tls_interact(self) -> layer.CommandGenerator[None]: """Retrieves all pending outgoing packets from aioquic and sends the data.""" # send all queued datagrams - assert self.quic is not None + assert self.quic for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): - if addr == self.conn.peername: - yield commands.SendData(self.conn, data) - else: - handler = self._routes.get(addr, None) - if handler is None: - yield commands.Log(f"{self.conn}: No route to {human.format_address(addr)}.") - else: - writer = handler.transports[handler.client].writer - assert writer is not None - writer.write(data) + assert addr == self.conn.peername + yield commands.SendData(self.tunnel_connection, data) # request a new wakeup if all pending requests trigger at a later time timer = self.quic.get_timer() @@ -632,11 +678,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # handle pre-handshake events while event := self.quic.next_event(): - if isinstance(event, quic_events.ConnectionIdIssued): - yield from self.issue_connection_id(event.connection_id) - elif isinstance(event, quic_events.ConnectionIdRetired): - yield from self.retire_connection_id(event.connection_id) - elif isinstance(event, quic_events.ConnectionTerminated): + if isinstance(event, quic_events.ConnectionTerminated): err = event.reason_phrase or error_code_to_str(event.error_code) return False, err elif isinstance(event, quic_events.HandshakeCompleted): @@ -666,7 +708,12 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo yield from self.tls_interact() return True, None - elif isinstance(event, (quic_events.PingAcknowledged, quic_events.ProtocolNegotiated)): + elif isinstance(event, ( + quic_events.ConnectionIdIssued, + quic_events.ConnectionIdRetired, + quic_events.PingAcknowledged, + quic_events.ProtocolNegotiated, + )): pass else: raise AssertionError(f"Unexpected event: {event!r}") @@ -692,11 +739,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: # handle post-handshake events while event := self.quic.next_event(): - if isinstance(event, quic_events.ConnectionIdIssued): - yield from self.issue_connection_id(event.connection_id) - elif isinstance(event, quic_events.ConnectionIdRetired): - yield from self.retire_connection_id(event.connection_id) - elif isinstance(event, quic_events.ConnectionTerminated): + if isinstance(event, quic_events.ConnectionTerminated): if self.debug: reason = event.reason_phrase or error_code_to_str(event.error_code) yield commands.Log( @@ -704,18 +747,18 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: ) yield commands.CloseConnection(self.conn) return # we don't handle any further events, nor do/can we transmit data, so exit - elif isinstance(event, quic_events.PingAcknowledged): - pass elif isinstance(event, quic_events.DatagramFrameReceived): - e = QuicDatagramReceived(self.conn, event.data) - yield from self.event_to_child(e) + yield from self.event_to_child(events.DataReceived(self.conn, event.data)) elif isinstance(event, quic_events.StreamDataReceived): - e = QuicStreamDataReceived(self.conn, event.data, event.stream_id, event.end_stream) - yield commands.Log(f"{e!r}") - yield from self.event_to_child(e) + yield from self.event_to_child(QuicStreamDataReceived(self.conn, event.data, event.stream_id, event.end_stream)) elif isinstance(event, quic_events.StreamReset): - e = QuicStreamReset(self.conn, event.stream_id, event.error_code) - yield from self.event_to_child(e) + yield from self.event_to_child(QuicStreamReset(self.conn, event.stream_id, event.error_code)) + elif isinstance(event, ( + quic_events.ConnectionIdIssued, + quic_events.ConnectionIdRetired, + quic_events.PingAcknowledged, + )): + pass else: raise AssertionError(f"Unexpected event: {event!r}") @@ -726,28 +769,19 @@ def receive_close(self) -> layer.CommandGenerator[None]: # unlike TLS we haven't sent CloseConnection before yield from super().receive_close() - def send_data(self, command: commands.SendData) -> layer.CommandGenerator[None]: - if isinstance(command, SendQuicDatagram): - self.quic.send_datagram_frame(command.data) - elif isinstance(command, SendQuicStreamData): - self.quic.send_stream_data(command.stream_id, command.data) - elif isinstance(command, QuicTransmit): - yield commands.Log("SendQuicTrigger") - assert not command.data - else: - raise AssertionError(f"Unexpected command: {command}") - + def send_data(self, data: bytes) -> layer.CommandGenerator[None]: + # non-stream data uses datagram frames + assert self.quic + if data: + self.quic.send_datagram_frame(data) yield from self.tls_interact() - def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: - if isinstance(command, CloseQuicStream): - if self.quic._stream_can_send(command.stream_id): - self.quic.send_stream_data(command.stream_id, b"", end_stream=True) - else: - if self.quic is not None: - self.quic.close() - yield from self.tls_interact() - yield from super().send_close(command) + def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: + # properly close the QUIC connection + if self.quic is not None: + self.quic.close() + yield from self.tls_interact() + yield from super().send_close(half_close) class ServerQuicLayer(QuicLayer): @@ -794,13 +828,8 @@ class ClientQuicLayer(QuicLayer): This layer establishes QUIC on a single client connection or roams to another connection. """ - connections: ClassVar[dict[tuple[connection.Address, bytes], ClientQuicLayer]] = dict() - """Mapping of (sockname, cid) tuples to QUIC client layers.""" - - server_layer: ServerQuicLayer | None - """The server layer sitting on top of this layer, or `None`.""" - is_top_level: bool - """Indicated whether this layer is receiving UDP packets directly.""" + server_tls_available: bool + """Indicates whether the parent layer is a ServerQuicLayer.""" def __init__(self, context: context.Context) -> None: # same as ClientTLSLayer, we might be nested in some other transport @@ -816,34 +845,7 @@ def __init__(self, context: context.Context) -> None: context.client.cipher_list = [] super().__init__(context, context.client) - parent_layer = self.context.layers[-2] - self.server_layer = parent_layer if isinstance(parent_layer, ServerQuicLayer) else None - self.is_top_level = len(context.layers) == (2 if self.server_layer is None else 3) - - def issue_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: - if self.is_top_level: - cid = (self.context.client.sockname, connection_id) - assert cid not in ClientQuicLayer.connections - ClientQuicLayer.connections[cid] = self - yield from super().issue_connection_id(connection_id) - - def retire_connection_id(self, connection_id: bytes) -> layer.CommandGenerator[None]: - if self.is_top_level: - cid = (self.context.client.sockname, connection_id) - assert ClientQuicLayer.connections[cid] == self - del ClientQuicLayer.connections[cid] - yield from super().retire_connection_id(connection_id) - - def replace_layer(self, initial_data: bytes, replacement_layer: layer.Layer) -> layer.CommandGenerator[tuple[bool, str | None]]: - """Replaces the QUIC layer(s) with another layer.""" - - # we need to replace the server layer as well, if there is one - layer_to_replace = self if self.server_layer is None else self.server_layer - layer_to_replace.handle_event = replacement_layer.handle_event # type: ignore - layer_to_replace._handle_event = replacement_layer._handle_event # type: ignore - yield from replacement_layer.handle_event(events.Start()) - yield from replacement_layer.handle_event(events.DataReceived(self.conn, initial_data)) - return True, None + self.server_tls_available = isinstance(self.context.layers[-2], ServerQuicLayer) def start_handshake(self) -> layer.CommandGenerator[None]: yield from () @@ -856,9 +858,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # fail if the received data is not a QUIC packet buffer = QuicBuffer(data=data) try: - header = pull_quic_header( - buffer, host_cid_length=self.context.options.quic_connection_id_length - ) + header = pull_quic_header(buffer) except ValueError as e: return False, f"Cannot parse QUIC header: {e} ({data.hex()})" @@ -879,24 +879,9 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo ) return False, None - # check if this is a new connection - target_layer = ClientQuicLayer.connections.get((self.context.client.sockname, header.destination_cid), None) - if target_layer is None: - # try to start QUIC - return (yield from self.start_client_tls(data, header)) - - else: - # ensure that we can roam - if (self.is_top_level and target_layer.context.client.proxy_mode is self.context.client.proxy_mode): - return False, "Connection cannot roam." - - # replace the layer with a roaming layer - return (yield from self.replace_layer(data, QuicRoamingLayer(self.context, target_layer))) - - def start_client_tls(self, data: bytes, header: QuicHeader) -> layer.CommandGenerator[tuple[bool, str | None]]: # ensure it's (likely) a client handshake packet if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: - return False, f"Invalid handshake received. ({data.hex()})" + return False, f"Invalid handshake received, roaming not supported. ({data.hex()})" # extract the client hello try: @@ -918,11 +903,19 @@ def start_client_tls(self, data: bytes, header: QuicHeader) -> layer.CommandGene ("ignore-conn", 0), ("ignore-conn", 0), self._loop.time(), transport_protocol="udp", proxy_mode=self.context.client.proxy_mode ) - if self.server_layer is not None: - self.server_layer.conn = self.server_layer.tunnel_connection = connection.Server( + + # we need to replace the server layer as well, if there is one + parent_layer = self.context.layers[self.context.layers.index(self) - 1] + if isinstance(parent_layer, ServerQuicLayer): + parent_layer.conn = parent_layer.tunnel_connection = connection.Server( None ) - return (yield from self.replace_layer(data, UDPLayer(self.context, ignore=True))) + replacement_layer = UDPLayer(self.context, ignore=True) + parent_layer.handle_event = replacement_layer.handle_event # type: ignore + parent_layer._handle_event = replacement_layer._handle_event # type: ignore + yield from parent_layer.handle_event(events.Start()) + yield from parent_layer.handle_event(events.DataReceived(self.conn, data)) + return True, None # start the server QUIC connection if demanded and available if ( @@ -947,7 +940,7 @@ def start_client_tls(self, data: bytes, header: QuicHeader) -> layer.CommandGene return (yield from super().receive_handshake_data(data)) def start_server_tls(self) -> layer.CommandGenerator[str | None]: - if self.server_layer is None: + if not self.server_tls_available: return f"No server QUIC available." err = yield commands.OpenConnection(self.context.server) return err diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 79350a12b7..22f45750c3 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -430,17 +430,17 @@ def receive_close(self) -> layer.CommandGenerator[None]: else: yield from super().receive_close() - def send_data(self, command: commands.SendData) -> layer.CommandGenerator[None]: + def send_data(self, data: bytes) -> layer.CommandGenerator[None]: try: - self.tls.sendall(command.data) + self.tls.sendall(data) except (SSL.ZeroReturnError, SSL.SysCallError): # The other peer may still be trying to send data over, which we discard here. pass yield from self.tls_interact() - def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: + def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: # We should probably shutdown the TLS connection properly here. - yield from super().send_close(command) + yield from super().send_close(half_close) class ServerTLSLayer(TLSLayer): diff --git a/mitmproxy/proxy/tunnel.py b/mitmproxy/proxy/tunnel.py index 1c8317f98b..7a481a7014 100644 --- a/mitmproxy/proxy/tunnel.py +++ b/mitmproxy/proxy/tunnel.py @@ -108,6 +108,37 @@ def _handshake_finished(self, err: Optional[str]): yield from self.event_to_child(evt) self._event_queue.clear() + def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[None]: + if ( + isinstance(command, commands.ConnectionCommand) + and command.connection == self.conn + ): + if isinstance(command, commands.SendData): + yield from self.send_data(command.data) + elif isinstance(command, commands.CloseConnection): + if self.conn != self.tunnel_connection: + if command.half_close: + self.conn.state &= ~connection.ConnectionState.CAN_WRITE + else: + self.conn.state = connection.ConnectionState.CLOSED + yield from self.send_close(command.half_close) + elif isinstance(command, commands.OpenConnection): + # create our own OpenConnection command object that blocks here. + self.command_to_reply_to = command + self.tunnel_state = TunnelState.ESTABLISHING + err = yield commands.OpenConnection(self.tunnel_connection) + if err: + yield from self.event_to_child( + events.OpenConnectionCompleted(command, err) + ) + self.tunnel_state = TunnelState.CLOSED + else: + yield from self.start_handshake() + else: # pragma: no cover + raise AssertionError(f"Unexpected command: {command}") + else: + yield command + def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: if ( self.tunnel_state is TunnelState.ESTABLISHING @@ -116,35 +147,7 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: self._event_queue.append(event) return for command in self.child_layer.handle_event(event): - if ( - isinstance(command, commands.ConnectionCommand) - and command.connection == self.conn - ): - if isinstance(command, commands.SendData): - yield from self.send_data(command) - elif isinstance(command, commands.CloseConnection): - if self.conn != self.tunnel_connection: - if command.half_close: - self.conn.state &= ~connection.ConnectionState.CAN_WRITE - else: - self.conn.state = connection.ConnectionState.CLOSED - yield from self.send_close(command) - elif isinstance(command, commands.OpenConnection): - # create our own OpenConnection command object that blocks here. - self.command_to_reply_to = command - self.tunnel_state = TunnelState.ESTABLISHING - err = yield commands.OpenConnection(self.tunnel_connection) - if err: - yield from self.event_to_child( - events.OpenConnectionCompleted(command, err) - ) - self.tunnel_state = TunnelState.CLOSED - else: - yield from self.start_handshake() - else: # pragma: no cover - raise AssertionError(f"Unexpected command: {command}") - else: - yield command + yield from self._handle_command(command) def start_handshake(self) -> layer.CommandGenerator[None]: yield from self._handle_event(events.DataReceived(self.tunnel_connection, b"")) @@ -166,11 +169,11 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: def receive_close(self) -> layer.CommandGenerator[None]: yield from self.event_to_child(events.ConnectionClosed(self.conn)) - def send_data(self, command: commands.SendData) -> layer.CommandGenerator[None]: - yield commands.SendData(self.tunnel_connection, command.data) + def send_data(self, data: bytes) -> layer.CommandGenerator[None]: + yield commands.SendData(self.tunnel_connection, data) - def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: - yield commands.CloseConnection(self.tunnel_connection, half_close=command.half_close) + def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: + yield commands.CloseConnection(self.tunnel_connection, half_close=half_close) class LayerStack: diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py new file mode 100644 index 0000000000..effb36afe7 --- /dev/null +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -0,0 +1,58 @@ +import pytest + +from mitmproxy.proxy.layers import quic + + +def test_error_code_to_str(): + assert quic.error_code_to_str(0x6) == "FINAL_SIZE_ERROR" + assert quic.error_code_to_str(0x104) == "H3_CLOSED_CRITICAL_STREAM" + assert quic.error_code_to_str(0xdead) == f"unknown error (0xdead)" + + +def test_is_success_error_code(): + assert quic.is_success_error_code(0x0) + assert not quic.is_success_error_code(0x6) + assert quic.is_success_error_code(0x100) + assert not quic.is_success_error_code(0x104) + assert not quic.is_success_error_code(0xdead) + + +client_hello = bytes.fromhex( + "ca0000000108c0618c84b54541320823fcce946c38d8210044e6a93bbb283593f75ffb6f2696b16cfdcb5b1255" + "577b2af5fc5894188c9568bc65eef253faf7f0520e41341cfa81d6aae573586665ce4e1e41676364820402feec" + "a81f3d22dbb476893422069066104a43e121c951a08c53b83f960becf99cf5304d5bc5346f52f472bd1a04d192" + "0bae025064990d27e5e4c325ac46121d3acadebe7babdb96192fb699693d65e2b2e21c53beeb4f40b50673a2f6" + "c22091cb7c76a845384fedee58df862464d1da505a280bfef91ca83a10bebbcb07855219dbc14aecf8a48da049" + "d03c77459b39d5355c95306cd03d6bdb471694fa998ca3b1f875ce87915b88ead15c5d6313a443f39aad808922" + "57ddfa6b4a898d773bb6fb520ede47ebd59d022431b1054a69e0bbbdf9f0fb32fc8bcc4b6879dd8cd5389474b1" + "99e18333e14d0347740a11916429a818bb8d93295d36e99840a373bb0e14c8b3adcf5e2165e70803f15316fd5e" + "5eeec04ae68d98f1adb22c54611c80fcd8ece619dbdf97b1510032ec374b7a71f94d9492b8b8cb56f56556dd97" + "edf1e50fa90e868ff93636a365678bdf3ee3f8e632588cd506b6f44fbfd4d99988238fbd5884c98f6a124108c1" + "878970780e42b111e3be6215776ef5be5a0205915e6d720d22c6a81a475c9e41ba94e4983b964cb5c8e1f40607" + "76d1d8d1adcef7587ea084231016bd6ee2643d11a3a35eb7fe4cca2b3f1a4b21e040b0d426412cca6c4271ea63" + "fb54ed7f57b41cd1af1be5507f87ea4f4a0c997367e883291de2f1b8a49bdaa52bae30064351b1139703400730" + "18a4104344ec6b4454b50a42e804bc70e78b9b3c82497273859c82ed241b643642d76df6ceab8f916392113a62" + "b231f228c7300624d74a846bec2f479ab8a8c3461f91c7bf806236e3bd2f54ba1ef8e2a1e0bfdde0c5ad227f7d" + "364c52510b1ade862ce0c8d7bd24b6d7d21c99b34de6d177eb3d575787b2af55060d76d6c2060befbb7953a816" + "6f66ad88ecf929dbb0ad3a16cf7dfd39d925e0b4b649c6d0c07ad46ed0229c17fb6a1395f16e1b138aab3af760" + "2b0ac762c4f611f7f3468997224ffbe500a7c53f92f65e41a3765a9f1d7e3f78208f5b4e147962d8c97d6c1a80" + "91ffc36090b2043d71853616f34c2185dc883c54ab6d66e10a6c18e0b9a4742597361f8554a42da3373241d0c8" + "54119bfadccffaf2335b2d97ffee627cb891bda8140a39399f853da4859f7e19682e152243efbaffb662edd19b" + "3819a74107c7dbe05ecb32e79dcdb1260f153b1ef133e978ccca3d9e400a7ed6c458d77e2956d2cb897b7a298b" + "fe144b5defdc23dfd2adf69f1fb0917840703402d524987ae3b1dcb85229843c9a419ef46e1ba0ba7783f2a2ec" + "d057a57518836aef2a7839ebd3688da98b54c942941f642e434727108d59ea25875b3050ca53d4637c76cbcbb9" + "e972c2b0b781131ee0a1403138b55486fe86bbd644920ee6aa578e3bab32d7d784b5c140295286d90c99b14823" + "1487f7ea64157001b745aa358c9ea6bec5a8d8b67a7534ec1f7648ff3b435911dfc3dff798d32fbf2efe2c1fcc" + "278865157590572387b76b78e727d3e7682cb501cdcdf9a0f17676f99d9aa67f10edccc9a92080294e88bf28c2" + "a9f32ae535fdb27fff7706540472abb9eab90af12b2bea005da189874b0ca69e6ae1690a6f2adf75be3853c94e" + "fd8098ed579c20cb37be6885d8d713af4ba52958cee383089b98ed9cb26e11127cf88d1b7d254f15f7903dd7ed" + "297c0013924e88248684fe8f2098326ce51aa6e5" +) + + +def test_parse_client_hello(): + assert quic.quic_parse_client_hello(client_hello).sni == "example.com" + with pytest.raises(ValueError): + quic.quic_parse_client_hello( + client_hello[:183] + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00" + ) diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index b9086834ad..ec42cbd64b 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -637,7 +637,7 @@ def test_cannot_parse_clienthello(self, tctx: context.Context): >> events.DataReceived(Server(None), b"data on other stream") << commands.Log(">> DataReceived(server, b'data on other stream')", "debug") << commands.Log( - "Swallowing DataReceived(server, b'data on other stream') as handshake failed.", + "[tls] Swallowing DataReceived(server, b'data on other stream') as handshake failed.", "debug", ) ) From b7831c05019a015077be5daac382a90d0f73af86 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 11 Sep 2022 18:32:04 +0200 Subject: [PATCH 059/695] [quic] mostly just var renaming --- mitmproxy/proxy/commands.py | 2 +- mitmproxy/proxy/events.py | 2 +- mitmproxy/proxy/layers/quic.py | 46 ++++++++++++++++------------- test/mitmproxy/proxy/test_tunnel.py | 4 +-- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/mitmproxy/proxy/commands.py b/mitmproxy/proxy/commands.py index 366f961979..f1cefb8440 100644 --- a/mitmproxy/proxy/commands.py +++ b/mitmproxy/proxy/commands.py @@ -74,7 +74,7 @@ def __init__(self, connection: Connection, data: bytes): def __repr__(self): target = str(self.connection).split("(", 1)[0].lower() - return f"{self.__class__.__name__}({target}, {self.data})" + return f"SendData({target}, {self.data})" class OpenConnection(ConnectionCommand): diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index 6afc7841ba..b571f3ad2f 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -47,7 +47,7 @@ class DataReceived(ConnectionEvent): def __repr__(self): target = type(self.connection).__name__.lower() - return f"{self.__class__.__name__}({target}, {self.data})" + return f"DataReceived({target}, {self.data})" class ConnectionClosed(ConnectionEvent): diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index c62f6f1e93..70a84dac22 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -448,11 +448,11 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, QuicStreamDataReceived): yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) if event.end_stream: - yield from self.close_read(stream_layer, conn) + yield from self.close_stream_layer(stream_layer, conn) elif isinstance(event, QuicStreamReset): if self.debug is not None: yield commands.Log(f"{self.debug}[quic] stream_reset (stream_id={event.stream_id}, error_code={event.error_code})") - yield from self.close_read(stream_layer, conn) + yield from self.close_stream_layer(stream_layer, conn) else: raise AssertionError(f"Unexpected stream event: {event!r}") @@ -468,10 +468,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.event_to_child(self.datagram_layer, event) # forward to either the client or server connection of stream layers - for conn, conn_layer in self.connections.items(): - if (conn is conn_layer.context.client) == (event.connection is self.context.client): + for conn, child_layer in self.connections.items(): + if ( + isinstance(child_layer, QuicStreamLayer) + and (conn is child_layer.context.client) == (event.connection is self.context.client) + ): conn.state &= ~connection.ConnectionState.CAN_WRITE - yield from self.close_read(conn_layer, conn) + yield from self.close_stream_layer(child_layer, conn) # all other connection events are routed to their corresponding layer elif isinstance(event, events.ConnectionEvent): @@ -480,25 +483,27 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unexpected event: {event!r}") - def close_read(self, layer: layer.Layer, conn: connection.Connection) -> layer.CommandGenerator[None]: + def close_stream_layer(self, stream_layer: QuicStreamLayer, conn: connection.Connection) -> layer.CommandGenerator[None]: """Closes the incoming part of a connection.""" if conn.state & connection.ConnectionState.CAN_READ: conn.state &= ~connection.ConnectionState.CAN_READ - yield from self.event_to_child(layer, events.ConnectionClosed(conn)) + yield from self.event_to_child(stream_layer, events.ConnectionClosed(conn)) + + def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: + """Forwards events to child layers and translates commands.""" - def event_to_child(self, layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: - for command in layer.handle_event(event): + for command in child_layer.handle_event(event): # intercept commands for streams connections if ( - isinstance(layer, QuicStreamLayer) + isinstance(child_layer, QuicStreamLayer) and isinstance(command, commands.ConnectionCommand) and command.connection in self.connections ): # get the target connection and stream ID - from_client = command.connection is layer.context.client + from_client = command.connection is child_layer.context.client conn = self.context.client if from_client else self.context.server - stream_id = layer.client_stream_id if from_client else layer.server_stream_id + stream_id = child_layer.client_stream_id if from_client else child_layer.server_stream_id assert stream_id is not None # write data and check CloseConnection wasn't called before @@ -513,16 +518,17 @@ def event_to_child(self, layer: layer.Layer, event: events.Event) -> layer.Comma yield SendQuicStreamData(conn, stream_id, b"", end_stream=True) if not command.half_close: yield StopQuicStream(conn, stream_id, QuicErrorCode.NO_ERROR) - yield from self.close_read(layer, conn) + yield from self.close_stream_layer(child_layer, conn) + # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): assert not from_client - assert layer.server_stream_id is None - layer.context.server.timestamp_start = time.time() - layer.server_stream_id = cast(int, (yield OpenQuicStream(conn))) - layer.context.server.state = connection.ConnectionState.OPEN - self.server_stream_ids[layer.server_stream_id] = layer - yield from self.event_to_child(layer, events.OpenConnectionCompleted(command, None)) + assert child_layer.server_stream_id is None + child_layer.context.server.timestamp_start = time.time() + child_layer.server_stream_id = cast(int, (yield OpenQuicStream(conn))) + child_layer.context.server.state = connection.ConnectionState.OPEN + self.server_stream_ids[child_layer.server_stream_id] = child_layer + yield from self.event_to_child(child_layer, events.OpenConnectionCompleted(command, None)) else: raise AssertionError(f"Unexpected stream connection command: {command!r}") @@ -530,7 +536,7 @@ def event_to_child(self, layer: layer.Layer, event: events.Event) -> layer.Comma # remember blocking and wakeup commands else: if command.blocking or isinstance(command, commands.RequestWakeup): - self.command_sources[command] = layer + self.command_sources[command] = child_layer yield command diff --git a/test/mitmproxy/proxy/test_tunnel.py b/test/mitmproxy/proxy/test_tunnel.py index f33f982525..24103637c6 100644 --- a/test/mitmproxy/proxy/test_tunnel.py +++ b/test/mitmproxy/proxy/test_tunnel.py @@ -45,8 +45,8 @@ def receive_handshake_data( else: return False, "handshake error" - def send_data(self, command: SendData) -> layer.CommandGenerator[None]: - yield SendData(self.tunnel_connection, b"tunneled-" + command.data) + def send_data(self, data: bytes) -> layer.CommandGenerator[None]: + yield SendData(self.tunnel_connection, b"tunneled-" + data) def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: yield from self.event_to_child( From 50d0aa2256ba32ecd59f642e4cc601d8ec0610e1 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 11 Sep 2022 21:28:33 +0200 Subject: [PATCH 060/695] [quic] minor fixes --- mitmproxy/proxy/layers/quic.py | 51 ++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 70a84dac22..cee0c44af8 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -315,20 +315,23 @@ class QuicStreamLayer(layer.Layer): Serves as a marker for NextLayer and keeps track of the connection states. """ + client: connection.Client client_stream_id: int + server: connection.Server server_stream_id: int | None child_layer: layer.Layer def __init__(self, context: context.Context, ignore: bool, client_stream_id: int, server_stream_id: int | None) -> None: - # We mustn't reuse the client or server from the QUIC connection as we have different states here. - context.client = connection.Client( + # We mustn't reuse client or server from the QUIC connection as we have different states here. + # Also, we store the original values to detect if the context has changed. + self.client = context.client = connection.Client( peername=context.client.peername, sockname=context.client.sockname, timestamp_start=time.time(), transport_protocol=context.client.transport_protocol, proxy_mode=context.client.proxy_mode, ) - context.server = connection.Server( + self.server = context.server = connection.Server( address=context.server.address, transport_protocol=context.server.transport_protocol, ) @@ -431,20 +434,17 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # create, register and start the layer stream_layer = QuicStreamLayer(self.context, self.ignore, client_stream_id, server_stream_id) - stream_layer.context.client.state = get_stream_connection_state(client_stream_id, is_client=False) + stream_layer.client.state = get_stream_connection_state(client_stream_id, is_client=False) self.client_stream_ids[client_stream_id] = stream_layer if server_stream_id is not None: - stream_layer.context.server.state = get_stream_connection_state(server_stream_id, is_client=True) + stream_layer.server.state = get_stream_connection_state(server_stream_id, is_client=True) self.server_stream_ids[server_stream_id] = stream_layer - self.connections[stream_layer.context.client] = stream_layer - self.connections[stream_layer.context.server] = stream_layer + self.connections[stream_layer.client] = stream_layer + self.connections[stream_layer.server] = stream_layer yield from self.event_to_child(stream_layer, events.Start()) - # get the target connection and ensure it is managed - conn = stream_layer.context.client if from_client else stream_layer.context.server - assert conn in self.connections - # forward the data and close events + conn = stream_layer.client if from_client else stream_layer.server if isinstance(event, QuicStreamDataReceived): yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) if event.end_stream: @@ -464,14 +464,16 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: or event.connection is self.context.server ) ): - # always for to the datagram layer + from_client = event.connection is self.context.client + + # always forward to the datagram layer yield from self.event_to_child(self.datagram_layer, event) # forward to either the client or server connection of stream layers for conn, child_layer in self.connections.items(): if ( isinstance(child_layer, QuicStreamLayer) - and (conn is child_layer.context.client) == (event.connection is self.context.client) + and ((conn is child_layer.client) if from_client else (conn is child_layer.server)) ): conn.state &= ~connection.ConnectionState.CAN_WRITE yield from self.close_stream_layer(child_layer, conn) @@ -498,21 +500,25 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer if ( isinstance(child_layer, QuicStreamLayer) and isinstance(command, commands.ConnectionCommand) - and command.connection in self.connections + and ( + command.connection is child_layer.client + or command.connection is child_layer.server + ) ): # get the target connection and stream ID - from_client = command.connection is child_layer.context.client + from_client = command.connection is child_layer.client conn = self.context.client if from_client else self.context.server stream_id = child_layer.client_stream_id if from_client else child_layer.server_stream_id - assert stream_id is not None # write data and check CloseConnection wasn't called before if isinstance(command, commands.SendData): + assert stream_id is not None assert conn.state & connection.ConnectionState.CAN_WRITE yield SendQuicStreamData(conn, stream_id, command.data) # send a FIN and optionally also a STOP frame elif isinstance(command, commands.CloseConnection): + assert stream_id is not None if conn.state & connection.ConnectionState.CAN_WRITE: conn.state &= ~connection.ConnectionState.CAN_WRITE yield SendQuicStreamData(conn, stream_id, b"", end_stream=True) @@ -523,11 +529,12 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): assert not from_client - assert child_layer.server_stream_id is None - child_layer.context.server.timestamp_start = time.time() - child_layer.server_stream_id = cast(int, (yield OpenQuicStream(conn))) - child_layer.context.server.state = connection.ConnectionState.OPEN - self.server_stream_ids[child_layer.server_stream_id] = child_layer + assert stream_id is None + child_layer.server.timestamp_start = time.time() + stream_id = cast(int, (yield OpenQuicStream(conn))) + child_layer.server_stream_id = stream_id + child_layer.server.state = get_stream_connection_state(stream_id, is_client=True) + self.server_stream_ids[stream_id] = child_layer yield from self.event_to_child(child_layer, events.OpenConnectionCompleted(command, None)) else: @@ -537,6 +544,8 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer else: if command.blocking or isinstance(command, commands.RequestWakeup): self.command_sources[command] = child_layer + if isinstance(command, commands.OpenConnection): + self.connections[command.connection] = child_layer yield command From 6ee9b5d039d2d66f2553c6dd11572838af30fb7d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 12 Sep 2022 15:08:49 +0200 Subject: [PATCH 061/695] bugfixes --- mitmproxy/proxy/events.py | 5 +++-- mitmproxy/proxy/layers/http/_http3.py | 1 - mitmproxy/proxy/layers/quic.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index b571f3ad2f..d3744eacf7 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -3,6 +3,7 @@ Events represent the only way for layers to receive new data from sockets. The counterpart to events are commands. """ +import typing import warnings from dataclasses import dataclass, is_dataclass from typing import Any, Generic, Optional, TypeVar @@ -72,7 +73,7 @@ def __new__(cls, *args, **kwargs): return super().__new__(cls) def __init_subclass__(cls, **kwargs): - command_cls = cls.__annotations__.get("command", None) + command_cls = typing.get_type_hints(cls).get("command", None) valid_command_subclass = ( isinstance(command_cls, type) and issubclass(command_cls, commands.Command) @@ -80,7 +81,7 @@ def __init_subclass__(cls, **kwargs): ) if not valid_command_subclass: warnings.warn( - f"{command_cls} needs a properly annotated command attribute.", + f"{cls} needs a properly annotated command attribute.", RuntimeWarning, ) if command_cls in command_reply_subclasses: diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index da9affdbc9..593665160b 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -19,7 +19,6 @@ from mitmproxy.proxy.layers.quic import ( ClientQuicLayer, QuicStreamDataReceived, QuicStreamReset, - QuicTransmit, ServerQuicLayer, error_code_to_str, ) from mitmproxy.proxy.utils import expect diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index cee0c44af8..0bc2c202eb 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -384,6 +384,7 @@ def __init__(self, context: context.Context, ignore: bool = False) -> None: context.server: self.datagram_layer, } self.command_sources = {} + self.ignore = ignore def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # we treat the datagram-layer as child layer, so forward Start @@ -765,7 +766,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: elif isinstance(event, quic_events.DatagramFrameReceived): yield from self.event_to_child(events.DataReceived(self.conn, event.data)) elif isinstance(event, quic_events.StreamDataReceived): - yield from self.event_to_child(QuicStreamDataReceived(self.conn, event.data, event.stream_id, event.end_stream)) + yield from self.event_to_child(QuicStreamDataReceived(self.conn, data=event.data, stream_id=event.stream_id, end_stream=event.end_stream)) elif isinstance(event, quic_events.StreamReset): yield from self.event_to_child(QuicStreamReset(self.conn, event.stream_id, event.error_code)) elif isinstance(event, ( From 08d26814bb8eaf094635039e2cec6a450cc9ccd9 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 18 Sep 2022 15:55:06 +0200 Subject: [PATCH 062/695] [quic] work on MockQuic --- mitmproxy/proxy/layers/http/_http3.py | 119 ++++++++++++++++++-------- mitmproxy/proxy/layers/quic.py | 52 +++++++---- 2 files changed, 118 insertions(+), 53 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 593665160b..e9665f5289 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,4 +1,5 @@ from abc import abstractmethod +from queue import Queue import time from typing import Dict, Optional, Union @@ -6,20 +7,26 @@ H3Connection, ErrorCode as H3ErrorCode, FrameUnexpected as H3FrameUnexpected, + Headers as H3Headers, HeadersState as H3HeadersState, ) from aioquic.h3 import events as h3_events from aioquic.quic import events as quic_events -from aioquic.quic.connection import QuicConnection, stream_is_client_initiated +from aioquic.quic.connection import stream_is_client_initiated, stream_is_unidirectional from aioquic.quic.packet import QuicErrorCode -from mitmproxy import http, version +from mitmproxy import connection, http, version from mitmproxy.net.http import status_codes from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( - ClientQuicLayer, QuicStreamDataReceived, + OpenQuicStream, + QuicStreamDataReceived, QuicStreamReset, - ServerQuicLayer, error_code_to_str, + ResetQuicStream, + SendQuicStreamData, + error_code_to_str, + get_connection_error, + set_connection_error, ) from mitmproxy.proxy.utils import expect @@ -54,13 +61,62 @@ class MockQuic: aioquic intermingles QUIC and HTTP/3. This is something we don't want to do because that makes testing much harder. Instead, we mock our QUIC connection object here and then take out the wire data to be sent. """ - pass - # TODO add mock for QuicConnection. + + def __init__(self, conn: connection.Connection) -> None: + self.conn = conn + self.pending_commands: Queue[commands.Command] = Queue() + self.available_stream_ids: Queue[int] = Queue() + + def close( + self, + error_code: int = QuicErrorCode.NO_ERROR, + frame_type: Optional[int] = None, + reason_phrase: str = "", + ) -> None: + set_connection_error(self.conn, quic_events.ConnectionTerminated( + error_code=error_code, + frame_type=frame_type, + reason_phrase=reason_phrase, + )) + self.pending_commands.put(commands.CloseConnection(self.conn)) + + def get_next_available_stream_id(self, is_unidirectional=False) -> int: + stream_id = self.available_stream_ids.get() + assert is_unidirectional == stream_is_unidirectional(stream_id) + return stream_id + + def send_stream_data( + self, stream_id: int, data: bytes, end_stream: bool = False + ) -> None: + self.pending_commands.put(SendQuicStreamData(self.conn, stream_id, data, end_stream)) + + +class LayeredH3Connection(H3Connection): + def __init__(self, connection: connection.Connection) -> None: + self._quic = MockQuic(connection) + super().__init__(quic=self._quic, enable_webtransport=False) + + def start(self) -> layer.CommandGenerator[None]: + # we need three unidirectional streams for `_init_connection` + for _ in range(1, 3): + self._quic.available_stream_ids.put((yield OpenQuicStream(self._quic.conn, is_unidirectional=False))) + + def create_webtransport_stream(self, session_id: int, is_unidirectional: bool = False) -> int: + raise NotImplementedError() # pragma: no cover + + def send_datagram(self, flow_id: int, data: bytes) -> None: + raise NotImplementedError() # pragma: no cover + + def send_push_promise(self, stream_id: int, headers: H3Headers) -> int: + raise NotImplementedError() # pragma: no cover + + def transmit(self) -> layer.CommandGenerator[None]: + while self._quic.pending_commands: + yield self._quic.pending_commands.get() class Http3Connection(HttpConnection): - quic: Optional[QuicConnection] = None - h3_conn: Optional[H3Connection] = None + h3_conn: Optional[LayeredH3Connection] = None ReceiveData: type[Union[RequestData, ResponseData]] ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] @@ -76,7 +132,8 @@ def parse_headers( def postprocess_outgoing_event(self, event: HttpEvent) -> HttpEvent: return event - def preprocess_incoming_event(self, event: HttpEvent) -> HttpEvent: + def preprocess_incoming_event(self, event: HttpEvent) -> layer.CommandGenerator[HttpEvent]: + yield from () return event @abstractmethod @@ -91,12 +148,11 @@ def state_done(self, _) -> layer.CommandGenerator[None]: @expect(HttpEvent, QuicStreamDataReceived, QuicStreamReset, events.ConnectionClosed) def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.quic is not None assert self.h3_conn is not None # send mitmproxy HTTP events over the H3 connection if isinstance(event, HttpEvent): - event = self.preprocess_incoming_event(event) + event = yield from self.preprocess_incoming_event(event) try: if isinstance(event, (RequestData, ResponseData)): @@ -141,7 +197,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: else: # transmit buffered data and re-arm timer - yield QuicTransmit(self.conn) + yield from self.h3_conn.transmit() elif isinstance(event, QuicStreamReset): if ( @@ -214,11 +270,12 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: receive_event = self.parse_headers(h3_event) except ValueError as e: # this will result in a ConnectionClosed event - self.quic.close( + set_connection_error(self.conn, quic_events.ConnectionTerminated( error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, + frame_type=None, reason_phrase=f"Invalid HTTP/3 request headers: {e}", - ) - yield QuicTransmit(self.conn) + )) + yield commands.CloseConnection(self.conn) else: yield ReceiveHttp( self.postprocess_outgoing_event(receive_event) @@ -247,7 +304,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, events.ConnectionClosed): for stream in self.h3_conn._stream.values(): if stream_is_client_initiated(stream.stream_id) and not stream.ended: - close_event = self.quic._close_event + close_event = get_connection_error(self.conn) yield ReceiveHttp( self.postprocess_outgoing_event( self.ReceiveProtocolError( @@ -272,23 +329,11 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: @expect(events.Start) def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: - assert isinstance(event, events.Start) - - # aioquic does not separate QUIC and HTTP/3, poke through the layer stack to get a reference to the QUIC - # connection object. - for x in reversed(self.context.layers): - if isinstance(x, ClientQuicLayer if isinstance(self, Http3Server) else ServerQuicLayer): - self.quic = x.quic - self.h3_conn = H3Connection(self.quic, enable_webtransport=False) - break - else: - raise AssertionError - - # self.quic = MockQuic() - # self.h3_conn = H3Connection(self.quic) + assert self.h3_conn is None + self.h3_conn = LayeredH3Connection(self.conn) self._handle_event = self.state_ready - yield from () + yield from self.h3_conn.start() _handle_event = state_start @@ -382,15 +427,14 @@ def postprocess_outgoing_event(self, event: HttpEvent) -> HttpEvent: event.stream_id = self._quic_to_event[event.stream_id] return event - def preprocess_incoming_event(self, event: HttpEvent) -> HttpEvent: + def preprocess_incoming_event(self, event: HttpEvent) -> layer.CommandGenerator[HttpEvent]: if event.stream_id in self._event_to_quic: event.stream_id = self._event_to_quic[event.stream_id] else: # QUIC and HTTP/3 would actually allow for direct stream ID mapping, but since we want # to support H2<->H3, we need to translate IDs. # NOTE: We always create bidirectional streams, as we can't safely infer unidirectionality. - assert self.quic is not None - stream_id = self.quic.get_next_available_stream_id() + stream_id = yield OpenQuicStream(self.conn) self._event_to_quic[event.stream_id] = stream_id self._quic_to_event[stream_id] = event.stream_id event.stream_id = stream_id @@ -398,13 +442,16 @@ def preprocess_incoming_event(self, event: HttpEvent) -> HttpEvent: def send_protocol_error(self, event: Union[RequestProtocolError, ResponseProtocolError]) -> None: assert isinstance(event, RequestProtocolError) - assert self.quic is not None # same as HTTP/2 code = event.code if code != H3ErrorCode.H3_REQUEST_CANCELLED: code = H3ErrorCode.H3_INTERNAL_ERROR - self.quic.reset_stream(stream_id=event.stream_id, error_code=code) + yield ResetQuicStream( + connection=self.conn, + stream_id=event.stream_id, + error_code=code, + ) __all__ = [ diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 0bc2c202eb..9bc442b574 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -120,10 +120,6 @@ class QuicStreamDataReceived(QuicStreamEvent): end_stream: bool """Whether the STREAM frame had the FIN bit set.""" - def __repr__(self): - target = type(self.connection).__name__.lower() - return f"{self.__class__.__name__}({target}, {self.stream_id}, {self.data}, {self.end_stream})" - @dataclass class QuicStreamReset(QuicStreamEvent): @@ -232,6 +228,16 @@ def error_code_to_str(error_code: int) -> str: return f"unknown error (0x{error_code:x})" +def get_connection_error(conn: connection.Connection) -> quic_events.ConnectionTerminated | None: + """Returns the QUIC close event that is associated with the given connection.""" + + close_event = getattr(conn, "quic_error", None) + if close_event is None: + return None + assert isinstance(close_event, quic_events.ConnectionTerminated) + return close_event + + def is_success_error_code(error_code: int) -> bool: """Returns whether the given error code actually indicates no error.""" @@ -250,6 +256,12 @@ def get_stream_connection_state(stream_id: int, is_client: bool) -> connection.C return state +def set_connection_error(conn: connection.Connection, close_event: quic_events.ConnectionTerminated) -> None: + """Stores the given close event for the given connection.""" + + setattr(conn, "quic_error", close_event) + + @dataclass class QuicClientHello(Exception): """Helper error only used in `quic_parse_client_hello`.""" @@ -357,8 +369,8 @@ class RawQuicLayer(layer.Layer): """Indicates whether traffic should be routed as-is.""" datagram_layer: layer.Layer """ - The layer handling datagrams over QUIC. It's like a child_layer, but with a forked context. - Instead of having a datagram equivalent for all stream classes, we use traditional `SendData` and `DataReceived`. + The layer that is handling datagrams over QUIC. It's like a child_layer, but with a forked context. + Instead of having a datagram-equivalent for all `QuicStream*` classes, we use `SendData` and `DataReceived` instead. There is also no need for another `NextLayer` marker, as a missing `QuicStreamLayer` implies UDP, and the connection state is the same as the one of the underlying QUIC connection. """ @@ -369,9 +381,11 @@ class RawQuicLayer(layer.Layer): connections: dict[connection.Connection, layer.Layer] """Maps connections to layers.""" command_sources: dict[commands.Command, layer.Layer] + """Keeps track of blocking commands and wakeup requests.""" def __init__(self, context: context.Context, ignore: bool = False) -> None: super().__init__(context) + self.ignore = ignore self.datagram_layer = ( UDPLayer(self.context.fork(), ignore=True) if ignore else @@ -384,18 +398,17 @@ def __init__(self, context: context.Context, ignore: bool = False) -> None: context.server: self.datagram_layer, } self.command_sources = {} - self.ignore = ignore def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - # we treat the datagram-layer as child layer, so forward Start + # we treat the datagram layer as child layer, so forward Start if isinstance(event, events.Start): yield from self.event_to_child(self.datagram_layer, event) - # properly forward stored completion events + # properly forward completion events based on their command elif isinstance(event, events.CommandCompleted): yield from self.event_to_child(self.command_sources.pop(event.command), event) - # route injection messages based on the flow's connections (prefer client, fallback to server) + # route injected messages based on their connections (prefer client, fallback to server) elif isinstance(event, events.MessageInjected): if event.flow.client_conn in self.connections: yield from self.event_to_child(self.connections[event.flow.client_conn], event) @@ -444,7 +457,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: self.connections[stream_layer.server] = stream_layer yield from self.event_to_child(stream_layer, events.Start()) - # forward the data and close events + # forward data and close events conn = stream_layer.client if from_client else stream_layer.server if isinstance(event, QuicStreamDataReceived): yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) @@ -507,9 +520,9 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer ) ): # get the target connection and stream ID - from_client = command.connection is child_layer.client - conn = self.context.client if from_client else self.context.server - stream_id = child_layer.client_stream_id if from_client else child_layer.server_stream_id + to_client = command.connection is child_layer.client + conn = self.context.client if to_client else self.context.server + stream_id = child_layer.client_stream_id if to_client else child_layer.server_stream_id # write data and check CloseConnection wasn't called before if isinstance(command, commands.SendData): @@ -529,7 +542,7 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): - assert not from_client + assert not to_client assert stream_id is None child_layer.server.timestamp_start = time.time() stream_id = cast(int, (yield OpenQuicStream(conn))) @@ -756,6 +769,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: # handle post-handshake events while event := self.quic.next_event(): if isinstance(event, quic_events.ConnectionTerminated): + set_connection_error(self.conn, event) if self.debug: reason = event.reason_phrase or error_code_to_str(event.error_code) yield commands.Log( @@ -766,7 +780,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: elif isinstance(event, quic_events.DatagramFrameReceived): yield from self.event_to_child(events.DataReceived(self.conn, event.data)) elif isinstance(event, quic_events.StreamDataReceived): - yield from self.event_to_child(QuicStreamDataReceived(self.conn, data=event.data, stream_id=event.stream_id, end_stream=event.end_stream)) + yield from self.event_to_child(QuicStreamDataReceived(self.conn, event.stream_id, event.data, event.end_stream)) elif isinstance(event, quic_events.StreamReset): yield from self.event_to_child(QuicStreamReset(self.conn, event.stream_id, event.error_code)) elif isinstance(event, ( @@ -795,7 +809,11 @@ def send_data(self, data: bytes) -> layer.CommandGenerator[None]: def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: # properly close the QUIC connection if self.quic is not None: - self.quic.close() + close_event = get_connection_error(self.conn) + if close_event is None: + self.quic.close() + else: + self.quic.close(close_event.error_code, close_event.frame_type, close_event.reason_phrase) yield from self.tls_interact() yield from super().send_close(half_close) From ba5f8f06d9854fea16e393c2731516b61b071bf0 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Wed, 21 Sep 2022 18:27:27 +0200 Subject: [PATCH 063/695] [quic] more unification with H2, QUIC abstraction --- mitmproxy/proxy/layers/http/_http3.py | 385 +++++++++++++------------- 1 file changed, 191 insertions(+), 194 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index e9665f5289..634969c3cc 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,7 +1,7 @@ from abc import abstractmethod from queue import Queue import time -from typing import Dict, Optional, Union +from typing import Dict, Iterable, Optional, Union from aioquic.h3.connection import ( H3Connection, @@ -19,8 +19,8 @@ from mitmproxy.net.http import status_codes from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( - OpenQuicStream, QuicStreamDataReceived, + QuicStreamEvent, QuicStreamReset, ResetQuicStream, SendQuicStreamData, @@ -62,10 +62,11 @@ class MockQuic: Instead, we mock our QUIC connection object here and then take out the wire data to be sent. """ - def __init__(self, conn: connection.Connection) -> None: + def __init__(self, conn: connection.Connection, is_client: bool) -> None: self.conn = conn self.pending_commands: Queue[commands.Command] = Queue() - self.available_stream_ids: Queue[int] = Queue() + self._next_stream_id: list[int, int, int, int] = [0, 1, 2, 3] + self._is_client = is_client def close( self, @@ -80,123 +81,196 @@ def close( )) self.pending_commands.put(commands.CloseConnection(self.conn)) - def get_next_available_stream_id(self, is_unidirectional=False) -> int: - stream_id = self.available_stream_ids.get() - assert is_unidirectional == stream_is_unidirectional(stream_id) + def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: + index = (int(is_unidirectional) << 1) | int(not self._is_client) + stream_id = self._next_stream_id[index] + self._next_stream_id[index] = stream_id + 4 return stream_id + def reset_stream(self, stream_id: int, error_code: int) -> None: + self.pending_commands.put(ResetQuicStream(self.conn, stream_id, error_code)) + + def send_datagram_frame(self, data: bytes) -> None: + self.pending_commands.put(commands.SendData(self.conn, data)) + def send_stream_data( self, stream_id: int, data: bytes, end_stream: bool = False ) -> None: self.pending_commands.put(SendQuicStreamData(self.conn, stream_id, data, end_stream)) + def stream_can_receive(self, stream_id: int) -> bool: + return ( + stream_is_client_initiated(stream_id) != self._is_client + or not stream_is_unidirectional(stream_id) + ) + class LayeredH3Connection(H3Connection): - def __init__(self, connection: connection.Connection) -> None: - self._quic = MockQuic(connection) - super().__init__(quic=self._quic, enable_webtransport=False) + def __init__(self, quic: MockQuic, enable_webtransport: bool = False) -> None: + self._quic = quic + super().__init__(quic, enable_webtransport) + + def _after_send(self, stream_id: int, end_stream: bool) -> None: + # if the stream ended, `QuicConnection` has an assert that no further data is being sent + # to catch this more early on, we set the header state on the `H3Stream` + if end_stream: + self._stream[stream_id].headers_send_state = H3HeadersState.AFTER_TRAILERS + + def end_stream(self, stream_id: int) -> None: + """Ends the given stream.""" + + self.send_data(stream_id, data=b"", end_stream=True) + + def get_next_available_stream_id(self, is_unidirectional: bool = False): + """Reserves and returns the next available stream ID.""" + + return self._quic.get_next_available_stream_id(is_unidirectional) + + def has_ended(self, stream_id: int) -> bool: + """Indicates whether the given stream has been closed by the peer.""" + + if not self._quic.stream_can_receive(stream_id): + return True + try: + return self._stream[stream_id].ended + except KeyError: + return False + + def has_sent_headers(self, stream_id: int) -> bool: + """Indicates whether headers have been sent over the given stream.""" + + try: + return self._stream[stream_id].headers_send_state != H3HeadersState.INITIAL + except KeyError: + return False + + @property + def open_stream_ids(self) -> Iterable[int]: + """Return all streams, that have not been closed by the peer yet.""" + + for stream in self._stream.values(): + if self._quic.stream_can_receive(stream.stream_id) and not stream.ended: + yield stream.stream_id + + def reset_stream(self, stream_id: int, error_code: int) -> None: + """Resets a stream that hasn't been ended locally yet.""" + + # we don't allow reset after FIN + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state == H3HeadersState.AFTER_TRAILERS: + raise H3FrameUnexpected("reset not allowed in this state") + + # set the header state and queue a reset event + stream.headers_send_state = H3HeadersState.AFTER_TRAILERS + self._quic.reset_stream(stream_id, error_code) + + def send_data(self, stream_id: int, data: bytes, end_stream: bool = False) -> None: + """Sends data over the given stream.""" - def start(self) -> layer.CommandGenerator[None]: - # we need three unidirectional streams for `_init_connection` - for _ in range(1, 3): - self._quic.available_stream_ids.put((yield OpenQuicStream(self._quic.conn, is_unidirectional=False))) + super().send_data(stream_id, data, end_stream) + self._after_send(stream_id, end_stream) - def create_webtransport_stream(self, session_id: int, is_unidirectional: bool = False) -> int: - raise NotImplementedError() # pragma: no cover + def send_headers(self, stream_id: int, headers: H3Headers, end_stream: bool = False) -> None: + """Sends headers over the given stream.""" - def send_datagram(self, flow_id: int, data: bytes) -> None: - raise NotImplementedError() # pragma: no cover + # ensure we haven't sent something before + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state != H3HeadersState.INITIAL: + raise H3FrameUnexpected("initial HEADERS frame is not allowed in this state") + super().send_headers(stream_id, headers, end_stream) + self._after_send(stream_id, end_stream) - def send_push_promise(self, stream_id: int, headers: H3Headers) -> int: - raise NotImplementedError() # pragma: no cover + def send_trailers(self, stream_id: int, trailers: H3Headers) -> None: + """Sends trailers over the given stream.""" + + # ensure we got some headers first + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state != H3HeadersState.AFTER_HEADERS: + raise H3FrameUnexpected("trailing HEADERS frame is not allowed in this state") + super().send_headers(stream_id, trailers, end_stream=True) + self._after_send(stream_id, end_stream=True) def transmit(self) -> layer.CommandGenerator[None]: + """Yields all pending commands for the upper QUIC layer.""" + while self._quic.pending_commands: yield self._quic.pending_commands.get() class Http3Connection(HttpConnection): - h3_conn: Optional[LayeredH3Connection] = None + h3_conn: LayeredH3Connection ReceiveData: type[Union[RequestData, ResponseData]] ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] + def __init__(self, context: context.Context, conn: connection.Connection): + super().__init__(context, conn) + self.h3_conn = LayeredH3Connection(MockQuic(self.conn, self.conn is self.context.client)) + @abstractmethod def parse_headers( self, event: h3_events.HeadersReceived ) -> Union[RequestHeaders, ResponseHeaders]: pass # pragma: no cover - def postprocess_outgoing_event(self, event: HttpEvent) -> HttpEvent: - return event - - def preprocess_incoming_event(self, event: HttpEvent) -> layer.CommandGenerator[HttpEvent]: - yield from () - return event - - @abstractmethod - def send_protocol_error( - self, event: Union[RequestProtocolError, ResponseProtocolError] - ) -> None: - pass # pragma: no cover - - @expect(HttpEvent, QuicStreamDataReceived, QuicStreamReset, events.ConnectionClosed) - def state_done(self, _) -> layer.CommandGenerator[None]: - yield from () - - @expect(HttpEvent, QuicStreamDataReceived, QuicStreamReset, events.ConnectionClosed) - def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.h3_conn is not None + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.Start): + pass # send mitmproxy HTTP events over the H3 connection if isinstance(event, HttpEvent): - event = yield from self.preprocess_incoming_event(event) try: - if isinstance(event, (RequestData, ResponseData)): - self.h3_conn.send_data( - stream_id=event.stream_id, data=event.data, end_stream=False - ) + self.h3_conn.send_data(event.stream_id, event.data) elif isinstance(event, (RequestHeaders, ResponseHeaders)): - self.h3_conn.send_headers( - stream_id=event.stream_id, - headers=( - yield from ( - format_h2_request_headers(self.context, event) - if isinstance(event, RequestHeaders) - else format_h2_response_headers(self.context, event) - ) - ), - end_stream=event.end_stream, + headers = yield from ( + format_h2_request_headers(self.context, event) + if isinstance(event, RequestHeaders) + else format_h2_response_headers(self.context, event) ) - if event.end_stream: - # this will prevent any further headers or data from being sent - self.h3_conn._stream[ - event.stream_id - ].headers_send_state = H3HeadersState.AFTER_TRAILERS + self.h3_conn.send_headers(event.stream_id, headers, end_stream=event.end_stream) elif isinstance(event, (RequestTrailers, ResponseTrailers)): - self.h3_conn.send_headers( - stream_id=event.stream_id, - headers=[*event.trailers.fields], - end_stream=True, - ) + trailers = [*event.trailers.fields] + self.h3_conn.send_trailers(event.stream_id, trailers) elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): - self.h3_conn.send_data( - stream_id=event.stream_id, data=b"", end_stream=True - ) + self.h3_conn.end_stream(event.stream_id) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): - self.send_protocol_error(event) + if not self.h3_conn.has_ended(event.stream_id): + code = { + status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED, + }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR) + send_error_message = ( + isinstance(event, ResponseProtocolError) + and not self.h3_conn.has_sent_headers(event.stream_id) + and event.code != status_codes.NO_RESPONSE + ) + if send_error_message: + self.h3_conn.send_headers( + event.stream_id, + [ + (b":status", b"%d" % event.code), + (b"server", version.MITMPROXY.encode()), + (b"content-type", b"text/html"), + ], + ) + self.h3_conn.send_data( + event.stream_id, + format_error(event.code, event.message), + end_stream=True, + ) + else: + self.h3_conn.reset_stream(event.stream_id, code) else: raise AssertionError(f"Unexpected event: {event!r}") - except H3FrameUnexpected: + except H3FrameUnexpected as e: # Http2Connection also ignores HttpEvents that violate the current stream state - pass + yield from commands.Log(f"Received {event!r} unexpectedly: {e}") else: - # transmit buffered data and re-arm timer + # transmit buffered data yield from self.h3_conn.transmit() elif isinstance(event, QuicStreamReset): @@ -207,7 +281,7 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: ): # mark the receiving part of the stream as ended # (H3Connection alas doesn't handle StreamReset) - self.h3_conn._stream[event.stream_id] = True + self.h3_conn._stream[event.stream_id].ended = True # report the protocol error (doing the same error code mingling as H2) code = ( @@ -216,12 +290,10 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: else self.ReceiveProtocolError.code ) yield ReceiveHttp( - self.postprocess_outgoing_event( - self.ReceiveProtocolError( - stream_id=event.stream_id, - message=f"stream reset by client ({error_code_to_str(event.error_code)})", - code=code, - ) + self.ReceiveProtocolError( + stream_id=event.stream_id, + message=f"stream reset by client ({error_code_to_str(event.error_code)})", + code=code, ) ) @@ -237,19 +309,9 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: isinstance(h3_event, h3_events.DataReceived) and stream_is_client_initiated(h3_event.stream_id) ): - yield ReceiveHttp( - self.postprocess_outgoing_event( - self.ReceiveData( - stream_id=h3_event.stream_id, data=h3_event.data - ) - ) - ) + yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) if h3_event.stream_ended: - yield ReceiveHttp( - self.postprocess_outgoing_event( - self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) - ) - ) + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) # report headers and trailers elif ( @@ -257,14 +319,8 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: and stream_is_client_initiated(h3_event.stream_id) ): if self.h3_conn._stream[h3_event.stream_id].headers_recv_state is H3HeadersState.AFTER_TRAILERS: - yield ReceiveHttp( - self.postprocess_outgoing_event( - self.ReceiveTrailers( - stream_id=h3_event.stream_id, - trailers=http.Headers(h3_event.headers), - ) - ) - ) + trailers = http.Headers(h3_event.headers) + yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, trailers)) else: try: receive_event = self.parse_headers(h3_event) @@ -277,17 +333,11 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: )) yield commands.CloseConnection(self.conn) else: - yield ReceiveHttp( - self.postprocess_outgoing_event(receive_event) - ) + yield ReceiveHttp(receive_event) # always report an EndOfMessage if the stream has ended if h3_event.stream_ended: - yield ReceiveHttp( - self.postprocess_outgoing_event( - self.ReceiveEndOfMessage(stream_id=h3_event.stream_id) - ) - ) + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) # we don't support push, web transport, etc. else: @@ -302,40 +352,22 @@ def state_ready(self, event: events.Event) -> layer.CommandGenerator[None]: # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): - for stream in self.h3_conn._stream.values(): - if stream_is_client_initiated(stream.stream_id) and not stream.ended: - close_event = get_connection_error(self.conn) - yield ReceiveHttp( - self.postprocess_outgoing_event( - self.ReceiveProtocolError( - stream_id=stream.stream_id, - message=( - "Connection closed." - if close_event is None else - close_event.reason_phrase - ), - code=( - QuicErrorCode.APPLICATION_ERROR - if close_event is None else - close_event.error_code - ), - ) - ) - ) - self._handle_event = self.state_done + close_event = get_connection_error(self.conn) + msg = ( + "peer closed connection" + if close_event is None else + close_event.reason_phrase or error_code_to_str(close_event.error_code) + ) + for stream_id in self.h3_conn.open_stream_ids: + yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) + self._handle_event = self.done else: raise AssertionError(f"Unexpected event: {event!r}") - @expect(events.Start) - def state_start(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.h3_conn is None - - self.h3_conn = LayeredH3Connection(self.conn) - self._handle_event = self.state_ready - yield from self.h3_conn.start() - - _handle_event = state_start + @expect(QuicStreamEvent, HttpEvent, events.ConnectionClosed) + def done(self, _) -> layer.CommandGenerator[None]: + yield from () class Http3Server(Http3Connection): @@ -372,29 +404,7 @@ def parse_headers(self, event: h3_events.HeadersReceived) -> Union[RequestHeader timestamp_start=time.time(), timestamp_end=None, ) - return RequestHeaders(stream_id=event.stream_id, request=request, end_stream=event.stream_ended) - - def send_protocol_error(self, event: Union[RequestProtocolError, ResponseProtocolError]) -> None: - assert self.h3_conn is not None - assert isinstance(event, ResponseProtocolError) - - # same as HTTP/2 - code = event.code - if code != status_codes.CLIENT_CLOSED_REQUEST: - code = status_codes.INTERNAL_SERVER_ERROR - self.h3_conn.send_headers( - stream_id=event.stream_id, - headers=[ - (b":status", b"%d" % code), - (b"server", version.MITMPROXY.encode()), - (b"content-type", b"text/html"), - ], - ) - self.h3_conn.send_data( - stream_id=event.stream_id, - data=format_error(code, event.message), - end_stream=True, - ) + return RequestHeaders(event.stream_id, request, end_stream=event.stream_ended) class Http3Client(Http3Connection): @@ -405,8 +415,25 @@ class Http3Client(Http3Connection): def __init__(self, context: context.Context): super().__init__(context, context.server) - self._event_to_quic: Dict[int, int] = {} - self._quic_to_event: Dict[int, int] = {} + self.our_stream_id: Dict[int, int] = {} + self.their_stream_id: Dict[int, int] = {} + + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + # QUIC and HTTP/3 would actually allow for direct stream ID mapping, but since we want + # to support H2<->H3, we need to translate IDs. + # NOTE: We always create bidirectional streams, as we can't safely infer unidirectionality. + if isinstance(event, HttpEvent): + ours = self.our_stream_id.get(event.stream_id, None) + if ours is None: + ours = self.h3_conn.get_next_available_stream_id() + self.our_stream_id[event.stream_id] = ours + self.their_stream_id[ours] = event.stream_id + event.stream_id = ours + + for cmd in super()._handle_event(event): + if isinstance(cmd, ReceiveHttp): + cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id] + yield cmd def parse_headers(self, event: h3_events.HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 @@ -421,37 +448,7 @@ def parse_headers(self, event: h3_events.HeadersReceived) -> Union[RequestHeader timestamp_start=time.time(), timestamp_end=None, ) - return ResponseHeaders(stream_id=event.stream_id, response=response, end_stream=event.stream_ended) - - def postprocess_outgoing_event(self, event: HttpEvent) -> HttpEvent: - event.stream_id = self._quic_to_event[event.stream_id] - return event - - def preprocess_incoming_event(self, event: HttpEvent) -> layer.CommandGenerator[HttpEvent]: - if event.stream_id in self._event_to_quic: - event.stream_id = self._event_to_quic[event.stream_id] - else: - # QUIC and HTTP/3 would actually allow for direct stream ID mapping, but since we want - # to support H2<->H3, we need to translate IDs. - # NOTE: We always create bidirectional streams, as we can't safely infer unidirectionality. - stream_id = yield OpenQuicStream(self.conn) - self._event_to_quic[event.stream_id] = stream_id - self._quic_to_event[stream_id] = event.stream_id - event.stream_id = stream_id - return event - - def send_protocol_error(self, event: Union[RequestProtocolError, ResponseProtocolError]) -> None: - assert isinstance(event, RequestProtocolError) - - # same as HTTP/2 - code = event.code - if code != H3ErrorCode.H3_REQUEST_CANCELLED: - code = H3ErrorCode.H3_INTERNAL_ERROR - yield ResetQuicStream( - connection=self.conn, - stream_id=event.stream_id, - error_code=code, - ) + return ResponseHeaders(event.stream_id, response, event.stream_ended) __all__ = [ From cada1b7bfe58322b299979f69ae314a13c89a814 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 22 Sep 2022 00:42:13 +0200 Subject: [PATCH 064/695] [quic] split H3 code, complete layer abstraction --- mitmproxy/proxy/layers/http/_http3.py | 291 +++++------------------- mitmproxy/proxy/layers/http/_http_h3.py | 272 ++++++++++++++++++++++ 2 files changed, 324 insertions(+), 239 deletions(-) create mode 100644 mitmproxy/proxy/layers/http/_http_h3.py diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 634969c3cc..d30cf16312 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,32 +1,20 @@ from abc import abstractmethod -from queue import Queue import time -from typing import Dict, Iterable, Optional, Union +from typing import Dict, Union from aioquic.h3.connection import ( - H3Connection, ErrorCode as H3ErrorCode, FrameUnexpected as H3FrameUnexpected, - Headers as H3Headers, - HeadersState as H3HeadersState, ) -from aioquic.h3 import events as h3_events -from aioquic.quic import events as quic_events -from aioquic.quic.connection import stream_is_client_initiated, stream_is_unidirectional -from aioquic.quic.packet import QuicErrorCode +from aioquic.h3.events import DataReceived, HeadersReceived from mitmproxy import connection, http, version from mitmproxy.net.http import status_codes from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( - QuicStreamDataReceived, QuicStreamEvent, - QuicStreamReset, - ResetQuicStream, - SendQuicStreamData, error_code_to_str, get_connection_error, - set_connection_error, ) from mitmproxy.proxy.utils import expect @@ -35,12 +23,12 @@ RequestEndOfMessage, RequestHeaders, RequestProtocolError, + RequestTrailers, ResponseData, ResponseEndOfMessage, ResponseHeaders, - RequestTrailers, - ResponseTrailers, ResponseProtocolError, + ResponseTrailers, ) from ._base import ( HttpConnection, @@ -54,147 +42,7 @@ parse_h2_request_headers, parse_h2_response_headers, ) - - -class MockQuic: - """ - aioquic intermingles QUIC and HTTP/3. This is something we don't want to do because that makes testing much harder. - Instead, we mock our QUIC connection object here and then take out the wire data to be sent. - """ - - def __init__(self, conn: connection.Connection, is_client: bool) -> None: - self.conn = conn - self.pending_commands: Queue[commands.Command] = Queue() - self._next_stream_id: list[int, int, int, int] = [0, 1, 2, 3] - self._is_client = is_client - - def close( - self, - error_code: int = QuicErrorCode.NO_ERROR, - frame_type: Optional[int] = None, - reason_phrase: str = "", - ) -> None: - set_connection_error(self.conn, quic_events.ConnectionTerminated( - error_code=error_code, - frame_type=frame_type, - reason_phrase=reason_phrase, - )) - self.pending_commands.put(commands.CloseConnection(self.conn)) - - def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: - index = (int(is_unidirectional) << 1) | int(not self._is_client) - stream_id = self._next_stream_id[index] - self._next_stream_id[index] = stream_id + 4 - return stream_id - - def reset_stream(self, stream_id: int, error_code: int) -> None: - self.pending_commands.put(ResetQuicStream(self.conn, stream_id, error_code)) - - def send_datagram_frame(self, data: bytes) -> None: - self.pending_commands.put(commands.SendData(self.conn, data)) - - def send_stream_data( - self, stream_id: int, data: bytes, end_stream: bool = False - ) -> None: - self.pending_commands.put(SendQuicStreamData(self.conn, stream_id, data, end_stream)) - - def stream_can_receive(self, stream_id: int) -> bool: - return ( - stream_is_client_initiated(stream_id) != self._is_client - or not stream_is_unidirectional(stream_id) - ) - - -class LayeredH3Connection(H3Connection): - def __init__(self, quic: MockQuic, enable_webtransport: bool = False) -> None: - self._quic = quic - super().__init__(quic, enable_webtransport) - - def _after_send(self, stream_id: int, end_stream: bool) -> None: - # if the stream ended, `QuicConnection` has an assert that no further data is being sent - # to catch this more early on, we set the header state on the `H3Stream` - if end_stream: - self._stream[stream_id].headers_send_state = H3HeadersState.AFTER_TRAILERS - - def end_stream(self, stream_id: int) -> None: - """Ends the given stream.""" - - self.send_data(stream_id, data=b"", end_stream=True) - - def get_next_available_stream_id(self, is_unidirectional: bool = False): - """Reserves and returns the next available stream ID.""" - - return self._quic.get_next_available_stream_id(is_unidirectional) - - def has_ended(self, stream_id: int) -> bool: - """Indicates whether the given stream has been closed by the peer.""" - - if not self._quic.stream_can_receive(stream_id): - return True - try: - return self._stream[stream_id].ended - except KeyError: - return False - - def has_sent_headers(self, stream_id: int) -> bool: - """Indicates whether headers have been sent over the given stream.""" - - try: - return self._stream[stream_id].headers_send_state != H3HeadersState.INITIAL - except KeyError: - return False - - @property - def open_stream_ids(self) -> Iterable[int]: - """Return all streams, that have not been closed by the peer yet.""" - - for stream in self._stream.values(): - if self._quic.stream_can_receive(stream.stream_id) and not stream.ended: - yield stream.stream_id - - def reset_stream(self, stream_id: int, error_code: int) -> None: - """Resets a stream that hasn't been ended locally yet.""" - - # we don't allow reset after FIN - stream = self._get_or_create_stream(stream_id) - if stream.headers_send_state == H3HeadersState.AFTER_TRAILERS: - raise H3FrameUnexpected("reset not allowed in this state") - - # set the header state and queue a reset event - stream.headers_send_state = H3HeadersState.AFTER_TRAILERS - self._quic.reset_stream(stream_id, error_code) - - def send_data(self, stream_id: int, data: bytes, end_stream: bool = False) -> None: - """Sends data over the given stream.""" - - super().send_data(stream_id, data, end_stream) - self._after_send(stream_id, end_stream) - - def send_headers(self, stream_id: int, headers: H3Headers, end_stream: bool = False) -> None: - """Sends headers over the given stream.""" - - # ensure we haven't sent something before - stream = self._get_or_create_stream(stream_id) - if stream.headers_send_state != H3HeadersState.INITIAL: - raise H3FrameUnexpected("initial HEADERS frame is not allowed in this state") - super().send_headers(stream_id, headers, end_stream) - self._after_send(stream_id, end_stream) - - def send_trailers(self, stream_id: int, trailers: H3Headers) -> None: - """Sends trailers over the given stream.""" - - # ensure we got some headers first - stream = self._get_or_create_stream(stream_id) - if stream.headers_send_state != H3HeadersState.AFTER_HEADERS: - raise H3FrameUnexpected("trailing HEADERS frame is not allowed in this state") - super().send_headers(stream_id, trailers, end_stream=True) - self._after_send(stream_id, end_stream=True) - - def transmit(self) -> layer.CommandGenerator[None]: - """Yields all pending commands for the upper QUIC layer.""" - - while self._quic.pending_commands: - yield self._quic.pending_commands.get() +from._http_h3 import LayeredH3Connection, StreamReset, TrailersReceived class Http3Connection(HttpConnection): @@ -207,13 +55,7 @@ class Http3Connection(HttpConnection): def __init__(self, context: context.Context, conn: connection.Connection): super().__init__(context, conn) - self.h3_conn = LayeredH3Connection(MockQuic(self.conn, self.conn is self.context.client)) - - @abstractmethod - def parse_headers( - self, event: h3_events.HeadersReceived - ) -> Union[RequestHeaders, ResponseHeaders]: - pass # pragma: no cover + self.h3_conn = LayeredH3Connection(self.conn, is_client=self.conn is self.context.client) def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): @@ -237,7 +79,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): self.h3_conn.end_stream(event.stream_id) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): - if not self.h3_conn.has_ended(event.stream_id): + if self.h3_conn.is_stream_open(event.stream_id): code = { status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED, }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR) @@ -273,82 +115,44 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # transmit buffered data yield from self.h3_conn.transmit() - elif isinstance(event, QuicStreamReset): - if ( - stream_is_client_initiated(event.stream_id) - and event.stream_id in self.h3_conn._stream - and not self.h3_conn._stream[event.stream_id].ended - ): - # mark the receiving part of the stream as ended - # (H3Connection alas doesn't handle StreamReset) - self.h3_conn._stream[event.stream_id].ended = True - - # report the protocol error (doing the same error code mingling as H2) - code = ( - status_codes.CLIENT_CLOSED_REQUEST - if event.error_code == H3ErrorCode.H3_REQUEST_CANCELLED - else self.ReceiveProtocolError.code - ) - yield ReceiveHttp( - self.ReceiveProtocolError( - stream_id=event.stream_id, - message=f"stream reset by client ({error_code_to_str(event.error_code)})", - code=code, + # forward stream messages from the QUIC layer to the H3 connection + elif isinstance(event, QuicStreamEvent): + for h3_event in self.h3_conn.handle_event(event): + if isinstance(h3_event, StreamReset) and h3_event.push_id is None: + err_str = error_code_to_str(h3_event.error_code) + err_code = { + H3ErrorCode.H3_REQUEST_CANCELLED: status_codes.CLIENT_CLOSED_REQUEST, + }.get(h3_event.error_code, self.ReceiveProtocolError.code) + yield ReceiveHttp( + self.ReceiveProtocolError( + h3_event.stream_id, + f"stream reset by client ({err_str})", + code=err_code, + ) ) - ) - - elif isinstance(event, QuicStreamDataReceived): - yield commands.Log(f"recvd data: {event=}") - # and convert back... - e = quic_events.StreamDataReceived(data=event.data, end_stream=event.end_stream, stream_id=event.stream_id) - for h3_event in self.h3_conn.handle_event(e): - yield commands.Log(f"{h3_event=}") - - # report received data - if ( - isinstance(h3_event, h3_events.DataReceived) - and stream_is_client_initiated(h3_event.stream_id) - ): + elif isinstance(h3_event, DataReceived) and h3_event.push_id is None: yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) if h3_event.stream_ended: yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - - # report headers and trailers - elif ( - isinstance(h3_event, h3_events.HeadersReceived) - and stream_is_client_initiated(h3_event.stream_id) - ): - if self.h3_conn._stream[h3_event.stream_id].headers_recv_state is H3HeadersState.AFTER_TRAILERS: - trailers = http.Headers(h3_event.headers) - yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, trailers)) + elif isinstance(h3_event, HeadersReceived) and h3_event.push_id is None: + try: + receive_event = self.parse_headers(h3_event) + except ValueError as e: + self.h3_conn.close_connection( + error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, + reason_phrase=f"Invalid HTTP/3 request headers: {e}", + ) + yield from self.h3_conn.transmit() else: - try: - receive_event = self.parse_headers(h3_event) - except ValueError as e: - # this will result in a ConnectionClosed event - set_connection_error(self.conn, quic_events.ConnectionTerminated( - error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, - frame_type=None, - reason_phrase=f"Invalid HTTP/3 request headers: {e}", - )) - yield commands.CloseConnection(self.conn) - else: - yield ReceiveHttp(receive_event) - - # always report an EndOfMessage if the stream has ended - if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - - # we don't support push, web transport, etc. + yield ReceiveHttp(receive_event) + if h3_event.stream_ended: + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + elif isinstance(h3_event, TrailersReceived) and h3_event.push_id is None: + trailers = http.Headers(h3_event.trailers) + yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, trailers)) else: - yield commands.Log( - f"Ignored unsupported H3 event: {h3_event!r}", - level=( - "info" - if stream_is_client_initiated(h3_event.stream_id) else - "debug" - ) - ) + # we don't support push, web transport, etc. + yield commands.Log(f"Ignored unsupported H3 event: {h3_event!r}") # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): @@ -369,6 +173,12 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: def done(self, _) -> layer.CommandGenerator[None]: yield from () + @abstractmethod + def parse_headers( + self, event: HeadersReceived + ) -> Union[RequestHeaders, ResponseHeaders]: + pass # pragma: no cover + class Http3Server(Http3Connection): ReceiveData = RequestData @@ -379,7 +189,7 @@ class Http3Server(Http3Connection): def __init__(self, context: context.Context): super().__init__(context, context.client) - def parse_headers(self, event: h3_events.HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers(self, event: HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 ( host, @@ -413,10 +223,13 @@ class Http3Client(Http3Connection): ReceiveProtocolError = ResponseProtocolError ReceiveTrailers = ResponseTrailers + our_stream_id: Dict[int, int] + their_stream_id: Dict[int, int] + def __init__(self, context: context.Context): super().__init__(context, context.server) - self.our_stream_id: Dict[int, int] = {} - self.their_stream_id: Dict[int, int] = {} + self.our_stream_id = {} + self.their_stream_id = {} def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # QUIC and HTTP/3 would actually allow for direct stream ID mapping, but since we want @@ -435,7 +248,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id] yield cmd - def parse_headers(self, event: h3_events.HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers(self, event: HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 status_code, headers = parse_h2_response_headers(event.headers) response = http.Response( diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py new file mode 100644 index 0000000000..dc05a0e8b0 --- /dev/null +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -0,0 +1,272 @@ +from dataclasses import dataclass +from queue import Queue +from typing import Iterable, Optional + +from aioquic.h3.connection import ( + H3Connection, + FrameUnexpected as H3FrameUnexpected, + H3Event, + H3Stream, + Headers, + HeadersState, +) +from aioquic.h3.events import HeadersReceived +from aioquic.quic import events as quic_events +from aioquic.quic.connection import stream_is_client_initiated, stream_is_unidirectional +from aioquic.quic.events import StreamDataReceived +from aioquic.quic.packet import QuicErrorCode + +from mitmproxy import connection +from mitmproxy.proxy import commands, layer +from mitmproxy.proxy.layers.quic import ( + QuicStreamDataReceived, + QuicStreamEvent, + QuicStreamReset, + ResetQuicStream, + SendQuicStreamData, + set_connection_error, +) + + +@dataclass +class TrailersReceived(H3Event): + """ + The TrailersReceived event is fired whenever trailers are received. + """ + + trailers: Headers + "The trailers." + + stream_id: int + "The ID of the stream the trailers were received for." + + push_id: Optional[int] = None + "The Push ID or `None` if this is not a push." + + +@dataclass +class StreamReset(H3Event): + """ + The StreamReset event is fired whenever a stream is reset by the peer. + """ + + stream_id: int + "The ID of the stream that was reset." + + error_code: int + """The error code indicating why the stream was reset.""" + + push_id: Optional[int] = None + "The Push ID or `None` if this is not a push." + + +class MockQuic: + """ + aioquic intermingles QUIC and HTTP/3. This is something we don't want to do because that makes testing much harder. + Instead, we mock our QUIC connection object here and then take out the wire data to be sent. + """ + + def __init__(self, conn: connection.Connection, is_client: bool) -> None: + self.conn = conn + self.configuration = type("Object", (), {"_is_client": is_client}) + self.pending_commands: Queue[commands.Command] = Queue() + self._next_stream_id: list[int, int, int, int] = [0, 1, 2, 3] + self._is_client = is_client + + def close( + self, + error_code: int = QuicErrorCode.NO_ERROR, + frame_type: Optional[int] = None, + reason_phrase: str = "", + ) -> None: + # we'll get closed if a protocol error occurs in `H3Connection.handle_event` + # we note the error on the connection and yield a CloseConnection + # this will then call `QuicConnection.close` with the proper values + # once the `Http3Connection` receives `ConnectionClosed`, it will send out `*ProtocolError` + set_connection_error(self.conn, quic_events.ConnectionTerminated( + error_code=error_code, + frame_type=frame_type, + reason_phrase=reason_phrase, + )) + self.pending_commands.put(commands.CloseConnection(self.conn)) + + def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: + # since we always reserve the ID, we have to "find" the next ID like `QuicConnection` does + index = (int(is_unidirectional) << 1) | int(not self._is_client) + stream_id = self._next_stream_id[index] + self._next_stream_id[index] = stream_id + 4 + return stream_id + + def reset_stream(self, stream_id: int, error_code: int) -> None: + self.pending_commands.put(ResetQuicStream(self.conn, stream_id, error_code)) + + def send_stream_data(self, stream_id: int, data: bytes, end_stream: bool = False) -> None: + self.pending_commands.put(SendQuicStreamData(self.conn, stream_id, data, end_stream)) + + +class LayeredH3Connection(H3Connection): + """ + Creates a H3 connection using a fake QUIC connection, which allows layer separation. + Also ensures that headers, data and trailers are sent in that order. + """ + + def __init__(self, conn: connection.Connection, is_client: bool, enable_webtransport: bool = False) -> None: + self._mock = MockQuic(conn, is_client) + super().__init__(self._mock, enable_webtransport) # type: ignore + + def _after_send(self, stream_id: int, end_stream: bool) -> None: + # if the stream ended, `QuicConnection` has an assert that no further data is being sent + # to catch this more early on, we set the header state on the `H3Stream` + if end_stream: + self._stream[stream_id].headers_send_state = HeadersState.AFTER_TRAILERS + + def _can_receive(self, stream_id: int) -> bool: + return ( + stream_is_client_initiated(stream_id) != self._is_client + or not stream_is_unidirectional(stream_id) + ) + + def _handle_request_or_push_frame( + self, + frame_type: int, + frame_data: Optional[bytes], + stream: H3Stream, + stream_ended: bool, + ) -> list[H3Event]: + # turn HeadersReceived into TrailersReceived for trailers + events = super()._handle_request_or_push_frame(frame_type, frame_data, stream, stream_ended) + for index, event in enumerate(events): + if ( + isinstance(event, HeadersReceived) + and self._stream[event.stream_id].headers_recv_state == HeadersState.AFTER_TRAILERS + ): + events[index] = TrailersReceived( + trailer=event.headers, + stream_id=event.stream_id, + push_id=event.push_id, + ) + return events + + def close_connection(self, error_code: int = QuicErrorCode.NO_ERROR, reason_phrase: str = "") -> None: + """Closes the underlying QUIC connection and ignores any incoming events.""" + + self._is_done = True + self._quic.close(error_code=error_code, reason_phrase=reason_phrase) + + def end_stream(self, stream_id: int) -> None: + """Ends the given stream.""" + + self.send_data(stream_id, data=b"", end_stream=True) + + def get_next_available_stream_id(self, is_unidirectional: bool = False): + """Reserves and returns the next available stream ID.""" + + return self._quic.get_next_available_stream_id(is_unidirectional) + + def handle_event(self, event: QuicStreamEvent) -> list[H3Event]: + # don't do anything if we're done + if self._is_done: + return [] + + # treat reset events similar to stream end events + elif isinstance(event, QuicStreamReset): + stream = self._get_or_create_stream(event.stream_id) + stream.ended = True + return [StreamReset( + stream_id=event.stream_id, + error_code=event.error_code, + push_id=stream.push_id, + )] + + # convert data events from the QUIC layer back to aioquic events + elif isinstance(event, QuicStreamDataReceived): + return super().handle_event(StreamDataReceived( + stream_id=event.stream_id, + data=event.data, + end_stream=event.end_stream, + )) + + # should never happen + else: + raise AssertionError(f"Unexpected event: {event!r}") + + def is_stream_open(self, stream_id: int) -> bool: + """Indicates whether the given stream is receivable.""" + + if not self._can_receive(stream_id): + return False + try: + return not self._stream[stream_id].ended + except KeyError: + return True + + def has_sent_headers(self, stream_id: int) -> bool: + """Indicates whether headers have been sent over the given stream.""" + + try: + return self._stream[stream_id].headers_send_state != HeadersState.INITIAL + except KeyError: + return False + + @property + def open_stream_ids(self) -> Iterable[int]: + """Returns all receivable streams.""" + + for stream in self._stream.values(): + if self._can_receive(stream.stream_id) and not stream.ended: + yield stream.stream_id + + def reset_stream(self, stream_id: int, error_code: int) -> None: + """Resets a stream that hasn't been ended locally yet.""" + + # we don't allow reset after FIN + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state == HeadersState.AFTER_TRAILERS: + raise H3FrameUnexpected("reset not allowed in this state") + + # set the header state and queue a reset event + stream.headers_send_state = HeadersState.AFTER_TRAILERS + self._quic.reset_stream(stream_id, error_code) + + def send_data(self, stream_id: int, data: bytes, end_stream: bool = False) -> None: + """Sends data over the given stream.""" + + super().send_data(stream_id, data, end_stream) + self._after_send(stream_id, end_stream) + + def send_datagram(self, flow_id: int, data: bytes) -> None: + # supporting datagrams would require additional information from the underlying QUIC connection + raise NotImplementedError() # pragma: no cover + + def send_headers(self, stream_id: int, headers: Headers, end_stream: bool = False) -> None: + """Sends headers over the given stream.""" + + # ensure we haven't sent something before + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state != HeadersState.INITIAL: + raise H3FrameUnexpected("initial HEADERS frame is not allowed in this state") + super().send_headers(stream_id, headers, end_stream) + self._after_send(stream_id, end_stream) + + def send_trailers(self, stream_id: int, trailers: Headers) -> None: + """Sends trailers over the given stream and ends it.""" + + # ensure we got some headers first + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state != HeadersState.AFTER_HEADERS: + raise H3FrameUnexpected("trailing HEADERS frame is not allowed in this state") + super().send_headers(stream_id, trailers, end_stream=True) + self._after_send(stream_id, end_stream=True) + + def transmit(self) -> layer.CommandGenerator[None]: + """Yields all pending commands for the upper QUIC layer.""" + + while self._mock.pending_commands: + yield self._mock.pending_commands.get() + + +__all__ = [ + "LayeredH3Connection", + "StreamReset", + "TrailersReceived", +] From 0b6472f8d9ab6d040a87377a2c19101b5a09d69c Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 22 Sep 2022 03:43:03 +0200 Subject: [PATCH 065/695] [quic] bugfixes, remove OpenQuicStream command --- mitmproxy/addons/next_layer.py | 2 + mitmproxy/proxy/layers/http/__init__.py | 7 +- mitmproxy/proxy/layers/http/_http3.py | 16 ++-- mitmproxy/proxy/layers/http/_http_h3.py | 32 +++----- mitmproxy/proxy/layers/quic.py | 99 ++++++++++--------------- 5 files changed, 62 insertions(+), 94 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index dd1cbf7973..a2de82928c 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -245,6 +245,8 @@ def s(*layers): elif context.client.transport_protocol == "udp": # unlike TCP, we make a decision immediately + if isinstance(context.layers[-1], layers.ServerQuicLayer): + return layers.ClientQuicLayer(context) tls = self.detect_udp_tls(data_client) is_quic = isinstance(context.layers[-1], layers.ClientQuicLayer) raw_layer_cls = layers.RawQuicLayer if is_quic else layers.UDPLayer diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 073734cb2d..61d668ef8d 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -884,8 +884,9 @@ def _handle_event(self, event: events.Event): if isinstance(event, events.ConnectionClosed): # The peer has closed it - let's close it too! yield commands.CloseConnection(event.connection) - elif isinstance(event, events.DataReceived): - # The peer has sent data. This can happen with HTTP/2 servers that already send a settings frame. + else: + # The peer has sent data or another connection activity occurred. + # This can happen with HTTP/2 servers that already send a settings frame. child_layer: HttpConnection if is_h3_alpn(self.context.server.alpn): child_layer = Http3Client(self.context.fork()) @@ -896,8 +897,6 @@ def _handle_event(self, event: events.Event): self.connections[self.context.server] = child_layer yield from self.event_to_child(child_layer, events.Start()) yield from self.event_to_child(child_layer, event) - else: - raise AssertionError(f"Unexpected event: {event}") else: handler = self.connections[event.connection] yield from self.event_to_child(handler, event) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index d30cf16312..a95ea80fef 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -42,7 +42,7 @@ parse_h2_request_headers, parse_h2_response_headers, ) -from._http_h3 import LayeredH3Connection, StreamReset, TrailersReceived +from ._http_h3 import LayeredH3Connection, StreamReset, TrailersReceived class Http3Connection(HttpConnection): @@ -55,14 +55,14 @@ class Http3Connection(HttpConnection): def __init__(self, context: context.Context, conn: connection.Connection): super().__init__(context, conn) - self.h3_conn = LayeredH3Connection(self.conn, is_client=self.conn is self.context.client) + self.h3_conn = LayeredH3Connection(self.conn, is_client=self.conn is self.context.server) def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): pass # send mitmproxy HTTP events over the H3 connection - if isinstance(event, HttpEvent): + elif isinstance(event, HttpEvent): try: if isinstance(event, (RequestData, ResponseData)): self.h3_conn.send_data(event.stream_id, event.data) @@ -74,8 +74,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) self.h3_conn.send_headers(event.stream_id, headers, end_stream=event.end_stream) elif isinstance(event, (RequestTrailers, ResponseTrailers)): - trailers = [*event.trailers.fields] - self.h3_conn.send_trailers(event.stream_id, trailers) + self.h3_conn.send_trailers(event.stream_id, [*event.trailers.fields]) elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): self.h3_conn.end_stream(event.stream_id) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): @@ -109,7 +108,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: except H3FrameUnexpected as e: # Http2Connection also ignores HttpEvents that violate the current stream state - yield from commands.Log(f"Received {event!r} unexpectedly: {e}") + yield commands.Log(f"Received {event!r} unexpectedly: {e}") else: # transmit buffered data @@ -148,8 +147,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if h3_event.stream_ended: yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) elif isinstance(h3_event, TrailersReceived) and h3_event.push_id is None: - trailers = http.Headers(h3_event.trailers) - yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, trailers)) + yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) else: # we don't support push, web transport, etc. yield commands.Log(f"Ignored unsupported H3 event: {h3_event!r}") @@ -164,7 +162,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) for stream_id in self.h3_conn.open_stream_ids: yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) - self._handle_event = self.done + self._handle_event = self.done # type: ignore else: raise AssertionError(f"Unexpected event: {event!r}") diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index dc05a0e8b0..fa1f442a44 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -11,9 +11,9 @@ HeadersState, ) from aioquic.h3.events import HeadersReceived -from aioquic.quic import events as quic_events from aioquic.quic.connection import stream_is_client_initiated, stream_is_unidirectional -from aioquic.quic.events import StreamDataReceived +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.events import ConnectionTerminated, StreamDataReceived from aioquic.quic.packet import QuicErrorCode from mitmproxy import connection @@ -68,11 +68,15 @@ class MockQuic: def __init__(self, conn: connection.Connection, is_client: bool) -> None: self.conn = conn - self.configuration = type("Object", (), {"_is_client": is_client}) self.pending_commands: Queue[commands.Command] = Queue() - self._next_stream_id: list[int, int, int, int] = [0, 1, 2, 3] + self._next_stream_id: list[int] = [0, 1, 2, 3] self._is_client = is_client + # the following fields are accessed by H3Connection + self.configuration = QuicConfiguration(is_client=is_client) + self._quic_logger = None + self._remote_max_datagram_frame_size = 0 + def close( self, error_code: int = QuicErrorCode.NO_ERROR, @@ -83,7 +87,7 @@ def close( # we note the error on the connection and yield a CloseConnection # this will then call `QuicConnection.close` with the proper values # once the `Http3Connection` receives `ConnectionClosed`, it will send out `*ProtocolError` - set_connection_error(self.conn, quic_events.ConnectionTerminated( + set_connection_error(self.conn, ConnectionTerminated( error_code=error_code, frame_type=frame_type, reason_phrase=reason_phrase, @@ -140,11 +144,7 @@ def _handle_request_or_push_frame( isinstance(event, HeadersReceived) and self._stream[event.stream_id].headers_recv_state == HeadersState.AFTER_TRAILERS ): - events[index] = TrailersReceived( - trailer=event.headers, - stream_id=event.stream_id, - push_id=event.push_id, - ) + events[index] = TrailersReceived(event.headers, event.stream_id, event.push_id) return events def close_connection(self, error_code: int = QuicErrorCode.NO_ERROR, reason_phrase: str = "") -> None: @@ -172,19 +172,11 @@ def handle_event(self, event: QuicStreamEvent) -> list[H3Event]: elif isinstance(event, QuicStreamReset): stream = self._get_or_create_stream(event.stream_id) stream.ended = True - return [StreamReset( - stream_id=event.stream_id, - error_code=event.error_code, - push_id=stream.push_id, - )] + return [StreamReset(event.stream_id, event.error_code, stream.push_id)] # convert data events from the QUIC layer back to aioquic events elif isinstance(event, QuicStreamDataReceived): - return super().handle_event(StreamDataReceived( - stream_id=event.stream_id, - data=event.data, - end_stream=event.end_stream, - )) + return super().handle_event(StreamDataReceived(event.data, event.end_stream, event.stream_id)) # should never happen else: diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 9bc442b574..d69e4e368f 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from ssl import VerifyMode import time -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import H3_ALPN, ErrorCode as H3ErrorCode @@ -176,27 +176,6 @@ def __init__(self, connection: connection.Connection, stream_id: int, error_code self.error_code = error_code -class OpenQuicStream(commands.ConnectionCommand): - """Command that allocates and returns the next available stream ID.""" - - is_unidirectional: bool - """Whether the stream should be unidirectional.""" - blocking = True - - def __init__(self, connection: connection.Connection, is_unidirectional: bool = False): - super().__init__(connection) - self.is_unidirectional = is_unidirectional - - -@dataclass(repr=False) -class OpenQuicStreamCompleted(events.CommandCompleted): - """Emitted when `OpenQuicStream` has been finished.""" - - command: OpenQuicStream - reply: int - """The stream ID for the next stream created by this endpoint.""" - - class QuicSecretsLogger: logger: tls.MasterSecretLogger @@ -340,12 +319,12 @@ def __init__(self, context: context.Context, ignore: bool, client_stream_id: int peername=context.client.peername, sockname=context.client.sockname, timestamp_start=time.time(), - transport_protocol=context.client.transport_protocol, + transport_protocol="tcp", proxy_mode=context.client.proxy_mode, ) self.server = context.server = connection.Server( address=context.server.address, - transport_protocol=context.server.transport_protocol, + transport_protocol="tcp", ) super().__init__(context) self.client_stream_id = client_stream_id @@ -382,6 +361,8 @@ class RawQuicLayer(layer.Layer): """Maps connections to layers.""" command_sources: dict[commands.Command, layer.Layer] """Keeps track of blocking commands and wakeup requests.""" + next_stream_id: list[int] + """List containing the next stream ID for all four is_unidirectional/is_client combinations.""" def __init__(self, context: context.Context, ignore: bool = False) -> None: super().__init__(context) @@ -398,6 +379,7 @@ def __init__(self, context: context.Context, ignore: bool = False) -> None: context.server: self.datagram_layer, } self.command_sources = {} + self.next_stream_id = [0, 1, 2, 3] def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # we treat the datagram layer as child layer, so forward Start @@ -440,17 +422,18 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: client_stream_id = event.stream_id server_stream_id = None else: - client_stream_id = cast(int, (yield OpenQuicStream( - connection=self.context.client, + client_stream_id = self.get_next_available_stream_id( + is_client=False, is_unidirectional=stream_is_unidirectional(event.stream_id), - ))) + ) server_stream_id = event.stream_id # create, register and start the layer - stream_layer = QuicStreamLayer(self.context, self.ignore, client_stream_id, server_stream_id) + stream_layer = QuicStreamLayer(self.context.fork(), self.ignore, client_stream_id, server_stream_id) stream_layer.client.state = get_stream_connection_state(client_stream_id, is_client=False) self.client_stream_ids[client_stream_id] = stream_layer if server_stream_id is not None: + stream_layer.server.timestamp_start = time.time() stream_layer.server.state = get_stream_connection_state(server_stream_id, is_client=True) self.server_stream_ids[server_stream_id] = stream_layer self.connections[stream_layer.client] = stream_layer @@ -499,11 +482,20 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unexpected event: {event!r}") - def close_stream_layer(self, stream_layer: QuicStreamLayer, conn: connection.Connection) -> layer.CommandGenerator[None]: + def close_stream_layer( + self, + stream_layer: QuicStreamLayer, + conn: connection.Connection, + force: bool = False, + ) -> layer.CommandGenerator[None]: """Closes the incoming part of a connection.""" if conn.state & connection.ConnectionState.CAN_READ: conn.state &= ~connection.ConnectionState.CAN_READ + if force: + stream_id = stream_layer.client_stream_id if conn is stream_layer.client else stream_layer.server_stream_id + if stream_id is not None: + yield StopQuicStream(conn, stream_id, QuicErrorCode.NO_ERROR) yield from self.event_to_child(stream_layer, events.ConnectionClosed(conn)) def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: @@ -537,15 +529,14 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer conn.state &= ~connection.ConnectionState.CAN_WRITE yield SendQuicStreamData(conn, stream_id, b"", end_stream=True) if not command.half_close: - yield StopQuicStream(conn, stream_id, QuicErrorCode.NO_ERROR) - yield from self.close_stream_layer(child_layer, conn) + yield from self.close_stream_layer(child_layer, conn, force=True) # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): assert not to_client assert stream_id is None child_layer.server.timestamp_start = time.time() - stream_id = cast(int, (yield OpenQuicStream(conn))) + stream_id = self.get_next_available_stream_id(is_client=True) child_layer.server_stream_id = stream_id child_layer.server.state = get_stream_connection_state(stream_id, is_client=True) self.server_stream_ids[stream_id] = child_layer @@ -562,6 +553,12 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer self.connections[command.connection] = child_layer yield command + def get_next_available_stream_id(self, is_client: bool, is_unidirectional: bool = False) -> int: + index = (int(is_unidirectional) << 1) | int(not is_client) + stream_id = self.next_stream_id[index] + self.next_stream_id[index] = stream_id + 4 + return stream_id + class QuicLayer(tunnel.TunnelLayer): quic: QuicConnection | None = None @@ -592,39 +589,19 @@ def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[N """Turns stream commands into aioquic connection invocations.""" if ( - isinstance(command, SendQuicStreamData) + isinstance(command, QuicStreamCommand) and command.connection is self.conn ): assert self.quic - self.quic.send_stream_data(command.stream_id, command.data, command.end_stream) - yield from self.tls_interact() - - elif ( - isinstance(command, ResetQuicStream) - and command.connection is self.conn - ): - assert self.quic - self.quic.reset_stream(command.stream_id, command.error_code) - yield from self.tls_interact() - - elif ( - isinstance(command, StopQuicStream) - and command.connection is self.conn - ): - assert self.quic - self.quic.stop_stream(command.stream_id, command.error_code) + if isinstance(command, SendQuicStreamData): + self.quic.send_stream_data(command.stream_id, command.data, command.end_stream) + elif isinstance(command, ResetQuicStream): + self.quic.reset_stream(command.stream_id, command.error_code) + elif isinstance(command, StopQuicStream): + self.quic.stop_stream(command.stream_id, command.error_code) + else: + raise AssertionError(f"Unexpected stream command: {command!r}") yield from self.tls_interact() - - elif ( - isinstance(command, OpenQuicStream) - and command.connection is self.conn - ): - assert self.quic - stream_id = self.quic.get_next_available_stream_id(command.is_unidirectional) - # the next operation is a no-op, but will allocate the stream ID - self.quic.send_stream_data(stream_id, data=b"", end_stream=False) - self.event_to_child(OpenQuicStreamCompleted(command, stream_id)) - else: yield from super()._handle_command(command) From 04c7892ba437c2e4a33ce7fa487fc5194aef9dc1 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 22 Sep 2022 06:15:11 +0200 Subject: [PATCH 066/695] [quic] more bugfixes --- mitmproxy/proxy/layers/quic.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index d69e4e368f..d3d7e13d80 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -461,7 +461,11 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: or event.connection is self.context.server ) ): + # copy the connection error from_client = event.connection is self.context.client + close_event = get_connection_error(event.connection) + if close_event is not None: + set_connection_error(self.context.server if from_client else self.context.client, close_event) # always forward to the datagram layer yield from self.event_to_child(self.datagram_layer, event) @@ -482,20 +486,11 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unexpected event: {event!r}") - def close_stream_layer( - self, - stream_layer: QuicStreamLayer, - conn: connection.Connection, - force: bool = False, - ) -> layer.CommandGenerator[None]: + def close_stream_layer(self, stream_layer: QuicStreamLayer, conn: connection.Connection) -> layer.CommandGenerator[None]: """Closes the incoming part of a connection.""" if conn.state & connection.ConnectionState.CAN_READ: conn.state &= ~connection.ConnectionState.CAN_READ - if force: - stream_id = stream_layer.client_stream_id if conn is stream_layer.client else stream_layer.server_stream_id - if stream_id is not None: - yield StopQuicStream(conn, stream_id, QuicErrorCode.NO_ERROR) yield from self.event_to_child(stream_layer, events.ConnectionClosed(conn)) def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: @@ -529,14 +524,22 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer conn.state &= ~connection.ConnectionState.CAN_WRITE yield SendQuicStreamData(conn, stream_id, b"", end_stream=True) if not command.half_close: - yield from self.close_stream_layer(child_layer, conn, force=True) + if ( + stream_is_client_initiated(stream_id) == to_client + or not stream_is_unidirectional(stream_id) + ): + yield StopQuicStream(conn, stream_id, QuicErrorCode.NO_ERROR) + yield from self.close_stream_layer(child_layer, conn) # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): assert not to_client assert stream_id is None child_layer.server.timestamp_start = time.time() - stream_id = self.get_next_available_stream_id(is_client=True) + stream_id = self.get_next_available_stream_id( + is_client=True, + is_unidirectional=stream_is_unidirectional(child_layer.client_stream_id) + ) child_layer.server_stream_id = stream_id child_layer.server.state = get_stream_connection_state(stream_id, is_client=True) self.server_stream_ids[stream_id] = child_layer @@ -598,7 +601,9 @@ def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[N elif isinstance(command, ResetQuicStream): self.quic.reset_stream(command.stream_id, command.error_code) elif isinstance(command, StopQuicStream): - self.quic.stop_stream(command.stream_id, command.error_code) + # the stream might have already been closed, check before stopping + if command.stream_id in self.quic._streams: + self.quic.stop_stream(command.stream_id, command.error_code) else: raise AssertionError(f"Unexpected stream command: {command!r}") yield from self.tls_interact() From 8ea0e1b17fe8578f293bb65183abd3b0d98a96c1 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 22 Sep 2022 08:51:11 +0200 Subject: [PATCH 067/695] [quic] replace queue with list --- mitmproxy/proxy/layers/http/_http_h3.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index fa1f442a44..3732da3393 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from queue import Queue from typing import Iterable, Optional from aioquic.h3.connection import ( @@ -68,7 +67,7 @@ class MockQuic: def __init__(self, conn: connection.Connection, is_client: bool) -> None: self.conn = conn - self.pending_commands: Queue[commands.Command] = Queue() + self.pending_commands: list[commands.Command] = [] self._next_stream_id: list[int] = [0, 1, 2, 3] self._is_client = is_client @@ -92,7 +91,7 @@ def close( frame_type=frame_type, reason_phrase=reason_phrase, )) - self.pending_commands.put(commands.CloseConnection(self.conn)) + self.pending_commands.append(commands.CloseConnection(self.conn)) def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: # since we always reserve the ID, we have to "find" the next ID like `QuicConnection` does @@ -102,10 +101,10 @@ def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: return stream_id def reset_stream(self, stream_id: int, error_code: int) -> None: - self.pending_commands.put(ResetQuicStream(self.conn, stream_id, error_code)) + self.pending_commands.append(ResetQuicStream(self.conn, stream_id, error_code)) def send_stream_data(self, stream_id: int, data: bytes, end_stream: bool = False) -> None: - self.pending_commands.put(SendQuicStreamData(self.conn, stream_id, data, end_stream)) + self.pending_commands.append(SendQuicStreamData(self.conn, stream_id, data, end_stream)) class LayeredH3Connection(H3Connection): @@ -254,7 +253,7 @@ def transmit(self) -> layer.CommandGenerator[None]: """Yields all pending commands for the upper QUIC layer.""" while self._mock.pending_commands: - yield self._mock.pending_commands.get() + yield self._mock.pending_commands.pop(0) __all__ = [ From a7b166f5cd5b00df2116bc7fdf2ec69cb3d83172 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 23 Sep 2022 11:36:12 +0200 Subject: [PATCH 068/695] [quic] reverse H3 running again --- mitmproxy/proxy/layers/http/_http3.py | 9 ++++-- mitmproxy/proxy/layers/http/_http_h3.py | 42 ++++++++++++------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index a95ea80fef..9ae66b3b0b 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -78,7 +78,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): self.h3_conn.end_stream(event.stream_id) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): - if self.h3_conn.is_stream_open(event.stream_id): + if not self.h3_conn.has_ended(event.stream_id): code = { status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED, }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR) @@ -130,7 +130,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) ) elif isinstance(h3_event, DataReceived) and h3_event.push_id is None: - yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) + if h3_event.data: + yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) if h3_event.stream_ended: yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) elif isinstance(h3_event, HeadersReceived) and h3_event.push_id is None: @@ -148,6 +149,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) elif isinstance(h3_event, TrailersReceived) and h3_event.push_id is None: yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) + if h3_event.stream_ended: + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) else: # we don't support push, web transport, etc. yield commands.Log(f"Ignored unsupported H3 event: {h3_event!r}") @@ -160,7 +163,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if close_event is None else close_event.reason_phrase or error_code_to_str(close_event.error_code) ) - for stream_id in self.h3_conn.open_stream_ids: + for stream_id in self.h3_conn.get_reserved_stream_ids(): yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) self._handle_event = self.done # type: ignore diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 3732da3393..977de4681c 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -10,7 +10,6 @@ HeadersState, ) from aioquic.h3.events import HeadersReceived -from aioquic.quic.connection import stream_is_client_initiated, stream_is_unidirectional from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.events import ConnectionTerminated, StreamDataReceived from aioquic.quic.packet import QuicErrorCode @@ -39,6 +38,9 @@ class TrailersReceived(H3Event): stream_id: int "The ID of the stream the trailers were received for." + stream_ended: bool + "Whether the STREAM frame had the FIN bit set." + push_id: Optional[int] = None "The Push ID or `None` if this is not a push." @@ -100,6 +102,10 @@ def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: self._next_stream_id[index] = stream_id + 4 return stream_id + def get_reserved_stream_ids(self, is_unidirectional: bool = False) -> Iterable[int]: + index = (int(is_unidirectional) << 1) | int(not self._is_client) + return range(index, self._next_stream_id[index] + 1, 4) + def reset_stream(self, stream_id: int, error_code: int) -> None: self.pending_commands.append(ResetQuicStream(self.conn, stream_id, error_code)) @@ -123,12 +129,6 @@ def _after_send(self, stream_id: int, end_stream: bool) -> None: if end_stream: self._stream[stream_id].headers_send_state = HeadersState.AFTER_TRAILERS - def _can_receive(self, stream_id: int) -> bool: - return ( - stream_is_client_initiated(stream_id) != self._is_client - or not stream_is_unidirectional(stream_id) - ) - def _handle_request_or_push_frame( self, frame_type: int, @@ -143,7 +143,7 @@ def _handle_request_or_push_frame( isinstance(event, HeadersReceived) and self._stream[event.stream_id].headers_recv_state == HeadersState.AFTER_TRAILERS ): - events[index] = TrailersReceived(event.headers, event.stream_id, event.push_id) + events[index] = TrailersReceived(event.headers, event.stream_id, event.stream_ended, event.push_id) return events def close_connection(self, error_code: int = QuicErrorCode.NO_ERROR, reason_phrase: str = "") -> None: @@ -153,15 +153,23 @@ def close_connection(self, error_code: int = QuicErrorCode.NO_ERROR, reason_phra self._quic.close(error_code=error_code, reason_phrase=reason_phrase) def end_stream(self, stream_id: int) -> None: - """Ends the given stream.""" + """Ends the given stream locally.""" - self.send_data(stream_id, data=b"", end_stream=True) + # check whether the stream hasn't been ended before + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state != HeadersState.AFTER_TRAILERS: + self.send_data(stream_id, data=b"", end_stream=True) def get_next_available_stream_id(self, is_unidirectional: bool = False): """Reserves and returns the next available stream ID.""" return self._quic.get_next_available_stream_id(is_unidirectional) + def get_reserved_stream_ids(self, is_unidirectional: bool = False) -> Iterable[int]: + """Returns all reserved stream IDs.""" + + return self._mock.get_reserved_stream_ids(is_unidirectional) + def handle_event(self, event: QuicStreamEvent) -> list[H3Event]: # don't do anything if we're done if self._is_done: @@ -181,11 +189,9 @@ def handle_event(self, event: QuicStreamEvent) -> list[H3Event]: else: raise AssertionError(f"Unexpected event: {event!r}") - def is_stream_open(self, stream_id: int) -> bool: - """Indicates whether the given stream is receivable.""" + def has_ended(self, stream_id: int) -> bool: + """Indicates whether the given stream has been ended by the peer.""" - if not self._can_receive(stream_id): - return False try: return not self._stream[stream_id].ended except KeyError: @@ -199,14 +205,6 @@ def has_sent_headers(self, stream_id: int) -> bool: except KeyError: return False - @property - def open_stream_ids(self) -> Iterable[int]: - """Returns all receivable streams.""" - - for stream in self._stream.values(): - if self._can_receive(stream.stream_id) and not stream.ended: - yield stream.stream_id - def reset_stream(self, stream_id: int, error_code: int) -> None: """Resets a stream that hasn't been ended locally yet.""" From 4d17c8f263cb9d7f1bdd40a3c9b5c33648cf5d4e Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 23 Sep 2022 13:02:53 +0200 Subject: [PATCH 069/695] [quic] new logging and cert loading --- mitmproxy/addons/proxyserver.py | 4 ++-- mitmproxy/addons/tlsconfig.py | 6 ++--- mitmproxy/certs.py | 8 +++---- mitmproxy/contentviews/http3.py | 9 ++++---- mitmproxy/proxy/context.py | 11 ++-------- mitmproxy/proxy/layers/http/_http3.py | 8 +++---- mitmproxy/proxy/layers/http/_http_h3.py | 4 ++-- mitmproxy/proxy/layers/quic.py | 29 +++++++++++++++---------- mitmproxy/proxy/server.py | 2 +- 9 files changed, 39 insertions(+), 42 deletions(-) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 0ef64c1ea5..b35b02ddd7 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -310,13 +310,13 @@ def inject_tcp(self, flow: Flow, to_client: bool, message: bytes): @command.command("inject.udp") def inject_udp(self, flow: Flow, to_client: bool, message: bytes): if not isinstance(flow, udp.UDPFlow): - ctx.log.warn("Cannot inject UDP messages into non-UDP flows.") + logger.warning("Cannot inject UDP messages into non-UDP flows.") event = UdpMessageInjected(flow, udp.UDPMessage(not to_client, message)) try: self.inject_event(event) except ValueError as e: - ctx.log.warn(str(e)) + logger.warning(str(e)) def server_connect(self, data: server_hooks.ServerConnectionHookData): if data.server.sockname is None: diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index a297494917..598442b492 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -325,7 +325,7 @@ def quic_start_client(self, tls_start: quic.QuicTlsData) -> None: client.cipher_list = ctx.options.ciphers_client.split(":") if client.cipher_list: tls_start.settings.cipher_suites = [ - CipherSuite(cipher) for cipher in client.cipher_list + CipherSuite[cipher] for cipher in client.cipher_list ] if ctx.options.add_upstream_certs_to_client_chain: @@ -356,13 +356,13 @@ def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: server.cipher_list = ctx.options.ciphers_server.split(":") if server.cipher_list: tls_start.settings.cipher_suites = [ - CipherSuite(cipher) for cipher in server.cipher_list + CipherSuite[cipher] for cipher in server.cipher_list ] client_cert = self.get_client_cert(server) if client_cert: config = QuicConfiguration() - config.load_cert_chain(client_cert) + config.load_cert_chain(Path(client_cert)) assert isinstance(config.certificate, x509.Certificate) tls_start.settings.certificate = config.certificate if config.private_key: diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index f982e3cf89..63c748d6d9 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import NewType, Optional, Union -from aioquic.tls import load_pem_x509_certificates from cryptography import x509 from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa, dsa, ec @@ -315,10 +314,9 @@ def __init__( self.default_chain_file = default_chain_file self.default_chain_certs = ( [ - Cert(cert) - for cert in load_pem_x509_certificates( - self.default_chain_file.read_bytes() - ) + Cert.from_pem(cert) + for cert in re.split(rb"(?<=-----END CERTIFICATE-----\n)", self.default_chain_file.read_bytes()) + if cert ] if self.default_chain_file else [default_ca] diff --git a/mitmproxy/contentviews/http3.py b/mitmproxy/contentviews/http3.py index bead71f8aa..0e8f70872d 100644 --- a/mitmproxy/contentviews/http3.py +++ b/mitmproxy/contentviews/http3.py @@ -1,13 +1,14 @@ from collections import defaultdict from collections.abc import Iterator from dataclasses import dataclass, field +from typing import Optional from aioquic.h3.connection import Setting, parse_settings from mitmproxy import flow, tcp from . import base from .hex import ViewHex -from ..proxy.layers.http import is_h3_alpn # type: ignore +from ..proxy.layers.http import is_h3_alpn from aioquic.buffer import Buffer, BufferReadError import pylsqpack @@ -72,8 +73,8 @@ def __init__(self): def __call__( self, data, - flow: flow.Flow | None = None, - tcp_message: tcp.TCPMessage | None = None, + flow: Optional[flow.Flow] = None, + tcp_message: Optional[tcp.TCPMessage] = None, **metadata ): assert isinstance(flow, tcp.TCPFlow) @@ -124,7 +125,7 @@ def __call__( def render_priority( self, data: bytes, - flow: flow.Flow | None = None, + flow: Optional[flow.Flow] = None, **metadata ) -> float: return 2 * float(bool(flow and is_h3_alpn(flow.client_conn.alpn))) * float(isinstance(flow, tcp.TCPFlow)) diff --git a/mitmproxy/proxy/context.py b/mitmproxy/proxy/context.py index 3e492b5b23..5edb977c25 100644 --- a/mitmproxy/proxy/context.py +++ b/mitmproxy/proxy/context.py @@ -1,11 +1,10 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from mitmproxy import connection from mitmproxy.options import Options if TYPE_CHECKING: import mitmproxy.proxy.layer - from mitmproxy.proxy.server import ConnectionHandler class Context: @@ -26,10 +25,6 @@ class Context: """ Provides access to options for proxy layers. Not intended for use by addons, use `mitmproxy.ctx.options` instead. """ - handler: Optional["ConnectionHandler"] - """ - The `ConnectionHandler` responsible for this context. - """ layers: list["mitmproxy.proxy.layer.Layer"] """ The protocol layer stack. @@ -39,18 +34,16 @@ def __init__( self, client: connection.Client, options: Options, - handler: Optional["ConnectionHandler"] = None, ) -> None: self.client = client self.options = options - self.handler = handler self.server = connection.Server( None, transport_protocol=client.transport_protocol ) self.layers = [] def fork(self) -> "Context": - ret = Context(self.client, self.options, self.handler) + ret = Context(self.client, self.options) ret.server = self.server ret.layers = self.layers.copy() return ret diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 9ae66b3b0b..fa7c975e02 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -80,8 +80,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): if not self.h3_conn.has_ended(event.stream_id): code = { - status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED, - }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR) + status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED.value, + }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR.value) send_error_message = ( isinstance(event, ResponseProtocolError) and not self.h3_conn.has_sent_headers(event.stream_id) @@ -116,11 +116,11 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # forward stream messages from the QUIC layer to the H3 connection elif isinstance(event, QuicStreamEvent): - for h3_event in self.h3_conn.handle_event(event): + for h3_event in self.h3_conn.handle_stream_event(event): if isinstance(h3_event, StreamReset) and h3_event.push_id is None: err_str = error_code_to_str(h3_event.error_code) err_code = { - H3ErrorCode.H3_REQUEST_CANCELLED: status_codes.CLIENT_CLOSED_REQUEST, + H3ErrorCode.H3_REQUEST_CANCELLED.value: status_codes.CLIENT_CLOSED_REQUEST, }.get(h3_event.error_code, self.ReceiveProtocolError.code) yield ReceiveHttp( self.ReceiveProtocolError( diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 977de4681c..3bc1447ad5 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -170,7 +170,7 @@ def get_reserved_stream_ids(self, is_unidirectional: bool = False) -> Iterable[i return self._mock.get_reserved_stream_ids(is_unidirectional) - def handle_event(self, event: QuicStreamEvent) -> list[H3Event]: + def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: # don't do anything if we're done if self._is_done: return [] @@ -183,7 +183,7 @@ def handle_event(self, event: QuicStreamEvent) -> list[H3Event]: # convert data events from the QUIC layer back to aioquic events elif isinstance(event, QuicStreamDataReceived): - return super().handle_event(StreamDataReceived(event.data, event.end_stream, event.stream_id)) + return self.handle_event(StreamDataReceived(event.data, event.end_stream, event.stream_id)) # should never happen else: diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index d3d7e13d80..c3724b8427 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio from dataclasses import dataclass, field +from logging import DEBUG, ERROR, WARNING from ssl import VerifyMode import time from typing import TYPE_CHECKING @@ -285,9 +286,9 @@ def initialize_replacement(peer_cid: bytes) -> None: try: return _initialize(peer_cid) finally: - quic.tls._server_handle_hello = server_handle_hello_replacement + quic.tls._server_handle_hello = server_handle_hello_replacement # type: ignore - quic._initialize = initialize_replacement + quic._initialize = initialize_replacement # type: ignore try: quic.receive_datagram(data, ("0.0.0.0", 0), now=0) except QuicClientHello as hello: @@ -448,7 +449,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.close_stream_layer(stream_layer, conn) elif isinstance(event, QuicStreamReset): if self.debug is not None: - yield commands.Log(f"{self.debug}[quic] stream_reset (stream_id={event.stream_id}, error_code={event.error_code})") + yield commands.Log( + f"{self.debug}[quic] stream_reset (stream_id={event.stream_id}, error_code={event.error_code})", DEBUG + ) yield from self.close_stream_layer(stream_layer, conn) else: raise AssertionError(f"Unexpected stream event: {event!r}") @@ -624,7 +627,7 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C else: yield QuicStartServerHook(tls_data) if tls_data.settings is None: - yield commands.Log(f"No QUIC context was provided, failing connection.", level="error") + yield commands.Log(f"No QUIC context was provided, failing connection.", ERROR) yield commands.CloseConnection(self.conn) return @@ -702,15 +705,17 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # set the connection's TLS properties self.conn.timestamp_tls_setup = self._loop.time() - self.conn.alpn = event.alpn_protocol.encode("ascii") + if event.alpn_protocol: + self.conn.alpn = event.alpn_protocol.encode("ascii") self.conn.certificate_list = [certs.Cert(cert) for cert in all_certs] + assert self.quic.tls.key_schedule self.conn.cipher = self.quic.tls.key_schedule.cipher_suite.name self.conn.tls_version = "QUIC" # log the result and report the success to addons if self.debug: yield commands.Log( - f"{self.debug}[quic] tls established: {self.conn}", "debug" + f"{self.debug}[quic] tls established: {self.conn}", DEBUG ) if self.conn is self.context.client: yield TlsEstablishedClientHook(QuicTlsData(self.conn, self.context, settings=self.tls)) @@ -755,7 +760,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: if self.debug: reason = event.reason_phrase or error_code_to_str(event.error_code) yield commands.Log( - f"{self.debug}[quic] close_notify {self.conn} (reason={reason})", level="debug" + f"{self.debug}[quic] close_notify {self.conn} (reason={reason})", DEBUG ) yield commands.CloseConnection(self.conn) return # we don't handle any further events, nor do/can we transmit data, so exit @@ -835,7 +840,7 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: yield from super().event_to_child(event) def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: - yield commands.Log(f"Server QUIC handshake failed. {err}", level="warn") + yield commands.Log(f"Server QUIC handshake failed. {err}", level=WARNING) yield from super().on_handshake_error(err) @@ -879,11 +884,11 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo return False, f"Cannot parse QUIC header: {e} ({data.hex()})" # negotiate version, support all versions known to aioquic - supported_versions = ( + supported_versions = [ version.value for version in QuicProtocolVersion if version is not QuicProtocolVersion.NEGOTIATION - ) + ] if header.version is not None and header.version not in supported_versions: yield commands.SendData( self.tunnel_connection, @@ -962,10 +967,10 @@ def start_server_tls(self) -> layer.CommandGenerator[str | None]: return err def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: - yield commands.Log(f"Client QUIC handshake failed. {err}", level="warn") + yield commands.Log(f"Client QUIC handshake failed. {err}", level=WARNING) yield from super().on_handshake_error(err) self.event_to_child = self.errored # type: ignore def errored(self, event: events.Event) -> layer.CommandGenerator[None]: if self.debug is not None: - yield commands.Log(f"{self.debug}[quic] Swallowing {event} as handshake failed.", "debug") + yield commands.Log(f"{self.debug}[quic] Swallowing {event} as handshake failed.", DEBUG) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index efaf6a9051..1341564ada 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -424,7 +424,7 @@ def __init__( time.time(), proxy_mode=mode, ) - context = Context(client, options, self) + context = Context(client, options) super().__init__(context) self.transports[client] = ConnectionIO( handler=None, reader=reader, writer=writer From a6d6bf91092d4013aa7e37d76362f9723d32fe1f Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 23 Sep 2022 13:23:59 +0200 Subject: [PATCH 070/695] [quic] suppress H3 push --- mitmproxy/proxy/layers/http/_http3.py | 74 +++++++++++++------------ mitmproxy/proxy/layers/http/_http_h3.py | 2 +- mitmproxy/proxy/mode_specs.py | 2 +- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index fa7c975e02..650d085b1a 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -6,7 +6,7 @@ ErrorCode as H3ErrorCode, FrameUnexpected as H3FrameUnexpected, ) -from aioquic.h3.events import DataReceived, HeadersReceived +from aioquic.h3.events import DataReceived, HeadersReceived, PushPromiseReceived from mitmproxy import connection, http, version from mitmproxy.net.http import status_codes @@ -117,43 +117,49 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # forward stream messages from the QUIC layer to the H3 connection elif isinstance(event, QuicStreamEvent): for h3_event in self.h3_conn.handle_stream_event(event): - if isinstance(h3_event, StreamReset) and h3_event.push_id is None: - err_str = error_code_to_str(h3_event.error_code) - err_code = { - H3ErrorCode.H3_REQUEST_CANCELLED.value: status_codes.CLIENT_CLOSED_REQUEST, - }.get(h3_event.error_code, self.ReceiveProtocolError.code) - yield ReceiveHttp( - self.ReceiveProtocolError( - h3_event.stream_id, - f"stream reset by client ({err_str})", - code=err_code, - ) - ) - elif isinstance(h3_event, DataReceived) and h3_event.push_id is None: - if h3_event.data: - yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) - if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - elif isinstance(h3_event, HeadersReceived) and h3_event.push_id is None: - try: - receive_event = self.parse_headers(h3_event) - except ValueError as e: - self.h3_conn.close_connection( - error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, - reason_phrase=f"Invalid HTTP/3 request headers: {e}", + if isinstance(h3_event, StreamReset): + if h3_event.push_id is None: + err_str = error_code_to_str(h3_event.error_code) + err_code = { + H3ErrorCode.H3_REQUEST_CANCELLED.value: status_codes.CLIENT_CLOSED_REQUEST, + }.get(h3_event.error_code, self.ReceiveProtocolError.code) + yield ReceiveHttp( + self.ReceiveProtocolError( + h3_event.stream_id, + f"stream reset by client ({err_str})", + code=err_code, + ) ) - yield from self.h3_conn.transmit() - else: - yield ReceiveHttp(receive_event) + elif isinstance(h3_event, DataReceived): + if h3_event.push_id is None: + if h3_event.data: + yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) + if h3_event.stream_ended: + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + elif isinstance(h3_event, HeadersReceived): + if h3_event.push_id is None: + try: + receive_event = self.parse_headers(h3_event) + except ValueError as e: + self.h3_conn.close_connection( + error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, + reason_phrase=f"Invalid HTTP/3 request headers: {e}", + ) + yield from self.h3_conn.transmit() + else: + yield ReceiveHttp(receive_event) + if h3_event.stream_ended: + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + elif isinstance(h3_event, TrailersReceived): + if h3_event.push_id is None: + yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) if h3_event.stream_ended: yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - elif isinstance(h3_event, TrailersReceived) and h3_event.push_id is None: - yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) - if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + elif isinstance(h3_event, PushPromiseReceived): + # we don't support push + pass else: - # we don't support push, web transport, etc. - yield commands.Log(f"Ignored unsupported H3 event: {h3_event!r}") + raise AssertionError(f"Unexpected event: {event!r}") # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 3bc1447ad5..56cd563060 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -104,7 +104,7 @@ def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: def get_reserved_stream_ids(self, is_unidirectional: bool = False) -> Iterable[int]: index = (int(is_unidirectional) << 1) | int(not self._is_client) - return range(index, self._next_stream_id[index] + 1, 4) + return range(index, self._next_stream_id[index], 4) def reset_stream(self, stream_id: int, error_code: int) -> None: self.pending_commands.append(ResetQuicStream(self.conn, stream_id, error_code)) diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index 4e38d6a0f5..73858b21ab 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -251,7 +251,7 @@ def __post_init__(self) -> None: class Http3Mode(ProxyMode): """ - A regular HTTP3 proxy that is interfaced with `HTTP CONNECT` calls (or absolute-form HTTP requests). + A regular HTTP3 proxy that is interfaced with absolute-form HTTP requests. (This class will be merged into `RegularMode` once the UDP implementation is deemed stable enough.) """ description = "HTTP3 proxy" From a5e5790f9e2d2077cd004679d99a0a57c47f042e Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sat, 24 Sep 2022 13:47:04 +0200 Subject: [PATCH 071/695] [quic] work on raw layer --- mitmproxy/addons/next_layer.py | 87 +++++++++++++++++++++------ mitmproxy/contentviews/__init__.py | 9 +-- mitmproxy/proxy/layers/__init__.py | 3 +- mitmproxy/proxy/layers/http/_http3.py | 2 +- mitmproxy/proxy/layers/quic.py | 16 ++--- 5 files changed, 82 insertions(+), 35 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index a2de82928c..b1cb179f5f 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -17,12 +17,12 @@ import re from collections.abc import Sequence import struct -from typing import Any, Callable, Iterable, Optional, Union +from typing import Any, Callable, Iterable, Optional, Union, cast from mitmproxy import ctx, dns, exceptions, connection from mitmproxy.net.tls import is_tls_record_magic from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy import context, layer, layers +from mitmproxy.proxy import context, layer, layers, mode_specs from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers.quic import quic_parse_client_hello from mitmproxy.proxy.layers.tls import HTTP_ALPNS, dtls_parse_client_hello, parse_client_hello @@ -155,6 +155,24 @@ def is_destination_in_hosts(self, context: context.Context, hosts: Iterable[re.P for rex in hosts ) + def is_reverse_proxy_scheme(self, context: context.Context, *args: str): + def s(*layers): + return stack_match(context, layers) + + # we allow all possible security layer combinations and rely on the correctness of ReverseProxy + return ( + ( + s(modes.ReverseProxy) + or + s(modes.ReverseProxy, layers.ClientTLSLayer) + or + s(modes.ReverseProxy, layers.ServerTLSLayer) + or + s(modes.ReverseProxy, layers.ServerTLSLayer, layers.ClientTLSLayer) + ) + and cast(mode_specs.ReverseMode, context.client.proxy_mode).scheme in args + ) + def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls]]: if len(data_client) == 0: return None @@ -195,7 +213,11 @@ def s(*layers): return stack_match(context, layers) if context.client.transport_protocol == "tcp": - if len(data_client) < 3 and not data_server: + if ( + len(data_client) < 3 + and not data_server + and not isinstance(context.layers[-1], layers.QuicStreamLayer) + ): return None # not enough data yet to make a decision # 1. check for --ignore/--allow @@ -229,7 +251,15 @@ def s(*layers): if self.is_destination_in_hosts(context, self.tcp_hosts): return layers.TCPLayer(context) - # 5. Check for raw tcp mode. + # 5. Check for raw reverse mode. + if self.is_reverse_proxy_scheme(context, "tcp", "tls"): + return layers.TCPLayer(context) + # NOTE at this point we are either + # - in http or https reverse mode + # - at the top level of a non-reverse/regular/upstream mode + # - at a deeper layer nesting level + + # 6. Check for raw tcp mode. very_likely_http = context.client.alpn and context.client.alpn in HTTP_ALPNS probably_no_http = not very_likely_http and ( not data_client[ @@ -240,16 +270,27 @@ def s(*layers): if ctx.options.rawtcp and probably_no_http: return layers.TCPLayer(context) - # 6. Assume HTTP by default. + # 7. Assume HTTP by default. return layers.HttpLayer(context, HTTPMode.transparent) elif context.client.transport_protocol == "udp": - # unlike TCP, we make a decision immediately - if isinstance(context.layers[-1], layers.ServerQuicLayer): + # for http3, upstream:http3 and reverse:quic/http3 proxies, there has to be a client quic layer + if ( + s(modes.HttpProxy) + or + s(modes.HttpUpstreamProxy) + or + s(modes.ReverseProxy, layers.ServerQuicLayer) + ): return layers.ClientQuicLayer(context) + + # unlike TCP, we make a decision immediately tls = self.detect_udp_tls(data_client) - is_quic = isinstance(context.layers[-1], layers.ClientQuicLayer) - raw_layer_cls = layers.RawQuicLayer if is_quic else layers.UDPLayer + raw_layer_cls = ( + layers.RawQuicLayer + if isinstance(context.layers[-1], layers.ClientQuicLayer) else + layers.UDPLayer + ) # 1. check for --ignore/--allow if self.ignore_connection( @@ -276,15 +317,25 @@ def s(*layers): if self.is_destination_in_hosts(context, self.udp_hosts): return raw_layer_cls(context) - # 5. Check for HTTP mode, but only on QUIC. - if ( - is_quic - and context.client.alpn - and (context.client.alpn == b"h3" or context.client.alpn.startswith(b"h3-")) - ): - return layers.HttpLayer(context, HTTPMode.transparent) + # 5. Check for raw reverse mode. + if self.is_reverse_proxy_scheme(context, "udp", "dtls"): + return layers.UDPLayer(context) + + # 6. Check for explicit QUIC reverse modes + if (s(modes.ReverseProxy, layers.ServerQuicLayer, layers.ClientQuicLayer)): + scheme = cast(mode_specs.ReverseMode, context.client.proxy_mode).scheme + if scheme == "quic": + return layers.RawQuicLayer(context) + if scheme == "http3": + return layers.HttpLayer(context, HTTPMode.transparent) + # 6b. ... or DNS mode + if self.is_reverse_proxy_scheme(context, "dns"): + return layers.DNSLayer(context) + # NOTE at this point we are either + # - at the top level of a non-reverse/regular/upstream mode + # - at a deeper layer nesting level - # 6. Check for DNS + # 7. Check for DNS try: dns.Message.unpack(data_client) except struct.error: @@ -292,7 +343,7 @@ def s(*layers): else: return layers.DNSLayer(context) - # 7. Use raw mode. + # 8. Use raw mode. return raw_layer_cls(context) else: diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 66dc3a98e9..aa7c35e4b8 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -36,13 +36,9 @@ graphql, grpc, mqtt, + http3, ) -try: - from . import http3 -except ImportError: - # FIXME: Remove once QUIC is merged. - http3 = None # type: ignore from .base import View, KEY_MAX, format_text, format_dict, TViewResult from ..http import HTTPFlow from ..tcp import TCPMessage, TCPFlow @@ -256,8 +252,7 @@ def get_content_view( add(msgpack.ViewMsgPack()) add(grpc.ViewGrpcProtobuf()) add(mqtt.ViewMQTT()) -if http3 is not None: - add(http3.ViewHttp3()) +add(http3.ViewHttp3()) __all__ = [ "View", diff --git a/mitmproxy/proxy/layers/__init__.py b/mitmproxy/proxy/layers/__init__.py index 97ae8fa47c..349c32cfcd 100644 --- a/mitmproxy/proxy/layers/__init__.py +++ b/mitmproxy/proxy/layers/__init__.py @@ -1,7 +1,7 @@ from . import modes from .dns import DNSLayer from .http import HttpLayer -from .quic import RawQuicLayer, ClientQuicLayer, ServerQuicLayer +from .quic import QuicStreamLayer, RawQuicLayer, ClientQuicLayer, ServerQuicLayer from .tcp import TCPLayer from .udp import UDPLayer from .tls import ClientTLSLayer, ServerTLSLayer @@ -11,6 +11,7 @@ "modes", "DNSLayer", "HttpLayer", + "QuicStreamLayer", "RawQuicLayer", "TCPLayer", "UDPLayer", diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 650d085b1a..f61cbb8a81 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -163,6 +163,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): + self._handle_event = self.done # type: ignore close_event = get_connection_error(self.conn) msg = ( "peer closed connection" @@ -171,7 +172,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) for stream_id in self.h3_conn.get_reserved_stream_ids(): yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) - self._handle_event = self.done # type: ignore else: raise AssertionError(f"Unexpected event: {event!r}") diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index c3724b8427..4dd397a660 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -511,28 +511,28 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer ): # get the target connection and stream ID to_client = command.connection is child_layer.client - conn = self.context.client if to_client else self.context.server + quic_conn = self.context.client if to_client else self.context.server stream_id = child_layer.client_stream_id if to_client else child_layer.server_stream_id # write data and check CloseConnection wasn't called before if isinstance(command, commands.SendData): assert stream_id is not None - assert conn.state & connection.ConnectionState.CAN_WRITE - yield SendQuicStreamData(conn, stream_id, command.data) + if command.connection.state & connection.ConnectionState.CAN_WRITE: + yield SendQuicStreamData(quic_conn, stream_id, command.data) # send a FIN and optionally also a STOP frame elif isinstance(command, commands.CloseConnection): assert stream_id is not None - if conn.state & connection.ConnectionState.CAN_WRITE: - conn.state &= ~connection.ConnectionState.CAN_WRITE - yield SendQuicStreamData(conn, stream_id, b"", end_stream=True) + if command.connection.state & connection.ConnectionState.CAN_WRITE: + command.connection.state &= ~connection.ConnectionState.CAN_WRITE + yield SendQuicStreamData(quic_conn, stream_id, b"", end_stream=True) if not command.half_close: if ( stream_is_client_initiated(stream_id) == to_client or not stream_is_unidirectional(stream_id) ): - yield StopQuicStream(conn, stream_id, QuicErrorCode.NO_ERROR) - yield from self.close_stream_layer(child_layer, conn) + yield StopQuicStream(quic_conn, stream_id, QuicErrorCode.NO_ERROR) + yield from self.close_stream_layer(child_layer, command.connection) # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): From 4cfd50bd830663b41f5f011bab40882256f3c74c Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 26 Sep 2022 03:18:28 +0200 Subject: [PATCH 072/695] [quic] minor changes --- mitmproxy/proxy/layers/quic.py | 97 +++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 36 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 4dd397a660..d6c12967b5 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -224,18 +224,6 @@ def is_success_error_code(error_code: int) -> bool: return error_code in (QuicErrorCode.NO_ERROR, H3ErrorCode.H3_NO_ERROR) -def get_stream_connection_state(stream_id: int, is_client: bool) -> connection.ConnectionState: - """Returns the initial connection state of a stream.""" - - state = connection.ConnectionState.OPEN - if stream_is_unidirectional(stream_id): - if stream_is_client_initiated(stream_id) == is_client: - state &= ~connection.ConnectionState.CAN_READ - else: - state &= ~connection.ConnectionState.CAN_WRITE - return state - - def set_connection_error(conn: connection.Connection, close_event: quic_events.ConnectionTerminated) -> None: """Stores the given close event for the given connection.""" @@ -308,14 +296,14 @@ class QuicStreamLayer(layer.Layer): """ client: connection.Client - client_stream_id: int + """Virtual client connection for this stream. Use this in QuicRawLayer instead of `context.client`.""" server: connection.Server - server_stream_id: int | None + """Virtual server connection for this stream. Use this in QuicRawLayer instead of `context.server`.""" child_layer: layer.Layer + """The stream's child layer.""" - def __init__(self, context: context.Context, ignore: bool, client_stream_id: int, server_stream_id: int | None) -> None: - # We mustn't reuse client or server from the QUIC connection as we have different states here. - # Also, we store the original values to detect if the context has changed. + def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> None: + # we mustn't reuse the client from the QUIC connection, as the state and protocol differs self.client = context.client = connection.Client( peername=context.client.peername, sockname=context.client.sockname, @@ -323,21 +311,52 @@ def __init__(self, context: context.Context, ignore: bool, client_stream_id: int transport_protocol="tcp", proxy_mode=context.client.proxy_mode, ) + + # unidirectional client streams are not fully open, set the appropriate state + if stream_is_unidirectional(stream_id): + self.client.state = ( + connection.ConnectionState.CAN_READ + if stream_is_client_initiated(stream_id) else + connection.ConnectionState.CAN_WRITE + ) + self._client_stream_id = stream_id + + # start with a closed server self.server = context.server = connection.Server( address=context.server.address, transport_protocol="tcp", ) + self._server_stream_id: int | None = None + + # ignored connections will be assigned a TCPLayer immediately super().__init__(context) - self.client_stream_id = client_stream_id - self.server_stream_id = server_stream_id self.child_layer = ( TCPLayer(context, ignore=True) if ignore else layer.NextLayer(context) ) + self.handle_event = self.child_layer.handle_event # type: ignore + self._handle_event = self.child_layer._handle_event # type: ignore def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - yield from self.child_layer.handle_event(event) + pass + + def open_server_stream(self, server_stream_id) -> None: + assert self._server_stream_id is None + self._server_stream_id = server_stream_id + self.server.timestamp_start = time.time() + self.server.state = ( + ( + connection.ConnectionState.CAN_WRITE + if stream_is_client_initiated(server_stream_id) else + connection.ConnectionState.CAN_READ + ) + if stream_is_unidirectional(server_stream_id) else + connection.ConnectionState.OPEN + ) + + def stream_id(self, client: bool) -> int | None: + return self._client_stream_id if client else self._server_stream_id class RawQuicLayer(layer.Layer): @@ -385,6 +404,11 @@ def __init__(self, context: context.Context, ignore: bool = False) -> None: def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # we treat the datagram layer as child layer, so forward Start if isinstance(event, events.Start): + if self.context.server.timestamp_start is None: + err = yield commands.OpenConnection(self.context.server) + if err: + yield commands.CloseConnection(self.context.client) + return yield from self.event_to_child(self.datagram_layer, event) # properly forward completion events based on their command @@ -430,12 +454,10 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: server_stream_id = event.stream_id # create, register and start the layer - stream_layer = QuicStreamLayer(self.context.fork(), self.ignore, client_stream_id, server_stream_id) - stream_layer.client.state = get_stream_connection_state(client_stream_id, is_client=False) + stream_layer = QuicStreamLayer(self.context.fork(), self.ignore, client_stream_id) self.client_stream_ids[client_stream_id] = stream_layer if server_stream_id is not None: - stream_layer.server.timestamp_start = time.time() - stream_layer.server.state = get_stream_connection_state(server_stream_id, is_client=True) + stream_layer.open_server_stream(server_stream_id) self.server_stream_ids[server_stream_id] = stream_layer self.connections[stream_layer.client] = stream_layer self.connections[stream_layer.server] = stream_layer @@ -446,13 +468,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, QuicStreamDataReceived): yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) if event.end_stream: - yield from self.close_stream_layer(stream_layer, conn) + yield from self.close_stream_layer(stream_layer, from_client) elif isinstance(event, QuicStreamReset): if self.debug is not None: yield commands.Log( f"{self.debug}[quic] stream_reset (stream_id={event.stream_id}, error_code={event.error_code})", DEBUG ) - yield from self.close_stream_layer(stream_layer, conn) + yield from self.close_stream_layer(stream_layer, from_client) else: raise AssertionError(f"Unexpected stream event: {event!r}") @@ -480,7 +502,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: and ((conn is child_layer.client) if from_client else (conn is child_layer.server)) ): conn.state &= ~connection.ConnectionState.CAN_WRITE - yield from self.close_stream_layer(child_layer, conn) + yield from self.close_stream_layer(child_layer, from_client) # all other connection events are routed to their corresponding layer elif isinstance(event, events.ConnectionEvent): @@ -489,11 +511,14 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unexpected event: {event!r}") - def close_stream_layer(self, stream_layer: QuicStreamLayer, conn: connection.Connection) -> layer.CommandGenerator[None]: + def close_stream_layer(self, stream_layer: QuicStreamLayer, client: bool) -> layer.CommandGenerator[None]: """Closes the incoming part of a connection.""" - if conn.state & connection.ConnectionState.CAN_READ: - conn.state &= ~connection.ConnectionState.CAN_READ + conn = stream_layer.client if client else stream_layer.server + conn.state &= ~connection.ConnectionState.CAN_READ + assert conn.timestamp_start is not None + if conn.timestamp_end is None: + conn.timestamp_end = time.time() yield from self.event_to_child(stream_layer, events.ConnectionClosed(conn)) def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: @@ -512,7 +537,7 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer # get the target connection and stream ID to_client = command.connection is child_layer.client quic_conn = self.context.client if to_client else self.context.server - stream_id = child_layer.client_stream_id if to_client else child_layer.server_stream_id + stream_id = child_layer.stream_id(to_client) # write data and check CloseConnection wasn't called before if isinstance(command, commands.SendData): @@ -532,19 +557,19 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer or not stream_is_unidirectional(stream_id) ): yield StopQuicStream(quic_conn, stream_id, QuicErrorCode.NO_ERROR) - yield from self.close_stream_layer(child_layer, command.connection) + yield from self.close_stream_layer(child_layer, to_client) # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): assert not to_client assert stream_id is None - child_layer.server.timestamp_start = time.time() + client_stream_id = child_layer.stream_id(client=True) + assert client_stream_id is not None stream_id = self.get_next_available_stream_id( is_client=True, - is_unidirectional=stream_is_unidirectional(child_layer.client_stream_id) + is_unidirectional=stream_is_unidirectional(client_stream_id) ) - child_layer.server_stream_id = stream_id - child_layer.server.state = get_stream_connection_state(stream_id, is_client=True) + child_layer.open_server_stream(stream_id) self.server_stream_ids[stream_id] = child_layer yield from self.event_to_child(child_layer, events.OpenConnectionCompleted(command, None)) From a0015d469585c5696698bbf56ef291f591903416 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 26 Sep 2022 04:20:00 +0200 Subject: [PATCH 073/695] [quic] fix re-entrance issue --- mitmproxy/proxy/layers/quic.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index d6c12967b5..2617c5b7ee 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -466,7 +466,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # forward data and close events conn = stream_layer.client if from_client else stream_layer.server if isinstance(event, QuicStreamDataReceived): - yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) + if event.data: + yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) if event.end_stream: yield from self.close_stream_layer(stream_layer, from_client) elif isinstance(event, QuicStreamReset): @@ -524,6 +525,8 @@ def close_stream_layer(self, stream_layer: QuicStreamLayer, client: bool) -> lay def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: """Forwards events to child layers and translates commands.""" + close_client = False + close_server = False for command in child_layer.handle_event(event): # intercept commands for streams connections if ( @@ -557,7 +560,10 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer or not stream_is_unidirectional(stream_id) ): yield StopQuicStream(quic_conn, stream_id, QuicErrorCode.NO_ERROR) - yield from self.close_stream_layer(child_layer, to_client) + if to_client: + close_client = True + else: + close_server = True # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): @@ -584,6 +590,12 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer self.connections[command.connection] = child_layer yield command + # we have to exhaust the previous generator to prevent re-entrance + if close_client: + yield from self.close_stream_layer(child_layer, client=True) + if close_server: + yield from self.close_stream_layer(child_layer, client=False) + def get_next_available_stream_id(self, is_client: bool, is_unidirectional: bool = False) -> int: index = (int(is_unidirectional) << 1) | int(not is_client) stream_id = self.next_stream_id[index] From 22a68b6d86840b3ed74c2b0b123a736e6257dd68 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 26 Sep 2022 04:31:35 +0200 Subject: [PATCH 074/695] [quic] fix H3 over raw QUIC --- mitmproxy/proxy/layers/quic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 2617c5b7ee..ba817ae502 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -689,6 +689,7 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C cipher_suites=tls_data.settings.cipher_suites, private_key=tls_data.settings.certificate_private_key, verify_mode=tls_data.settings.verify_mode, + max_datagram_frame_size=65536, ) self.quic = QuicConnection( configuration=configuration, From 995ff0207cb3dd8ebced8f9fc7b8772d662966d9 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 26 Sep 2022 10:40:39 +0200 Subject: [PATCH 075/695] [quic] alternative fix for re-entrance --- mitmproxy/proxy/layers/quic.py | 13 +------------ mitmproxy/proxy/layers/tcp.py | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index ba817ae502..6668442e93 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -525,8 +525,6 @@ def close_stream_layer(self, stream_layer: QuicStreamLayer, client: bool) -> lay def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: """Forwards events to child layers and translates commands.""" - close_client = False - close_server = False for command in child_layer.handle_event(event): # intercept commands for streams connections if ( @@ -560,10 +558,7 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer or not stream_is_unidirectional(stream_id) ): yield StopQuicStream(quic_conn, stream_id, QuicErrorCode.NO_ERROR) - if to_client: - close_client = True - else: - close_server = True + yield from self.close_stream_layer(child_layer, to_client) # open server connections by reserving the next stream ID elif isinstance(command, commands.OpenConnection): @@ -590,12 +585,6 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer self.connections[command.connection] = child_layer yield command - # we have to exhaust the previous generator to prevent re-entrance - if close_client: - yield from self.close_stream_layer(child_layer, client=True) - if close_server: - yield from self.close_stream_layer(child_layer, client=False) - def get_next_available_stream_id(self, is_client: bool, is_unidirectional: bool = False) -> int: index = (int(is_unidirectional) << 1) | int(not is_client) stream_id = self.next_stream_id[index] diff --git a/mitmproxy/proxy/layers/tcp.py b/mitmproxy/proxy/layers/tcp.py index 296adb80b3..d3c9f9bd69 100644 --- a/mitmproxy/proxy/layers/tcp.py +++ b/mitmproxy/proxy/layers/tcp.py @@ -123,11 +123,11 @@ def relay_messages(self, event: events.Event) -> layer.CommandGenerator[None]: or (self.context.server.state & ConnectionState.CAN_READ) ) if all_done: + self._handle_event = self.done if self.context.server.state is not ConnectionState.CLOSED: yield commands.CloseConnection(self.context.server) if self.context.client.state is not ConnectionState.CLOSED: yield commands.CloseConnection(self.context.client) - self._handle_event = self.done if self.flow: yield TcpEndHook(self.flow) self.flow.live = False From 0d59b57791d4ab242d2d7be68e419bee46233694 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 13 Oct 2022 17:37:13 +0200 Subject: [PATCH 076/695] nits --- mitmproxy/proxy/layers/http/__init__.py | 5 ++++- mitmproxy/proxy/layers/modes.py | 11 ++++++----- mitmproxy/proxy/layers/quic.py | 12 ++++++++---- mitmproxy/tools/main.py | 1 + web/src/js/ducks/_options_gen.ts | 2 -- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index d30b08f443..3cf117e017 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -45,6 +45,7 @@ from ._http1 import Http1Client, Http1Connection, Http1Server from ._http2 import Http2Client, Http2Server from ._http3 import Http3Client, Http3Server +from ..quic import QuicStreamEvent from ...context import Context from ...mode_specs import ReverseMode, UpstreamMode @@ -886,7 +887,7 @@ def _handle_event(self, event: events.Event): if isinstance(event, events.ConnectionClosed): # The peer has closed it - let's close it too! yield commands.CloseConnection(event.connection) - else: + elif isinstance(event, (events.DataReceived, QuicStreamEvent)): # The peer has sent data or another connection activity occurred. # This can happen with HTTP/2 servers that already send a settings frame. child_layer: HttpConnection @@ -899,6 +900,8 @@ def _handle_event(self, event: events.Event): self.connections[self.context.server] = child_layer yield from self.event_to_child(child_layer, events.Start()) yield from self.event_to_child(child_layer, event) + else: + raise AssertionError(f"Unexpected event: {event}") else: handler = self.connections[event.connection] yield from self.event_to_child(handler, event) diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 3b3b7d53e3..1a58d10dfc 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -60,13 +60,14 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert isinstance(spec, ReverseMode) self.context.server.address = spec.address - if spec.scheme in ("https", "http3", "quic", "tls", "dtls"): + if spec.scheme in ("http3", "quic"): if not self.context.options.keep_host_header: self.context.server.sni = spec.address[0] - if spec.scheme == "http3" or spec.scheme == "quic": - self.child_layer = quic.ServerQuicLayer(self.context) - else: - self.child_layer = tls.ServerTLSLayer(self.context) + self.child_layer = quic.ServerQuicLayer(self.context) + elif spec.scheme in ("https", "tls", "dtls"): + if not self.context.options.keep_host_header: + self.context.server.sni = spec.address[0] + self.child_layer = tls.ServerTLSLayer(self.context) elif spec.scheme == "udp": self.child_layer = udp.UDPLayer(self.context) elif spec.scheme == "http" or spec.scheme == "tcp": diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 6668442e93..979a907c58 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -605,21 +605,24 @@ def __init__(self, context: context.Context, conn: connection.Connection) -> Non conn.tls = True def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - # turn Wakeup events into empty DataReceived events if ( isinstance(event, events.Wakeup) and event.command in self._wakeup_commands ): + # TunnelLayer has no understanding of wakeups, so we turn this into an empty DataReceived event + # which TunnelLayer recognizes as belonging to our connection. assert self.quic timer = self._wakeup_commands.pop(event.command) if self.quic._state is not QuicConnectionState.TERMINATED: self.quic.handle_timer(now=max(timer, self._loop.time())) - event = events.DataReceived(self.tunnel_connection, b"") - yield from super()._handle_event(event) + yield from super()._handle_event( + events.DataReceived(self.tunnel_connection, b"") + ) + else: + yield from super()._handle_event(event) def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[None]: """Turns stream commands into aioquic connection invocations.""" - if ( isinstance(command, QuicStreamCommand) and command.connection is self.conn @@ -801,6 +804,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: quic_events.ConnectionIdIssued, quic_events.ConnectionIdRetired, quic_events.PingAcknowledged, + quic_events.ProtocolNegotiated, )): pass else: diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 749356a632..239689b4a3 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -54,6 +54,7 @@ async def main() -> T: logging.getLogger("tornado").setLevel(logging.WARNING) logging.getLogger("asyncio").setLevel(logging.WARNING) logging.getLogger("hpack").setLevel(logging.WARNING) + logging.getLogger("quic").setLevel(logging.WARNING) # aioquic uses a different prefix... debug.register_info_dumpers() opts = options.Options() diff --git a/web/src/js/ducks/_options_gen.ts b/web/src/js/ducks/_options_gen.ts index 64f821065b..32f7ae1bdc 100644 --- a/web/src/js/ducks/_options_gen.ts +++ b/web/src/js/ducks/_options_gen.ts @@ -43,7 +43,6 @@ export interface OptionsState { proxy_debug: boolean proxyauth: string | undefined rawtcp: boolean - rawudp: boolean readfile_filter: string | undefined rfile: string | undefined save_stream_file: string | undefined @@ -135,7 +134,6 @@ export const defaultState: OptionsState = { proxy_debug: false, proxyauth: undefined, rawtcp: true, - rawudp: true, readfile_filter: undefined, rfile: undefined, save_stream_file: undefined, From 0812b36f20ad35921760ea24d212d51e0d7fec47 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 13 Oct 2022 17:51:21 +0200 Subject: [PATCH 077/695] experimental: move `half_close` out of `CloseConnection` --- mitmproxy/proxy/commands.py | 2 ++ mitmproxy/proxy/layers/http/__init__.py | 3 ++- mitmproxy/proxy/layers/http/_http1.py | 2 +- mitmproxy/proxy/layers/quic.py | 4 ++-- mitmproxy/proxy/layers/tcp.py | 2 +- mitmproxy/proxy/layers/tls.py | 4 ++-- mitmproxy/proxy/server.py | 4 +++- mitmproxy/proxy/tunnel.py | 12 +++++------- test/mitmproxy/proxy/layers/test_tcp.py | 6 +++--- test/mitmproxy/proxy/test_tunnel.py | 6 +++--- test/mitmproxy/proxy/tutils.py | 9 ++++----- 11 files changed, 28 insertions(+), 26 deletions(-) diff --git a/mitmproxy/proxy/commands.py b/mitmproxy/proxy/commands.py index d4173ebc9b..e5c3627f30 100644 --- a/mitmproxy/proxy/commands.py +++ b/mitmproxy/proxy/commands.py @@ -94,6 +94,8 @@ class CloseConnection(ConnectionCommand): all other connections will ultimately be closed during cleanup. """ + +class CloseTcpConnection(CloseConnection): half_close: bool """ If True, only close our half of the connection by sending a FIN packet. diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 3cf117e017..4f9cd26acd 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -791,7 +791,8 @@ def passthrough(self, event: events.Event) -> layer.CommandGenerator[None]: # The easiest approach for this is to just always full close for now. # Alternatively, we could signal that we want a half close only through ResponseProtocolError, # but that is more complex to implement. - command.half_close = False + if isinstance(command, commands.CloseTcpConnection): + command = commands.CloseConnection(command.connection) yield command else: yield command diff --git a/mitmproxy/proxy/layers/http/_http1.py b/mitmproxy/proxy/layers/http/_http1.py index 2cf410fb9a..e645b54695 100644 --- a/mitmproxy/proxy/layers/http/_http1.py +++ b/mitmproxy/proxy/layers/http/_http1.py @@ -369,7 +369,7 @@ def send(self, event: HttpEvent) -> layer.CommandGenerator[None]: if "chunked" in self.request.headers.get("transfer-encoding", "").lower(): yield commands.SendData(self.conn, b"0\r\n\r\n") elif http1.expected_http_body_size(self.request, self.response) == -1: - yield commands.CloseConnection(self.conn, half_close=True) + yield commands.CloseTcpConnection(self.conn, half_close=True) yield from self.mark_done(request=True) else: raise AssertionError(f"Unexpected event: {event}") diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 979a907c58..6e3393f4a6 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -824,7 +824,7 @@ def send_data(self, data: bytes) -> layer.CommandGenerator[None]: self.quic.send_datagram_frame(data) yield from self.tls_interact() - def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: + def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: # properly close the QUIC connection if self.quic is not None: close_event = get_connection_error(self.conn) @@ -833,7 +833,7 @@ def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: else: self.quic.close(close_event.error_code, close_event.frame_type, close_event.reason_phrase) yield from self.tls_interact() - yield from super().send_close(half_close) + yield from super().send_close(command) class ServerQuicLayer(QuicLayer): diff --git a/mitmproxy/proxy/layers/tcp.py b/mitmproxy/proxy/layers/tcp.py index d3c9f9bd69..2d1ff7305b 100644 --- a/mitmproxy/proxy/layers/tcp.py +++ b/mitmproxy/proxy/layers/tcp.py @@ -132,7 +132,7 @@ def relay_messages(self, event: events.Event) -> layer.CommandGenerator[None]: yield TcpEndHook(self.flow) self.flow.live = False else: - yield commands.CloseConnection(send_to, half_close=True) + yield commands.CloseTcpConnection(send_to, half_close=True) else: raise AssertionError(f"Unexpected event: {event}") diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index ab63d51d35..d79cbcfc69 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -440,9 +440,9 @@ def send_data(self, data: bytes) -> layer.CommandGenerator[None]: pass yield from self.tls_interact() - def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: + def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: # We should probably shutdown the TLS connection properly here. - yield from super().send_close(half_close) + yield from super().send_close(command) class ServerTLSLayer(TLSLayer): diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 1341564ada..ca83be9ac0 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -369,8 +369,10 @@ def server_event(self, event: events.Event) -> None: assert writer if not writer.is_closing(): writer.write(command.data) - elif isinstance(command, commands.CloseConnection): + elif isinstance(command, commands.CloseTcpConnection): self.close_connection(command.connection, command.half_close) + elif isinstance(command, commands.CloseConnection): + self.close_connection(command.connection, False) elif isinstance(command, commands.StartHook): asyncio_utils.create_task( self.hook_task(command), diff --git a/mitmproxy/proxy/tunnel.py b/mitmproxy/proxy/tunnel.py index 7a481a7014..435693de6a 100644 --- a/mitmproxy/proxy/tunnel.py +++ b/mitmproxy/proxy/tunnel.py @@ -117,11 +117,9 @@ def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[N yield from self.send_data(command.data) elif isinstance(command, commands.CloseConnection): if self.conn != self.tunnel_connection: - if command.half_close: - self.conn.state &= ~connection.ConnectionState.CAN_WRITE - else: - self.conn.state = connection.ConnectionState.CLOSED - yield from self.send_close(command.half_close) + self.conn.state &= ~connection.ConnectionState.CAN_WRITE + command.connection = self.tunnel_connection + yield from self.send_close(command) elif isinstance(command, commands.OpenConnection): # create our own OpenConnection command object that blocks here. self.command_to_reply_to = command @@ -172,8 +170,8 @@ def receive_close(self) -> layer.CommandGenerator[None]: def send_data(self, data: bytes) -> layer.CommandGenerator[None]: yield commands.SendData(self.tunnel_connection, data) - def send_close(self, half_close: bool) -> layer.CommandGenerator[None]: - yield commands.CloseConnection(self.tunnel_connection, half_close=half_close) + def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: + yield command class LayerStack: diff --git a/test/mitmproxy/proxy/layers/test_tcp.py b/test/mitmproxy/proxy/layers/test_tcp.py index d4f4947b09..df01fa9f98 100644 --- a/test/mitmproxy/proxy/layers/test_tcp.py +++ b/test/mitmproxy/proxy/layers/test_tcp.py @@ -1,6 +1,6 @@ import pytest -from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData +from mitmproxy.proxy.commands import CloseConnection, CloseTcpConnection, OpenConnection, SendData from mitmproxy.proxy.events import ConnectionClosed, DataReceived from mitmproxy.proxy.layers import tcp from mitmproxy.proxy.layers.tcp import TcpMessageInjected @@ -52,7 +52,7 @@ def test_simple(tctx): >> reply() << SendData(tctx.client, b"hi") >> ConnectionClosed(tctx.server) - << CloseConnection(tctx.client, half_close=True) + << CloseTcpConnection(tctx.client, half_close=True) >> ConnectionClosed(tctx.client) << CloseConnection(tctx.server) << tcp.TcpEndHook(f) @@ -88,7 +88,7 @@ def test_receive_data_after_half_close(tctx): >> DataReceived(tctx.client, b"eof-delimited-request") << SendData(tctx.server, b"eof-delimited-request") >> ConnectionClosed(tctx.client) - << CloseConnection(tctx.server, half_close=True) + << CloseTcpConnection(tctx.server, half_close=True) >> DataReceived(tctx.server, b"i'm late") << SendData(tctx.client, b"i'm late") >> ConnectionClosed(tctx.server) diff --git a/test/mitmproxy/proxy/test_tunnel.py b/test/mitmproxy/proxy/test_tunnel.py index 24103637c6..004c7ab69f 100644 --- a/test/mitmproxy/proxy/test_tunnel.py +++ b/test/mitmproxy/proxy/test_tunnel.py @@ -3,7 +3,7 @@ import pytest from mitmproxy.proxy import tunnel, layer -from mitmproxy.proxy.commands import SendData, Log, CloseConnection, OpenConnection +from mitmproxy.proxy.commands import CloseTcpConnection, SendData, Log, CloseConnection, OpenConnection from mitmproxy.connection import Server, ConnectionState from mitmproxy.proxy.context import Context from mitmproxy.proxy.events import Event, DataReceived, Start, ConnectionClosed @@ -24,7 +24,7 @@ def _handle_event(self, event: Event) -> layer.CommandGenerator[None]: err = yield OpenConnection(self.context.server) yield Log(f"Opened: {err=}. Server state: {self.context.server.state.name}") elif isinstance(event, DataReceived) and event.data == b"half-close": - err = yield CloseConnection(event.connection, half_close=True) + err = yield CloseTcpConnection(event.connection, half_close=True) elif isinstance(event, ConnectionClosed): yield Log(f"Got {event.connection.__class__.__name__.lower()} close.") yield CloseConnection(event.connection) @@ -164,7 +164,7 @@ def test_tunnel_default_impls(tctx: Context): >> reply(None) << Log("Opened: err=None. Server state: OPEN") >> DataReceived(server, b"half-close") - << CloseConnection(server, half_close=True) + << CloseTcpConnection(server, half_close=True) ) diff --git a/test/mitmproxy/proxy/tutils.py b/test/mitmproxy/proxy/tutils.py index 61b7489b1a..087db71316 100644 --- a/test/mitmproxy/proxy/tutils.py +++ b/test/mitmproxy/proxy/tutils.py @@ -224,11 +224,10 @@ def __bool__(self): for cmd in cmds: pos += 1 assert self.actual[pos] == cmd - if isinstance(cmd, commands.CloseConnection): - if cmd.half_close: - cmd.connection.state &= ~ConnectionState.CAN_WRITE - else: - cmd.connection.state = ConnectionState.CLOSED + if isinstance(cmd, commands.CloseTcpConnection) and cmd.half_close: + cmd.connection.state &= ~ConnectionState.CAN_WRITE + elif isinstance(cmd, commands.CloseConnection): + cmd.connection.state = ConnectionState.CLOSED elif isinstance(cmd, commands.Log): need_to_emulate_log = ( not self.logs From eb494033c10d2c266dd3ab38cd1be424c083fb0c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 13 Oct 2022 18:36:48 +0200 Subject: [PATCH 078/695] `get/set_connection_error` -> event/command subclasses --- mitmproxy/proxy/layers/http/_http3.py | 12 ++-- mitmproxy/proxy/layers/http/_http_h3.py | 13 ++-- mitmproxy/proxy/layers/quic.py | 86 +++++++++++++++++-------- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index f61cbb8a81..9c46d77f9b 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -12,9 +12,9 @@ from mitmproxy.net.http import status_codes from mitmproxy.proxy import commands, context, events, layer from mitmproxy.proxy.layers.quic import ( + QuicConnectionClosed, QuicStreamEvent, error_code_to_str, - get_connection_error, ) from mitmproxy.proxy.utils import expect @@ -164,12 +164,10 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): self._handle_event = self.done # type: ignore - close_event = get_connection_error(self.conn) - msg = ( - "peer closed connection" - if close_event is None else - close_event.reason_phrase or error_code_to_str(close_event.error_code) - ) + if isinstance(event, QuicConnectionClosed): + msg = event.reason_phrase or error_code_to_str(event.error_code) + else: + msg = "peer closed connection" for stream_id in self.h3_conn.get_reserved_stream_ids(): yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 56cd563060..78d82a40a2 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -17,12 +17,12 @@ from mitmproxy import connection from mitmproxy.proxy import commands, layer from mitmproxy.proxy.layers.quic import ( + CloseQuicConnection, QuicStreamDataReceived, QuicStreamEvent, QuicStreamReset, ResetQuicStream, SendQuicStreamData, - set_connection_error, ) @@ -87,13 +87,10 @@ def close( # we'll get closed if a protocol error occurs in `H3Connection.handle_event` # we note the error on the connection and yield a CloseConnection # this will then call `QuicConnection.close` with the proper values - # once the `Http3Connection` receives `ConnectionClosed`, it will send out `*ProtocolError` - set_connection_error(self.conn, ConnectionTerminated( - error_code=error_code, - frame_type=frame_type, - reason_phrase=reason_phrase, - )) - self.pending_commands.append(commands.CloseConnection(self.conn)) + # once the `Http3Connection` receives `ConnectionClosed`, it will send out `ProtocolError` + self.pending_commands.append( + CloseQuicConnection(self.conn, error_code, frame_type, reason_phrase) + ) def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: # since we always reserve the ID, we have to "find" the next ID like `QuicConnection` does diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 6e3393f4a6..fd2afd9f40 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -177,6 +177,56 @@ def __init__(self, connection: connection.Connection, stream_id: int, error_code self.error_code = error_code +class CloseQuicConnection(commands.CloseConnection): + """Close a QUIC connection.""" + + error_code: int + "The error code which was specified when closing the connection." + + frame_type: int | None + "The frame type which caused the connection to be closed, or `None`." + + reason_phrase: str + "The human-readable reason for which the connection was closed." + + # XXX: A bit much boilerplate right now. Should switch to dataclasses. + def __init__( + self, + conn: connection.Connection, + error_code: int, + frame_type: int | None, + reason_phrase: str, + ): + super().__init__(conn) + self.error_code = error_code + self.frame_type = frame_type + self.reason_phrase = reason_phrase + + +class QuicConnectionClosed(events.ConnectionClosed): + """QUIC connection has been closed.""" + error_code: int + "The error code which was specified when closing the connection." + + frame_type: int | None + "The frame type which caused the connection to be closed, or `None`." + + reason_phrase: str + "The human-readable reason for which the connection was closed." + + def __init__( + self, + conn: connection.Connection, + error_code: int, + frame_type: int | None, + reason_phrase: str, + ): + super().__init__(conn) + self.error_code = error_code + self.frame_type = frame_type + self.reason_phrase = reason_phrase + + class QuicSecretsLogger: logger: tls.MasterSecretLogger @@ -208,28 +258,12 @@ def error_code_to_str(error_code: int) -> str: return f"unknown error (0x{error_code:x})" -def get_connection_error(conn: connection.Connection) -> quic_events.ConnectionTerminated | None: - """Returns the QUIC close event that is associated with the given connection.""" - - close_event = getattr(conn, "quic_error", None) - if close_event is None: - return None - assert isinstance(close_event, quic_events.ConnectionTerminated) - return close_event - - def is_success_error_code(error_code: int) -> bool: """Returns whether the given error code actually indicates no error.""" return error_code in (QuicErrorCode.NO_ERROR, H3ErrorCode.H3_NO_ERROR) -def set_connection_error(conn: connection.Connection, close_event: quic_events.ConnectionTerminated) -> None: - """Stores the given close event for the given connection.""" - - setattr(conn, "quic_error", close_event) - - @dataclass class QuicClientHello(Exception): """Helper error only used in `quic_parse_client_hello`.""" @@ -481,17 +515,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # handle close events that target this context elif ( - isinstance(event, events.ConnectionClosed) + isinstance(event, QuicConnectionClosed) and ( event.connection is self.context.client or event.connection is self.context.server ) ): - # copy the connection error from_client = event.connection is self.context.client - close_event = get_connection_error(event.connection) - if close_event is not None: - set_connection_error(self.context.server if from_client else self.context.client, close_event) # always forward to the datagram layer yield from self.event_to_child(self.datagram_layer, event) @@ -552,7 +582,9 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer if command.connection.state & connection.ConnectionState.CAN_WRITE: command.connection.state &= ~connection.ConnectionState.CAN_WRITE yield SendQuicStreamData(quic_conn, stream_id, b"", end_stream=True) - if not command.half_close: + # XXX: Use `command.connection.state & connection.ConnectionState.CAN_READ` instead? + only_close_our_half = isinstance(command, commands.CloseTcpConnection) and command.half_close + if not only_close_our_half: if ( stream_is_client_initiated(stream_id) == to_client or not stream_is_unidirectional(stream_id) @@ -786,13 +818,12 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: # handle post-handshake events while event := self.quic.next_event(): if isinstance(event, quic_events.ConnectionTerminated): - set_connection_error(self.conn, event) if self.debug: reason = event.reason_phrase or error_code_to_str(event.error_code) yield commands.Log( f"{self.debug}[quic] close_notify {self.conn} (reason={reason})", DEBUG ) - yield commands.CloseConnection(self.conn) + yield CloseQuicConnection(self.conn, event.error_code, event.frame_type, event.reason_phrase) return # we don't handle any further events, nor do/can we transmit data, so exit elif isinstance(event, quic_events.DatagramFrameReceived): yield from self.event_to_child(events.DataReceived(self.conn, event.data)) @@ -827,11 +858,10 @@ def send_data(self, data: bytes) -> layer.CommandGenerator[None]: def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: # properly close the QUIC connection if self.quic is not None: - close_event = get_connection_error(self.conn) - if close_event is None: - self.quic.close() + if isinstance(command, CloseQuicConnection): + self.quic.close(command.error_code, command.frame_type, command.reason_phrase) else: - self.quic.close(close_event.error_code, close_event.frame_type, close_event.reason_phrase) + self.quic.close() yield from self.tls_interact() yield from super().send_close(command) From 7b1d18857604545b11753cf7c2ed3adee59bd2a2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 13 Oct 2022 20:07:41 +0200 Subject: [PATCH 079/695] add QUIC metadata to streams --- mitmproxy/proxy/layers/quic.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index fd2afd9f40..e48dd32865 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -333,7 +333,7 @@ class QuicStreamLayer(layer.Layer): """Virtual client connection for this stream. Use this in QuicRawLayer instead of `context.client`.""" server: connection.Server """Virtual server connection for this stream. Use this in QuicRawLayer instead of `context.server`.""" - child_layer: layer.Layer + child_layer: TCPLayer """The stream's child layer.""" def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> None: @@ -369,11 +369,21 @@ def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> No if ignore else layer.NextLayer(context) ) + if ignore: + self.child_layer = TCPLayer(context, ignore=True) + else: + tcp_layer = TCPLayer(context) + # This can potentially move to a smarter place later on, + # but it's useful debugging info in mitmproxy for now. + tcp_layer.flow.metadata["quic_is_unidirectional"] = stream_is_unidirectional(stream_id) + tcp_layer.flow.metadata["quic_initiator"] = "client" if stream_is_client_initiated(stream_id) else "server" + tcp_layer.flow.metadata["quic_stream_id_client"] = stream_id + self.child_layer = tcp_layer self.handle_event = self.child_layer.handle_event # type: ignore self._handle_event = self.child_layer._handle_event # type: ignore def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - pass + raise AssertionError def open_server_stream(self, server_stream_id) -> None: assert self._server_stream_id is None @@ -388,6 +398,8 @@ def open_server_stream(self, server_stream_id) -> None: if stream_is_unidirectional(server_stream_id) else connection.ConnectionState.OPEN ) + if self.child_layer.flow: + self.child_layer.flow.metadata["quic_stream_id_server"] = server_stream_id def stream_id(self, client: bool) -> int | None: return self._client_stream_id if client else self._server_stream_id From 519b91c9a2b254f5a932f5c92601d663dfeb1cc3 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 24 Oct 2022 03:47:20 +0200 Subject: [PATCH 080/695] [quic] assign metadata with NextLayer support --- mitmproxy/contentviews/http3.py | 6 +-- mitmproxy/proxy/layers/http/_http_h3.py | 2 +- mitmproxy/proxy/layers/quic.py | 67 +++++++++++++++++-------- 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/mitmproxy/contentviews/http3.py b/mitmproxy/contentviews/http3.py index 97f34ca68e..4861694f08 100644 --- a/mitmproxy/contentviews/http3.py +++ b/mitmproxy/contentviews/http3.py @@ -1,7 +1,7 @@ from collections import defaultdict from collections.abc import Iterator from dataclasses import dataclass, field -from typing import Optional +from typing import Optional, Union from aioquic.h3.connection import Setting, parse_settings @@ -76,7 +76,7 @@ def pretty(self): @dataclass class ConnectionState: message_count: int = 0 - frames: dict[int, list[Frame | StreamType]] = field(default_factory=dict) + frames: dict[int, list[Union[Frame, StreamType]]] = field(default_factory=dict) client_buf: bytearray = field(default_factory=bytearray) server_buf: bytearray = field(default_factory=bytearray) @@ -154,7 +154,7 @@ def render_priority( return 2 * float(bool(flow and is_h3_alpn(flow.client_conn.alpn))) * float(isinstance(flow, tcp.TCPFlow)) -def fmt_frames(frames: list[Frame | StreamType]) -> Iterator[base.TViewLine]: +def fmt_frames(frames: list[Union[Frame, StreamType]]) -> Iterator[base.TViewLine]: for i, frame in enumerate(frames): if i > 0: yield [("text", "")] diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 78d82a40a2..d70a441633 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -11,7 +11,7 @@ ) from aioquic.h3.events import HeadersReceived from aioquic.quic.configuration import QuicConfiguration -from aioquic.quic.events import ConnectionTerminated, StreamDataReceived +from aioquic.quic.events import StreamDataReceived from aioquic.quic.packet import QuicErrorCode from mitmproxy import connection diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index e48dd32865..3841a60425 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -27,7 +27,7 @@ ) from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection +from mitmproxy import certs, connection, flow from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, tunnel from mitmproxy.proxy.layers.tcp import TCPLayer @@ -136,7 +136,7 @@ class QuicStreamCommand(commands.ConnectionCommand): stream_id: int """The ID of the stream the command was issued for.""" - def __init__(self, connection: connection.Connection, stream_id: int): + def __init__(self, connection: connection.Connection, stream_id: int) -> None: super().__init__(connection) self.stream_id = stream_id @@ -149,7 +149,7 @@ class SendQuicStreamData(QuicStreamCommand): end_stream: bool """Whether the FIN bit should be set in the STREAM frame.""" - def __init__(self, connection: connection.Connection, stream_id: int, data: bytes, end_stream: bool = False): + def __init__(self, connection: connection.Connection, stream_id: int, data: bytes, end_stream: bool = False) -> None: super().__init__(connection, stream_id) self.data = data self.end_stream = end_stream @@ -161,7 +161,7 @@ class ResetQuicStream(QuicStreamCommand): error_code: int """An error code indicating why the stream is being reset.""" - def __init__(self, connection: connection.Connection, stream_id: int, error_code: int): + def __init__(self, connection: connection.Connection, stream_id: int, error_code: int) -> None: super().__init__(connection, stream_id) self.error_code = error_code @@ -172,7 +172,7 @@ class StopQuicStream(QuicStreamCommand): error_code: int """An error code indicating why the stream is being stopped.""" - def __init__(self, connection: connection.Connection, stream_id: int, error_code: int): + def __init__(self, connection: connection.Connection, stream_id: int, error_code: int) -> None: super().__init__(connection, stream_id) self.error_code = error_code @@ -196,7 +196,7 @@ def __init__( error_code: int, frame_type: int | None, reason_phrase: str, - ): + ) -> None: super().__init__(conn) self.error_code = error_code self.frame_type = frame_type @@ -220,7 +220,7 @@ def __init__( error_code: int, frame_type: int | None, reason_phrase: str, - ): + ) -> None: super().__init__(conn) self.error_code = error_code self.frame_type = frame_type @@ -323,6 +323,25 @@ def initialize_replacement(peer_cid: bytes) -> None: raise ValueError("No ClientHello returned.") +class QuicStreamNextLayer(layer.NextLayer): + """`NextLayer` variant that callbacks `QuicStreamLayer` after layer decision.""" + + def __init__(self, context: context.Context, stream: QuicStreamLayer, ask_on_start: bool = False) -> None: + super().__init__(context, ask_on_start) + self._stream = stream + self._layer: layer.Layer | None = None + + @property + def layer(self) -> layer.Layer | None: + return self._layer + + @layer.setter + def layer(self, value: layer.Layer | None) -> None: + self._layer = value + if self._layer: + self._stream.refresh_metadata() + + class QuicStreamLayer(layer.Layer): """ Layer for QUIC streams. @@ -333,7 +352,7 @@ class QuicStreamLayer(layer.Layer): """Virtual client connection for this stream. Use this in QuicRawLayer instead of `context.client`.""" server: connection.Server """Virtual server connection for this stream. Use this in QuicRawLayer instead of `context.server`.""" - child_layer: TCPLayer + child_layer: layer.Layer """The stream's child layer.""" def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> None: @@ -367,18 +386,11 @@ def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> No self.child_layer = ( TCPLayer(context, ignore=True) if ignore else - layer.NextLayer(context) + QuicStreamNextLayer(context, self) ) - if ignore: - self.child_layer = TCPLayer(context, ignore=True) - else: - tcp_layer = TCPLayer(context) - # This can potentially move to a smarter place later on, - # but it's useful debugging info in mitmproxy for now. - tcp_layer.flow.metadata["quic_is_unidirectional"] = stream_is_unidirectional(stream_id) - tcp_layer.flow.metadata["quic_initiator"] = "client" if stream_is_client_initiated(stream_id) else "server" - tcp_layer.flow.metadata["quic_stream_id_client"] = stream_id - self.child_layer = tcp_layer + self.refresh_metadata() + + # we don't handle any events, pass everything to the child layer self.handle_event = self.child_layer.handle_event # type: ignore self._handle_event = self.child_layer._handle_event # type: ignore @@ -398,8 +410,21 @@ def open_server_stream(self, server_stream_id) -> None: if stream_is_unidirectional(server_stream_id) else connection.ConnectionState.OPEN ) - if self.child_layer.flow: - self.child_layer.flow.metadata["quic_stream_id_server"] = server_stream_id + self.refresh_metadata() + + def refresh_metadata(self) -> None: + # find the first non-NextLayer + child_layer = self.child_layer + while isinstance(child_layer, layer.NextLayer): + child_layer = child_layer.layer + if child_layer: + # try to get the layer's flow + f = getattr(child_layer, "flow", None) + if isinstance(f, flow.Flow): + f.metadata["quic_is_unidirectional"] = stream_is_unidirectional(self._client_stream_id) + f.metadata["quic_initiator"] = "client" if stream_is_client_initiated(self._client_stream_id) else "server" + f.metadata["quic_stream_id_client"] = self._client_stream_id + f.metadata["quic_stream_id_server"] = self._server_stream_id def stream_id(self, client: bool) -> int | None: return self._client_stream_id if client else self._server_stream_id From d45a7f262eb43bda137a9fe7abe75d959d8e2eb2 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 25 Oct 2022 07:11:05 +0200 Subject: [PATCH 081/695] [quic] first tests --- mitmproxy/proxy/layers/quic.py | 59 +++-- test/mitmproxy/proxy/layers/test_quic.py | 300 ++++++++++++++++++++++- 2 files changed, 341 insertions(+), 18 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 3841a60425..013c8b77a7 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -27,7 +27,7 @@ ) from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection, flow +from mitmproxy import certs, connection from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, tunnel from mitmproxy.proxy.layers.tcp import TCPLayer @@ -395,7 +395,7 @@ def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> No self._handle_event = self.child_layer._handle_event # type: ignore def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - raise AssertionError + raise AssertionError # pragma: no cover def open_server_stream(self, server_stream_id) -> None: assert self._server_stream_id is None @@ -413,18 +413,20 @@ def open_server_stream(self, server_stream_id) -> None: self.refresh_metadata() def refresh_metadata(self) -> None: - # find the first non-NextLayer + # find the first transport layer child_layer = self.child_layer - while isinstance(child_layer, layer.NextLayer): - child_layer = child_layer.layer - if child_layer: - # try to get the layer's flow - f = getattr(child_layer, "flow", None) - if isinstance(f, flow.Flow): - f.metadata["quic_is_unidirectional"] = stream_is_unidirectional(self._client_stream_id) - f.metadata["quic_initiator"] = "client" if stream_is_client_initiated(self._client_stream_id) else "server" - f.metadata["quic_stream_id_client"] = self._client_stream_id - f.metadata["quic_stream_id_server"] = self._server_stream_id + while True: + if isinstance(child_layer, layer.NextLayer): + child_layer = child_layer.layer + elif isinstance(child_layer, tunnel.TunnelLayer): + child_layer = child_layer.child_layer + else: + break + if isinstance(child_layer, (UDPLayer, TCPLayer)) and child_layer.flow: + child_layer.flow.metadata["quic_is_unidirectional"] = stream_is_unidirectional(self._client_stream_id) + child_layer.flow.metadata["quic_initiator"] = "client" if stream_is_client_initiated(self._client_stream_id) else "server" + child_layer.flow.metadata["quic_stream_id_client"] = self._client_stream_id + child_layer.flow.metadata["quic_stream_id_server"] = self._server_stream_id def stream_id(self, client: bool) -> int | None: return self._client_stream_id if client else self._server_stream_id @@ -479,6 +481,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: err = yield commands.OpenConnection(self.context.server) if err: yield commands.CloseConnection(self.context.client) + self._handle_event = self.done return yield from self.event_to_child(self.datagram_layer, event) @@ -559,18 +562,37 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) ): from_client = event.connection is self.context.client + other_conn = self.context.server if from_client else self.context.client - # always forward to the datagram layer - yield from self.event_to_child(self.datagram_layer, event) + # be done if both connections are closed + if not other_conn.connected: + self._handle_event = self.done + + # always forward to the datagram layer and swallow `CloseConnection` commands + for command in self.event_to_child(self.datagram_layer, event): + if ( + not isinstance(command, commands.CloseConnection) + or command.connection is not other_conn + ): + yield command - # forward to either the client or server connection of stream layers + # forward to either the client or server connection of stream layers and swallow empty stream end for conn, child_layer in self.connections.items(): if ( isinstance(child_layer, QuicStreamLayer) and ((conn is child_layer.client) if from_client else (conn is child_layer.server)) ): conn.state &= ~connection.ConnectionState.CAN_WRITE - yield from self.close_stream_layer(child_layer, from_client) + for command in self.close_stream_layer(child_layer, from_client): + if ( + not isinstance(command, SendQuicStreamData) + or command.data + ): + yield command + + # forward the close event + if other_conn.connected: + yield CloseQuicConnection(other_conn, event.error_code, event.frame_type, event.reason_phrase) # all other connection events are routed to their corresponding layer elif isinstance(event, events.ConnectionEvent): @@ -660,6 +682,9 @@ def get_next_available_stream_id(self, is_client: bool, is_unidirectional: bool self.next_stream_id[index] = stream_id + 4 return stream_id + def done(self, _) -> layer.CommandGenerator[None]: + yield from () + class QuicLayer(tunnel.TunnelLayer): quic: QuicConnection | None = None diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index effb36afe7..c09021887f 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -1,6 +1,33 @@ +import ssl +import time +from aioquic.buffer import Buffer as QuicBuffer +from aioquic.quic import events as quic_events +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.connection import QuicConnection, pull_quic_header +from typing import Optional +from unittest.mock import MagicMock import pytest +from mitmproxy import connection, options +from mitmproxy.addons.proxyserver import Proxyserver +from mitmproxy.proxy import commands, context, events, layer, tunnel +from mitmproxy.proxy import layers +from mitmproxy.proxy.layers import quic, tls +from mitmproxy.utils import data +from test.mitmproxy.proxy import tutils -from mitmproxy.proxy.layers import quic +from mitmproxy.tcp import TCPFlow + + +tlsdata = data.Data(__name__) + + +@pytest.fixture +def tctx() -> context.Context: + opts = options.Options() + Proxyserver().load(opts) + return context.Context( + connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts + ) def test_error_code_to_str(): @@ -56,3 +83,274 @@ def test_parse_client_hello(): quic.quic_parse_client_hello( client_hello[:183] + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00" ) + + +@pytest.mark.parametrize("value", ["s1 s2\n", "s1 s2"]) +def test_secrets_logger(value: str): + logger = MagicMock() + quic_logger = quic.QuicSecretsLogger(logger) + assert quic_logger.write(value) == 6 + quic_logger.flush() + logger.assert_called_once() + logger.assert_called_once_with(None, b"s1 s2") + + +def test_quic_stream_layer_ignored(tctx: context.Context): + quic_layer = quic.QuicStreamLayer(tctx, True, 1) + assert isinstance(quic_layer.child_layer, layers.TCPLayer) + assert not quic_layer.child_layer.flow + quic_layer.child_layer.flow = TCPFlow(MagicMock(), MagicMock()) + quic_layer.refresh_metadata() + assert quic_layer.child_layer.flow.metadata["quic_is_unidirectional"] is False + assert quic_layer.child_layer.flow.metadata["quic_initiator"] == "server" + assert quic_layer.child_layer.flow.metadata["quic_stream_id_client"] == 1 + assert quic_layer.child_layer.flow.metadata["quic_stream_id_server"] is None + assert quic_layer.stream_id(True) == 1 + assert quic_layer.stream_id(False) is None + + +def test_quic_stream_layer(tctx: context.Context): + quic_layer = quic.QuicStreamLayer(tctx, False, 2) + assert isinstance(quic_layer.child_layer, layer.NextLayer) + tunnel_layer = tunnel.TunnelLayer(tctx, MagicMock(), MagicMock()) + quic_layer.child_layer.layer = tunnel_layer + tcp_layer = layers.TCPLayer(tctx) + tunnel_layer.child_layer = tcp_layer + quic_layer.open_server_stream(3) + assert tcp_layer.flow.metadata["quic_is_unidirectional"] is True + assert tcp_layer.flow.metadata["quic_initiator"] == "client" + assert tcp_layer.flow.metadata["quic_stream_id_client"] == 2 + assert tcp_layer.flow.metadata["quic_stream_id_server"] == 3 + assert quic_layer.stream_id(True) == 2 + assert quic_layer.stream_id(False) == 3 + + +@pytest.mark.parametrize("ignore", [True, False]) +def test_raw_quic_layer_error(tctx: context.Context, ignore: bool): + quic_layer = quic.RawQuicLayer(tctx, ignore=ignore) + assert ( + tutils.Playbook(quic_layer) + << commands.OpenConnection(tctx.server) + >> tutils.reply("failed to open") + << commands.CloseConnection(tctx.client) + ) + assert quic_layer._handle_event == quic_layer.done + + +def test_raw_quic_layer_ignored(tctx: context.Context): + quic_layer = quic.RawQuicLayer(tctx, ignore=True) + assert ( + tutils.Playbook(quic_layer) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> events.DataReceived(tctx.client, b"msg1") + << commands.SendData(tctx.server, b"msg1") + >> events.DataReceived(tctx.server, b"msg2") + << commands.SendData(tctx.client, b"msg2") + >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg3", end_stream=False) + << quic.SendQuicStreamData(tctx.server, 0, b"msg3", end_stream=False) + >> quic.QuicStreamDataReceived(tctx.client, 6, b"msg4", end_stream=False) + << quic.SendQuicStreamData(tctx.server, 2, b"msg4", end_stream=False) + >> quic.QuicStreamDataReceived(tctx.server, 9, b"msg5", end_stream=False) + << quic.SendQuicStreamData(tctx.client, 1, b"msg5", end_stream=False) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"", end_stream=True) + << quic.SendQuicStreamData(tctx.server, 0, b"", end_stream=True) + >> quic.QuicConnectionClosed(tctx.client, 42, None, "closed") + << quic.CloseQuicConnection(tctx.server, 42, None, "closed") + >> quic.QuicConnectionClosed(tctx.server, 42, None, "closed") + << None + ) + assert quic_layer._handle_event == quic_layer.done + + +class SSLTest: + """Helper container for Python's builtin SSL object.""" + + def __init__( + self, + server_side: bool = False, + alpn: Optional[list[str]] = None, + sni: Optional[bytes] = b"example.mitmproxy.org", + ): + self.ctx = QuicConfiguration( + is_client=not server_side, + max_datagram_frame_size=65536, + ) + + self.ctx.verify_mode = ssl.CERT_OPTIONAL + self.ctx.load_verify_locations( + cafile=tlsdata.path("../../net/data/verificationcerts/trusted-root.crt"), + ) + + if alpn: + self.ctx.alpn_protocols = alpn + if server_side: + if sni == b"192.0.2.42": + filename = "trusted-leaf-ip" + else: + filename = "trusted-leaf" + self.ctx.load_cert_chain( + certfile=tlsdata.path( + f"../../net/data/verificationcerts/{filename}.crt" + ), + keyfile=tlsdata.path( + f"../../net/data/verificationcerts/{filename}.key" + ), + ) + + self.ctx.server_name = None if server_side else sni + + self.quic = None if server_side else QuicConnection(configuration=self.ctx) + + def write(self, buf: bytes) -> int: + if self.quic is None: + quic_buf = QuicBuffer(data=buf) + header = pull_quic_header(quic_buf, host_cid_length=8) + self.quic = QuicConnection( + configuration=self.ctx, + original_destination_connection_id=header.destination_cid, + ) + self.quic.receive_datagram(buf, ("0.0.0.0", 0), time.time()) + + def read(self) -> bytes: + buf = b"" + has_data = False + for datagram, addr in self.quic.datagrams_to_send(time.time()): + assert addr == ("0.0.0.0", 0) + buf += datagram + has_data = True + if not has_data: + raise AssertionError("no datagrams to send") + return buf + + def handshake_completed(self) -> bool: + while event := self.quic.next_event(): + if isinstance(event, quic_events.HandshakeCompleted): + return True + else: + return False + + +def _test_echo( + playbook: tutils.Playbook, tssl: SSLTest, conn: connection.Connection +) -> None: + tssl.quic.send_datagram_frame(b"Hello World") + data = tutils.Placeholder(bytes) + assert ( + playbook + >> events.DataReceived(conn, tssl.read()) + << commands.SendData(conn, data) + ) + tssl.write(data()) + while event := tssl.quic.next_event(): + if isinstance(event, quic_events.DatagramFrameReceived): + assert event.data == b"hello world" + else: + raise AssertionError() + + +class TlsEchoLayer(tutils.EchoLayer): + err: Optional[str] = None + + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.DataReceived) and event.data == b"open-connection": + err = yield commands.OpenConnection(self.context.server) + if err: + yield commands.SendData( + event.connection, f"open-connection failed: {err}".encode() + ) + else: + yield from super()._handle_event(event) + + +def finish_handshake( + playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest +): + data = tutils.Placeholder(bytes) + tls_hook_data = tutils.Placeholder(tls.TlsData) + if isinstance(conn, connection.Client): + established_hook = tls.TlsEstablishedClientHook(tls_hook_data) + else: + established_hook = tls.TlsEstablishedServerHook(tls_hook_data) + assert ( + playbook + >> events.DataReceived(conn, tssl.read()) + << established_hook + >> tutils.reply() + << commands.SendData(conn, data) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(TlsEchoLayer) + ) + assert tls_hook_data().conn.error is None + tssl.write(data()) + + +def reply_tls_start_server(*args, **kwargs) -> tutils.reply: + """ + Helper function to simplify the syntax for quic_start_server hooks. + """ + + def make_server_conn(tls_start: quic.QuicTlsData) -> None: + # ssl_context = SSL.Context(Method.TLS_METHOD) + # ssl_context.set_min_proto_version(SSL.TLS1_3_VERSION) + tls_start.settings = quic.QuicTlsSettings( + ca_file=tlsdata.path("../../net/data/verificationcerts/trusted-root.crt"), + verify_mode=ssl.CERT_REQUIRED, + ) + + return tutils.reply(*args, side_effect=make_server_conn, **kwargs) + + +class TestServerTLS: + def test_repr(self, tctx: context.Context): + assert repr(quic.ServerQuicLayer(tctx)) + + def test_not_connected(self, tctx: context.Context): + """Test that we don't do anything if no server connection exists.""" + layer = quic.ServerQuicLayer(tctx) + layer.child_layer = TlsEchoLayer(tctx) + + assert ( + tutils.Playbook(layer) + >> events.DataReceived(tctx.client, b"Hello World") + << commands.SendData(tctx.client, b"hello world") + ) + + def test_simple(self, tctx: context.Context): + playbook = tutils.Playbook(quic.ServerQuicLayer(tctx)) + tctx.server.address = ("example.mitmproxy.org", 443) + tctx.server.state = connection.ConnectionState.OPEN + tctx.server.sni = "example.mitmproxy.org" + + tssl = SSLTest(server_side=True) + + # send ClientHello, receive ClientHello + data = tutils.Placeholder(bytes) + assert ( + playbook + << quic.QuicStartServerHook(tutils.Placeholder()) + >> reply_tls_start_server() + << commands.SendData(tctx.server, data) + << commands.RequestWakeup(tutils.Placeholder()) + ) + tssl.write(data()) + assert not tssl.handshake_completed() + + # finish handshake (mitmproxy) + finish_handshake(playbook, tctx.server, tssl) + + # finish handshake (locally) + assert tssl.handshake_completed() + playbook >> events.DataReceived(tctx.server, tssl.read()) + playbook << None + assert playbook + + assert tctx.server.tls_established + + # Echo + assert ( + playbook + >> events.DataReceived(tctx.client, b"foo") + << commands.SendData(tctx.client, b"foo") + ) + From ec04569027d0b208659b97d1b22e186896174485 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 25 Oct 2022 20:50:40 +0200 Subject: [PATCH 082/695] [quic] explicit timing, more tests --- mitmproxy/proxy/layers/quic.py | 57 +++++++++++--------- test/mitmproxy/proxy/layers/test_quic.py | 66 +++++++++++++++++++----- 2 files changed, 86 insertions(+), 37 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 013c8b77a7..5fb1b18068 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -4,7 +4,7 @@ from logging import DEBUG, ERROR, WARNING from ssl import VerifyMode import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Callable from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import H3_ALPN, ErrorCode as H3ErrorCode @@ -690,10 +690,10 @@ class QuicLayer(tunnel.TunnelLayer): quic: QuicConnection | None = None tls: QuicTlsSettings | None = None - def __init__(self, context: context.Context, conn: connection.Connection) -> None: + def __init__(self, context: context.Context, conn: connection.Connection, time: Callable[[], float] | None) -> None: super().__init__(context, tunnel_connection=conn, conn=conn) self.child_layer = layer.NextLayer(self.context, ask_on_start=True) - self._loop = asyncio.get_event_loop() + self._time = time or asyncio.get_running_loop().time self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() self._routes: dict[connection.Address, ConnectionHandler | None] = dict() conn.tls = True @@ -708,7 +708,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.quic timer = self._wakeup_commands.pop(event.command) if self.quic._state is not QuicConnectionState.TERMINATED: - self.quic.handle_timer(now=max(timer, self._loop.time())) + self.quic.handle_timer(now=max(timer, self._time())) yield from super()._handle_event( events.DataReceived(self.tunnel_connection, b"") ) @@ -749,7 +749,7 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C yield QuicStartClientHook(tls_data) else: yield QuicStartServerHook(tls_data) - if tls_data.settings is None: + if not tls_data.settings: yield commands.Log(f"No QUIC context was provided, failing connection.", ERROR) yield commands.CloseConnection(self.conn) return @@ -785,7 +785,7 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C # if we act as client, connect to upstream if original_destination_connection_id is None: - self.quic.connect(self.conn.peername, now=self._loop.time()) + self.quic.connect(self.conn.peername, now=self._time()) yield from self.tls_interact() def tls_interact(self) -> layer.CommandGenerator[None]: @@ -793,7 +793,7 @@ def tls_interact(self) -> layer.CommandGenerator[None]: # send all queued datagrams assert self.quic - for data, addr in self.quic.datagrams_to_send(now=self._loop.time()): + for data, addr in self.quic.datagrams_to_send(now=self._time()): assert addr == self.conn.peername yield commands.SendData(self.tunnel_connection, data) @@ -803,7 +803,7 @@ def tls_interact(self) -> layer.CommandGenerator[None]: timer is not None and not any(existing <= timer for existing in self._wakeup_commands.values()) ): - command = commands.RequestWakeup(timer - self._loop.time()) + command = commands.RequestWakeup(timer - self._time()) self._wakeup_commands[command] = timer yield command @@ -812,7 +812,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # forward incoming data to aioquic if data: - self.quic.receive_datagram(data, self.conn.peername, now=self._loop.time()) + self.quic.receive_datagram(data, self.conn.peername, now=self._time()) # handle pre-handshake events while event := self.quic.next_event(): @@ -822,13 +822,13 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo elif isinstance(event, quic_events.HandshakeCompleted): # concatenate all peer certificates all_certs: list[x509.Certificate] = [] - if self.quic.tls._peer_certificate is not None: + if self.quic.tls._peer_certificate: all_certs.append(self.quic.tls._peer_certificate) - if self.quic.tls._peer_certificate_chain is not None: + if self.quic.tls._peer_certificate_chain: all_certs.extend(self.quic.tls._peer_certificate_chain) # set the connection's TLS properties - self.conn.timestamp_tls_setup = self._loop.time() + self.conn.timestamp_tls_setup = time.time() if event.alpn_protocol: self.conn.alpn = event.alpn_protocol.encode("ascii") self.conn.certificate_list = [certs.Cert(cert) for cert in all_certs] @@ -875,7 +875,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: # forward incoming data to aioquic if data: - self.quic.receive_datagram(data, self.conn.peername, now=self._loop.time()) + self.quic.receive_datagram(data, self.conn.peername, now=self._time()) # handle post-handshake events while event := self.quic.next_event(): @@ -885,7 +885,12 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: yield commands.Log( f"{self.debug}[quic] close_notify {self.conn} (reason={reason})", DEBUG ) - yield CloseQuicConnection(self.conn, event.error_code, event.frame_type, event.reason_phrase) + # We don't rely on `ConnectionTerminated` to dispatch `QuicConnectionClosed`, because + # after aioquic receives a termination frame, it still waits for the next `handle_timer` + # before returning `ConnectionTerminated in `next_event`. In the meantime, the underlying + # connection could be closed. Therefore, we dispatch when `ConnectionClosed` and simply + # close the connection here. + yield commands.CloseConnection(self.tunnel_connection) return # we don't handle any further events, nor do/can we transmit data, so exit elif isinstance(event, quic_events.DatagramFrameReceived): yield from self.event_to_child(events.DataReceived(self.conn, event.data)) @@ -907,8 +912,14 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: yield from self.tls_interact() def receive_close(self) -> layer.CommandGenerator[None]: - # unlike TLS we haven't sent CloseConnection before - yield from super().receive_close() + # if `_close_event` is not set, the underlying connection has been closed + # we turn this into a QUIC close event as well + close_event = self.quic._close_event or quic_events.ConnectionTerminated( + QuicErrorCode.APPLICATION_ERROR, None, "Connection closed." + ) + yield from self.event_to_child( + QuicConnectionClosed(self.conn, close_event.error_code, close_event.frame_type, close_event.reason_phrase) + ) def send_data(self, data: bytes) -> layer.CommandGenerator[None]: # non-stream data uses datagram frames @@ -919,7 +930,7 @@ def send_data(self, data: bytes) -> layer.CommandGenerator[None]: def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: # properly close the QUIC connection - if self.quic is not None: + if self.quic: if isinstance(command, CloseQuicConnection): self.quic.close(command.error_code, command.frame_type, command.reason_phrase) else: @@ -935,8 +946,8 @@ class ServerQuicLayer(QuicLayer): wait_for_clienthello: bool = False - def __init__(self, context: context.Context, conn: connection.Server | None = None): - super().__init__(context, conn or context.server) + def __init__(self, context: context.Context, conn: connection.Server | None = None, time: Callable[[], float] | None = None): + super().__init__(context, conn or context.server, time) def start_handshake(self) -> layer.CommandGenerator[None]: wait_for_clienthello = ( @@ -975,7 +986,7 @@ class ClientQuicLayer(QuicLayer): server_tls_available: bool """Indicates whether the parent layer is a ServerQuicLayer.""" - def __init__(self, context: context.Context) -> None: + def __init__(self, context: context.Context, time: Callable[[], float] | None = None) -> None: # same as ClientTLSLayer, we might be nested in some other transport if context.client.tls: context.client.alpn = None @@ -988,7 +999,7 @@ def __init__(self, context: context.Context) -> None: context.client.alpn_offers = [] context.client.cipher_list = [] - super().__init__(context, context.client) + super().__init__(context, context.client, time) self.server_tls_available = isinstance(self.context.layers[-2], ServerQuicLayer) def start_handshake(self) -> layer.CommandGenerator[None]: @@ -996,7 +1007,7 @@ def start_handshake(self) -> layer.CommandGenerator[None]: def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bool, str | None]]: # if we already had a valid client hello, don't process further packets - if self.tls is not None: + if self.tls: return (yield from super().receive_handshake_data(data)) # fail if the received data is not a QUIC packet @@ -1044,7 +1055,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # replace the QUIC layer with an UDP layer if requested if tls_clienthello.ignore_connection: self.conn = self.tunnel_connection = connection.Client( - ("ignore-conn", 0), ("ignore-conn", 0), self._loop.time(), + ("ignore-conn", 0), ("ignore-conn", 0), time.time(), transport_protocol="udp", proxy_mode=self.context.client.proxy_mode ) diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index c09021887f..e10f5ba16e 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -1,10 +1,9 @@ import ssl -import time from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import QuicConnection, pull_quic_header -from typing import Optional +from typing import Optional, TypeVar from unittest.mock import MagicMock import pytest from mitmproxy import connection, options @@ -21,6 +20,9 @@ tlsdata = data.Data(__name__) +T = TypeVar('T', bound=layer.Layer) + + @pytest.fixture def tctx() -> context.Context: opts = options.Options() @@ -200,9 +202,11 @@ def __init__( self.ctx.server_name = None if server_side else sni + self.now = 0.0 self.quic = None if server_side else QuicConnection(configuration=self.ctx) def write(self, buf: bytes) -> int: + self.now = self.now + 0.1 if self.quic is None: quic_buf = QuicBuffer(data=buf) header = pull_quic_header(quic_buf, host_cid_length=8) @@ -210,12 +214,13 @@ def write(self, buf: bytes) -> int: configuration=self.ctx, original_destination_connection_id=header.destination_cid, ) - self.quic.receive_datagram(buf, ("0.0.0.0", 0), time.time()) + self.quic.receive_datagram(buf, ("0.0.0.0", 0), self.now) def read(self) -> bytes: + self.now = self.now + 0.1 buf = b"" has_data = False - for datagram, addr in self.quic.datagrams_to_send(time.time()): + for datagram, addr in self.quic.datagrams_to_send(self.now): assert addr == ("0.0.0.0", 0) buf += datagram has_data = True @@ -245,12 +250,14 @@ def _test_echo( while event := tssl.quic.next_event(): if isinstance(event, quic_events.DatagramFrameReceived): assert event.data == b"hello world" + break else: raise AssertionError() class TlsEchoLayer(tutils.EchoLayer): err: Optional[str] = None + closed: Optional[quic.QuicConnectionClosed] = None def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.DataReceived) and event.data == b"open-connection": @@ -259,13 +266,25 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield commands.SendData( event.connection, f"open-connection failed: {err}".encode() ) + elif isinstance(event, quic.QuicConnectionClosed): + self.closed = event else: yield from super()._handle_event(event) def finish_handshake( - playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest -): + playbook: tutils.Playbook, + conn: connection.Connection, + tssl: SSLTest, + child_layer: type[T] +) -> T: + result: Optional[T] = None + + def set_layer(next_layer: layer.NextLayer) -> None: + nonlocal result + result = child_layer(next_layer.context) + next_layer.layer = result + data = tutils.Placeholder(bytes) tls_hook_data = tutils.Placeholder(tls.TlsData) if isinstance(conn, connection.Client): @@ -279,11 +298,14 @@ def finish_handshake( >> tutils.reply() << commands.SendData(conn, data) << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(TlsEchoLayer) + >> tutils.reply(side_effect=set_layer) ) assert tls_hook_data().conn.error is None tssl.write(data()) + assert result + return result + def reply_tls_start_server(*args, **kwargs) -> tutils.reply: """ @@ -303,11 +325,11 @@ def make_server_conn(tls_start: quic.QuicTlsData) -> None: class TestServerTLS: def test_repr(self, tctx: context.Context): - assert repr(quic.ServerQuicLayer(tctx)) + assert repr(quic.ServerQuicLayer(tctx, time=lambda: 0)) def test_not_connected(self, tctx: context.Context): """Test that we don't do anything if no server connection exists.""" - layer = quic.ServerQuicLayer(tctx) + layer = quic.ServerQuicLayer(tctx, time=lambda: 0) layer.child_layer = TlsEchoLayer(tctx) assert ( @@ -317,13 +339,13 @@ def test_not_connected(self, tctx: context.Context): ) def test_simple(self, tctx: context.Context): - playbook = tutils.Playbook(quic.ServerQuicLayer(tctx)) + tssl = SSLTest(server_side=True) + + playbook = tutils.Playbook(quic.ServerQuicLayer(tctx, time=lambda: tssl.now)) tctx.server.address = ("example.mitmproxy.org", 443) tctx.server.state = connection.ConnectionState.OPEN tctx.server.sni = "example.mitmproxy.org" - tssl = SSLTest(server_side=True) - # send ClientHello, receive ClientHello data = tutils.Placeholder(bytes) assert ( @@ -331,13 +353,13 @@ def test_simple(self, tctx: context.Context): << quic.QuicStartServerHook(tutils.Placeholder()) >> reply_tls_start_server() << commands.SendData(tctx.server, data) - << commands.RequestWakeup(tutils.Placeholder()) + << commands.RequestWakeup(0.2) ) tssl.write(data()) assert not tssl.handshake_completed() # finish handshake (mitmproxy) - finish_handshake(playbook, tctx.server, tssl) + echo = finish_handshake(playbook, tctx.server, tssl, TlsEchoLayer) # finish handshake (locally) assert tssl.handshake_completed() @@ -353,4 +375,20 @@ def test_simple(self, tctx: context.Context): >> events.DataReceived(tctx.client, b"foo") << commands.SendData(tctx.client, b"foo") ) + _test_echo(playbook, tssl, tctx.server) + tssl.quic.close(42, None, "goodbye from simple") + playbook >> events.DataReceived(tctx.server, tssl.read()) + playbook << None + assert playbook + tssl.now = tssl.now + 60 + assert ( + playbook + >> events.Wakeup(playbook.actual[4]) + << commands.CloseConnection(tctx.server) + >> events.ConnectionClosed(tctx.server) + << None + ) + assert echo.closed + assert echo.closed.error_code == 42 + assert echo.closed.reason_phrase == "goodbye from simple" From 83dac8dc39d57419f20f5bcf2ad3b1e3e268acc7 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 27 Oct 2022 03:55:17 +0200 Subject: [PATCH 083/695] [quic] refinements and tests --- mitmproxy/proxy/layers/quic.py | 30 +-- test/mitmproxy/proxy/layers/test_quic.py | 243 +++++++++++++++++++++-- 2 files changed, 242 insertions(+), 31 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 5fb1b18068..0725ad9a5c 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -276,7 +276,7 @@ def quic_parse_client_hello(data: bytes) -> ClientHello: # ensure the first packet is indeed the initial one buffer = QuicBuffer(data=data) - header = pull_quic_header(buffer) + header = pull_quic_header(buffer, 8) if header.packet_type != PACKET_TYPE_INITIAL: raise ValueError("Packet is not initial one.") @@ -545,11 +545,17 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if event.end_stream: yield from self.close_stream_layer(stream_layer, from_client) elif isinstance(event, QuicStreamReset): - if self.debug is not None: - yield commands.Log( - f"{self.debug}[quic] stream_reset (stream_id={event.stream_id}, error_code={event.error_code})", DEBUG - ) - yield from self.close_stream_layer(stream_layer, from_client) + # preserve stream resets + for command in self.close_stream_layer(stream_layer, from_client): + if ( + isinstance(command, SendQuicStreamData) + and command.stream_id == stream_layer.stream_id(not from_client) + and command.end_stream + and not command.data + ): + yield ResetQuicStream(command.connection, command.stream_id, event.error_code) + else: + yield command else: raise AssertionError(f"Unexpected stream event: {event!r}") @@ -565,7 +571,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: other_conn = self.context.server if from_client else self.context.client # be done if both connections are closed - if not other_conn.connected: + if other_conn.connected: + yield CloseQuicConnection(other_conn, event.error_code, event.frame_type, event.reason_phrase) + else: self._handle_event = self.done # always forward to the datagram layer and swallow `CloseConnection` commands @@ -590,10 +598,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ): yield command - # forward the close event - if other_conn.connected: - yield CloseQuicConnection(other_conn, event.error_code, event.frame_type, event.reason_phrase) - # all other connection events are routed to their corresponding layer elif isinstance(event, events.ConnectionEvent): yield from self.event_to_child(self.connections[event.connection], event) @@ -887,8 +891,8 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: ) # We don't rely on `ConnectionTerminated` to dispatch `QuicConnectionClosed`, because # after aioquic receives a termination frame, it still waits for the next `handle_timer` - # before returning `ConnectionTerminated in `next_event`. In the meantime, the underlying - # connection could be closed. Therefore, we dispatch when `ConnectionClosed` and simply + # before returning `ConnectionTerminated` in `next_event`. In the meantime, the underlying + # connection could be closed. Therefore, we instead dispatch on `ConnectionClosed` and simply # close the connection here. yield commands.CloseConnection(self.tunnel_connection) return # we don't handle any further events, nor do/can we transmit data, so exit diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index e10f5ba16e..acecf85308 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -10,7 +10,8 @@ from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.proxy import commands, context, events, layer, tunnel from mitmproxy.proxy import layers -from mitmproxy.proxy.layers import quic, tls +from mitmproxy.proxy.layers import quic, tcp, tls, udp +from mitmproxy.udp import UDPFlow, UDPMessage from mitmproxy.utils import data from test.mitmproxy.proxy import tutils @@ -85,6 +86,39 @@ def test_parse_client_hello(): quic.quic_parse_client_hello( client_hello[:183] + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00" ) + with pytest.raises(ValueError, match="not initial"): + quic.quic_parse_client_hello( + b'\\s\xd8\xd8\xa5dT\x8bc\xd3\xae\x1c\xb2\x8a7-\x1d\x19j\x85\xb0~\x8c\x80\xa5\x8cY\xac\x0ecK\x7fC2f\xbcm\x1b\xac~' + ) + + +def test_parse_client_hello_invalid(monkeypatch): + class InvalidClientHello(Exception): + @property + def data(self): + raise EOFError() + + monkeypatch.setattr(quic, "QuicClientHello", InvalidClientHello) + with pytest.raises(ValueError, match="Invalid ClientHello"): + quic.quic_parse_client_hello(client_hello) + + +def test_parse_client_hello_conn_err(monkeypatch): + def raise_conn_err(self, data, addr, now): + raise quic.QuicConnectionError(0, 0, "Conn err") + + monkeypatch.setattr(QuicConnection, "receive_datagram", raise_conn_err) + with pytest.raises(ValueError, match="Conn err"): + quic.quic_parse_client_hello(client_hello) + + +def test_parse_client_hello_none(monkeypatch): + def do_nothing(self, data, addr, now): + pass + + monkeypatch.setattr(QuicConnection, "receive_datagram", do_nothing) + with pytest.raises(ValueError, match="No ClientHello"): + quic.quic_parse_client_hello(client_hello) @pytest.mark.parametrize("value", ["s1 s2\n", "s1 s2"]) @@ -157,6 +191,8 @@ def test_raw_quic_layer_ignored(tctx: context.Context): << quic.SendQuicStreamData(tctx.client, 1, b"msg5", end_stream=False) >> quic.QuicStreamDataReceived(tctx.client, 0, b"", end_stream=True) << quic.SendQuicStreamData(tctx.server, 0, b"", end_stream=True) + >> quic.QuicStreamReset(tctx.client, 6, 142) + << quic.ResetQuicStream(tctx.server, 2, 142) >> quic.QuicConnectionClosed(tctx.client, 42, None, "closed") << quic.CloseQuicConnection(tctx.server, 42, None, "closed") >> quic.QuicConnectionClosed(tctx.server, 42, None, "closed") @@ -165,6 +201,194 @@ def test_raw_quic_layer_ignored(tctx: context.Context): assert quic_layer._handle_event == quic_layer.done +def test_raw_quic_layer_msg_inject(tctx: context.Context): + udpflow = tutils.Placeholder(UDPFlow) + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + playbook + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> events.DataReceived(tctx.client, b"msg1") + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(udp.UDPLayer) + << udp.UdpStartHook(udpflow) + >> tutils.reply() + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg1") + >> udp.UdpMessageInjected(udpflow, UDPMessage(True, b"msg2")) + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg2") + >> udp.UdpMessageInjected(UDPFlow(("other", 80), tctx.server), UDPMessage(True, b"msg3")) + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg3") + ) + with pytest.raises(AssertionError, match="not associated"): + playbook >> udp.UdpMessageInjected(UDPFlow(("notfound", 0), ("noexist", 0)), UDPMessage(True, b"msg2")) + assert playbook + + +def test_raw_quic_layer_reset_with_end_hook(tctx: context.Context): + tcpflow = tutils.Placeholder(TCPFlow) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 2, b"msg1", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(tcp.TCPLayer) + << tcp.TcpStartHook(tcpflow) + >> tutils.reply() + << tcp.TcpMessageHook(tcpflow) + >> tutils.reply() + << quic.SendQuicStreamData(tctx.server, 2, b"msg1", end_stream=False) + >> quic.QuicStreamReset(tctx.client, 2, 42) + << quic.ResetQuicStream(tctx.server, 2, 42) + << tcp.TcpEndHook(tcpflow) + >> tutils.reply() + ) + + +def test_raw_quic_layer_close_with_end_hooks(tctx: context.Context): + udpflow = tutils.Placeholder(UDPFlow) + tcpflow = tutils.Placeholder(TCPFlow) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> events.DataReceived(tctx.client, b"msg1") + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(udp.UDPLayer) + << udp.UdpStartHook(udpflow) + >> tutils.reply() + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg1") + >> quic.QuicStreamDataReceived(tctx.client, 2, b"msg2", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(tcp.TCPLayer) + << tcp.TcpStartHook(tcpflow) + >> tutils.reply() + << tcp.TcpMessageHook(tcpflow) + >> tutils.reply() + << quic.SendQuicStreamData(tctx.server, 2, b"msg2", end_stream=False) + >> quic.QuicConnectionClosed(tctx.client, 42, None, "bye") + << quic.CloseQuicConnection(tctx.server, 42, None, "bye") + << tcp.TcpEndHook(tcpflow) + >> tutils.reply() + >> quic.QuicConnectionClosed(tctx.server, 42, None, "bye") + << udp.UdpEndHook(udpflow) + >> tutils.reply() + ) + + +class InvalidStreamEvent(quic.QuicStreamEvent): + pass + + +def test_raw_quic_layer_invalid_stream_event(tctx: context.Context): + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + ) + with pytest.raises(AssertionError, match="Unexpected stream event"): + playbook >> InvalidStreamEvent(tctx.client, 0) + assert playbook + + +class InvalidEvent(events.Event): + pass + + +def test_raw_quic_layer_invalid_event(tctx: context.Context): + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + ) + with pytest.raises(AssertionError, match="Unexpected event"): + playbook >> InvalidEvent() + assert playbook + + +def test_raw_quic_layer_full_close(tctx: context.Context): + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg1", end_stream=True) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(lambda ctx: udp.UDPLayer(ctx, ignore=True)) + << quic.SendQuicStreamData(tctx.server, 0, b"msg1", end_stream=False) + << quic.SendQuicStreamData(tctx.server, 0, b"", end_stream=True) + << quic.StopQuicStream(tctx.server, 0, 0) + ) + + +class InvalidConnectionCommand(commands.ConnectionCommand): + pass + + +class TlsEchoLayer(tutils.EchoLayer): + err: Optional[str] = None + closed: Optional[quic.QuicConnectionClosed] = None + + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.DataReceived) and event.data == b"open-connection": + err = yield commands.OpenConnection(self.context.server) + if err: + yield commands.SendData( + event.connection, f"open-connection failed: {err}".encode() + ) + elif isinstance(event, events.DataReceived) and event.data == b"invalid-command": + yield InvalidConnectionCommand(event.connection) + elif isinstance(event, quic.QuicConnectionClosed): + self.closed = event + else: + yield from super()._handle_event(event) + + +def test_raw_quic_layer_open_connection(tctx: context.Context): + server = connection.Server(("other", 80)) + + def echo_new_server(ctx: context.Context): + echo_layer = TlsEchoLayer(ctx) + echo_layer.context.server = server + return echo_layer + + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"open-connection", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(echo_new_server) + << commands.OpenConnection(server) + >> tutils.reply(None) + ) + + +def test_raw_quic_layer_invalid_connection_command(tctx: context.Context): + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + playbook + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg1", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(TlsEchoLayer) + << quic.SendQuicStreamData(tctx.client, 0, b"msg1", end_stream=False) + ) + with pytest.raises(AssertionError, match="Unexpected stream connection command"): + playbook >> quic.QuicStreamDataReceived(tctx.client, 0, b"invalid-command", end_stream=False) + assert playbook + + class SSLTest: """Helper container for Python's builtin SSL object.""" @@ -255,23 +479,6 @@ def _test_echo( raise AssertionError() -class TlsEchoLayer(tutils.EchoLayer): - err: Optional[str] = None - closed: Optional[quic.QuicConnectionClosed] = None - - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.DataReceived) and event.data == b"open-connection": - err = yield commands.OpenConnection(self.context.server) - if err: - yield commands.SendData( - event.connection, f"open-connection failed: {err}".encode() - ) - elif isinstance(event, quic.QuicConnectionClosed): - self.closed = event - else: - yield from super()._handle_event(event) - - def finish_handshake( playbook: tutils.Playbook, conn: connection.Connection, From d6c629cfe294436b01f19e2a3e22c3a879b00711 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 28 Oct 2022 00:41:47 +0200 Subject: [PATCH 084/695] [quic] more tests --- test/mitmproxy/proxy/layers/test_quic.py | 723 ++++++++++++++--------- 1 file changed, 428 insertions(+), 295 deletions(-) diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index acecf85308..60f00a6df3 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -1,3 +1,4 @@ +from logging import WARNING import ssl from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events @@ -33,18 +34,35 @@ def tctx() -> context.Context: ) -def test_error_code_to_str(): - assert quic.error_code_to_str(0x6) == "FINAL_SIZE_ERROR" - assert quic.error_code_to_str(0x104) == "H3_CLOSED_CRITICAL_STREAM" - assert quic.error_code_to_str(0xdead) == f"unknown error (0xdead)" +class InvalidStreamEvent(quic.QuicStreamEvent): + pass -def test_is_success_error_code(): - assert quic.is_success_error_code(0x0) - assert not quic.is_success_error_code(0x6) - assert quic.is_success_error_code(0x100) - assert not quic.is_success_error_code(0x104) - assert not quic.is_success_error_code(0xdead) +class InvalidEvent(events.Event): + pass + + +class InvalidConnectionCommand(commands.ConnectionCommand): + pass + + +class TlsEchoLayer(tutils.EchoLayer): + err: Optional[str] = None + closed: Optional[quic.QuicConnectionClosed] = None + + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + if isinstance(event, events.DataReceived) and event.data == b"open-connection": + err = yield commands.OpenConnection(self.context.server) + if err: + yield commands.SendData( + event.connection, f"open-connection failed: {err}".encode() + ) + elif isinstance(event, events.DataReceived) and event.data == b"invalid-command": + yield InvalidConnectionCommand(event.connection) + elif isinstance(event, quic.QuicConnectionClosed): + self.closed = event + else: + yield from super()._handle_event(event) client_hello = bytes.fromhex( @@ -80,45 +98,18 @@ def test_is_success_error_code(): ) -def test_parse_client_hello(): - assert quic.quic_parse_client_hello(client_hello).sni == "example.com" - with pytest.raises(ValueError): - quic.quic_parse_client_hello( - client_hello[:183] + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00" - ) - with pytest.raises(ValueError, match="not initial"): - quic.quic_parse_client_hello( - b'\\s\xd8\xd8\xa5dT\x8bc\xd3\xae\x1c\xb2\x8a7-\x1d\x19j\x85\xb0~\x8c\x80\xa5\x8cY\xac\x0ecK\x7fC2f\xbcm\x1b\xac~' - ) - - -def test_parse_client_hello_invalid(monkeypatch): - class InvalidClientHello(Exception): - @property - def data(self): - raise EOFError() - - monkeypatch.setattr(quic, "QuicClientHello", InvalidClientHello) - with pytest.raises(ValueError, match="Invalid ClientHello"): - quic.quic_parse_client_hello(client_hello) - - -def test_parse_client_hello_conn_err(monkeypatch): - def raise_conn_err(self, data, addr, now): - raise quic.QuicConnectionError(0, 0, "Conn err") - - monkeypatch.setattr(QuicConnection, "receive_datagram", raise_conn_err) - with pytest.raises(ValueError, match="Conn err"): - quic.quic_parse_client_hello(client_hello) - +def test_error_code_to_str(): + assert quic.error_code_to_str(0x6) == "FINAL_SIZE_ERROR" + assert quic.error_code_to_str(0x104) == "H3_CLOSED_CRITICAL_STREAM" + assert quic.error_code_to_str(0xdead) == f"unknown error (0xdead)" -def test_parse_client_hello_none(monkeypatch): - def do_nothing(self, data, addr, now): - pass - monkeypatch.setattr(QuicConnection, "receive_datagram", do_nothing) - with pytest.raises(ValueError, match="No ClientHello"): - quic.quic_parse_client_hello(client_hello) +def test_is_success_error_code(): + assert quic.is_success_error_code(0x0) + assert not quic.is_success_error_code(0x6) + assert quic.is_success_error_code(0x100) + assert not quic.is_success_error_code(0x104) + assert not quic.is_success_error_code(0xdead) @pytest.mark.parametrize("value", ["s1 s2\n", "s1 s2"]) @@ -131,262 +122,264 @@ def test_secrets_logger(value: str): logger.assert_called_once_with(None, b"s1 s2") -def test_quic_stream_layer_ignored(tctx: context.Context): - quic_layer = quic.QuicStreamLayer(tctx, True, 1) - assert isinstance(quic_layer.child_layer, layers.TCPLayer) - assert not quic_layer.child_layer.flow - quic_layer.child_layer.flow = TCPFlow(MagicMock(), MagicMock()) - quic_layer.refresh_metadata() - assert quic_layer.child_layer.flow.metadata["quic_is_unidirectional"] is False - assert quic_layer.child_layer.flow.metadata["quic_initiator"] == "server" - assert quic_layer.child_layer.flow.metadata["quic_stream_id_client"] == 1 - assert quic_layer.child_layer.flow.metadata["quic_stream_id_server"] is None - assert quic_layer.stream_id(True) == 1 - assert quic_layer.stream_id(False) is None - - -def test_quic_stream_layer(tctx: context.Context): - quic_layer = quic.QuicStreamLayer(tctx, False, 2) - assert isinstance(quic_layer.child_layer, layer.NextLayer) - tunnel_layer = tunnel.TunnelLayer(tctx, MagicMock(), MagicMock()) - quic_layer.child_layer.layer = tunnel_layer - tcp_layer = layers.TCPLayer(tctx) - tunnel_layer.child_layer = tcp_layer - quic_layer.open_server_stream(3) - assert tcp_layer.flow.metadata["quic_is_unidirectional"] is True - assert tcp_layer.flow.metadata["quic_initiator"] == "client" - assert tcp_layer.flow.metadata["quic_stream_id_client"] == 2 - assert tcp_layer.flow.metadata["quic_stream_id_server"] == 3 - assert quic_layer.stream_id(True) == 2 - assert quic_layer.stream_id(False) == 3 - - -@pytest.mark.parametrize("ignore", [True, False]) -def test_raw_quic_layer_error(tctx: context.Context, ignore: bool): - quic_layer = quic.RawQuicLayer(tctx, ignore=ignore) - assert ( - tutils.Playbook(quic_layer) - << commands.OpenConnection(tctx.server) - >> tutils.reply("failed to open") - << commands.CloseConnection(tctx.client) - ) - assert quic_layer._handle_event == quic_layer.done - - -def test_raw_quic_layer_ignored(tctx: context.Context): - quic_layer = quic.RawQuicLayer(tctx, ignore=True) - assert ( - tutils.Playbook(quic_layer) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - >> events.DataReceived(tctx.client, b"msg1") - << commands.SendData(tctx.server, b"msg1") - >> events.DataReceived(tctx.server, b"msg2") - << commands.SendData(tctx.client, b"msg2") - >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg3", end_stream=False) - << quic.SendQuicStreamData(tctx.server, 0, b"msg3", end_stream=False) - >> quic.QuicStreamDataReceived(tctx.client, 6, b"msg4", end_stream=False) - << quic.SendQuicStreamData(tctx.server, 2, b"msg4", end_stream=False) - >> quic.QuicStreamDataReceived(tctx.server, 9, b"msg5", end_stream=False) - << quic.SendQuicStreamData(tctx.client, 1, b"msg5", end_stream=False) - >> quic.QuicStreamDataReceived(tctx.client, 0, b"", end_stream=True) - << quic.SendQuicStreamData(tctx.server, 0, b"", end_stream=True) - >> quic.QuicStreamReset(tctx.client, 6, 142) - << quic.ResetQuicStream(tctx.server, 2, 142) - >> quic.QuicConnectionClosed(tctx.client, 42, None, "closed") - << quic.CloseQuicConnection(tctx.server, 42, None, "closed") - >> quic.QuicConnectionClosed(tctx.server, 42, None, "closed") - << None - ) - assert quic_layer._handle_event == quic_layer.done - - -def test_raw_quic_layer_msg_inject(tctx: context.Context): - udpflow = tutils.Placeholder(UDPFlow) - playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) - assert ( - playbook - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - >> events.DataReceived(tctx.client, b"msg1") - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(udp.UDPLayer) - << udp.UdpStartHook(udpflow) - >> tutils.reply() - << udp.UdpMessageHook(udpflow) - >> tutils.reply() - << commands.SendData(tctx.server, b"msg1") - >> udp.UdpMessageInjected(udpflow, UDPMessage(True, b"msg2")) - << udp.UdpMessageHook(udpflow) - >> tutils.reply() - << commands.SendData(tctx.server, b"msg2") - >> udp.UdpMessageInjected(UDPFlow(("other", 80), tctx.server), UDPMessage(True, b"msg3")) - << udp.UdpMessageHook(udpflow) - >> tutils.reply() - << commands.SendData(tctx.server, b"msg3") - ) - with pytest.raises(AssertionError, match="not associated"): - playbook >> udp.UdpMessageInjected(UDPFlow(("notfound", 0), ("noexist", 0)), UDPMessage(True, b"msg2")) - assert playbook - - -def test_raw_quic_layer_reset_with_end_hook(tctx: context.Context): - tcpflow = tutils.Placeholder(TCPFlow) - assert ( - tutils.Playbook(quic.RawQuicLayer(tctx)) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - >> quic.QuicStreamDataReceived(tctx.client, 2, b"msg1", end_stream=False) - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(tcp.TCPLayer) - << tcp.TcpStartHook(tcpflow) - >> tutils.reply() - << tcp.TcpMessageHook(tcpflow) - >> tutils.reply() - << quic.SendQuicStreamData(tctx.server, 2, b"msg1", end_stream=False) - >> quic.QuicStreamReset(tctx.client, 2, 42) - << quic.ResetQuicStream(tctx.server, 2, 42) - << tcp.TcpEndHook(tcpflow) - >> tutils.reply() - ) - - -def test_raw_quic_layer_close_with_end_hooks(tctx: context.Context): - udpflow = tutils.Placeholder(UDPFlow) - tcpflow = tutils.Placeholder(TCPFlow) - assert ( - tutils.Playbook(quic.RawQuicLayer(tctx)) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - >> events.DataReceived(tctx.client, b"msg1") - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(udp.UDPLayer) - << udp.UdpStartHook(udpflow) - >> tutils.reply() - << udp.UdpMessageHook(udpflow) - >> tutils.reply() - << commands.SendData(tctx.server, b"msg1") - >> quic.QuicStreamDataReceived(tctx.client, 2, b"msg2", end_stream=False) - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(tcp.TCPLayer) - << tcp.TcpStartHook(tcpflow) - >> tutils.reply() - << tcp.TcpMessageHook(tcpflow) - >> tutils.reply() - << quic.SendQuicStreamData(tctx.server, 2, b"msg2", end_stream=False) - >> quic.QuicConnectionClosed(tctx.client, 42, None, "bye") - << quic.CloseQuicConnection(tctx.server, 42, None, "bye") - << tcp.TcpEndHook(tcpflow) - >> tutils.reply() - >> quic.QuicConnectionClosed(tctx.server, 42, None, "bye") - << udp.UdpEndHook(udpflow) - >> tutils.reply() - ) - - -class InvalidStreamEvent(quic.QuicStreamEvent): - pass - - -def test_raw_quic_layer_invalid_stream_event(tctx: context.Context): - playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) - assert ( - tutils.Playbook(quic.RawQuicLayer(tctx)) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - ) - with pytest.raises(AssertionError, match="Unexpected stream event"): - playbook >> InvalidStreamEvent(tctx.client, 0) - assert playbook - - -class InvalidEvent(events.Event): - pass +class TestParseClientHello: + def test_input(self): + assert quic.quic_parse_client_hello(client_hello).sni == "example.com" + with pytest.raises(ValueError): + quic.quic_parse_client_hello( + client_hello[:183] + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00" + ) + with pytest.raises(ValueError, match="not initial"): + quic.quic_parse_client_hello( + b'\\s\xd8\xd8\xa5dT\x8bc\xd3\xae\x1c\xb2\x8a7-\x1d\x19j\x85\xb0~\x8c\x80\xa5\x8cY\xac\x0ecK\x7fC2f\xbcm\x1b\xac~' + ) -def test_raw_quic_layer_invalid_event(tctx: context.Context): - playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) - assert ( - tutils.Playbook(quic.RawQuicLayer(tctx)) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - ) - with pytest.raises(AssertionError, match="Unexpected event"): - playbook >> InvalidEvent() - assert playbook - + def test_invalid(self, monkeypatch): + class InvalidClientHello(Exception): + @property + def data(self): + raise EOFError() + + monkeypatch.setattr(quic, "QuicClientHello", InvalidClientHello) + with pytest.raises(ValueError, match="Invalid ClientHello"): + quic.quic_parse_client_hello(client_hello) + + def test_connection_error(self, monkeypatch): + def raise_conn_err(self, data, addr, now): + raise quic.QuicConnectionError(0, 0, "Conn err") + + monkeypatch.setattr(QuicConnection, "receive_datagram", raise_conn_err) + with pytest.raises(ValueError, match="Conn err"): + quic.quic_parse_client_hello(client_hello) + + def test_no_return(self, monkeypatch): + def do_nothing(self, data, addr, now): + pass + + monkeypatch.setattr(QuicConnection, "receive_datagram", do_nothing) + with pytest.raises(ValueError, match="No ClientHello"): + quic.quic_parse_client_hello(client_hello) + + +class TestQuicStreamLayer: + def test_ignored(self, tctx: context.Context): + quic_layer = quic.QuicStreamLayer(tctx, True, 1) + assert isinstance(quic_layer.child_layer, layers.TCPLayer) + assert not quic_layer.child_layer.flow + quic_layer.child_layer.flow = TCPFlow(MagicMock(), MagicMock()) + quic_layer.refresh_metadata() + assert quic_layer.child_layer.flow.metadata["quic_is_unidirectional"] is False + assert quic_layer.child_layer.flow.metadata["quic_initiator"] == "server" + assert quic_layer.child_layer.flow.metadata["quic_stream_id_client"] == 1 + assert quic_layer.child_layer.flow.metadata["quic_stream_id_server"] is None + assert quic_layer.stream_id(True) == 1 + assert quic_layer.stream_id(False) is None -def test_raw_quic_layer_full_close(tctx: context.Context): - assert ( - tutils.Playbook(quic.RawQuicLayer(tctx)) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg1", end_stream=True) - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(lambda ctx: udp.UDPLayer(ctx, ignore=True)) - << quic.SendQuicStreamData(tctx.server, 0, b"msg1", end_stream=False) - << quic.SendQuicStreamData(tctx.server, 0, b"", end_stream=True) - << quic.StopQuicStream(tctx.server, 0, 0) - ) + def test_simple(self, tctx: context.Context): + quic_layer = quic.QuicStreamLayer(tctx, False, 2) + assert isinstance(quic_layer.child_layer, layer.NextLayer) + tunnel_layer = tunnel.TunnelLayer(tctx, MagicMock(), MagicMock()) + quic_layer.child_layer.layer = tunnel_layer + tcp_layer = layers.TCPLayer(tctx) + tunnel_layer.child_layer = tcp_layer + quic_layer.open_server_stream(3) + assert tcp_layer.flow.metadata["quic_is_unidirectional"] is True + assert tcp_layer.flow.metadata["quic_initiator"] == "client" + assert tcp_layer.flow.metadata["quic_stream_id_client"] == 2 + assert tcp_layer.flow.metadata["quic_stream_id_server"] == 3 + assert quic_layer.stream_id(True) == 2 + assert quic_layer.stream_id(False) == 3 + + +class TestRawQuicLayer: + @pytest.mark.parametrize("ignore", [True, False]) + def test_error(self, tctx: context.Context, ignore: bool): + quic_layer = quic.RawQuicLayer(tctx, ignore=ignore) + assert ( + tutils.Playbook(quic_layer) + << commands.OpenConnection(tctx.server) + >> tutils.reply("failed to open") + << commands.CloseConnection(tctx.client) + ) + assert quic_layer._handle_event == quic_layer.done + def test_ignored(self, tctx: context.Context): + quic_layer = quic.RawQuicLayer(tctx, ignore=True) + assert ( + tutils.Playbook(quic_layer) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> events.DataReceived(tctx.client, b"msg1") + << commands.SendData(tctx.server, b"msg1") + >> events.DataReceived(tctx.server, b"msg2") + << commands.SendData(tctx.client, b"msg2") + >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg3", end_stream=False) + << quic.SendQuicStreamData(tctx.server, 0, b"msg3", end_stream=False) + >> quic.QuicStreamDataReceived(tctx.client, 6, b"msg4", end_stream=False) + << quic.SendQuicStreamData(tctx.server, 2, b"msg4", end_stream=False) + >> quic.QuicStreamDataReceived(tctx.server, 9, b"msg5", end_stream=False) + << quic.SendQuicStreamData(tctx.client, 1, b"msg5", end_stream=False) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"", end_stream=True) + << quic.SendQuicStreamData(tctx.server, 0, b"", end_stream=True) + >> quic.QuicStreamReset(tctx.client, 6, 142) + << quic.ResetQuicStream(tctx.server, 2, 142) + >> quic.QuicConnectionClosed(tctx.client, 42, None, "closed") + << quic.CloseQuicConnection(tctx.server, 42, None, "closed") + >> quic.QuicConnectionClosed(tctx.server, 42, None, "closed") + << None + ) + assert quic_layer._handle_event == quic_layer.done -class InvalidConnectionCommand(commands.ConnectionCommand): - pass + def test_msg_inject(self, tctx: context.Context): + udpflow = tutils.Placeholder(UDPFlow) + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + playbook + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> events.DataReceived(tctx.client, b"msg1") + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(udp.UDPLayer) + << udp.UdpStartHook(udpflow) + >> tutils.reply() + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg1") + >> udp.UdpMessageInjected(udpflow, UDPMessage(True, b"msg2")) + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg2") + >> udp.UdpMessageInjected(UDPFlow(("other", 80), tctx.server), UDPMessage(True, b"msg3")) + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg3") + ) + with pytest.raises(AssertionError, match="not associated"): + playbook >> udp.UdpMessageInjected(UDPFlow(("notfound", 0), ("noexist", 0)), UDPMessage(True, b"msg2")) + assert playbook + def test_reset_with_end_hook(self, tctx: context.Context): + tcpflow = tutils.Placeholder(TCPFlow) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 2, b"msg1", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(tcp.TCPLayer) + << tcp.TcpStartHook(tcpflow) + >> tutils.reply() + << tcp.TcpMessageHook(tcpflow) + >> tutils.reply() + << quic.SendQuicStreamData(tctx.server, 2, b"msg1", end_stream=False) + >> quic.QuicStreamReset(tctx.client, 2, 42) + << quic.ResetQuicStream(tctx.server, 2, 42) + << tcp.TcpEndHook(tcpflow) + >> tutils.reply() + ) -class TlsEchoLayer(tutils.EchoLayer): - err: Optional[str] = None - closed: Optional[quic.QuicConnectionClosed] = None + def test_close_with_end_hooks(self, tctx: context.Context): + udpflow = tutils.Placeholder(UDPFlow) + tcpflow = tutils.Placeholder(TCPFlow) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> events.DataReceived(tctx.client, b"msg1") + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(udp.UDPLayer) + << udp.UdpStartHook(udpflow) + >> tutils.reply() + << udp.UdpMessageHook(udpflow) + >> tutils.reply() + << commands.SendData(tctx.server, b"msg1") + >> quic.QuicStreamDataReceived(tctx.client, 2, b"msg2", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(tcp.TCPLayer) + << tcp.TcpStartHook(tcpflow) + >> tutils.reply() + << tcp.TcpMessageHook(tcpflow) + >> tutils.reply() + << quic.SendQuicStreamData(tctx.server, 2, b"msg2", end_stream=False) + >> quic.QuicConnectionClosed(tctx.client, 42, None, "bye") + << quic.CloseQuicConnection(tctx.server, 42, None, "bye") + << tcp.TcpEndHook(tcpflow) + >> tutils.reply() + >> quic.QuicConnectionClosed(tctx.server, 42, None, "bye") + << udp.UdpEndHook(udpflow) + >> tutils.reply() + ) - def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, events.DataReceived) and event.data == b"open-connection": - err = yield commands.OpenConnection(self.context.server) - if err: - yield commands.SendData( - event.connection, f"open-connection failed: {err}".encode() - ) - elif isinstance(event, events.DataReceived) and event.data == b"invalid-command": - yield InvalidConnectionCommand(event.connection) - elif isinstance(event, quic.QuicConnectionClosed): - self.closed = event - else: - yield from super()._handle_event(event) + def test_invalid_stream_event(self, tctx: context.Context): + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + ) + with pytest.raises(AssertionError, match="Unexpected stream event"): + playbook >> InvalidStreamEvent(tctx.client, 0) + assert playbook + def test_invalid_event(self, tctx: context.Context): + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + ) + with pytest.raises(AssertionError, match="Unexpected event"): + playbook >> InvalidEvent() + assert playbook -def test_raw_quic_layer_open_connection(tctx: context.Context): - server = connection.Server(("other", 80)) + def test_full_close(self, tctx: context.Context): + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg1", end_stream=True) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(lambda ctx: udp.UDPLayer(ctx, ignore=True)) + << quic.SendQuicStreamData(tctx.server, 0, b"msg1", end_stream=False) + << quic.SendQuicStreamData(tctx.server, 0, b"", end_stream=True) + << quic.StopQuicStream(tctx.server, 0, 0) + ) - def echo_new_server(ctx: context.Context): - echo_layer = TlsEchoLayer(ctx) - echo_layer.context.server = server - return echo_layer + def test_open_connection(self, tctx: context.Context): + server = connection.Server(("other", 80)) - assert ( - tutils.Playbook(quic.RawQuicLayer(tctx)) - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - >> quic.QuicStreamDataReceived(tctx.client, 0, b"open-connection", end_stream=False) - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(echo_new_server) - << commands.OpenConnection(server) - >> tutils.reply(None) - ) + def echo_new_server(ctx: context.Context): + echo_layer = TlsEchoLayer(ctx) + echo_layer.context.server = server + return echo_layer + assert ( + tutils.Playbook(quic.RawQuicLayer(tctx)) + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"open-connection", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(echo_new_server) + << commands.OpenConnection(server) + >> tutils.reply("uhoh") + << quic.SendQuicStreamData(tctx.client, 0, b"open-connection failed: uhoh", end_stream=False) + ) -def test_raw_quic_layer_invalid_connection_command(tctx: context.Context): - playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) - assert ( - playbook - << commands.OpenConnection(tctx.server) - >> tutils.reply(None) - >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg1", end_stream=False) - << layer.NextLayerHook(tutils.Placeholder()) - >> tutils.reply_next_layer(TlsEchoLayer) - << quic.SendQuicStreamData(tctx.client, 0, b"msg1", end_stream=False) - ) - with pytest.raises(AssertionError, match="Unexpected stream connection command"): - playbook >> quic.QuicStreamDataReceived(tctx.client, 0, b"invalid-command", end_stream=False) - assert playbook + def test_invalid_connection_command(self, tctx: context.Context): + playbook = tutils.Playbook(quic.RawQuicLayer(tctx)) + assert ( + playbook + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + >> quic.QuicStreamDataReceived(tctx.client, 0, b"msg1", end_stream=False) + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(TlsEchoLayer) + << quic.SendQuicStreamData(tctx.client, 0, b"msg1", end_stream=False) + ) + with pytest.raises(AssertionError, match="Unexpected stream connection command"): + playbook >> quic.QuicStreamDataReceived(tctx.client, 0, b"invalid-command", end_stream=False) + assert playbook class SSLTest: @@ -396,7 +389,7 @@ def __init__( self, server_side: bool = False, alpn: Optional[list[str]] = None, - sni: Optional[bytes] = b"example.mitmproxy.org", + sni: Optional[str] = "example.mitmproxy.org", ): self.ctx = QuicConfiguration( is_client=not server_side, @@ -411,7 +404,7 @@ def __init__( if alpn: self.ctx.alpn_protocols = alpn if server_side: - if sni == b"192.0.2.42": + if sni == "192.0.2.42": filename = "trusted-leaf-ip" else: filename = "trusted-leaf" @@ -427,6 +420,7 @@ def __init__( self.ctx.server_name = None if server_side else sni self.now = 0.0 + self.address = (sni, 443) self.quic = None if server_side else QuicConnection(configuration=self.ctx) def write(self, buf: bytes) -> int: @@ -438,14 +432,14 @@ def write(self, buf: bytes) -> int: configuration=self.ctx, original_destination_connection_id=header.destination_cid, ) - self.quic.receive_datagram(buf, ("0.0.0.0", 0), self.now) + self.quic.receive_datagram(buf, self.address, self.now) def read(self) -> bytes: self.now = self.now + 0.1 buf = b"" has_data = False for datagram, addr in self.quic.datagrams_to_send(self.now): - assert addr == ("0.0.0.0", 0) + assert addr == self.address buf += datagram has_data = True if not has_data: @@ -514,14 +508,32 @@ def set_layer(next_layer: layer.NextLayer) -> None: return result +def reply_tls_start_client(*args, **kwargs) -> tutils.reply: + """ + Helper function to simplify the syntax for quic_start_client hooks. + """ + + def make_client_conn(tls_start: quic.QuicTlsData) -> None: + config = QuicConfiguration() + config.load_cert_chain( + tlsdata.path("../../net/data/verificationcerts/trusted-leaf.crt"), + tlsdata.path("../../net/data/verificationcerts/trusted-leaf.key"), + ) + tls_start.settings = quic.QuicTlsSettings( + certificate = config.certificate, + certificate_chain = config.certificate_chain, + certificate_private_key = config.private_key, + ) + + return tutils.reply(*args, side_effect=make_client_conn, **kwargs) + + def reply_tls_start_server(*args, **kwargs) -> tutils.reply: """ Helper function to simplify the syntax for quic_start_server hooks. """ def make_server_conn(tls_start: quic.QuicTlsData) -> None: - # ssl_context = SSL.Context(Method.TLS_METHOD) - # ssl_context.set_min_proto_version(SSL.TLS1_3_VERSION) tls_start.settings = quic.QuicTlsSettings( ca_file=tlsdata.path("../../net/data/verificationcerts/trusted-root.crt"), verify_mode=ssl.CERT_REQUIRED, @@ -599,3 +611,124 @@ def test_simple(self, tctx: context.Context): assert echo.closed assert echo.closed.error_code == 42 assert echo.closed.reason_phrase == "goodbye from simple" + + def test_untrusted_cert(self, tctx: context.Context): + """If the certificate is not trusted, we should fail.""" + tssl = SSLTest(server_side=True) + + playbook = tutils.Playbook(quic.ServerQuicLayer(tctx, time=lambda: tssl.now)) + tctx.server.address = ("wrong.host.mitmproxy.org", 443) + tctx.server.sni = "wrong.host.mitmproxy.org" + + # send ClientHello + data = tutils.Placeholder(bytes) + assert ( + playbook + << layer.NextLayerHook(tutils.Placeholder()) + >> tutils.reply_next_layer(TlsEchoLayer) + >> events.DataReceived(tctx.client, b"open-connection") + << commands.OpenConnection(tctx.server) + >> tutils.reply(None) + << quic.QuicStartServerHook(tutils.Placeholder()) + >> reply_tls_start_server() + << commands.SendData(tctx.server, data) + << commands.RequestWakeup(0.2) + ) + + # receive ServerHello, finish client handshake + tssl.write(data()) + assert not tssl.handshake_completed() + + # exchange termination data + data = tutils.Placeholder(bytes) + assert ( + playbook + >> events.DataReceived(tctx.server, tssl.read()) + << commands.SendData(tctx.server, data) + ) + tssl.write(data()) + tssl.now = tssl.now + 60 + + tls_hook_data = tutils.Placeholder(quic.QuicTlsData) + assert ( + playbook + >> events.Wakeup(playbook.actual[9]) + << commands.Log( + "Server QUIC handshake failed. hostname 'wrong.host.mitmproxy.org' doesn't match 'example.mitmproxy.org'", + WARNING + ) + << tls.TlsFailedServerHook(tls_hook_data) + >> tutils.reply() + << commands.CloseConnection(tctx.server) + << commands.SendData( + tctx.client, + b"open-connection failed: hostname 'wrong.host.mitmproxy.org' doesn't match 'example.mitmproxy.org'", + ) + ) + assert ( + tls_hook_data().conn.error == "hostname 'wrong.host.mitmproxy.org' doesn't match 'example.mitmproxy.org'" + ) + assert not tctx.server.tls_established + + +def make_client_tls_layer( + tctx: context.Context, **kwargs +) -> tuple[tutils.Playbook, tls.ClientTLSLayer, SSLTest]: + tssl_client = SSLTest(alpn=quic.H3_ALPN, **kwargs) + tssl_client.quic.connect(("example.mitmproxy.org", 443), 0) + + # This is a bit contrived as the client layer expects a server layer as parent. + # We also set child layers manually to avoid NextLayer noise. + server_layer = quic.ServerQuicLayer(tctx, time=lambda: tssl_client.now) + client_layer = quic.ClientQuicLayer(tctx, time=lambda: tssl_client.now) + server_layer.child_layer = client_layer + playbook = tutils.Playbook(server_layer) + + # Add some server config, this is needed anyways. + tctx.server.__dict__["address"] = ( + "example.mitmproxy.org", + 443, + ) # .address fails because connection is open + tctx.server.sni = "example.mitmproxy.org" + + # Start handshake. + assert not tssl_client.handshake_completed() + + return playbook, client_layer, tssl_client + + +class TestClientTLS: + def test_client_only(self, tctx: context.Context): + """Test TLS with client only""" + playbook, client_layer, tssl_client = make_client_tls_layer(tctx) + client_layer.debug = " " + assert not tctx.client.tls_established + + # Send ClientHello, receive ServerHello + data = tutils.Placeholder(bytes) + assert ( + playbook + >> events.DataReceived(tctx.client, tssl_client.read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply() + << quic.QuicStartClientHook(tutils.Placeholder()) + >> reply_tls_start_client() + << commands.SendData(tctx.client, data) + << commands.RequestWakeup(tutils.Placeholder()) + ) + tssl_client.write(data()) + assert tssl_client.handshake_completed() + # Finish Handshake + finish_handshake(playbook, tctx.client, tssl_client, TlsEchoLayer) + + assert tssl_client.quic.tls._peer_certificate + assert tctx.client.tls_established + + # Echo + _test_echo(playbook, tssl_client, tctx.client) + other_server = connection.Server(None) + assert ( + playbook + >> events.DataReceived(other_server, b"Plaintext") + << commands.SendData(other_server, b"plaintext") + ) \ No newline at end of file From 54350292d94b91c5d3a462f7ebd9b2fd2ae5b33b Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 28 Oct 2022 02:13:38 +0200 Subject: [PATCH 085/695] [quic] refine alpn handling --- mitmproxy/addons/tlsconfig.py | 38 ++++++++-- mitmproxy/proxy/layers/quic.py | 21 +++--- test/mitmproxy/proxy/layers/test_quic.py | 90 ++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 23 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index e97b363f9b..e3879613c3 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -308,25 +308,37 @@ def quic_start_client(self, tls_start: quic.QuicTlsData) -> None: return # a user addon has already provided the settings. tls_start.settings = quic.QuicTlsSettings() + # keep the following part in sync with `tls_start_client` assert isinstance(tls_start.conn, connection.Client) client: connection.Client = tls_start.conn server: connection.Server = tls_start.context.server entry = self.get_cert(tls_start.context) - tls_start.settings.certificate = entry.cert._cert - tls_start.settings.certificate_private_key = entry.privatekey - tls_start.settings.certificate_chain = [cert._cert for cert in entry.chain_certs] if not client.cipher_list and ctx.options.ciphers_client: client.cipher_list = ctx.options.ciphers_client.split(":") + + if ctx.options.add_upstream_certs_to_client_chain: + extra_chain_certs = server.certificate_list + else: + extra_chain_certs = [] + + # set context parameters if client.cipher_list: tls_start.settings.cipher_suites = [ CipherSuite[cipher] for cipher in client.cipher_list ] + tls_start.settings.alpn_protocols = [ + alpn.decode("ascii") for alpn in (client.alpn, server.alpn) if alpn + ] - if ctx.options.add_upstream_certs_to_client_chain: - tls_start.settings.certificate_chain.extend(cert._cert for cert in server.certificate_list) + # set the certificates + tls_start.settings.certificate = entry.cert._cert + tls_start.settings.certificate_private_key = entry.privatekey + tls_start.settings.certificate_chain = [ + cert._cert for cert in (*entry.chain_certs, *extra_chain_certs) + ] def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: """Establish QUIC between proxy and server.""" @@ -334,6 +346,7 @@ def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: return # a user addon has already provided the settings. tls_start.settings = quic.QuicTlsSettings() + # keep the following part in sync with `tls_start_server` assert isinstance(tls_start.conn, connection.Server) client: connection.Client = tls_start.context.client @@ -342,20 +355,32 @@ def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: if ctx.options.ssl_insecure: tls_start.settings.verify_mode = ssl.CERT_NONE + else: + tls_start.settings.verify_mode = ssl.CERT_REQUIRED if server.sni is None: server.sni = client.sni or server.address[0] if not server.alpn_offers: - server.alpn_offers = client.alpn_offers + if client.alpn_offers: + server.alpn_offers = tuple(client.alpn_offers) + else: + server.alpn_offers = [] if not server.cipher_list and ctx.options.ciphers_server: server.cipher_list = ctx.options.ciphers_server.split(":") + + # set context parameters if server.cipher_list: tls_start.settings.cipher_suites = [ CipherSuite[cipher] for cipher in server.cipher_list ] + if server.alpn_offers: + tls_start.settings.alpn_protocols = [ + alpn.decode("ascii") for alpn in server.alpn_offers + ] + # set the certificates client_cert = self.get_client_cert(server) if client_cert: config = QuicConfiguration() @@ -366,7 +391,6 @@ def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: assert isinstance(config.private_key, (dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey)) tls_start.settings.certificate_private_key = config.private_key tls_start.settings.certificate_chain = config.certificate_chain - tls_start.settings.ca_path = ctx.options.ssl_verify_upstream_trusted_confdir tls_start.settings.ca_file = ctx.options.ssl_verify_upstream_trusted_ca diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 0725ad9a5c..287b1d3cf4 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Callable from aioquic.buffer import Buffer as QuicBuffer -from aioquic.h3.connection import H3_ALPN, ErrorCode as H3ErrorCode +from aioquic.h3.connection import ErrorCode as H3ErrorCode from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import ( @@ -51,6 +51,8 @@ class QuicTlsSettings: Settings necessary to establish QUIC's TLS context. """ + alpn_protocols: list[str] | None = None + """A list of supported ALPN protocols.""" certificate: x509.Certificate | None = None """The certificate to use for the connection.""" certificate_chain: list[x509.Certificate] = field(default_factory=list) @@ -331,8 +333,8 @@ def __init__(self, context: context.Context, stream: QuicStreamLayer, ask_on_sta self._stream = stream self._layer: layer.Layer | None = None - @property - def layer(self) -> layer.Layer | None: + @property # type: ignore + def layer(self) -> layer.Layer | None: # type: ignore return self._layer @layer.setter @@ -414,7 +416,7 @@ def open_server_stream(self, server_stream_id) -> None: def refresh_metadata(self) -> None: # find the first transport layer - child_layer = self.child_layer + child_layer: layer.Layer | None = self.child_layer while True: if isinstance(child_layer, layer.NextLayer): child_layer = child_layer.layer @@ -481,7 +483,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: err = yield commands.OpenConnection(self.context.server) if err: yield commands.CloseConnection(self.context.client) - self._handle_event = self.done + self._handle_event = self.done # type: ignore return yield from self.event_to_child(self.datagram_layer, event) @@ -574,7 +576,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if other_conn.connected: yield CloseQuicConnection(other_conn, event.error_code, event.frame_type, event.reason_phrase) else: - self._handle_event = self.done + self._handle_event = self.done # type: ignore # always forward to the datagram layer and swallow `CloseConnection` commands for command in self.event_to_child(self.datagram_layer, event): @@ -760,11 +762,7 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C # build the aioquic connection configuration = QuicConfiguration( - alpn_protocols=( - [offer.decode("ascii") for offer in self.conn.alpn_offers] - if self.conn.alpn_offers else - H3_ALPN - ), + alpn_protocols=tls_data.settings.alpn_protocols, is_client=self.conn is self.context.server, secrets_log_file=( QuicSecretsLogger(tls.log_master_secret) # type: ignore @@ -916,6 +914,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: yield from self.tls_interact() def receive_close(self) -> layer.CommandGenerator[None]: + assert self.quic # if `_close_event` is not set, the underlying connection has been closed # we turn this into a QUIC close event as well close_event = self.quic._close_event or quic_events.ConnectionTerminated( diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index 60f00a6df3..c3a2ba3383 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -4,7 +4,7 @@ from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.connection import QuicConnection, pull_quic_header -from typing import Optional, TypeVar +from typing import Literal, Optional, TypeVar from unittest.mock import MagicMock import pytest from mitmproxy import connection, options @@ -508,7 +508,9 @@ def set_layer(next_layer: layer.NextLayer) -> None: return result -def reply_tls_start_client(*args, **kwargs) -> tutils.reply: +def reply_tls_start_client( + alpn: Optional[str] = None, *args, **kwargs +) -> tutils.reply: """ Helper function to simplify the syntax for quic_start_client hooks. """ @@ -524,11 +526,15 @@ def make_client_conn(tls_start: quic.QuicTlsData) -> None: certificate_chain = config.certificate_chain, certificate_private_key = config.private_key, ) + if alpn is not None: + tls_start.settings.alpn_protocols = [alpn] return tutils.reply(*args, side_effect=make_client_conn, **kwargs) -def reply_tls_start_server(*args, **kwargs) -> tutils.reply: +def reply_tls_start_server( + alpn: Optional[str] = None, *args, **kwargs +) -> tutils.reply: """ Helper function to simplify the syntax for quic_start_server hooks. """ @@ -538,6 +544,8 @@ def make_server_conn(tls_start: quic.QuicTlsData) -> None: ca_file=tlsdata.path("../../net/data/verificationcerts/trusted-root.crt"), verify_mode=ssl.CERT_REQUIRED, ) + if alpn is not None: + tls_start.settings.alpn_protocols = [alpn] return tutils.reply(*args, side_effect=make_server_conn, **kwargs) @@ -674,7 +682,7 @@ def test_untrusted_cert(self, tctx: context.Context): def make_client_tls_layer( tctx: context.Context, **kwargs ) -> tuple[tutils.Playbook, tls.ClientTLSLayer, SSLTest]: - tssl_client = SSLTest(alpn=quic.H3_ALPN, **kwargs) + tssl_client = SSLTest(**kwargs) tssl_client.quic.connect(("example.mitmproxy.org", 443), 0) # This is a bit contrived as the client layer expects a server layer as parent. @@ -731,4 +739,76 @@ def test_client_only(self, tctx: context.Context): playbook >> events.DataReceived(other_server, b"Plaintext") << commands.SendData(other_server, b"plaintext") - ) \ No newline at end of file + ) + + @pytest.mark.parametrize("server_state", ["open", "closed"]) + def test_server_required(self, tctx: context.Context, server_state: Literal["open", "closed"]): + """ + Test the scenario where a server connection is required (for example, because of an unknown ALPN) + to establish TLS with the client. + """ + if server_state == "open": + tctx.server.state = connection.ConnectionState.OPEN + tssl_server = SSLTest(server_side=True, alpn=["quux"]) + playbook, client_layer, tssl_client = make_client_tls_layer(tctx, alpn=["quux"]) + + # We should now get instructed to open a server connection. + data = tutils.Placeholder(bytes) + + def require_server_conn(client_hello: tls.ClientHelloData) -> None: + client_hello.establish_server_tls_first = True + + ( + playbook + >> events.DataReceived(tctx.client, tssl_client.read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply(side_effect=require_server_conn) + ) + if server_state == "closed": + playbook << commands.OpenConnection(tctx.server) + playbook >> tutils.reply(None) + assert ( + playbook + << quic.QuicStartServerHook(tutils.Placeholder()) + >> reply_tls_start_server(alpn="quux") + << commands.SendData(tctx.server, data) + << commands.RequestWakeup(tutils.Placeholder()) + ) + + # Establish TLS with the server... + tssl_server.write(data()) + assert not tssl_server.handshake_completed() + + data = tutils.Placeholder(bytes) + assert ( + playbook + >> events.DataReceived(tctx.server, tssl_server.read()) + << tls.TlsEstablishedServerHook(tutils.Placeholder()) + >> tutils.reply() + << commands.SendData(tctx.server, data) + << commands.RequestWakeup(tutils.Placeholder()) + << quic.QuicStartClientHook(tutils.Placeholder()) + ) + tssl_server.write(data()) + assert tctx.server.tls_established + # Server TLS is established, we can now reply to the client handshake... + + data = tutils.Placeholder(bytes) + assert ( + playbook + >> reply_tls_start_client(alpn="quux") + << commands.SendData(tctx.client, data) + << commands.RequestWakeup(tutils.Placeholder()) + ) + tssl_client.write(data()) + assert tssl_client.handshake_completed() + finish_handshake(playbook, tctx.client, tssl_client, TlsEchoLayer) + + # Both handshakes completed! + assert tctx.client.tls_established + assert tctx.server.tls_established + assert tctx.server.sni == tctx.client.sni + assert tctx.client.alpn == b"quux" + assert tctx.server.alpn == b"quux" + _test_echo(playbook, tssl_server, tctx.server) + _test_echo(playbook, tssl_client, tctx.client) From 1ac59fe369c04ad2ff7e10a36c696f3e0034a811 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 28 Oct 2022 03:04:12 +0200 Subject: [PATCH 086/695] [quic] fix ignore_connection issue --- mitmproxy/proxy/layers/quic.py | 2 +- test/mitmproxy/proxy/layers/test_quic.py | 113 ++++++++++++++++++++++- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 287b1d3cf4..eee197f149 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1072,7 +1072,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo parent_layer.handle_event = replacement_layer.handle_event # type: ignore parent_layer._handle_event = replacement_layer._handle_event # type: ignore yield from parent_layer.handle_event(events.Start()) - yield from parent_layer.handle_event(events.DataReceived(self.conn, data)) + yield from parent_layer.handle_event(events.DataReceived(self.context.client, data)) return True, None # start the server QUIC connection if demanded and available diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index c3a2ba3383..4f65a5a1c6 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -1,5 +1,6 @@ -from logging import WARNING +from logging import DEBUG, WARNING import ssl +import time from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration @@ -683,7 +684,7 @@ def make_client_tls_layer( tctx: context.Context, **kwargs ) -> tuple[tutils.Playbook, tls.ClientTLSLayer, SSLTest]: tssl_client = SSLTest(**kwargs) - tssl_client.quic.connect(("example.mitmproxy.org", 443), 0) + tssl_client.quic.connect(tssl_client.address, 0) # This is a bit contrived as the client layer expects a server layer as parent. # We also set child layers manually to avoid NextLayer noise. @@ -812,3 +813,111 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: assert tctx.server.alpn == b"quux" _test_echo(playbook, tssl_server, tctx.server) _test_echo(playbook, tssl_client, tctx.client) + + @pytest.mark.parametrize("server_state", ["open", "closed"]) + def test_passthrough_from_clienthello(self, tctx: context.Context, server_state: Literal["open", "closed"]): + """ + Test the scenario where the connection is moved to passthrough mode in the tls_clienthello hook. + """ + if server_state == "open": + tctx.server.timestamp_start = time.time() + tctx.server.state = connection.ConnectionState.OPEN + + playbook, client_layer, tssl_client = make_client_tls_layer(tctx, alpn=["quux"]) + client_layer.child_layer = TlsEchoLayer(client_layer.context) + + def make_passthrough(client_hello: tls.ClientHelloData) -> None: + client_hello.ignore_connection = True + + client_hello = tssl_client.read() + ( + playbook + >> events.DataReceived(tctx.client, client_hello) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply(side_effect=make_passthrough) + ) + if server_state == "closed": + playbook << commands.OpenConnection(tctx.server) + playbook >> tutils.reply(None) + assert ( + playbook + << commands.SendData(tctx.server, client_hello) # passed through unmodified + >> events.DataReceived( + tctx.server, b"ServerHello" + ) # and the same for the serverhello. + << commands.SendData(tctx.client, b"ServerHello") + ) + + def test_cannot_parse_clienthello(self, tctx: context.Context): + """Test the scenario where we cannot parse the ClientHello""" + playbook, client_layer, tssl_client = make_client_tls_layer(tctx) + tls_hook_data = tutils.Placeholder(quic.QuicTlsData) + + invalid = b"\x16\x03\x01\x00\x00" + + assert ( + playbook + >> events.DataReceived(tctx.client, invalid) + << commands.Log( + f"Client QUIC handshake failed. Cannot parse QUIC header: Packet fixed bit is zero ({invalid.hex()})", + level=WARNING, + ) + << tls.TlsFailedClientHook(tls_hook_data) + >> tutils.reply() + << commands.CloseConnection(tctx.client) + ) + assert tls_hook_data().conn.error + assert not tctx.client.tls_established + + # Make sure that an active server connection does not cause child layers to spawn. + client_layer.debug = "" + assert ( + playbook + >> events.DataReceived(connection.Server(None), b"data on other stream") + << commands.Log(">> DataReceived(server, b'data on other stream')", DEBUG) + << commands.Log( + "[quic] Swallowing DataReceived(server, b'data on other stream') as handshake failed.", + DEBUG, + ) + ) + + def test_mitmproxy_ca_is_untrusted(self, tctx: context.Context): + """Test the scenario where the client doesn't trust the mitmproxy CA.""" + playbook, client_layer, tssl_client = make_client_tls_layer( + tctx, sni="wrong.host.mitmproxy.org" + ) + playbook.logs = True + + data = tutils.Placeholder(bytes) + assert ( + playbook + >> events.DataReceived(tctx.client, tssl_client.read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply() + << quic.QuicStartClientHook(tutils.Placeholder()) + >> reply_tls_start_client() + << commands.SendData(tctx.client, data) + << commands.RequestWakeup(tutils.Placeholder()) + ) + tssl_client.write(data()) + assert not tssl_client.handshake_completed() + + # Finish Handshake + tls_hook_data = tutils.Placeholder(quic.QuicTlsData) + playbook >> events.DataReceived(tctx.client, tssl_client.read()) + assert playbook + tssl_client.now = tssl_client.now + 60 + assert ( + playbook + >> events.Wakeup(playbook.actual[7]) + << commands.Log( + "Client QUIC handshake failed. hostname 'wrong.host.mitmproxy.org' doesn't match 'example.mitmproxy.org'", + WARNING, + ) + << tls.TlsFailedClientHook(tls_hook_data) + >> tutils.reply() + << commands.CloseConnection(tctx.client) + >> events.ConnectionClosed(tctx.client) + ) + assert not tctx.client.tls_established + assert tls_hook_data().conn.error From 26b2545dc2ab5ea4f006b6d5a2cf5b55e9899896 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sat, 29 Oct 2022 20:11:53 +0200 Subject: [PATCH 087/695] [quic] full quic.py coverage --- mitmproxy/proxy/layers/quic.py | 14 +- test/mitmproxy/proxy/layers/test_quic.py | 283 +++++++++++++++++++++-- 2 files changed, 267 insertions(+), 30 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index eee197f149..46a0b13046 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -4,7 +4,7 @@ from logging import DEBUG, ERROR, WARNING from ssl import VerifyMode import time -from typing import TYPE_CHECKING, Callable +from typing import Callable from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import ErrorCode as H3ErrorCode @@ -41,9 +41,6 @@ from mitmproxy.proxy.layers.udp import UDPLayer from mitmproxy.tls import ClientHello, ClientHelloData, TlsData -if TYPE_CHECKING: - from mitmproxy.proxy.server import ConnectionHandler - @dataclass class QuicTlsSettings: @@ -423,7 +420,7 @@ def refresh_metadata(self) -> None: elif isinstance(child_layer, tunnel.TunnelLayer): child_layer = child_layer.child_layer else: - break + break # pragma: no cover if isinstance(child_layer, (UDPLayer, TCPLayer)) and child_layer.flow: child_layer.flow.metadata["quic_is_unidirectional"] = stream_is_unidirectional(self._client_stream_id) child_layer.flow.metadata["quic_initiator"] = "client" if stream_is_client_initiated(self._client_stream_id) else "server" @@ -701,7 +698,6 @@ def __init__(self, context: context.Context, conn: connection.Connection, time: self.child_layer = layer.NextLayer(self.context, ask_on_start=True) self._time = time or asyncio.get_running_loop().time self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() - self._routes: dict[connection.Address, ConnectionHandler | None] = dict() conn.tls = True def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -826,8 +822,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo all_certs: list[x509.Certificate] = [] if self.quic.tls._peer_certificate: all_certs.append(self.quic.tls._peer_certificate) - if self.quic.tls._peer_certificate_chain: - all_certs.extend(self.quic.tls._peer_certificate_chain) + all_certs.extend(self.quic.tls._peer_certificate_chain) # set the connection's TLS properties self.conn.timestamp_tls_setup = time.time() @@ -983,7 +978,7 @@ def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: class ClientQuicLayer(QuicLayer): """ - This layer establishes QUIC on a single client connection or roams to another connection. + This layer establishes QUIC on a single client connection. """ server_tls_available: bool @@ -1091,6 +1086,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # start the client QUIC connection yield from self.start_tls(header.destination_cid) + # XXX copied from TLS, we assume that `CloseConnection` in `start_tls` takes effect immediately if not self.conn.connected: return False, "connection closed early" diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index 4f65a5a1c6..f6fb30d7da 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -1,4 +1,4 @@ -from logging import DEBUG, WARNING +from logging import DEBUG, ERROR, WARNING import ssl import time from aioquic.buffer import Buffer as QuicBuffer @@ -35,16 +35,12 @@ def tctx() -> context.Context: ) -class InvalidStreamEvent(quic.QuicStreamEvent): - pass +class DummyLayer(layer.Layer): + child_layer: Optional[layer.Layer] - -class InvalidEvent(events.Event): - pass - - -class InvalidConnectionCommand(commands.ConnectionCommand): - pass + def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: + assert self.child_layer + return self.child_layer.handle_event(event) class TlsEchoLayer(tutils.EchoLayer): @@ -58,10 +54,26 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield commands.SendData( event.connection, f"open-connection failed: {err}".encode() ) + elif isinstance(event, events.DataReceived) and event.data == b"close-connection": + yield commands.CloseConnection(event.connection) + elif isinstance(event, events.DataReceived) and event.data == b"close-connection-error": + yield quic.CloseQuicConnection(event.connection, ~0, None, "error") + elif isinstance(event, events.DataReceived) and event.data == b"stop-stream": + yield quic.StopQuicStream(event.connection, 24, 123) elif isinstance(event, events.DataReceived) and event.data == b"invalid-command": + class InvalidConnectionCommand(commands.ConnectionCommand): + pass yield InvalidConnectionCommand(event.connection) + elif isinstance(event, events.DataReceived) and event.data == b"invalid-stream-command": + class InvalidStreamCommand(quic.QuicStreamCommand): + pass + yield InvalidStreamCommand(event.connection, 42) elif isinstance(event, quic.QuicConnectionClosed): self.closed = event + elif isinstance(event, quic.QuicStreamDataReceived): + yield quic.SendQuicStreamData(event.connection, event.stream_id, event.data, event.end_stream) + elif isinstance(event, quic.QuicStreamReset): + yield quic.ResetQuicStream(event.connection, event.stream_id, event.error_code) else: yield from super()._handle_event(event) @@ -124,7 +136,6 @@ def test_secrets_logger(value: str): class TestParseClientHello: - def test_input(self): assert quic.quic_parse_client_hello(client_hello).sni == "example.com" with pytest.raises(ValueError): @@ -154,13 +165,9 @@ def raise_conn_err(self, data, addr, now): with pytest.raises(ValueError, match="Conn err"): quic.quic_parse_client_hello(client_hello) - def test_no_return(self, monkeypatch): - def do_nothing(self, data, addr, now): - pass - - monkeypatch.setattr(QuicConnection, "receive_datagram", do_nothing) + def test_no_return(self): with pytest.raises(ValueError, match="No ClientHello"): - quic.quic_parse_client_hello(client_hello) + quic.quic_parse_client_hello(client_hello[0:1200] + b'\x00' + client_hello[1200:]) class TestQuicStreamLayer: @@ -320,6 +327,8 @@ def test_invalid_stream_event(self, tctx: context.Context): >> tutils.reply(None) ) with pytest.raises(AssertionError, match="Unexpected stream event"): + class InvalidStreamEvent(quic.QuicStreamEvent): + pass playbook >> InvalidStreamEvent(tctx.client, 0) assert playbook @@ -331,6 +340,8 @@ def test_invalid_event(self, tctx: context.Context): >> tutils.reply(None) ) with pytest.raises(AssertionError, match="Unexpected event"): + class InvalidEvent(events.Event): + pass playbook >> InvalidEvent() assert playbook @@ -383,6 +394,125 @@ def test_invalid_connection_command(self, tctx: context.Context): assert playbook +class MockQuic(QuicConnection): + def __init__(self, event) -> None: + super().__init__(configuration=QuicConfiguration(is_client=True)) + self.event = event + + def next_event(self): + event = self.event + self.event = None + return event + + def datagrams_to_send(self, now: float): + return [] + + def get_timer(self): + return None + + +def make_mock_quic( + tctx: context.Context, + event: Optional[quic_events.QuicEvent] = None, + established: bool = True +) -> tuple[tutils.Playbook, MockQuic]: + tctx.client.state = connection.ConnectionState.CLOSED + quic_layer = quic.QuicLayer(tctx, tctx.client, time=lambda: 0) + quic_layer.child_layer = TlsEchoLayer(tctx) + mock = MockQuic(event) + quic_layer.quic = mock + quic_layer.tunnel_state = ( + tls.tunnel.TunnelState.OPEN + if established else + tls.tunnel.TunnelState.ESTABLISHING + ) + return tutils.Playbook(quic_layer), mock + + +class TestQuicLayer: + @pytest.mark.parametrize("established", [True, False]) + def test_invalid_event(self, tctx: context.Context, established: bool): + class InvalidEvent(quic_events.QuicEvent): + pass + playbook, conn = make_mock_quic( + tctx, event=InvalidEvent(), established=established + ) + with pytest.raises(AssertionError, match="Unexpected event"): + assert ( + playbook + >> events.DataReceived(tctx.client, b"") + ) + + def test_invalid_stream_command(self, tctx: context.Context): + playbook, conn = make_mock_quic( + tctx, quic_events.DatagramFrameReceived(b"invalid-stream-command") + ) + with pytest.raises(AssertionError, match="Unexpected stream command"): + assert (playbook >> events.DataReceived(tctx.client, b"")) + + def test_close(self, tctx: context.Context): + playbook, conn = make_mock_quic( + tctx, quic_events.DatagramFrameReceived(b"close-connection") + ) + assert not conn._close_event + assert ( + playbook + >> events.DataReceived(tctx.client, b"") + << commands.CloseConnection(tctx.client) + ) + assert conn._close_event + assert conn._close_event.error_code == 0 + + def test_close_error(self, tctx: context.Context): + playbook, conn = make_mock_quic( + tctx, quic_events.DatagramFrameReceived(b"close-connection-error") + ) + assert not conn._close_event + assert ( + playbook + >> events.DataReceived(tctx.client, b"") + << quic.CloseQuicConnection(tctx.client, ~0, None, "error") + ) + assert conn._close_event + assert conn._close_event.error_code == ~0 + + def test_datagram(self, tctx: context.Context): + playbook, conn = make_mock_quic( + tctx, quic_events.DatagramFrameReceived(b"packet") + ) + assert not conn._datagrams_pending + assert (playbook >> events.DataReceived(tctx.client, b"")) + assert len(conn._datagrams_pending) == 1 + assert conn._datagrams_pending[0] == b"packet" + + def test_stream_data(self, tctx: context.Context): + playbook, conn = make_mock_quic( + tctx, quic_events.StreamDataReceived(b"packet", False, 42) + ) + assert 42 not in conn._streams + assert (playbook >> events.DataReceived(tctx.client, b"")) + assert b"packet" == conn._streams[42].sender._buffer + + def test_stream_reset(self, tctx: context.Context): + playbook, conn = make_mock_quic( + tctx, quic_events.StreamReset(123, 42) + ) + assert 42 not in conn._streams + assert (playbook >> events.DataReceived(tctx.client, b"")) + assert conn._streams[42].sender.reset_pending + assert conn._streams[42].sender._reset_error_code == 123 + + def test_stream_stop(self, tctx: context.Context): + playbook, conn = make_mock_quic( + tctx, quic_events.DatagramFrameReceived(b"stop-stream") + ) + assert 24 not in conn._streams + conn._get_or_create_stream_for_send(24) + assert (playbook >> events.DataReceived(tctx.client, b"")) + assert conn._streams[24].receiver.stop_pending + assert conn._streams[24].receiver._stop_error_code == 123 + + class SSLTest: """Helper container for Python's builtin SSL object.""" @@ -391,6 +521,7 @@ def __init__( server_side: bool = False, alpn: Optional[list[str]] = None, sni: Optional[str] = "example.mitmproxy.org", + version: Optional[int] = None, ): self.ctx = QuicConfiguration( is_client=not server_side, @@ -420,6 +551,9 @@ def __init__( self.ctx.server_name = None if server_side else sni + if version is not None: + self.ctx.supported_versions = [version] + self.now = 0.0 self.address = (sni, 443) self.quic = None if server_side else QuicConnection(configuration=self.ctx) @@ -681,14 +815,13 @@ def test_untrusted_cert(self, tctx: context.Context): def make_client_tls_layer( - tctx: context.Context, **kwargs -) -> tuple[tutils.Playbook, tls.ClientTLSLayer, SSLTest]: + tctx: context.Context, no_server: bool = False, **kwargs +) -> tuple[tutils.Playbook, quic.ClientQuicLayer, SSLTest]: tssl_client = SSLTest(**kwargs) - tssl_client.quic.connect(tssl_client.address, 0) # This is a bit contrived as the client layer expects a server layer as parent. # We also set child layers manually to avoid NextLayer noise. - server_layer = quic.ServerQuicLayer(tctx, time=lambda: tssl_client.now) + server_layer = DummyLayer(tctx) if no_server else quic.ServerQuicLayer(tctx, time=lambda: tssl_client.now) client_layer = quic.ClientQuicLayer(tctx, time=lambda: tssl_client.now) server_layer.child_layer = client_layer playbook = tutils.Playbook(server_layer) @@ -701,6 +834,7 @@ def make_client_tls_layer( tctx.server.sni = "example.mitmproxy.org" # Start handshake. + tssl_client.quic.connect(tssl_client.address, now=tssl_client.now) assert not tssl_client.handshake_completed() return playbook, client_layer, tssl_client @@ -742,6 +876,16 @@ def test_client_only(self, tctx: context.Context): << commands.SendData(other_server, b"plaintext") ) + # test the close log + tssl_client.now = tssl_client.now + 60 + assert ( + playbook + >> events.Wakeup(playbook.actual[16]) + << commands.Log(" >> Wakeup(command=RequestWakeup({'delay': 0.20000000000000004}))", DEBUG) + << commands.Log(" [quic] close_notify Client(client:1234, state=open, tls) (reason=Idle timeout)", DEBUG) + << commands.CloseConnection(tctx.client) + ) + @pytest.mark.parametrize("server_state", ["open", "closed"]) def test_server_required(self, tctx: context.Context, server_state: Literal["open", "closed"]): """ @@ -921,3 +1065,100 @@ def test_mitmproxy_ca_is_untrusted(self, tctx: context.Context): ) assert not tctx.client.tls_established assert tls_hook_data().conn.error + + def test_server_unavailable_and_no_settings(self, tctx: context.Context): + playbook, client_layer, tssl_client = make_client_tls_layer(tctx) + + def require_server_conn(client_hello: tls.ClientHelloData) -> None: + client_hello.establish_server_tls_first = True + + assert ( + playbook + >> events.DataReceived(tctx.client, tssl_client.read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply(side_effect=require_server_conn) + << commands.OpenConnection(tctx.server) + >> tutils.reply("I cannot open the server, Dave") + << commands.Log( + f"Unable to establish QUIC connection with server (I cannot open the server, Dave). " + f"Trying to establish QUIC with client anyway. " + f"If you plan to redirect requests away from this server, " + f"consider setting `connection_strategy` to `lazy` to suppress early connections." + ) + << quic.QuicStartClientHook(tutils.Placeholder()) + ) + tctx.client.state = connection.ConnectionState.CLOSED + assert ( + playbook + >> tutils.reply() + << commands.Log(f"No QUIC context was provided, failing connection.", ERROR) + << commands.CloseConnection(tctx.client) + << commands.Log("Client QUIC handshake failed. connection closed early", WARNING) + << tls.TlsFailedClientHook(tutils.Placeholder()) + ) + + def test_no_server_tls(self, tctx: context.Context): + playbook, client_layer, tssl_client = make_client_tls_layer(tctx, no_server=True) + + def require_server_conn(client_hello: tls.ClientHelloData) -> None: + client_hello.establish_server_tls_first = True + + assert ( + playbook + >> events.DataReceived(tctx.client, tssl_client.read()) + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply(side_effect=require_server_conn) + << commands.Log( + f"Unable to establish QUIC connection with server (No server QUIC available.). " + f"Trying to establish QUIC with client anyway. " + f"If you plan to redirect requests away from this server, " + f"consider setting `connection_strategy` to `lazy` to suppress early connections." + ) + << quic.QuicStartClientHook(tutils.Placeholder()) + ) + + def test_version_negotiation(self, tctx: context.Context): + playbook, client_layer, tssl_client = make_client_tls_layer(tctx, version=0) + assert ( + playbook + >> events.DataReceived(tctx.client, tssl_client.read()) + << commands.SendData(tctx.client, tutils.Placeholder()) + ) + assert client_layer.tunnel_state == tls.tunnel.TunnelState.ESTABLISHING + + def test_non_init_clienthello(self, tctx: context.Context): + playbook, client_layer, tssl_client = make_client_tls_layer(tctx) + data = ( + b'\xc2\x00\x00\x00\x01\x08q\xda\x98\x03X-\x13o\x08y\xa5RQv\xbe\xe3\xeb\x00@a\x98\x19\xf95t\xad-\x1c\\a\xdd\x8c\xd0\x15F' + b'\xdf\xdc\x87cb\x1eu\xb0\x95*\xac\xa8\xf7a \xb8\nQ\xbd=\xf5x\xca\r\xe6\x8b\x05 w\x9f\xcd\x8d\xcb\xa0\x06\x1e \x8d.\x8f' + b'T\xda\x12et\xe4\x83\x93X\x8aa\xd1\xb2\x18\xb6\xa7\xf50y\x9b\xc5T\xe1\x87\xdd\x9fqv\xb0\x90\xa7s' + b'\xee\x00\x00\x00\x01\x08q\xda\x98\x03X-\x13o\x08y\xa5RQv\xbe\xe3\xeb@a*.\xa8j\x90\x1b\x1a\x7fZ\x04\x0b\\\xc7\x00\x03' + b'\xd7sC\xf8G\x84\x1e\xba\xcf\x08Z\xdd\x98+\xaa\x98J\xca\xe3\xb7u1\x89\x00\xdf\x8e\x16`\xd9^\xc0@i\x1a\x10\x99\r\xd8' + b'\x1dv3\xc6\xb8"\xb9\xa8F\x95K\x9a/\xbc\'\xd8\xd8\x94\x8f\xe7B/\x05\x9d\xfb\x80\xa9\xda@\xe6\xb0J\xfe\xe0\x0f\x02L}' + b'\xd9\xed\xd2L\xa7\xcf' + ) + assert ( + playbook + >> events.DataReceived(tctx.client, data) + << commands.Log(f"Client QUIC handshake failed. Invalid handshake received, roaming not supported. ({data.hex()})", WARNING) + << tls.TlsFailedClientHook(tutils.Placeholder()) + ) + assert client_layer.tunnel_state == tls.tunnel.TunnelState.ESTABLISHING + + def test_invalid_clienthello(self, tctx: context.Context): + playbook, client_layer, tssl_client = make_client_tls_layer(tctx) + data = client_hello[0:1200] + b'\x00' + client_hello[1200:] + assert ( + playbook + >> events.DataReceived(tctx.client, data) + << commands.Log(f"Client QUIC handshake failed. Cannot parse ClientHello: No ClientHello returned. ({data.hex()})", WARNING) + << tls.TlsFailedClientHook(tutils.Placeholder()) + ) + assert client_layer.tunnel_state == tls.tunnel.TunnelState.ESTABLISHING + + def test_tls_reset(self, tctx: context.Context): + tctx.client.tls = True + tctx.client.sni = "some" + DummyLayer(tctx) + quic.ClientQuicLayer(tctx, time=lambda: 0) + assert tctx.client.sni is None From 3d39c52048e64a626aedaa24163bc2a4cac99264 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 31 Oct 2022 02:23:51 +0100 Subject: [PATCH 088/695] [quic] mode tests --- docs/scripts/api-events.py | 11 ++++- mitmproxy/proxy/layers/quic.py | 50 +++++++++++++++-------- setup.cfg | 1 - test/mitmproxy/addons/test_proxyserver.py | 5 +++ test/mitmproxy/proxy/layers/test_modes.py | 48 +++++++++++++++++++++- test/mitmproxy/proxy/layers/test_quic.py | 15 ++++--- test/mitmproxy/proxy/test_commands.py | 1 + test/mitmproxy/proxy/test_mode_servers.py | 2 +- test/mitmproxy/proxy/test_mode_specs.py | 6 +++ 9 files changed, 109 insertions(+), 30 deletions(-) diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index 7d9971b62f..d4d934bf3f 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -6,7 +6,7 @@ from mitmproxy import hooks, log, addonmanager from mitmproxy.proxy import server_hooks, layer -from mitmproxy.proxy.layers import dns, http, modes, tcp, tls, udp, websocket +from mitmproxy.proxy.layers import dns, http, modes, quic, tcp, tls, udp, websocket known = set() @@ -139,6 +139,15 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: ], ) + category( + "QUIC", + "", + [ + quic.QuicStartClientHook, + quic.QuicStartServerHook, + ], + ) + category( "TLS", "", diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 46a0b13046..ad52219424 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -263,6 +263,33 @@ def is_success_error_code(error_code: int) -> bool: return error_code in (QuicErrorCode.NO_ERROR, H3ErrorCode.H3_NO_ERROR) +def tls_settings_to_configuration( + settings: QuicTlsSettings, + server_side: bool, + sni: str | None = None, +) -> QuicConfiguration: + """Converts `QuicTlsSettings` to `QuicConfiguration`.""" + + return QuicConfiguration( + alpn_protocols=settings.alpn_protocols, + is_client=server_side, + secrets_log_file=( + QuicSecretsLogger(tls.log_master_secret) # type: ignore + if tls.log_master_secret is not None + else None + ), + server_name=sni, + cafile=settings.ca_file, + capath=settings.ca_path, + certificate=settings.certificate, + certificate_chain=settings.certificate_chain, + cipher_suites=settings.cipher_suites, + private_key=settings.certificate_private_key, + verify_mode=settings.verify_mode, + max_datagram_frame_size=65536, + ) + + @dataclass class QuicClientHello(Exception): """Helper error only used in `quic_parse_client_hello`.""" @@ -696,7 +723,7 @@ class QuicLayer(tunnel.TunnelLayer): def __init__(self, context: context.Context, conn: connection.Connection, time: Callable[[], float] | None) -> None: super().__init__(context, tunnel_connection=conn, conn=conn) self.child_layer = layer.NextLayer(self.context, ask_on_start=True) - self._time = time or asyncio.get_running_loop().time + self._time = time or asyncio.get_event_loop().time self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() conn.tls = True @@ -757,23 +784,10 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C return # build the aioquic connection - configuration = QuicConfiguration( - alpn_protocols=tls_data.settings.alpn_protocols, - is_client=self.conn is self.context.server, - secrets_log_file=( - QuicSecretsLogger(tls.log_master_secret) # type: ignore - if tls.log_master_secret is not None - else None - ), - server_name=self.conn.sni, - cafile=tls_data.settings.ca_file, - capath=tls_data.settings.ca_path, - certificate=tls_data.settings.certificate, - certificate_chain=tls_data.settings.certificate_chain, - cipher_suites=tls_data.settings.cipher_suites, - private_key=tls_data.settings.certificate_private_key, - verify_mode=tls_data.settings.verify_mode, - max_datagram_frame_size=65536, + configuration = tls_settings_to_configuration( + settings=tls_data.settings, + server_side=self.conn is self.context.server, + sni=self.conn.sni, ) self.quic = QuicConnection( configuration=configuration, diff --git a/setup.cfg b/setup.cfg index a43c808d59..e167ead672 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,7 +56,6 @@ exclude = mitmproxy/connections.py mitmproxy/contentviews/base.py mitmproxy/contentviews/grpc.py - mitmproxy/contentviews/http3.py mitmproxy/ctx.py mitmproxy/exceptions.py mitmproxy/flow.py diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index e8eecf80de..b31303acf8 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -146,6 +146,11 @@ async def test_inject_fail(caplog) -> None: ps.inject_tcp(tflow.tflow(), True, b"test") assert "Cannot inject TCP messages into non-TCP flows." in caplog.text + ps.inject_udp(tflow.tflow(), True, b"test") + assert "Cannot inject UDP messages into non-UDP flows." in caplog.text + ps.inject_udp(tflow.tudpflow(), True, b"test") + assert "Flow is not from a live connection." in caplog.text + ps.inject_websocket(tflow.twebsocketflow(), True, b"test") assert "Flow is not from a live connection." in caplog.text ps.inject_websocket(tflow.ttcpflow(), True, b"test") diff --git a/test/mitmproxy/proxy/layers/test_modes.py b/test/mitmproxy/proxy/layers/test_modes.py index 3937213751..e3f4a70306 100644 --- a/test/mitmproxy/proxy/layers/test_modes.py +++ b/test/mitmproxy/proxy/layers/test_modes.py @@ -10,12 +10,13 @@ CloseConnection, Log, OpenConnection, + RequestWakeup, SendData, ) from mitmproxy.proxy.context import Context from mitmproxy.proxy.events import ConnectionClosed, DataReceived from mitmproxy.proxy.layer import NextLayer, NextLayerHook -from mitmproxy.proxy.layers import http, modes, tcp, tls +from mitmproxy.proxy.layers import http, modes, quic, tcp, tls, udp from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy.layers.tcp import TcpMessageHook, TcpStartHook from mitmproxy.proxy.layers.tls import ( @@ -26,6 +27,7 @@ from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.tcp import TCPFlow from mitmproxy.test import tflow +from mitmproxy.udp import UDPFlow from test.mitmproxy.proxy.layers.test_tls import ( reply_tls_start_client, reply_tls_start_server, @@ -168,6 +170,50 @@ def test_reverse_dns(tctx): assert server().address == ("8.8.8.8", 53) +@pytest.mark.parametrize("keep_host_header", [True, False]) +def test_quic(tctx: Context, keep_host_header: bool): + tctx.options.keep_host_header = keep_host_header + tctx.server.sni = "other" + tctx.client.proxy_mode = ProxyMode.parse("reverse:quic://1.2.3.4:5") + client_hello = Placeholder(bytes) + + def set_settings(data: quic.QuicTlsData): + data.settings = quic.QuicTlsSettings() + + assert ( + Playbook(modes.ReverseProxy(tctx)) + << OpenConnection(tctx.server) + >> reply(None) + << quic.QuicStartServerHook(Placeholder(quic.QuicTlsData)) + >> reply(side_effect=set_settings) + << SendData(tctx.server, client_hello) + << RequestWakeup(Placeholder(float)) + ) + assert tctx.server.address == ("1.2.3.4", 5) + assert quic.quic_parse_client_hello(client_hello()).sni == ( + "other" if keep_host_header else "1.2.3.4" + ) + + +def test_udp(tctx: Context): + tctx.client.proxy_mode = ProxyMode.parse("reverse:udp://1.2.3.4:5") + flow = Placeholder(UDPFlow) + assert ( + Playbook(modes.ReverseProxy(tctx)) + << OpenConnection(tctx.server) + >> reply(None) + << udp.UdpStartHook(flow) + >> reply() + >> DataReceived(tctx.client, b"test-input") + << udp.UdpMessageHook(flow) + >> reply() + << SendData(tctx.server, b"test-input") + ) + assert tctx.server.address == ("1.2.3.4", 5) + assert len(flow().messages) == 1 + assert flow().messages[0].content == b"test-input" + + @pytest.mark.parametrize("patch", [True, False]) @pytest.mark.parametrize("connection_strategy", ["eager", "lazy"]) def test_reverse_proxy_tcp_over_tls( diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index f6fb30d7da..13e98c1af8 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -57,7 +57,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, events.DataReceived) and event.data == b"close-connection": yield commands.CloseConnection(event.connection) elif isinstance(event, events.DataReceived) and event.data == b"close-connection-error": - yield quic.CloseQuicConnection(event.connection, ~0, None, "error") + yield quic.CloseQuicConnection(event.connection, 123, None, "error") elif isinstance(event, events.DataReceived) and event.data == b"stop-stream": yield quic.StopQuicStream(event.connection, 24, 123) elif isinstance(event, events.DataReceived) and event.data == b"invalid-command": @@ -131,7 +131,6 @@ def test_secrets_logger(value: str): quic_logger = quic.QuicSecretsLogger(logger) assert quic_logger.write(value) == 6 quic_logger.flush() - logger.assert_called_once() logger.assert_called_once_with(None, b"s1 s2") @@ -175,7 +174,7 @@ def test_ignored(self, tctx: context.Context): quic_layer = quic.QuicStreamLayer(tctx, True, 1) assert isinstance(quic_layer.child_layer, layers.TCPLayer) assert not quic_layer.child_layer.flow - quic_layer.child_layer.flow = TCPFlow(MagicMock(), MagicMock()) + quic_layer.child_layer.flow = TCPFlow(tctx.client, tctx.server) quic_layer.refresh_metadata() assert quic_layer.child_layer.flow.metadata["quic_is_unidirectional"] is False assert quic_layer.child_layer.flow.metadata["quic_initiator"] == "server" @@ -187,7 +186,7 @@ def test_ignored(self, tctx: context.Context): def test_simple(self, tctx: context.Context): quic_layer = quic.QuicStreamLayer(tctx, False, 2) assert isinstance(quic_layer.child_layer, layer.NextLayer) - tunnel_layer = tunnel.TunnelLayer(tctx, MagicMock(), MagicMock()) + tunnel_layer = tunnel.TunnelLayer(tctx, tctx.client, tctx.server) quic_layer.child_layer.layer = tunnel_layer tcp_layer = layers.TCPLayer(tctx) tunnel_layer.child_layer = tcp_layer @@ -414,7 +413,7 @@ def get_timer(self): def make_mock_quic( tctx: context.Context, event: Optional[quic_events.QuicEvent] = None, - established: bool = True + established: bool = True, ) -> tuple[tutils.Playbook, MockQuic]: tctx.client.state = connection.ConnectionState.CLOSED quic_layer = quic.QuicLayer(tctx, tctx.client, time=lambda: 0) @@ -471,10 +470,10 @@ def test_close_error(self, tctx: context.Context): assert ( playbook >> events.DataReceived(tctx.client, b"") - << quic.CloseQuicConnection(tctx.client, ~0, None, "error") + << quic.CloseQuicConnection(tctx.client, 123, None, "error") ) assert conn._close_event - assert conn._close_event.error_code == ~0 + assert conn._close_event.error_code == 123 def test_datagram(self, tctx: context.Context): playbook, conn = make_mock_quic( @@ -514,7 +513,7 @@ def test_stream_stop(self, tctx: context.Context): class SSLTest: - """Helper container for Python's builtin SSL object.""" + """Helper container for QuicConnection object.""" def __init__( self, diff --git a/test/mitmproxy/proxy/test_commands.py b/test/mitmproxy/proxy/test_commands.py index c69591e32b..0007d12adb 100644 --- a/test/mitmproxy/proxy/test_commands.py +++ b/test/mitmproxy/proxy/test_commands.py @@ -17,6 +17,7 @@ def test_dataclasses(tconn): assert repr(commands.SendData(tconn, b"foo")) assert repr(commands.OpenConnection(tconn)) assert repr(commands.CloseConnection(tconn)) + assert repr(commands.CloseTcpConnection(tconn, half_close=True)) assert repr(commands.Log("hello")) diff --git a/test/mitmproxy/proxy/test_mode_servers.py b/test/mitmproxy/proxy/test_mode_servers.py index c1228b3caa..eea437ebbf 100644 --- a/test/mitmproxy/proxy/test_mode_servers.py +++ b/test/mitmproxy/proxy/test_mode_servers.py @@ -18,7 +18,7 @@ def test_make(): context = MagicMock() assert ServerInstance.make("regular", manager) - for mode in ["regular", "upstream:example.com", "transparent", "reverse:example.com", "socks5"]: + for mode in ["regular", "http3", "upstream:example.com", "transparent", "reverse:example.com", "socks5"]: inst = ServerInstance.make(mode, manager) assert inst assert inst.make_top_layer(context) diff --git a/test/mitmproxy/proxy/test_mode_specs.py b/test/mitmproxy/proxy/test_mode_specs.py index e2e7eecf93..c6c2a57982 100644 --- a/test/mitmproxy/proxy/test_mode_specs.py +++ b/test/mitmproxy/proxy/test_mode_specs.py @@ -51,9 +51,12 @@ def test_listen_addr(): def test_parse_specific_modes(): assert ProxyMode.parse("regular") + assert ProxyMode.parse("http3") assert ProxyMode.parse("transparent") assert ProxyMode.parse("upstream:https://proxy") + assert ProxyMode.parse("upstream:http3://proxy") assert ProxyMode.parse("reverse:https://host@443") + assert ProxyMode.parse("reverse:http3://host@443") assert ProxyMode.parse("socks5") assert ProxyMode.parse("dns") assert ProxyMode.parse("reverse:dns://8.8.8.8") @@ -68,6 +71,9 @@ def test_parse_specific_modes(): with pytest.raises(ValueError, match="takes no arguments"): ProxyMode.parse("regular:configuration") + with pytest.raises(ValueError, match="takes no arguments"): + ProxyMode.parse("http3:configuration") + with pytest.raises(ValueError, match="invalid upstream proxy scheme"): ProxyMode.parse("upstream:dns://example.com") From 3aba003099c26531eda5d9802cb205f2e6a55115 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 31 Oct 2022 06:23:58 +0100 Subject: [PATCH 089/695] [quic] tls tests --- mitmproxy/addons/tlsconfig.py | 42 ++++------ mitmproxy/proxy/layers/quic.py | 12 +-- test/mitmproxy/addons/test_tlsconfig.py | 100 ++++++++++++++++++++++- test/mitmproxy/proxy/layers/test_quic.py | 65 +++++++++------ 4 files changed, 156 insertions(+), 63 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index e3879613c3..5e068a220e 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -5,10 +5,7 @@ import ssl from typing import Any, Optional, TypedDict -from aioquic.quic.configuration import QuicConfiguration from aioquic.tls import CipherSuite -from cryptography import x509 -from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa from OpenSSL import SSL, crypto from mitmproxy import certs, ctx, exceptions, connection, tls from mitmproxy.net import tls as net_tls @@ -202,19 +199,6 @@ def tls_start_client(self, tls_start: tls.TlsData) -> None: ) tls_start.ssl_conn.set_accept_state() - def get_client_cert(self, server: connection.Server) -> Optional[str]: - if ctx.options.client_certs: - client_certs = os.path.expanduser(ctx.options.client_certs) - if os.path.isfile(client_certs): - return client_certs - else: - assert server.address - server_name: str = server.sni or server.address[0] - p = os.path.join(client_certs, f"{server_name}.pem") - if os.path.isfile(p): - return p - return None - def tls_start_server(self, tls_start: tls.TlsData) -> None: """Establish TLS or DTLS between proxy and server.""" if tls_start.ssl_conn is not None: @@ -259,6 +243,17 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: # don't assign to client.cipher_list, doesn't need to be stored. cipher_list = server.cipher_list or DEFAULT_CIPHERS + client_cert: Optional[str] = None + if ctx.options.client_certs: + client_certs = os.path.expanduser(ctx.options.client_certs) + if os.path.isfile(client_certs): + client_cert = client_certs + else: + server_name: str = server.sni or server.address[0] + p = os.path.join(client_certs, f"{server_name}.pem") + if os.path.isfile(p): + client_cert = p + ssl_ctx = net_tls.create_proxy_server_context( method=net_tls.Method.DTLS_CLIENT_METHOD if tls_start.is_dtls else net_tls.Method.TLS_CLIENT_METHOD, min_version=net_tls.Version[ctx.options.tls_version_server_min], @@ -267,7 +262,7 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: verify=verify, ca_path=ctx.options.ssl_verify_upstream_trusted_confdir, ca_pemfile=ctx.options.ssl_verify_upstream_trusted_ca, - client_cert=self.get_client_cert(server), + client_cert=client_cert, ) tls_start.ssl_conn = SSL.Connection(ssl_ctx) @@ -319,7 +314,7 @@ def quic_start_client(self, tls_start: quic.QuicTlsData) -> None: if not client.cipher_list and ctx.options.ciphers_client: client.cipher_list = ctx.options.ciphers_client.split(":") - if ctx.options.add_upstream_certs_to_client_chain: + if ctx.options.add_upstream_certs_to_client_chain: # pragma: no cover extra_chain_certs = server.certificate_list else: extra_chain_certs = [] @@ -381,16 +376,7 @@ def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: ] # set the certificates - client_cert = self.get_client_cert(server) - if client_cert: - config = QuicConfiguration() - config.load_cert_chain(Path(client_cert)) - assert isinstance(config.certificate, x509.Certificate) - tls_start.settings.certificate = config.certificate - if config.private_key: - assert isinstance(config.private_key, (dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey)) - tls_start.settings.certificate_private_key = config.private_key - tls_start.settings.certificate_chain = config.certificate_chain + # NOTE client certificates are not supported tls_start.settings.ca_path = ctx.options.ssl_verify_upstream_trusted_confdir tls_start.settings.ca_file = ctx.options.ssl_verify_upstream_trusted_ca diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index ad52219424..cb08812417 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -265,20 +265,20 @@ def is_success_error_code(error_code: int) -> bool: def tls_settings_to_configuration( settings: QuicTlsSettings, - server_side: bool, - sni: str | None = None, + is_client: bool, + server_name: str | None = None, ) -> QuicConfiguration: """Converts `QuicTlsSettings` to `QuicConfiguration`.""" return QuicConfiguration( alpn_protocols=settings.alpn_protocols, - is_client=server_side, + is_client=is_client, secrets_log_file=( QuicSecretsLogger(tls.log_master_secret) # type: ignore if tls.log_master_secret is not None else None ), - server_name=sni, + server_name=server_name, cafile=settings.ca_file, capath=settings.ca_path, certificate=settings.certificate, @@ -786,8 +786,8 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C # build the aioquic connection configuration = tls_settings_to_configuration( settings=tls_data.settings, - server_side=self.conn is self.context.server, - sni=self.conn.sni, + is_client=self.conn is self.context.server, + server_name=self.conn.sni, ) self.quic = QuicConnection( configuration=configuration, diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 535d30f428..d5c955c787 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -5,13 +5,14 @@ import pytest +from cryptography import x509 from OpenSSL import SSL from mitmproxy import certs, connection, tls from mitmproxy.addons import tlsconfig from mitmproxy.proxy import context -from mitmproxy.proxy.layers import modes, tls as proxy_tls +from mitmproxy.proxy.layers import modes, quic, tls as proxy_tls from mitmproxy.test import taddons -from test.mitmproxy.proxy.layers import test_tls +from test.mitmproxy.proxy.layers import test_quic, test_tls def test_alpn_select_callback(): @@ -162,6 +163,19 @@ def do_handshake( return True + def quic_do_handshake( + self, + tssl_client: test_quic.SSLTest, + tssl_server: test_quic.SSLTest, + ) -> bool: + tssl_server.write(tssl_client.read()) + tssl_client.write(tssl_server.read()) + tssl_server.write(tssl_client.read()) + return ( + tssl_client.handshake_completed() + and tssl_server.handshake_completed() + ) + def test_tls_start_client(self, tdata): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: @@ -192,6 +206,37 @@ def test_tls_start_client(self, tdata): ("DNS", "example.mitmproxy.org"), ) + def test_quic_start_client(self, tdata): + ta = tlsconfig.TlsConfig() + with taddons.context(ta) as tctx: + ta.configure(["confdir"]) + tctx.configure( + ta, + certs=[ + tdata.path("mitmproxy/net/data/verificationcerts/trusted-leaf.pem") + ], + ciphers_client="CHACHA20_POLY1305_SHA256", + ) + ctx = context.Context( + connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), + tctx.options, + ) + + tls_start = quic.QuicTlsData(ctx.client, context=ctx) + ta.quic_start_client(tls_start) + settings_server = tls_start.settings + settings_server.alpn_protocols = ["h3"] + tssl_server = test_quic.SSLTest(server_side=True, settings=settings_server) + + # assert that a preexisting settings is not overwritten + ta.quic_start_client(tls_start) + assert settings_server is tls_start.settings + + tssl_client = test_quic.SSLTest(alpn=["h3"]) + assert self.quic_do_handshake(tssl_client, tssl_server) + san = tssl_client.quic.tls._peer_certificate.extensions.get_extension_for_class(x509.SubjectAlternativeName) + assert san.value.get_values_for_type(x509.DNSName) == ["example.mitmproxy.org"] + def test_tls_start_server_cannot_verify(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: @@ -251,6 +296,35 @@ def test_tls_start_server_verify_ok(self, hostname, tdata): tssl_server = test_tls.SSLTest(server_side=True, sni=hostname.encode()) assert self.do_handshake(tssl_client, tssl_server) + @pytest.mark.parametrize("hostname", ["example.mitmproxy.org", "192.0.2.42"]) + def test_quic_start_server_verify_ok(self, hostname, tdata): + ta = tlsconfig.TlsConfig() + with taddons.context(ta) as tctx: + ctx = context.Context( + connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), + tctx.options, + ) + ctx.server.address = (hostname, 443) + tctx.configure( + ta, + ssl_verify_upstream_trusted_ca=tdata.path( + "mitmproxy/net/data/verificationcerts/trusted-root.crt" + ), + ) + + tls_start = quic.QuicTlsData(ctx.server, context=ctx) + ta.quic_start_server(tls_start) + settings_client = tls_start.settings + settings_client.alpn_protocols = ["h3"] + tssl_client = test_quic.SSLTest(settings=settings_client) + + # assert that a preexisting ssl_conn is not overwritten + ta.quic_start_server(tls_start) + assert settings_client is tls_start.settings + + tssl_server = test_quic.SSLTest(server_side=True, sni=hostname.encode(), alpn=["h3"]) + assert self.quic_do_handshake(tssl_client, tssl_server) + def test_tls_start_server_insecure(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: @@ -273,6 +347,28 @@ def test_tls_start_server_insecure(self): tssl_server = test_tls.SSLTest(server_side=True) assert self.do_handshake(tssl_client, tssl_server) + def test_quic_start_server_insecure(self): + ta = tlsconfig.TlsConfig() + with taddons.context(ta) as tctx: + ctx = context.Context( + connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), + tctx.options, + ) + ctx.server.address = ("example.mitmproxy.org", 443) + ctx.client.alpn_offers = [b"h3"] + + tctx.configure( + ta, + ssl_verify_upstream_trusted_ca=None, + ssl_insecure=True, + ciphers_server="CHACHA20_POLY1305_SHA256", + ) + tls_start = quic.QuicTlsData(ctx.server, context=ctx) + ta.quic_start_server(tls_start) + tssl_client = test_quic.SSLTest(settings=tls_start.settings) + tssl_server = test_quic.SSLTest(server_side=True, alpn=["h3"]) + assert self.quic_do_handshake(tssl_client, tssl_server) + def test_alpn_selection(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index 13e98c1af8..6b707a8114 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -521,41 +521,53 @@ def __init__( alpn: Optional[list[str]] = None, sni: Optional[str] = "example.mitmproxy.org", version: Optional[int] = None, + settings: Optional[quic.QuicTlsSettings] = None, ): - self.ctx = QuicConfiguration( - is_client=not server_side, - max_datagram_frame_size=65536, - ) - - self.ctx.verify_mode = ssl.CERT_OPTIONAL - self.ctx.load_verify_locations( - cafile=tlsdata.path("../../net/data/verificationcerts/trusted-root.crt"), - ) + if settings is None: + self.ctx = QuicConfiguration( + is_client=not server_side, + max_datagram_frame_size=65536, + ) - if alpn: - self.ctx.alpn_protocols = alpn - if server_side: - if sni == "192.0.2.42": - filename = "trusted-leaf-ip" - else: - filename = "trusted-leaf" - self.ctx.load_cert_chain( - certfile=tlsdata.path( - f"../../net/data/verificationcerts/{filename}.crt" - ), - keyfile=tlsdata.path( - f"../../net/data/verificationcerts/{filename}.key" - ), + self.ctx.verify_mode = ssl.CERT_OPTIONAL + self.ctx.load_verify_locations( + cafile=tlsdata.path("../../net/data/verificationcerts/trusted-root.crt"), ) - self.ctx.server_name = None if server_side else sni + if alpn: + self.ctx.alpn_protocols = alpn + if server_side: + if sni == "192.0.2.42": + filename = "trusted-leaf-ip" + else: + filename = "trusted-leaf" + self.ctx.load_cert_chain( + certfile=tlsdata.path( + f"../../net/data/verificationcerts/{filename}.crt" + ), + keyfile=tlsdata.path( + f"../../net/data/verificationcerts/{filename}.key" + ), + ) + + self.ctx.server_name = None if server_side else sni - if version is not None: - self.ctx.supported_versions = [version] + if version is not None: + self.ctx.supported_versions = [version] + else: + assert alpn is None + assert version is None + self.ctx = quic.tls_settings_to_configuration( + settings=settings, + is_client=not server_side, + server_name=sni, + ) self.now = 0.0 self.address = (sni, 443) self.quic = None if server_side else QuicConnection(configuration=self.ctx) + if not server_side: + self.quic.connect(self.address, now=self.now) def write(self, buf: bytes) -> int: self.now = self.now + 0.1 @@ -833,7 +845,6 @@ def make_client_tls_layer( tctx.server.sni = "example.mitmproxy.org" # Start handshake. - tssl_client.quic.connect(tssl_client.address, now=tssl_client.now) assert not tssl_client.handshake_completed() return playbook, client_layer, tssl_client From f8dc5a66832183bd7f0acc99a96359bf32b69ed8 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 31 Oct 2022 23:02:14 +0100 Subject: [PATCH 090/695] [quic] next layer work --- mitmproxy/addons/next_layer.py | 163 +++++++++---------- test/mitmproxy/addons/test_next_layer.py | 193 +++++++++++++++++++++-- 2 files changed, 259 insertions(+), 97 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index b1cb179f5f..0404bb597b 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -155,23 +155,26 @@ def is_destination_in_hosts(self, context: context.Context, hosts: Iterable[re.P for rex in hosts ) - def is_reverse_proxy_scheme(self, context: context.Context, *args: str): + def get_http_layer(self, context: context.Context) -> Optional[layers.HttpLayer]: def s(*layers): return stack_match(context, layers) - # we allow all possible security layer combinations and rely on the correctness of ReverseProxy - return ( - ( - s(modes.ReverseProxy) - or - s(modes.ReverseProxy, layers.ClientTLSLayer) - or - s(modes.ReverseProxy, layers.ServerTLSLayer) - or - s(modes.ReverseProxy, layers.ServerTLSLayer, layers.ClientTLSLayer) - ) - and cast(mode_specs.ReverseMode, context.client.proxy_mode).scheme in args - ) + # Setup the HTTP layer for a regular HTTP proxy ... + if ( + s(modes.HttpProxy) + or + # or a "Secure Web Proxy", see https://www.chromium.org/developers/design-documents/secure-web-proxy + s(modes.HttpProxy, (layers.ClientTLSLayer, layers.ClientQuicLayer)) + ): + return layers.HttpLayer(context, HTTPMode.regular) + # ... or an upstream proxy. + if ( + s(modes.HttpUpstreamProxy) + or + s(modes.HttpUpstreamProxy, (layers.ClientTLSLayer, layers.ClientQuicLayer)) + ): + return layers.HttpLayer(context, HTTPMode.upstream) + return None def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls]]: if len(data_client) == 0: @@ -195,6 +198,36 @@ def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, Clie # that's all we currently have to offer return None + def raw_udp_layer(self, context: context.Context, ignore: bool = False) -> layer.Layer: + def s(*layers): + return stack_match(context, layers) + + # for regular and upstream HTTP3, if we already created a client QUIC layer + # we need a server and raw QUIC layer as well + if ( + s(modes.HttpProxy, layers.ClientQuicLayer) + or + s(modes.HttpUpstreamProxy, layers.ClientQuicLayer) + ): + server_layer = layers.ServerQuicLayer(context) + server_layer.child_layer = layers.RawQuicLayer(context, ignore=ignore) + return server_layer + + # for reverse HTTP3 and QUIC, we need a client and raw QUIC layer + elif (s(modes.ReverseProxy, layers.ServerQuicLayer)): + client_layer = layers.ClientQuicLayer(context) + client_layer.child_layer = layers.RawQuicLayer(context, ignore=ignore) + return client_layer + + # in other cases we assume `setup_tls_layer` happened, so if the + # top layer is `ClientQuicLayer` we return a raw QUIC layer... + elif isinstance(context.layers[-1], layers.ClientQuicLayer): + return layers.RawQuicLayer(context, ignore=ignore) + + # ... otherwise an UDP layer + else: + return layers.UDPLayer(context, ignore=ignore) + def next_layer(self, nextlayer: layer.NextLayer): if nextlayer.layer is None: nextlayer.layer = self._next_layer( @@ -208,10 +241,6 @@ def _next_layer( ) -> Optional[layer.Layer]: assert context.layers - # helper function to quickly check if the existing layer stack matches a particular configuration. - def s(*layers): - return stack_match(context, layers) - if context.client.transport_protocol == "tcp": if ( len(data_client) < 3 @@ -231,35 +260,15 @@ def s(*layers): if is_tls_record_magic(data_client): return self.setup_tls_layer(context) - # 3. Setup the HTTP layer for a regular HTTP proxy - if ( - s(modes.HttpProxy) - or - # or a "Secure Web Proxy", see https://www.chromium.org/developers/design-documents/secure-web-proxy - s(modes.HttpProxy, layers.ClientTLSLayer) - ): - return layers.HttpLayer(context, HTTPMode.regular) - # 3b. ... or an upstream proxy. - if ( - s(modes.HttpUpstreamProxy) - or - s(modes.HttpUpstreamProxy, layers.ClientTLSLayer) - ): - return layers.HttpLayer(context, HTTPMode.upstream) + # 3. Check for HTTP + if http_layer := self.get_http_layer(context): + return http_layer # 4. Check for --tcp if self.is_destination_in_hosts(context, self.tcp_hosts): return layers.TCPLayer(context) - # 5. Check for raw reverse mode. - if self.is_reverse_proxy_scheme(context, "tcp", "tls"): - return layers.TCPLayer(context) - # NOTE at this point we are either - # - in http or https reverse mode - # - at the top level of a non-reverse/regular/upstream mode - # - at a deeper layer nesting level - - # 6. Check for raw tcp mode. + # 5. Check for raw tcp mode. very_likely_http = context.client.alpn and context.client.alpn in HTTP_ALPNS probably_no_http = not very_likely_http and ( not data_client[ @@ -270,27 +279,12 @@ def s(*layers): if ctx.options.rawtcp and probably_no_http: return layers.TCPLayer(context) - # 7. Assume HTTP by default. + # 6. Assume HTTP by default. return layers.HttpLayer(context, HTTPMode.transparent) elif context.client.transport_protocol == "udp": - # for http3, upstream:http3 and reverse:quic/http3 proxies, there has to be a client quic layer - if ( - s(modes.HttpProxy) - or - s(modes.HttpUpstreamProxy) - or - s(modes.ReverseProxy, layers.ServerQuicLayer) - ): - return layers.ClientQuicLayer(context) - # unlike TCP, we make a decision immediately tls = self.detect_udp_tls(data_client) - raw_layer_cls = ( - layers.RawQuicLayer - if isinstance(context.layers[-1], layers.ClientQuicLayer) else - layers.UDPLayer - ) # 1. check for --ignore/--allow if self.ignore_connection( @@ -299,43 +293,42 @@ def s(*layers): is_tls=lambda _: tls is not None, client_hello=lambda _: None if tls is None else tls[0] ): - return raw_layer_cls(context, ignore=True) + return self.raw_udp_layer(context, ignore=True) # 2. Check for DTLS/QUIC if tls is not None: _, client_layer_cls, server_layer_cls = tls return self.setup_tls_layer(context, client_layer_cls, server_layer_cls) - # 3. Setup the HTTP layer for a regular HTTP proxy - if s(modes.HttpProxy, layers.ClientQuicLayer): - return layers.HttpLayer(context, HTTPMode.regular) - # 3b. ... or an upstream proxy. - if s(modes.HttpUpstreamProxy, layers.ClientQuicLayer): - return layers.HttpLayer(context, HTTPMode.upstream) + # 3. Check for HTTP + if http_layer := self.get_http_layer(context): + return http_layer # 4. Check for --udp if self.is_destination_in_hosts(context, self.udp_hosts): - return raw_layer_cls(context) + return self.raw_udp_layer(context) - # 5. Check for raw reverse mode. - if self.is_reverse_proxy_scheme(context, "udp", "dtls"): - return layers.UDPLayer(context) - - # 6. Check for explicit QUIC reverse modes - if (s(modes.ReverseProxy, layers.ServerQuicLayer, layers.ClientQuicLayer)): + # 5. Check for reverse modes + if (isinstance(context.layers[0], modes.ReverseProxy)): scheme = cast(mode_specs.ReverseMode, context.client.proxy_mode).scheme - if scheme == "quic": - return layers.RawQuicLayer(context) - if scheme == "http3": + if scheme in ("udp", "dtls"): + return layers.UDPLayer(context) + elif scheme == "http3": return layers.HttpLayer(context, HTTPMode.transparent) - # 6b. ... or DNS mode - if self.is_reverse_proxy_scheme(context, "dns"): - return layers.DNSLayer(context) - # NOTE at this point we are either - # - at the top level of a non-reverse/regular/upstream mode - # - at a deeper layer nesting level - - # 7. Check for DNS + elif scheme == "quic": + # if the client supports QUIC, we use QUIC raw layer, + # otherwise we only use the QUIC datagram only + return ( + layers.RawQuicLayer(context) + if isinstance(context.layers[-1], layers.ClientQuicLayer) else + layers.UDPLayer(context) + ) + elif scheme == "dns": + return layers.DNSLayer(context) + else: + raise AssertionError(scheme) + + # 6. Check for DNS try: dns.Message.unpack(data_client) except struct.error: @@ -343,8 +336,8 @@ def s(*layers): else: return layers.DNSLayer(context) - # 8. Use raw mode. - return raw_layer_cls(context) + # 7. Use raw mode. + return self.raw_udp_layer(context) else: raise AssertionError(context.client.transport_protocol) diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index 741f47fe15..ced57924be 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -46,6 +46,39 @@ def tctx(): ) +quic_client_hello = bytes.fromhex( + "ca0000000108c0618c84b54541320823fcce946c38d8210044e6a93bbb283593f75ffb6f2696b16cfdcb5b1255" + "577b2af5fc5894188c9568bc65eef253faf7f0520e41341cfa81d6aae573586665ce4e1e41676364820402feec" + "a81f3d22dbb476893422069066104a43e121c951a08c53b83f960becf99cf5304d5bc5346f52f472bd1a04d192" + "0bae025064990d27e5e4c325ac46121d3acadebe7babdb96192fb699693d65e2b2e21c53beeb4f40b50673a2f6" + "c22091cb7c76a845384fedee58df862464d1da505a280bfef91ca83a10bebbcb07855219dbc14aecf8a48da049" + "d03c77459b39d5355c95306cd03d6bdb471694fa998ca3b1f875ce87915b88ead15c5d6313a443f39aad808922" + "57ddfa6b4a898d773bb6fb520ede47ebd59d022431b1054a69e0bbbdf9f0fb32fc8bcc4b6879dd8cd5389474b1" + "99e18333e14d0347740a11916429a818bb8d93295d36e99840a373bb0e14c8b3adcf5e2165e70803f15316fd5e" + "5eeec04ae68d98f1adb22c54611c80fcd8ece619dbdf97b1510032ec374b7a71f94d9492b8b8cb56f56556dd97" + "edf1e50fa90e868ff93636a365678bdf3ee3f8e632588cd506b6f44fbfd4d99988238fbd5884c98f6a124108c1" + "878970780e42b111e3be6215776ef5be5a0205915e6d720d22c6a81a475c9e41ba94e4983b964cb5c8e1f40607" + "76d1d8d1adcef7587ea084231016bd6ee2643d11a3a35eb7fe4cca2b3f1a4b21e040b0d426412cca6c4271ea63" + "fb54ed7f57b41cd1af1be5507f87ea4f4a0c997367e883291de2f1b8a49bdaa52bae30064351b1139703400730" + "18a4104344ec6b4454b50a42e804bc70e78b9b3c82497273859c82ed241b643642d76df6ceab8f916392113a62" + "b231f228c7300624d74a846bec2f479ab8a8c3461f91c7bf806236e3bd2f54ba1ef8e2a1e0bfdde0c5ad227f7d" + "364c52510b1ade862ce0c8d7bd24b6d7d21c99b34de6d177eb3d575787b2af55060d76d6c2060befbb7953a816" + "6f66ad88ecf929dbb0ad3a16cf7dfd39d925e0b4b649c6d0c07ad46ed0229c17fb6a1395f16e1b138aab3af760" + "2b0ac762c4f611f7f3468997224ffbe500a7c53f92f65e41a3765a9f1d7e3f78208f5b4e147962d8c97d6c1a80" + "91ffc36090b2043d71853616f34c2185dc883c54ab6d66e10a6c18e0b9a4742597361f8554a42da3373241d0c8" + "54119bfadccffaf2335b2d97ffee627cb891bda8140a39399f853da4859f7e19682e152243efbaffb662edd19b" + "3819a74107c7dbe05ecb32e79dcdb1260f153b1ef133e978ccca3d9e400a7ed6c458d77e2956d2cb897b7a298b" + "fe144b5defdc23dfd2adf69f1fb0917840703402d524987ae3b1dcb85229843c9a419ef46e1ba0ba7783f2a2ec" + "d057a57518836aef2a7839ebd3688da98b54c942941f642e434727108d59ea25875b3050ca53d4637c76cbcbb9" + "e972c2b0b781131ee0a1403138b55486fe86bbd644920ee6aa578e3bab32d7d784b5c140295286d90c99b14823" + "1487f7ea64157001b745aa358c9ea6bec5a8d8b67a7534ec1f7648ff3b435911dfc3dff798d32fbf2efe2c1fcc" + "278865157590572387b76b78e727d3e7682cb501cdcdf9a0f17676f99d9aa67f10edccc9a92080294e88bf28c2" + "a9f32ae535fdb27fff7706540472abb9eab90af12b2bea005da189874b0ca69e6ae1690a6f2adf75be3853c94e" + "fd8098ed579c20cb37be6885d8d713af4ba52958cee383089b98ed9cb26e11127cf88d1b7d254f15f7903dd7ed" + "297c0013924e88248684fe8f2098326ce51aa6e5" +) + + class TestNextLayer: def test_configure(self): nl = NextLayer() @@ -147,44 +180,180 @@ def test_next_layer2(self): assert isinstance(nl._next_layer(ctx, b"GET /foo", b""), layers.HttpLayer) assert isinstance(nl._next_layer(ctx, b"", b"hello"), layers.TCPLayer) - def test_next_layer_udp(self): + @pytest.mark.parametrize( + ("client_hello", "client_layer", "server_layer"), + [ + (dtls_client_hello_with_extensions, layers.ClientTLSLayer, layers.ServerTLSLayer), + (quic_client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer), + ] + ) + def test_next_layer_udp( + self, + client_hello: bytes, + client_layer: layer.Layer, + server_layer: layer.Layer, + ): def is_ignored_udp(layer: Optional[layer.Layer]): return isinstance(layer, layers.UDPLayer) and layer.flow is None def is_intercepted_udp(layer: Optional[layer.Layer]): return isinstance(layer, layers.UDPLayer) and layer.flow is not None + def is_http(layer: Optional[layer.Layer], mode: HTTPMode): + return ( + isinstance(layer, layers.HttpLayer) + and layer.mode is mode + ) + nl = NextLayer() ctx = MagicMock() ctx.client.alpn = None ctx.server.address = ("example.com", 443) ctx.client.transport_protocol = "udp" with taddons.context(nl) as tctx: - ctx.layers = [layers.modes.HttpProxy(ctx)] - assert is_intercepted_udp(nl._next_layer(ctx, b"", b"")) + ctx.layers = [layers.modes.HttpProxy(ctx), client_layer(ctx)] + assert is_http(nl._next_layer(ctx, b"", b""), HTTPMode.regular) - ctx.layers = [layers.modes.HttpProxy(ctx)] + ctx.layers = [layers.modes.HttpUpstreamProxy(ctx), client_layer(ctx)] + assert is_http(nl._next_layer(ctx, b"", b""), HTTPMode.upstream) + + ctx.layers = [layers.modes.TransparentProxy(ctx)] + is_intercepted_udp(nl._next_layer(ctx, b"", b"")) + + ctx.layers = [layers.modes.TransparentProxy(ctx)] ctx.server.address = ("nomatch.com", 443) tctx.configure(nl, ignore_hosts=["example.com"]) - assert is_intercepted_udp(nl._next_layer(ctx, dtls_client_hello_with_extensions[:50], b"")) - assert is_ignored_udp(nl._next_layer(ctx, dtls_client_hello_with_extensions, b"")) + assert is_intercepted_udp(nl._next_layer(ctx, client_hello[:50], b"")) + assert is_ignored_udp(nl._next_layer(ctx, client_hello, b"")) - ctx.layers = [layers.modes.HttpProxy(ctx)] + ctx.layers = [layers.modes.TransparentProxy(ctx)] ctx.server.address = ("example.com", 443) - assert is_ignored_udp(nl._next_layer(ctx, dtls_client_hello_with_extensions[:50], b"")) + assert is_ignored_udp(nl._next_layer(ctx, client_hello[:50], b"")) - ctx.layers = [layers.modes.HttpProxy(ctx)] + ctx.layers = [layers.modes.TransparentProxy(ctx)] tctx.configure(nl, ignore_hosts=[]) - assert isinstance(nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), layers.ClientTLSLayer) + decision = nl._next_layer(ctx, client_hello, b"") + assert isinstance(decision, server_layer) + assert isinstance(decision.child_layer, client_layer) - ctx.layers = [layers.modes.HttpProxy(ctx)] + ctx.layers = [layers.modes.ReverseProxy(ctx), server_layer(ctx)] + tctx.configure(nl, ignore_hosts=[]) + assert isinstance(nl._next_layer(ctx, client_hello, b""), client_layer) + + ctx.layers = [layers.modes.TransparentProxy(ctx)] tctx.configure(nl, udp_hosts=["example.com"]) assert isinstance(nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.UDPLayer) - ctx.layers = [layers.modes.HttpProxy(ctx)] + ctx.layers = [layers.modes.TransparentProxy(ctx)] tctx.configure(nl, udp_hosts=[]) assert isinstance(nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.DNSLayer) + def test_next_layer_reverse_raw(self): + nl = NextLayer() + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + with taddons.context(nl) as tctx: + tctx.configure(nl, ignore_hosts=["example.com"]) + + ctx.layers = [layers.modes.HttpProxy(ctx), layers.ClientQuicLayer(ctx)] + decision = nl._next_layer(ctx, b"", b"") + assert isinstance(decision, layers.ServerQuicLayer) + assert isinstance(decision.child_layer, layers.RawQuicLayer) + + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), layers.ClientQuicLayer(ctx)] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) + + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx)] + decision = nl._next_layer(ctx, b"", b"") + assert isinstance(decision, layers.ClientQuicLayer) + assert isinstance(decision.child_layer, layers.RawQuicLayer) + + tctx.configure(nl, ignore_hosts=[]) + + def test_next_layer_reverse_quic_mode(self): + nl = NextLayer() + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "quic" + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + layers.ClientQuicLayer(ctx), + ] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + ] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) + + def test_next_layer_reverse_http3_mode(self): + nl = NextLayer() + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "http3" + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + layers.ClientQuicLayer(ctx), + ] + decision = nl._next_layer(ctx, b"", b"") + assert isinstance(decision, layers.HttpLayer) + assert decision.mode is HTTPMode.transparent + + def test_next_layer_reverse_invalid_mode(self): + nl = NextLayer() + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "invalidscheme" + ctx.layers = [layers.modes.ReverseProxy(ctx)] + with pytest.raises(AssertionError, match="invalidscheme"): + nl._next_layer(ctx, b"", b"") + + def test_next_layer_reverse_dtls_mode(self): + nl = NextLayer() + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "dtls" + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx)] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx), layers.ClientTLSLayer(ctx)] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) + + def test_next_layer_reverse_udp_mode(self): + nl = NextLayer() + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "udp" + ctx.layers = [layers.modes.ReverseProxy(ctx)] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ClientTLSLayer(ctx)] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) + + def test_next_layer_reverse_dns_mode(self): + nl = NextLayer() + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "dns" + ctx.layers = [layers.modes.ReverseProxy(ctx)] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ClientTLSLayer(ctx)] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) + def test_next_layer_invalid_proto(self): nl = NextLayer() ctx = MagicMock() From 49cca5301c8eb3b461db4028861f4bcd6b948362 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 31 Oct 2022 23:09:13 +0100 Subject: [PATCH 091/695] [quic] fix lint issues --- test/mitmproxy/addons/test_next_layer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index ced57924be..c76d1ccde0 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -256,20 +256,20 @@ def test_next_layer_reverse_raw(self): ctx.client.transport_protocol = "udp" with taddons.context(nl) as tctx: tctx.configure(nl, ignore_hosts=["example.com"]) - + ctx.layers = [layers.modes.HttpProxy(ctx), layers.ClientQuicLayer(ctx)] decision = nl._next_layer(ctx, b"", b"") assert isinstance(decision, layers.ServerQuicLayer) assert isinstance(decision.child_layer, layers.RawQuicLayer) - + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), layers.ClientQuicLayer(ctx)] assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) - + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx)] decision = nl._next_layer(ctx, b"", b"") assert isinstance(decision, layers.ClientQuicLayer) assert isinstance(decision.child_layer, layers.RawQuicLayer) - + tctx.configure(nl, ignore_hosts=[]) def test_next_layer_reverse_quic_mode(self): From 78c1a23bf39c91747f8d43c7a950ab6211575b25 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 1 Nov 2022 22:28:40 +0100 Subject: [PATCH 092/695] [quic] better chain file handling --- mitmproxy/certs.py | 6 +++--- test/mitmproxy/test_certs.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 63c748d6d9..70d847075f 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -314,9 +314,9 @@ def __init__( self.default_chain_file = default_chain_file self.default_chain_certs = ( [ - Cert.from_pem(cert) - for cert in re.split(rb"(?<=-----END CERTIFICATE-----\n)", self.default_chain_file.read_bytes()) - if cert + Cert.from_pem(chunk) + for chunk in re.split(rb"(?=-----BEGIN( [A-Z]+)+-----)", self.default_chain_file.read_bytes()) + if chunk.startswith(b"-----BEGIN CERTIFICATE-----") ] if self.default_chain_file else [default_ca] diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index 7ccb52d5c2..448efdfea3 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -65,10 +65,12 @@ def test_chain_file(self, tdata, tmp_path): (tmp_path / "mitmproxy-ca.pem").write_bytes(cert) ca = certs.CertStore.from_store(tmp_path, "mitmproxy", 2048) assert ca.default_chain_file is None + assert len(ca.default_chain_certs) == 1 (tmp_path / "mitmproxy-ca.pem").write_bytes(2 * cert) ca = certs.CertStore.from_store(tmp_path, "mitmproxy", 2048) assert ca.default_chain_file == (tmp_path / "mitmproxy-ca.pem") + assert len(ca.default_chain_certs) == 2 def test_sans(self, tstore): c1 = tstore.get_cert("foo.com", ["*.bar.com"]) From 9da97d6db4877d2396f9bb9f01b1cd55f0ea9929 Mon Sep 17 00:00:00 2001 From: mitmproxy release bot Date: Wed, 2 Nov 2022 11:13:57 +0000 Subject: [PATCH 093/695] reopen main for development --- mitmproxy/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 6a3372c669..d1405ffe1a 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -2,7 +2,7 @@ import subprocess import sys -VERSION = "9.0.1" +VERSION = "10.0.0.dev" MITMPROXY = "mitmproxy " + VERSION # Serialization format version. This is displayed nowhere, it just needs to be incremented by one From cf27d0af732301b0f3c6f003025654f72eba8945 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 2 Nov 2022 11:40:24 +0000 Subject: [PATCH 094/695] add `mode` change to CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0edba889a3..6b8b6af0d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,12 @@ * Deprecate `mitmproxy.ctx.log` in favor of Python's builtin `logging` module. See [the docs](https://docs.mitmproxy.org/dev/addons-api-changelog/) for details and upgrade instructions. ([#5590](https://github.com/mitmproxy/mitmproxy/pull/5590), @mhils) + +### Breaking Changes + + * The `mode` option is now a list of server specs instead of a single spec. + The CLI interface is unaffected, but users may need to update their `config.yaml`. + ([#5393](https://github.com/mitmproxy/mitmproxy/pull/5393), @mhils) ### Full Changelog From c9ccd6f4b3917b33300d8d187bb7eee4d1a5b41e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Nov 2022 17:47:27 +0000 Subject: [PATCH 095/695] wireguard: create confdir on startup, fix #5715 --- mitmproxy/proxy/mode_servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index ca00d4389c..b3ee872711 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -308,6 +308,7 @@ async def start(self) -> None: try: if not conf_path.exists(): + conf_path.parent.mkdir(parents=True, exist_ok=True) conf_path.write_text(json.dumps({ "server_key": wg.genkey(), "client_key": wg.genkey(), From 77ed92d26976c4f332d8383a7380606a1d4e09f2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Nov 2022 17:51:59 +0000 Subject: [PATCH 096/695] add WireGuard docs, fix #5706 --- mitmproxy/options.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index bf21823ab7..4b2f0e9cda 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -103,8 +103,9 @@ def __init__(self, **kwargs) -> None: The proxy server type(s) to spawn. Can be passed multiple times. Mitmproxy supports "regular" (HTTP), "transparent", "socks5", "reverse:SPEC", - and "upstream:SPEC" proxy servers. For reverse and upstream proxy modes, SPEC - is host specification in the form of "http[s]://host[:port]". + "upstream:SPEC", and "wireguard[:PATH]" proxy servers. For reverse and upstream proxy modes, SPEC + is host specification in the form of "http[s]://host[:port]". For WireGuard mode, PATH may point to + a file containing key material. If no such file exists, it will be created on startup. You may append `@listen_port` or `@listen_host:listen_port` to override `listen_host` or `listen_port` for a specific proxy mode. Features such as client playback will use the first mode to determine From d3f439cfd1f4c56545543f001132b95422fb506c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 4 Nov 2022 10:28:16 +0000 Subject: [PATCH 097/695] make asgi/wsgi apps listen on all ports by default, remove `onboarding_host` --- CHANGELOG.md | 6 ++++++ mitmproxy/addons/asgiapp.py | 8 +++++--- mitmproxy/addons/onboarding.py | 7 +------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b8b6af0d3..74ebc06ada 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,13 @@ ## Unreleased: mitmproxy next +* ASGI/WSGI apps can now listen on all ports for a specific hostname. + This makes it simpler to accept both HTTP and HTTPS. +### Breaking Changes + +* The `onboarding_port` option has been removed. The onboarding app now responds + to all requests for the hostname specified in `onboarding_host`. ## 02 November 2022: mitmproxy 9.0.1 diff --git a/mitmproxy/addons/asgiapp.py b/mitmproxy/addons/asgiapp.py index 9c9b1ca299..3a077a44e8 100644 --- a/mitmproxy/addons/asgiapp.py +++ b/mitmproxy/addons/asgiapp.py @@ -2,6 +2,7 @@ import logging import traceback import urllib.parse +from typing import Optional import asgiref.compatibility import asgiref.wsgi @@ -20,7 +21,7 @@ class ASGIApp: - It currently only implements the HTTP protocol (Lifespan and WebSocket are unimplemented). """ - def __init__(self, asgi_app, host: str, port: int): + def __init__(self, asgi_app, host: str, port: Optional[int]): asgi_app = asgiref.compatibility.guarantee_single_callable(asgi_app) self.asgi_app, self.host, self.port = asgi_app, host, port @@ -30,7 +31,8 @@ def name(self) -> str: def should_serve(self, flow: http.HTTPFlow) -> bool: return bool( - (flow.request.pretty_host, flow.request.port) == (self.host, self.port) + flow.request.pretty_host == self.host + and (self.port is None or flow.request.port == self.port) and flow.live and not flow.error and not flow.response @@ -42,7 +44,7 @@ async def request(self, flow: http.HTTPFlow) -> None: class WSGIApp(ASGIApp): - def __init__(self, wsgi_app, host: str, port: int): + def __init__(self, wsgi_app, host: str, port: Optional[int]): asgi_app = asgiref.wsgi.WsgiToAsgi(wsgi_app) super().__init__(asgi_app, host, port) diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 272068cf96..78ff110334 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -3,14 +3,13 @@ from mitmproxy import ctx APP_HOST = "mitm.it" -APP_PORT = 80 class Onboarding(asgiapp.WSGIApp): name = "onboarding" def __init__(self): - super().__init__(app, APP_HOST, APP_PORT) + super().__init__(app, APP_HOST, None) def load(self, loader): loader.add_option( @@ -25,13 +24,9 @@ def load(self, loader): entry for the app domain is not present. """, ) - loader.add_option( - "onboarding_port", int, APP_PORT, "Port to serve the onboarding app from." - ) def configure(self, updated): self.host = ctx.options.onboarding_host - self.port = ctx.options.onboarding_port app.config["CONFDIR"] = ctx.options.confdir async def request(self, f): From 0ad8630487218f6c851605a0869555b7bf473c88 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 4 Nov 2022 10:41:46 +0000 Subject: [PATCH 098/695] add self-test addon for binaries --- release/build-and-deploy-docker.py | 8 +++--- release/build.py | 24 ++++++++++------- release/selftest.py | 43 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 13 deletions(-) create mode 100644 release/selftest.py diff --git a/release/build-and-deploy-docker.py b/release/build-and-deploy-docker.py index 15ee80b4db..0a2d6f56e4 100644 --- a/release/build-and-deploy-docker.py +++ b/release/build-and-deploy-docker.py @@ -21,7 +21,7 @@ elif ref.startswith("refs/tags/"): tag = ref.replace("refs/tags/", "") else: - raise AssertionError + raise AssertionError("Failed to parse $GITHUB_REF") (whl,) = root.glob("release/dist/mitmproxy-*-py3-none-any.whl") docker_build_dir = root / "release/docker" @@ -47,15 +47,17 @@ "docker", "run", "--rm", + "-v", + f"{root / 'release'}:/release" "localtesting", "mitmdump", - "--version", + "-s", "/release/selftest.py", ], check=True, capture_output=True, ) print(r.stdout.decode()) -assert "Mitmproxy: " in r.stdout.decode() +assert "Self-test successful" in r.stdout.decode() # Now we can deploy. subprocess.check_call( diff --git a/release/build.py b/release/build.py index 2f33b38430..c5efff9380 100644 --- a/release/build.py +++ b/release/build.py @@ -121,15 +121,13 @@ def standalone_binaries(): with archive(DIST_DIR / f"mitmproxy-{version()}-{operating_system()}") as f: _pyinstaller("standalone.spec") + _test_binaries(TEMP_DIR / "pyinstaller/dist") + for tool in ["mitmproxy", "mitmdump", "mitmweb"]: executable = TEMP_DIR / "pyinstaller/dist" / tool if platform.system() == "Windows": executable = executable.with_suffix(".exe") - # Test if it works at all O:-) - print(f"> {executable} --version") - subprocess.check_call([executable, "--version"]) - f.add(str(executable), str(executable.name)) print(f"Packed {f.name}.") @@ -138,11 +136,21 @@ def _ensure_pyinstaller_onedir(): if not (TEMP_DIR / "pyinstaller/dist/onedir").exists(): _pyinstaller("windows-dir.spec") + _test_binaries(TEMP_DIR / "pyinstaller/dist/onedir") + + +def _test_binaries(binary_directory: Path) -> None: for tool in ["mitmproxy", "mitmdump", "mitmweb"]: + executable = binary_directory / tool + if platform.system() == "Windows": + executable = executable.with_suffix(".exe") + print(f"> {tool} --version") - executable = (TEMP_DIR / "pyinstaller/dist/onedir" / tool).with_suffix(".exe") subprocess.check_call([executable, "--version"]) + print(f"> {tool} -s selftest.py") + subprocess.check_call([executable, "-s", here / "selftest.py"]) + @cli.command() def msix_installer(): @@ -256,11 +264,7 @@ def report(block, blocksize, total): subprocess.run( [installer, "--mode", "unattended", "--unattendedmodeui", "none"], check=True ) - MITMPROXY_INSTALL_DIR = Path(rf"C:\Program Files\mitmproxy\bin") - for tool in ["mitmproxy", "mitmdump", "mitmweb"]: - executable = (MITMPROXY_INSTALL_DIR / tool).with_suffix(".exe") - print(f"> {executable} --version") - subprocess.check_call([executable, "--version"]) + _test_binaries(Path(r"C:\Program Files\mitmproxy\bin")) if __name__ == "__main__": diff --git a/release/selftest.py b/release/selftest.py new file mode 100644 index 0000000000..6052a5ca83 --- /dev/null +++ b/release/selftest.py @@ -0,0 +1,43 @@ +""" +This addons is used for binaries to perform a minimal selftest. Use like so: + + mitmdump -s selftest.py -p 0 +""" +import asyncio +import logging +import ssl +import sys +from pathlib import Path + +from mitmproxy import ctx + + +def load(_): + # force a random port + ctx.options.listen_port = 0 + + +def running(): + # attach is somewhere so that it's not collected. + ctx.task = asyncio.create_task(make_request()) + + +async def make_request(): + try: + cafile = Path(ctx.options.confdir).expanduser() / "mitmproxy-ca.pem" + ssl_ctx = ssl.create_default_context(cafile=cafile) + port = ctx.master.addons.get("proxyserver").listen_addrs()[0][1] + reader, writer = await asyncio.open_connection( + "127.0.0.1", port, + ssl=ssl_ctx + ) + writer.write(b"GET / HTTP/1.1\r\nHost: mitm.it\r\nConnection: close\r\n\r\n") + await writer.drain() + resp = await reader.read() + if b"This page is served by your local mitmproxy instance" not in resp: + raise RuntimeError(resp) + logging.info("Self-test successful.") + ctx.master.shutdown() + except Exception as e: + print(f"{e!r}") + sys.exit(1) From a308d3dabcbb938a79902bfd02cf2be0b711c308 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 6 Nov 2022 18:42:30 +0100 Subject: [PATCH 099/695] [quic] first test for H3 --- mitmproxy/proxy/layers/http/_http3.py | 4 +- .../mitmproxy/proxy/layers/http/test_http3.py | 379 ++++++++++++++++++ 2 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 test/mitmproxy/proxy/layers/http/test_http3.py diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 9c46d77f9b..4ee46c94fd 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -59,7 +59,7 @@ def __init__(self, context: context.Context, conn: connection.Connection): def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): - pass + yield from self.h3_conn.transmit() # send mitmproxy HTTP events over the H3 connection elif isinstance(event, HttpEvent): @@ -145,7 +145,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, reason_phrase=f"Invalid HTTP/3 request headers: {e}", ) - yield from self.h3_conn.transmit() else: yield ReceiveHttp(receive_event) if h3_event.stream_ended: @@ -160,6 +159,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: pass else: raise AssertionError(f"Unexpected event: {event!r}") + yield from self.h3_conn.transmit() # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, events.ConnectionClosed): diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py new file mode 100644 index 0000000000..ece395971f --- /dev/null +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -0,0 +1,379 @@ +import collections.abc +from typing import Callable, Iterable, Optional +import pytest +import pylsqpack + +from aioquic._buffer import Buffer +from aioquic.h3.connection import FrameType, StreamType, Headers, Setting, encode_frame, encode_uint_var, encode_settings, parse_settings + +from mitmproxy import connection +from mitmproxy.http import HTTPFlow +from mitmproxy.proxy import commands, context, layers +from mitmproxy.proxy.layers import http, quic +from test.mitmproxy.proxy import tutils + + +example_request_headers = [ + (b":method", b"GET"), + (b":scheme", b"http"), + (b":path", b"/"), + (b":authority", b"example.com"), +] + + +class CallbackPlaceholder(tutils._Placeholder[bytes]): + """Data placeholder that invokes a callback once its bytes get set.""" + def __init__(self, cb: Callable[[bytes], None]): + super().__init__(bytes) + self._cb = cb + + def setdefault(self, value: bytes) -> None: + if self._obj is None: + self._cb(value) + return super().setdefault(value) + + +class DelayedPlaceholder(tutils._Placeholder[bytes]): + """Data placeholder that resolves its bytes when needed.""" + def __init__(self, resolve: Callable[[], bytes]): + super().__init__(bytes) + self._resolve = resolve + + def __call__(self) -> bytes: + if self._obj is None: + self._obj = self._resolve() + return super().__call__() + + +class MultiPlaybook(tutils.Playbook): + """Playbook that allows multiple events and commands to be registered at once.""" + def __lshift__(self, c): + if isinstance(c, collections.abc.Iterable): + for c_i in c: + super().__lshift__(c_i) + else: + super().__lshift__(c) + return self + + def __rshift__(self, e): + if isinstance(e, collections.abc.Iterable): + for e_i in e: + super().__rshift__(e_i) + else: + super().__rshift__(e) + return self + + +class FrameFactory: + """Helper class for generating QUIC stream events and commands.""" + def __init__( + self, + conn: connection.Connection, + is_client: bool + ) -> None: + self.conn = conn + self.is_client = is_client + self.decoder = pylsqpack.Decoder( + max_table_capacity=4096, + blocked_streams=16, + ) + self.decoder_placeholder: Optional[tutils.Placeholder(bytes)] = None + self.encoder = pylsqpack.Encoder() + self.encoder_placeholder: Optional[tutils.Placeholder(bytes)] = None + self.peer_stream_id: dict[StreamType, int] = {} + self.local_stream_id: dict[StreamType, int] = {} + self.max_push_id: Optional[int] = None + + def get_default_stream_id( + self, + stream_type: StreamType, + for_local: bool + ) -> int: + if stream_type == StreamType.CONTROL: + stream_id = 2 + elif stream_type == StreamType.QPACK_ENCODER: + stream_id = 6 + elif stream_type == StreamType.QPACK_DECODER: + stream_id = 10 + else: + raise AssertionError(stream_type) + if self.is_client is not for_local: + stream_id = stream_id + 1 + return stream_id + + def send_stream_type( + self, + stream_type: StreamType, + stream_id: Optional[int] = None, + ) -> quic.SendQuicStreamData: + assert stream_type not in self.peer_stream_id + if stream_id is None: + stream_id = self.get_default_stream_id( + stream_type, for_local=False + ) + self.peer_stream_id[stream_type] = stream_id + return quic.SendQuicStreamData( + connection=self.conn, + stream_id=stream_id, + data=encode_uint_var(stream_type), + end_stream=False, + ) + + def receive_stream_type( + self, + stream_type: StreamType, + stream_id: Optional[int] = None, + ) -> quic.QuicStreamDataReceived: + assert stream_type not in self.local_stream_id + if stream_id is None: + stream_id = self.get_default_stream_id( + stream_type, for_local=True + ) + self.local_stream_id[stream_type] = stream_id + return quic.QuicStreamDataReceived( + connection=self.conn, + stream_id=stream_id, + data=encode_uint_var(stream_type), + end_stream=False, + ) + + def send_settings(self) -> quic.SendQuicStreamData: + assert self.encoder_placeholder is None + placeholder = tutils.Placeholder(bytes) + self.encoder_placeholder = placeholder + + def cb(data: bytes) -> None: + buf = Buffer(data=data) + assert buf.pull_uint_var() == FrameType.SETTINGS + settings = parse_settings(buf.pull_bytes(buf.pull_uint_var())) + placeholder.setdefault(self.encoder.apply_settings( + max_table_capacity=settings[Setting.QPACK_MAX_TABLE_CAPACITY], + blocked_streams=settings[Setting.QPACK_BLOCKED_STREAMS], + )) + + return quic.SendQuicStreamData( + connection=self.conn, + stream_id=self.peer_stream_id[StreamType.CONTROL], + data=CallbackPlaceholder(cb), + end_stream=False, + ) + + def send_max_push_id(self) -> quic.SendQuicStreamData: + def cb(data: bytes) -> None: + buf = Buffer(data=data) + assert buf.pull_uint_var() == FrameType.MAX_PUSH_ID + buf = Buffer(data=buf.pull_bytes(buf.pull_uint_var())) + self.max_push_id = buf.pull_uint_var() + assert buf.eof() + + return quic.SendQuicStreamData( + connection=self.conn, + stream_id=self.peer_stream_id[StreamType.CONTROL], + data=CallbackPlaceholder(cb), + end_stream=False, + ) + + def receive_settings( + self, + settings: dict[int, int] = { + Setting.QPACK_MAX_TABLE_CAPACITY: 4096, + Setting.QPACK_BLOCKED_STREAMS: 16, + Setting.ENABLE_CONNECT_PROTOCOL: 1, + Setting.DUMMY: 1, + }, + ) -> quic.QuicStreamDataReceived: + return quic.QuicStreamDataReceived( + connection=self.conn, + stream_id=self.local_stream_id[StreamType.CONTROL], + data=encode_frame(FrameType.SETTINGS, encode_settings(settings)), + end_stream=False, + ) + + def send_encoder(self) -> quic.SendQuicStreamData: + def cb(data: bytes) -> bytes: + self.decoder.feed_encoder(data) + return data + + return quic.SendQuicStreamData( + connection=self.conn, + stream_id=self.peer_stream_id[StreamType.QPACK_ENCODER], + data=CallbackPlaceholder(cb), + end_stream=False, + ) + + def receive_encoder(self) -> quic.QuicStreamDataReceived: + assert self.encoder_placeholder is not None + placeholder = self.encoder_placeholder + self.encoder_placeholder = None + + return quic.QuicStreamDataReceived( + connection=self.conn, + stream_id=self.local_stream_id[StreamType.QPACK_ENCODER], + data=placeholder, + end_stream=False, + ) + + def send_data( + self, + data: bytes, + stream_id: int = 0, + end_stream: bool = False, + ) -> quic.SendQuicStreamData: + return quic.SendQuicStreamData( + self.conn, + stream_id=stream_id, + data=encode_frame(FrameType.DATA, data), + end_stream=end_stream, + ) + + def send_decoder(self) -> quic.SendQuicStreamData: + def cb(data: bytes) -> None: + self.encoder.feed_decoder(data) + + return quic.SendQuicStreamData( + self.conn, + stream_id=self.peer_stream_id[StreamType.QPACK_DECODER], + data=CallbackPlaceholder(cb), + end_stream=False, + ) + + def receive_decoder(self) -> quic.QuicStreamDataReceived: + assert self.decoder_placeholder is not None + placeholder = self.decoder_placeholder + self.decoder_placeholder = None + + return quic.QuicStreamDataReceived( + self.conn, + stream_id=self.local_stream_id[StreamType.QPACK_DECODER], + data=placeholder, + end_stream=False, + ) + + def receive_headers( + self, + headers: Headers, + stream_id: int = 0, + end_stream: bool = False, + ) -> Iterable[quic.QuicStreamDataReceived]: + data = tutils.Placeholder(bytes) + + def encode() -> bytes: + encoder, frame_data = self.encoder.encode(stream_id, headers) + data.setdefault(encode_frame(FrameType.HEADERS, frame_data)) + return encoder + + yield quic.QuicStreamDataReceived( + connection=self.conn, + stream_id=self.local_stream_id[StreamType.QPACK_ENCODER], + data=DelayedPlaceholder(encode), + end_stream=False, + ) + yield quic.QuicStreamDataReceived( + connection=self.conn, + stream_id=stream_id, + data=data, + end_stream=end_stream, + ) + + def send_headers( + self, + headers: Headers, + stream_id: int = 0, + end_stream: bool = False, + ) -> Iterable[quic.SendQuicStreamData]: + assert self.decoder_placeholder is None + placeholder = tutils.Placeholder(bytes) + self.decoder_placeholder = placeholder + + def decode(data: bytes) -> None: + buf = Buffer(data=data) + assert buf.pull_uint_var() == FrameType.HEADERS + frame_data = buf.pull_bytes(buf.pull_uint_var()) + decoder, headers = self.decoder.feed_header(stream_id, frame_data) + placeholder.setdefault(decoder) + assert headers == headers + + yield self.send_encoder() + yield quic.SendQuicStreamData( + connection=self.conn, + stream_id=stream_id, + data=CallbackPlaceholder(decode), + end_stream=end_stream, + ) + + def receive_data( + self, + data: bytes, + stream_id: int = 0, + end_stream: bool = False, + ) -> quic.QuicStreamDataReceived: + return quic.QuicStreamDataReceived( + connection=self.conn, + stream_id=stream_id, + data=encode_frame(FrameType.DATA, data), + end_stream=end_stream, + ) + + def send_server_init(self) -> Iterable[quic.SendQuicStreamData]: + yield self.send_stream_type(StreamType.CONTROL) + yield self.send_settings() + yield self.send_max_push_id() + yield self.send_stream_type(StreamType.QPACK_ENCODER) + yield self.send_stream_type(StreamType.QPACK_DECODER) + + +@pytest.fixture +def open_h3_server_conn(): + # this is a bit fake here (port 80, with alpn, but no tls - c'mon), + # but we don't want to pollute our tests with TLS handshakes. + server = connection.Server(("example.com", 80), transport_protocol="udp") + server.state = connection.ConnectionState.OPEN + server.alpn = b"h3" + return server + + +def start_h3_client(tctx: context.Context) -> tuple[tutils.Playbook, FrameFactory]: + tctx.client.alpn = b"h3" + tctx.client.transport_protocol = "udp" + + playbook = MultiPlaybook(layers.HttpLayer(tctx, layers.http.HTTPMode.regular)) + cff = FrameFactory(conn=tctx.client, is_client=True) + assert ( + playbook + << cff.send_stream_type(StreamType.CONTROL) + << cff.send_settings() + << cff.send_stream_type(StreamType.QPACK_ENCODER) + << cff.send_stream_type(StreamType.QPACK_DECODER) + >> cff.receive_stream_type(StreamType.CONTROL) + >> cff.receive_settings() + << cff.send_encoder() + >> cff.receive_stream_type(StreamType.QPACK_ENCODER) + >> cff.receive_stream_type(StreamType.QPACK_DECODER) + >> cff.receive_encoder() + ) + return playbook, cff + + +def make_h3(open_connection: commands.OpenConnection) -> None: + open_connection.connection.alpn = b"h3" + open_connection.connection.transport_protocol = "udp" + + +def test_simple(tctx: context.Context): + playbook, cff = start_h3_client(tctx) + flow = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + sff = FrameFactory(server, is_client=False) + assert ( + playbook + >> cff.receive_headers(example_request_headers, end_stream=True) + << http.HttpRequestHeadersHook(flow) + << cff.send_decoder() + >> tutils.reply(to=http.HttpRequestHeadersHook(flow)) + << http.HttpRequestHook(flow) + >> tutils.reply() + << commands.OpenConnection(server) + >> tutils.reply(None, side_effect=make_h3) + << sff.send_server_init() + << sff.send_headers(example_request_headers, end_stream=True) + ) From 201f03082af882e6895b0a116d39e5137495344d Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 7 Nov 2022 01:22:59 +0100 Subject: [PATCH 100/695] [quic] more h3 tests --- .../mitmproxy/proxy/layers/http/test_http3.py | 298 ++++++++++++++---- 1 file changed, 230 insertions(+), 68 deletions(-) diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index ece395971f..00770a5c1b 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -4,9 +4,18 @@ import pylsqpack from aioquic._buffer import Buffer -from aioquic.h3.connection import FrameType, StreamType, Headers, Setting, encode_frame, encode_uint_var, encode_settings, parse_settings - -from mitmproxy import connection +from aioquic.h3.connection import ( + FrameType, + Headers, + Setting, + StreamType, + encode_frame, + encode_uint_var, + encode_settings, + parse_settings, +) + +from mitmproxy import connection, version from mitmproxy.http import HTTPFlow from mitmproxy.proxy import commands, context, layers from mitmproxy.proxy.layers import http, quic @@ -20,6 +29,16 @@ (b":authority", b"example.com"), ] +example_response_headers = [(b":status", b"200")] + +example_response_trailers = [(b"resp-trailer-a", b"a"), (b"resp-trailer-b", b"b")] + + +def decode_frame(frame_type: int, frame_data: bytes) -> bytes: + buf = Buffer(data=frame_data) + assert buf.pull_uint_var() == frame_type + return buf.pull_bytes(buf.pull_uint_var()) + class CallbackPlaceholder(tutils._Placeholder[bytes]): """Data placeholder that invokes a callback once its bytes get set.""" @@ -77,7 +96,7 @@ def __init__( max_table_capacity=4096, blocked_streams=16, ) - self.decoder_placeholder: Optional[tutils.Placeholder(bytes)] = None + self.decoder_placeholders: list[tutils.Placeholder(bytes)] = [] self.encoder = pylsqpack.Encoder() self.encoder_placeholder: Optional[tutils.Placeholder(bytes)] = None self.peer_stream_id: dict[StreamType, int] = {} @@ -137,6 +156,21 @@ def receive_stream_type( end_stream=False, ) + def send_max_push_id(self) -> quic.SendQuicStreamData: + def cb(data: bytes) -> None: + buf = Buffer(data=data) + assert buf.pull_uint_var() == FrameType.MAX_PUSH_ID + buf = Buffer(data=buf.pull_bytes(buf.pull_uint_var())) + self.max_push_id = buf.pull_uint_var() + assert buf.eof() + + return quic.SendQuicStreamData( + connection=self.conn, + stream_id=self.peer_stream_id[StreamType.CONTROL], + data=CallbackPlaceholder(cb), + end_stream=False, + ) + def send_settings(self) -> quic.SendQuicStreamData: assert self.encoder_placeholder is None placeholder = tutils.Placeholder(bytes) @@ -158,21 +192,6 @@ def cb(data: bytes) -> None: end_stream=False, ) - def send_max_push_id(self) -> quic.SendQuicStreamData: - def cb(data: bytes) -> None: - buf = Buffer(data=data) - assert buf.pull_uint_var() == FrameType.MAX_PUSH_ID - buf = Buffer(data=buf.pull_bytes(buf.pull_uint_var())) - self.max_push_id = buf.pull_uint_var() - assert buf.eof() - - return quic.SendQuicStreamData( - connection=self.conn, - stream_id=self.peer_stream_id[StreamType.CONTROL], - data=CallbackPlaceholder(cb), - end_stream=False, - ) - def receive_settings( self, settings: dict[int, int] = { @@ -213,19 +232,6 @@ def receive_encoder(self) -> quic.QuicStreamDataReceived: end_stream=False, ) - def send_data( - self, - data: bytes, - stream_id: int = 0, - end_stream: bool = False, - ) -> quic.SendQuicStreamData: - return quic.SendQuicStreamData( - self.conn, - stream_id=stream_id, - data=encode_frame(FrameType.DATA, data), - end_stream=end_stream, - ) - def send_decoder(self) -> quic.SendQuicStreamData: def cb(data: bytes) -> None: self.encoder.feed_decoder(data) @@ -238,9 +244,8 @@ def cb(data: bytes) -> None: ) def receive_decoder(self) -> quic.QuicStreamDataReceived: - assert self.decoder_placeholder is not None - placeholder = self.decoder_placeholder - self.decoder_placeholder = None + assert self.decoder_placeholders + placeholder = self.decoder_placeholders.pop(0) return quic.QuicStreamDataReceived( self.conn, @@ -249,6 +254,31 @@ def receive_decoder(self) -> quic.QuicStreamDataReceived: end_stream=False, ) + def send_headers( + self, + headers: Headers, + stream_id: int = 0, + end_stream: bool = False, + ) -> Iterable[quic.SendQuicStreamData]: + placeholder = tutils.Placeholder(bytes) + self.decoder_placeholders.append(placeholder) + + def decode(data: bytes) -> None: + buf = Buffer(data=data) + assert buf.pull_uint_var() == FrameType.HEADERS + frame_data = buf.pull_bytes(buf.pull_uint_var()) + decoder, actual_headers = self.decoder.feed_header(stream_id, frame_data) + placeholder.setdefault(decoder) + assert headers == actual_headers + + yield self.send_encoder() + yield quic.SendQuicStreamData( + connection=self.conn, + stream_id=stream_id, + data=CallbackPlaceholder(decode), + end_stream=end_stream, + ) + def receive_headers( self, headers: Headers, @@ -275,29 +305,16 @@ def encode() -> bytes: end_stream=end_stream, ) - def send_headers( + def send_data( self, - headers: Headers, + data: bytes, stream_id: int = 0, end_stream: bool = False, - ) -> Iterable[quic.SendQuicStreamData]: - assert self.decoder_placeholder is None - placeholder = tutils.Placeholder(bytes) - self.decoder_placeholder = placeholder - - def decode(data: bytes) -> None: - buf = Buffer(data=data) - assert buf.pull_uint_var() == FrameType.HEADERS - frame_data = buf.pull_bytes(buf.pull_uint_var()) - decoder, headers = self.decoder.feed_header(stream_id, frame_data) - placeholder.setdefault(decoder) - assert headers == headers - - yield self.send_encoder() - yield quic.SendQuicStreamData( - connection=self.conn, + ) -> quic.SendQuicStreamData: + return quic.SendQuicStreamData( + self.conn, stream_id=stream_id, - data=CallbackPlaceholder(decode), + data=encode_frame(FrameType.DATA, data), end_stream=end_stream, ) @@ -314,13 +331,27 @@ def receive_data( end_stream=end_stream, ) - def send_server_init(self) -> Iterable[quic.SendQuicStreamData]: + def send_init(self) -> Iterable[quic.SendQuicStreamData]: yield self.send_stream_type(StreamType.CONTROL) yield self.send_settings() - yield self.send_max_push_id() + if not self.is_client: + yield self.send_max_push_id() yield self.send_stream_type(StreamType.QPACK_ENCODER) yield self.send_stream_type(StreamType.QPACK_DECODER) + def receive_init(self) -> Iterable[quic.QuicStreamDataReceived]: + yield self.receive_stream_type(StreamType.CONTROL) + yield self.receive_stream_type(StreamType.QPACK_ENCODER) + yield self.receive_stream_type(StreamType.QPACK_DECODER) + yield self.receive_settings() + + @property + def is_done(self) -> bool: + return ( + self.encoder_placeholder is None + and not self.decoder_placeholders + ) + @pytest.fixture def open_h3_server_conn(): @@ -340,15 +371,9 @@ def start_h3_client(tctx: context.Context) -> tuple[tutils.Playbook, FrameFactor cff = FrameFactory(conn=tctx.client, is_client=True) assert ( playbook - << cff.send_stream_type(StreamType.CONTROL) - << cff.send_settings() - << cff.send_stream_type(StreamType.QPACK_ENCODER) - << cff.send_stream_type(StreamType.QPACK_DECODER) - >> cff.receive_stream_type(StreamType.CONTROL) - >> cff.receive_settings() + << cff.send_init() + >> cff.receive_init() << cff.send_encoder() - >> cff.receive_stream_type(StreamType.QPACK_ENCODER) - >> cff.receive_stream_type(StreamType.QPACK_DECODER) >> cff.receive_encoder() ) return playbook, cff @@ -366,14 +391,151 @@ def test_simple(tctx: context.Context): sff = FrameFactory(server, is_client=False) assert ( playbook + # request client >> cff.receive_headers(example_request_headers, end_stream=True) - << http.HttpRequestHeadersHook(flow) - << cff.send_decoder() - >> tutils.reply(to=http.HttpRequestHeadersHook(flow)) + << (request := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) << http.HttpRequestHook(flow) >> tutils.reply() + # request server << commands.OpenConnection(server) >> tutils.reply(None, side_effect=make_h3) - << sff.send_server_init() + << sff.send_init() + << sff.send_headers(example_request_headers, end_stream=True) + >> sff.receive_init() + << sff.send_encoder() + >> sff.receive_encoder() + >> sff.receive_decoder() # for send_headers + # response server + >> sff.receive_headers(example_response_headers) + << (response := http.HttpResponseHeadersHook(flow)) + << sff.send_decoder() # for receive_headers + >> tutils.reply(to=response) + >> sff.receive_data(b"Hello, World!", end_stream=True) + << http.HttpResponseHook(flow) + >> tutils.reply() + # response client + << cff.send_headers(example_response_headers) + << cff.send_data(b"Hello, World!") + << cff.send_data(b"", end_stream=True) + >> cff.receive_decoder() # for send_headers + ) + assert cff.is_done and sff.is_done + assert flow().request.url == "http://example.com/" + assert flow().response.text == "Hello, World!" + + +@pytest.mark.parametrize("stream", [True, False]) +def test_response_trailers( + tctx: context.Context, + open_h3_server_conn: connection.Server, + stream: bool, +): + playbook, cff = start_h3_client(tctx) + tctx.server = open_h3_server_conn + sff = FrameFactory(tctx.server, is_client=False) + + def enable_streaming(flow: HTTPFlow): + flow.response.stream = stream + + flow = tutils.Placeholder(HTTPFlow) + ( + playbook + # request client + >> cff.receive_headers(example_request_headers, end_stream=True) + << (request := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) + << http.HttpRequestHook(flow) + >> tutils.reply() + # request server + << sff.send_init() << sff.send_headers(example_request_headers, end_stream=True) + >> sff.receive_init() + << sff.send_encoder() + >> sff.receive_encoder() + >> sff.receive_decoder() # for send_headers + # response server + >> sff.receive_headers(example_response_headers) + << (response_headers := http.HttpResponseHeadersHook(flow)) + << sff.send_decoder() # for receive_headers + >> tutils.reply(to=response_headers, side_effect=enable_streaming) + ) + if stream: + ( + playbook + << cff.send_headers(example_response_headers) + >> cff.receive_decoder() # for send_headers + >> sff.receive_data(b"Hello, World!") + << cff.send_data(b"Hello, World!") + ) + else: + playbook >> sff.receive_data(b"Hello, World!") + assert ( + playbook + >> sff.receive_headers(example_response_trailers, end_stream=True) + << (response := http.HttpResponseHook(flow)) + << sff.send_decoder() # for receive_headers + ) + + def modify_tailers(flow: HTTPFlow) -> None: + assert flow.response.trailers + del flow.response.trailers["resp-trailer-a"] + + if stream: + assert ( + playbook + >> tutils.reply(to=response, side_effect=modify_tailers) + << cff.send_headers(example_response_trailers[1:], end_stream=True) + >> cff.receive_decoder() # for send_headers + ) + else: + assert ( + playbook + >> tutils.reply(to=response, side_effect=modify_tailers) + << cff.send_headers(example_response_headers) + << cff.send_data(b"Hello, World!") + << cff.send_headers(example_response_trailers[1:], end_stream=True) + >> cff.receive_decoder() # for send_headers + >> cff.receive_decoder() # for send_headers + ) + assert cff.is_done and sff.is_done + + +def test_upstream_error(tctx: context.Context): + playbook, cff = start_h3_client(tctx) + flow = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + err = tutils.Placeholder(bytes) + assert ( + playbook + # request client + >> cff.receive_headers(example_request_headers, end_stream=True) + << (request := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) + << http.HttpRequestHook(flow) + >> tutils.reply() + # request server + << commands.OpenConnection(server) + >> tutils.reply("oops server <> error") + << http.HttpErrorHook(flow) + >> tutils.reply() + << cff.send_headers([ + (b":status", b"502"), + (b'server', version.MITMPROXY.encode()), + (b'content-type', b'text/html'), + ]) + << quic.SendQuicStreamData( + tctx.client, + stream_id=0, + data=err, + end_stream=True, + ) + >> cff.receive_decoder() # for send_headers ) + assert cff.is_done + data = decode_frame(FrameType.DATA, err()) + assert b"502 Bad Gateway" in data + assert b"server <> error" in data From 97ca30ce6fd566931ceff92602813c39196a37aa Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 7 Nov 2022 03:49:32 +0100 Subject: [PATCH 101/695] [quic] fix next layer handling --- mitmproxy/addons/next_layer.py | 22 ++++++++++++++-------- mitmproxy/addons/tlsconfig.py | 4 +++- mitmproxy/proxy/layers/modes.py | 8 ++------ mitmproxy/proxy/layers/quic.py | 6 +++--- test/mitmproxy/addons/test_next_layer.py | 10 ++++++++-- test/mitmproxy/proxy/layers/test_modes.py | 6 +++++- 6 files changed, 35 insertions(+), 21 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 0404bb597b..3b4b1b68f8 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -314,15 +314,21 @@ def _next_layer( if scheme in ("udp", "dtls"): return layers.UDPLayer(context) elif scheme == "http3": - return layers.HttpLayer(context, HTTPMode.transparent) + if isinstance(context.layers[-1], layers.ClientQuicLayer): + return layers.HttpLayer(context, HTTPMode.transparent) + else: + return layers.ClientQuicLayer(context) elif scheme == "quic": - # if the client supports QUIC, we use QUIC raw layer, - # otherwise we only use the QUIC datagram only - return ( - layers.RawQuicLayer(context) - if isinstance(context.layers[-1], layers.ClientQuicLayer) else - layers.UDPLayer(context) - ) + if isinstance(context.layers[-1], layers.ClientQuicLayer): + # the client supports QUIC, use raw layer + return layers.RawQuicLayer(context) + elif data_client: + # we have received data, which was not a handshake, use UDP + # on the client, and send datagrams over QUIC to the server + return layers.UDPLayer(context) + else: + # wait for client data to make a decision + return None elif scheme == "dns": return layers.DNSLayer(context) else: diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 5e068a220e..58b2503acf 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -5,6 +5,7 @@ import ssl from typing import Any, Optional, TypedDict +from aioquic.h3.connection import H3_ALPN from aioquic.tls import CipherSuite from OpenSSL import SSL, crypto from mitmproxy import certs, ctx, exceptions, connection, tls @@ -360,7 +361,8 @@ def quic_start_server(self, tls_start: quic.QuicTlsData) -> None: if client.alpn_offers: server.alpn_offers = tuple(client.alpn_offers) else: - server.alpn_offers = [] + # aioquic fails if no ALPN is offered, so use H3 + server.alpn_offers = tuple(alpn.encode("ascii") for alpn in H3_ALPN) if not server.cipher_list and ctx.options.ciphers_server: server.cipher_list = ctx.options.ciphers_server.split(":") diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 1a58d10dfc..5a433778ad 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -7,7 +7,7 @@ from mitmproxy import connection from mitmproxy.proxy import commands, events, layer from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.layers import dns, quic, tls, udp +from mitmproxy.proxy.layers import quic, tls from mitmproxy.proxy.mode_specs import ReverseMode from mitmproxy.proxy.utils import expect @@ -68,12 +68,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if not self.context.options.keep_host_header: self.context.server.sni = spec.address[0] self.child_layer = tls.ServerTLSLayer(self.context) - elif spec.scheme == "udp": - self.child_layer = udp.UDPLayer(self.context) - elif spec.scheme == "http" or spec.scheme == "tcp": + elif spec.scheme in ("tcp", "http", "udp", "dns"): self.child_layer = layer.NextLayer(self.context) - elif spec.scheme == "dns": - self.child_layer = dns.DNSLayer(self.context) else: raise AssertionError(spec.scheme) # pragma: no cover diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index cb08812417..32cac1154b 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1057,8 +1057,8 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo return False, f"Cannot parse ClientHello: {str(e)} ({data.hex()})" # copy the client hello information - self.context.client.sni = client_hello.sni - self.context.client.alpn_offers = client_hello.alpn_protocols + self.conn.sni = client_hello.sni + self.conn.alpn_offers = client_hello.alpn_protocols # check with addons what we shall do tls_clienthello = ClientHelloData(self.context, client_hello) @@ -1068,7 +1068,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo if tls_clienthello.ignore_connection: self.conn = self.tunnel_connection = connection.Client( ("ignore-conn", 0), ("ignore-conn", 0), time.time(), - transport_protocol="udp", proxy_mode=self.context.client.proxy_mode + transport_protocol="udp" ) # we need to replace the server layer as well, if there is one diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index c76d1ccde0..ce52a771f8 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -289,7 +289,13 @@ def test_next_layer_reverse_quic_mode(self): layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), ] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) + assert nl._next_layer(ctx, b"", b"") is None + assert isinstance(nl._next_layer(ctx, b"notahandshake", b""), layers.UDPLayer) + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + ] + assert isinstance(nl._next_layer(ctx, quic_client_hello, b""), layers.ClientQuicLayer) def test_next_layer_reverse_http3_mode(self): nl = NextLayer() @@ -301,8 +307,8 @@ def test_next_layer_reverse_http3_mode(self): ctx.layers = [ layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), - layers.ClientQuicLayer(ctx), ] + assert isinstance(nl._next_layer(ctx, b"notahandshakebutignore", b""), layers.ClientQuicLayer) decision = nl._next_layer(ctx, b"", b"") assert isinstance(decision, layers.HttpLayer) assert decision.mode is HTTPMode.transparent diff --git a/test/mitmproxy/proxy/layers/test_modes.py b/test/mitmproxy/proxy/layers/test_modes.py index e3f4a70306..e8610d0f02 100644 --- a/test/mitmproxy/proxy/layers/test_modes.py +++ b/test/mitmproxy/proxy/layers/test_modes.py @@ -161,6 +161,8 @@ def test_reverse_dns(tctx): assert ( Playbook(modes.ReverseProxy(tctx), hooks=False) >> DataReceived(tctx.client, tflow.tdnsreq().packed) + << NextLayerHook(Placeholder(NextLayer)) + >> reply_next_layer(layers.DNSLayer) << layers.dns.DnsRequestHook(f) >> reply(None) << OpenConnection(server) @@ -202,9 +204,11 @@ def test_udp(tctx: Context): Playbook(modes.ReverseProxy(tctx)) << OpenConnection(tctx.server) >> reply(None) + >> DataReceived(tctx.client, b"test-input") + << NextLayerHook(Placeholder(NextLayer)) + >> reply_next_layer(layers.UDPLayer) << udp.UdpStartHook(flow) >> reply() - >> DataReceived(tctx.client, b"test-input") << udp.UdpMessageHook(flow) >> reply() << SendData(tctx.server, b"test-input") From 206e2f50eb9b6f9154f07316285d8964a4c9018d Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Mon, 7 Nov 2022 04:12:35 +0100 Subject: [PATCH 102/695] [quic] improve H3 close connection --- mitmproxy/proxy/layers/http/_http3.py | 16 ++++++++++------ mitmproxy/proxy/layers/http/_http_h3.py | 9 +++++++-- mitmproxy/proxy/layers/quic.py | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 4ee46c94fd..3275fcd1a9 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -162,19 +162,23 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield from self.h3_conn.transmit() # report a protocol error for all remaining open streams when a connection is closed - elif isinstance(event, events.ConnectionClosed): + elif isinstance(event, QuicConnectionClosed): self._handle_event = self.done # type: ignore - if isinstance(event, QuicConnectionClosed): - msg = event.reason_phrase or error_code_to_str(event.error_code) - else: - msg = "peer closed connection" + msg = event.reason_phrase or error_code_to_str(event.error_code) for stream_id in self.h3_conn.get_reserved_stream_ids(): yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) + # turn `QuicErrorCode.NO_ERROR` into `H3ErrorCode.H3_NO_ERROR` + self.h3_conn.close_connection( + event.error_code or H3ErrorCode.H3_NO_ERROR, + event.frame_type, + event.reason_phrase, + ) + yield from self.h3_conn.transmit() else: raise AssertionError(f"Unexpected event: {event!r}") - @expect(QuicStreamEvent, HttpEvent, events.ConnectionClosed) + @expect(HttpEvent, QuicStreamEvent, QuicConnectionClosed) def done(self, _) -> layer.CommandGenerator[None]: yield from () diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index d70a441633..d4573ca852 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -143,11 +143,16 @@ def _handle_request_or_push_frame( events[index] = TrailersReceived(event.headers, event.stream_id, event.stream_ended, event.push_id) return events - def close_connection(self, error_code: int = QuicErrorCode.NO_ERROR, reason_phrase: str = "") -> None: + def close_connection( + self, + error_code: int = QuicErrorCode.NO_ERROR, + frame_type: Optional[int] = None, + reason_phrase: str = "", + ) -> None: """Closes the underlying QUIC connection and ignores any incoming events.""" self._is_done = True - self._quic.close(error_code=error_code, reason_phrase=reason_phrase) + self._quic.close(error_code, frame_type, reason_phrase) def end_stream(self, stream_id: int) -> None: """Ends the given stream locally.""" diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 32cac1154b..b5fdb41c1b 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -927,7 +927,7 @@ def receive_close(self) -> layer.CommandGenerator[None]: # if `_close_event` is not set, the underlying connection has been closed # we turn this into a QUIC close event as well close_event = self.quic._close_event or quic_events.ConnectionTerminated( - QuicErrorCode.APPLICATION_ERROR, None, "Connection closed." + QuicErrorCode.NO_ERROR, None, "Connection closed." ) yield from self.event_to_child( QuicConnectionClosed(self.conn, close_event.error_code, close_event.frame_type, close_event.reason_phrase) From 6cf2a1202aaa24156b471e6f0a4c1fd58ad57602 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 8 Nov 2022 05:38:59 +0100 Subject: [PATCH 103/695] [quic] full-stack test --- mitmproxy/addons/tlsconfig.py | 6 +- test/mitmproxy/addons/test_next_layer.py | 13 +- test/mitmproxy/addons/test_proxyserver.py | 341 ++++++++++++++++++++++ 3 files changed, 356 insertions(+), 4 deletions(-) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 58b2503acf..40bbee735b 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -325,8 +325,12 @@ def quic_start_client(self, tls_start: quic.QuicTlsData) -> None: tls_start.settings.cipher_suites = [ CipherSuite[cipher] for cipher in client.cipher_list ] + # if we don't have upstream ALPN, we allow all offered by the client tls_start.settings.alpn_protocols = [ - alpn.decode("ascii") for alpn in (client.alpn, server.alpn) if alpn + alpn.decode("ascii") + for alpn in [ + alpn for alpn in (client.alpn, server.alpn) if alpn + ] or client.alpn_offers ] # set the certificates diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index ce52a771f8..fc06a609dd 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -309,6 +309,7 @@ def test_next_layer_reverse_http3_mode(self): layers.ServerQuicLayer(ctx), ] assert isinstance(nl._next_layer(ctx, b"notahandshakebutignore", b""), layers.ClientQuicLayer) + assert len(ctx.layers) == 3 decision = nl._next_layer(ctx, b"", b"") assert isinstance(decision, layers.HttpLayer) assert decision.mode is HTTPMode.transparent @@ -333,7 +334,9 @@ def test_next_layer_reverse_dtls_mode(self): ctx.client.proxy_mode.scheme = "dtls" ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx)] assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) - ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx), layers.ClientTLSLayer(ctx)] + ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx)] + assert isinstance(nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), layers.ClientTLSLayer) + assert len(ctx.layers) == 3 assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) def test_next_layer_reverse_udp_mode(self): @@ -345,7 +348,9 @@ def test_next_layer_reverse_udp_mode(self): ctx.client.proxy_mode.scheme = "udp" ctx.layers = [layers.modes.ReverseProxy(ctx)] assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) - ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ClientTLSLayer(ctx)] + ctx.layers = [layers.modes.ReverseProxy(ctx)] + assert isinstance(nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), layers.ClientTLSLayer) + assert len(ctx.layers) == 2 assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) def test_next_layer_reverse_dns_mode(self): @@ -357,7 +362,9 @@ def test_next_layer_reverse_dns_mode(self): ctx.client.proxy_mode.scheme = "dns" ctx.layers = [layers.modes.ReverseProxy(ctx)] assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) - ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ClientTLSLayer(ctx)] + ctx.layers = [layers.modes.ReverseProxy(ctx)] + assert isinstance(nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), layers.ClientTLSLayer) + assert len(ctx.layers) == 2 assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) def test_next_layer_invalid_proto(self): diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index b31303acf8..cda5bcc485 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -1,9 +1,21 @@ import asyncio from contextlib import asynccontextmanager +from dataclasses import dataclass import socket +import ssl +from typing import AsyncGenerator, Optional, TypeVar from unittest.mock import Mock +from aioquic.asyncio.protocol import QuicConnectionProtocol +from aioquic.asyncio.server import QuicServer +from aioquic.h3 import events as h3_events +from aioquic.h3.connection import H3Connection, FrameUnexpected +from aioquic.quic import events as quic_events +from aioquic.quic.configuration import QuicConfiguration +from aioquic.quic.connection import QuicConnection, QuicConnectionError import pytest +from mitmproxy.addons.next_layer import NextLayer +from mitmproxy.addons.tlsconfig import TlsConfig import mitmproxy.platform from mitmproxy import dns, exceptions @@ -17,6 +29,10 @@ from mitmproxy.test import taddons, tflow from mitmproxy.test.tflow import tclient_conn, tserver_conn from mitmproxy.test.tutils import tdnsreq +from mitmproxy.utils import data + + +tlsdata = data.Data(__name__) class HelperAddon: @@ -352,3 +368,328 @@ def server_handler( assert len(ps.connections) == 1 tctx.configure(ps, server=False) await caplog_async.await_log("Stopped reverse proxy to dtls") + + +class H3EchoServer(QuicConnectionProtocol): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._seen_headers: set[int] = set() + self.http: Optional[H3Connection] = None + + def http_headers_received(self, event: h3_events.HeadersReceived) -> None: + assert event.push_id is None + headers: dict[bytes, bytes] = {} + for name, value in event.headers: + headers[name] = value + response = [] + if event.stream_id not in self._seen_headers: + self._seen_headers.add(event.stream_id) + assert headers[b":authority"] == b"example.mitmproxy.org" + assert headers[b":method"] == b"GET" + assert headers[b":path"] == b"/test" + response.append((b":status", b"200")) + response.append((b"x-response", headers[b"x-request"])) + self.http.send_headers( + stream_id=event.stream_id, + headers=response, + end_stream=event.stream_ended + ) + self.transmit() + + def http_data_received(self, event: h3_events.DataReceived) -> None: + assert event.push_id is None + assert event.stream_id in self._seen_headers + try: + self.http.send_data( + stream_id=event.stream_id, + data=event.data, + end_stream=event.stream_ended, + ) + except FrameUnexpected: + if event.data or not event.stream_ended: + raise + self._quic.send_stream_data( + stream_id=event.stream_id, + data=b"", + end_stream=True, + ) + self.transmit() + + def http_event_received(self, event: h3_events.H3Event) -> None: + if isinstance(event, h3_events.HeadersReceived): + self.http_headers_received(event) + elif isinstance(event, h3_events.DataReceived): + self.http_data_received(event) + else: + raise AssertionError(event) + + def quic_event_received(self, event: quic_events.QuicEvent) -> None: + if isinstance(event, quic_events.ProtocolNegotiated): + assert event.alpn_protocol == "h3" + self.http = H3Connection(self._quic) + if self.http is not None: + for http_event in self.http.handle_event(event): + self.http_event_received(http_event) + + +@asynccontextmanager +async def quic_server(create_protocol, alpn: list[str]) -> AsyncGenerator[Address, None]: + configuration = QuicConfiguration(is_client=False, alpn_protocols=alpn) + configuration.load_cert_chain( + certfile=tlsdata.path("../net/data/verificationcerts/trusted-leaf.crt"), + keyfile=tlsdata.path("../net/data/verificationcerts/trusted-leaf.key"), + ) + loop = asyncio.get_running_loop() + transport, server = await loop.create_datagram_endpoint( + lambda: QuicServer( + configuration=configuration, + create_protocol=create_protocol, + ), + local_addr=("127.0.0.1", 0), + ) + try: + yield transport.get_extra_info("sockname") + finally: + server.close() + + +@dataclass +class H3Response: + waiter: asyncio.Future + headers: Optional[h3_events.H3Event] = None + data: Optional[bytes] = None + trailers: Optional[h3_events.H3Event] = None + + +class QuicClient(QuicConnectionProtocol): + TIMEOUT = 5 + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._waiter = self._loop.create_future() + + def quic_event_received(self, event: quic_events.QuicEvent) -> None: + if not self._waiter.done(): + if isinstance(event, quic_events.ConnectionTerminated): + self._waiter.set_exception(QuicConnectionError( + event.error_code, event.frame_type, event.reason_phrase + )) + elif isinstance(event, quic_events.HandshakeCompleted): + self._waiter.set_result(None) + + def connection_lost(self, exc: Optional[Exception]) -> None: + if not self._waiter.done(): + self._waiter.set_exception(exc) + return super().connection_lost(exc) + + async def wait_handshake(self) -> None: + return await asyncio.wait_for(self._waiter, timeout=self.TIMEOUT) + + +class H3Client(QuicClient): + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._responses: dict[int, H3Response] = dict() + self.http = H3Connection(self._quic) + + def http_headers_received(self, event: h3_events.HeadersReceived) -> None: + assert event.push_id is None + response = self._responses[event.stream_id] + if response.waiter.done(): + return + if response.headers is None: + response.headers = event.headers + if event.stream_ended: + response.waiter.set_result(response) + elif response.trailers is None: + response.trailers = event.headers + if event.stream_ended: + response.waiter.set_result(response) + else: + response.waiter.set_exception(Exception("Headers after trailers received.")) + + def http_data_received(self, event: h3_events.DataReceived) -> None: + assert event.push_id is None + response = self._responses[event.stream_id] + if response.waiter.done(): + return + if response.headers is None: + response.waiter.set_exception(Exception("Data without headers received.")) + elif response.trailers is None: + if response.data is None: + response.data = event.data + else: + response.data = response.data + event.data + if event.stream_ended: + response.waiter.set_result(response) + elif event.data or not event.stream_ended: + response.waiter.set_exception(Exception("Data after trailers received.")) + else: + response.waiter.set_result(response) + + def http_event_received(self, event: h3_events.H3Event) -> None: + if isinstance(event, h3_events.HeadersReceived): + self.http_headers_received(event) + elif isinstance(event, h3_events.DataReceived): + self.http_data_received(event) + else: + raise AssertionError(event) + + def quic_event_received(self, event: quic_events.QuicEvent) -> None: + super().quic_event_received(event) + for http_event in self.http.handle_event(event): + self.http_event_received(http_event) + + async def request( + self, + headers: h3_events.H3Event, + data: Optional[bytes] = None, + trailers: Optional[h3_events.H3Event] = None, + ) -> H3Response: + stream_id = self._quic.get_next_available_stream_id() + self.http.send_headers( + stream_id=stream_id, + headers=headers, + end_stream=data is None and trailers is None, + ) + if data is not None: + self.http.send_data( + stream_id=stream_id, + data=data, + end_stream=trailers is None, + ) + if trailers is not None: + self.http.send_headers( + stream_id=stream_id, + headers=trailers, + end_stream=True, + ) + waiter = self._loop.create_future() + self._responses[stream_id] = H3Response(waiter=waiter) + self.transmit() + return await asyncio.wait_for(waiter, timeout=self.TIMEOUT) + + +T = TypeVar("T", bound=QuicClient) + + +@asynccontextmanager +async def quic_connect( + cls: type[T], + alpn: list[str], + address: Address, +) -> AsyncGenerator[T, None]: + configuration = QuicConfiguration( + is_client=True, + alpn_protocols=alpn, + server_name="example.mitmproxy.org", + verify_mode=ssl.CERT_NONE, + ) + loop = asyncio.get_running_loop() + transport, protocol = await loop.create_datagram_endpoint( + lambda: cls(QuicConnection(configuration=configuration)), + local_addr=("127.0.0.1", 0), + ) + assert isinstance(protocol, cls) + try: + protocol.connect(address) + await protocol.wait_handshake() + yield protocol + finally: + protocol.close() + await protocol.wait_closed() + transport.close() + + +@pytest.mark.parametrize("connection_strategy", ["lazy", "eager"]) +@pytest.mark.parametrize("scheme", ["http3", "quic"]) +async def test_reverse_http3( + caplog_async, scheme: str, connection_strategy: str +) -> None: + def assert_no_data(response: H3Response): + # http3 is more strict + if scheme == "http3": + assert response.data is None + else: + assert not response.data + + caplog_async.set_level("INFO") + ps = Proxyserver() + nl = NextLayer() + ta = TlsConfig() + with taddons.context(ps, nl, ta) as tctx: + tctx.options.keep_host_header = True + tctx.options.connection_strategy = connection_strategy + ta.configure(["confdir"]) + async with quic_server(H3EchoServer, alpn=["h3"]) as server_addr: + mode = f"reverse:{scheme}://{server_addr[0]}:{server_addr[1]}@127.0.0.1:0" + tctx.configure( + ta, + ssl_verify_upstream_trusted_ca=tlsdata.path( + "../net/data/verificationcerts/trusted-root.crt" + ), + ) + tctx.configure(ps, mode=[mode]) + assert await ps.setup_servers() + ps.running() + await caplog_async.await_log(f"reverse proxy to {scheme}://{server_addr[0]}:{server_addr[1]} listening") + assert ps.servers + addr = ps.servers[mode].listen_addrs[0] + async with quic_connect(H3Client, alpn=["h3"], address=addr) as client: + headers = [ + (b":scheme", b"https"), + (b":authority", b"example.mitmproxy.org"), + (b":method", b"GET"), + (b":path", b"/test"), + ] + r1 = await client.request( + headers=headers + [(b"x-request", b"justheaders")], + data=None, + trailers=None, + ) + assert r1.headers == [ + (b":status", b"200"), + (b"x-response", b"justheaders"), + ] + assert_no_data(r1) + assert r1.trailers is None + + r2 = await client.request( + headers=headers + [(b"x-request", b"hasdata")], + data=b"echo", + trailers=None, + ) + assert r2.headers == [ + (b":status", b"200"), + (b"x-response", b"hasdata"), + ] + assert r2.data == b"echo" + assert r2.trailers is None + + r3 = await client.request( + headers=headers + [(b"x-request", b"nodata")], + data=None, + trailers=[(b"x-request", b"buttrailers")], + ) + assert r3.headers == [ + (b":status", b"200"), + (b"x-response", b"nodata"), + ] + assert_no_data(r3) + assert r3.trailers == [(b"x-response", b"buttrailers")] + + r4 = await client.request( + headers=headers + [(b"x-request", b"this")], + data=b"has", + trailers=[(b"x-request", b"everything")], + ) + assert r4.headers == [ + (b":status", b"200"), + (b"x-response", b"this"), + ] + assert r4.data == b"has" + assert r4.trailers == [(b"x-response", b"everything")] + + tctx.configure(ps, server=False) + await caplog_async.await_log(f"Stopped reverse proxy to {scheme}") From 64fa241825bd5b61a1da9c3f3bc90b5125a5919a Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 8 Nov 2022 06:34:34 +0100 Subject: [PATCH 104/695] [quic] datagram test --- test/mitmproxy/addons/test_proxyserver.py | 141 ++++++++++++++++++---- 1 file changed, 117 insertions(+), 24 deletions(-) diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index cda5bcc485..7958b82104 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -3,7 +3,8 @@ from dataclasses import dataclass import socket import ssl -from typing import AsyncGenerator, Optional, TypeVar +from typing import AsyncGenerator, ClassVar, Optional, TypeVar +from typing_extensions import Self from unittest.mock import Mock from aioquic.asyncio.protocol import QuicConnectionProtocol @@ -425,16 +426,26 @@ def http_event_received(self, event: h3_events.H3Event) -> None: def quic_event_received(self, event: quic_events.QuicEvent) -> None: if isinstance(event, quic_events.ProtocolNegotiated): - assert event.alpn_protocol == "h3" self.http = H3Connection(self._quic) if self.http is not None: for http_event in self.http.handle_event(event): self.http_event_received(http_event) +class QuicDatagramEchoServer(QuicConnectionProtocol): + def quic_event_received(self, event: quic_events.QuicEvent) -> None: + if isinstance(event, quic_events.DatagramFrameReceived): + self._quic.send_datagram_frame(event.data) + self.transmit() + + @asynccontextmanager async def quic_server(create_protocol, alpn: list[str]) -> AsyncGenerator[Address, None]: - configuration = QuicConfiguration(is_client=False, alpn_protocols=alpn) + configuration = QuicConfiguration( + is_client=False, + alpn_protocols=alpn, + max_datagram_frame_size=65536, + ) configuration.load_cert_chain( certfile=tlsdata.path("../net/data/verificationcerts/trusted-leaf.crt"), keyfile=tlsdata.path("../net/data/verificationcerts/trusted-leaf.key"), @@ -453,16 +464,8 @@ async def quic_server(create_protocol, alpn: list[str]) -> AsyncGenerator[Addres server.close() -@dataclass -class H3Response: - waiter: asyncio.Future - headers: Optional[h3_events.H3Event] = None - data: Optional[bytes] = None - trailers: Optional[h3_events.H3Event] = None - - class QuicClient(QuicConnectionProtocol): - TIMEOUT = 5 + TIMEOUT: ClassVar[int] = 5 def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -483,7 +486,42 @@ def connection_lost(self, exc: Optional[Exception]) -> None: return super().connection_lost(exc) async def wait_handshake(self) -> None: - return await asyncio.wait_for(self._waiter, timeout=self.TIMEOUT) + return await asyncio.wait_for(self._waiter, timeout=QuicClient.TIMEOUT) + + +class QuicDatagramClient(QuicClient): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._datagram: asyncio.Future[bytes] = self._loop.create_future() + + def quic_event_received(self, event: quic_events.QuicEvent) -> None: + super().quic_event_received(event) + if not self._datagram.done(): + if isinstance(event, quic_events.DatagramFrameReceived): + self._datagram.set_result(event.data) + elif isinstance(event, quic_events.ConnectionTerminated): + self._datagram.set_exception(QuicConnectionError( + event.error_code, event.frame_type, event.reason_phrase + )) + + def send_datagram(self, data: bytes) -> None: + self._quic.send_datagram_frame(data) + self.transmit() + + async def recv_datagram(self) -> bytes: + return await asyncio.wait_for(self._datagram, timeout=QuicClient.TIMEOUT) + + +@dataclass +class H3Response: + waiter: asyncio.Future[Self] + stream_id: int + headers: Optional[h3_events.H3Event] = None + data: Optional[bytes] = None + trailers: Optional[h3_events.H3Event] = None + + async def wait_result(self) -> Self: + return await asyncio.wait_for(self.waiter, timeout=QuicClient.TIMEOUT) class H3Client(QuicClient): @@ -541,34 +579,36 @@ def quic_event_received(self, event: quic_events.QuicEvent) -> None: for http_event in self.http.handle_event(event): self.http_event_received(http_event) - async def request( + def request( self, headers: h3_events.H3Event, data: Optional[bytes] = None, trailers: Optional[h3_events.H3Event] = None, + end_stream: bool = True, ) -> H3Response: stream_id = self._quic.get_next_available_stream_id() self.http.send_headers( stream_id=stream_id, headers=headers, - end_stream=data is None and trailers is None, + end_stream=data is None and trailers is None and end_stream, ) if data is not None: self.http.send_data( stream_id=stream_id, data=data, - end_stream=trailers is None, + end_stream=trailers is None and end_stream, ) if trailers is not None: self.http.send_headers( stream_id=stream_id, headers=trailers, - end_stream=True, + end_stream=end_stream, ) waiter = self._loop.create_future() - self._responses[stream_id] = H3Response(waiter=waiter) + response = H3Response(waiter=waiter, stream_id=stream_id) + self._responses[stream_id] = response self.transmit() - return await asyncio.wait_for(waiter, timeout=self.TIMEOUT) + return response T = TypeVar("T", bound=QuicClient) @@ -585,6 +625,7 @@ async def quic_connect( alpn_protocols=alpn, server_name="example.mitmproxy.org", verify_mode=ssl.CERT_NONE, + max_datagram_frame_size=65536, ) loop = asyncio.get_running_loop() transport, protocol = await loop.create_datagram_endpoint( @@ -604,7 +645,7 @@ async def quic_connect( @pytest.mark.parametrize("connection_strategy", ["lazy", "eager"]) @pytest.mark.parametrize("scheme", ["http3", "quic"]) -async def test_reverse_http3( +async def test_reverse_http3_and_quic_stream( caplog_async, scheme: str, connection_strategy: str ) -> None: def assert_no_data(response: H3Response): @@ -647,7 +688,7 @@ def assert_no_data(response: H3Response): headers=headers + [(b"x-request", b"justheaders")], data=None, trailers=None, - ) + ).wait_result() assert r1.headers == [ (b":status", b"200"), (b"x-response", b"justheaders"), @@ -659,7 +700,7 @@ def assert_no_data(response: H3Response): headers=headers + [(b"x-request", b"hasdata")], data=b"echo", trailers=None, - ) + ).wait_result() assert r2.headers == [ (b":status", b"200"), (b"x-response", b"hasdata"), @@ -671,7 +712,7 @@ def assert_no_data(response: H3Response): headers=headers + [(b"x-request", b"nodata")], data=None, trailers=[(b"x-request", b"buttrailers")], - ) + ).wait_result() assert r3.headers == [ (b":status", b"200"), (b"x-response", b"nodata"), @@ -683,7 +724,7 @@ def assert_no_data(response: H3Response): headers=headers + [(b"x-request", b"this")], data=b"has", trailers=[(b"x-request", b"everything")], - ) + ).wait_result() assert r4.headers == [ (b":status", b"200"), (b"x-response", b"this"), @@ -691,5 +732,57 @@ def assert_no_data(response: H3Response): assert r4.data == b"has" assert r4.trailers == [(b"x-response", b"everything")] + r5 = client.request( + headers=headers + [(b"x-request", b"this")], + data=b"has", + trailers=[(b"x-request", b"everything")], + end_stream=False, + ) + client._quic.send_stream_data( + stream_id=r5.stream_id, + data=b"", + end_stream=True, + ) + client.transmit() + await r5.wait_result() + assert r5.headers == [ + (b":status", b"200"), + (b"x-response", b"this"), + ] + assert r5.data == b"has" + assert r5.trailers == [(b"x-response", b"everything")] + tctx.configure(ps, server=False) await caplog_async.await_log(f"Stopped reverse proxy to {scheme}") + + +async def test_reverse_quic_datagram(caplog_async) -> None: + caplog_async.set_level("INFO") + ps = Proxyserver() + nl = NextLayer() + ta = TlsConfig() + with taddons.context(ps, nl, ta) as tctx: + tctx.options.keep_host_header = True + # eager is not (yet) support for non-H3 + tctx.options.connection_strategy = "lazy" + ta.configure(["confdir"]) + async with quic_server(QuicDatagramEchoServer, alpn=["dgram"]) as server_addr: + mode = f"reverse:quic://{server_addr[0]}:{server_addr[1]}@127.0.0.1:0" + tctx.configure( + ta, + ssl_verify_upstream_trusted_ca=tlsdata.path( + "../net/data/verificationcerts/trusted-root.crt" + ), + ) + tctx.configure(ps, mode=[mode]) + assert await ps.setup_servers() + ps.running() + await caplog_async.await_log(f"reverse proxy to quic://{server_addr[0]}:{server_addr[1]} listening") + assert ps.servers + addr = ps.servers[mode].listen_addrs[0] + async with quic_connect(QuicDatagramClient, alpn=["dgram"], address=addr) as client: + client.send_datagram(b"echo") + assert await client.recv_datagram() == b"echo" + + tctx.configure(ps, server=False) + await caplog_async.await_log("Stopped reverse proxy to quic") From e1d5f4b838b16c63f4e4b51aa3e53939c7c47625 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 8 Nov 2022 07:06:01 +0100 Subject: [PATCH 105/695] [quic] replace get_event_loop --- mitmproxy/proxy/layers/quic.py | 5 +- test/mitmproxy/addons/test_next_layer.py | 127 ++++++++++++---------- test/mitmproxy/addons/test_proxyserver.py | 7 +- 3 files changed, 76 insertions(+), 63 deletions(-) diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index b5fdb41c1b..2f45da810f 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,5 +1,4 @@ from __future__ import annotations -import asyncio from dataclasses import dataclass, field from logging import DEBUG, ERROR, WARNING from ssl import VerifyMode @@ -27,7 +26,7 @@ ) from cryptography import x509 from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection +from mitmproxy import certs, connection, ctx from mitmproxy.net import tls from mitmproxy.proxy import commands, context, events, layer, tunnel from mitmproxy.proxy.layers.tcp import TCPLayer @@ -723,7 +722,7 @@ class QuicLayer(tunnel.TunnelLayer): def __init__(self, context: context.Context, conn: connection.Connection, time: Callable[[], float] | None) -> None: super().__init__(context, tunnel_connection=conn, conn=conn) self.child_layer = layer.NextLayer(self.context, ask_on_start=True) - self._time = time or asyncio.get_event_loop().time + self._time = time or ctx.master.event_loop.time self._wakeup_commands: dict[commands.RequestWakeup, float] = dict() conn.tls = True diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index fc06a609dd..cdbbe54300 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -250,69 +250,82 @@ def is_http(layer: Optional[layer.Layer], mode: HTTPMode): def test_next_layer_reverse_raw(self): nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - with taddons.context(nl) as tctx: - tctx.configure(nl, ignore_hosts=["example.com"]) - - ctx.layers = [layers.modes.HttpProxy(ctx), layers.ClientQuicLayer(ctx)] - decision = nl._next_layer(ctx, b"", b"") - assert isinstance(decision, layers.ServerQuicLayer) - assert isinstance(decision.child_layer, layers.RawQuicLayer) - - ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), layers.ClientQuicLayer(ctx)] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) - - ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx)] - decision = nl._next_layer(ctx, b"", b"") - assert isinstance(decision, layers.ClientQuicLayer) - assert isinstance(decision.child_layer, layers.RawQuicLayer) - - tctx.configure(nl, ignore_hosts=[]) + with taddons.context(nl): + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + with taddons.context(nl) as tctx: + tctx.configure(nl, ignore_hosts=["example.com"]) + + ctx.layers = [ + layers.modes.HttpProxy(ctx), + layers.ClientQuicLayer(ctx), + ] + decision = nl._next_layer(ctx, b"", b"") + assert isinstance(decision, layers.ServerQuicLayer) + assert isinstance(decision.child_layer, layers.RawQuicLayer) + + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + layers.ClientQuicLayer(ctx,), + ] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) + + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + ] + decision = nl._next_layer(ctx, b"", b"") + assert isinstance(decision, layers.ClientQuicLayer) + assert isinstance(decision.child_layer, layers.RawQuicLayer) + + tctx.configure(nl, ignore_hosts=[]) def test_next_layer_reverse_quic_mode(self): nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "quic" - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - layers.ClientQuicLayer(ctx), - ] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - ] - assert nl._next_layer(ctx, b"", b"") is None - assert isinstance(nl._next_layer(ctx, b"notahandshake", b""), layers.UDPLayer) - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - ] - assert isinstance(nl._next_layer(ctx, quic_client_hello, b""), layers.ClientQuicLayer) + with taddons.context(nl): + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "quic" + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + layers.ClientQuicLayer(ctx), + ] + assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + ] + assert nl._next_layer(ctx, b"", b"") is None + assert isinstance(nl._next_layer(ctx, b"notahandshake", b""), layers.UDPLayer) + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + ] + assert isinstance(nl._next_layer(ctx, quic_client_hello, b""), layers.ClientQuicLayer) def test_next_layer_reverse_http3_mode(self): nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "http3" - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - ] - assert isinstance(nl._next_layer(ctx, b"notahandshakebutignore", b""), layers.ClientQuicLayer) - assert len(ctx.layers) == 3 - decision = nl._next_layer(ctx, b"", b"") - assert isinstance(decision, layers.HttpLayer) - assert decision.mode is HTTPMode.transparent + with taddons.context(nl): + ctx = MagicMock() + ctx.client.alpn = None + ctx.server.address = ("example.com", 443) + ctx.client.transport_protocol = "udp" + ctx.client.proxy_mode.scheme = "http3" + ctx.layers = [ + layers.modes.ReverseProxy(ctx), + layers.ServerQuicLayer(ctx), + ] + assert isinstance(nl._next_layer(ctx, b"notahandshakebutignore", b""), layers.ClientQuicLayer) + assert len(ctx.layers) == 3 + decision = nl._next_layer(ctx, b"", b"") + assert isinstance(decision, layers.HttpLayer) + assert decision.mode is HTTPMode.transparent def test_next_layer_reverse_invalid_mode(self): nl = NextLayer() diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 7958b82104..9bb21d4c10 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -1,10 +1,11 @@ +from __future__ import annotations + import asyncio from contextlib import asynccontextmanager from dataclasses import dataclass import socket import ssl from typing import AsyncGenerator, ClassVar, Optional, TypeVar -from typing_extensions import Self from unittest.mock import Mock from aioquic.asyncio.protocol import QuicConnectionProtocol @@ -514,13 +515,13 @@ async def recv_datagram(self) -> bytes: @dataclass class H3Response: - waiter: asyncio.Future[Self] + waiter: asyncio.Future[H3Response] stream_id: int headers: Optional[h3_events.H3Event] = None data: Optional[bytes] = None trailers: Optional[h3_events.H3Event] = None - async def wait_result(self) -> Self: + async def wait_result(self) -> H3Response: return await asyncio.wait_for(self.waiter, timeout=QuicClient.TIMEOUT) From 2bda324d94143cda4c6d21e06c678e4ad6d71bb3 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 8 Nov 2022 07:51:25 +0100 Subject: [PATCH 106/695] [quic] improve and fix tests --- test/mitmproxy/addons/test_next_layer.py | 12 ++++--- test/mitmproxy/addons/test_proxyserver.py | 6 ++-- test/mitmproxy/proxy/layers/test_modes.py | 41 ++++++++++++----------- test/mitmproxy/proxy/layers/test_quic.py | 12 +------ 4 files changed, 33 insertions(+), 38 deletions(-) diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index cdbbe54300..ff724f6822 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -181,15 +181,15 @@ def test_next_layer2(self): assert isinstance(nl._next_layer(ctx, b"", b"hello"), layers.TCPLayer) @pytest.mark.parametrize( - ("client_hello", "client_layer", "server_layer"), + ("protocol", "client_layer", "server_layer"), [ - (dtls_client_hello_with_extensions, layers.ClientTLSLayer, layers.ServerTLSLayer), - (quic_client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer), + ("dtls", layers.ClientTLSLayer, layers.ServerTLSLayer), + ("quic", layers.ClientQuicLayer, layers.ServerQuicLayer), ] ) def test_next_layer_udp( self, - client_hello: bytes, + protocol: str, client_layer: layer.Layer, server_layer: layer.Layer, ): @@ -205,6 +205,10 @@ def is_http(layer: Optional[layer.Layer], mode: HTTPMode): and layer.mode is mode ) + client_hello = { + "dtls": dtls_client_hello_with_extensions, + "quic": quic_client_hello, + }[protocol] nl = NextLayer() ctx = MagicMock() ctx.client.alpn = None diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 9bb21d4c10..4d3a0d728f 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -757,15 +757,15 @@ def assert_no_data(response: H3Response): await caplog_async.await_log(f"Stopped reverse proxy to {scheme}") -async def test_reverse_quic_datagram(caplog_async) -> None: +@pytest.mark.parametrize("connection_strategy", ["lazy", "eager"]) +async def test_reverse_quic_datagram(caplog_async, connection_strategy: str) -> None: caplog_async.set_level("INFO") ps = Proxyserver() nl = NextLayer() ta = TlsConfig() with taddons.context(ps, nl, ta) as tctx: tctx.options.keep_host_header = True - # eager is not (yet) support for non-H3 - tctx.options.connection_strategy = "lazy" + tctx.options.connection_strategy = connection_strategy ta.configure(["confdir"]) async with quic_server(QuicDatagramEchoServer, alpn=["dgram"]) as server_addr: mode = f"reverse:quic://{server_addr[0]}:{server_addr[1]}@127.0.0.1:0" diff --git a/test/mitmproxy/proxy/layers/test_modes.py b/test/mitmproxy/proxy/layers/test_modes.py index e8610d0f02..10addd1030 100644 --- a/test/mitmproxy/proxy/layers/test_modes.py +++ b/test/mitmproxy/proxy/layers/test_modes.py @@ -26,7 +26,7 @@ ) from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.tcp import TCPFlow -from mitmproxy.test import tflow +from mitmproxy.test import taddons, tflow from mitmproxy.udp import UDPFlow from test.mitmproxy.proxy.layers.test_tls import ( reply_tls_start_client, @@ -174,27 +174,28 @@ def test_reverse_dns(tctx): @pytest.mark.parametrize("keep_host_header", [True, False]) def test_quic(tctx: Context, keep_host_header: bool): - tctx.options.keep_host_header = keep_host_header - tctx.server.sni = "other" - tctx.client.proxy_mode = ProxyMode.parse("reverse:quic://1.2.3.4:5") - client_hello = Placeholder(bytes) + with taddons.context(): + tctx.options.keep_host_header = keep_host_header + tctx.server.sni = "other" + tctx.client.proxy_mode = ProxyMode.parse("reverse:quic://1.2.3.4:5") + client_hello = Placeholder(bytes) - def set_settings(data: quic.QuicTlsData): - data.settings = quic.QuicTlsSettings() + def set_settings(data: quic.QuicTlsData): + data.settings = quic.QuicTlsSettings() - assert ( - Playbook(modes.ReverseProxy(tctx)) - << OpenConnection(tctx.server) - >> reply(None) - << quic.QuicStartServerHook(Placeholder(quic.QuicTlsData)) - >> reply(side_effect=set_settings) - << SendData(tctx.server, client_hello) - << RequestWakeup(Placeholder(float)) - ) - assert tctx.server.address == ("1.2.3.4", 5) - assert quic.quic_parse_client_hello(client_hello()).sni == ( - "other" if keep_host_header else "1.2.3.4" - ) + assert ( + Playbook(modes.ReverseProxy(tctx)) + << OpenConnection(tctx.server) + >> reply(None) + << quic.QuicStartServerHook(Placeholder(quic.QuicTlsData)) + >> reply(side_effect=set_settings) + << SendData(tctx.server, client_hello) + << RequestWakeup(Placeholder(float)) + ) + assert tctx.server.address == ("1.2.3.4", 5) + assert quic.quic_parse_client_hello(client_hello()).sni == ( + "other" if keep_host_header else "1.2.3.4" + ) def test_udp(tctx: Context): diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index 6b707a8114..3c2f486147 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -8,8 +8,7 @@ from typing import Literal, Optional, TypeVar from unittest.mock import MagicMock import pytest -from mitmproxy import connection, options -from mitmproxy.addons.proxyserver import Proxyserver +from mitmproxy import connection from mitmproxy.proxy import commands, context, events, layer, tunnel from mitmproxy.proxy import layers from mitmproxy.proxy.layers import quic, tcp, tls, udp @@ -26,15 +25,6 @@ T = TypeVar('T', bound=layer.Layer) -@pytest.fixture -def tctx() -> context.Context: - opts = options.Options() - Proxyserver().load(opts) - return context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts - ) - - class DummyLayer(layer.Layer): child_layer: Optional[layer.Layer] From 5aa6c9b9f9b7c1bf4de71d561a9b900b5ee0aea6 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Tue, 8 Nov 2022 22:08:12 +0100 Subject: [PATCH 107/695] [quic] minor improvements --- mitmproxy/addons/next_layer.py | 5 +++-- mitmproxy/proxy/layers/quic.py | 8 +++++++- test/mitmproxy/addons/test_proxyserver.py | 21 ++++++++++++++++++--- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 3b4b1b68f8..bf4be48dff 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -242,10 +242,11 @@ def _next_layer( assert context.layers if context.client.transport_protocol == "tcp": + is_quic_stream = isinstance(context.layers[-1], layers.QuicStreamLayer) if ( len(data_client) < 3 and not data_server - and not isinstance(context.layers[-1], layers.QuicStreamLayer) + and not is_quic_stream ): return None # not enough data yet to make a decision @@ -253,7 +254,7 @@ def _next_layer( ignore = self.ignore_connection(context.server.address, data_client) if ignore is True: return layers.TCPLayer(context, ignore=True) - if ignore is None: + if ignore is None and not is_quic_stream: return None # 2. Check for TLS diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 2f45da810f..3395fd4bd4 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -743,6 +743,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: yield from super()._handle_event(event) + def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: + # the parent will call _handle_command multiple times, we transmit cumulative afterwards + # this will reduce the number of sends, especially if data=b"" and end_stream=True + yield from super().event_to_child(event) + if self.quic: + yield from self.tls_interact() + def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[None]: """Turns stream commands into aioquic connection invocations.""" if ( @@ -760,7 +767,6 @@ def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[N self.quic.stop_stream(command.stream_id, command.error_code) else: raise AssertionError(f"Unexpected stream command: {command!r}") - yield from self.tls_interact() else: yield from super()._handle_command(command) diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 4d3a0d728f..8fc7e534a2 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -5,7 +5,7 @@ from dataclasses import dataclass import socket import ssl -from typing import AsyncGenerator, ClassVar, Optional, TypeVar +from typing import Any, AsyncGenerator, Callable, ClassVar, Optional, TypeVar from unittest.mock import Mock from aioquic.asyncio.protocol import QuicConnectionProtocol @@ -520,10 +520,16 @@ class H3Response: headers: Optional[h3_events.H3Event] = None data: Optional[bytes] = None trailers: Optional[h3_events.H3Event] = None + callback: Optional[Callable[[str], None]] = None async def wait_result(self) -> H3Response: return await asyncio.wait_for(self.waiter, timeout=QuicClient.TIMEOUT) + def __setattr__(self, name: str, value: Any) -> None: + super().__setattr__(name, value) + if self.callback: + self.callback(name) + class H3Client(QuicClient): @@ -733,12 +739,21 @@ def assert_no_data(response: H3Response): assert r4.data == b"has" assert r4.trailers == [(b"x-response", b"everything")] + # the following test makes sure that we behave properly if end_stream is sent separately r5 = client.request( headers=headers + [(b"x-request", b"this")], data=b"has", - trailers=[(b"x-request", b"everything")], + trailers=[(b"x-request", b"everything but end_stream")], end_stream=False, ) + if scheme == "quic": + trailer_waiter = asyncio.get_running_loop().create_future() + r5.callback = lambda name: name != "trailers" or trailer_waiter.set_result(None) + await asyncio.wait_for(trailer_waiter, timeout=QuicClient.TIMEOUT) + assert r5.trailers is not None + assert not r5.waiter.done() + else: + await asyncio.sleep(0) client._quic.send_stream_data( stream_id=r5.stream_id, data=b"", @@ -751,7 +766,7 @@ def assert_no_data(response: H3Response): (b"x-response", b"this"), ] assert r5.data == b"has" - assert r5.trailers == [(b"x-response", b"everything")] + assert r5.trailers == [(b"x-response", b"everything but end_stream")] tctx.configure(ps, server=False) await caplog_async.await_log(f"Stopped reverse proxy to {scheme}") From a22f6c4d469a85a3c9071ef25fc11eaea8118eaf Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 9 Nov 2022 17:35:12 +0100 Subject: [PATCH 108/695] fix stray quickhelp entry, refs #4515 --- mitmproxy/tools/console/quickhelp.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mitmproxy/tools/console/quickhelp.py b/mitmproxy/tools/console/quickhelp.py index 58878c3d4b..18e6c90d79 100644 --- a/mitmproxy/tools/console/quickhelp.py +++ b/mitmproxy/tools/console/quickhelp.py @@ -67,10 +67,11 @@ def make( "Export": "Export this flow to file", "Delete": "Delete flow from view", } - if focused_flow.marked: - top_items["Unmark"] = "Toggle mark on this flow" - else: - top_items["Mark"] = "Toggle mark on this flow" + if widget == FlowListBox: + if focused_flow.marked: + top_items["Unmark"] = "Toggle mark on this flow" + else: + top_items["Mark"] = "Toggle mark on this flow" if focused_flow.intercepted: top_items["Resume"] = "Resume this intercepted flow" if focused_flow.modified(): From 3b8e1f6e2af57afeceebeb9d8558def820814200 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Nov 2022 13:23:10 +0100 Subject: [PATCH 109/695] skip self-test for console ui --- release/build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/build.py b/release/build.py index c5efff9380..6878a9e7f2 100644 --- a/release/build.py +++ b/release/build.py @@ -148,6 +148,9 @@ def _test_binaries(binary_directory: Path) -> None: print(f"> {tool} --version") subprocess.check_call([executable, "--version"]) + if tool == "mitmproxy": + continue # requires a TTY, which we don't have here. + print(f"> {tool} -s selftest.py") subprocess.check_call([executable, "-s", here / "selftest.py"]) From f8bd7516ccd482786bbca4919f48921a2aebddd5 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Nov 2022 17:05:47 +0100 Subject: [PATCH 110/695] unbreak docker ci --- release/build-and-deploy-docker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/build-and-deploy-docker.py b/release/build-and-deploy-docker.py index 0a2d6f56e4..50a6ce4963 100644 --- a/release/build-and-deploy-docker.py +++ b/release/build-and-deploy-docker.py @@ -48,16 +48,16 @@ "run", "--rm", "-v", - f"{root / 'release'}:/release" + f"{root / 'release'}:/release", "localtesting", "mitmdump", "-s", "/release/selftest.py", ], - check=True, capture_output=True, ) print(r.stdout.decode()) assert "Self-test successful" in r.stdout.decode() +assert r.returncode == 0 # Now we can deploy. subprocess.check_call( From fcee888f35d6248efb1946069d24bf4fdcd86d49 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 10 Nov 2022 23:24:36 +0100 Subject: [PATCH 111/695] [quic] regular proxy test --- test/mitmproxy/addons/test_proxyserver.py | 222 +++++++++++++--------- test/mitmproxy/proxy/layers/test_quic.py | 2 +- 2 files changed, 134 insertions(+), 90 deletions(-) diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 8fc7e534a2..5912d8eac9 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -650,18 +650,102 @@ async def quic_connect( transport.close() -@pytest.mark.parametrize("connection_strategy", ["lazy", "eager"]) -@pytest.mark.parametrize("scheme", ["http3", "quic"]) -async def test_reverse_http3_and_quic_stream( - caplog_async, scheme: str, connection_strategy: str -) -> None: +async def _test_echo(client: H3Client, strict: bool) -> None: def assert_no_data(response: H3Response): - # http3 is more strict - if scheme == "http3": + if strict: assert response.data is None else: assert not response.data + headers = [ + (b":scheme", b"https"), + (b":authority", b"example.mitmproxy.org"), + (b":method", b"GET"), + (b":path", b"/test"), + ] + r1 = await client.request( + headers=headers + [(b"x-request", b"justheaders")], + data=None, + trailers=None, + ).wait_result() + assert r1.headers == [ + (b":status", b"200"), + (b"x-response", b"justheaders"), + ] + assert_no_data(r1) + assert r1.trailers is None + + r2 = await client.request( + headers=headers + [(b"x-request", b"hasdata")], + data=b"echo", + trailers=None, + ).wait_result() + assert r2.headers == [ + (b":status", b"200"), + (b"x-response", b"hasdata"), + ] + assert r2.data == b"echo" + assert r2.trailers is None + + r3 = await client.request( + headers=headers + [(b"x-request", b"nodata")], + data=None, + trailers=[(b"x-request", b"buttrailers")], + ).wait_result() + assert r3.headers == [ + (b":status", b"200"), + (b"x-response", b"nodata"), + ] + assert_no_data(r3) + assert r3.trailers == [(b"x-response", b"buttrailers")] + + r4 = await client.request( + headers=headers + [(b"x-request", b"this")], + data=b"has", + trailers=[(b"x-request", b"everything")], + ).wait_result() + assert r4.headers == [ + (b":status", b"200"), + (b"x-response", b"this"), + ] + assert r4.data == b"has" + assert r4.trailers == [(b"x-response", b"everything")] + + # the following test makes sure that we behave properly if end_stream is sent separately + r5 = client.request( + headers=headers + [(b"x-request", b"this")], + data=b"has", + trailers=[(b"x-request", b"everything but end_stream")], + end_stream=False, + ) + if not strict: + trailer_waiter = asyncio.get_running_loop().create_future() + r5.callback = lambda name: name != "trailers" or trailer_waiter.set_result(None) + await asyncio.wait_for(trailer_waiter, timeout=QuicClient.TIMEOUT) + assert r5.trailers is not None + assert not r5.waiter.done() + else: + await asyncio.sleep(0) + client._quic.send_stream_data( + stream_id=r5.stream_id, + data=b"", + end_stream=True, + ) + client.transmit() + await r5.wait_result() + assert r5.headers == [ + (b":status", b"200"), + (b"x-response", b"this"), + ] + assert r5.data == b"has" + assert r5.trailers == [(b"x-response", b"everything but end_stream")] + + +@pytest.mark.parametrize("connection_strategy", ["lazy", "eager"]) +@pytest.mark.parametrize("scheme", ["http3", "quic"]) +async def test_reverse_http3_and_quic_stream( + caplog_async, scheme: str, connection_strategy: str +) -> None: caplog_async.set_level("INFO") ps = Proxyserver() nl = NextLayer() @@ -685,88 +769,8 @@ def assert_no_data(response: H3Response): assert ps.servers addr = ps.servers[mode].listen_addrs[0] async with quic_connect(H3Client, alpn=["h3"], address=addr) as client: - headers = [ - (b":scheme", b"https"), - (b":authority", b"example.mitmproxy.org"), - (b":method", b"GET"), - (b":path", b"/test"), - ] - r1 = await client.request( - headers=headers + [(b"x-request", b"justheaders")], - data=None, - trailers=None, - ).wait_result() - assert r1.headers == [ - (b":status", b"200"), - (b"x-response", b"justheaders"), - ] - assert_no_data(r1) - assert r1.trailers is None - - r2 = await client.request( - headers=headers + [(b"x-request", b"hasdata")], - data=b"echo", - trailers=None, - ).wait_result() - assert r2.headers == [ - (b":status", b"200"), - (b"x-response", b"hasdata"), - ] - assert r2.data == b"echo" - assert r2.trailers is None - - r3 = await client.request( - headers=headers + [(b"x-request", b"nodata")], - data=None, - trailers=[(b"x-request", b"buttrailers")], - ).wait_result() - assert r3.headers == [ - (b":status", b"200"), - (b"x-response", b"nodata"), - ] - assert_no_data(r3) - assert r3.trailers == [(b"x-response", b"buttrailers")] - - r4 = await client.request( - headers=headers + [(b"x-request", b"this")], - data=b"has", - trailers=[(b"x-request", b"everything")], - ).wait_result() - assert r4.headers == [ - (b":status", b"200"), - (b"x-response", b"this"), - ] - assert r4.data == b"has" - assert r4.trailers == [(b"x-response", b"everything")] - - # the following test makes sure that we behave properly if end_stream is sent separately - r5 = client.request( - headers=headers + [(b"x-request", b"this")], - data=b"has", - trailers=[(b"x-request", b"everything but end_stream")], - end_stream=False, - ) - if scheme == "quic": - trailer_waiter = asyncio.get_running_loop().create_future() - r5.callback = lambda name: name != "trailers" or trailer_waiter.set_result(None) - await asyncio.wait_for(trailer_waiter, timeout=QuicClient.TIMEOUT) - assert r5.trailers is not None - assert not r5.waiter.done() - else: - await asyncio.sleep(0) - client._quic.send_stream_data( - stream_id=r5.stream_id, - data=b"", - end_stream=True, - ) - client.transmit() - await r5.wait_result() - assert r5.headers == [ - (b":status", b"200"), - (b"x-response", b"this"), - ] - assert r5.data == b"has" - assert r5.trailers == [(b"x-response", b"everything but end_stream")] + await _test_echo(client, strict=scheme == "http3") + assert len(ps.connections) == 1 tctx.configure(ps, server=False) await caplog_async.await_log(f"Stopped reverse proxy to {scheme}") @@ -802,3 +806,43 @@ async def test_reverse_quic_datagram(caplog_async, connection_strategy: str) -> tctx.configure(ps, server=False) await caplog_async.await_log("Stopped reverse proxy to quic") + + +async def test_regular_http3(caplog_async, monkeypatch) -> None: + caplog_async.set_level("INFO") + ps = Proxyserver() + nl = NextLayer() + ta = TlsConfig() + with taddons.context(ps, nl, ta) as tctx: + ta.configure(["confdir"]) + async with quic_server(H3EchoServer, alpn=["h3"]) as server_addr: + orig_open_connection = udp.open_connection + + def open_connection_path( + host: str, port: int, *args, **kwargs + ) -> udp.UdpClient: + if host == "example.mitmproxy.org" and port == 443: + host = server_addr[0] + port = server_addr[1] + return orig_open_connection(host, port, *args, **kwargs) + + monkeypatch.setattr(udp, "open_connection", open_connection_path) + mode = f"http3@127.0.0.1:0" + tctx.configure( + ta, + ssl_verify_upstream_trusted_ca=tlsdata.path( + "../net/data/verificationcerts/trusted-root.crt" + ), + ) + tctx.configure(ps, mode=[mode]) + assert await ps.setup_servers() + ps.running() + await caplog_async.await_log(f"HTTP3 proxy listening") + assert ps.servers + addr = ps.servers[mode].listen_addrs[0] + async with quic_connect(H3Client, alpn=["h3"], address=addr) as client: + await _test_echo(client=client, strict=True) + assert len(ps.connections) == 1 + + tctx.configure(ps, server=False) + await caplog_async.await_log("Stopped HTTP3 proxy") diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index 3c2f486147..cc7609a497 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -955,8 +955,8 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: assert tctx.server.sni == tctx.client.sni assert tctx.client.alpn == b"quux" assert tctx.server.alpn == b"quux" - _test_echo(playbook, tssl_server, tctx.server) _test_echo(playbook, tssl_client, tctx.client) + _test_echo(playbook, tssl_server, tctx.server) @pytest.mark.parametrize("server_state", ["open", "closed"]) def test_passthrough_from_clienthello(self, tctx: context.Context, server_state: Literal["open", "closed"]): From 0bbb0215c16bbeaf3b048c023ed0ee55f57b0de8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 14 Nov 2022 07:04:34 +1300 Subject: [PATCH 112/695] more mypy (#5724) Co-authored-by: requires.io Co-authored-by: Maximilian Hils --- examples/addons/filter-flows.py | 8 ++--- mitmproxy/addons/asgiapp.py | 1 + mitmproxy/addons/blocklist.py | 2 +- mitmproxy/addons/clientplayback.py | 1 + mitmproxy/addons/dumper.py | 3 +- mitmproxy/addons/errorcheck.py | 4 +-- mitmproxy/addons/eventstore.py | 2 +- mitmproxy/addons/maplocal.py | 2 +- mitmproxy/addons/mapremote.py | 2 +- mitmproxy/addons/modifybody.py | 2 +- mitmproxy/addons/modifyheaders.py | 2 +- mitmproxy/addons/proxyauth.py | 2 +- mitmproxy/addons/proxyserver.py | 2 +- mitmproxy/addons/save.py | 1 + mitmproxy/addons/script.py | 4 +-- mitmproxy/addons/stickycookie.py | 4 +-- mitmproxy/addons/view.py | 12 ++++---- mitmproxy/connection.py | 10 +++---- mitmproxy/contentviews/__init__.py | 7 ++--- mitmproxy/contentviews/grpc.py | 14 +++++---- mitmproxy/contentviews/http3.py | 2 +- mitmproxy/contentviews/mqtt.py | 2 ++ mitmproxy/contentviews/protobuf.py | 2 +- mitmproxy/coretypes/multidict.py | 4 +-- mitmproxy/flow.py | 4 +-- mitmproxy/flowfilter.py | 30 +++++++++---------- mitmproxy/hooks.py | 4 +-- mitmproxy/http.py | 4 +-- mitmproxy/io/compat.py | 6 ++-- mitmproxy/master.py | 5 ++-- mitmproxy/net/http/url.py | 3 +- mitmproxy/optmanager.py | 4 +-- mitmproxy/platform/windows.py | 22 +++++++------- mitmproxy/proxy/commands.py | 2 +- mitmproxy/proxy/events.py | 2 +- mitmproxy/proxy/layer.py | 8 +++-- mitmproxy/proxy/layers/http/__init__.py | 10 ++++--- mitmproxy/proxy/layers/modes.py | 11 +++---- mitmproxy/proxy/layers/tls.py | 2 +- mitmproxy/proxy/mode_servers.py | 2 +- mitmproxy/stateobject.py | 2 +- mitmproxy/tools/console/commands.py | 2 ++ mitmproxy/tools/console/common.py | 8 ++--- mitmproxy/tools/console/grideditor/base.py | 16 +++++----- .../tools/console/grideditor/col_text.py | 4 +-- mitmproxy/tools/console/keybindings.py | 1 + mitmproxy/tools/console/keymap.py | 15 ++++++---- mitmproxy/tools/console/layoutwidget.py | 5 +++- mitmproxy/tools/console/master.py | 11 +++---- mitmproxy/tools/console/options.py | 2 ++ mitmproxy/tools/console/palettes.py | 10 ++++--- mitmproxy/tools/console/statusbar.py | 10 +++---- mitmproxy/tools/main.py | 3 +- mitmproxy/tools/web/app.py | 19 +++++++----- mitmproxy/tools/web/master.py | 5 ++-- mitmproxy/utils/arg_check.py | 7 +++-- mitmproxy/utils/data.py | 4 ++- mitmproxy/utils/debug.py | 5 +++- mitmproxy/utils/signals.py | 4 +-- release/build.py | 2 +- release/selftest.py | 2 +- setup.cfg | 1 + test/mitmproxy/tools/console/test_keymap.py | 8 ++--- tox.ini | 8 ++--- web/src/js/ducks/_options_gen.ts | 2 -- 65 files changed, 206 insertions(+), 164 deletions(-) diff --git a/examples/addons/filter-flows.py b/examples/addons/filter-flows.py index 94ace9d99e..b69406c503 100644 --- a/examples/addons/filter-flows.py +++ b/examples/addons/filter-flows.py @@ -1,19 +1,19 @@ """ Use mitmproxy's filter pattern in scripts. """ +from __future__ import annotations import logging -from mitmproxy import ctx, http +from mitmproxy import http from mitmproxy import flowfilter class Filter: - def __init__(self): - self.filter: flowfilter.TFilter = None + filter: flowfilter.TFilter def configure(self, updated): if "flowfilter" in updated: - self.filter = flowfilter.parse(ctx.options.flowfilter) + self.filter = flowfilter.parse(".") def load(self, l): l.add_option("flowfilter", str, "", "Check that flow matches filter.") diff --git a/mitmproxy/addons/asgiapp.py b/mitmproxy/addons/asgiapp.py index 3a077a44e8..f85a88ebcb 100644 --- a/mitmproxy/addons/asgiapp.py +++ b/mitmproxy/addons/asgiapp.py @@ -126,6 +126,7 @@ async def send(event): ) flow.response.decode() elif event["type"] == "http.response.body": + assert flow.response flow.response.content += event.get("body", b"") if not event.get("more_body", False): nonlocal sent_response diff --git a/mitmproxy/addons/blocklist.py b/mitmproxy/addons/blocklist.py index 4945fa4997..4d5b7bedff 100644 --- a/mitmproxy/addons/blocklist.py +++ b/mitmproxy/addons/blocklist.py @@ -36,7 +36,7 @@ def parse_spec(option: str) -> BlockSpec: class BlockList: - def __init__(self): + def __init__(self) -> None: self.items: list[BlockSpec] = [] def load(self, loader): diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index ac3ee8de1d..6981cb0361 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -152,6 +152,7 @@ async def playback(self): while True: self.inflight = await self.queue.get() try: + assert self.inflight h = ReplayHandler(self.inflight, self.options) if ctx.options.client_replay_concurrency == -1: asyncio_utils.create_task( diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 73e45da5da..9979c91a4a 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -1,3 +1,4 @@ +from __future__ import annotations import logging import itertools @@ -29,7 +30,7 @@ def indent(n: int, text: str) -> str: return "\n".join(pad + i for i in l) -CONTENTVIEW_STYLES = { +CONTENTVIEW_STYLES: dict[str, dict[str, str | bool]] = { "highlight": dict(bold=True), "offset": dict(fg="blue"), "header": dict(fg="green", bold=True), diff --git a/mitmproxy/addons/errorcheck.py b/mitmproxy/addons/errorcheck.py index f82748ad76..1015c9efb5 100644 --- a/mitmproxy/addons/errorcheck.py +++ b/mitmproxy/addons/errorcheck.py @@ -9,7 +9,7 @@ class ErrorCheck: """Monitor startup for error log entries, and terminate immediately if there are some.""" - def __init__(self, log_to_stderr: bool = False): + def __init__(self, log_to_stderr: bool = False) -> None: self.log_to_stderr = log_to_stderr self.logger = ErrorCheckHandler() @@ -31,7 +31,7 @@ async def shutdown_if_errored(self): class ErrorCheckHandler(log.MitmLogHandler): - def __init__(self): + def __init__(self) -> None: super().__init__(logging.ERROR) self.has_errored: list[logging.LogRecord] = [] diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py index d973842c4d..6f597b9c3d 100644 --- a/mitmproxy/addons/eventstore.py +++ b/mitmproxy/addons/eventstore.py @@ -10,7 +10,7 @@ class EventStore: - def __init__(self, size=10000): + def __init__(self, size: int = 10000) -> None: self.data: collections.deque[LogEntry] = collections.deque(maxlen=size) self.sig_add = signals.SyncSignal(lambda entry: None) self.sig_refresh = signals.SyncSignal(lambda: None) diff --git a/mitmproxy/addons/maplocal.py b/mitmproxy/addons/maplocal.py index 2701f14737..54c522d93e 100644 --- a/mitmproxy/addons/maplocal.py +++ b/mitmproxy/addons/maplocal.py @@ -76,7 +76,7 @@ def file_candidates(url: str, spec: MapLocalSpec) -> list[Path]: class MapLocal: - def __init__(self): + def __init__(self) -> None: self.replacements: list[MapLocalSpec] = [] def load(self, loader): diff --git a/mitmproxy/addons/mapremote.py b/mitmproxy/addons/mapremote.py index 2fe6c2d2ef..245323a034 100644 --- a/mitmproxy/addons/mapremote.py +++ b/mitmproxy/addons/mapremote.py @@ -24,7 +24,7 @@ def parse_map_remote_spec(option: str) -> MapRemoteSpec: class MapRemote: - def __init__(self): + def __init__(self) -> None: self.replacements: list[MapRemoteSpec] = [] def load(self, loader): diff --git a/mitmproxy/addons/modifybody.py b/mitmproxy/addons/modifybody.py index 7d3287aecc..b82059afe1 100644 --- a/mitmproxy/addons/modifybody.py +++ b/mitmproxy/addons/modifybody.py @@ -7,7 +7,7 @@ class ModifyBody: - def __init__(self): + def __init__(self) -> None: self.replacements: list[ModifySpec] = [] def load(self, loader): diff --git a/mitmproxy/addons/modifyheaders.py b/mitmproxy/addons/modifyheaders.py index 370b9abb92..995005f305 100644 --- a/mitmproxy/addons/modifyheaders.py +++ b/mitmproxy/addons/modifyheaders.py @@ -51,7 +51,7 @@ def parse_modify_spec(option: str, subject_is_regex: bool) -> ModifySpec: class ModifyHeaders: - def __init__(self): + def __init__(self) -> None: self.replacements: list[ModifySpec] = [] def load(self, loader): diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 96013d8d49..653fa48ae0 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -22,7 +22,7 @@ class ProxyAuth: validator: Validator | None = None - def __init__(self): + def __init__(self) -> None: self.authenticated: MutableMapping[ connection.Client, tuple[str, str] ] = weakref.WeakKeyDictionary() diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 9d592ccd79..faf935e3eb 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -195,7 +195,7 @@ def load(self, loader): def running(self): self.is_running = True - def configure(self, updated): + def configure(self, updated) -> None: if "stream_large_bodies" in updated: try: human.parse_size(ctx.options.stream_large_bodies) diff --git a/mitmproxy/addons/save.py b/mitmproxy/addons/save.py index 8faa188ee6..2a3db39b71 100644 --- a/mitmproxy/addons/save.py +++ b/mitmproxy/addons/save.py @@ -77,6 +77,7 @@ def configure(self, updated): self.maybe_rotate_to_new_file() except OSError as e: raise exceptions.OptionsError(str(e)) from e + assert self.stream self.stream.flt = self.filt else: self.done() diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index d39fda4eb4..1c37e4a439 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -79,7 +79,7 @@ def __init__(self, path: str, reload: bool) -> None: self.name = "scriptmanager:" + path self.path = path self.fullpath = os.path.expanduser(path.strip("'\" ")) - self.ns = None + self.ns: types.ModuleType | None = None self.is_running = False if not os.path.isfile(self.fullpath): @@ -126,7 +126,7 @@ def loadscript(self): ctx.master.addons.invoke_addon_sync(self.ns, hooks.RunningHook()) async def watcher(self): - last_mtime = 0 + last_mtime = 0.0 while True: try: mtime = os.stat(self.fullpath).st_mtime diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index df0abfbd98..ef33f5bc41 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -30,8 +30,8 @@ def domain_match(a: str, b: str) -> bool: class StickyCookie: - def __init__(self): - self.jar: dict[TOrigin, dict[str, str]] = collections.defaultdict(dict) + def __init__(self) -> None: + self.jar: collections.defaultdict[TOrigin, dict[str, str]] = collections.defaultdict(dict) self.flt: Optional[flowfilter.TFilter] = None def load(self, loader): diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 7b683d4a24..9a3e2116aa 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -142,9 +142,9 @@ def _sig_view_remove(flow: mitmproxy.flow.Flow, index: int) -> None: class View(collections.abc.Sequence): - def __init__(self): + def __init__(self) -> None: super().__init__() - self._store = collections.OrderedDict() + self._store: collections.OrderedDict[str, mitmproxy.flow.Flow] = collections.OrderedDict() self.filter = flowfilter.match_all # Should we show only marked flows? self.show_marked = False @@ -156,7 +156,7 @@ def __init__(self): url=OrderRequestURL(self), size=OrderKeySize(self), ) - self.order_key = self.default_order + self.order_key: _OrderKey = self.default_order self.order_reversed = False self.focus_follow = False @@ -316,9 +316,9 @@ def set_order(self, order_key: str) -> None: """ if order_key not in self.orders: raise exceptions.CommandError("Unknown flow order: %s" % order_key) - order_key = self.orders[order_key] - self.order_key = order_key - newview = sortedcontainers.SortedListWithKey(key=order_key) + key = self.orders[order_key] + self.order_key = key + newview = sortedcontainers.SortedListWithKey(key=key) newview.update(self._view) self._view = newview diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index 042fa9437b..b08394892e 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -227,7 +227,7 @@ def from_state(cls, state) -> "Client": return client def set_state(self, state): - self.peername = tuple(state["address"]) if state["address"] else None + self.peername = tuple(state["address"]) if state["address"] else None # type: ignore self.alpn = state["alpn"] self.cipher = state["cipher_name"] self.id = state["id"] @@ -238,7 +238,7 @@ def set_state(self, state): self.tls_version = state["tls_version"] # only used in sans-io self.state = ConnectionState(state["state"]) - self.sockname = tuple(state["sockname"]) if state["sockname"] else None + self.sockname = tuple(state["sockname"]) if state["sockname"] else None # type: ignore self.error = state["error"] self.tls = state["tls"] self.certificate_list = [ @@ -394,13 +394,13 @@ def from_state(cls, state) -> "Server": return server def set_state(self, state): - self.address = tuple(state["address"]) if state["address"] else None + self.address = tuple(state["address"]) if state["address"] else None # type: ignore self.alpn = state["alpn"] self.id = state["id"] - self.peername = tuple(state["ip_address"]) if state["ip_address"] else None + self.peername = tuple(state["ip_address"]) if state["ip_address"] else None # type: ignore self.sni = state["sni"] self.sockname = ( - tuple(state["source_address"]) if state["source_address"] else None + tuple(state["source_address"]) if state["source_address"] else None # type: ignore ) self.timestamp_end = state["timestamp_end"] self.timestamp_start = state["timestamp_start"] diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 86fff00b25..522e8bb971 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -44,9 +44,8 @@ # FIXME: Remove once QUIC is merged. http3 = None # type: ignore from .base import View, KEY_MAX, format_text, format_dict, TViewResult -from ..http import HTTPFlow -from ..tcp import TCPMessage, TCPFlow -from ..udp import UDPMessage, UDPFlow +from ..tcp import TCPMessage +from ..udp import UDPMessage from ..websocket import WebSocketMessage views: list[View] = [] @@ -101,7 +100,7 @@ def safe_to_print(lines, encoding="utf8"): def get_message_content_view( viewname: str, message: Union[http.Message, TCPMessage, UDPMessage, WebSocketMessage], - flow: Union[HTTPFlow, TCPFlow, UDPFlow], + flow: flow.Flow, ): """ Like get_content_view, but also handles message encoding. diff --git a/mitmproxy/contentviews/grpc.py b/mitmproxy/contentviews/grpc.py index 9db6c6b40b..332310af27 100644 --- a/mitmproxy/contentviews/grpc.py +++ b/mitmproxy/contentviews/grpc.py @@ -504,9 +504,11 @@ def apply_rules(self, only_first_hit=True): if match: if only_first_hit: # only first match - self.name = fd.name - self.preferred_decoding = fd.intended_decoding - self.try_unpack = fd.as_packed + if fd.name is not None: + self.name = fd.name + if fd.intended_decoding is not None: + self.preferred_decoding = fd.intended_decoding + self.try_unpack = bool(fd.as_packed) return else: # overwrite matches till last rule was inspected @@ -773,8 +775,8 @@ def gen_flat_decoded_field_dicts(self) -> Generator[dict, None, None]: def __init__( self, data: bytes, - rules: list[ProtoParser.ParserRule] = None, - parser_options: ParserOptions = None, + rules: list[ProtoParser.ParserRule] | None = None, + parser_options: ParserOptions | None = None, ) -> None: self.data: bytes = data if parser_options is None: @@ -979,7 +981,7 @@ class ViewGrpcProtobuf(base.View): ] # allows to take external ParserOptions object. goes with defaults otherwise - def __init__(self, config: ViewConfig = None) -> None: + def __init__(self, config: ViewConfig | None = None) -> None: super().__init__() if config is None: config = ViewConfig() diff --git a/mitmproxy/contentviews/http3.py b/mitmproxy/contentviews/http3.py index a354a901cf..fe41dce5ea 100644 --- a/mitmproxy/contentviews/http3.py +++ b/mitmproxy/contentviews/http3.py @@ -83,7 +83,7 @@ class ConnectionState: class ViewHttp3(base.View): name = "HTTP/3 Frames" - def __init__(self): + def __init__(self) -> None: self.connections: defaultdict[tcp.TCPFlow, ConnectionState] = defaultdict(ConnectionState) def __call__( diff --git a/mitmproxy/contentviews/mqtt.py b/mitmproxy/contentviews/mqtt.py index c344fed20f..1b870341c8 100644 --- a/mitmproxy/contentviews/mqtt.py +++ b/mitmproxy/contentviews/mqtt.py @@ -89,6 +89,7 @@ def pprint(self): s = f"[{self.Names[self.packet_type]}]" if self.packet_type == self.CONNECT: + assert self.payload s += f""" Client Id: {self.payload['ClientId']} @@ -101,6 +102,7 @@ def pprint(self): s += " sent topic filters: " s += ", ".join([f"'{tf}'" for tf in self.topic_filters]) elif self.packet_type == self.PUBLISH: + assert self.payload topic_name = strutils.bytes_to_escaped_str(self.topic_name) payload = strutils.bytes_to_escaped_str(self.payload) diff --git a/mitmproxy/contentviews/protobuf.py b/mitmproxy/contentviews/protobuf.py index 7aea0c1760..50d349eb59 100644 --- a/mitmproxy/contentviews/protobuf.py +++ b/mitmproxy/contentviews/protobuf.py @@ -57,7 +57,7 @@ def format_pbuf(raw): body = pair.value try: - pairs = _parse_proto(body) + pairs = _parse_proto(body) # type: ignore stack.extend([(pair, indent_level + 2) for pair in pairs[::-1]]) write_buf(out, pair.field_tag, None, indent_level) except: diff --git a/mitmproxy/coretypes/multidict.py b/mitmproxy/coretypes/multidict.py index 15f24568a9..6710346e62 100644 --- a/mitmproxy/coretypes/multidict.py +++ b/mitmproxy/coretypes/multidict.py @@ -148,7 +148,7 @@ class MultiDict(_MultiDict[KT, VT], serializable.Serializable): def __init__(self, fields=()): super().__init__() - self.fields = tuple(tuple(i) for i in fields) + self.fields = tuple(tuple(i) for i in fields) # type: ignore @staticmethod def _reduce_values(values): @@ -162,7 +162,7 @@ def get_state(self): return self.fields def set_state(self, state): - self.fields = tuple(tuple(x) for x in state) + self.fields = tuple(tuple(x) for x in state) # type: ignore @classmethod def from_state(cls, state): diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 3f4db28db9..42e1974585 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -44,7 +44,7 @@ def __repr__(self): def from_state(cls, state): # the default implementation assumes an empty constructor. Override # accordingly. - f = cls(None) + f = cls("") f.set_state(state) return f @@ -180,7 +180,7 @@ def from_state(cls, state): flow_cls = Flow.__types[state["type"]] except KeyError: raise ValueError(f"Unknown flow type: {state['type']}") - f = flow_cls(None, None) # noqa + f = flow_cls(None, None) # type: ignore f.set_state(state) return f diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index b0aa45a899..aaec9e1f5f 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -288,19 +288,19 @@ class FBod(_Rex): @only(http.HTTPFlow, tcp.TCPFlow, udp.UDPFlow, dns.DNSFlow) def __call__(self, f): if isinstance(f, http.HTTPFlow): - if f.request and f.request.raw_content: - if self.re.search(f.request.get_content(strict=False)): + if f.request and (content := f.request.get_content(strict=False)) is not None: + if self.re.search(content): return True - if f.response and f.response.raw_content: - if self.re.search(f.response.get_content(strict=False)): + if f.response and (content := f.response.get_content(strict=False)) is not None: + if self.re.search(content): return True if f.websocket: - for msg in f.websocket.messages: - if self.re.search(msg.content): + for wmsg in f.websocket.messages: + if wmsg.content is not None and self.re.search(wmsg.content): return True elif isinstance(f, (tcp.TCPFlow, udp.UDPFlow)): for msg in f.messages: - if self.re.search(msg.content): + if msg.content is not None and self.re.search(msg.content): return True elif isinstance(f, dns.DNSFlow): if f.request and self.re.search(f.request.content): @@ -318,12 +318,12 @@ class FBodRequest(_Rex): @only(http.HTTPFlow, tcp.TCPFlow, udp.UDPFlow, dns.DNSFlow) def __call__(self, f): if isinstance(f, http.HTTPFlow): - if f.request and f.request.raw_content: - if self.re.search(f.request.get_content(strict=False)): + if f.request and (content := f.request.get_content(strict=False)) is not None: + if self.re.search(content): return True if f.websocket: - for msg in f.websocket.messages: - if msg.from_client and self.re.search(msg.content): + for wmsg in f.websocket.messages: + if wmsg.from_client and self.re.search(wmsg.content): return True elif isinstance(f, (tcp.TCPFlow, udp.UDPFlow)): for msg in f.messages: @@ -342,12 +342,12 @@ class FBodResponse(_Rex): @only(http.HTTPFlow, tcp.TCPFlow, udp.UDPFlow, dns.DNSFlow) def __call__(self, f): if isinstance(f, http.HTTPFlow): - if f.response and f.response.raw_content: - if self.re.search(f.response.get_content(strict=False)): + if f.response and (content := f.response.get_content(strict=False)) is not None: + if self.re.search(content): return True if f.websocket: - for msg in f.websocket.messages: - if not msg.from_client and self.re.search(msg.content): + for wmsg in f.websocket.messages: + if not wmsg.from_client and self.re.search(wmsg.content): return True elif isinstance(f, (tcp.TCPFlow, udp.UDPFlow)): for msg in f.messages: diff --git a/mitmproxy/hooks.py b/mitmproxy/hooks.py index d0c2934fa1..2c6c8574e8 100644 --- a/mitmproxy/hooks.py +++ b/mitmproxy/hooks.py @@ -43,8 +43,8 @@ def __init_subclass__(cls, **kwargs): all_hooks[cls.name] = cls # define a custom hash and __eq__ function so that events are hashable and not comparable. - cls.__hash__ = object.__hash__ - cls.__eq__ = object.__eq__ + cls.__hash__ = object.__hash__ # type: ignore + cls.__eq__ = object.__eq__ # type: ignore all_hooks: dict[str, type[Hook]] = {} diff --git a/mitmproxy/http.py b/mitmproxy/http.py index e88242530c..14e024462f 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -156,7 +156,7 @@ def get_all(self, name: Union[str, bytes]) -> list[str]: name = _always_bytes(name) return [_native(x) for x in super().get_all(name)] - def set_all(self, name: Union[str, bytes], values: list[Union[str, bytes]]): + def set_all(self, name: Union[str, bytes], values: Iterable[Union[str, bytes]]): """ Explicitly set multiple headers for the given key. See `Headers.get_all`. @@ -981,7 +981,7 @@ def _get_multipart_form(self): is_valid_content_type = ( "multipart/form-data" in self.headers.get("content-type", "").lower() ) - if is_valid_content_type: + if is_valid_content_type and self.content is not None: try: return multipart.decode(self.headers.get("content-type"), self.content) except ValueError: diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 19229a4f14..5ae6b08c03 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -6,7 +6,7 @@ version number, this prevents issues with developer builds and snapshots. """ import uuid -from typing import Any, Mapping, Union +from typing import Any, Union from mitmproxy import version from mitmproxy.utils import strutils @@ -139,8 +139,8 @@ def convert_300_4(data): return data -client_connections: Mapping[str, str] = {} -server_connections: Mapping[str, str] = {} +client_connections: dict[tuple[str, ...], str] = {} +server_connections: dict[tuple[str, ...], str] = {} def convert_4_5(data): diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 3e5f248b62..0947e6eb01 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -22,7 +22,7 @@ class Master: event_loop: asyncio.AbstractEventLoop - def __init__(self, opts, event_loop: Optional[asyncio.AbstractEventLoop] = None): + def __init__(self, opts: options.Options, event_loop: Optional[asyncio.AbstractEventLoop] = None): self.options: options.Options = opts or options.Options() self.commands = command.CommandManager(self) self.addons = addonmanager.AddonManager(self) @@ -79,7 +79,7 @@ async def done(self) -> None: await self.addons.trigger_event(hooks.DoneHook()) self._legacy_log_events.uninstall() - def _asyncio_exception_handler(self, loop, context): + def _asyncio_exception_handler(self, loop, context) -> None: try: exc: Exception = context["exception"] except KeyError: @@ -108,6 +108,7 @@ async def load_flow(self, f): # easy to replay saved flows against a different host. # We may change this in the future so that clientplayback always replays to the first mode. mode = ReverseMode.parse(self.options.mode[0]) + assert isinstance(mode, ReverseMode) f.request.host, f.request.port, *_ = mode.address f.request.scheme = mode.scheme diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index 9468302e12..274f229fb1 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -1,3 +1,4 @@ +from __future__ import annotations import re import urllib.parse from collections.abc import Sequence @@ -85,7 +86,7 @@ def unparse(scheme: str, host: str, port: int, path: str = "") -> str: return f"{scheme}://{authority}{path}" -def encode(s: Sequence[tuple[str, str]], similar_to: str = None) -> str: +def encode(s: Sequence[tuple[str, str]], similar_to: str | None = None) -> str: """ Takes a list of (key, value) tuples and returns a urlencoded string. If similar_to is passed, the output is formatted similar to the provided urlencoded string. diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 033819be7c..ae3bb54ac6 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -103,7 +103,7 @@ class OptManager: mutation doesn't change the option state inadvertently. """ - def __init__(self): + def __init__(self) -> None: self.deferred: dict[str, Any] = {} self.changed = signals.SyncSignal(_sig_changed_spec) self.changed.connect(self._notify_subscribers) @@ -526,7 +526,7 @@ def parse(text): snip = v.problem_mark.get_snippet() raise exceptions.OptionsError( "Config error at line %s:\n%s\n%s" - % (v.problem_mark.line + 1, snip, v.problem) + % (v.problem_mark.line + 1, snip, getattr(v, 'problem', '')) ) else: raise exceptions.OptionsError("Could not parse options.") diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index 0e0515bd51..1ff8876618 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -1,9 +1,9 @@ +from __future__ import annotations import collections import collections.abc import contextlib import ctypes import ctypes.wintypes -import io import json import os import re @@ -12,7 +12,7 @@ import threading import time from collections.abc import Callable -from typing import Any, ClassVar, Optional +from typing import Any, ClassVar, IO, Optional, cast import pydivert import pydivert.consts @@ -27,20 +27,20 @@ # Resolver -def read(rfile: io.BufferedReader) -> Any: +def read(rfile: IO[bytes]) -> Any: x = rfile.readline().strip() if not x: return None return json.loads(x) -def write(data, wfile: io.BufferedWriter) -> None: +def write(data, wfile: IO[bytes]) -> None: wfile.write(json.dumps(data).encode() + b"\n") wfile.flush() class Resolver: - sock: socket.socket + sock: socket.socket | None lock: threading.RLock def __init__(self): @@ -84,7 +84,9 @@ class APIRequestHandler(socketserver.StreamRequestHandler): for each received pickled client address, port tuple. """ - def handle(self): + server: APIServer + + def handle(self) -> None: proxifier: TransparentProxy = self.server.proxifier try: pid: int = read(self.rfile) @@ -96,7 +98,7 @@ def handle(self): if c is None: return try: - server = proxifier.client_server_map[tuple(c)] + server = proxifier.client_server_map[cast(tuple[str, int], tuple(c))] except KeyError: server = None write(server, self.wfile) @@ -205,7 +207,7 @@ def refresh(self): self._refresh_ipv6() def _refresh_ipv4(self): - ret = ctypes.windll.iphlpapi.GetExtendedTcpTable( + ret = ctypes.windll.iphlpapi.GetExtendedTcpTable( # type: ignore ctypes.byref(self._tcp), ctypes.byref(self._tcp_size), False, @@ -228,7 +230,7 @@ def _refresh_ipv4(self): ) def _refresh_ipv6(self): - ret = ctypes.windll.iphlpapi.GetExtendedTcpTable( + ret = ctypes.windll.iphlpapi.GetExtendedTcpTable( # type: ignore ctypes.byref(self._tcp6), ctypes.byref(self._tcp6_size), False, @@ -275,7 +277,7 @@ def run(self): try: packet = self.windivert.recv() except OSError as e: - if e.winerror == 995: + if getattr(e, "winerror", None) == 995: return else: raise diff --git a/mitmproxy/proxy/commands.py b/mitmproxy/proxy/commands.py index d4173ebc9b..326290fc12 100644 --- a/mitmproxy/proxy/commands.py +++ b/mitmproxy/proxy/commands.py @@ -76,7 +76,7 @@ def __init__(self, connection: Connection, data: bytes): def __repr__(self): target = str(self.connection).split("(", 1)[0].lower() - return f"SendData({target}, {self.data})" + return f"SendData({target}, {self.data!r})" class OpenConnection(ConnectionCommand): diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index b571f3ad2f..813e4e174d 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -47,7 +47,7 @@ class DataReceived(ConnectionEvent): def __repr__(self): target = type(self.connection).__name__.lower() - return f"DataReceived({target}, {self.data})" + return f"DataReceived({target}, {self.data!r})" class ConnectionClosed(ConnectionEvent): diff --git a/mitmproxy/proxy/layer.py b/mitmproxy/proxy/layer.py index 2b2868fb51..d486e9b8f0 100644 --- a/mitmproxy/proxy/layer.py +++ b/mitmproxy/proxy/layer.py @@ -4,6 +4,7 @@ import collections import textwrap from abc import abstractmethod +from collections.abc import Callable from dataclasses import dataclass from logging import DEBUG from typing import Any, ClassVar, Generator, NamedTuple, Optional, TypeVar @@ -98,6 +99,7 @@ def __debug(self, message): message = message[:256] + "…" else: Layer.__last_debug_message = message + assert self.debug is not None return commands.Log(textwrap.indent(message, self.debug), DEBUG) @property @@ -247,7 +249,7 @@ def __init__(self, context: Context, ask_on_start: bool = False) -> None: self.layer = None self.events = [] self._ask_on_start = ask_on_start - self._handle = None + self._handle: Callable[[mevents.Event], CommandGenerator[None]] | None = None def __repr__(self): return f"NextLayer:{repr(self.layer)}" @@ -296,8 +298,8 @@ def _ask(self): # 2. This layer is not needed anymore, so we directly reassign .handle_event. # 3. Some layers may however still have a reference to the old .handle_event. # ._handle is just an optimization to reduce the callstack in these cases. - self.handle_event = self.layer.handle_event - self._handle_event = self.layer.handle_event + self.handle_event = self.layer.handle_event # type: ignore + self._handle_event = self.layer.handle_event # type: ignore self._handle = self.layer.handle_event # Utility methods for whoever decides what the next layer is going to be. diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 0cce0d0327..e1d56f4350 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -129,12 +129,13 @@ class HttpStream(layer.Layer): child_layer: Optional[layer.Layer] = None @cached_property - def mode(self): + def mode(self) -> HTTPMode: i = self.context.layers.index(self) - parent: HttpLayer = self.context.layers[i - 1] + parent = self.context.layers[i - 1] + assert isinstance(parent, HttpLayer) return parent.mode - def __init__(self, context: Context, stream_id: int): + def __init__(self, context: Context, stream_id: int) -> None: super().__init__(context) self.request_body_buf = b"" self.response_body_buf = b"" @@ -482,10 +483,11 @@ def send_response(self, already_streamed: bool = False): if self.client_state == self.state_done: yield from self.flow_done() - def flow_done(self): + def flow_done(self) -> layer.CommandGenerator[None]: if not self.flow.websocket: self.flow.live = False + assert self.flow.response if self.flow.response.status_code == 101: if self.flow.websocket: self.child_layer = websocket.WebsocketLayer(self.context, self.flow) diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index d320b513ff..bfbe9fbe19 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -1,8 +1,9 @@ +from __future__ import annotations import socket import struct from abc import ABCMeta from dataclasses import dataclass -from typing import Optional +from typing import Callable, Optional from mitmproxy import connection from mitmproxy.proxy import commands, events, layer @@ -154,7 +155,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unknown event: {event}") - def state_greet(self): + def state_greet(self) -> layer.CommandGenerator[None]: if len(self.buf) < 2: return @@ -194,9 +195,9 @@ def state_greet(self): self.buf = self.buf[2 + n_methods :] yield from self.state() - state = state_greet + state: Callable[..., layer.CommandGenerator[None]] = state_greet - def state_auth(self): + def state_auth(self) -> layer.CommandGenerator[None]: if len(self.buf) < 3: return @@ -225,7 +226,7 @@ def state_auth(self): self.state = self.state_connect yield from self.state() - def state_connect(self): + def state_connect(self) -> layer.CommandGenerator[None]: # Parse Connect Request if len(self.buf) < 5: return diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 05307d6e12..20d39bbb9e 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -257,7 +257,7 @@ def __init__(self, context: context.Context, conn: connection.Connection): conn.tls = True def __repr__(self): - return super().__repr__().replace(")", f" {self.conn.sni} {self.conn.alpn})") + return super().__repr__().replace(")", f" {self.conn.sni!r} {self.conn.alpn!r})") @property def is_dtls(self): diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index b3ee872711..34d01a4ae5 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -81,7 +81,7 @@ def __init__(self, mode: M, manager: ServerManager): def __init_subclass__(cls, **kwargs): """Register all subclasses so that make() finds them.""" # extract mode from Generic[Mode]. - mode = get_args(cls.__orig_bases__[0])[0] + mode = get_args(cls.__orig_bases__[0])[0] # type: ignore if not isinstance(mode, TypeVar): assert issubclass(mode, mode_specs.ProxyMode) assert mode.type_name not in ServerInstance.__modes diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 4f87a52279..73117eddf2 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -41,7 +41,7 @@ def set_state(self, state): setattr(self, attr, val) else: curr = getattr(self, attr, None) - if hasattr(curr, "set_state"): + if curr is not None and hasattr(curr, "set_state"): curr.set_state(val) else: setattr(self, attr, make_object(cls, val)) diff --git a/mitmproxy/tools/console/commands.py b/mitmproxy/tools/console/commands.py index 6b890c2d32..78564ef176 100644 --- a/mitmproxy/tools/console/commands.py +++ b/mitmproxy/tools/console/commands.py @@ -125,6 +125,8 @@ class Commands(urwid.Pile, layoutwidget.LayoutWidget): title = "Command Reference" keyctx = "commands" + focus_position: int + def __init__(self, master): oh = CommandHelp(master) super().__init__( diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 61c2ef0801..94ff65836a 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -242,11 +242,11 @@ def rle_append_beginning_modify(rle, a_r): rle[0:0] = [(a, r)] -def colorize_host(host): +def colorize_host(host: str): tld = get_tld(host) sld = get_sld(host) - attr = [] + attr: list = [] tld_size = len(tld) sld_size = len(sld) - tld_size @@ -268,14 +268,14 @@ def colorize_host(host): return attr -def colorize_req(s): +def colorize_req(s: str): path = s.split("?", 2)[0] i_query = len(path) i_last_slash = path.rfind("/") i_ext = path[i_last_slash + 1 :].rfind(".") i_ext = i_last_slash + i_ext if i_ext >= 0 else len(s) in_val = False - attr = [] + attr: list = [] for i in range(len(s)): c = s[i] if ( diff --git a/mitmproxy/tools/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py index 8c20a23fb0..5759c0cf38 100644 --- a/mitmproxy/tools/console/grideditor/base.py +++ b/mitmproxy/tools/console/grideditor/base.py @@ -1,8 +1,8 @@ import abc import copy import os -from collections.abc import Callable, Container, Iterable, Sequence -from typing import Any, AnyStr, Optional +from collections.abc import Callable, Container, Iterable, MutableSequence, Sequence +from typing import Any, AnyStr, ClassVar, Optional import urwid @@ -117,7 +117,7 @@ class GridWalker(urwid.ListWalker): """ def __init__(self, lst: Iterable[list], editor: "GridEditor") -> None: - self.lst: Sequence[tuple[Any, set]] = [(i, set()) for i in lst] + self.lst: MutableSequence[tuple[Any, set]] = [(i, set()) for i in lst] self.editor = editor self.focus = 0 self.focus_col = 0 @@ -150,7 +150,7 @@ def set_value(self, val, focus, focus_col, errors=None): errors = set() row = list(self.lst[focus][0]) row[focus_col] = val - self.lst[focus] = [tuple(row), errors] + self.lst[focus] = [tuple(row), errors] # type: ignore self._modified() def delete_focus(self): @@ -180,7 +180,7 @@ def start_edit(self): self._modified() def stop_edit(self): - if self.edit_row: + if self.edit_row and self.edit_row.edit_col: try: val = self.edit_row.edit_col.get_data() except ValueError: @@ -242,7 +242,7 @@ def __init__(self, lw): class BaseGridEditor(urwid.WidgetWrap): title: str = "" - keyctx = "grideditor" + keyctx: ClassVar[str] = "grideditor" def __init__( self, @@ -388,7 +388,7 @@ def cmd_spawn_editor(self): class GridEditor(BaseGridEditor): title = "" columns: Sequence[Column] = () - keyctx = "grideditor" + keyctx: ClassVar[str] = "grideditor" def __init__( self, @@ -408,7 +408,7 @@ class FocusEditor(urwid.WidgetWrap, layoutwidget.LayoutWidget): A specialised GridEditor that edits the current focused flow. """ - keyctx = "grideditor" + keyctx: ClassVar[str] = "grideditor" def __init__(self, master): self.master = master diff --git a/mitmproxy/tools/console/grideditor/col_text.py b/mitmproxy/tools/console/grideditor/col_text.py index ec49ce2a05..d5ad1cba03 100644 --- a/mitmproxy/tools/console/grideditor/col_text.py +++ b/mitmproxy/tools/console/grideditor/col_text.py @@ -28,10 +28,10 @@ def blank(self): class EncodingMixin: def __init__(self, data, encoding_args): self.encoding_args = encoding_args - super().__init__(data.__str__().encode(*self.encoding_args)) + super().__init__(str(data).encode(*self.encoding_args)) # type: ignore def get_data(self): - data = super().get_data() + data = super().get_data() # type: ignore try: return data.decode(*self.encoding_args) except ValueError: diff --git a/mitmproxy/tools/console/keybindings.py b/mitmproxy/tools/console/keybindings.py index eb2db10aa8..903f71e439 100644 --- a/mitmproxy/tools/console/keybindings.py +++ b/mitmproxy/tools/console/keybindings.py @@ -130,6 +130,7 @@ def sig_mod(self, txt): class KeyBindings(urwid.Pile, layoutwidget.LayoutWidget): title = "Key Bindings" keyctx = "keybindings" + focus_position: int def __init__(self, master): oh = KeyHelp(master) diff --git a/mitmproxy/tools/console/keymap.py b/mitmproxy/tools/console/keymap.py index c9539dd750..d4fbcaf819 100644 --- a/mitmproxy/tools/console/keymap.py +++ b/mitmproxy/tools/console/keymap.py @@ -71,7 +71,7 @@ def sortkey(self): class Keymap: def __init__(self, master): self.executor = commandexecutor.CommandExecutor(master) - self.keys = {} + self.keys: dict[str, dict[str, Binding]] = {} for c in Contexts: self.keys[c] = {} self.bindings = [] @@ -161,7 +161,8 @@ def handle(self, context: str, key: str) -> Optional[str]: """ b = self.get(context, key) or self.get("global", key) if b: - return self.executor(b.command) + self.executor(b.command) + return None return key def handle_only(self, context: str, key: str) -> Optional[str]: @@ -171,7 +172,8 @@ def handle_only(self, context: str, key: str) -> Optional[str]: """ b = self.get(context, key) if b: - return self.executor(b.command) + self.executor(b.command) + return None return key @@ -187,10 +189,13 @@ def handle_only(self, context: str, key: str) -> Optional[str]: class KeymapConfig: defaultFile = "keys.yaml" + def __init__(self, master): + self.master = master + @command.command("console.keymap.load") def keymap_load_path(self, path: mitmproxy.types.Path) -> None: try: - self.load_path(ctx.master.keymap, path) # type: ignore + self.load_path(self.master.keymap, path) # type: ignore except (OSError, KeyBindingError) as e: raise exceptions.CommandError("Could not load key bindings - %s" % e) from e @@ -198,7 +203,7 @@ def running(self): p = os.path.join(os.path.expanduser(ctx.options.confdir), self.defaultFile) if os.path.exists(p): try: - self.load_path(ctx.master.keymap, p) + self.load_path(self.master.keymap, p) except KeyBindingError as e: logging.error(e) diff --git a/mitmproxy/tools/console/layoutwidget.py b/mitmproxy/tools/console/layoutwidget.py index dd6e910021..5443c4f0db 100644 --- a/mitmproxy/tools/console/layoutwidget.py +++ b/mitmproxy/tools/console/layoutwidget.py @@ -1,3 +1,6 @@ +from typing import ClassVar + + class LayoutWidget: """ All top-level layout widgets and all widgets that may be set in an @@ -6,7 +9,7 @@ class LayoutWidget: # Title is only required for windows, not overlay components title = "" - keyctx = "" + keyctx: ClassVar[str] = "" def key_responder(self): """ diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index a5f4d0c474..415e4a6751 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -18,6 +18,7 @@ from mitmproxy import addons from mitmproxy import master +from mitmproxy import options from mitmproxy import log from mitmproxy.addons import errorcheck, intercept from mitmproxy.addons import eventstore @@ -36,7 +37,7 @@ class ConsoleMaster(master.Master): - def __init__(self, opts): + def __init__(self, opts: options.Options) -> None: super().__init__(opts) self.view: view.View = view.View() @@ -48,8 +49,6 @@ def __init__(self, opts): defaultkeys.map(self.keymap) self.options.errored.connect(self.options_error) - self.view_stack = [] - self.addons.add(*addons.default_addons()) self.addons.add( intercept.Intercept(), @@ -57,11 +56,11 @@ def __init__(self, opts): self.events, readfile.ReadFile(), consoleaddons.ConsoleAddon(self), - keymap.KeymapConfig(), + keymap.KeymapConfig(self), errorcheck.ErrorCheck(log_to_stderr=True), ) - self.window = None + self.window: window.Window | None = None def __setattr__(self, name, value): super().__setattr__(name, value) @@ -241,9 +240,11 @@ async def done(self): await super().done() def overlay(self, widget, **kwargs): + assert self.window self.window.set_overlay(widget, **kwargs) def switch_view(self, name): + assert self.window self.window.push(name) def quit(self, a): diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index fbfe3e7465..01b055f9e1 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -244,6 +244,8 @@ class Options(urwid.Pile, layoutwidget.LayoutWidget): title = "Options" keyctx = "options" + focus_position: int + def __init__(self, master): oh = OptionHelp(master) self.optionslist = OptionsList(master, oh) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 4e86258e21..f51855c35d 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -3,6 +3,7 @@ # # http://urwid.org/manual/displayattributes.html # +from __future__ import annotations from collections.abc import Mapping, Sequence from typing import Optional @@ -89,9 +90,10 @@ class Palette: ] _fields.extend(["gradient_%02d" % i for i in range(100)]) high: Optional[Mapping[str, Sequence[str]]] = None + low: Mapping[str, Sequence[str]] - def palette(self, transparent): - l = [] + def palette(self, transparent: bool): + l: list[Sequence[str | None]] = [] highback, lowback = None, None if not transparent: if self.high and self.high.get("background"): @@ -102,14 +104,14 @@ def palette(self, transparent): if transparent and i == "background": l.append(["background", "default", "default"]) else: - v = [i] + v: list[str | None] = [i] low = list(self.low[i]) if lowback and low[1] == "default": low[1] = lowback v.extend(low) if self.high and i in self.high: v.append(None) - high = list(self.high[i]) + high: list[str | None] = list(self.high[i]) if highback and high[1] == "default": high[1] = highback v.extend(high) diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index a8ca015909..1ea3fe7fcb 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -150,11 +150,11 @@ def keypress(self, size, k): return k def show_quickhelp(self) -> None: - try: - s = self.master.window.focus_stack() + if w := self.master.window: + s = w.focus_stack() focused_widget = type(s.top_widget()) is_top_widget = len(s.stack) == 1 - except AttributeError: # on startup + else: # on startup focused_widget = flowlist.FlowListBox is_top_widget = True focused_flow = self.master.view.focus.flow @@ -196,7 +196,7 @@ def refresh(self) -> None: self.redraw() signals.call_in.send(seconds=self.REFRESHTIME, callback=self.refresh) - def sig_update(self, flow=None, updated=None): + def sig_update(self, *args, **kwargs) -> None: self.redraw() def keypress(self, *args, **kwargs): @@ -297,7 +297,7 @@ def get_status(self) -> list[tuple[str, str] | str]: def redraw(self) -> None: fc = self.master.commands.execute("view.properties.length") - if self.master.view.focus.flow is None: + if self.master.view.focus.index is None: offset = 0 else: offset = self.master.view.focus.index + 1 diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 749356a632..fa05722212 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -1,3 +1,4 @@ +from __future__ import annotations import argparse import asyncio import logging @@ -42,7 +43,7 @@ def run( master_cls: type[T], make_parser: Callable[[options.Options], argparse.ArgumentParser], arguments: Sequence[str], - extra: Callable[[Any], dict] = None, + extra: Callable[[Any], dict] | None = None, ) -> T: # pragma: no cover """ extra: Extra argument processing callable which returns a dict of diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index c085934057..4b58dc2710 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -5,7 +5,7 @@ import logging import os.path import re -from collections.abc import Sequence +from collections.abc import Callable, Sequence from io import BytesIO from itertools import islice from typing import ClassVar, Optional, Union @@ -313,16 +313,18 @@ def get(self): class DumpFlows(RequestHandler): - def get(self): + def get(self) -> None: self.set_header("Content-Disposition", "attachment; filename=flows") self.set_header("Content-Type", "application/octet-stream") + match: Callable[[mitmproxy.flow.Flow], bool] try: match = flowfilter.parse(self.request.arguments["filter"][0].decode()) except ValueError: # thrown py flowfilter.parse if filter is invalid raise APIError(400, f"Invalid filter argument / regex") except (KeyError, IndexError): # Key+Index: ["filter"][0] can fail, if it's not set - match = bool # returns always true + def match(_) -> bool: + return True with BytesIO() as bio: fw = io.FlowWriter(bio) @@ -381,7 +383,7 @@ def delete(self, flow_id): self.flow.kill() self.view.remove([self.flow]) - def put(self, flow_id): + def put(self, flow_id) -> None: flow: mitmproxy.flow.Flow = self.flow flow.backup() try: @@ -469,13 +471,13 @@ def post(self, flow_id, message): def get(self, flow_id, message): message = getattr(self.flow, message) + assert isinstance(self.flow, HTTPFlow) original_cd = message.headers.get("Content-Disposition", None) filename = None if original_cd: - filename = re.search(r'filename=([-\w" .()]+)', original_cd) - if filename: - filename = filename.group(1) + if m := re.search(r'filename=([-\w" .()]+)', original_cd): + filename = m.group(1) if not filename: filename = self.flow.request.path.split("?")[0].split("/")[-1] @@ -509,7 +511,7 @@ def message_to_json( description=description, ) - def get(self, flow_id, message, content_view): + def get(self, flow_id, message, content_view) -> None: flow = self.flow assert isinstance(flow, (HTTPFlow, TCPFlow, UDPFlow)) @@ -519,6 +521,7 @@ def get(self, flow_id, message, content_view): max_lines = None if message == "messages": + messages: list[TCPMessage] | list[UDPMessage] | list[WebSocketMessage] if isinstance(flow, HTTPFlow) and flow.websocket: messages = flow.websocket.messages elif isinstance(flow, (TCPFlow, UDPFlow)): diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 0a67108c0a..62119e09c0 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -9,6 +9,7 @@ from mitmproxy import log from mitmproxy import master from mitmproxy import optmanager +from mitmproxy import options from mitmproxy.addons import errorcheck, eventstore from mitmproxy.addons import intercept from mitmproxy.addons import readfile @@ -22,8 +23,8 @@ class WebMaster(master.Master): - def __init__(self, options, with_termlog=True): - super().__init__(options) + def __init__(self, opts: options.Options, with_termlog: bool = True): + super().__init__(opts) self.view = view.View() self.view.sig_view_add.connect(self._sig_view_add) self.view.sig_view_remove.connect(self._sig_view_remove) diff --git a/mitmproxy/utils/arg_check.py b/mitmproxy/utils/arg_check.py index b6736e3ab2..ad43eaf9b8 100644 --- a/mitmproxy/utils/arg_check.py +++ b/mitmproxy/utils/arg_check.py @@ -131,10 +131,11 @@ def check(): for option in REPLACED.splitlines(): if option in args: - if isinstance(REPLACEMENTS.get(option), list): - new_options = REPLACEMENTS.get(option) + r = REPLACEMENTS.get(option) + if isinstance(r, list): + new_options = r else: - new_options = [REPLACEMENTS.get(option)] + new_options = [r] print( "{} is deprecated.\n" "Please use `{}` instead.".format(option, "` or `".join(new_options)) diff --git a/mitmproxy/utils/data.py b/mitmproxy/utils/data.py index b715072178..091640ec9f 100644 --- a/mitmproxy/utils/data.py +++ b/mitmproxy/utils/data.py @@ -7,7 +7,9 @@ class Data: def __init__(self, name): self.name = name m = importlib.import_module(name) - dirname = os.path.dirname(inspect.getsourcefile(m)) + f = inspect.getsourcefile(m) + assert f is not None + dirname = os.path.dirname(f) self.dirname = os.path.abspath(dirname) def push(self, subpath): diff --git a/mitmproxy/utils/debug.py b/mitmproxy/utils/debug.py index a522d46a6f..5e01ff6b0b 100644 --- a/mitmproxy/utils/debug.py +++ b/mitmproxy/utils/debug.py @@ -18,11 +18,14 @@ def dump_system_info(): mitmproxy_version = version.get_dev_version() + openssl_version = SSL.SSLeay_version(SSL.SSLEAY_VERSION) + if isinstance(openssl_version, bytes): + openssl_version = openssl_version.decode() data = [ f"Mitmproxy: {mitmproxy_version}", f"Python: {platform.python_version()}", - f"OpenSSL: {SSL.SSLeay_version(SSL.SSLEAY_VERSION).decode()}", + f"OpenSSL: {openssl_version}", f"Platform: {platform.platform()}", ] return "\n".join(data) diff --git a/mitmproxy/utils/signals.py b/mitmproxy/utils/signals.py index 423dddc334..37900f68b1 100644 --- a/mitmproxy/utils/signals.py +++ b/mitmproxy/utils/signals.py @@ -18,7 +18,7 @@ from typing import ParamSpec except ImportError: # pragma: no cover # Python 3.9 - from typing_extensions import ParamSpec + from typing_extensions import ParamSpec # type: ignore P = ParamSpec("P") R = TypeVar("R") @@ -37,7 +37,7 @@ def make_weak_ref(obj: Any) -> weakref.ReferenceType: # We're running into https://github.com/python/mypy/issues/6073 here, # which is why the base class is a mixin and not a generic superclass. class _SignalMixin: - def __init__(self): + def __init__(self) -> None: self.receivers: list[weakref.ref[Callable]] = [] def connect(self, receiver: Callable) -> None: diff --git a/release/build.py b/release/build.py index 6878a9e7f2..60177e31a2 100644 --- a/release/build.py +++ b/release/build.py @@ -129,7 +129,7 @@ def standalone_binaries(): executable = executable.with_suffix(".exe") f.add(str(executable), str(executable.name)) - print(f"Packed {f.name}.") + print(f"Packed {f.name!r}.") def _ensure_pyinstaller_onedir(): diff --git a/release/selftest.py b/release/selftest.py index 6052a5ca83..cf1130d6a5 100644 --- a/release/selftest.py +++ b/release/selftest.py @@ -19,7 +19,7 @@ def load(_): def running(): # attach is somewhere so that it's not collected. - ctx.task = asyncio.create_task(make_request()) + ctx.task = asyncio.create_task(make_request()) # type: ignore async def make_request(): diff --git a/setup.cfg b/setup.cfg index a43c808d59..a2c237e86c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ exclude_lines = \.\.\. [mypy] +check_untyped_defs = True ignore_missing_imports = True files = mitmproxy,examples/addons,release diff --git a/test/mitmproxy/tools/console/test_keymap.py b/test/mitmproxy/tools/console/test_keymap.py index 12e12f7c0d..fb73471e37 100644 --- a/test/mitmproxy/tools/console/test_keymap.py +++ b/test/mitmproxy/tools/console/test_keymap.py @@ -75,8 +75,8 @@ def test_remove(): def test_load_path(tmpdir): dst = str(tmpdir.join("conf")) - kmc = keymap.KeymapConfig() - with taddons.context(kmc) as tctx: + with taddons.context() as tctx: + kmc = keymap.KeymapConfig(tctx.master) km = keymap.Keymap(tctx.master) tctx.master.keymap = km @@ -148,8 +148,8 @@ def test_load_path(tmpdir): def test_parse(): - kmc = keymap.KeymapConfig() - with taddons.context(kmc): + with taddons.context() as tctx: + kmc = keymap.KeymapConfig(tctx.master) assert kmc.parse("") == [] assert kmc.parse("\n\n\n \n") == [] with pytest.raises(keymap.KeyBindingError, match="expected a list of keys"): diff --git a/tox.ini b/tox.ini index 4d0d6ed23b..98ff06b338 100644 --- a/tox.ini +++ b/tox.ini @@ -29,13 +29,13 @@ commands = [testenv:mypy] deps = - mypy==0.982 + mypy==0.990 types-certifi==2021.10.8.3 types-Flask==1.1.6 types-Werkzeug==1.0.9 - types-requests==2.28.11.2 - types-cryptography==3.3.23.1 - types-pyOpenSSL==22.1.0.1 + types-requests==2.28.11.4 + types-cryptography==3.3.23.2 + types-pyOpenSSL==22.1.0.2 -e .[dev] commands = diff --git a/web/src/js/ducks/_options_gen.ts b/web/src/js/ducks/_options_gen.ts index 64f821065b..246ed3e6ba 100644 --- a/web/src/js/ducks/_options_gen.ts +++ b/web/src/js/ducks/_options_gen.ts @@ -39,7 +39,6 @@ export interface OptionsState { normalize_outbound_headers: boolean onboarding: boolean onboarding_host: string - onboarding_port: number proxy_debug: boolean proxyauth: string | undefined rawtcp: boolean @@ -131,7 +130,6 @@ export const defaultState: OptionsState = { normalize_outbound_headers: true, onboarding: true, onboarding_host: "mitm.it", - onboarding_port: 80, proxy_debug: false, proxyauth: undefined, rawtcp: true, From d5f1d1c623dec98f92d764a78b2e43f3e035fa59 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 18 Nov 2022 10:42:48 +0100 Subject: [PATCH 113/695] Improve error message for missing packages in frozen binaries (#5740) --- mitmproxy/addons/script.py | 11 +++++++++++ test/mitmproxy/addons/test_script.py | 8 ++++++++ test/mitmproxy/data/addonscripts/import_error.py | 1 + 3 files changed, 20 insertions(+) create mode 100644 test/mitmproxy/data/addonscripts/import_error.py diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 1c37e4a439..12586d327c 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -39,6 +39,17 @@ def load_script(path: str) -> Optional[types.ModuleType]: loader.exec_module(m) if not getattr(m, "name", None): m.name = path # type: ignore + except ImportError as e: + err_msg = str(e) + if getattr(sys, "frozen", False): + err_msg = ( + f"{err_msg}. \n" + f"Note that mitmproxy's binaries include their own Python environment. " + f"If your addon requires the installation of additional dependencies, " + f"please install mitmproxy from PyPI " + f"(https://docs.mitmproxy.org/stable/overview-installation/#installation-from-the-python-package-index-pypi)." + ) + script_error_handler(path, e, msg=err_msg) except Exception as e: script_error_handler(path, e, msg=str(e)) finally: diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index fba36a5562..0971fa0e9d 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -123,6 +123,14 @@ async def test_exception(self, tdata, caplog_async): await caplog_async.await_log("error.py") sc.done() + async def test_import_error(self, monkeypatch, tdata, caplog): + monkeypatch.setattr(sys, "frozen", True, raising=False) + script.Script( + tdata.path("mitmproxy/data/addonscripts/import_error.py"), + False, + ) + assert "Note that mitmproxy's binaries include their own Python environment" in caplog.text + async def test_optionexceptions(self, tdata, caplog_async): with taddons.context() as tctx: sc = script.Script( diff --git a/test/mitmproxy/data/addonscripts/import_error.py b/test/mitmproxy/data/addonscripts/import_error.py new file mode 100644 index 0000000000..e868bcc347 --- /dev/null +++ b/test/mitmproxy/data/addonscripts/import_error.py @@ -0,0 +1 @@ +import nonexistent From 6b7106614187a22f0b61cde024e9c559191ed0ad Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 21 Nov 2022 01:27:31 +0100 Subject: [PATCH 114/695] cleanup `mitmproxy.connection`, introduce dataclass-based serialization --- CHANGELOG.md | 2 + docs/src/content/addons-api-changelog.md | 4 + examples/contrib/change_upstream_proxy.py | 2 +- mitmproxy/addons/clientplayback.py | 2 +- mitmproxy/addons/view.py | 4 +- mitmproxy/connection.py | 196 +++--------------- mitmproxy/coretypes/serializable.py | 114 ++++++++++ mitmproxy/flow.py | 3 +- mitmproxy/io/compat.py | 34 ++- mitmproxy/io/io.py | 4 +- mitmproxy/proxy/context.py | 3 +- mitmproxy/proxy/layers/http/__init__.py | 2 +- .../proxy/layers/http/_upstream_proxy.py | 2 +- mitmproxy/proxy/layers/tls.py | 6 +- mitmproxy/proxy/server.py | 6 +- mitmproxy/stateobject.py | 3 +- mitmproxy/test/tflow.py | 89 ++++---- mitmproxy/version.py | 2 +- test/mitmproxy/addons/test_block.py | 2 +- test/mitmproxy/addons/test_next_layer.py | 2 +- test/mitmproxy/addons/test_tlsconfig.py | 59 ++---- test/mitmproxy/coretypes/test_serializable.py | 76 +++++++ test/mitmproxy/proxy/conftest.py | 2 +- .../mitmproxy/proxy/layers/http/test_http2.py | 2 +- .../proxy/layers/http/test_http_fuzz.py | 27 ++- test/mitmproxy/proxy/layers/test_modes.py | 4 +- .../proxy/layers/test_socks5_fuzz.py | 6 +- test/mitmproxy/proxy/layers/test_tls.py | 4 +- test/mitmproxy/proxy/test_commands.py | 2 +- test/mitmproxy/proxy/test_events.py | 2 +- test/mitmproxy/proxy/test_layer.py | 8 +- test/mitmproxy/proxy/test_tunnel.py | 12 +- test/mitmproxy/test_connection.py | 8 +- 33 files changed, 381 insertions(+), 313 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74ebc06ada..59edf1d7f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ * The `onboarding_port` option has been removed. The onboarding app now responds to all requests for the hostname specified in `onboarding_host`. +* `connection.Client` and `connection.Server` now accept keyword arguments only. + This is a breaking change for custom addons that use these classes directly. ## 02 November 2022: mitmproxy 9.0.1 diff --git a/docs/src/content/addons-api-changelog.md b/docs/src/content/addons-api-changelog.md index c4dfad60ed..7fd2327f67 100644 --- a/docs/src/content/addons-api-changelog.md +++ b/docs/src/content/addons-api-changelog.md @@ -10,6 +10,10 @@ menu: We try to avoid them, but this page lists breaking changes in the mitmproxy addon API. +## mitmproxy >= 9.1 + +`mitmproxy.connection.Client` and `mitmproxy.connection.Server` now accept keyword arguments only. + ## mitmproxy 9.0 #### Logging diff --git a/examples/contrib/change_upstream_proxy.py b/examples/contrib/change_upstream_proxy.py index ddcbabf100..7f6d56bc4a 100644 --- a/examples/contrib/change_upstream_proxy.py +++ b/examples/contrib/change_upstream_proxy.py @@ -32,5 +32,5 @@ def request(flow: http.HTTPFlow) -> None: if is_proxy_change and server_connection_already_open: # server_conn already refers to an existing connection (which cannot be modified), # so we need to replace it with a new server connection object. - flow.server_conn = Server(flow.server_conn.address) + flow.server_conn = Server(address=flow.server_conn.address) flow.server_conn.via = ServerSpec("http", address) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 6981cb0361..d435324e86 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -84,7 +84,7 @@ def __init__(self, flow: http.HTTPFlow, options: Options) -> None: client.state = ConnectionState.OPEN context = Context(client, options) - context.server = Server((flow.request.host, flow.request.port)) + context.server = Server(address=(flow.request.host, flow.request.port)) context.server.tls = flow.request.scheme == "https" if options.mode and options.mode[0].startswith("upstream:"): mode = UpstreamMode.parse(options.mode[0]) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 9a3e2116aa..965d9fc483 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -475,8 +475,8 @@ def create(self, method: str, url: str) -> None: except ValueError as e: raise exceptions.CommandError("Invalid URL: %s" % e) - c = connection.Client(("", 0), ("", 0), req.timestamp_start - 0.0001) - s = connection.Server((req.host, req.port)) + c = connection.Client(peername=("", 0), sockname=("", 0), timestamp_start=req.timestamp_start - 0.0001) + s = connection.Server(address=(req.host, req.port)) f = http.HTTPFlow(c, s) f.request = req diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index b08394892e..278a7677ed 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -1,3 +1,6 @@ +import dataclasses +import time +from dataclasses import dataclass, field import uuid import warnings from abc import ABCMeta @@ -30,7 +33,9 @@ class ConnectionState(Flag): Address = tuple[str, int] -class Connection(serializable.Serializable, metaclass=ABCMeta): +# noinspection PyDataclass +@dataclass(kw_only=True) +class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): """ Base class for client and server connections. @@ -41,11 +46,11 @@ class Connection(serializable.Serializable, metaclass=ABCMeta): # all connections have a unique id. While # f.client_conn == f2.client_conn already holds true for live flows (where we have object identity), # we also want these semantics for recorded flows. - id: str + id: str = field(default_factory=lambda: str(uuid.uuid4())) """A unique UUID to identify the connection.""" state: ConnectionState """The current connection state.""" - transport_protocol: TransportProtocol + transport_protocol: TransportProtocol = field(default="tcp") """The connection protocol in use.""" peername: Optional[Address] """The remote's `(ip, port)` tuple for this connection.""" @@ -99,7 +104,7 @@ class Connection(serializable.Serializable, metaclass=ABCMeta): The [Server Name Indication (SNI)](https://en.wikipedia.org/wiki/Server_Name_Indication) sent in the ClientHello. """ - timestamp_start: Optional[float] + timestamp_start: Optional[float] = None timestamp_end: Optional[float] = None """*Timestamp:* Connection has been closed.""" timestamp_tls_setup: Optional[float] = None @@ -124,16 +129,20 @@ def __hash__(self): return hash(self.id) def __repr__(self): - attrs = repr( - { - k: { - "cipher_list": lambda: f"<{len(v)} ciphers>", - "id": lambda: f"…{v[-6:]}", - }.get(k, lambda: v)() - for k, v in self.__dict__.items() - } - ) - return f"{type(self).__name__}({attrs})" + attrs = { + # ensure these come first. + "id": None, + "address": None, + } + for f in dataclasses.fields(self): + val = getattr(self, f.name) + if val != f.default: + if f.name == "cipher_list": + val = f"<{len(val)} ciphers>" + elif f.name == "id": + val = f"…{val[-6:]}" + attrs[f.name] = val + return f"{type(self).__name__}({attrs!r})" @property def alpn_proto_negotiated(self) -> Optional[bytes]: # pragma: no cover @@ -146,6 +155,8 @@ def alpn_proto_negotiated(self) -> Optional[bytes]: # pragma: no cover return self.alpn +# noinspection PyDataclass +@dataclass(kw_only=True, eq=False, repr=False) class Client(Connection): """A connection between a client and mitmproxy.""" @@ -154,34 +165,19 @@ class Client(Connection): sockname: Address """The local address we received this connection on.""" + state: ConnectionState = field(default=ConnectionState.OPEN) + mitmcert: Optional[certs.Cert] = None """ The certificate used by mitmproxy to establish TLS with the client. """ - proxy_mode: mode_specs.ProxyMode + proxy_mode: mode_specs.ProxyMode = field(default=mode_specs.ProxyMode.parse("regular")) """The proxy server type this client has been connecting to.""" - timestamp_start: float + timestamp_start: float = field(default_factory=time.time) """*Timestamp:* TCP SYN received""" - def __init__( - self, - peername: Address, - sockname: Address, - timestamp_start: float, - *, - transport_protocol: TransportProtocol = "tcp", - proxy_mode: mode_specs.ProxyMode = mode_specs.ProxyMode.parse("regular"), - ): - self.id = str(uuid.uuid4()) - self.peername = peername - self.sockname = sockname - self.timestamp_start = timestamp_start - self.state = ConnectionState.OPEN - self.transport_protocol = transport_protocol - self.proxy_mode = proxy_mode - def __str__(self): if self.alpn: tls_state = f", alpn={self.alpn.decode(errors='replace')}" @@ -191,68 +187,6 @@ def __str__(self): tls_state = "" return f"Client({human.format_address(self.peername)}, state={self.state.name.lower()}{tls_state})" - def get_state(self): - # Important: Retain full compatibility with old proxy core for now! - # This means we need to add all new fields to the old implementation. - return { - "address": self.peername, - "alpn": self.alpn, - "cipher_name": self.cipher, - "id": self.id, - "mitmcert": self.mitmcert.get_state() - if self.mitmcert is not None - else None, - "sni": self.sni, - "timestamp_end": self.timestamp_end, - "timestamp_start": self.timestamp_start, - "timestamp_tls_setup": self.timestamp_tls_setup, - "tls_established": self.tls_established, - "tls_extensions": [], - "tls_version": self.tls_version, - # only used in sans-io - "state": self.state.value, - "sockname": self.sockname, - "error": self.error, - "tls": self.tls, - "certificate_list": [x.get_state() for x in self.certificate_list], - "alpn_offers": self.alpn_offers, - "cipher_list": self.cipher_list, - "proxy_mode": self.proxy_mode.get_state(), - } - - @classmethod - def from_state(cls, state) -> "Client": - client = Client(state["address"], ("mitmproxy", 8080), state["timestamp_start"]) - client.set_state(state) - return client - - def set_state(self, state): - self.peername = tuple(state["address"]) if state["address"] else None # type: ignore - self.alpn = state["alpn"] - self.cipher = state["cipher_name"] - self.id = state["id"] - self.sni = state["sni"] - self.timestamp_end = state["timestamp_end"] - self.timestamp_start = state["timestamp_start"] - self.timestamp_tls_setup = state["timestamp_tls_setup"] - self.tls_version = state["tls_version"] - # only used in sans-io - self.state = ConnectionState(state["state"]) - self.sockname = tuple(state["sockname"]) if state["sockname"] else None # type: ignore - self.error = state["error"] - self.tls = state["tls"] - self.certificate_list = [ - certs.Cert.from_state(x) for x in state["certificate_list"] - ] - self.mitmcert = ( - certs.Cert.from_state(state["mitmcert"]) - if state["mitmcert"] is not None - else None - ) - self.alpn_offers = state["alpn_offers"] - self.cipher_list = state["cipher_list"] - self.proxy_mode = mode_specs.ProxyMode.from_state(state["proxy_mode"]) - @property def address(self): # pragma: no cover """*Deprecated:* An outdated alias for Client.peername.""" @@ -308,6 +242,8 @@ def clientcert(self, val): # pragma: no cover self.certificate_list = [] +# noinspection PyDataclass +@dataclass(kw_only=True, eq=False, repr=False) class Server(Connection): """A connection between mitmproxy and an upstream server.""" @@ -317,6 +253,8 @@ class Server(Connection): address: Optional[Address] """The server's `(host, port)` address tuple. The host can either be a domain or a plain IP address.""" + state: ConnectionState = field(default=ConnectionState.CLOSED) + timestamp_start: Optional[float] = None """*Timestamp:* TCP SYN sent.""" timestamp_tcp_setup: Optional[float] = None @@ -325,17 +263,6 @@ class Server(Connection): via: Optional[server_spec.ServerSpec] = None """An optional proxy server specification via which the connection should be established.""" - def __init__( - self, - address: Optional[Address], - *, - transport_protocol: TransportProtocol = "tcp", - ): - self.id = str(uuid.uuid4()) - self.address = address - self.state = ConnectionState.CLOSED - self.transport_protocol = transport_protocol - def __str__(self): if self.alpn: tls_state = f", alpn={self.alpn.decode(errors='replace')}" @@ -361,63 +288,6 @@ def __setattr__(self, name, value): raise RuntimeError(f"Cannot change server.{name} on open connection.") return super().__setattr__(name, value) - def get_state(self): - return { - "address": self.address, - "alpn": self.alpn, - "id": self.id, - "ip_address": self.peername, - "sni": self.sni, - "source_address": self.sockname, - "timestamp_end": self.timestamp_end, - "timestamp_start": self.timestamp_start, - "timestamp_tcp_setup": self.timestamp_tcp_setup, - "timestamp_tls_setup": self.timestamp_tls_setup, - "tls_established": self.tls_established, - "tls_version": self.tls_version, - "via": None, - # only used in sans-io - "state": self.state.value, - "error": self.error, - "tls": self.tls, - "certificate_list": [x.get_state() for x in self.certificate_list], - "alpn_offers": self.alpn_offers, - "cipher_name": self.cipher, - "cipher_list": self.cipher_list, - "via2": self.via, - } - - @classmethod - def from_state(cls, state) -> "Server": - server = Server(None) - server.set_state(state) - return server - - def set_state(self, state): - self.address = tuple(state["address"]) if state["address"] else None # type: ignore - self.alpn = state["alpn"] - self.id = state["id"] - self.peername = tuple(state["ip_address"]) if state["ip_address"] else None # type: ignore - self.sni = state["sni"] - self.sockname = ( - tuple(state["source_address"]) if state["source_address"] else None # type: ignore - ) - self.timestamp_end = state["timestamp_end"] - self.timestamp_start = state["timestamp_start"] - self.timestamp_tcp_setup = state["timestamp_tcp_setup"] - self.timestamp_tls_setup = state["timestamp_tls_setup"] - self.tls_version = state["tls_version"] - self.state = ConnectionState(state["state"]) - self.error = state["error"] - self.tls = state["tls"] - self.certificate_list = [ - certs.Cert.from_state(x) for x in state["certificate_list"] - ] - self.alpn_offers = state["alpn_offers"] - self.cipher = state["cipher_name"] - self.cipher_list = state["cipher_list"] - self.via = state["via2"] - @property def ip_address(self) -> Optional[Address]: # pragma: no cover """*Deprecated:* An outdated alias for `Server.peername`.""" diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index df09598b6e..9b6f1c2ac5 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -1,4 +1,9 @@ import abc +import collections.abc +import dataclasses +import enum +import types +import typing import uuid from typing import TypeVar @@ -37,3 +42,112 @@ def copy(self: T) -> T: if isinstance(state, dict) and "id" in state: state["id"] = str(uuid.uuid4()) return self.from_state(state) + + +@dataclasses.dataclass +class SerializableDataclass(Serializable): + + def get_state(self): + state = {} + for field in dataclasses.fields(self): + val = getattr(self, field.name) + state[field.name] = _to_state(val, field.type, field.name) + return state + + @classmethod + def from_state(cls: type[T], state) -> T: + # state = state.copy() + for field in dataclasses.fields(cls): + try: + state_val = state[field.name] + except KeyError: + if field.default is dataclasses.MISSING: + raise ValueError(f"Missing state attribute: {field.name}") + else: + continue + state[field.name] = _to_val(state_val, field.type, field.name) + try: + return cls(**state) # type: ignore + except TypeError as e: + raise ValueError(f"Invalid state for {cls}: {e} ({state=})") from e + + def set_state(self, state): + for field in dataclasses.fields(self): + current = getattr(self, field.name) + if isinstance(current, Serializable): + current.set_state(state.pop(field.name)) + else: + val = _to_val(state.pop(field.name), field.type, field.name) + setattr(self, field.name, val) + + if state: + raise RuntimeWarning(f"Unexpected fields in SerializableDataclass.set_state: {state}") + + +U = TypeVar("U") + + +def _process(attr_val: typing.Any, attr_type: type[U], attr_name: str, make: bool) -> U: + origin = typing.get_origin(attr_type) + if origin is typing.Literal: + assert attr_val in typing.get_args(attr_type), "Literal does not match." + return attr_val + if origin in (types.UnionType, typing.Union): + attr_type, nt = typing.get_args(attr_type) + assert nt is types.NoneType, f"{attr_name}: only `x | None` union types are supported`" # noqa + if attr_val is None: + return None # type: ignore + else: + return _process(attr_val, attr_type, attr_name, make) + else: + if attr_val is None: + raise ValueError(f"Attribute {attr_name} must not be None.") + + if make and hasattr(attr_type, "from_state"): + return attr_type.from_state(attr_val) # type: ignore + elif not make and hasattr(attr_type, "get_state"): + return attr_val.get_state() + + if origin in (list, collections.abc.Sequence): + (T,) = typing.get_args(attr_type) + return [_process(x, T, attr_name, make) for x in attr_val] # type: ignore + elif origin is tuple: + # We don't have a good way to represent tuple[str,int] | tuple[str,int,int,int], so we do a dirty hack here. + if attr_name in ("peername", "sockname"): + return tuple( + _process(x, T, attr_name, make) for x, T in zip(attr_val, [str, int, int, int]) + ) # type: ignore + Ts = typing.get_args(attr_type) + if len(Ts) != len(attr_val): + raise ValueError(f"Invalid data for {attr_name}. Expected {Ts}, got {attr_val}.") + return tuple(_process(x, T, attr_name, make) for T, x in zip(Ts, attr_val)) # type: ignore + elif origin is dict: + k_cls, v_cls = typing.get_args(attr_type) + return { + _process(k, k_cls, attr_name, make): _process(v, v_cls, attr_name, make) for k, v in attr_val.items() + } # type: ignore + elif attr_type in (int, float): + if not isinstance(attr_val, (int, float)): + raise ValueError(f"Invalid value for {attr_name}. Expected {attr_type}, got {attr_val} ({type(attr_val)}).") + return attr_type(attr_val) # type: ignore + elif attr_type in (str, bytes, bool): + if not isinstance(attr_val, attr_type): + raise ValueError(f"Invalid value for {attr_name}. Expected {attr_type}, got {attr_val} ({type(attr_val)}).") + return attr_type(attr_val) # type: ignore + elif isinstance(attr_type, type) and issubclass(attr_type, enum.Enum): + if make: + return attr_type(attr_val) # type: ignore + else: + return attr_val.value + else: + raise TypeError(f"Unexpected type for {attr_name}: {attr_type}") + + +def _to_val(state: typing.Any, attr_type: type[U], attr_name: str) -> U: + """Create an object based on the state given in val.""" + return _process(state, attr_type, attr_name, True) + + +def _to_state(value: typing.Any, attr_type: type[U], attr_name: str) -> U: + """Get the state of the object given as val.""" + return _process(value, attr_type, attr_name, False) diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 42e1974585..e7ce47ff71 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -1,4 +1,5 @@ import asyncio +import copy import time import uuid from typing import Any, ClassVar, Optional @@ -163,7 +164,7 @@ def get_state(self): d = super().get_state() d.update(version=version.FLOW_FORMAT_VERSION, type=self.type) if self._backup and self._backup != d: - d.update(backup=self._backup) + d.update(backup=copy.deepcopy(self._backup)) return d def set_state(self, state): diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 5ae6b08c03..bed19945ec 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -5,6 +5,7 @@ v3.0.0dev) and versioning. Every change or migration gets a new flow file version number, this prevents issues with developer builds and snapshots. """ +import copy import uuid from typing import Any, Union @@ -283,7 +284,7 @@ def convert_11_12(data): data["version"] = 12 if "websocket" in data["metadata"]: - _websocket_handshakes[data["id"]] = data + _websocket_handshakes[data["id"]] = copy.deepcopy(data) if "websocket_handshake" in data["metadata"]: ws_flow = data @@ -386,6 +387,36 @@ def convert_17_18(data): return data +def convert_18_19(data): + data["version"] = 19 + data["client_conn"]["peername"] = data["client_conn"].pop("address", None) + if data["client_conn"].get("timestamp_start") is None: + data["client_conn"]["timestamp_start"] = 0.0 + data["client_conn"].pop("tls_extensions") + + data["server_conn"]["peername"] = data["server_conn"].pop("ip_address", None) + data["server_conn"]["sockname"] = data["server_conn"].pop("source_address", None) + data["server_conn"]["via"] = data["server_conn"].pop("via2", None) + + for conn in ["client_conn", "server_conn"]: + if data[conn].get("timestamp_tls_setup") is None and data[conn].get("tls_established"): + data[conn]["timestamp_tls_setup"] = 1.0 + data[conn].pop("tls_established") + + data[conn]["cipher"] = data[conn].pop("cipher_name", None) + + for name in ["peername", "sockname", "address"]: + if data[conn].get(name) and isinstance(data[conn][name][0], bytes): + data[conn][name][0] = data[conn][name][0].decode(errors="backslashreplace") + + if data["server_conn"]["sni"] is True: + data["server_conn"]["sni"] = data["server_conn"]["address"][0] + if data["server_conn"]["sni"] is False: + data["server_conn"]["sni"] = None + + return data + + def _convert_dict_keys(o: Any) -> Any: if isinstance(o, dict): return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()} @@ -448,6 +479,7 @@ def convert_unicode(data: dict) -> dict: 15: convert_15_16, 16: convert_16_17, 17: convert_17_18, + 18: convert_18_19, } diff --git a/mitmproxy/io/io.py b/mitmproxy/io/io.py index 402d3630c0..cd2b095bbf 100644 --- a/mitmproxy/io/io.py +++ b/mitmproxy/io/io.py @@ -35,11 +35,11 @@ def stream(self) -> Iterable[flow.Flow]: try: yield flow.Flow.from_state(compat.migrate_flow(loaded)) except ValueError as e: - raise exceptions.FlowReadException(e) + raise exceptions.FlowReadException(e) from e except (ValueError, TypeError, IndexError) as e: if str(e) == "not a tnetstring: empty file": return # Error is due to EOF - raise exceptions.FlowReadException("Invalid data format.") + raise exceptions.FlowReadException("Invalid data format.") from e class FilteredFlowWriter: diff --git a/mitmproxy/proxy/context.py b/mitmproxy/proxy/context.py index 5edb977c25..1be73b39b1 100644 --- a/mitmproxy/proxy/context.py +++ b/mitmproxy/proxy/context.py @@ -38,7 +38,8 @@ def __init__( self.client = client self.options = options self.server = connection.Server( - None, transport_protocol=client.transport_protocol + address=None, + transport_protocol=client.transport_protocol ) self.layers = [] diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index e1d56f4350..b2ee80d8ee 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -1000,7 +1000,7 @@ def get_connection( if not can_use_context_connection: - context.server = Server(event.address) + context.server = Server(address=event.address) if event.via: context.server.via = event.via diff --git a/mitmproxy/proxy/layers/http/_upstream_proxy.py b/mitmproxy/proxy/layers/http/_upstream_proxy.py index 62909bead2..4029d5347c 100644 --- a/mitmproxy/proxy/layers/http/_upstream_proxy.py +++ b/mitmproxy/proxy/layers/http/_upstream_proxy.py @@ -32,7 +32,7 @@ def make(cls, ctx: context.Context, send_connect: bool) -> tunnel.LayerStack: scheme, address = ctx.server.via assert scheme in ("http", "https") - http_proxy = connection.Server(address) + http_proxy = connection.Server(address=address) stack = tunnel.LayerStack() if scheme == "https": diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 20d39bbb9e..f04ab01661 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -570,12 +570,12 @@ def receive_handshake_data( # we've figured out that we don't want to intercept this connection, so we assign fake connection objects # to all TLS layers. This makes the real connection contents just go through. self.conn = self.tunnel_connection = connection.Client( - ("ignore-conn", 0), ("ignore-conn", 0), time.time() + peername=("ignore-conn", 0), sockname=("ignore-conn", 0) ) parent_layer = self.context.layers[self.context.layers.index(self) - 1] if isinstance(parent_layer, ServerTLSLayer): parent_layer.conn = parent_layer.tunnel_connection = connection.Server( - None + address=None ) if self.is_dtls: self.child_layer = udp.UDPLayer(self.context, ignore=True) @@ -670,4 +670,4 @@ class MockTLSLayer(TLSLayer): """ def __init__(self, ctx: context.Context): - super().__init__(ctx, connection.Server(None)) + super().__init__(ctx, connection.Server(address=None)) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 1341564ada..2ee5fd0796 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -419,9 +419,9 @@ def __init__( mode: mode_specs.ProxyMode, ) -> None: client = Client( - writer.get_extra_info("peername"), - writer.get_extra_info("sockname"), - time.time(), + peername=writer.get_extra_info("peername"), + sockname=writer.get_extra_info("sockname"), + timestamp_start=time.time(), proxy_mode=mode, ) context = Context(client, options) diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 73117eddf2..ef94e8fcf5 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -9,8 +9,7 @@ class StateObject(serializable.Serializable): """ An object with serializable state. - State attributes can either be serializable types(str, tuple, bool, ...) - or StateObject instances themselves. + New code should look into adopting SerializableDataclass instead of this class. """ _stateobject_attributes: typing.ClassVar[abc.MutableMapping[str, typing.Any]] diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index f448eab0ee..bcb2d211b4 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -8,6 +8,7 @@ from mitmproxy import tcp from mitmproxy import udp from mitmproxy import websocket +from mitmproxy.connection import ConnectionState from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.test.tutils import tdnsreq, tdnsresp from mitmproxy.test.tutils import treq, tresp @@ -211,58 +212,50 @@ def tdummyflow(client_conn=True, server_conn=True, err=None) -> DummyFlow: def tclient_conn() -> connection.Client: - c = connection.Client.from_state( - dict( - id=str(uuid.uuid4()), - address=("127.0.0.1", 22), - mitmcert=None, - tls_established=True, - timestamp_start=946681200, - timestamp_tls_setup=946681201, - timestamp_end=946681206, - sni="address", - cipher_name="cipher", - alpn=b"http/1.1", - tls_version="TLSv1.2", - tls_extensions=[(0x00, bytes.fromhex("000e00000b6578616d"))], - state=0, - sockname=("", 0), - error=None, - tls=False, - certificate_list=[], - alpn_offers=[], - cipher_list=[], - proxy_mode="regular", - ) + c = connection.Client( + id=str(uuid.uuid4()), + peername=("127.0.0.1", 22), + sockname=("", 0), + mitmcert=None, + timestamp_start=946681200, + timestamp_tls_setup=946681201, + timestamp_end=946681206, + sni="address", + cipher="cipher", + alpn=b"http/1.1", + tls_version="TLSv1.2", + state=ConnectionState.OPEN, + error=None, + tls=False, + certificate_list=[], + alpn_offers=[], + cipher_list=[], + proxy_mode=ProxyMode.parse("regular"), ) return c def tserver_conn() -> connection.Server: - c = connection.Server.from_state( - dict( - id=str(uuid.uuid4()), - address=("address", 22), - source_address=("address", 22), - ip_address=("192.168.0.1", 22), - timestamp_start=946681202, - timestamp_tcp_setup=946681203, - timestamp_tls_setup=946681204, - timestamp_end=946681205, - tls_established=True, - sni="address", - alpn=None, - tls_version="TLSv1.2", - via=None, - state=0, - error=None, - tls=False, - certificate_list=[], - alpn_offers=[], - cipher_name=None, - cipher_list=[], - via2=None, - ) + c = connection.Server( + id=str(uuid.uuid4()), + address=("address", 22), + peername=("192.168.0.1", 22), + sockname=("address", 22), + timestamp_start=946681202, + timestamp_tcp_setup=946681203, + timestamp_tls_setup=946681204, + timestamp_end=946681205, + sni="address", + alpn=None, + tls_version="TLSv1.2", + via=None, + state=ConnectionState.CLOSED, + error=None, + tls=False, + certificate_list=[], + alpn_offers=[], + cipher=None, + cipher_list=[], ) return c @@ -300,4 +293,4 @@ def tflows() -> list[flow.Flow]: tudpflow(err=True), tdnsflow(resp=True), tdnsflow(err=True), - ] \ No newline at end of file + ] diff --git a/mitmproxy/version.py b/mitmproxy/version.py index d1405ffe1a..b4c46ef150 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -7,7 +7,7 @@ # Serialization format version. This is displayed nowhere, it just needs to be incremented by one # for each change in the file format. -FLOW_FORMAT_VERSION = 18 +FLOW_FORMAT_VERSION = 19 def get_dev_version() -> str: diff --git a/test/mitmproxy/addons/test_block.py b/test/mitmproxy/addons/test_block.py index 3a7d0d4837..3fad8f5b8c 100644 --- a/test/mitmproxy/addons/test_block.py +++ b/test/mitmproxy/addons/test_block.py @@ -56,6 +56,6 @@ async def test_block_global(block_global, block_private, should_be_killed, addre ar = block.Block() with taddons.context(ar) as tctx: tctx.configure(ar, block_global=block_global, block_private=block_private) - client = connection.Client(address, ("127.0.0.1", 8080), 1607699500) + client = connection.Client(peername=address, sockname=("127.0.0.1", 8080)) ar.client_connected(client) assert bool(client.error) == should_be_killed diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index 8b592cb5b1..bc113b46ce 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -14,7 +14,7 @@ @pytest.fixture def tctx(): context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), + connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), tctx.options, ) diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 535d30f428..9f23ada275 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -6,7 +6,7 @@ import pytest from OpenSSL import SSL -from mitmproxy import certs, connection, tls +from mitmproxy import certs, connection, tls, options from mitmproxy.addons import tlsconfig from mitmproxy.proxy import context from mitmproxy.proxy.layers import modes, tls as proxy_tls @@ -60,6 +60,13 @@ def test_alpn_select_callback(): here = Path(__file__).parent +def _ctx(opts: options.Options) -> context.Context: + return context.Context( + connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), + opts, + ) + + class TestTlsConfig: def test_configure(self, tdata): ta = tlsconfig.TlsConfig() @@ -92,10 +99,7 @@ def test_get_cert(self, tdata): with taddons.context(ta) as tctx: ta.configure(["confdir"]) - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) # Edge case first: We don't have _any_ idea about the server nor is there a SNI, # so we just return our local IP as subject. @@ -132,10 +136,7 @@ def test_tls_clienthello(self): # only really testing for coverage here, there's no point in mirroring the individual conditions ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ch = tls.ClientHelloData(ctx, None) # type: ignore ta.tls_clienthello(ch) assert not ch.establish_server_tls_first @@ -173,10 +174,7 @@ def test_tls_start_client(self, tdata): ], ciphers_client="ECDHE-ECDSA-AES128-GCM-SHA256", ) - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) tls_start = tls.TlsData(ctx.client, context=ctx) ta.tls_start_client(tls_start) @@ -195,10 +193,7 @@ def test_tls_start_client(self, tdata): def test_tls_start_server_cannot_verify(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.server.address = ("example.mitmproxy.org", 443) ctx.server.sni = "" # explicitly opt out of using the address. @@ -209,10 +204,7 @@ def test_tls_start_server_cannot_verify(self): def test_tls_start_server_verify_failed(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.client.alpn_offers = [b"h2"] ctx.client.cipher_list = ["TLS_AES_256_GCM_SHA384", "ECDHE-RSA-AES128-SHA"] ctx.server.address = ("example.mitmproxy.org", 443) @@ -228,10 +220,7 @@ def test_tls_start_server_verify_failed(self): def test_tls_start_server_verify_ok(self, hostname, tdata): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.server.address = (hostname, 443) tctx.configure( ta, @@ -254,10 +243,7 @@ def test_tls_start_server_verify_ok(self, hostname, tdata): def test_tls_start_server_insecure(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.server.address = ("example.mitmproxy.org", 443) tctx.configure( @@ -276,10 +262,7 @@ def test_tls_start_server_insecure(self): def test_alpn_selection(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.server.address = ("example.mitmproxy.org", 443) tls_start = tls.TlsData(ctx.server, context=ctx) @@ -319,10 +302,7 @@ def test_no_h2_proxy(self, tdata): ], ) - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) # mock up something that looks like a secure web proxy. ctx.layers = [modes.HttpProxy(ctx), 123] tls_start = tls.TlsData(ctx.client, context=ctx) @@ -339,10 +319,7 @@ def test_no_h2_proxy(self, tdata): def test_client_cert_file(self, tdata, client_certs): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.server.address = ("example.mitmproxy.org", 443) tctx.configure( ta, diff --git a/test/mitmproxy/coretypes/test_serializable.py b/test/mitmproxy/coretypes/test_serializable.py index 8617a75ee6..068d4e6a18 100644 --- a/test/mitmproxy/coretypes/test_serializable.py +++ b/test/mitmproxy/coretypes/test_serializable.py @@ -1,6 +1,11 @@ import copy +import enum +from dataclasses import dataclass + +import pytest from mitmproxy.coretypes import serializable +from mitmproxy.coretypes.serializable import SerializableDataclass class SerializableDummy(serializable.Serializable): @@ -34,3 +39,74 @@ def test_copy_id(self): b = a.copy() assert a.get_state()["id"] != b.get_state()["id"] assert a.get_state()["foo"] == b.get_state()["foo"] + + +@dataclass +class Simple(SerializableDataclass): + x: int + y: str | None + + +@dataclass +class SerializableChild(SerializableDataclass): + foo: Simple + maybe_foo: Simple | None + + +@dataclass +class Inheritance(Simple): + z: bool + + +class TEnum(enum.Enum): + A = 1 + B = 2 + + +@dataclass +class BuiltinChildren(SerializableDataclass): + a: list[int] | None + b: dict[str, int] | None + c: tuple[int, int] | None + d: list[Simple] + e: TEnum | None + + +@dataclass +class Defaults(SerializableDataclass): + z: int | None = 42 + + +class TestSerializableDataclass: + @pytest.mark.parametrize("cls, state", [ + (Simple, {"x": 42, "y": 'foo'}), + (Simple, {"x": 42, "y": None}), + (SerializableChild, {"foo": {"x": 42, "y": "foo"}, "maybe_foo": None}), + (SerializableChild, {"foo": {"x": 42, "y": "foo"}, "maybe_foo": {"x": 42, "y": "foo"}}), + (Inheritance, {"x": 42, "y": "foo", "z": True}), + (BuiltinChildren, {"a": [1, 2, 3], "b": {"foo": 42}, "c": (1, 2), "d": [{"x": 42, "y": "foo"}], "e": 1}), + (BuiltinChildren, {"a": None, "b": None, "c": None, "d": [], "e": None}), + ]) + def test_roundtrip(self, cls, state): + a = cls.from_state(copy.deepcopy(state)) + assert a.get_state() == state + + def test_invalid_none(self): + with pytest.raises(ValueError): + Simple.from_state({"x": None, "y": "foo"}) + + def test_defaults(self): + a = Defaults.from_state({}) + assert a.get_state() == {"z": 42} + + def test_invalid_type(self): + with pytest.raises(ValueError): + Simple.from_state({"x": 42, "y": 42}) + + def test_invalid_key(self): + with pytest.raises(ValueError): + Simple.from_state({"x": 42, "y": "foo", "z": True}) + + def test_invalid_type_in_list(self): + with pytest.raises(ValueError): + BuiltinChildren.from_state({"w": [{"x": "foo", "y": "foo"}], "x": None, "y": None, "z": None}) diff --git a/test/mitmproxy/proxy/conftest.py b/test/mitmproxy/proxy/conftest.py index ba256ff887..77713aa226 100644 --- a/test/mitmproxy/proxy/conftest.py +++ b/test/mitmproxy/proxy/conftest.py @@ -13,7 +13,7 @@ def tctx() -> context.Context: opts = options.Options() Proxyserver().load(opts) return context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts + connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), opts ) diff --git a/test/mitmproxy/proxy/layers/http/test_http2.py b/test/mitmproxy/proxy/layers/http/test_http2.py index 6609f96ead..e28bf72f35 100644 --- a/test/mitmproxy/proxy/layers/http/test_http2.py +++ b/test/mitmproxy/proxy/layers/http/test_http2.py @@ -42,7 +42,7 @@ def open_h2_server_conn(): # this is a bit fake here (port 80, with alpn, but no tls - c'mon), # but we don't want to pollute our tests with TLS handshakes. - s = Server(("example.com", 80)) + s = Server(address=("example.com", 80)) s.state = ConnectionState.OPEN s.alpn = b"h2" return s diff --git a/test/mitmproxy/proxy/layers/http/test_http_fuzz.py b/test/mitmproxy/proxy/layers/http/test_http_fuzz.py index 6ad352efcf..67478a02fa 100644 --- a/test/mitmproxy/proxy/layers/http/test_http_fuzz.py +++ b/test/mitmproxy/proxy/layers/http/test_http_fuzz.py @@ -133,9 +133,7 @@ def h2_responses(draw): @given(chunks(mutations(h1_requests()))) def test_fuzz_h1_request(data): - tctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts - ) + tctx = _tctx() layer = http.HttpLayer(tctx, HTTPMode.regular) for _ in layer.handle_event(Start()): @@ -148,9 +146,7 @@ def test_fuzz_h1_request(data): @given(chunks(mutations(h2_responses()))) @example([b"0 OK\r\n\r\n", b"\r\n", b"5\r\n12345\r\n0\r\n\r\n"]) def test_fuzz_h1_response(data): - tctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts - ) + tctx = _tctx() server = Placeholder(connection.Server) playbook = Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False) assert ( @@ -276,9 +272,7 @@ def h2_frames(draw): def h2_layer(opts): - tctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts - ) + tctx = _tctx() tctx.options.http2_ping_keepalive = 0 tctx.client.alpn = b"h2" @@ -322,10 +316,15 @@ def test_fuzz_h2_request_mutations(chunks): _h2_request(chunks) -def _h2_response(chunks): - tctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts +def _tctx() -> context.Context: + return context.Context( + connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), + opts ) + + +def _h2_response(chunks): + tctx = _tctx() playbook = Playbook(http.HttpLayer(tctx, HTTPMode.regular), hooks=False) server = Placeholder(connection.Server) assert ( @@ -427,9 +426,7 @@ def _test_cancel(stream_req, stream_resp, draw): """ Test that we don't raise an exception if someone disconnects. """ - tctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts - ) + tctx = _tctx() playbook, cff = start_h2_client(tctx) flow = Placeholder(HTTPFlow) server = Placeholder(Server) diff --git a/test/mitmproxy/proxy/layers/test_modes.py b/test/mitmproxy/proxy/layers/test_modes.py index 3937213751..7fc0215f6b 100644 --- a/test/mitmproxy/proxy/layers/test_modes.py +++ b/test/mitmproxy/proxy/layers/test_modes.py @@ -43,12 +43,12 @@ def test_upstream_https(tctx): curl -x localhost:8080 -k http://example.com """ tctx1 = Context( - Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), + Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), copy.deepcopy(tctx.options), ) tctx1.client.proxy_mode = ProxyMode.parse("upstream:https://example.mitmproxy.org:8081") tctx2 = Context( - Client(("client", 4321), ("127.0.0.1", 8080), 1605699329), + Client(peername=("client", 4321), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), copy.deepcopy(tctx.options), ) assert tctx2.client.proxy_mode == ProxyMode.parse("regular") diff --git a/test/mitmproxy/proxy/layers/test_socks5_fuzz.py b/test/mitmproxy/proxy/layers/test_socks5_fuzz.py index 4e882889a5..bbefa4b01c 100644 --- a/test/mitmproxy/proxy/layers/test_socks5_fuzz.py +++ b/test/mitmproxy/proxy/layers/test_socks5_fuzz.py @@ -8,7 +8,11 @@ from mitmproxy.proxy.layers.modes import Socks5Proxy opts = options.Options() -tctx = Context(Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), opts) +tctx = Context(Client( + peername=("client", 1234), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329 +), opts) @given(binary()) diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 9f1ae5a138..8476894e8e 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -501,7 +501,7 @@ def test_client_only(self, tctx: context.Context): # Echo _test_echo(playbook, tssl_client, tctx.client) - other_server = Server(None) + other_server = Server(address=None) assert ( playbook >> events.DataReceived(other_server, b"Plaintext") @@ -636,7 +636,7 @@ def test_cannot_parse_clienthello(self, tctx: context.Context): client_layer.debug = "" assert ( playbook - >> events.DataReceived(Server(None), b"data on other stream") + >> events.DataReceived(Server(address=None), b"data on other stream") << commands.Log(">> DataReceived(server, b'data on other stream')", DEBUG) << commands.Log( "Swallowing DataReceived(server, b'data on other stream') as handshake failed.", diff --git a/test/mitmproxy/proxy/test_commands.py b/test/mitmproxy/proxy/test_commands.py index c69591e32b..5e48226244 100644 --- a/test/mitmproxy/proxy/test_commands.py +++ b/test/mitmproxy/proxy/test_commands.py @@ -9,7 +9,7 @@ @pytest.fixture def tconn() -> connection.Server: - return connection.Server(None) + return connection.Server(address=None) def test_dataclasses(tconn): diff --git a/test/mitmproxy/proxy/test_events.py b/test/mitmproxy/proxy/test_events.py index 2061f27c8c..c415fadad2 100644 --- a/test/mitmproxy/proxy/test_events.py +++ b/test/mitmproxy/proxy/test_events.py @@ -8,7 +8,7 @@ @pytest.fixture def tconn() -> connection.Server: - return connection.Server(None) + return connection.Server(address=None) def test_dataclasses(tconn): diff --git a/test/mitmproxy/proxy/test_layer.py b/test/mitmproxy/proxy/test_layer.py index 59aebef6dd..1d4baef7e2 100644 --- a/test/mitmproxy/proxy/test_layer.py +++ b/test/mitmproxy/proxy/test_layer.py @@ -51,8 +51,7 @@ def state_bar(self, event: events.Event) -> layer.CommandGenerator[None]: tutils.Playbook(tlayer, hooks=True, logs=True) << commands.Log(" >> Start({})", DEBUG) << commands.Log( - " << OpenConnection({'connection': Server({'id': '…rverid', 'address': None, " - "'state': , 'transport_protocol': 'tcp'})})", + " << OpenConnection({'connection': Server({'id': '…rverid', 'address': None})})", DEBUG, ) << commands.OpenConnection(tctx.server) @@ -60,9 +59,8 @@ def state_bar(self, event: events.Event) -> layer.CommandGenerator[None]: << commands.Log(" >! DataReceived(client, b'foo')", DEBUG) >> tutils.reply(None, to=-3) << commands.Log( - " >> Reply(OpenConnection({'connection': Server(" - "{'id': '…rverid', 'address': None, 'state': , " - "'transport_protocol': 'tcp', 'timestamp_start': 1624544785})}), None)", + " >> Reply(OpenConnection({'connection': Server({'id': '…rverid', 'address': None, " + "'state': , 'timestamp_start': 1624544785})}), None)", DEBUG, ) << commands.Log(" !> DataReceived(client, b'foo')", DEBUG) diff --git a/test/mitmproxy/proxy/test_tunnel.py b/test/mitmproxy/proxy/test_tunnel.py index 24103637c6..060de4f762 100644 --- a/test/mitmproxy/proxy/test_tunnel.py +++ b/test/mitmproxy/proxy/test_tunnel.py @@ -56,7 +56,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: @pytest.mark.parametrize("success", ["success", "fail"]) def test_tunnel_handshake_start(tctx: Context, success): - server = Server(("proxy", 1234)) + server = Server(address=("proxy", 1234)) server.state = ConnectionState.OPEN tl = TTunnelLayer(tctx, server, tctx.server) @@ -87,7 +87,7 @@ def test_tunnel_handshake_start(tctx: Context, success): @pytest.mark.parametrize("success", ["success", "fail"]) def test_tunnel_handshake_command(tctx: Context, success): - server = Server(("proxy", 1234)) + server = Server(address=("proxy", 1234)) tl = TTunnelLayer(tctx, server, tctx.server) tl.child_layer = TChildLayer(tctx) @@ -137,7 +137,7 @@ def test_tunnel_default_impls(tctx: Context): Some tunnels don't need certain features, so the default behaviour should be to be transparent. """ - server = Server(None) + server = Server(address=None) server.state = ConnectionState.OPEN tl = tunnel.TunnelLayer(tctx, server, tctx.server) tl.child_layer = TChildLayer(tctx) @@ -169,7 +169,7 @@ def test_tunnel_default_impls(tctx: Context): def test_tunnel_openconnection_error(tctx: Context): - server = Server(("proxy", 1234)) + server = Server(address=("proxy", 1234)) tl = TTunnelLayer(tctx, server, tctx.server) tl.child_layer = TChildLayer(tctx) @@ -192,7 +192,7 @@ def test_tunnel_openconnection_error(tctx: Context): @pytest.mark.parametrize("disconnect", ["client", "server"]) def test_disconnect_during_handshake_start(tctx: Context, disconnect): - server = Server(("proxy", 1234)) + server = Server(address=("proxy", 1234)) server.state = ConnectionState.OPEN tl = TTunnelLayer(tctx, server, tctx.server) @@ -224,7 +224,7 @@ def test_disconnect_during_handshake_start(tctx: Context, disconnect): @pytest.mark.parametrize("disconnect", ["client", "server"]) def test_disconnect_during_handshake_command(tctx: Context, disconnect): - server = Server(("proxy", 1234)) + server = Server(address=("proxy", 1234)) tl = TTunnelLayer(tctx, server, tctx.server) tl.child_layer = TChildLayer(tctx) diff --git a/test/mitmproxy/test_connection.py b/test/mitmproxy/test_connection.py index 27761e2ac9..9bd041716c 100644 --- a/test/mitmproxy/test_connection.py +++ b/test/mitmproxy/test_connection.py @@ -6,7 +6,7 @@ class TestConnection: def test_basic(self): - c = Client(("127.0.0.1", 52314), ("127.0.0.1", 8080), 1607780791) + c = Client(peername=("127.0.0.1", 52314), sockname=("127.0.0.1", 8080), timestamp_start=1607780791) assert not c.tls_established c.timestamp_tls_setup = 1607780792 assert c.tls_established @@ -28,7 +28,7 @@ def test_eq(self): class TestClient: def test_basic(self): - c = Client(("127.0.0.1", 52314), ("127.0.0.1", 8080), 1607780791) + c = Client(peername=("127.0.0.1", 52314), sockname=("127.0.0.1", 8080), timestamp_start=1607780791) assert repr(c) assert str(c) c.timestamp_tls_setup = 1607780791 @@ -55,7 +55,7 @@ def test_state(self): class TestServer: def test_basic(self): - s = Server(("address", 22)) + s = Server(address=("address", 22)) assert repr(s) assert str(s) s.timestamp_tls_setup = 1607780791 @@ -72,7 +72,7 @@ def test_state(self): assert c2.get_state() == c.get_state() def test_address(self): - s = Server(("address", 22)) + s = Server(address=("address", 22)) s.address = ("example.com", 443) s.state = ConnectionState.OPEN with pytest.raises(RuntimeError): From c2a65c049c300f7fae81848d9e933e5becabb916 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 21 Nov 2022 01:55:40 +0100 Subject: [PATCH 115/695] replace most StateObjects with SerializableDataclass --- mitmproxy/coretypes/serializable.py | 33 +++++++++++++++++----- mitmproxy/dns.py | 44 ++++------------------------- mitmproxy/flow.py | 22 ++++----------- mitmproxy/websocket.py | 24 +++------------- 4 files changed, 40 insertions(+), 83 deletions(-) diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index 9b6f1c2ac5..7049f4ed0c 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -5,6 +5,7 @@ import types import typing import uuid +from functools import cache from typing import TypeVar T = TypeVar("T", bound="Serializable") @@ -44,20 +45,38 @@ def copy(self: T) -> T: return self.from_state(state) +U = TypeVar("U", bound="SerializableDataclass") + + @dataclasses.dataclass class SerializableDataclass(Serializable): + @classmethod + @cache + def __fields(cls) -> tuple[dataclasses.Field, ...]: + # with from __future__ import annotations, `field.type` is a string, + # see https://github.com/python/cpython/issues/83623. + hints = None + fields = [] + for field in dataclasses.fields(cls): + if isinstance(field.type, str): + if hints is None: + hints = typing.get_type_hints(cls) + field.type = hints[field.name] + fields.append(field) + return tuple(fields) + def get_state(self): state = {} - for field in dataclasses.fields(self): + for field in self.__fields(): val = getattr(self, field.name) state[field.name] = _to_state(val, field.type, field.name) return state @classmethod - def from_state(cls: type[T], state) -> T: + def from_state(cls: type[U], state) -> U: # state = state.copy() - for field in dataclasses.fields(cls): + for field in cls.__fields(): try: state_val = state[field.name] except KeyError: @@ -72,7 +91,7 @@ def from_state(cls: type[T], state) -> T: raise ValueError(f"Invalid state for {cls}: {e} ({state=})") from e def set_state(self, state): - for field in dataclasses.fields(self): + for field in self.__fields(): current = getattr(self, field.name) if isinstance(current, Serializable): current.set_state(state.pop(field.name)) @@ -84,10 +103,10 @@ def set_state(self, state): raise RuntimeWarning(f"Unexpected fields in SerializableDataclass.set_state: {state}") -U = TypeVar("U") +V = TypeVar("V") -def _process(attr_val: typing.Any, attr_type: type[U], attr_name: str, make: bool) -> U: +def _process(attr_val: typing.Any, attr_type: type[V], attr_name: str, make: bool) -> V: origin = typing.get_origin(attr_type) if origin is typing.Literal: assert attr_val in typing.get_args(attr_type), "Literal does not match." @@ -140,7 +159,7 @@ def _process(attr_val: typing.Any, attr_type: type[U], attr_name: str, make: boo else: return attr_val.value else: - raise TypeError(f"Unexpected type for {attr_name}: {attr_type}") + raise TypeError(f"Unexpected type for {attr_name}: {attr_type!r}") def _to_val(state: typing.Any, attr_type: type[U], attr_name: str) -> U: diff --git a/mitmproxy/dns.py b/mitmproxy/dns.py index 4bfe25f7b2..74649e1a6d 100644 --- a/mitmproxy/dns.py +++ b/mitmproxy/dns.py @@ -7,26 +7,21 @@ import time from typing import ClassVar -from mitmproxy import flow, stateobject +from mitmproxy import flow +from mitmproxy.coretypes import serializable from mitmproxy.net.dns import classes, domain_names, op_codes, response_codes, types # DNS parameters taken from https://www.iana.org/assignments/dns-parameters/dns-parameters.xml @dataclass -class Question(stateobject.StateObject): +class Question(serializable.SerializableDataclass): HEADER: ClassVar[struct.Struct] = struct.Struct("!HH") name: str type: int class_: int - _stateobject_attributes = dict(name=str, type=int, class_=int) - - @classmethod - def from_state(cls, state): - return cls(**state) - def __str__(self) -> str: return self.name @@ -43,7 +38,7 @@ def to_json(self) -> dict: @dataclass -class ResourceRecord(stateobject.StateObject): +class ResourceRecord(serializable.SerializableDataclass): DEFAULT_TTL: ClassVar[int] = 60 HEADER: ClassVar[struct.Struct] = struct.Struct("!HHIH") @@ -53,12 +48,6 @@ class ResourceRecord(stateobject.StateObject): ttl: int data: bytes - _stateobject_attributes = dict(name=str, type=int, class_=int, ttl=int, data=bytes) - - @classmethod - def from_state(cls, state): - return cls(**state) - def __str__(self) -> str: try: if self.type == types.A: @@ -150,7 +139,7 @@ def TXT(cls, name: str, text: str, *, ttl: int = DEFAULT_TTL) -> ResourceRecord: # comments are taken from rfc1035 @dataclass -class Message(stateobject.StateObject): +class Message(serializable.SerializableDataclass): HEADER: ClassVar[struct.Struct] = struct.Struct("!HHHHHH") timestamp: float @@ -194,29 +183,6 @@ class Message(stateobject.StateObject): additionals: list[ResourceRecord] """Third resource record section.""" - _stateobject_attributes = dict( - timestamp=float, - id=int, - query=bool, - op_code=int, - authoritative_answer=bool, - truncation=bool, - recursion_desired=bool, - recursion_available=bool, - reserved=int, - response_code=int, - questions=list[Question], - answers=list[ResourceRecord], - authorities=list[ResourceRecord], - additionals=list[ResourceRecord], - ) - - @classmethod - def from_state(cls, state): - obj = cls.__new__(cls) # `cls(**state)` won't work recursively - obj.set_state(state) - return obj - def __str__(self) -> str: return "\r\n".join( map( diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index e7ce47ff71..d531aa369d 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -2,15 +2,18 @@ import copy import time import uuid +from dataclasses import dataclass, field from typing import Any, ClassVar, Optional from mitmproxy import connection from mitmproxy import exceptions from mitmproxy import stateobject from mitmproxy import version +from mitmproxy.coretypes import serializable -class Error(stateobject.StateObject): +@dataclass +class Error(serializable.SerializableDataclass): """ An Error. @@ -23,32 +26,17 @@ class Error(stateobject.StateObject): msg: str """Message describing the error.""" - timestamp: float + timestamp: float = field(default_factory=time.time) """Unix timestamp of when this error happened.""" KILLED_MESSAGE: ClassVar[str] = "Connection killed." - def __init__(self, msg: str, timestamp: Optional[float] = None) -> None: - """Create an error. If no timestamp is passed, the current time is used.""" - self.msg = msg - self.timestamp = timestamp or time.time() - - _stateobject_attributes = dict(msg=str, timestamp=float) - def __str__(self): return self.msg def __repr__(self): return self.msg - @classmethod - def from_state(cls, state): - # the default implementation assumes an empty constructor. Override - # accordingly. - f = cls("") - f.set_state(state) - return f - class Flow(stateobject.StateObject): """ diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 7cac0a712c..6f301922c4 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -7,10 +7,10 @@ """ import time import warnings +from dataclasses import dataclass, field from typing import Union from typing import Optional -from mitmproxy import stateobject from mitmproxy.coretypes import serializable from wsproto.frame_protocol import Opcode @@ -144,13 +144,14 @@ def text(self, value: str) -> None: self.content = value.encode() -class WebSocketData(stateobject.StateObject): +@dataclass +class WebSocketData(serializable.SerializableDataclass): """ A data container for everything related to a single WebSocket connection. This is typically accessed as `mitmproxy.http.HTTPFlow.websocket`. """ - messages: list[WebSocketMessage] + messages: list[WebSocketMessage] = field(default_factory=list) """All `WebSocketMessage`s transferred over this connection.""" closed_by_client: Optional[bool] = None @@ -167,22 +168,5 @@ class WebSocketData(stateobject.StateObject): timestamp_end: Optional[float] = None """*Timestamp:* WebSocket connection closed.""" - _stateobject_attributes = dict( - messages=list[WebSocketMessage], - closed_by_client=bool, - close_code=int, - close_reason=str, - timestamp_end=float, - ) - - def __init__(self): - self.messages = [] - def __repr__(self): return f"" - - @classmethod - def from_state(cls, state): - d = WebSocketData() - d.set_state(state) - return d From c1d0385782f0c4b53d67054030b0f4d68b66bbc3 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 21 Nov 2022 02:48:21 +0100 Subject: [PATCH 116/695] tests++ --- mitmproxy/coretypes/serializable.py | 21 ++++---- mitmproxy/io/compat.py | 5 +- test/mitmproxy/coretypes/test_serializable.py | 48 ++++++++++++++++++- test/mitmproxy/io/test_compat.py | 1 + 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index 7049f4ed0c..0fd2ef4c22 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -77,14 +77,7 @@ def get_state(self): def from_state(cls: type[U], state) -> U: # state = state.copy() for field in cls.__fields(): - try: - state_val = state[field.name] - except KeyError: - if field.default is dataclasses.MISSING: - raise ValueError(f"Missing state attribute: {field.name}") - else: - continue - state[field.name] = _to_val(state_val, field.type, field.name) + state[field.name] = _to_val(state[field.name], field.type, field.name) try: return cls(**state) # type: ignore except TypeError as e: @@ -93,14 +86,15 @@ def from_state(cls: type[U], state) -> U: def set_state(self, state): for field in self.__fields(): current = getattr(self, field.name) - if isinstance(current, Serializable): - current.set_state(state.pop(field.name)) + f_state = state.pop(field.name) + if isinstance(current, Serializable) and f_state is not None: + current.set_state(f_state) else: - val = _to_val(state.pop(field.name), field.type, field.name) + val = _to_val(f_state, field.type, field.name) setattr(self, field.name, val) if state: - raise RuntimeWarning(f"Unexpected fields in SerializableDataclass.set_state: {state}") + raise ValueError(f"Unexpected fields in {type(self).__name__}.set_state: {state}") V = TypeVar("V") @@ -109,7 +103,8 @@ def set_state(self, state): def _process(attr_val: typing.Any, attr_type: type[V], attr_name: str, make: bool) -> V: origin = typing.get_origin(attr_type) if origin is typing.Literal: - assert attr_val in typing.get_args(attr_type), "Literal does not match." + if attr_val not in typing.get_args(attr_type): + raise ValueError(f"Invalid value for {attr_name}: {attr_val!r} does not match any literal value.") return attr_val if origin in (types.UnionType, typing.Union): attr_type, nt = typing.get_args(attr_type) diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index bed19945ec..7b571ad902 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -399,11 +399,10 @@ def convert_18_19(data): data["server_conn"]["via"] = data["server_conn"].pop("via2", None) for conn in ["client_conn", "server_conn"]: - if data[conn].get("timestamp_tls_setup") is None and data[conn].get("tls_established"): - data[conn]["timestamp_tls_setup"] = 1.0 data[conn].pop("tls_established") data[conn]["cipher"] = data[conn].pop("cipher_name", None) + data[conn].setdefault("transport_protocol", "tcp") for name in ["peername", "sockname", "address"]: if data[conn].get(name) and isinstance(data[conn][name][0], bytes): @@ -411,8 +410,6 @@ def convert_18_19(data): if data["server_conn"]["sni"] is True: data["server_conn"]["sni"] = data["server_conn"]["address"][0] - if data["server_conn"]["sni"] is False: - data["server_conn"]["sni"] = None return data diff --git a/test/mitmproxy/coretypes/test_serializable.py b/test/mitmproxy/coretypes/test_serializable.py index 068d4e6a18..4295dae1af 100644 --- a/test/mitmproxy/coretypes/test_serializable.py +++ b/test/mitmproxy/coretypes/test_serializable.py @@ -1,6 +1,10 @@ +from __future__ import annotations + import copy import enum +from collections.abc import Mapping from dataclasses import dataclass +from typing import Literal import pytest @@ -63,6 +67,11 @@ class TEnum(enum.Enum): B = 2 +@dataclass +class TLiteral(SerializableDataclass): + l: Literal["foo", "bar"] + + @dataclass class BuiltinChildren(SerializableDataclass): a: list[int] | None @@ -77,6 +86,16 @@ class Defaults(SerializableDataclass): z: int | None = 42 +@dataclass +class Unsupported(SerializableDataclass): + a: Mapping[str, int] + + +@dataclass +class Addr(SerializableDataclass): + peername: tuple[str, int] + + class TestSerializableDataclass: @pytest.mark.parametrize("cls, state", [ (Simple, {"x": 42, "y": 'foo'}), @@ -86,27 +105,52 @@ class TestSerializableDataclass: (Inheritance, {"x": 42, "y": "foo", "z": True}), (BuiltinChildren, {"a": [1, 2, 3], "b": {"foo": 42}, "c": (1, 2), "d": [{"x": 42, "y": "foo"}], "e": 1}), (BuiltinChildren, {"a": None, "b": None, "c": None, "d": [], "e": None}), + (TLiteral, {"l": "foo"}), ]) def test_roundtrip(self, cls, state): a = cls.from_state(copy.deepcopy(state)) assert a.get_state() == state + def test_set(self): + s = SerializableChild(foo=Simple(x=42, y=None), maybe_foo=Simple(x=43, y=None)) + s.set_state({"foo": {"x": 44, "y": None}, "maybe_foo": None}) + assert s.foo.x == 44 + assert s.maybe_foo is None + with pytest.raises(ValueError, match="Unexpected fields"): + Simple(0, "").set_state({"x": 42, "y": "foo", "z": True}) + + def test_invalid_none(self): with pytest.raises(ValueError): Simple.from_state({"x": None, "y": "foo"}) def test_defaults(self): - a = Defaults.from_state({}) + a = Defaults() assert a.get_state() == {"z": 42} def test_invalid_type(self): with pytest.raises(ValueError): Simple.from_state({"x": 42, "y": 42}) + with pytest.raises(ValueError): + BuiltinChildren.from_state({"a": None, "b": None, "c": ("foo",), "d": [], "e": None}) def test_invalid_key(self): with pytest.raises(ValueError): Simple.from_state({"x": 42, "y": "foo", "z": True}) def test_invalid_type_in_list(self): + with pytest.raises(ValueError, match="Invalid value for x"): + BuiltinChildren.from_state({"a": None, "b": None, "c": None, "d": [{"x": "foo", "y": "foo"}], "e": None}) + + def test_unsupported_type(self): + with pytest.raises(TypeError): + Unsupported.from_state({"a": "foo"}) + + def test_literal(self): + assert TLiteral.from_state({"l": "foo"}).get_state() == {"l": "foo"} with pytest.raises(ValueError): - BuiltinChildren.from_state({"w": [{"x": "foo", "y": "foo"}], "x": None, "y": None, "z": None}) + TLiteral.from_state({"l": "unknown"}) + + def test_peername(self): + assert Addr.from_state({"peername": ("addr", 42)}).get_state() == {"peername": ("addr", 42)} + assert Addr.from_state({"peername": ("addr", 42, 0, 0)}).get_state() == {"peername": ("addr", 42, 0, 0)} diff --git a/test/mitmproxy/io/test_compat.py b/test/mitmproxy/io/test_compat.py index e19c8909b8..85ba5a0eea 100644 --- a/test/mitmproxy/io/test_compat.py +++ b/test/mitmproxy/io/test_compat.py @@ -11,6 +11,7 @@ ["dumpfile-018.mitm", "https://www.example.com/", 1], ["dumpfile-019.mitm", "https://webrv.rtb-seller.com/", 1], ["dumpfile-7-websocket.mitm", "https://echo.websocket.org/", 6], + ["dumpfile-7.mitm", "https://example.com/", 2], ["dumpfile-10.mitm", "https://example.com/", 1], ], ) From cb2eade466ab8ea09ce7a408c899a1819067eb8c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 21 Nov 2022 03:03:49 +0100 Subject: [PATCH 117/695] fix Python 3.9 compatibility --- mitmproxy/connection.py | 34 +++++++++++++------ mitmproxy/coretypes/serializable.py | 12 +++++-- test/mitmproxy/coretypes/test_serializable.py | 17 +++++----- test/mitmproxy/test_connection.py | 7 +++- 4 files changed, 46 insertions(+), 24 deletions(-) diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index 278a7677ed..a6f9be704c 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -1,4 +1,5 @@ import dataclasses +import sys import time from dataclasses import dataclass, field import uuid @@ -32,9 +33,14 @@ class ConnectionState(Flag): # this version at least provides useful type checking messages. Address = tuple[str, int] +if sys.version_info < (3, 10): # pragma: no cover + kw_only = {} +else: + kw_only = {"kw_only": True} + # noinspection PyDataclass -@dataclass(kw_only=True) +@dataclass(**kw_only) class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): """ Base class for client and server connections. @@ -42,20 +48,21 @@ class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): The connection object only exposes metadata about the connection, but not the underlying socket object. This is intentional, all I/O should be handled by `mitmproxy.proxy.server` exclusively. """ + peername: Optional[Address] + """The remote's `(ip, port)` tuple for this connection.""" + sockname: Optional[Address] + """Our local `(ip, port)` tuple for this connection.""" + + state: ConnectionState + """The current connection state.""" # all connections have a unique id. While # f.client_conn == f2.client_conn already holds true for live flows (where we have object identity), # we also want these semantics for recorded flows. id: str = field(default_factory=lambda: str(uuid.uuid4())) """A unique UUID to identify the connection.""" - state: ConnectionState - """The current connection state.""" transport_protocol: TransportProtocol = field(default="tcp") """The connection protocol in use.""" - peername: Optional[Address] - """The remote's `(ip, port)` tuple for this connection.""" - sockname: Optional[Address] - """Our local `(ip, port)` tuple for this connection.""" error: Optional[str] = None """ A string describing a general error with connections to this address. @@ -156,7 +163,7 @@ def alpn_proto_negotiated(self) -> Optional[bytes]: # pragma: no cover # noinspection PyDataclass -@dataclass(kw_only=True, eq=False, repr=False) +@dataclass(eq=False, repr=False, **kw_only) class Client(Connection): """A connection between a client and mitmproxy.""" @@ -243,15 +250,20 @@ def clientcert(self, val): # pragma: no cover # noinspection PyDataclass -@dataclass(kw_only=True, eq=False, repr=False) +@dataclass(eq=False, repr=False, **kw_only) class Server(Connection): """A connection between mitmproxy and an upstream server.""" + address: Optional[Address] # type: ignore + """The server's `(host, port)` address tuple. The host can either be a domain or a plain IP address.""" + + if sys.version_info < (3, 10): # pragma: no cover + # no keyword-only arguments here. + address: Optional[Address] = None + peername: Optional[Address] = None """The server's resolved `(ip, port)` tuple. Will be set during connection establishment.""" sockname: Optional[Address] = None - address: Optional[Address] - """The server's `(host, port)` address tuple. The host can either be a domain or a plain IP address.""" state: ConnectionState = field(default=ConnectionState.CLOSED) diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index 0fd2ef4c22..7161f9cd71 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -2,12 +2,18 @@ import collections.abc import dataclasses import enum -import types import typing import uuid from functools import cache from typing import TypeVar +try: + from types import UnionType, NoneType +except ImportError: # pragma: no cover + class UnionType: # type: ignore + pass + NoneType = type(None) # type: ignore + T = TypeVar("T", bound="Serializable") @@ -106,9 +112,9 @@ def _process(attr_val: typing.Any, attr_type: type[V], attr_name: str, make: boo if attr_val not in typing.get_args(attr_type): raise ValueError(f"Invalid value for {attr_name}: {attr_val!r} does not match any literal value.") return attr_val - if origin in (types.UnionType, typing.Union): + if origin in (UnionType, typing.Union): attr_type, nt = typing.get_args(attr_type) - assert nt is types.NoneType, f"{attr_name}: only `x | None` union types are supported`" # noqa + assert nt is NoneType, f"{attr_name}: only `x | None` union types are supported`" # noqa if attr_val is None: return None # type: ignore else: diff --git a/test/mitmproxy/coretypes/test_serializable.py b/test/mitmproxy/coretypes/test_serializable.py index 4295dae1af..f8f55499ff 100644 --- a/test/mitmproxy/coretypes/test_serializable.py +++ b/test/mitmproxy/coretypes/test_serializable.py @@ -4,7 +4,7 @@ import enum from collections.abc import Mapping from dataclasses import dataclass -from typing import Literal +from typing import Literal, Optional import pytest @@ -48,13 +48,13 @@ def test_copy_id(self): @dataclass class Simple(SerializableDataclass): x: int - y: str | None + y: Optional[str] @dataclass class SerializableChild(SerializableDataclass): foo: Simple - maybe_foo: Simple | None + maybe_foo: Optional[Simple] @dataclass @@ -74,16 +74,16 @@ class TLiteral(SerializableDataclass): @dataclass class BuiltinChildren(SerializableDataclass): - a: list[int] | None - b: dict[str, int] | None - c: tuple[int, int] | None + a: Optional[list[int]] + b: Optional[dict[str, int]] + c: Optional[tuple[int, int]] d: list[Simple] - e: TEnum | None + e: Optional[TEnum] @dataclass class Defaults(SerializableDataclass): - z: int | None = 42 + z: Optional[int] = 42 @dataclass @@ -119,7 +119,6 @@ def test_set(self): with pytest.raises(ValueError, match="Unexpected fields"): Simple(0, "").set_state({"x": 42, "y": "foo", "z": True}) - def test_invalid_none(self): with pytest.raises(ValueError): Simple.from_state({"x": None, "y": "foo"}) diff --git a/test/mitmproxy/test_connection.py b/test/mitmproxy/test_connection.py index 9bd041716c..5bc0aeeb98 100644 --- a/test/mitmproxy/test_connection.py +++ b/test/mitmproxy/test_connection.py @@ -28,7 +28,12 @@ def test_eq(self): class TestClient: def test_basic(self): - c = Client(peername=("127.0.0.1", 52314), sockname=("127.0.0.1", 8080), timestamp_start=1607780791) + c = Client( + peername=("127.0.0.1", 52314), + sockname=("127.0.0.1", 8080), + timestamp_start=1607780791, + cipher_list=["foo", "bar"] + ) assert repr(c) assert str(c) c.timestamp_tls_setup = 1607780791 From 07a40208a32cb2d48a1f2a24d2569894b5a378a0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 22 Nov 2022 00:37:42 +0100 Subject: [PATCH 118/695] `rm -rf stateobject` --- mitmproxy/coretypes/serializable.py | 31 +++-- mitmproxy/dns.py | 14 +- mitmproxy/flow.py | 78 ++++++----- mitmproxy/http.py | 18 ++- mitmproxy/proxy/mode_specs.py | 3 +- mitmproxy/stateobject.py | 95 ------------- mitmproxy/tcp.py | 11 +- mitmproxy/udp.py | 11 +- test/mitmproxy/coretypes/test_serializable.py | 18 +++ test/mitmproxy/proxy/test_mode_specs.py | 4 +- test/mitmproxy/test_http.py | 4 +- test/mitmproxy/test_stateobject.py | 126 ------------------ 12 files changed, 134 insertions(+), 279 deletions(-) delete mode 100644 mitmproxy/stateobject.py delete mode 100644 test/mitmproxy/test_stateobject.py diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index 7161f9cd71..f3911b6c92 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -16,6 +16,8 @@ class UnionType: # type: ignore T = TypeVar("T", bound="Serializable") +State = typing.Any + class Serializable(metaclass=abc.ABCMeta): """ @@ -27,11 +29,12 @@ class Serializable(metaclass=abc.ABCMeta): def from_state(cls: type[T], state) -> T: """ Create a new object from the given state. + Consumes the passed state. """ raise NotImplementedError() @abc.abstractmethod - def get_state(self): + def get_state(self) -> State: """ Retrieve object state. """ @@ -40,7 +43,8 @@ def get_state(self): @abc.abstractmethod def set_state(self, state): """ - Set object state to the given state. + Set object state to the given state. Consumes the passed state. + May return a `dataclasses.FrozenInstanceError` if the object is immutable. """ raise NotImplementedError() @@ -54,7 +58,6 @@ def copy(self: T) -> T: U = TypeVar("U", bound="SerializableDataclass") -@dataclasses.dataclass class SerializableDataclass(Serializable): @classmethod @@ -62,17 +65,16 @@ class SerializableDataclass(Serializable): def __fields(cls) -> tuple[dataclasses.Field, ...]: # with from __future__ import annotations, `field.type` is a string, # see https://github.com/python/cpython/issues/83623. - hints = None + hints = typing.get_type_hints(cls) fields = [] + # noinspection PyDataclass for field in dataclasses.fields(cls): if isinstance(field.type, str): - if hints is None: - hints = typing.get_type_hints(cls) field.type = hints[field.name] fields.append(field) return tuple(fields) - def get_state(self): + def get_state(self) -> State: state = {} for field in self.__fields(): val = getattr(self, field.name) @@ -89,15 +91,22 @@ def from_state(cls: type[U], state) -> U: except TypeError as e: raise ValueError(f"Invalid state for {cls}: {e} ({state=})") from e - def set_state(self, state): + def set_state(self, state: State) -> None: for field in self.__fields(): current = getattr(self, field.name) f_state = state.pop(field.name) if isinstance(current, Serializable) and f_state is not None: - current.set_state(f_state) - else: - val = _to_val(f_state, field.type, field.name) + try: + current.set_state(f_state) + continue + except dataclasses.FrozenInstanceError: + pass + val = _to_val(f_state, field.type, field.name) + try: setattr(self, field.name, val) + except dataclasses.FrozenInstanceError: + state[field.name] = f_state # restore state dict. + raise if state: raise ValueError(f"Unexpected fields in {type(self).__name__}.set_state: {state}") diff --git a/mitmproxy/dns.py b/mitmproxy/dns.py index 74649e1a6d..8ccd48c05e 100644 --- a/mitmproxy/dns.py +++ b/mitmproxy/dns.py @@ -431,9 +431,17 @@ class DNSFlow(flow.Flow): response: Message | None = None """The DNS response.""" - _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes["request"] = Message - _stateobject_attributes["response"] = Message + def get_state(self) -> serializable.State: + return { + **super().get_state(), + "request": self.request.get_state(), + "response": self.response.get_state() if self.response else None, + } + + def set_state(self, state: serializable.State) -> None: + self.request = Message.from_state(state.pop("request")) + self.response = Message.from_state(r) if (r := state.pop("response")) else None + super().set_state(state) def __repr__(self) -> str: return f"" diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index d531aa369d..69514f2981 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -1,3 +1,4 @@ +from __future__ import annotations import asyncio import copy import time @@ -7,7 +8,6 @@ from mitmproxy import connection from mitmproxy import exceptions -from mitmproxy import stateobject from mitmproxy import version from mitmproxy.coretypes import serializable @@ -38,7 +38,7 @@ def __repr__(self): return self.msg -class Flow(stateobject.StateObject): +class Flow(serializable.Serializable): """ Base class for network flows. A flow is a collection of objects, for example HTTP request/response pairs or a list of TCP messages. @@ -126,20 +126,7 @@ def __init__( self.metadata: dict[str, Any] = dict() self.comment: str = "" - _stateobject_attributes = dict( - id=str, - error=Error, - client_conn=connection.Client, - server_conn=connection.Server, - intercepted=bool, - is_replay=str, - marked=str, - metadata=dict[str, Any], - comment=str, - timestamp_created=float, - ) - - __types: dict[str, type["Flow"]] = {} + __types: dict[str, type[Flow]] = {} type: ClassVar[str] # automatically derived from the class name in __init_subclass__ """The flow type, for example `http`, `tcp`, or `dns`.""" @@ -148,28 +135,55 @@ def __init_subclass__(cls, **kwargs): cls.type = cls.__name__.removesuffix("Flow").lower() Flow.__types[cls.type] = cls - def get_state(self): - d = super().get_state() - d.update(version=version.FLOW_FORMAT_VERSION, type=self.type) - if self._backup and self._backup != d: - d.update(backup=copy.deepcopy(self._backup)) - return d - - def set_state(self, state): - state = state.copy() - state.pop("version") - state.pop("type") - if "backup" in state: - self._backup = state.pop("backup") - super().set_state(state) + def get_state(self) -> serializable.State: + state = { + "version": version.FLOW_FORMAT_VERSION, + "type": self.type, + "id": self.id, + "error": self.error.get_state() if self.error else None, + "client_conn": self.client_conn.get_state(), + "server_conn": self.server_conn.get_state(), + "intercepted": self.intercepted, + "is_replay": self.is_replay, + "marked": self.marked, + "metadata": copy.deepcopy(self.metadata), + "comment": self.comment, + "timestamp_created": self.timestamp_created, + } + state["backup"] = copy.deepcopy(self._backup) if self._backup != state else None + return state + + def set_state(self, state: serializable.State) -> None: + assert state.pop("version") == version.FLOW_FORMAT_VERSION + assert state.pop("type") == self.type + self.id = state.pop("id") + if state["error"]: + if self.error: + self.error.set_state(state.pop("error")) + else: + self.error = Error.from_state(state.pop("error")) + else: + self.error = state.pop("error") + self.client_conn.set_state(state.pop("client_conn")) + self.server_conn.set_state(state.pop("server_conn")) + self.intercepted = state.pop("intercepted") + self.is_replay = state.pop("is_replay") + self.marked = state.pop("marked") + self.metadata = state.pop("metadata") + self.comment = state.pop("comment") + self.timestamp_created = state.pop("timestamp_created") + self._backup = state.pop("backup", None) + assert state == {} @classmethod - def from_state(cls, state): + def from_state(cls, state: serializable.State) -> Flow: try: flow_cls = Flow.__types[state["type"]] except KeyError: raise ValueError(f"Unknown flow type: {state['type']}") - f = flow_cls(None, None) # type: ignore + client = connection.Client(peername=("", 0), sockname=("", 0)) + server = connection.Server(address=None) + f = flow_cls(client, server) f.set_state(state) return f diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 14e024462f..0a3440f329 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -1247,11 +1247,19 @@ class HTTPFlow(flow.Flow): If this HTTP flow initiated a WebSocket connection, this attribute contains all associated WebSocket data. """ - _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - # mypy doesn't support update with kwargs - _stateobject_attributes.update( - dict(request=Request, response=Response, websocket=WebSocketData) - ) + def get_state(self) -> serializable.State: + return { + **super().get_state(), + "request": self.request.get_state(), + "response": self.response.get_state() if self.response else None, + "websocket": self.websocket.get_state() if self.websocket else None, + } + + def set_state(self, state: serializable.State) -> None: + self.request = Request.from_state(state.pop("request")) + self.response = Response.from_state(r) if (r := state.pop("response")) else None + self.websocket = WebSocketData.from_state(w) if (w := state.pop("websocket")) else None + super().set_state(state) def __repr__(self): s = " class-or-type dict containing all attributes that - should be serialized. If the attribute is a class, it must implement the - Serializable protocol. - """ - - def get_state(self): - """ - Retrieve object state. - """ - state = {} - for attr, cls in self._stateobject_attributes.items(): - val = getattr(self, attr) - state[attr] = get_state(cls, val) - return state - - def set_state(self, state): - """ - Load object state from data returned by a get_state call. - """ - state = state.copy() - for attr, cls in self._stateobject_attributes.items(): - val = state.pop(attr) - if val is None: - setattr(self, attr, val) - else: - curr = getattr(self, attr, None) - if curr is not None and hasattr(curr, "set_state"): - curr.set_state(val) - else: - setattr(self, attr, make_object(cls, val)) - if state: - raise RuntimeWarning(f"Unexpected State in __setstate__: {state}") - - -def _process(typeinfo: typecheck.Type, val: typing.Any, make: bool) -> typing.Any: - if val is None: - return None - elif make and hasattr(typeinfo, "from_state"): - return typeinfo.from_state(val) - elif not make and hasattr(val, "get_state"): - return val.get_state() - - origin = typing.get_origin(typeinfo) - - if origin is list: - T = typing.get_args(typeinfo)[0] - return [_process(T, x, make) for x in val] - elif origin is tuple: - Ts = typing.get_args(typeinfo) - if len(Ts) != len(val): - raise ValueError(f"Invalid data. Expected {Ts}, got {val}.") - return tuple(_process(T, x, make) for T, x in zip(Ts, val)) - elif origin is dict: - k_cls, v_cls = typing.get_args(typeinfo) - return { - _process(k_cls, k, make): _process(v_cls, v, make) for k, v in val.items() - } - elif typeinfo is typing.Any: - # This requires a bit of explanation. We can't import our IO layer here, - # because it causes a circular import. Rather than restructuring the - # code for this, we use JSON serialization, which has similar primitive - # type restrictions as tnetstring, to check for conformance. - try: - json.dumps(val) - except TypeError: - raise ValueError(f"Data not serializable: {val}") - return val - else: - return typeinfo(val) - - -def make_object(typeinfo: typecheck.Type, val: typing.Any) -> typing.Any: - """Create an object based on the state given in val.""" - return _process(typeinfo, val, True) - - -def get_state(typeinfo: typecheck.Type, val: typing.Any) -> typing.Any: - """Get the state of the object given as val.""" - return _process(typeinfo, val, False) diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py index a2512cfb83..2ad13336c4 100644 --- a/mitmproxy/tcp.py +++ b/mitmproxy/tcp.py @@ -54,8 +54,15 @@ def __init__( super().__init__(client_conn, server_conn, live) self.messages = [] - _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes["messages"] = list[TCPMessage] + def get_state(self) -> serializable.State: + return { + **super().get_state(), + "messages": [m.get_state() for m in self.messages], + } + + def set_state(self, state: serializable.State) -> None: + self.messages = [TCPMessage.from_state(m) for m in state.pop("messages")] + super().set_state(state) def __repr__(self): return f"" diff --git a/mitmproxy/udp.py b/mitmproxy/udp.py index accc294d41..e719de0ac1 100644 --- a/mitmproxy/udp.py +++ b/mitmproxy/udp.py @@ -51,8 +51,15 @@ def __init__( super().__init__(client_conn, server_conn, live) self.messages = [] - _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes["messages"] = list[UDPMessage] + def get_state(self) -> serializable.State: + return { + **super().get_state(), + "messages": [m.get_state() for m in self.messages], + } + + def set_state(self, state: serializable.State) -> None: + self.messages = [UDPMessage.from_state(m) for m in state.pop("messages")] + super().set_state(state) def __repr__(self): return f"" diff --git a/test/mitmproxy/coretypes/test_serializable.py b/test/mitmproxy/coretypes/test_serializable.py index f8f55499ff..70980a625e 100644 --- a/test/mitmproxy/coretypes/test_serializable.py +++ b/test/mitmproxy/coretypes/test_serializable.py @@ -1,6 +1,7 @@ from __future__ import annotations import copy +import dataclasses import enum from collections.abc import Mapping from dataclasses import dataclass @@ -96,6 +97,16 @@ class Addr(SerializableDataclass): peername: tuple[str, int] +@dataclass(frozen=True) +class Frozen(SerializableDataclass): + x: int + + +@dataclass +class FrozenWrapper(SerializableDataclass): + f: Frozen + + class TestSerializableDataclass: @pytest.mark.parametrize("cls, state", [ (Simple, {"x": 42, "y": 'foo'}), @@ -153,3 +164,10 @@ def test_literal(self): def test_peername(self): assert Addr.from_state({"peername": ("addr", 42)}).get_state() == {"peername": ("addr", 42)} assert Addr.from_state({"peername": ("addr", 42, 0, 0)}).get_state() == {"peername": ("addr", 42, 0, 0)} + + def test_set_immutable(self): + w = FrozenWrapper(Frozen(42)) + with pytest.raises(dataclasses.FrozenInstanceError): + w.f.set_state({"x": 43}) + w.set_state({"f": {"x": 43}}) + assert w.f.x == 43 diff --git a/test/mitmproxy/proxy/test_mode_specs.py b/test/mitmproxy/proxy/test_mode_specs.py index e2e7eecf93..06dd45aae8 100644 --- a/test/mitmproxy/proxy/test_mode_specs.py +++ b/test/mitmproxy/proxy/test_mode_specs.py @@ -1,3 +1,5 @@ +import dataclasses + import pytest from mitmproxy.proxy.mode_specs import ProxyMode, Socks5Mode @@ -24,7 +26,7 @@ def test_parse(): ProxyMode.parse("regular@99999") m.set_state(m.get_state()) - with pytest.raises(RuntimeError, match="Proxy modes are frozen"): + with pytest.raises(dataclasses.FrozenInstanceError): m.set_state("regular") diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index c44bde049a..169aafd71d 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -931,7 +931,9 @@ def test_serializable(self): resp = tresp() resp.trailers = Headers() resp2 = Response.from_state(resp.get_state()) - assert resp.data == resp2.data + resp3 = tresp() + resp3.set_state(resp.get_state()) + assert resp.data == resp2.data == resp3.data def test_content_length_update(self): resp = tresp() diff --git a/test/mitmproxy/test_stateobject.py b/test/mitmproxy/test_stateobject.py deleted file mode 100644 index 8c7147de51..0000000000 --- a/test/mitmproxy/test_stateobject.py +++ /dev/null @@ -1,126 +0,0 @@ -from typing import Any - -import pytest - -from mitmproxy.stateobject import StateObject - - -class TObject(StateObject): - def __init__(self, x): - self.x = x - - @classmethod - def from_state(cls, state): - obj = cls(None) - obj.set_state(state) - return obj - - -class Child(TObject): - _stateobject_attributes = dict(x=int) - - def __eq__(self, other): - return isinstance(other, Child) and self.x == other.x - - -class TTuple(TObject): - _stateobject_attributes = dict(x=tuple[int, Child]) - - -class TList(TObject): - _stateobject_attributes = dict(x=list[Child]) - - -class TDict(TObject): - _stateobject_attributes = dict(x=dict[str, Child]) - - -class TAny(TObject): - _stateobject_attributes = dict(x=Any) - - -class TSerializableChild(TObject): - _stateobject_attributes = dict(x=Child) - - -def test_simple(): - a = Child(42) - assert a.get_state() == {"x": 42} - b = a.copy() - a.set_state({"x": 44}) - assert a.x == 44 - assert b.x == 42 - - -def test_serializable_child(): - child = Child(42) - a = TSerializableChild(child) - assert a.get_state() == {"x": {"x": 42}} - a.set_state({"x": {"x": 43}}) - assert a.x.x == 43 - assert a.x is child - b = a.copy() - assert a.x == b.x - assert a.x is not b.x - - -def test_tuple(): - a = TTuple((42, Child(43))) - assert a.get_state() == {"x": (42, {"x": 43})} - b = a.copy() - a.set_state({"x": (44, {"x": 45})}) - assert a.x == (44, Child(45)) - assert b.x == (42, Child(43)) - - -def test_tuple_err(): - a = TTuple(None) - with pytest.raises(ValueError, match="Invalid data"): - a.set_state({"x": (42,)}) - - -def test_list(): - a = TList([Child(1), Child(2)]) - assert a.get_state() == { - "x": [{"x": 1}, {"x": 2}], - } - copy = a.copy() - assert len(copy.x) == 2 - assert copy.x is not a.x - assert copy.x[0] is not a.x[0] - - -def test_dict(): - a = TDict({"foo": Child(42)}) - assert a.get_state() == {"x": {"foo": {"x": 42}}} - b = a.copy() - assert list(a.x.items()) == list(b.x.items()) - assert a.x is not b.x - assert a.x["foo"] is not b.x["foo"] - - -def test_any(): - a = TAny(42) - b = a.copy() - assert a.x == b.x - - a = TAny(object()) - with pytest.raises(ValueError): - a.get_state() - - -def test_too_much_state(): - a = Child(42) - s = a.get_state() - s["foo"] = "bar" - - with pytest.raises(RuntimeWarning): - a.set_state(s) - - -def test_none(): - a = Child(None) - assert a.get_state() == {"x": None} - a = Child(42) - a.set_state({"x": None}) - assert a.x is None From e24f93e263743d101190826049f0f2ef372bad7c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 22 Nov 2022 13:17:05 +1300 Subject: [PATCH 119/695] [requires.io] dependency update on main branch (#5734) * [requires.io] dependency update * [requires.io] dependency update * [requires.io] dependency update Co-authored-by: requires.io --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 98ff06b338..ed7e548d2d 100644 --- a/tox.ini +++ b/tox.ini @@ -29,11 +29,11 @@ commands = [testenv:mypy] deps = - mypy==0.990 + mypy==0.991 types-certifi==2021.10.8.3 types-Flask==1.1.6 types-Werkzeug==1.0.9 - types-requests==2.28.11.4 + types-requests==2.28.11.5 types-cryptography==3.3.23.2 types-pyOpenSSL==22.1.0.2 -e .[dev] From b9dc95cb8c3e2ea65800a0f01d9348d056dd32f6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 22 Nov 2022 14:04:06 +0100 Subject: [PATCH 120/695] fix a race condition in `ConnectionHandler.drain_writers` (#5749) --- mitmproxy/proxy/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 2ee5fd0796..39b598de95 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -307,7 +307,7 @@ async def drain_writers(self): write buffers, so if we cannot write fast enough our own read buffers run full and the TCP recv stream is throttled. """ async with self._drain_lock: - for transport in self.transports.values(): + for transport in list(self.transports.values()): if transport.writer is not None: try: await transport.writer.drain() From 8f845191f58a4381fa6aa21335e8993b6360c159 Mon Sep 17 00:00:00 2001 From: Mark Storus Date: Tue, 22 Nov 2022 18:29:59 -0800 Subject: [PATCH 121/695] fix Wireshark TLS (Pre)-Master-Secret link (#5752) --- docs/src/content/howto-wireshark-tls.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md index 6b8dc890a7..d6784432b0 100644 --- a/docs/src/content/howto-wireshark-tls.md +++ b/docs/src/content/howto-wireshark-tls.md @@ -9,7 +9,7 @@ menu: The SSL/TLS master keys can be logged by mitmproxy so that external programs can decrypt SSL/TLS connections both from and to the proxy. Recent versions of -Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. +Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/TLS#using-the-pre-master-secret) for more information. Key logging is enabled by setting the environment variable `SSLKEYLOGFILE` so that it points to a writable text file: From 7285c250ff1b7e4c39ee6e1bab4dc1c11540331f Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 24 Nov 2022 04:01:59 +0100 Subject: [PATCH 122/695] [quic] more h3 tests and http1 stream_id fix --- mitmproxy/proxy/layers/http/_http1.py | 6 +- mitmproxy/proxy/layers/quic.py | 4 +- .../mitmproxy/proxy/layers/http/test_http3.py | 252 +++++++++++++++++- 3 files changed, 251 insertions(+), 11 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http1.py b/mitmproxy/proxy/layers/http/_http1.py index e645b54695..c7c79cac87 100644 --- a/mitmproxy/proxy/layers/http/_http1.py +++ b/mitmproxy/proxy/layers/http/_http1.py @@ -77,7 +77,7 @@ def start(self, _) -> layer.CommandGenerator[None]: state = start def read_body(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.stream_id + assert self.stream_id is not None while True: try: if isinstance(event, events.DataReceived): @@ -331,7 +331,7 @@ def send(self, event: HttpEvent) -> layer.CommandGenerator[None]: yield commands.CloseConnection(self.conn) return - if not self.stream_id: + if self.stream_id is None: assert isinstance(event, RequestHeaders) self.stream_id = event.stream_id self.request = event.request @@ -383,7 +383,7 @@ def read_headers( yield commands.Log(f"Unexpected data from server: {bytes(self.buf)!r}") yield commands.CloseConnection(self.conn) return - assert self.stream_id + assert self.stream_id is not None response_head = self.buf.maybe_extract_lines() if response_head: diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 3395fd4bd4..402f46a268 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1072,7 +1072,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # replace the QUIC layer with an UDP layer if requested if tls_clienthello.ignore_connection: self.conn = self.tunnel_connection = connection.Client( - ("ignore-conn", 0), ("ignore-conn", 0), time.time(), + peername=("ignore-conn", 0), sockname=("ignore-conn", 0), transport_protocol="udp" ) @@ -1080,7 +1080,7 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo parent_layer = self.context.layers[self.context.layers.index(self) - 1] if isinstance(parent_layer, ServerQuicLayer): parent_layer.conn = parent_layer.tunnel_connection = connection.Server( - None + address=None ) replacement_layer = UDPLayer(self.context, ignore=True) parent_layer.handle_event = replacement_layer.handle_event # type: ignore diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index 00770a5c1b..e217582170 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -5,6 +5,7 @@ from aioquic._buffer import Buffer from aioquic.h3.connection import ( + ErrorCode, FrameType, Headers, Setting, @@ -17,7 +18,7 @@ from mitmproxy import connection, version from mitmproxy.http import HTTPFlow -from mitmproxy.proxy import commands, context, layers +from mitmproxy.proxy import commands, context, events, layers from mitmproxy.proxy.layers import http, quic from test.mitmproxy.proxy import tutils @@ -31,6 +32,8 @@ example_response_headers = [(b":status", b"200")] +example_request_trailers = [(b"req-trailer-a", b"a"), (b"req-trailer-b", b"b")] + example_response_trailers = [(b"resp-trailer-a", b"a"), (b"resp-trailer-b", b"b")] @@ -46,7 +49,7 @@ def __init__(self, cb: Callable[[bytes], None]): super().__init__(bytes) self._cb = cb - def setdefault(self, value: bytes) -> None: + def setdefault(self, value: bytes) -> bytes: if self._obj is None: self._cb(value) return super().setdefault(value) @@ -96,9 +99,9 @@ def __init__( max_table_capacity=4096, blocked_streams=16, ) - self.decoder_placeholders: list[tutils.Placeholder(bytes)] = [] + self.decoder_placeholders: list[tutils.Placeholder[bytes]] = [] self.encoder = pylsqpack.Encoder() - self.encoder_placeholder: Optional[tutils.Placeholder(bytes)] = None + self.encoder_placeholder: Optional[tutils.Placeholder[bytes]] = None self.peer_stream_id: dict[StreamType, int] = {} self.local_stream_id: dict[StreamType, int] = {} self.max_push_id: Optional[int] = None @@ -331,6 +334,22 @@ def receive_data( end_stream=end_stream, ) + def send_reset(self, error_code: int, stream_id: int = 0) -> quic.ResetQuicStream: + return quic.ResetQuicStream( + connection=self.conn, + stream_id=stream_id, + error_code=error_code, + ) + + def receive_reset( + self, error_code: int, stream_id: int = 0 + ) -> quic.QuicStreamReset: + return quic.QuicStreamReset( + connection=self.conn, + stream_id=stream_id, + error_code=error_code, + ) + def send_init(self) -> Iterable[quic.SendQuicStreamData]: yield self.send_stream_type(StreamType.CONTROL) yield self.send_settings() @@ -357,7 +376,7 @@ def is_done(self) -> bool: def open_h3_server_conn(): # this is a bit fake here (port 80, with alpn, but no tls - c'mon), # but we don't want to pollute our tests with TLS handshakes. - server = connection.Server(("example.com", 80), transport_protocol="udp") + server = connection.Server(address=("example.com", 80), transport_protocol="udp") server.state = connection.ConnectionState.OPEN server.alpn = b"h3" return server @@ -366,6 +385,7 @@ def open_h3_server_conn(): def start_h3_client(tctx: context.Context) -> tuple[tutils.Playbook, FrameFactory]: tctx.client.alpn = b"h3" tctx.client.transport_protocol = "udp" + tctx.server.transport_protocol = "udp" playbook = MultiPlaybook(layers.HttpLayer(tctx, layers.http.HTTPMode.regular)) cff = FrameFactory(conn=tctx.client, is_client=True) @@ -381,7 +401,6 @@ def start_h3_client(tctx: context.Context) -> tuple[tutils.Playbook, FrameFactor def make_h3(open_connection: commands.OpenConnection) -> None: open_connection.connection.alpn = b"h3" - open_connection.connection.transport_protocol = "udp" def test_simple(tctx: context.Context): @@ -503,6 +522,70 @@ def modify_tailers(flow: HTTPFlow) -> None: assert cff.is_done and sff.is_done +@pytest.mark.parametrize("stream", [True, False]) +def test_request_trailers( + tctx: context.Context, + open_h3_server_conn: connection.Server, + stream: bool, +): + playbook, cff = start_h3_client(tctx) + tctx.server = open_h3_server_conn + sff = FrameFactory(tctx.server, is_client=False) + + def enable_streaming(flow: HTTPFlow): + flow.request.stream = stream + + flow = tutils.Placeholder(HTTPFlow) + ( + playbook + # request client + >> cff.receive_headers(example_request_headers) + << (request_headers := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> cff.receive_data(b"Hello World!") + >> tutils.reply(to=request_headers, side_effect=enable_streaming) + ) + if not stream: + ( + playbook + >> cff.receive_headers(example_request_trailers, end_stream=True) + << (request := http.HttpRequestHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) + ) + ( + playbook + # request server + << sff.send_init() + << sff.send_headers(example_request_headers) + << sff.send_data(b"Hello World!") + ) + if not stream: + playbook << sff.send_headers(example_request_trailers, end_stream=True) + ( + playbook + >> sff.receive_init() + << sff.send_encoder() + >> sff.receive_encoder() + >> sff.receive_decoder() # for send_headers + ) + if stream: + ( + playbook + >> cff.receive_headers(example_request_trailers, end_stream=True) + << (request := http.HttpRequestHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) + << sff.send_headers(example_request_trailers, end_stream=True) + ) + assert ( + playbook + >> sff.receive_decoder() # for send_headers + ) + + assert cff.is_done and sff.is_done + + def test_upstream_error(tctx: context.Context): playbook, cff = start_h3_client(tctx) flow = tutils.Placeholder(HTTPFlow) @@ -539,3 +622,160 @@ def test_upstream_error(tctx: context.Context): data = decode_frame(FrameType.DATA, err()) assert b"502 Bad Gateway" in data assert b"server <> error" in data + + +def test_cancel_then_server_disconnect(tctx: context.Context): + """ + Test that we properly handle the case of the following event sequence: + - client cancels a stream + - we start an error hook + - server disconnects + - error hook completes. + """ + playbook, cff = start_h3_client(tctx) + flow = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + assert ( + playbook + # request client + >> cff.receive_headers(example_request_headers, end_stream=True) + << (request := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) + << http.HttpRequestHook(flow) + >> tutils.reply() + # request server + << commands.OpenConnection(server) + >> tutils.reply(None) + << commands.SendData(server, b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n") + # cancel + >> cff.receive_reset(error_code=ErrorCode.H3_REQUEST_CANCELLED) + << commands.CloseConnection(server) + << http.HttpErrorHook(flow) + >> tutils.reply() + >> events.ConnectionClosed(server) + << None + ) + assert cff.is_done + + +def test_cancel_during_response_hook(tctx: context.Context): + """ + Test that we properly handle the case of the following event sequence: + - we receive a server response + - we trigger the response hook + - the client cancels the stream + - the response hook completes + + Given that we have already triggered the response hook, we don't want to trigger the error hook. + """ + playbook, cff = start_h3_client(tctx) + flow = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + assert ( + playbook + # request client + >> cff.receive_headers(example_request_headers, end_stream=True) + << (request := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) + << http.HttpRequestHook(flow) + >> tutils.reply() + # request server + << commands.OpenConnection(server) + >> tutils.reply(None) + << commands.SendData(server, b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n") + # response server + >> events.DataReceived(server, b"HTTP/1.1 204 No Content\r\n\r\n") + << (reponse_headers := http.HttpResponseHeadersHook(flow)) + << commands.CloseConnection(server) + >> tutils.reply(to=reponse_headers) + << (response := http.HttpResponseHook(flow)) + >> cff.receive_reset(error_code=ErrorCode.H3_REQUEST_CANCELLED) + >> tutils.reply(to=response) + << cff.send_reset(error_code=ErrorCode.H3_INTERNAL_ERROR) + ) + assert cff.is_done + + +def test_stream_concurrency(tctx: context.Context): + """Test that we can send an intercepted request with a lower stream id than one that has already been sent.""" + playbook, cff = start_h3_client(tctx) + flow1 = tutils.Placeholder(HTTPFlow) + flow2 = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + sff = FrameFactory(server, is_client=False) + headers1 = [*example_request_headers, (b"x-order", b"1")] + headers2 = [*example_request_headers, (b"x-order", b"2")] + assert ( + playbook + # request client + >> cff.receive_headers( + headers1, stream_id=0, end_stream=True + ) + << (request_header1 := http.HttpRequestHeadersHook(flow1)) + << cff.send_decoder() # for receive_headers + >> cff.receive_headers( + headers2, stream_id=4, end_stream=True + ) + << (request_header2 := http.HttpRequestHeadersHook(flow2)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request_header1) + << (request1 := http.HttpRequestHook(flow1)) + >> tutils.reply(to=request_header2) + << (request2 := http.HttpRequestHook(flow2)) + # req 2 overtakes 1 and we already have a reply: + >> tutils.reply(to=request2) + # request server + << commands.OpenConnection(server) + >> tutils.reply(None, side_effect=make_h3) + << sff.send_init() + << sff.send_headers( + headers2, stream_id=0, end_stream=True + ) + >> sff.receive_init() + << sff.send_encoder() + >> sff.receive_encoder() + >> sff.receive_decoder() # for send_headers + >> tutils.reply(to=request1) + << sff.send_headers( + headers1, stream_id=4, end_stream=True + ) + >> sff.receive_decoder() # for send_headers + ) + assert cff.is_done and sff.is_done + + +def test_stream_concurrent_get_connection(tctx: context.Context): + """Test that an immediate second request for the same domain does not trigger a second connection attempt.""" + playbook, cff = start_h3_client(tctx) + playbook.hooks = False + flow = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + sff = FrameFactory(server, is_client=False) + assert ( + playbook + >> cff.receive_headers( + example_request_headers, stream_id=0, end_stream=True + ) + << cff.send_decoder() # for receive_headers + << (o := commands.OpenConnection(server)) + >> cff.receive_headers( + example_request_headers, stream_id=4, end_stream=True + ) + << cff.send_decoder() # for receive_headers + >> tutils.reply(None, to=o, side_effect=make_h3) + << sff.send_init() + << sff.send_headers( + example_request_headers, stream_id=0, end_stream=True + ) + << sff.send_headers( + example_request_headers, stream_id=4, end_stream=True + ) + >> sff.receive_init() + << sff.send_encoder() + >> sff.receive_encoder() + >> sff.receive_decoder() # for send_headers + >> sff.receive_decoder() # for send_headers + ) + assert cff.is_done and sff.is_done From 6360f388df72684c516626762bed397a6dc06f47 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 24 Nov 2022 11:19:48 +0100 Subject: [PATCH 123/695] [quic] dataclass changes --- test/mitmproxy/addons/test_tlsconfig.py | 15 +++------------ test/mitmproxy/proxy/layers/http/test_http3.py | 1 - test/mitmproxy/proxy/layers/test_quic.py | 6 +++--- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 5a9b8c64ab..9bd6142b90 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -215,10 +215,7 @@ def test_quic_start_client(self, tdata): ], ciphers_client="CHACHA20_POLY1305_SHA256", ) - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) tls_start = quic.QuicTlsData(ctx.client, context=ctx) ta.quic_start_client(tls_start) @@ -289,10 +286,7 @@ def test_tls_start_server_verify_ok(self, hostname, tdata): def test_quic_start_server_verify_ok(self, hostname, tdata): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.server.address = (hostname, 443) tctx.configure( ta, @@ -336,10 +330,7 @@ def test_tls_start_server_insecure(self): def test_quic_start_server_insecure(self): ta = tlsconfig.TlsConfig() with taddons.context(ta) as tctx: - ctx = context.Context( - connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), - tctx.options, - ) + ctx = _ctx(tctx.options) ctx.server.address = ("example.mitmproxy.org", 443) ctx.client.alpn_offers = [b"h3"] diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index e217582170..694bfe917c 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -750,7 +750,6 @@ def test_stream_concurrent_get_connection(tctx: context.Context): """Test that an immediate second request for the same domain does not trigger a second connection attempt.""" playbook, cff = start_h3_client(tctx) playbook.hooks = False - flow = tutils.Placeholder(HTTPFlow) server = tutils.Placeholder(connection.Server) sff = FrameFactory(server, is_client=False) assert ( diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index cc7609a497..06da7725cf 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -348,7 +348,7 @@ def test_full_close(self, tctx: context.Context): ) def test_open_connection(self, tctx: context.Context): - server = connection.Server(("other", 80)) + server = connection.Server(address=("other", 80)) def echo_new_server(ctx: context.Context): echo_layer = TlsEchoLayer(ctx) @@ -869,7 +869,7 @@ def test_client_only(self, tctx: context.Context): # Echo _test_echo(playbook, tssl_client, tctx.client) - other_server = connection.Server(None) + other_server = connection.Server(address=None) assert ( playbook >> events.DataReceived(other_server, b"Plaintext") @@ -1017,7 +1017,7 @@ def test_cannot_parse_clienthello(self, tctx: context.Context): client_layer.debug = "" assert ( playbook - >> events.DataReceived(connection.Server(None), b"data on other stream") + >> events.DataReceived(connection.Server(address=None), b"data on other stream") << commands.Log(">> DataReceived(server, b'data on other stream')", DEBUG) << commands.Log( "[quic] Swallowing DataReceived(server, b'data on other stream') as handshake failed.", From d4e8b05619a76cd65634d294ded363b9895679b7 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 24 Nov 2022 20:07:15 +0100 Subject: [PATCH 124/695] [quic] more h3 tests and fixes --- mitmproxy/proxy/layers/http/_http3.py | 53 +++++----- mitmproxy/proxy/layers/http/_http_h3.py | 16 +--- mitmproxy/proxy/mode_specs.py | 6 +- .../mitmproxy/proxy/layers/http/test_http3.py | 96 ++++++++++++++++++- test/mitmproxy/proxy/test_mode_specs.py | 1 - 5 files changed, 128 insertions(+), 44 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 3275fcd1a9..39607a5260 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -14,6 +14,7 @@ from mitmproxy.proxy.layers.quic import ( QuicConnectionClosed, QuicStreamEvent, + StopQuicStream, error_code_to_str, ) from mitmproxy.proxy.utils import expect @@ -76,33 +77,35 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, (RequestTrailers, ResponseTrailers)): self.h3_conn.send_trailers(event.stream_id, [*event.trailers.fields]) elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): - self.h3_conn.end_stream(event.stream_id) + if not self.h3_conn.has_sent_end_stream(event.stream_id): + self.h3_conn.send_data(event.stream_id, b"", end_stream=True) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): - if not self.h3_conn.has_ended(event.stream_id): - code = { - status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED.value, - }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR.value) - send_error_message = ( - isinstance(event, ResponseProtocolError) - and not self.h3_conn.has_sent_headers(event.stream_id) - and event.code != status_codes.NO_RESPONSE + code = { + status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED.value, + }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR.value) + send_error_message = ( + isinstance(event, ResponseProtocolError) + and not self.h3_conn.has_sent_headers(event.stream_id) + and event.code != status_codes.NO_RESPONSE + ) + if send_error_message: + self.h3_conn.send_headers( + event.stream_id, + [ + (b":status", b"%d" % event.code), + (b"server", version.MITMPROXY.encode()), + (b"content-type", b"text/html"), + ], ) - if send_error_message: - self.h3_conn.send_headers( - event.stream_id, - [ - (b":status", b"%d" % event.code), - (b"server", version.MITMPROXY.encode()), - (b"content-type", b"text/html"), - ], - ) - self.h3_conn.send_data( - event.stream_id, - format_error(event.code, event.message), - end_stream=True, - ) - else: - self.h3_conn.reset_stream(event.stream_id, code) + self.h3_conn.send_data( + event.stream_id, + format_error(event.code, event.message), + end_stream=True, + ) + elif self.h3_conn.has_sent_end_stream(event.stream_id): + yield StopQuicStream(self.conn, event.stream_id, code) + else: + self.h3_conn.reset_stream(event.stream_id, code) else: raise AssertionError(f"Unexpected event: {event!r}") diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index d4573ca852..95414410dd 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -154,14 +154,6 @@ def close_connection( self._is_done = True self._quic.close(error_code, frame_type, reason_phrase) - def end_stream(self, stream_id: int) -> None: - """Ends the given stream locally.""" - - # check whether the stream hasn't been ended before - stream = self._get_or_create_stream(stream_id) - if stream.headers_send_state != HeadersState.AFTER_TRAILERS: - self.send_data(stream_id, data=b"", end_stream=True) - def get_next_available_stream_id(self, is_unidirectional: bool = False): """Reserves and returns the next available stream ID.""" @@ -191,13 +183,13 @@ def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: else: raise AssertionError(f"Unexpected event: {event!r}") - def has_ended(self, stream_id: int) -> bool: - """Indicates whether the given stream has been ended by the peer.""" + def has_sent_end_stream(self, stream_id: int) -> bool: + """Indicates whether the given stream has been ended locally.""" try: - return not self._stream[stream_id].ended + return self._stream[stream_id].headers_send_state == HeadersState.AFTER_TRAILERS except KeyError: - return True + return False def has_sent_headers(self, stream_id: int) -> bool: """Indicates whether headers have been sent over the given stream.""" diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index a56c26ab78..0092c8077a 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -196,16 +196,14 @@ class UpstreamMode(ProxyMode): """A regular HTTP(S) proxy, but all connections are forwarded to a second upstream HTTP(S) proxy.""" description = "HTTP(S) proxy (upstream mode)" transport_protocol = TCP - scheme: Literal["http", "https", "http3"] + scheme: Literal["http", "https"] address: tuple[str, int] # noinspection PyDataclass def __post_init__(self) -> None: scheme, self.address = server_spec.parse(self.data, default_scheme="http") - if scheme != "http" and scheme != "https" and scheme != "http3": + if scheme != "http" and scheme != "https": raise ValueError("invalid upstream proxy scheme") - if scheme == "http3": - self.transport_protocol = UDP self.scheme = scheme diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index 694bfe917c..e309fc8cb2 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -17,9 +17,11 @@ ) from mitmproxy import connection, version -from mitmproxy.http import HTTPFlow +from mitmproxy.flow import Error +from mitmproxy.http import HTTPFlow, Request from mitmproxy.proxy import commands, context, events, layers from mitmproxy.proxy.layers import http, quic +from mitmproxy.proxy.layers.http._http3 import Http3Client from test.mitmproxy.proxy import tutils @@ -693,7 +695,7 @@ def test_cancel_during_response_hook(tctx: context.Context): << (response := http.HttpResponseHook(flow)) >> cff.receive_reset(error_code=ErrorCode.H3_REQUEST_CANCELLED) >> tutils.reply(to=response) - << cff.send_reset(error_code=ErrorCode.H3_INTERNAL_ERROR) + << cff.send_reset(ErrorCode.H3_INTERNAL_ERROR) ) assert cff.is_done @@ -778,3 +780,93 @@ def test_stream_concurrent_get_connection(tctx: context.Context): >> sff.receive_decoder() # for send_headers ) assert cff.is_done and sff.is_done + + +def test_kill_stream(tctx: context.Context): + """Test that we can kill individual streams.""" + playbook, cff = start_h3_client(tctx) + flow1 = tutils.Placeholder(HTTPFlow) + flow2 = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + sff = FrameFactory(server, is_client=False) + headers1 = [*example_request_headers, (b"x-order", b"1")] + headers2 = [*example_request_headers, (b"x-order", b"2")] + + def kill(flow: HTTPFlow): + # Can't use flow.kill() here because that currently still depends on a reply object. + flow.error = Error(Error.KILLED_MESSAGE) + + assert ( + playbook + # request client + >> cff.receive_headers( + headers1, stream_id=0, end_stream=True + ) + << (request_header1 := http.HttpRequestHeadersHook(flow1)) + << cff.send_decoder() # for receive_headers + >> cff.receive_headers( + headers2, stream_id=4, end_stream=True + ) + << (request_header2 := http.HttpRequestHeadersHook(flow2)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request_header2, side_effect=kill) + << http.HttpErrorHook(flow2) + >> tutils.reply() + << cff.send_reset(ErrorCode.H3_INTERNAL_ERROR, stream_id=4) + >> tutils.reply(to=request_header1) + << http.HttpRequestHook(flow1) + >> tutils.reply() + # request server + << commands.OpenConnection(server) + >> tutils.reply(None, side_effect=make_h3) + << sff.send_init() + << sff.send_headers( + headers1, stream_id=0, end_stream=True + ) + >> sff.receive_init() + << sff.send_encoder() + >> sff.receive_encoder() + >> sff.receive_decoder() # for send_headers + ) + assert cff.is_done and sff.is_done + + +class TestClient: + @pytest.mark.parametrize("end_stream", [True, False]) + def test_no_data_on_closed_stream(self, tctx: context.Context, end_stream: bool): + frame_factory = FrameFactory(tctx.server, is_client=False) + playbook = MultiPlaybook(Http3Client(tctx)) + req = Request.make("GET", "http://example.com/") + resp = [(b":status", b"200")] + ( + playbook + << frame_factory.send_init() + >> frame_factory.receive_init() + << frame_factory.send_encoder() + >> frame_factory.receive_encoder() + >> http.RequestHeaders(1, req, end_stream=end_stream) + << frame_factory.send_headers([ + (b":method", b"GET"), + (b':scheme', b'http'), + (b':path', b'/'), + (b'content-length', b'0'), + ], end_stream=end_stream) + >> frame_factory.receive_decoder() # for send_headers + ) + if end_stream: + playbook >> http.RequestEndOfMessage(1) + ( + playbook + >> frame_factory.receive_headers(resp) + << http.ReceiveHttp(tutils.Placeholder(http.ResponseHeaders)) + << frame_factory.send_decoder() # for receive_headers + >> http.RequestProtocolError( + 1, "cancelled", code=http.status_codes.CLIENT_CLOSED_REQUEST + ) + ) + if end_stream: + playbook << quic.StopQuicStream(frame_factory.conn, stream_id=0, error_code=ErrorCode.H3_REQUEST_CANCELLED) + else: + playbook << frame_factory.send_reset(ErrorCode.H3_REQUEST_CANCELLED) + assert playbook + assert frame_factory.is_done diff --git a/test/mitmproxy/proxy/test_mode_specs.py b/test/mitmproxy/proxy/test_mode_specs.py index cf67dfac7e..be83e5238a 100644 --- a/test/mitmproxy/proxy/test_mode_specs.py +++ b/test/mitmproxy/proxy/test_mode_specs.py @@ -56,7 +56,6 @@ def test_parse_specific_modes(): assert ProxyMode.parse("http3") assert ProxyMode.parse("transparent") assert ProxyMode.parse("upstream:https://proxy") - assert ProxyMode.parse("upstream:http3://proxy") assert ProxyMode.parse("reverse:https://host@443") assert ProxyMode.parse("reverse:http3://host@443") assert ProxyMode.parse("socks5") From f23a1887bb76501f6dbb57573847767e7c1538f1 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Thu, 24 Nov 2022 22:18:07 +0100 Subject: [PATCH 125/695] [quic] fix h3 double-close issue --- mitmproxy/proxy/layers/http/_http3.py | 7 -- mitmproxy/proxy/layers/http/_http_h3.py | 8 ++- .../mitmproxy/proxy/layers/http/test_http3.py | 72 +++++++++++++++++++ 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 39607a5260..20c16bcf8a 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -170,13 +170,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: msg = event.reason_phrase or error_code_to_str(event.error_code) for stream_id in self.h3_conn.get_reserved_stream_ids(): yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) - # turn `QuicErrorCode.NO_ERROR` into `H3ErrorCode.H3_NO_ERROR` - self.h3_conn.close_connection( - event.error_code or H3ErrorCode.H3_NO_ERROR, - event.frame_type, - event.reason_phrase, - ) - yield from self.h3_conn.transmit() else: raise AssertionError(f"Unexpected event: {event!r}") diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 95414410dd..4071f37e92 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -177,7 +177,13 @@ def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: # convert data events from the QUIC layer back to aioquic events elif isinstance(event, QuicStreamDataReceived): - return self.handle_event(StreamDataReceived(event.data, event.end_stream, event.stream_id)) + if self._get_or_create_stream(event.stream_id).ended: + # aioquic will not send us any events once a stream has ended. + # Instead, it will close the connection. We simulate this here for H3 tests. + self.close_connection(error_code=QuicErrorCode.PROTOCOL_VIOLATION, reason_phrase="stream already ended") + return [] + else: + return self.handle_event(StreamDataReceived(event.data, event.end_stream, event.stream_id)) # should never happen else: diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index e309fc8cb2..4c10878291 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -626,6 +626,48 @@ def test_upstream_error(tctx: context.Context): assert b"server <> error" in data +def test_rst_then_close(tctx): + """ + Test that we properly handle the case of a client that first causes protocol errors and then disconnects. + + This is slightly different to H2, as QUIC will close the connection immediately. + """ + playbook, cff = start_h3_client(tctx) + flow = tutils.Placeholder(HTTPFlow) + server = tutils.Placeholder(connection.Server) + err = tutils.Placeholder(str) + + assert ( + playbook + # request client + >> cff.receive_headers(example_request_headers, end_stream=True) + << (request := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request) + << http.HttpRequestHook(flow) + >> tutils.reply() + # request server + << (open := commands.OpenConnection(server)) + >> cff.receive_data(b"unexpected data frame") + << quic.CloseQuicConnection( + tctx.client, + error_code=quic.QuicErrorCode.PROTOCOL_VIOLATION, + frame_type=None, + reason_phrase=err, + ) + >> quic.QuicConnectionClosed( + tctx.client, + error_code=quic.QuicErrorCode.PROTOCOL_VIOLATION, + frame_type=None, + reason_phrase=err, + ) + >> tutils.reply("connection cancelled", to=open) + << http.HttpErrorHook(flow) + >> tutils.reply() + ) + assert flow().error.msg == "connection cancelled" + + def test_cancel_then_server_disconnect(tctx: context.Context): """ Test that we properly handle the case of the following event sequence: @@ -870,3 +912,33 @@ def test_no_data_on_closed_stream(self, tctx: context.Context, end_stream: bool) playbook << frame_factory.send_reset(ErrorCode.H3_REQUEST_CANCELLED) assert playbook assert frame_factory.is_done + + +def test_early_server_data(tctx: context.Context): + playbook, cff = start_h3_client(tctx) + sff = FrameFactory(tctx.server, is_client=False) + + tctx.server.address = ("example.com", 80) + tctx.server.state = connection.ConnectionState.OPEN + tctx.server.alpn = b"h3" + + flow = tutils.Placeholder(HTTPFlow) + assert ( + playbook + >> cff.receive_headers(example_request_headers, end_stream=True) + << (request_header := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + >> tutils.reply(to=request_header) + << (request := http.HttpRequestHook(flow)) + # Surprise! We get data from the server before the request hook finishes. + >> sff.receive_stream_type(StreamType.CONTROL) + << sff.send_init() + >> sff.receive_stream_type(StreamType.QPACK_ENCODER) + >> sff.receive_stream_type(StreamType.QPACK_DECODER) + >> sff.receive_settings() + << sff.send_encoder() + >> sff.receive_encoder() + # Request hook finishes... + >> tutils.reply(to=request) + << sff.send_headers(example_request_headers, end_stream=True) + ) From a8b3784f8d2709cd4b546f5ecf9ee128e8fad566 Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Fri, 25 Nov 2022 14:15:21 +0100 Subject: [PATCH 126/695] [quic] switch to aioquic_mitmproxy --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f1a3c203d9..c1c4617fca 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#install-requires # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ - "aioquic>=0.9.20,<0.10", + "aioquic_mitmproxy>=0.9.20,<0.10", "asgiref>=3.2.10,<3.6", "Brotli>=1.0,<1.1", "certifi>=2019.9.11", # no semver here - this should always be on the last release! From 8bd88541f658b6257c230b35ec73f6ef0ec601fb Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 27 Nov 2022 04:17:57 +0100 Subject: [PATCH 127/695] [quic] H3 stream reset refinements --- mitmproxy/proxy/layers/http/_http3.py | 99 +++++----- mitmproxy/proxy/layers/http/_http_h3.py | 60 ++++--- .../mitmproxy/proxy/layers/http/test_http3.py | 170 +++++++++++++++--- 3 files changed, 232 insertions(+), 97 deletions(-) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 20c16bcf8a..0c750dbc3e 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -57,6 +57,7 @@ class Http3Connection(HttpConnection): def __init__(self, context: context.Context, conn: connection.Connection): super().__init__(context, conn) self.h3_conn = LayeredH3Connection(self.conn, is_client=self.conn is self.context.server) + self._stream_protocol_errors: dict[int, int] = {} def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.Start): @@ -77,12 +78,12 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(event, (RequestTrailers, ResponseTrailers)): self.h3_conn.send_trailers(event.stream_id, [*event.trailers.fields]) elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): - if not self.h3_conn.has_sent_end_stream(event.stream_id): - self.h3_conn.send_data(event.stream_id, b"", end_stream=True) + self.h3_conn.end_stream(event.stream_id) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): code = { status_codes.CLIENT_CLOSED_REQUEST: H3ErrorCode.H3_REQUEST_CANCELLED.value, }.get(event.code, H3ErrorCode.H3_INTERNAL_ERROR.value) + self._stream_protocol_errors[event.stream_id] = code send_error_message = ( isinstance(event, ResponseProtocolError) and not self.h3_conn.has_sent_headers(event.stream_id) @@ -102,8 +103,6 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: format_error(event.code, event.message), end_stream=True, ) - elif self.h3_conn.has_sent_end_stream(event.stream_id): - yield StopQuicStream(self.conn, event.stream_id, code) else: self.h3_conn.reset_stream(event.stream_id, code) else: @@ -119,56 +118,66 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # forward stream messages from the QUIC layer to the H3 connection elif isinstance(event, QuicStreamEvent): - for h3_event in self.h3_conn.handle_stream_event(event): - if isinstance(h3_event, StreamReset): - if h3_event.push_id is None: - err_str = error_code_to_str(h3_event.error_code) - err_code = { - H3ErrorCode.H3_REQUEST_CANCELLED.value: status_codes.CLIENT_CLOSED_REQUEST, - }.get(h3_event.error_code, self.ReceiveProtocolError.code) - yield ReceiveHttp( - self.ReceiveProtocolError( - h3_event.stream_id, - f"stream reset by client ({err_str})", - code=err_code, - ) - ) - elif isinstance(h3_event, DataReceived): - if h3_event.push_id is None: - if h3_event.data: - yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) - if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - elif isinstance(h3_event, HeadersReceived): - if h3_event.push_id is None: - try: - receive_event = self.parse_headers(h3_event) - except ValueError as e: - self.h3_conn.close_connection( - error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, - reason_phrase=f"Invalid HTTP/3 request headers: {e}", + h3_events = self.h3_conn.handle_stream_event(event) + if event.stream_id in self._stream_protocol_errors: + # we already reset or ended the stream, tell the peer to stop + # (this is a noop if the peer already did the same) + yield StopQuicStream( + self.conn, + event.stream_id, + self._stream_protocol_errors[event.stream_id], + ) + else: + for h3_event in h3_events: + if isinstance(h3_event, StreamReset): + if h3_event.push_id is None: + err_str = error_code_to_str(h3_event.error_code) + err_code = { + H3ErrorCode.H3_REQUEST_CANCELLED.value: status_codes.CLIENT_CLOSED_REQUEST, + }.get(h3_event.error_code, self.ReceiveProtocolError.code) + yield ReceiveHttp( + self.ReceiveProtocolError( + h3_event.stream_id, + f"stream reset by client ({err_str})", + code=err_code, + ) ) - else: - yield ReceiveHttp(receive_event) + elif isinstance(h3_event, DataReceived): + if h3_event.push_id is None: + if h3_event.data: + yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) if h3_event.stream_ended: yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - elif isinstance(h3_event, TrailersReceived): - if h3_event.push_id is None: - yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) - if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - elif isinstance(h3_event, PushPromiseReceived): - # we don't support push - pass - else: - raise AssertionError(f"Unexpected event: {event!r}") + elif isinstance(h3_event, HeadersReceived): + if h3_event.push_id is None: + try: + receive_event = self.parse_headers(h3_event) + except ValueError as e: + self.h3_conn.close_connection( + error_code=H3ErrorCode.H3_GENERAL_PROTOCOL_ERROR, + reason_phrase=f"Invalid HTTP/3 request headers: {e}", + ) + else: + yield ReceiveHttp(receive_event) + if h3_event.stream_ended: + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + elif isinstance(h3_event, TrailersReceived): + if h3_event.push_id is None: + yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) + if h3_event.stream_ended: + yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + elif isinstance(h3_event, PushPromiseReceived): + # we don't support push + pass + else: + raise AssertionError(f"Unexpected event: {event!r}") yield from self.h3_conn.transmit() # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, QuicConnectionClosed): self._handle_event = self.done # type: ignore msg = event.reason_phrase or error_code_to_str(event.error_code) - for stream_id in self.h3_conn.get_reserved_stream_ids(): + for stream_id in self.h3_conn.get_open_stream_ids(push_id=None): yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) else: diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 4071f37e92..a81f8b13de 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -2,12 +2,13 @@ from typing import Iterable, Optional from aioquic.h3.connection import ( + FrameUnexpected, H3Connection, - FrameUnexpected as H3FrameUnexpected, H3Event, H3Stream, Headers, HeadersState, + StreamType, ) from aioquic.h3.events import HeadersReceived from aioquic.quic.configuration import QuicConfiguration @@ -99,10 +100,6 @@ def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: self._next_stream_id[index] = stream_id + 4 return stream_id - def get_reserved_stream_ids(self, is_unidirectional: bool = False) -> Iterable[int]: - index = (int(is_unidirectional) << 1) | int(not self._is_client) - return range(index, self._next_stream_id[index], 4) - def reset_stream(self, stream_id: int, error_code: int) -> None: self.pending_commands.append(ResetQuicStream(self.conn, stream_id, error_code)) @@ -154,31 +151,56 @@ def close_connection( self._is_done = True self._quic.close(error_code, frame_type, reason_phrase) + def end_stream(self, stream_id: int) -> None: + """Ends the given stream if not already done so.""" + + stream = self._get_or_create_stream(stream_id) + if stream.headers_send_state != HeadersState.AFTER_TRAILERS: + super().send_data(stream_id, b"", end_stream=True) + stream.headers_send_state = HeadersState.AFTER_TRAILERS + def get_next_available_stream_id(self, is_unidirectional: bool = False): """Reserves and returns the next available stream ID.""" return self._quic.get_next_available_stream_id(is_unidirectional) - def get_reserved_stream_ids(self, is_unidirectional: bool = False) -> Iterable[int]: - """Returns all reserved stream IDs.""" + def get_open_stream_ids(self, push_id: Optional[int]) -> Iterable[int]: + """Iterates over all non-special open streams, optionally for a given push id.""" - return self._mock.get_reserved_stream_ids(is_unidirectional) + return ( + stream.stream_id + for stream in self._stream.values() + if ( + stream.push_id == push_id + and stream.stream_type == ( + None + if push_id is None else + StreamType.PUSH + ) + and not ( + stream.headers_recv_state == HeadersState.AFTER_TRAILERS + and stream.headers_send_state == HeadersState.AFTER_TRAILERS + ) + ) + ) def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: # don't do anything if we're done if self._is_done: return [] - # treat reset events similar to stream end events + # treat reset events similar to data events with end_stream=True + # We can receive multiple reset events as long as the final size does not change. elif isinstance(event, QuicStreamReset): stream = self._get_or_create_stream(event.stream_id) stream.ended = True + stream.headers_recv_state = HeadersState.AFTER_TRAILERS return [StreamReset(event.stream_id, event.error_code, stream.push_id)] # convert data events from the QUIC layer back to aioquic events elif isinstance(event, QuicStreamDataReceived): if self._get_or_create_stream(event.stream_id).ended: - # aioquic will not send us any events once a stream has ended. + # aioquic will not send us any data events once a stream has ended. # Instead, it will close the connection. We simulate this here for H3 tests. self.close_connection(error_code=QuicErrorCode.PROTOCOL_VIOLATION, reason_phrase="stream already ended") return [] @@ -189,14 +211,6 @@ def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: else: raise AssertionError(f"Unexpected event: {event!r}") - def has_sent_end_stream(self, stream_id: int) -> bool: - """Indicates whether the given stream has been ended locally.""" - - try: - return self._stream[stream_id].headers_send_state == HeadersState.AFTER_TRAILERS - except KeyError: - return False - def has_sent_headers(self, stream_id: int) -> bool: """Indicates whether headers have been sent over the given stream.""" @@ -208,12 +222,8 @@ def has_sent_headers(self, stream_id: int) -> bool: def reset_stream(self, stream_id: int, error_code: int) -> None: """Resets a stream that hasn't been ended locally yet.""" - # we don't allow reset after FIN - stream = self._get_or_create_stream(stream_id) - if stream.headers_send_state == HeadersState.AFTER_TRAILERS: - raise H3FrameUnexpected("reset not allowed in this state") - # set the header state and queue a reset event + stream = self._get_or_create_stream(stream_id) stream.headers_send_state = HeadersState.AFTER_TRAILERS self._quic.reset_stream(stream_id, error_code) @@ -233,7 +243,7 @@ def send_headers(self, stream_id: int, headers: Headers, end_stream: bool = Fals # ensure we haven't sent something before stream = self._get_or_create_stream(stream_id) if stream.headers_send_state != HeadersState.INITIAL: - raise H3FrameUnexpected("initial HEADERS frame is not allowed in this state") + raise FrameUnexpected("initial HEADERS frame is not allowed in this state") super().send_headers(stream_id, headers, end_stream) self._after_send(stream_id, end_stream) @@ -243,7 +253,7 @@ def send_trailers(self, stream_id: int, trailers: Headers) -> None: # ensure we got some headers first stream = self._get_or_create_stream(stream_id) if stream.headers_send_state != HeadersState.AFTER_HEADERS: - raise H3FrameUnexpected("trailing HEADERS frame is not allowed in this state") + raise FrameUnexpected("trailing HEADERS frame is not allowed in this state") super().send_headers(stream_id, trailers, end_stream=True) self._after_send(stream_id, end_stream=True) diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index 4c10878291..bb66cac2eb 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -447,11 +447,11 @@ def test_simple(tctx: context.Context): assert flow().response.text == "Hello, World!" -@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize("stream", ["stream", ""]) def test_response_trailers( tctx: context.Context, open_h3_server_conn: connection.Server, - stream: bool, + stream: str, ): playbook, cff = start_h3_client(tctx) tctx.server = open_h3_server_conn @@ -499,22 +499,19 @@ def enable_streaming(flow: HTTPFlow): << (response := http.HttpResponseHook(flow)) << sff.send_decoder() # for receive_headers ) - - def modify_tailers(flow: HTTPFlow) -> None: - assert flow.response.trailers - del flow.response.trailers["resp-trailer-a"] - + assert flow().response.trailers + del flow().response.trailers["resp-trailer-a"] if stream: assert ( playbook - >> tutils.reply(to=response, side_effect=modify_tailers) + >> tutils.reply(to=response) << cff.send_headers(example_response_trailers[1:], end_stream=True) >> cff.receive_decoder() # for send_headers ) else: assert ( playbook - >> tutils.reply(to=response, side_effect=modify_tailers) + >> tutils.reply(to=response) << cff.send_headers(example_response_headers) << cff.send_data(b"Hello, World!") << cff.send_headers(example_response_trailers[1:], end_stream=True) @@ -524,11 +521,11 @@ def modify_tailers(flow: HTTPFlow) -> None: assert cff.is_done and sff.is_done -@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize("stream", ["stream", ""]) def test_request_trailers( tctx: context.Context, open_h3_server_conn: connection.Server, - stream: bool, + stream: str, ): playbook, cff = start_h3_client(tctx) tctx.server = open_h3_server_conn @@ -626,6 +623,131 @@ def test_upstream_error(tctx: context.Context): assert b"server <> error" in data +@pytest.mark.parametrize("stream", ["stream", ""]) +@pytest.mark.parametrize("when", ["request", "response"]) +@pytest.mark.parametrize("how", ["RST", "disconnect", "RST+disconnect"]) +def test_http3_client_aborts( + tctx: context.Context, stream: str, when: str, how: str +): + """ + Test handling of the case where a client aborts during request or response transmission. + + If the client aborts the request transmission, we must trigger an error hook, + if the client disconnects during response transmission, no error hook is triggered. + """ + server = tutils.Placeholder(connection.Server) + flow = tutils.Placeholder(HTTPFlow) + playbook, cff = start_h3_client(tctx) + + def enable_request_streaming(flow: HTTPFlow): + flow.request.stream = True + + def enable_response_streaming(flow: HTTPFlow): + flow.response.stream = True + + assert ( + playbook + >> cff.receive_headers(example_request_headers) + << (request_headers := http.HttpRequestHeadersHook(flow)) + << cff.send_decoder() # for receive_headers + ) + if stream and when == "request": + assert ( + playbook + >> tutils.reply( + side_effect=enable_request_streaming, to=request_headers + ) + << commands.OpenConnection(server) + >> tutils.reply(None) + << commands.SendData(server, b"GET / HTTP/1.1\r\n" b"Host: example.com\r\n\r\n") + ) + else: + assert playbook >> tutils.reply(to=request_headers) + + if when == "request": + if "RST" in how: + playbook >> cff.receive_reset(ErrorCode.H3_REQUEST_CANCELLED) + else: + playbook >> quic.QuicConnectionClosed( + tctx.client, + error_code=ErrorCode.H3_REQUEST_CANCELLED, + frame_type=None, + reason_phrase="peer closed connection" + ) + + if stream: + playbook << commands.CloseConnection(server) + playbook << http.HttpErrorHook(flow) + playbook >> tutils.reply() + + if how == "RST+disconnect": + playbook >> quic.QuicConnectionClosed( + tctx.client, + error_code=ErrorCode.H3_NO_ERROR, + frame_type=None, + reason_phrase="peer closed connection" + ) + assert playbook + assert ( + "stream reset" in flow().error.msg + or "peer closed connection" in flow().error.msg + ) + return + + assert ( + playbook + >> cff.receive_data(b"", end_stream=True) + << http.HttpRequestHook(flow) + >> tutils.reply() + << commands.OpenConnection(server) + >> tutils.reply(None) + << commands.SendData(server, b"GET / HTTP/1.1\r\n" b"Host: example.com\r\n\r\n") + >> events.DataReceived(server, b"HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\n123") + << http.HttpResponseHeadersHook(flow) + ) + if stream: + assert ( + playbook + >> tutils.reply(side_effect=enable_response_streaming) + << cff.send_headers([ + (b":status", b"200"), + (b"content-length", b"6"), + ]) + << cff.send_data(b"123") + ) + else: + assert playbook >> tutils.reply() + + if "RST" in how: + playbook >> cff.receive_reset(ErrorCode.H3_REQUEST_CANCELLED) + else: + playbook >> quic.QuicConnectionClosed( + tctx.client, + error_code=ErrorCode.H3_REQUEST_CANCELLED, + frame_type=None, + reason_phrase="peer closed connection" + ) + + playbook << commands.CloseConnection(server) + playbook << http.HttpErrorHook(flow) + playbook >> tutils.reply() + assert playbook + + if how == "RST+disconnect": + playbook >> quic.QuicConnectionClosed( + tctx.client, + error_code=ErrorCode.H3_REQUEST_CANCELLED, + frame_type=None, + reason_phrase="peer closed connection" + ) + assert playbook + + if "RST" in how: + assert "stream reset" in flow().error.msg + else: + assert "peer closed connection" in flow().error.msg + + def test_rst_then_close(tctx): """ Test that we properly handle the case of a client that first causes protocol errors and then disconnects. @@ -874,43 +996,37 @@ def kill(flow: HTTPFlow): class TestClient: - @pytest.mark.parametrize("end_stream", [True, False]) - def test_no_data_on_closed_stream(self, tctx: context.Context, end_stream: bool): + def test_no_data_on_closed_stream(self, tctx: context.Context): frame_factory = FrameFactory(tctx.server, is_client=False) playbook = MultiPlaybook(Http3Client(tctx)) req = Request.make("GET", "http://example.com/") resp = [(b":status", b"200")] - ( + assert ( playbook << frame_factory.send_init() >> frame_factory.receive_init() << frame_factory.send_encoder() >> frame_factory.receive_encoder() - >> http.RequestHeaders(1, req, end_stream=end_stream) + >> http.RequestHeaders(1, req, end_stream=True) << frame_factory.send_headers([ (b":method", b"GET"), (b':scheme', b'http'), (b':path', b'/'), (b'content-length', b'0'), - ], end_stream=end_stream) + ], end_stream=True) >> frame_factory.receive_decoder() # for send_headers - ) - if end_stream: - playbook >> http.RequestEndOfMessage(1) - ( - playbook + >> http.RequestEndOfMessage(1) >> frame_factory.receive_headers(resp) << http.ReceiveHttp(tutils.Placeholder(http.ResponseHeaders)) << frame_factory.send_decoder() # for receive_headers >> http.RequestProtocolError( 1, "cancelled", code=http.status_codes.CLIENT_CLOSED_REQUEST ) - ) - if end_stream: - playbook << quic.StopQuicStream(frame_factory.conn, stream_id=0, error_code=ErrorCode.H3_REQUEST_CANCELLED) - else: - playbook << frame_factory.send_reset(ErrorCode.H3_REQUEST_CANCELLED) - assert playbook + << frame_factory.send_reset(ErrorCode.H3_REQUEST_CANCELLED) + >> frame_factory.receive_data(b"foo") + << quic.StopQuicStream(tctx.server, 0, ErrorCode.H3_REQUEST_CANCELLED) + ) # important: no ResponseData event here! + assert frame_factory.is_done From 5d0947acb7d4b4ef2cf87bbaf2ac5ffbfe2fc18e Mon Sep 17 00:00:00 2001 From: Manuel Meitinger Date: Sun, 27 Nov 2022 06:01:04 +0100 Subject: [PATCH 128/695] [quic] 100% H3 coverage --- CHANGELOG.md | 2 + mitmproxy/proxy/layers/http/_http3.py | 11 ++- mitmproxy/proxy/layers/http/_http_h3.py | 6 +- .../mitmproxy/proxy/layers/http/test_http3.py | 93 ++++++++++++++++++- 4 files changed, 102 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59edf1d7f3..16003b2600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased: mitmproxy next +* Add QUIC support. + ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) * ASGI/WSGI apps can now listen on all ports for a specific hostname. This makes it simpler to accept both HTTP and HTTPS. diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 0c750dbc3e..8a696a24fb 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -105,7 +105,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: ) else: self.h3_conn.reset_stream(event.stream_id, code) - else: + else: # pragma: no cover raise AssertionError(f"Unexpected event: {event!r}") except H3FrameUnexpected as e: @@ -121,7 +121,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: h3_events = self.h3_conn.handle_stream_event(event) if event.stream_id in self._stream_protocol_errors: # we already reset or ended the stream, tell the peer to stop - # (this is a noop if the peer already did the same) + # (this is a noop if the peer already did the same) yield StopQuicStream( self.conn, event.stream_id, @@ -166,21 +166,22 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) if h3_event.stream_ended: yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) - elif isinstance(h3_event, PushPromiseReceived): + elif isinstance(h3_event, PushPromiseReceived): # pragma: no cover # we don't support push pass - else: + else: # pragma: no cover raise AssertionError(f"Unexpected event: {event!r}") yield from self.h3_conn.transmit() # report a protocol error for all remaining open streams when a connection is closed elif isinstance(event, QuicConnectionClosed): self._handle_event = self.done # type: ignore + self.h3_conn.handle_connection_closed(event) msg = event.reason_phrase or error_code_to_str(event.error_code) for stream_id in self.h3_conn.get_open_stream_ids(push_id=None): yield ReceiveHttp(self.ReceiveProtocolError(stream_id, msg)) - else: + else: # pragma: no cover raise AssertionError(f"Unexpected event: {event!r}") @expect(HttpEvent, QuicStreamEvent, QuicConnectionClosed) diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index a81f8b13de..849f3b1594 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -19,6 +19,7 @@ from mitmproxy.proxy import commands, layer from mitmproxy.proxy.layers.quic import ( CloseQuicConnection, + QuicConnectionClosed, QuicStreamDataReceived, QuicStreamEvent, QuicStreamReset, @@ -184,6 +185,9 @@ def get_open_stream_ids(self, push_id: Optional[int]) -> Iterable[int]: ) ) + def handle_connection_closed(self, event: QuicConnectionClosed) -> None: + self._is_done = True + def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: # don't do anything if we're done if self._is_done: @@ -208,7 +212,7 @@ def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: return self.handle_event(StreamDataReceived(event.data, event.end_stream, event.stream_id)) # should never happen - else: + else: # pragma: no cover raise AssertionError(f"Unexpected event: {event!r}") def has_sent_headers(self, stream_id: int) -> bool: diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index bb66cac2eb..eccdd7bf43 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -7,7 +7,7 @@ from aioquic.h3.connection import ( ErrorCode, FrameType, - Headers, + Headers as H3Headers, Setting, StreamType, encode_frame, @@ -18,7 +18,7 @@ from mitmproxy import connection, version from mitmproxy.flow import Error -from mitmproxy.http import HTTPFlow, Request +from mitmproxy.http import Headers, HTTPFlow, Request from mitmproxy.proxy import commands, context, events, layers from mitmproxy.proxy.layers import http, quic from mitmproxy.proxy.layers.http._http3 import Http3Client @@ -261,7 +261,7 @@ def receive_decoder(self) -> quic.QuicStreamDataReceived: def send_headers( self, - headers: Headers, + headers: H3Headers, stream_id: int = 0, end_stream: bool = False, ) -> Iterable[quic.SendQuicStreamData]: @@ -286,7 +286,7 @@ def decode(data: bytes) -> None: def receive_headers( self, - headers: Headers, + headers: H3Headers, stream_id: int = 0, end_stream: bool = False, ) -> Iterable[quic.QuicStreamDataReceived]: @@ -405,6 +405,50 @@ def make_h3(open_connection: commands.OpenConnection) -> None: open_connection.connection.alpn = b"h3" +def test_ignore_push(tctx: context.Context): + playbook, cff = start_h3_client(tctx) + + +def test_fail_without_header(tctx: context.Context): + playbook = MultiPlaybook(layers.http.Http3Server(tctx)) + cff = FrameFactory(tctx.client, is_client=True) + assert ( + playbook + << cff.send_init() + >> cff.receive_init() + << cff.send_encoder() + >> cff.receive_encoder() + >> http.ResponseProtocolError(0, "first message", http.status_codes.NO_RESPONSE) + << cff.send_reset(ErrorCode.H3_INTERNAL_ERROR) + ) + assert cff.is_done + + +def test_invalid_header(tctx: context.Context): + playbook, cff = start_h3_client(tctx) + assert ( + playbook + >> cff.receive_headers([ + (b":method", b"CONNECT"), + (b":path", b"/"), + (b":authority", b"example.com"), + ], end_stream=True) + << cff.send_decoder() # for receive_headers + << quic.CloseQuicConnection( + tctx.client, + error_code=ErrorCode.H3_GENERAL_PROTOCOL_ERROR, + frame_type=None, + reason_phrase="Invalid HTTP/3 request headers: Required pseudo header is missing: b':scheme'", + ) + # ensure that once we close, we don't process messages anymore + >> cff.receive_headers([ + (b":method", b"CONNECT"), + (b":path", b"/"), + (b":authority", b"example.com"), + ], end_stream=True) + ) + + def test_simple(tctx: context.Context): playbook, cff = start_h3_client(tctx) flow = tutils.Placeholder(HTTPFlow) @@ -1029,6 +1073,47 @@ def test_no_data_on_closed_stream(self, tctx: context.Context): assert frame_factory.is_done + def test_ignore_wrong_order(self, tctx: context.Context): + frame_factory = FrameFactory(tctx.server, is_client=False) + playbook = MultiPlaybook(Http3Client(tctx)) + req = Request.make("GET", "http://example.com/") + assert ( + playbook + << frame_factory.send_init() + >> frame_factory.receive_init() + << frame_factory.send_encoder() + >> frame_factory.receive_encoder() + >> http.RequestTrailers(1, Headers([(b"x-trailer", b"")])) + << commands.Log( + "Received RequestTrailers(stream_id=0, trailers=Headers[(b'x-trailer', b'')]) unexpectedly: " + "trailing HEADERS frame is not allowed in this state" + ) + >> http.RequestEndOfMessage(1) + << commands.Log( + "Received RequestEndOfMessage(stream_id=0) unexpectedly: " + "DATA frame is not allowed in this state" + ) + >> http.RequestData(1, b"123") + << commands.Log( + "Received RequestData(stream_id=0, data=b'123') unexpectedly: " + "DATA frame is not allowed in this state" + ) + >> http.RequestHeaders(1, req, end_stream=False) + << frame_factory.send_headers([ + (b":method", b"GET"), + (b':scheme', b'http'), + (b':path', b'/'), + (b'content-length', b'0'), + ], end_stream=False) + >> frame_factory.receive_decoder() # for send_headers + >> http.RequestHeaders(1, req, end_stream=False) + << commands.Log( + "Received RequestHeaders(stream_id=0, request=Request(GET example.com:80/)," + " end_stream=False, replay_flow=None) unexpectedly: " + "initial HEADERS frame is not allowed in this state" + ) + ) + def test_early_server_data(tctx: context.Context): playbook, cff = start_h3_client(tctx) From 6c9b4fec24f60d964ac6500a0d08d943d9fc8bf9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 27 Nov 2022 20:07:16 +0100 Subject: [PATCH 129/695] [requires.io] dependency update on main branch (#5754) * [requires.io] dependency update * Update tox.ini Co-authored-by: requires.io --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ed7e548d2d..b41164d32a 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ commands = [testenv:flake8] deps = - flake8>=3.8.4,<5.1 + flake8>=3.8.4,<6.1 flake8-tidy-imports>=4.2.0,<5 commands = flake8 --jobs 8 mitmproxy examples test release {posargs} From e20aec75b069deb2252708ebffc000988605e3c0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 28 Nov 2022 12:06:00 +0100 Subject: [PATCH 130/695] fix #5764 (#5766) --- web/src/js/components/common/Splitter.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/src/js/components/common/Splitter.tsx b/web/src/js/components/common/Splitter.tsx index dc522fca0c..b1aa1d8979 100644 --- a/web/src/js/components/common/Splitter.tsx +++ b/web/src/js/components/common/Splitter.tsx @@ -87,8 +87,12 @@ export default class Splitter extends Component { const node = ReactDOM.findDOMNode(this) - node.previousElementSibling.style.flex = '' - node.nextElementSibling.style.flex = '' + if (node.previousElementSibling){ + node.previousElementSibling.style.flex = '' + } + if (node.nextElementSibling) { + node.nextElementSibling.style.flex = '' + } if (!willUnmount) { this.setState({ applied: false }) From 30135ea36a40d78cecde621630b403814e6a744c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 28 Nov 2022 17:30:41 +0100 Subject: [PATCH 131/695] fix nits --- CHANGELOG.md | 2 +- mitmproxy/addons/dumper.py | 12 +++++++++--- mitmproxy/contentviews/http3.py | 1 - 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16003b2600..c460a474f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased: mitmproxy next -* Add QUIC support. +* Add experimental QUIC support. ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) * ASGI/WSGI apps can now listen on all ports for a specific hostname. This makes it simpler to accept both HTTP and HTTPS. diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 090123250d..a947199466 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -16,6 +16,7 @@ from mitmproxy import flowfilter from mitmproxy import http from mitmproxy.contrib import click as miniclick +from mitmproxy.net.dns import response_codes from mitmproxy.tcp import TCPFlow, TCPMessage from mitmproxy.udp import UDPFlow, UDPMessage from mitmproxy.utils import human @@ -377,9 +378,14 @@ def dns_response(self, f: dns.DNSFlow): self._echo_dns_query(f) arrows = self.style(" <<", bold=True) - answers = ", ".join( - self.style(str(x), fg="bright_blue") for x in f.response.answers - ) + if f.response.answers: + answers = ", ".join( + self.style(str(x), fg="bright_blue") for x in f.response.answers + ) + else: + answers = self.style(response_codes.to_str( + f.response.response_code, + ), fg="red") self.echo(f"{arrows} {answers}") def dns_error(self, f: dns.DNSFlow): diff --git a/mitmproxy/contentviews/http3.py b/mitmproxy/contentviews/http3.py index 8c280dd6c1..df014b99db 100644 --- a/mitmproxy/contentviews/http3.py +++ b/mitmproxy/contentviews/http3.py @@ -110,7 +110,6 @@ def __call__( h3_buf = Buffer(data=bytes(buf[:8])) stream_type = h3_buf.pull_uint_var() consumed = h3_buf.tell() - assert consumed == 1 del buf[:consumed] state.frames[0] = [ StreamType(stream_type) From 5ec4bbf496075c66c818dce644493ad936de95c2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 28 Nov 2022 17:32:23 +0100 Subject: [PATCH 132/695] don't persist connection state, fix #5524 --- mitmproxy/connection.py | 6 +----- mitmproxy/coretypes/serializable.py | 2 ++ mitmproxy/io/compat.py | 8 ++++++++ mitmproxy/proxy/layers/quic.py | 4 +++- mitmproxy/proxy/server.py | 1 + mitmproxy/version.py | 2 +- test/mitmproxy/proxy/conftest.py | 4 +++- test/mitmproxy/proxy/layers/test_modes.py | 6 +++--- test/mitmproxy/test_connection.py | 6 ++++-- 9 files changed, 26 insertions(+), 13 deletions(-) diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index a6f9be704c..20782057ad 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -53,7 +53,7 @@ class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): sockname: Optional[Address] """Our local `(ip, port)` tuple for this connection.""" - state: ConnectionState + state: ConnectionState = field(default=ConnectionState.CLOSED, metadata={"serialize": False}) """The current connection state.""" # all connections have a unique id. While @@ -172,8 +172,6 @@ class Client(Connection): sockname: Address """The local address we received this connection on.""" - state: ConnectionState = field(default=ConnectionState.OPEN) - mitmcert: Optional[certs.Cert] = None """ The certificate used by mitmproxy to establish TLS with the client. @@ -265,8 +263,6 @@ class Server(Connection): """The server's resolved `(ip, port)` tuple. Will be set during connection establishment.""" sockname: Optional[Address] = None - state: ConnectionState = field(default=ConnectionState.CLOSED) - timestamp_start: Optional[float] = None """*Timestamp:* TCP SYN sent.""" timestamp_tcp_setup: Optional[float] = None diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index f3911b6c92..8bfd959d15 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -69,6 +69,8 @@ def __fields(cls) -> tuple[dataclasses.Field, ...]: fields = [] # noinspection PyDataclass for field in dataclasses.fields(cls): + if field.metadata.get("serialize", True) is False: + continue if isinstance(field.type, str): field.type = hints[field.name] fields.append(field) diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 7b571ad902..466c14e03d 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -414,6 +414,13 @@ def convert_18_19(data): return data +def convert_19_20(data): + data["version"] = 20 + data["client_conn"].pop("state", None) + data["server_conn"].pop("state", None) + return data + + def _convert_dict_keys(o: Any) -> Any: if isinstance(o, dict): return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()} @@ -477,6 +484,7 @@ def convert_unicode(data: dict) -> dict: 16: convert_16_17, 17: convert_17_18, 18: convert_18_19, + 19: convert_19_20, } diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 402f46a268..1b2c900ecf 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -388,6 +388,7 @@ def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> No timestamp_start=time.time(), transport_protocol="tcp", proxy_mode=context.client.proxy_mode, + state=connection.ConnectionState.OPEN, ) # unidirectional client streams are not fully open, set the appropriate state @@ -1073,7 +1074,8 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo if tls_clienthello.ignore_connection: self.conn = self.tunnel_connection = connection.Client( peername=("ignore-conn", 0), sockname=("ignore-conn", 0), - transport_protocol="udp" + transport_protocol="udp", + state=connection.ConnectionState.OPEN, ) # we need to replace the server layer as well, if there is one diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index b5ff406b17..24959848c5 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -425,6 +425,7 @@ def __init__( sockname=writer.get_extra_info("sockname"), timestamp_start=time.time(), proxy_mode=mode, + state=ConnectionState.OPEN, ) context = Context(client, options) super().__init__(context) diff --git a/mitmproxy/version.py b/mitmproxy/version.py index b4c46ef150..6022c261d9 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -7,7 +7,7 @@ # Serialization format version. This is displayed nowhere, it just needs to be incremented by one # for each change in the file format. -FLOW_FORMAT_VERSION = 19 +FLOW_FORMAT_VERSION = 20 def get_dev_version() -> str: diff --git a/test/mitmproxy/proxy/conftest.py b/test/mitmproxy/proxy/conftest.py index 77713aa226..fe8f564b53 100644 --- a/test/mitmproxy/proxy/conftest.py +++ b/test/mitmproxy/proxy/conftest.py @@ -13,7 +13,9 @@ def tctx() -> context.Context: opts = options.Options() Proxyserver().load(opts) return context.Context( - connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), opts + connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, state=connection.ConnectionState.OPEN), + opts ) diff --git a/test/mitmproxy/proxy/layers/test_modes.py b/test/mitmproxy/proxy/layers/test_modes.py index 43e1ed7e69..96ca863f5a 100644 --- a/test/mitmproxy/proxy/layers/test_modes.py +++ b/test/mitmproxy/proxy/layers/test_modes.py @@ -4,7 +4,7 @@ from mitmproxy import dns from mitmproxy.addons.proxyauth import ProxyAuth -from mitmproxy.connection import Client, Server +from mitmproxy.connection import Client, ConnectionState, Server from mitmproxy.proxy import layers from mitmproxy.proxy.commands import ( CloseConnection, @@ -45,12 +45,12 @@ def test_upstream_https(tctx): curl -x localhost:8080 -k http://example.com """ tctx1 = Context( - Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), + Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329, state=ConnectionState.OPEN), copy.deepcopy(tctx.options), ) tctx1.client.proxy_mode = ProxyMode.parse("upstream:https://example.mitmproxy.org:8081") tctx2 = Context( - Client(peername=("client", 4321), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), + Client(peername=("client", 4321), sockname=("127.0.0.1", 8080), timestamp_start=1605699329, state=ConnectionState.OPEN), copy.deepcopy(tctx.options), ) assert tctx2.client.proxy_mode == ProxyMode.parse("regular") diff --git a/test/mitmproxy/test_connection.py b/test/mitmproxy/test_connection.py index 5bc0aeeb98..300015c76e 100644 --- a/test/mitmproxy/test_connection.py +++ b/test/mitmproxy/test_connection.py @@ -6,7 +6,9 @@ class TestConnection: def test_basic(self): - c = Client(peername=("127.0.0.1", 52314), sockname=("127.0.0.1", 8080), timestamp_start=1607780791) + c = Client(peername=("127.0.0.1", 52314), sockname=("127.0.0.1", 8080), + timestamp_start=1607780791, + state=ConnectionState.OPEN) assert not c.tls_established c.timestamp_tls_setup = 1607780792 assert c.tls_established @@ -39,7 +41,7 @@ def test_basic(self): c.timestamp_tls_setup = 1607780791 assert str(c) c.alpn = b"foo" - assert str(c) == "Client(127.0.0.1:52314, state=open, alpn=foo)" + assert str(c) == "Client(127.0.0.1:52314, state=closed, alpn=foo)" def test_state(self): c = tclient_conn() From 9a41fe08ebc8179ece25aa8b95cc4a27511f0c9f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 28 Nov 2022 17:58:38 +0100 Subject: [PATCH 133/695] display quic as such in the ui --- mitmproxy/addons/dumper.py | 8 ++++++-- mitmproxy/proxy/layers/quic.py | 11 +++-------- mitmproxy/tools/console/common.py | 7 ++++++- mitmproxy/tools/console/palettes.py | 4 ++++ .../web/static/images/resourceQuicIcon.png | Bin 0 -> 1317 bytes test/mitmproxy/addons/test_dumper.py | 17 +++++++++++++++++ web/src/css/sprites.less | 4 ++++ web/src/images/resourceQuicIcon.png | Bin 0 -> 1317 bytes .../js/__tests__/components/FlowViewSpec.tsx | 4 ++-- .../__snapshots__/FlowViewSpec.tsx.snap | 14 ++++---------- .../js/components/FlowTable/FlowColumns.tsx | 3 +++ web/src/js/components/FlowView/Messages.tsx | 4 +++- web/src/js/components/FlowView/TcpMessages.tsx | 3 +-- web/src/js/components/FlowView/UdpMessages.tsx | 3 +-- 14 files changed, 54 insertions(+), 28 deletions(-) create mode 100644 mitmproxy/tools/web/static/images/resourceQuicIcon.png create mode 100644 web/src/images/resourceQuicIcon.png diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index a947199466..8b290d1717 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -336,16 +336,20 @@ def tcp_error(self, f): def udp_error(self, f): self._proto_error(f) - def _proto_message(self, f): + def _proto_message(self, f: Union[TCPFlow, UDPFlow]) -> None: if self.match(f): message = f.messages[-1] direction = "->" if message.from_client else "<-" + if f.client_conn.tls_version == "QUIC": + type_ = f"quic/{f.type}" + else: + type_ = f.type self.echo( "{client} {direction} {type} {direction} {server}".format( client=human.format_address(f.client_conn.peername), server=human.format_address(f.server_conn.address), direction=direction, - type=f.type, + type=type_, ) ) if ctx.options.flow_detail >= 3: diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 1b2c900ecf..657df37669 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -382,14 +382,9 @@ class QuicStreamLayer(layer.Layer): def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> None: # we mustn't reuse the client from the QUIC connection, as the state and protocol differs - self.client = context.client = connection.Client( - peername=context.client.peername, - sockname=context.client.sockname, - timestamp_start=time.time(), - transport_protocol="tcp", - proxy_mode=context.client.proxy_mode, - state=connection.ConnectionState.OPEN, - ) + self.client = context.client = context.client.copy() + self.client.transport_protocol = "tcp" + self.client.state = connection.ConnectionState.OPEN # unidirectional client streams are not fully open, set the appropriate state if stream_is_unidirectional(stream_id): diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 94ff65836a..65c12db15d 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -119,6 +119,7 @@ def fcol(s: str, attr: str) -> tuple[str, int, urwid.Text]: "tcp": "scheme_tcp", "udp": "scheme_udp", "dns": "scheme_dns", + "quic": "scheme_quic", } HTTP_REQUEST_METHOD_STYLES = { "GET": "method_get", @@ -763,12 +764,16 @@ def format_flow( duration = f.messages[-1].timestamp - f.client_conn.timestamp_start else: duration = None + if f.client_conn.tls_version == "QUIC": + protocol = "quic" + else: + protocol = f.type return format_message_flow( render_mode=render_mode, focused=focused, timestamp_start=f.client_conn.timestamp_start, marked=f.marked, - protocol=f.type, + protocol=protocol, client_address=f.client_conn.peername, server_address=f.server_conn.address, total_size=total_size, diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index f51855c35d..afbec3a011 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -40,6 +40,7 @@ class Palette: "scheme_tcp", "scheme_udp", "scheme_dns", + "scheme_quic", "scheme_other", "url_punctuation", "url_domain", @@ -180,6 +181,7 @@ class LowDark(Palette): scheme_tcp=("dark magenta", "default"), scheme_udp=("dark magenta", "default"), scheme_dns=("dark blue", "default"), + scheme_quic=("brown", "default"), scheme_other=("dark magenta", "default"), url_punctuation=("light gray", "default"), url_domain=("white", "default"), @@ -280,6 +282,7 @@ class LowLight(Palette): scheme_tcp=("light magenta", "default"), scheme_udp=("light magenta", "default"), scheme_dns=("light blue", "default"), + scheme_quic=("brown", "default"), scheme_other=("light magenta", "default"), url_punctuation=("dark gray", "default"), url_domain=("dark gray", "default"), @@ -401,6 +404,7 @@ class SolarizedLight(LowLight): scheme_tcp=("light magenta", "default"), scheme_udp=("light magenta", "default"), scheme_dns=("light blue", "default"), + scheme_quic=(sol_orange, "default"), scheme_other=("light magenta", "default"), url_punctuation=("dark gray", "default"), url_domain=("dark gray", "default"), diff --git a/mitmproxy/tools/web/static/images/resourceQuicIcon.png b/mitmproxy/tools/web/static/images/resourceQuicIcon.png new file mode 100644 index 0000000000000000000000000000000000000000..b5c871c546f441461c0106dbc6f81039c8c4ac4d GIT binary patch literal 1317 zcma)4X;4#F6n+F47Pqi;kYyBTsTH+aH)JsZij_qHfoh0$R1^z^2}=NRVOYYXEP@EF z>?)QP8A{L=TOZw%003M3e0@S-w%3-iKD_$W=22nN!-t&o)?@YVO^1z1yl*%j05(}@OAn~1+718+ z)8JEKC*VyO=H}*}9|DC!!2!IY{{vmizu`yL*47ptWoBjuL6AnHQLEJ|m1=!`eQj+` zsZ@d>xVpLuHz*W}m6a7($mMdGOa==`CWMN1L)m6f|3jz>t$tmt?yFWm@&%U{K@Lqr zKti{H)dAAd(vr6M763KJL9D+ul_*d=Ef;YVtr_ySjo^!PsZ^?M==>3SatV z)`o(&>ZMuL%WNg%*?Mj0l3;w9MUn8Bl96_iNTm4_r=03t7OyFG&MPR`8C9335H zGMPg|LktFkL?U%{bx|mk&d$z;hKAwc;oRI@I-O3XQgd>01_lP;JLrH!BH;(pL6=xs zTIxdgxbP=%L`n(?3wzX9U>s;2MCJW zusTk7Z}IP?P9^Cm`Qv=Ur)-G-!Li>NpiBRF>W8rL4CpYCeg%K{H9sG(u+*x?qUiG> zUfYSibo#;b&y%)_{e})l;~9R~+ShkHv;2pfN6un9oa8wVB3zt3_w<>A{?L^}`aOM- zX)^UquIx4bxM}yz31U@2qDYRt!!;_wr(~qgx>Nmg98t)$^&z!wwgv6Buej*wwc6WRcLj@%zMvL3XHTE79 z12pr9Mb<;iQYh_^_XGD%tJpTH{R!4$T%@R}^&)w~-r@PauFu_Rfu-a+mrrZ|iY!A3 zTbk;28ryXx<#9Iy4&@R%y9kWU>H!a)9wO*^0X;5!&7CFwWo##a2)KTi-e9#^so#D8 ziMih7K_w5bfQbi?h1nS$^c!g3HoMc1Vz*7?)QP8A{L=TOZw%003M3e0@S-w%3-iKD_$W=22nN!-t&o)?@YVO^1z1yl*%j05(}@OAn~1+718+ z)8JEKC*VyO=H}*}9|DC!!2!IY{{vmizu`yL*47ptWoBjuL6AnHQLEJ|m1=!`eQj+` zsZ@d>xVpLuHz*W}m6a7($mMdGOa==`CWMN1L)m6f|3jz>t$tmt?yFWm@&%U{K@Lqr zKti{H)dAAd(vr6M763KJL9D+ul_*d=Ef;YVtr_ySjo^!PsZ^?M==>3SatV z)`o(&>ZMuL%WNg%*?Mj0l3;w9MUn8Bl96_iNTm4_r=03t7OyFG&MPR`8C9335H zGMPg|LktFkL?U%{bx|mk&d$z;hKAwc;oRI@I-O3XQgd>01_lP;JLrH!BH;(pL6=xs zTIxdgxbP=%L`n(?3wzX9U>s;2MCJW zusTk7Z}IP?P9^Cm`Qv=Ur)-G-!Li>NpiBRF>W8rL4CpYCeg%K{H9sG(u+*x?qUiG> zUfYSibo#;b&y%)_{e})l;~9R~+ShkHv;2pfN6un9oa8wVB3zt3_w<>A{?L^}`aOM- zX)^UquIx4bxM}yz31U@2qDYRt!!;_wr(~qgx>Nmg98t)$^&z!wwgv6Buej*wwc6WRcLj@%zMvL3XHTE79 z12pr9Mb<;iQYh_^_XGD%tJpTH{R!4$T%@R}^&)w~-r@PauFu_Rfu-a+mrrZ|iY!A3 zTbk;28ryXx<#9Iy4&@R%y9kWU>H!a)9wO*^0X;5!&7CFwWo##a2)KTi-e9#^so#D8 ziMih7K_w5bfQbi?h1nS$^c!g3HoMc1Vz*7 { store.dispatch(flowActions.select(store.getState().flows.list[2].id)); - fireEvent.click(screen.getByText("TCP Messages")); + fireEvent.click(screen.getByText("Stream Data")); expect(asFragment()).toMatchSnapshot(); fireEvent.click(screen.getByText("Error")); @@ -49,7 +49,7 @@ test("FlowView", async () => { store.dispatch(flowActions.select(store.getState().flows.list[4].id)); - fireEvent.click(screen.getByText("UDP Messages")); + fireEvent.click(screen.getByText("Datagrams")); expect(asFragment()).toMatchSnapshot(); fireEvent.click(screen.getByText("Error")); diff --git a/web/src/js/__tests__/components/__snapshots__/FlowViewSpec.tsx.snap b/web/src/js/__tests__/components/__snapshots__/FlowViewSpec.tsx.snap index 6c8e33a20e..c748ffd718 100644 --- a/web/src/js/__tests__/components/__snapshots__/FlowViewSpec.tsx.snap +++ b/web/src/js/__tests__/components/__snapshots__/FlowViewSpec.tsx.snap @@ -1006,7 +1006,7 @@ exports[`FlowView 7`] = ` class="active" href="#" > - TCP Messages + Stream Data -

- TCP Data -

@@ -1079,7 +1076,7 @@ exports[`FlowView 8`] = ` class="" href="#" > - TCP Messages + Stream Data - UDP Messages + Datagrams -

- UDP Data -

@@ -1516,7 +1510,7 @@ exports[`FlowView 13`] = ` class="" href="#" > - UDP Messages + Datagrams getIcon(flow) const getIcon = (flow: Flow): string => { if (flow.type !== "http") { + if (flow.client_conn.tls_version === "QUIC") { + return `resource-icon-quic`; + } return `resource-icon-${flow.type}` } if (flow.websocket) { diff --git a/web/src/js/components/FlowView/Messages.tsx b/web/src/js/components/FlowView/Messages.tsx index 89ae95cdaa..1f5f5a8c91 100644 --- a/web/src/js/components/FlowView/Messages.tsx +++ b/web/src/js/components/FlowView/Messages.tsx @@ -29,7 +29,9 @@ export default function Messages({flow, messages_meta}: MessagesPropTypes) { try { return JSON.parse(content) } catch (e) { - const err: ContentViewData = {"description": "Network Error", lines: [[["error", `${content}`]]]}; + const err: ContentViewData[] = [ + {"description": "Network Error", lines: [[["error", `${content}`]]]} + ]; return err; } } diff --git a/web/src/js/components/FlowView/TcpMessages.tsx b/web/src/js/components/FlowView/TcpMessages.tsx index ce04beb884..115caf7d3d 100644 --- a/web/src/js/components/FlowView/TcpMessages.tsx +++ b/web/src/js/components/FlowView/TcpMessages.tsx @@ -6,9 +6,8 @@ import Messages from "./Messages"; export default function TcpMessages({flow}: { flow: TCPFlow }) { return (
-

TCP Data

) } -TcpMessages.displayName = "TCP Messages" +TcpMessages.displayName = "Stream Data" diff --git a/web/src/js/components/FlowView/UdpMessages.tsx b/web/src/js/components/FlowView/UdpMessages.tsx index d40900406f..0dafda693e 100644 --- a/web/src/js/components/FlowView/UdpMessages.tsx +++ b/web/src/js/components/FlowView/UdpMessages.tsx @@ -6,9 +6,8 @@ import Messages from "./Messages"; export default function UdpMessages({flow}: { flow: UDPFlow }) { return (
-

UDP Data

) } -UdpMessages.displayName = "UDP Messages" +UdpMessages.displayName = "Datagrams" From d0f297f6a54179559d7d227266f579ac13a82250 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 28 Nov 2022 18:13:35 +0100 Subject: [PATCH 134/695] update mitmweb assets --- mitmproxy/tools/web/static/app.css | 2 +- mitmproxy/tools/web/static/app.js | 50 +++++++++++++++--------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/mitmproxy/tools/web/static/app.css b/mitmproxy/tools/web/static/app.css index 6cb8d58282..bfe8297430 100644 --- a/mitmproxy/tools/web/static/app.css +++ b/mitmproxy/tools/web/static/app.css @@ -1,2 +1,2 @@ -html{box-sizing:border-box}*,:after,:before{box-sizing:inherit}.resource-icon{width:32px;height:32px}.resource-icon-css{background-image:url(images/chrome-devtools/resourceCSSIcon.png)}.resource-icon-document{background-image:url(images/chrome-devtools/resourceDocumentIcon.png)}.resource-icon-js{background-image:url(images/chrome-devtools/resourceJSIcon.png)}.resource-icon-plain{background-image:url(images/chrome-devtools/resourcePlainIcon.png)}.resource-icon-executable{background-image:url(images/resourceExecutableIcon.png)}.resource-icon-flash{background-image:url(images/resourceFlashIcon.png)}.resource-icon-image{background-image:url(images/resourceImageIcon.png)}.resource-icon-java{background-image:url(images/resourceJavaIcon.png)}.resource-icon-not-modified{background-image:url(images/resourceNotModifiedIcon.png)}.resource-icon-redirect{background-image:url(images/resourceRedirectIcon.png)}.resource-icon-websocket{background-image:url(images/resourceWebSocketIcon.png)}.resource-icon-tcp{background-image:url(images/resourceTcpIcon.png)}.resource-icon-udp{background-image:url(images/resourceUdpIcon.png)}.resource-icon-dns{background-image:url(images/resourceDnsIcon.png)}#container,#mitmproxy,body,html{height:100%;margin:0;overflow:hidden}#container{display:flex;flex-direction:column;outline:0}#container>.eventlog,#container>footer,#container>header{flex:0 0 auto}.main-view{flex:1 1 auto;height:0;display:flex;flex-direction:row}.main-view.vertical{flex-direction:column}.main-view .flow-detail,.main-view .flow-table{flex:1 1 auto}.splitter{flex:0 0 1px;background-color:#aaa;position:relative}.splitter>div{position:absolute}.splitter.splitter-x{cursor:col-resize}.splitter.splitter-x>div{margin-left:-1px;width:4px;height:100%}.splitter.splitter-y{cursor:row-resize}.splitter.splitter-y>div{margin-top:-1px;height:4px;width:100%}.nav-tabs{border-bottom:solid #a6a6a6 1px}.nav-tabs>a{display:inline-block;border:solid transparent 1px;text-decoration:none}.nav-tabs>a.active{background-color:#fff;border-color:#a6a6a6;border-bottom-color:#fff}.nav-tabs>a.special{color:#fff;background-color:#396cad;border-bottom-color:#396cad}.nav-tabs>a.special:hover{background-color:#5386c6}.nav-tabs-lg>a{padding:3px 14px;margin:0 2px -1px}.nav-tabs-sm>a{padding:0 7px;margin:2px 2px -1px}header{padding-top:6px;background-color:#fff}header>div{display:block;margin:0;padding:0;border-bottom:solid #a6a6a6 1px;height:95px;overflow:visible}.menu-group{margin:0 5px 0 6px;display:inline-block;height:95px}.menu-content{height:79px;display:flow-root}.menu-content>a{display:inline-block}.menu-content>.btn,.menu-content>a>.btn{height:79px;text-align:center;margin:0 1px;padding:12px 5px;border:none;border-radius:0}.menu-content>.btn i,.menu-content>a>.btn i{font-size:20px;display:block;margin:0 auto 5px}.menu-content>.btn.btn-sm{height:26.33333333px;padding:0 5px}.menu-content>.btn.btn-sm i{display:inline-block;font-size:14px;margin:0}.menu-entry{text-align:left;height:26.33333333px;line-height:1;padding:.5rem 1rem}.menu-entry label{font-size:1.2rem;font-weight:400;margin:0}.menu-entry input[type=checkbox]{margin:0 2px;vertical-align:middle}.menu-legend{color:#777;height:16px;text-align:center;font-size:12px;padding:0 5px}.menu-group+.menu-group:before{margin-left:-6px;content:" ";border-left:solid 1px #e6e6e6;margin-top:10px;height:75px;position:absolute}.main-menu{display:flex}.main-menu .menu-group{width:50%}.main-menu .btn-sm{margin-top:6px}.filter-input{margin:4px 0}.filter-input .popover{top:27px;left:43px;display:block;max-width:none;opacity:.9}@media (max-width:767px){.filter-input .popover{top:16px;left:29px;right:2px}}.filter-input .popover .popover-content{max-height:500px;overflow-y:auto}.filter-input .popover .popover-content tr{cursor:pointer}.filter-input .popover .popover-content tr:hover{background-color:hsla(209,52%,84%,.5)!important}.connection-indicator{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em;float:right;margin:5px;opacity:1;transition:all 1s linear}a.connection-indicator:focus,a.connection-indicator:hover{color:#fff;text-decoration:none;cursor:pointer}.connection-indicator:empty{display:none}.btn .connection-indicator{position:relative;top:-1px}.connection-indicator.fetching,.connection-indicator.init{background-color:#5bc0de}.connection-indicator.established{background-color:#5cb85c;opacity:0}.connection-indicator.error{background-color:#d9534f;transition:all .2s linear}.connection-indicator.offline{background-color:#f0ad4e;opacity:1}.flow-table{width:100%;overflow-y:scroll;overflow-x:hidden}.flow-table table{width:100%;table-layout:fixed}.flow-table thead tr{background-color:#f2f2f2;border-bottom:solid #bebebe 1px;line-height:23px}.flow-table th{font-weight:400;position:relative!important;padding-left:1px;-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.flow-table th.sort-asc,.flow-table th.sort-desc{background-color:#fafafa}.flow-table th.sort-asc:after,.flow-table th.sort-desc:after{font:normal normal normal 14px/1 FontAwesome;position:absolute;right:3px;top:3px;padding:2px;background-color:rgba(250,250,250,.8)}.flow-table th.sort-asc:after{content:"\f0de"}.flow-table th.sort-desc:after{content:"\f0dd"}.flow-table tr{cursor:pointer;background-color:#fff}.flow-table tr:nth-child(even){background-color:#f2f2f2}.flow-table tr.selected{background-color:#e0ebf5!important}.flow-table tr.selected.highlighted{background-color:#7bbefc!important}.flow-table tr.highlighted{background-color:#ffeb99}.flow-table tr.highlighted:nth-child(even){background-color:#ffe57f}.flow-table td{overflow:hidden;white-space:nowrap;text-overflow:ellipsis}.flow-table tr.intercepted:not(.has-response) .col-method,.flow-table tr.intercepted:not(.has-response) .col-path{color:#ff7f00}.flow-table tr.intercepted.has-response .col-size,.flow-table tr.intercepted.has-response .col-status,.flow-table tr.intercepted.has-response .col-time{color:#ff7f00}.flow-table .fa{line-height:inherit}.flow-table .col-tls{width:10px}.flow-table .col-tls-https{background-color:rgba(0,185,0,.5)}.flow-table .col-icon{width:32px}.flow-table .col-path .fa{margin-left:0;font-size:16px}.flow-table .col-path .fa-repeat{color:green}.flow-table .col-path .fa-pause{color:#ff7f00}.flow-table .col-path .fa-exclamation,.flow-table .col-path .fa-times{color:#8b0000}.flow-table .col-method{width:60px}.flow-table .col-status{width:50px}.flow-table .col-size{width:70px}.flow-table .col-time{width:50px}.flow-table .col-timestamp{width:170px}.flow-table td.col-size,.flow-table td.col-time,.flow-table td.col-timestamp{text-align:right}.flow-table .col-quickactions{width:0;direction:rtl;overflow:hidden;background-color:inherit;font-size:20px}.flow-table .col-quickactions *{direction:ltr}.flow-table .col-quickactions.hover,.flow-table tr:hover .col-quickactions{overflow:visible}.flow-table .col-quickactions>div{height:32px;background-color:inherit;display:inline-flex;align-items:center}.flow-table .col-quickactions>div>a{margin-right:2px;height:32px;width:32px;border-radius:16px;text-align:center}.flow-table .col-quickactions>div>a:hover{background-color:rgba(0,0,0,.05)}.flow-table .col-quickactions .fa-play{transform:translate(1px,2px)}.flow-table .col-quickactions .fa-repeat{transform:translate(0,2px)}.flow-detail{width:100%;overflow:hidden;display:flex;flex-direction:column}.flow-detail nav{background-color:#f2f2f2}.flow-detail section{overflow-y:scroll;flex:1;padding:5px 12px 10px}.flow-detail section>footer{box-shadow:0 0 3px gray;padding:2px;margin:0;height:23px}.flow-detail .first-line{font-family:Menlo,Monaco,Consolas,"Courier New",monospace;background-color:#428bca;color:#fff;margin:0 -8px 2px;padding:4px 8px;border-radius:5px;word-break:break-all;max-height:100px;overflow-y:auto}.flow-detail .contentview{margin:0 -12px;padding:0 12px}.flow-detail .contentview .controls{display:flex;align-items:center}.flow-detail .contentview .controls h5{flex:1;font-size:12px;font-weight:700;margin:10px 0}.flow-detail .contentview pre button:not(:only-child){margin-top:6px}.flow-detail hr{margin:0}.inline-input{display:inline;margin:0 -3px;padding:0 3px;border:solid transparent 1px}.inline-input:hover{box-shadow:0 0 0 1px rgba(0,0,0,.0125),0 2px 4px rgba(0,0,0,.05),0 2px 6px rgba(0,0,0,.025);background-color:rgba(255,255,255,.1)}.inline-input[placeholder]:empty:not(:focus-visible):before{content:attr(placeholder);color:#d3d3d3;font-style:italic}.inline-input[contenteditable]{outline-width:0;box-shadow:0 0 0 1px rgba(0,0,0,.05),0 2px 4px rgba(0,0,0,.2),0 2px 6px rgba(0,0,0,.1);background-color:rgba(255,255,255,.2)}.inline-input[contenteditable].has-warning{color:#ffb8b8}.certificate-table,.connection-table,.timing-table{width:100%;table-layout:fixed;word-break:break-all}.certificate-table td:nth-child(2),.connection-table td:nth-child(2),.timing-table td:nth-child(2){font-family:Menlo,Monaco,Consolas,"Courier New",monospace;width:70%}.certificate-table tr:not(:first-child),.connection-table tr:not(:first-child),.timing-table tr:not(:first-child){border-top:1px solid #f7f7f7}.certificate-table td,.connection-table td,.timing-table td{vertical-align:top}.connection-table td:first-child{padding-right:1em}.headers,.trailers{position:relative;min-height:2ex;overflow-wrap:break-word}.headers .kv-row,.trailers .kv-row{margin-bottom:.3em;max-height:12.4ex;overflow-y:auto}.headers .kv-key,.trailers .kv-key{font-weight:700}.headers .kv-value,.trailers .kv-value{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}.headers .inline-input,.trailers .inline-input{background-color:#fff}.headers .kv-add-row,.trailers .kv-add-row{opacity:0;color:#666;position:absolute;bottom:4px;right:4px;transition:all .1s ease-in-out}.headers:hover .kv-add-row,.trailers:hover .kv-add-row{opacity:1}.connection-table td,.timing-table td{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}dl.cert-attributes{display:flex;flex-flow:row;flex-wrap:wrap;margin-bottom:0}dl.cert-attributes dd,dl.cert-attributes dt{text-overflow:ellipsis;overflow:hidden}dl.cert-attributes dt{flex:0 0 2em}dl.cert-attributes dd{flex:0 0 calc(100% - 2em)}.dns-request table td,.dns-request table th,.dns-response table td,.dns-response table th{padding-right:1rem}.flowview-image{text-align:center;padding:10px 0}.flowview-image img{max-width:100%;max-height:100%}.edit-flow-container{position:fixed;right:20px}.edit-flow{cursor:pointer;position:absolute;right:0;top:5px;height:40px;width:40px;border-radius:20px;z-index:10000;background-color:rgba(255,255,255,.7);border:solid 2px rgba(248,145,59,.7);text-align:center;font-size:22px;line-height:37px;transition:all .1s ease-in-out}.edit-flow:hover{background-color:rgba(239,108,0,.7);color:rgba(0,0,0,.8);border:solid 2px transparent}.eventlog{height:200px;flex:0 0 auto;display:flex;flex-direction:column}.eventlog>div{background-color:#f2f2f2;padding:0 5px;flex:0 0 auto;border-top:1px solid #aaa;cursor:row-resize}.eventlog>pre{flex:1 1 auto;margin:0;border-radius:0;overflow-x:auto;overflow-y:scroll;background-color:#fcfcfc}.eventlog .fa-close{cursor:pointer;float:right;color:grey;padding:3px 0;padding-left:10px}.eventlog .fa-close:hover{color:#000}.eventlog .btn-toggle{margin-top:-2px;margin-left:3px;padding:2px 2px;font-size:10px;line-height:10px;border-radius:2px}.eventlog .label{cursor:pointer;vertical-align:middle;display:inline-block;margin-top:-2px;margin-left:3px}footer{box-shadow:0 -1px 3px #d3d3d3;padding:0 0 4px 3px}footer .label{margin-right:3px}.CodeMirror{border:1px solid #ccc;height:auto!important}.CodeMirror{font-family:monospace;height:300px;color:#000;direction:ltr}.CodeMirror-lines{padding:4px 0}.CodeMirror pre.CodeMirror-line,.CodeMirror pre.CodeMirror-line-like{padding:0 4px}.CodeMirror-gutter-filler,.CodeMirror-scrollbar-filler{background-color:#fff}.CodeMirror-gutters{border-right:1px solid #ddd;background-color:#f7f7f7;white-space:nowrap}.CodeMirror-linenumber{padding:0 3px 0 5px;min-width:20px;text-align:right;color:#999;white-space:nowrap}.CodeMirror-guttermarker{color:#000}.CodeMirror-guttermarker-subtle{color:#999}.CodeMirror-cursor{border-left:1px solid #000;border-right:none;width:0}.CodeMirror div.CodeMirror-secondarycursor{border-left:1px solid silver}.cm-fat-cursor .CodeMirror-cursor{width:auto;border:0!important;background:#7e7}.cm-fat-cursor div.CodeMirror-cursors{z-index:1}.cm-fat-cursor-mark{background-color:rgba(20,255,20,.5);-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite}.cm-animate-fat-cursor{width:auto;-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite;background-color:#7e7}@-moz-keyframes blink{50%{background-color:transparent}}@-webkit-keyframes blink{50%{background-color:transparent}}@keyframes blink{50%{background-color:transparent}}.cm-tab{display:inline-block;text-decoration:inherit}.CodeMirror-rulers{position:absolute;left:0;right:0;top:-50px;bottom:0;overflow:hidden}.CodeMirror-ruler{border-left:1px solid #ccc;top:0;bottom:0;position:absolute}.cm-s-default .cm-header{color:#00f}.cm-s-default .cm-quote{color:#090}.cm-negative{color:#d44}.cm-positive{color:#292}.cm-header,.cm-strong{font-weight:700}.cm-em{font-style:italic}.cm-link{text-decoration:underline}.cm-strikethrough{text-decoration:line-through}.cm-s-default .cm-keyword{color:#708}.cm-s-default .cm-atom{color:#219}.cm-s-default .cm-number{color:#164}.cm-s-default .cm-def{color:#00f}.cm-s-default .cm-variable-2{color:#05a}.cm-s-default .cm-type,.cm-s-default .cm-variable-3{color:#085}.cm-s-default .cm-comment{color:#a50}.cm-s-default .cm-string{color:#a11}.cm-s-default .cm-string-2{color:#f50}.cm-s-default .cm-meta{color:#555}.cm-s-default .cm-qualifier{color:#555}.cm-s-default .cm-builtin{color:#30a}.cm-s-default .cm-bracket{color:#997}.cm-s-default .cm-tag{color:#170}.cm-s-default .cm-attribute{color:#00c}.cm-s-default .cm-hr{color:#999}.cm-s-default .cm-link{color:#00c}.cm-s-default .cm-error{color:red}.cm-invalidchar{color:red}.CodeMirror-composing{border-bottom:2px solid}div.CodeMirror span.CodeMirror-matchingbracket{color:#0b0}div.CodeMirror span.CodeMirror-nonmatchingbracket{color:#a22}.CodeMirror-matchingtag{background:rgba(255,150,0,.3)}.CodeMirror-activeline-background{background:#e8f2ff}.CodeMirror{position:relative;overflow:hidden;background:#fff}.CodeMirror-scroll{overflow:scroll!important;margin-bottom:-50px;margin-right:-50px;padding-bottom:50px;height:100%;outline:0;position:relative}.CodeMirror-sizer{position:relative;border-right:50px solid transparent}.CodeMirror-gutter-filler,.CodeMirror-hscrollbar,.CodeMirror-scrollbar-filler,.CodeMirror-vscrollbar{position:absolute;z-index:6;display:none;outline:0}.CodeMirror-vscrollbar{right:0;top:0;overflow-x:hidden;overflow-y:scroll}.CodeMirror-hscrollbar{bottom:0;left:0;overflow-y:hidden;overflow-x:scroll}.CodeMirror-scrollbar-filler{right:0;bottom:0}.CodeMirror-gutter-filler{left:0;bottom:0}.CodeMirror-gutters{position:absolute;left:0;top:0;min-height:100%;z-index:3}.CodeMirror-gutter{white-space:normal;height:100%;display:inline-block;vertical-align:top;margin-bottom:-50px}.CodeMirror-gutter-wrapper{position:absolute;z-index:4;background:0 0!important;border:none!important}.CodeMirror-gutter-background{position:absolute;top:0;bottom:0;z-index:4}.CodeMirror-gutter-elt{position:absolute;cursor:default;z-index:4}.CodeMirror-gutter-wrapper ::selection{background-color:transparent}.CodeMirror-gutter-wrapper ::-moz-selection{background-color:transparent}.CodeMirror-lines{cursor:text;min-height:1px}.CodeMirror pre.CodeMirror-line,.CodeMirror pre.CodeMirror-line-like{-moz-border-radius:0;-webkit-border-radius:0;border-radius:0;border-width:0;background:0 0;font-family:inherit;font-size:inherit;margin:0;white-space:pre;word-wrap:normal;line-height:inherit;color:inherit;z-index:2;position:relative;overflow:visible;-webkit-tap-highlight-color:transparent;-webkit-font-variant-ligatures:contextual;font-variant-ligatures:contextual}.CodeMirror-wrap pre.CodeMirror-line,.CodeMirror-wrap pre.CodeMirror-line-like{word-wrap:break-word;white-space:pre-wrap;word-break:normal}.CodeMirror-linebackground{position:absolute;left:0;right:0;top:0;bottom:0;z-index:0}.CodeMirror-linewidget{position:relative;z-index:2;padding:.1px}.CodeMirror-rtl pre{direction:rtl}.CodeMirror-code{outline:0}.CodeMirror-gutter,.CodeMirror-gutters,.CodeMirror-linenumber,.CodeMirror-scroll,.CodeMirror-sizer{-moz-box-sizing:content-box;box-sizing:content-box}.CodeMirror-measure{position:absolute;width:100%;height:0;overflow:hidden;visibility:hidden}.CodeMirror-cursor{position:absolute;pointer-events:none}.CodeMirror-measure pre{position:static}div.CodeMirror-cursors{visibility:hidden;position:relative;z-index:3}div.CodeMirror-dragcursors{visibility:visible}.CodeMirror-focused div.CodeMirror-cursors{visibility:visible}.CodeMirror-selected{background:#d9d9d9}.CodeMirror-focused .CodeMirror-selected{background:#d7d4f0}.CodeMirror-crosshair{cursor:crosshair}.CodeMirror-line::selection,.CodeMirror-line>span::selection,.CodeMirror-line>span>span::selection{background:#d7d4f0}.CodeMirror-line::-moz-selection,.CodeMirror-line>span::-moz-selection,.CodeMirror-line>span>span::-moz-selection{background:#d7d4f0}.cm-searching{background-color:#ffa;background-color:rgba(255,255,0,.4)}.cm-force-border{padding-right:.1px}@media print{.CodeMirror div.CodeMirror-cursors{visibility:hidden}}.cm-tab-wrap-hack:after{content:''}span.CodeMirror-selectedtext{background:0 0}.contentview .header{font-weight:700}.contentview .highlight{font-weight:700}.contentview .offset{color:#00f}.contentview .codeeditor{margin-bottom:12px}.contentview .Token_Name_Tag{color:#006400}.contentview .Token_Literal_String{color:#b22222}.contentview .Token_Literal_Number{color:purple}.contentview .Token_Keyword_Constant{color:#00f}.modal-visible{display:block}.modal-dialog{overflow-y:initial!important}.modal-body{max-height:calc(100vh - 200px);overflow-y:auto}.dropdown-menu{margin:0!important}.dropdown-menu>li>a{padding:3px 10px}.command-title{background-color:#f2f2f2;border:1px solid #aaa}.command-result{display:block;margin:0;background-color:#fcfcfc;height:100px;max-height:100px;overflow:auto}.command-suggestion{background-color:#9c9c9c}.argument-suggestion{background-color:hsla(209,52%,84%,.5)!important}.command>.popover{display:block;position:relative;max-width:none}.available-commands{overflow:auto}.wireguard-config{margin:1rem 0;display:flex;flex-wrap:wrap;column-gap:2rem;align-items:center}.wireguard-config>*{margin:0} +html{box-sizing:border-box}*,:after,:before{box-sizing:inherit}.resource-icon{width:32px;height:32px}.resource-icon-css{background-image:url(images/chrome-devtools/resourceCSSIcon.png)}.resource-icon-document{background-image:url(images/chrome-devtools/resourceDocumentIcon.png)}.resource-icon-js{background-image:url(images/chrome-devtools/resourceJSIcon.png)}.resource-icon-plain{background-image:url(images/chrome-devtools/resourcePlainIcon.png)}.resource-icon-executable{background-image:url(images/resourceExecutableIcon.png)}.resource-icon-flash{background-image:url(images/resourceFlashIcon.png)}.resource-icon-image{background-image:url(images/resourceImageIcon.png)}.resource-icon-java{background-image:url(images/resourceJavaIcon.png)}.resource-icon-not-modified{background-image:url(images/resourceNotModifiedIcon.png)}.resource-icon-redirect{background-image:url(images/resourceRedirectIcon.png)}.resource-icon-websocket{background-image:url(images/resourceWebSocketIcon.png)}.resource-icon-tcp{background-image:url(images/resourceTcpIcon.png)}.resource-icon-udp{background-image:url(images/resourceUdpIcon.png)}.resource-icon-dns{background-image:url(images/resourceDnsIcon.png)}.resource-icon-quic{background-image:url(images/resourceQuicIcon.png)}#container,#mitmproxy,body,html{height:100%;margin:0;overflow:hidden}#container{display:flex;flex-direction:column;outline:0}#container>.eventlog,#container>footer,#container>header{flex:0 0 auto}.main-view{flex:1 1 auto;height:0;display:flex;flex-direction:row}.main-view.vertical{flex-direction:column}.main-view .flow-detail,.main-view .flow-table{flex:1 1 auto}.splitter{flex:0 0 1px;background-color:#aaa;position:relative}.splitter>div{position:absolute}.splitter.splitter-x{cursor:col-resize}.splitter.splitter-x>div{margin-left:-1px;width:4px;height:100%}.splitter.splitter-y{cursor:row-resize}.splitter.splitter-y>div{margin-top:-1px;height:4px;width:100%}.nav-tabs{border-bottom:solid #a6a6a6 1px}.nav-tabs>a{display:inline-block;border:solid transparent 1px;text-decoration:none}.nav-tabs>a.active{background-color:#fff;border-color:#a6a6a6;border-bottom-color:#fff}.nav-tabs>a.special{color:#fff;background-color:#396cad;border-bottom-color:#396cad}.nav-tabs>a.special:hover{background-color:#5386c6}.nav-tabs-lg>a{padding:3px 14px;margin:0 2px -1px}.nav-tabs-sm>a{padding:0 7px;margin:2px 2px -1px}header{padding-top:6px;background-color:#fff}header>div{display:block;margin:0;padding:0;border-bottom:solid #a6a6a6 1px;height:95px;overflow:visible}.menu-group{margin:0 5px 0 6px;display:inline-block;height:95px}.menu-content{height:79px;display:flow-root}.menu-content>a{display:inline-block}.menu-content>.btn,.menu-content>a>.btn{height:79px;text-align:center;margin:0 1px;padding:12px 5px;border:none;border-radius:0}.menu-content>.btn i,.menu-content>a>.btn i{font-size:20px;display:block;margin:0 auto 5px}.menu-content>.btn.btn-sm{height:26.33333333px;padding:0 5px}.menu-content>.btn.btn-sm i{display:inline-block;font-size:14px;margin:0}.menu-entry{text-align:left;height:26.33333333px;line-height:1;padding:.5rem 1rem}.menu-entry label{font-size:1.2rem;font-weight:400;margin:0}.menu-entry input[type=checkbox]{margin:0 2px;vertical-align:middle}.menu-legend{color:#777;height:16px;text-align:center;font-size:12px;padding:0 5px}.menu-group+.menu-group:before{margin-left:-6px;content:" ";border-left:solid 1px #e6e6e6;margin-top:10px;height:75px;position:absolute}.main-menu{display:flex}.main-menu .menu-group{width:50%}.main-menu .btn-sm{margin-top:6px}.filter-input{margin:4px 0}.filter-input .popover{top:27px;left:43px;display:block;max-width:none;opacity:.9}@media (max-width:767px){.filter-input .popover{top:16px;left:29px;right:2px}}.filter-input .popover .popover-content{max-height:500px;overflow-y:auto}.filter-input .popover .popover-content tr{cursor:pointer}.filter-input .popover .popover-content tr:hover{background-color:hsla(209,52%,84%,.5)!important}.connection-indicator{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em;float:right;margin:5px;opacity:1;transition:all 1s linear}a.connection-indicator:focus,a.connection-indicator:hover{color:#fff;text-decoration:none;cursor:pointer}.connection-indicator:empty{display:none}.btn .connection-indicator{position:relative;top:-1px}.connection-indicator.fetching,.connection-indicator.init{background-color:#5bc0de}.connection-indicator.established{background-color:#5cb85c;opacity:0}.connection-indicator.error{background-color:#d9534f;transition:all .2s linear}.connection-indicator.offline{background-color:#f0ad4e;opacity:1}.flow-table{width:100%;overflow-y:scroll;overflow-x:hidden}.flow-table table{width:100%;table-layout:fixed}.flow-table thead tr{background-color:#f2f2f2;border-bottom:solid #bebebe 1px;line-height:23px}.flow-table th{font-weight:400;position:relative!important;padding-left:1px;-webkit-touch-callout:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.flow-table th.sort-asc,.flow-table th.sort-desc{background-color:#fafafa}.flow-table th.sort-asc:after,.flow-table th.sort-desc:after{font:normal normal normal 14px/1 FontAwesome;position:absolute;right:3px;top:3px;padding:2px;background-color:rgba(250,250,250,.8)}.flow-table th.sort-asc:after{content:"\f0de"}.flow-table th.sort-desc:after{content:"\f0dd"}.flow-table tr{cursor:pointer;background-color:#fff}.flow-table tr:nth-child(even){background-color:#f2f2f2}.flow-table tr.selected{background-color:#e0ebf5!important}.flow-table tr.selected.highlighted{background-color:#7bbefc!important}.flow-table tr.highlighted{background-color:#ffeb99}.flow-table tr.highlighted:nth-child(even){background-color:#ffe57f}.flow-table td{overflow:hidden;white-space:nowrap;text-overflow:ellipsis}.flow-table tr.intercepted:not(.has-response) .col-method,.flow-table tr.intercepted:not(.has-response) .col-path{color:#ff7f00}.flow-table tr.intercepted.has-response .col-size,.flow-table tr.intercepted.has-response .col-status,.flow-table tr.intercepted.has-response .col-time{color:#ff7f00}.flow-table .fa{line-height:inherit}.flow-table .col-tls{width:10px}.flow-table .col-tls-https{background-color:rgba(0,185,0,.5)}.flow-table .col-icon{width:32px}.flow-table .col-path .fa{margin-left:0;font-size:16px}.flow-table .col-path .fa-repeat{color:green}.flow-table .col-path .fa-pause{color:#ff7f00}.flow-table .col-path .fa-exclamation,.flow-table .col-path .fa-times{color:#8b0000}.flow-table .col-method{width:60px}.flow-table .col-status{width:50px}.flow-table .col-size{width:70px}.flow-table .col-time{width:50px}.flow-table .col-timestamp{width:170px}.flow-table td.col-size,.flow-table td.col-time,.flow-table td.col-timestamp{text-align:right}.flow-table .col-quickactions{width:0;direction:rtl;overflow:hidden;background-color:inherit;font-size:20px}.flow-table .col-quickactions *{direction:ltr}.flow-table .col-quickactions.hover,.flow-table tr:hover .col-quickactions{overflow:visible}.flow-table .col-quickactions>div{height:32px;background-color:inherit;display:inline-flex;align-items:center}.flow-table .col-quickactions>div>a{margin-right:2px;height:32px;width:32px;border-radius:16px;text-align:center}.flow-table .col-quickactions>div>a:hover{background-color:rgba(0,0,0,.05)}.flow-table .col-quickactions .fa-play{transform:translate(1px,2px)}.flow-table .col-quickactions .fa-repeat{transform:translate(0,2px)}.flow-detail{width:100%;overflow:hidden;display:flex;flex-direction:column}.flow-detail nav{background-color:#f2f2f2}.flow-detail section{overflow-y:scroll;flex:1;padding:5px 12px 10px}.flow-detail section>footer{box-shadow:0 0 3px gray;padding:2px;margin:0;height:23px}.flow-detail .first-line{font-family:Menlo,Monaco,Consolas,"Courier New",monospace;background-color:#428bca;color:#fff;margin:0 -8px 2px;padding:4px 8px;border-radius:5px;word-break:break-all;max-height:100px;overflow-y:auto}.flow-detail .contentview{margin:0 -12px;padding:0 12px}.flow-detail .contentview .controls{display:flex;align-items:center}.flow-detail .contentview .controls h5{flex:1;font-size:12px;font-weight:700;margin:10px 0}.flow-detail .contentview pre button:not(:only-child){margin-top:6px}.flow-detail hr{margin:0}.inline-input{display:inline;margin:0 -3px;padding:0 3px;border:solid transparent 1px}.inline-input:hover{box-shadow:0 0 0 1px rgba(0,0,0,.0125),0 2px 4px rgba(0,0,0,.05),0 2px 6px rgba(0,0,0,.025);background-color:rgba(255,255,255,.1)}.inline-input[placeholder]:empty:not(:focus-visible):before{content:attr(placeholder);color:#d3d3d3;font-style:italic}.inline-input[contenteditable]{outline-width:0;box-shadow:0 0 0 1px rgba(0,0,0,.05),0 2px 4px rgba(0,0,0,.2),0 2px 6px rgba(0,0,0,.1);background-color:rgba(255,255,255,.2)}.inline-input[contenteditable].has-warning{color:#ffb8b8}.certificate-table,.connection-table,.timing-table{width:100%;table-layout:fixed;word-break:break-all}.certificate-table td:nth-child(2),.connection-table td:nth-child(2),.timing-table td:nth-child(2){font-family:Menlo,Monaco,Consolas,"Courier New",monospace;width:70%}.certificate-table tr:not(:first-child),.connection-table tr:not(:first-child),.timing-table tr:not(:first-child){border-top:1px solid #f7f7f7}.certificate-table td,.connection-table td,.timing-table td{vertical-align:top}.connection-table td:first-child{padding-right:1em}.headers,.trailers{position:relative;min-height:2ex;overflow-wrap:break-word}.headers .kv-row,.trailers .kv-row{margin-bottom:.3em;max-height:12.4ex;overflow-y:auto}.headers .kv-key,.trailers .kv-key{font-weight:700}.headers .kv-value,.trailers .kv-value{font-family:Menlo,Monaco,Consolas,"Courier New",monospace}.headers .inline-input,.trailers .inline-input{background-color:#fff}.headers .kv-add-row,.trailers .kv-add-row{opacity:0;color:#666;position:absolute;bottom:4px;right:4px;transition:all .1s ease-in-out}.headers:hover .kv-add-row,.trailers:hover .kv-add-row{opacity:1}.connection-table td,.timing-table td{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}dl.cert-attributes{display:flex;flex-flow:row;flex-wrap:wrap;margin-bottom:0}dl.cert-attributes dd,dl.cert-attributes dt{text-overflow:ellipsis;overflow:hidden}dl.cert-attributes dt{flex:0 0 2em}dl.cert-attributes dd{flex:0 0 calc(100% - 2em)}.dns-request table td,.dns-request table th,.dns-response table td,.dns-response table th{padding-right:1rem}.flowview-image{text-align:center;padding:10px 0}.flowview-image img{max-width:100%;max-height:100%}.edit-flow-container{position:fixed;right:20px}.edit-flow{cursor:pointer;position:absolute;right:0;top:5px;height:40px;width:40px;border-radius:20px;z-index:10000;background-color:rgba(255,255,255,.7);border:solid 2px rgba(248,145,59,.7);text-align:center;font-size:22px;line-height:37px;transition:all .1s ease-in-out}.edit-flow:hover{background-color:rgba(239,108,0,.7);color:rgba(0,0,0,.8);border:solid 2px transparent}.eventlog{height:200px;flex:0 0 auto;display:flex;flex-direction:column}.eventlog>div{background-color:#f2f2f2;padding:0 5px;flex:0 0 auto;border-top:1px solid #aaa;cursor:row-resize}.eventlog>pre{flex:1 1 auto;margin:0;border-radius:0;overflow-x:auto;overflow-y:scroll;background-color:#fcfcfc}.eventlog .fa-close{cursor:pointer;float:right;color:grey;padding:3px 0;padding-left:10px}.eventlog .fa-close:hover{color:#000}.eventlog .btn-toggle{margin-top:-2px;margin-left:3px;padding:2px 2px;font-size:10px;line-height:10px;border-radius:2px}.eventlog .label{cursor:pointer;vertical-align:middle;display:inline-block;margin-top:-2px;margin-left:3px}footer{box-shadow:0 -1px 3px #d3d3d3;padding:0 0 4px 3px}footer .label{margin-right:3px}.CodeMirror{border:1px solid #ccc;height:auto!important}.CodeMirror{font-family:monospace;height:300px;color:#000;direction:ltr}.CodeMirror-lines{padding:4px 0}.CodeMirror pre.CodeMirror-line,.CodeMirror pre.CodeMirror-line-like{padding:0 4px}.CodeMirror-gutter-filler,.CodeMirror-scrollbar-filler{background-color:#fff}.CodeMirror-gutters{border-right:1px solid #ddd;background-color:#f7f7f7;white-space:nowrap}.CodeMirror-linenumber{padding:0 3px 0 5px;min-width:20px;text-align:right;color:#999;white-space:nowrap}.CodeMirror-guttermarker{color:#000}.CodeMirror-guttermarker-subtle{color:#999}.CodeMirror-cursor{border-left:1px solid #000;border-right:none;width:0}.CodeMirror div.CodeMirror-secondarycursor{border-left:1px solid silver}.cm-fat-cursor .CodeMirror-cursor{width:auto;border:0!important;background:#7e7}.cm-fat-cursor div.CodeMirror-cursors{z-index:1}.cm-fat-cursor-mark{background-color:rgba(20,255,20,.5);-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite}.cm-animate-fat-cursor{width:auto;-webkit-animation:blink 1.06s steps(1) infinite;-moz-animation:blink 1.06s steps(1) infinite;animation:blink 1.06s steps(1) infinite;background-color:#7e7}@-moz-keyframes blink{50%{background-color:transparent}}@-webkit-keyframes blink{50%{background-color:transparent}}@keyframes blink{50%{background-color:transparent}}.cm-tab{display:inline-block;text-decoration:inherit}.CodeMirror-rulers{position:absolute;left:0;right:0;top:-50px;bottom:0;overflow:hidden}.CodeMirror-ruler{border-left:1px solid #ccc;top:0;bottom:0;position:absolute}.cm-s-default .cm-header{color:#00f}.cm-s-default .cm-quote{color:#090}.cm-negative{color:#d44}.cm-positive{color:#292}.cm-header,.cm-strong{font-weight:700}.cm-em{font-style:italic}.cm-link{text-decoration:underline}.cm-strikethrough{text-decoration:line-through}.cm-s-default .cm-keyword{color:#708}.cm-s-default .cm-atom{color:#219}.cm-s-default .cm-number{color:#164}.cm-s-default .cm-def{color:#00f}.cm-s-default .cm-variable-2{color:#05a}.cm-s-default .cm-type,.cm-s-default .cm-variable-3{color:#085}.cm-s-default .cm-comment{color:#a50}.cm-s-default .cm-string{color:#a11}.cm-s-default .cm-string-2{color:#f50}.cm-s-default .cm-meta{color:#555}.cm-s-default .cm-qualifier{color:#555}.cm-s-default .cm-builtin{color:#30a}.cm-s-default .cm-bracket{color:#997}.cm-s-default .cm-tag{color:#170}.cm-s-default .cm-attribute{color:#00c}.cm-s-default .cm-hr{color:#999}.cm-s-default .cm-link{color:#00c}.cm-s-default .cm-error{color:red}.cm-invalidchar{color:red}.CodeMirror-composing{border-bottom:2px solid}div.CodeMirror span.CodeMirror-matchingbracket{color:#0b0}div.CodeMirror span.CodeMirror-nonmatchingbracket{color:#a22}.CodeMirror-matchingtag{background:rgba(255,150,0,.3)}.CodeMirror-activeline-background{background:#e8f2ff}.CodeMirror{position:relative;overflow:hidden;background:#fff}.CodeMirror-scroll{overflow:scroll!important;margin-bottom:-50px;margin-right:-50px;padding-bottom:50px;height:100%;outline:0;position:relative}.CodeMirror-sizer{position:relative;border-right:50px solid transparent}.CodeMirror-gutter-filler,.CodeMirror-hscrollbar,.CodeMirror-scrollbar-filler,.CodeMirror-vscrollbar{position:absolute;z-index:6;display:none;outline:0}.CodeMirror-vscrollbar{right:0;top:0;overflow-x:hidden;overflow-y:scroll}.CodeMirror-hscrollbar{bottom:0;left:0;overflow-y:hidden;overflow-x:scroll}.CodeMirror-scrollbar-filler{right:0;bottom:0}.CodeMirror-gutter-filler{left:0;bottom:0}.CodeMirror-gutters{position:absolute;left:0;top:0;min-height:100%;z-index:3}.CodeMirror-gutter{white-space:normal;height:100%;display:inline-block;vertical-align:top;margin-bottom:-50px}.CodeMirror-gutter-wrapper{position:absolute;z-index:4;background:0 0!important;border:none!important}.CodeMirror-gutter-background{position:absolute;top:0;bottom:0;z-index:4}.CodeMirror-gutter-elt{position:absolute;cursor:default;z-index:4}.CodeMirror-gutter-wrapper ::selection{background-color:transparent}.CodeMirror-gutter-wrapper ::-moz-selection{background-color:transparent}.CodeMirror-lines{cursor:text;min-height:1px}.CodeMirror pre.CodeMirror-line,.CodeMirror pre.CodeMirror-line-like{-moz-border-radius:0;-webkit-border-radius:0;border-radius:0;border-width:0;background:0 0;font-family:inherit;font-size:inherit;margin:0;white-space:pre;word-wrap:normal;line-height:inherit;color:inherit;z-index:2;position:relative;overflow:visible;-webkit-tap-highlight-color:transparent;-webkit-font-variant-ligatures:contextual;font-variant-ligatures:contextual}.CodeMirror-wrap pre.CodeMirror-line,.CodeMirror-wrap pre.CodeMirror-line-like{word-wrap:break-word;white-space:pre-wrap;word-break:normal}.CodeMirror-linebackground{position:absolute;left:0;right:0;top:0;bottom:0;z-index:0}.CodeMirror-linewidget{position:relative;z-index:2;padding:.1px}.CodeMirror-rtl pre{direction:rtl}.CodeMirror-code{outline:0}.CodeMirror-gutter,.CodeMirror-gutters,.CodeMirror-linenumber,.CodeMirror-scroll,.CodeMirror-sizer{-moz-box-sizing:content-box;box-sizing:content-box}.CodeMirror-measure{position:absolute;width:100%;height:0;overflow:hidden;visibility:hidden}.CodeMirror-cursor{position:absolute;pointer-events:none}.CodeMirror-measure pre{position:static}div.CodeMirror-cursors{visibility:hidden;position:relative;z-index:3}div.CodeMirror-dragcursors{visibility:visible}.CodeMirror-focused div.CodeMirror-cursors{visibility:visible}.CodeMirror-selected{background:#d9d9d9}.CodeMirror-focused .CodeMirror-selected{background:#d7d4f0}.CodeMirror-crosshair{cursor:crosshair}.CodeMirror-line::selection,.CodeMirror-line>span::selection,.CodeMirror-line>span>span::selection{background:#d7d4f0}.CodeMirror-line::-moz-selection,.CodeMirror-line>span::-moz-selection,.CodeMirror-line>span>span::-moz-selection{background:#d7d4f0}.cm-searching{background-color:#ffa;background-color:rgba(255,255,0,.4)}.cm-force-border{padding-right:.1px}@media print{.CodeMirror div.CodeMirror-cursors{visibility:hidden}}.cm-tab-wrap-hack:after{content:''}span.CodeMirror-selectedtext{background:0 0}.contentview .header{font-weight:700}.contentview .highlight{font-weight:700}.contentview .offset{color:#00f}.contentview .codeeditor{margin-bottom:12px}.contentview .Token_Name_Tag{color:#006400}.contentview .Token_Literal_String{color:#b22222}.contentview .Token_Literal_Number{color:purple}.contentview .Token_Keyword_Constant{color:#00f}.modal-visible{display:block}.modal-dialog{overflow-y:initial!important}.modal-body{max-height:calc(100vh - 200px);overflow-y:auto}.dropdown-menu{margin:0!important}.dropdown-menu>li>a{padding:3px 10px}.command-title{background-color:#f2f2f2;border:1px solid #aaa}.command-result{display:block;margin:0;background-color:#fcfcfc;height:100px;max-height:100px;overflow:auto}.command-suggestion{background-color:#9c9c9c}.argument-suggestion{background-color:hsla(209,52%,84%,.5)!important}.command>.popover{display:block;position:relative;max-width:none}.available-commands{overflow:auto}.wireguard-config{margin:1rem 0;display:flex;flex-wrap:wrap;column-gap:2rem;align-items:center}.wireguard-config>*{margin:0} /*# sourceMappingURL=app.css.map */ diff --git a/mitmproxy/tools/web/static/app.js b/mitmproxy/tools/web/static/app.js index 228782695f..8828bc7537 100644 --- a/mitmproxy/tools/web/static/app.js +++ b/mitmproxy/tools/web/static/app.js @@ -1,14 +1,14 @@ -(()=>{var fD=Object.create;var qc=Object.defineProperty,cD=Object.defineProperties,pD=Object.getOwnPropertyDescriptor,dD=Object.getOwnPropertyDescriptors,hD=Object.getOwnPropertyNames,Ag=Object.getOwnPropertySymbols,mD=Object.getPrototypeOf,Cw=Object.prototype.hasOwnProperty,Fb=Object.prototype.propertyIsEnumerable;var bw=(e,t,n)=>t in e?qc(e,t,{enumerable:!0,configurable:!0,writable:!0,value:n}):e[t]=n,ke=(e,t)=>{for(var n in t||(t={}))Cw.call(t,n)&&bw(e,n,t[n]);if(Ag)for(var n of Ag(t))Fb.call(t,n)&&bw(e,n,t[n]);return e},Pt=(e,t)=>cD(e,dD(t)),Bb=e=>qc(e,"__esModule",{value:!0}),o=(e,t)=>qc(e,"name",{value:t,configurable:!0});var Ws=(e,t)=>{var n={};for(var l in e)Cw.call(e,l)&&t.indexOf(l)<0&&(n[l]=e[l]);if(e!=null&&Ag)for(var l of Ag(e))t.indexOf(l)<0&&Fb.call(e,l)&&(n[l]=e[l]);return n};var Ue=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports),Hb=(e,t)=>{Bb(e);for(var n in t)qc(e,n,{get:t[n],enumerable:!0})},gD=(e,t,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let l of hD(t))!Cw.call(e,l)&&l!=="default"&&qc(e,l,{get:()=>t[l],enumerable:!(n=pD(t,l))||n.enumerable});return e},fe=e=>gD(Bb(qc(e!=null?fD(mD(e)):{},"default",e&&e.__esModule&&"default"in e?{get:()=>e.default,enumerable:!0}:{value:e,enumerable:!0})),e);var Vc=(e,t,n)=>(bw(e,typeof t!="symbol"?t+"":t,n),n);var Ia=(e,t,n)=>new Promise((l,d)=>{var h=C=>{try{v(n.next(C))}catch(k){d(k)}},c=C=>{try{v(n.throw(C))}catch(k){d(k)}},v=C=>C.done?l(C.value):Promise.resolve(C.value).then(h,c);v((n=n.apply(e,t)).next())});var Ew=Ue((f4,Ub)=>{"use strict";var Wb=Object.getOwnPropertySymbols,vD=Object.prototype.hasOwnProperty,yD=Object.prototype.propertyIsEnumerable;function wD(e){if(e==null)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}o(wD,"toObject");function xD(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de",Object.getOwnPropertyNames(e)[0]==="5")return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;var l=Object.getOwnPropertyNames(t).map(function(h){return t[h]});if(l.join("")!=="0123456789")return!1;var d={};return"abcdefghijklmnopqrst".split("").forEach(function(h){d[h]=h}),Object.keys(Object.assign({},d)).join("")==="abcdefghijklmnopqrst"}catch(h){return!1}}o(xD,"shouldUseNative");Ub.exports=xD()?Object.assign:function(e,t){for(var n,l=wD(e),d,h=1;h{"use strict";var _w=Ew(),Kc=60103,zb=60106;Et.Fragment=60107;Et.StrictMode=60108;Et.Profiler=60114;var $b=60109,jb=60110,qb=60112;Et.Suspense=60113;var Vb=60115,Kb=60116;typeof Symbol=="function"&&Symbol.for&&(Co=Symbol.for,Kc=Co("react.element"),zb=Co("react.portal"),Et.Fragment=Co("react.fragment"),Et.StrictMode=Co("react.strict_mode"),Et.Profiler=Co("react.profiler"),$b=Co("react.provider"),jb=Co("react.context"),qb=Co("react.forward_ref"),Et.Suspense=Co("react.suspense"),Vb=Co("react.memo"),Kb=Co("react.lazy"));var Co,Gb=typeof Symbol=="function"&&Symbol.iterator;function SD(e){return e===null||typeof e!="object"?null:(e=Gb&&e[Gb]||e["@@iterator"],typeof e=="function"?e:null)}o(SD,"y");function Yd(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n{"use strict";iE.exports=nE()});var fE=Ue(Rt=>{"use strict";var Yc,Xd,Ig,Ow;typeof performance=="object"&&typeof performance.now=="function"?(oE=performance,Rt.unstable_now=function(){return oE.now()}):(Mw=Date,sE=Mw.now(),Rt.unstable_now=function(){return Mw.now()-sE});var oE,Mw,sE;typeof window=="undefined"||typeof MessageChannel!="function"?(Xc=null,Aw=null,Dw=o(function(){if(Xc!==null)try{var e=Rt.unstable_now();Xc(!0,e),Xc=null}catch(t){throw setTimeout(Dw,0),t}},"w"),Yc=o(function(e){Xc!==null?setTimeout(Yc,0,e):(Xc=e,setTimeout(Dw,0))},"f"),Xd=o(function(e,t){Aw=setTimeout(e,t)},"g"),Ig=o(function(){clearTimeout(Aw)},"h"),Rt.unstable_shouldYield=function(){return!1},Ow=Rt.unstable_forceFrameRate=function(){}):(lE=window.setTimeout,aE=window.clearTimeout,typeof console!="undefined"&&(uE=window.cancelAnimationFrame,typeof window.requestAnimationFrame!="function"&&console.error("This browser doesn't support requestAnimationFrame. Make sure that you load a polyfill in older browsers. https://reactjs.org/link/react-polyfills"),typeof uE!="function"&&console.error("This browser doesn't support cancelAnimationFrame. Make sure that you load a polyfill in older browsers. https://reactjs.org/link/react-polyfills")),Qd=!1,Zd=null,Fg=-1,Rw=5,Iw=0,Rt.unstable_shouldYield=function(){return Rt.unstable_now()>=Iw},Ow=o(function(){},"k"),Rt.unstable_forceFrameRate=function(e){0>e||125>>1,d=e[l];if(d!==void 0&&0Wg(c,n))C!==void 0&&0>Wg(C,c)?(e[l]=C,e[v]=n,l=v):(e[l]=c,e[h]=n,l=h);else if(C!==void 0&&0>Wg(C,n))e[l]=C,e[v]=n,l=v;else break e}}return t}return null}o(Hg,"K");function Wg(e,t){var n=e.sortIndex-t.sortIndex;return n!==0?n:e.id-t.id}o(Wg,"I");var Us=[],Fa=[],TD=1,bo=null,Yn=3,Ug=!1,df=!1,Jd=!1;function Hw(e){for(var t=os(Fa);t!==null;){if(t.callback===null)Hg(Fa);else if(t.startTime<=e)Hg(Fa),t.sortIndex=t.expirationTime,Bw(Us,t);else break;t=os(Fa)}}o(Hw,"T");function Ww(e){if(Jd=!1,Hw(e),!df)if(os(Us)!==null)df=!0,Yc(Uw);else{var t=os(Fa);t!==null&&Xd(Ww,t.startTime-e)}}o(Ww,"U");function Uw(e,t){df=!1,Jd&&(Jd=!1,Ig()),Ug=!0;var n=Yn;try{for(Hw(t),bo=os(Us);bo!==null&&(!(bo.expirationTime>t)||e&&!Rt.unstable_shouldYield());){var l=bo.callback;if(typeof l=="function"){bo.callback=null,Yn=bo.priorityLevel;var d=l(bo.expirationTime<=t);t=Rt.unstable_now(),typeof d=="function"?bo.callback=d:bo===os(Us)&&Hg(Us),Hw(t)}else Hg(Us);bo=os(Us)}if(bo!==null)var h=!0;else{var c=os(Fa);c!==null&&Xd(Ww,c.startTime-t),h=!1}return h}finally{bo=null,Yn=n,Ug=!1}}o(Uw,"V");var kD=Ow;Rt.unstable_IdlePriority=5;Rt.unstable_ImmediatePriority=1;Rt.unstable_LowPriority=4;Rt.unstable_NormalPriority=3;Rt.unstable_Profiling=null;Rt.unstable_UserBlockingPriority=2;Rt.unstable_cancelCallback=function(e){e.callback=null};Rt.unstable_continueExecution=function(){df||Ug||(df=!0,Yc(Uw))};Rt.unstable_getCurrentPriorityLevel=function(){return Yn};Rt.unstable_getFirstCallbackNode=function(){return os(Us)};Rt.unstable_next=function(e){switch(Yn){case 1:case 2:case 3:var t=3;break;default:t=Yn}var n=Yn;Yn=t;try{return e()}finally{Yn=n}};Rt.unstable_pauseExecution=function(){};Rt.unstable_requestPaint=kD;Rt.unstable_runWithPriority=function(e,t){switch(e){case 1:case 2:case 3:case 4:case 5:break;default:e=3}var n=Yn;Yn=e;try{return t()}finally{Yn=n}};Rt.unstable_scheduleCallback=function(e,t,n){var l=Rt.unstable_now();switch(typeof n=="object"&&n!==null?(n=n.delay,n=typeof n=="number"&&0l?(e.sortIndex=n,Bw(Fa,e),os(Us)===null&&e===os(Fa)&&(Jd?Ig():Jd=!0,Xd(Ww,n-l))):(e.sortIndex=d,Bw(Us,e),df||Ug||(df=!0,Yc(Uw))),e};Rt.unstable_wrapCallback=function(e){var t=Yn;return function(){var n=Yn;Yn=t;try{return e.apply(this,arguments)}finally{Yn=n}}}});var pE=Ue((h4,cE)=>{"use strict";cE.exports=fE()});var ZT=Ue(Lo=>{"use strict";var zg=Oe(),hr=Ew(),Tn=pE();function we(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;nt}return!1}o(OD,"na");function vi(e,t,n,l,d,h,c){this.acceptsBooleans=t===2||t===3||t===4,this.attributeName=l,this.attributeNamespace=d,this.mustUseProperty=n,this.propertyName=e,this.type=t,this.sanitizeURL=h,this.removeEmptyString=c}o(vi,"B");var Rn={};"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style".split(" ").forEach(function(e){Rn[e]=new vi(e,0,!1,e,null,!1,!1)});[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach(function(e){var t=e[0];Rn[t]=new vi(t,1,!1,e[1],null,!1,!1)});["contentEditable","draggable","spellCheck","value"].forEach(function(e){Rn[e]=new vi(e,2,!1,e.toLowerCase(),null,!1,!1)});["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach(function(e){Rn[e]=new vi(e,2,!1,e,null,!1,!1)});"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture disableRemotePlayback formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope".split(" ").forEach(function(e){Rn[e]=new vi(e,3,!1,e.toLowerCase(),null,!1,!1)});["checked","multiple","muted","selected"].forEach(function(e){Rn[e]=new vi(e,3,!0,e,null,!1,!1)});["capture","download"].forEach(function(e){Rn[e]=new vi(e,4,!1,e,null,!1,!1)});["cols","rows","size","span"].forEach(function(e){Rn[e]=new vi(e,6,!1,e,null,!1,!1)});["rowSpan","start"].forEach(function(e){Rn[e]=new vi(e,5,!1,e.toLowerCase(),null,!1,!1)});var zw=/[\-:]([a-z])/g;function $w(e){return e[1].toUpperCase()}o($w,"pa");"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x image-rendering letter-spacing lighting-color marker-end marker-mid marker-start overline-position overline-thickness paint-order panose-1 pointer-events rendering-intent shape-rendering stop-color stop-opacity strikethrough-position strikethrough-thickness stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width text-anchor text-decoration text-rendering underline-position underline-thickness unicode-bidi unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical vector-effect vert-adv-y vert-origin-x vert-origin-y word-spacing writing-mode xmlns:xlink x-height".split(" ").forEach(function(e){var t=e.replace(zw,$w);Rn[t]=new vi(t,1,!1,e,null,!1,!1)});"xlink:actuate xlink:arcrole xlink:role xlink:show xlink:title xlink:type".split(" ").forEach(function(e){var t=e.replace(zw,$w);Rn[t]=new vi(t,1,!1,e,"http://www.w3.org/1999/xlink",!1,!1)});["xml:base","xml:lang","xml:space"].forEach(function(e){var t=e.replace(zw,$w);Rn[t]=new vi(t,1,!1,e,"http://www.w3.org/XML/1998/namespace",!1,!1)});["tabIndex","crossOrigin"].forEach(function(e){Rn[e]=new vi(e,1,!1,e.toLowerCase(),null,!1,!1)});Rn.xlinkHref=new vi("xlinkHref",1,!1,"xlink:href","http://www.w3.org/1999/xlink",!0,!1);["src","href","action","formAction"].forEach(function(e){Rn[e]=new vi(e,1,!1,e.toLowerCase(),null,!0,!0)});function jw(e,t,n,l){var d=Rn.hasOwnProperty(t)?Rn[t]:null,h=d!==null?d.type===0:l?!1:!(!(2{var fD=Object.create;var qc=Object.defineProperty,cD=Object.defineProperties,pD=Object.getOwnPropertyDescriptor,dD=Object.getOwnPropertyDescriptors,hD=Object.getOwnPropertyNames,Ag=Object.getOwnPropertySymbols,mD=Object.getPrototypeOf,xw=Object.prototype.hasOwnProperty,Fb=Object.prototype.propertyIsEnumerable;var Sw=(e,t,n)=>t in e?qc(e,t,{enumerable:!0,configurable:!0,writable:!0,value:n}):e[t]=n,ke=(e,t)=>{for(var n in t||(t={}))xw.call(t,n)&&Sw(e,n,t[n]);if(Ag)for(var n of Ag(t))Fb.call(t,n)&&Sw(e,n,t[n]);return e},Pt=(e,t)=>cD(e,dD(t)),Bb=e=>qc(e,"__esModule",{value:!0}),o=(e,t)=>qc(e,"name",{value:t,configurable:!0});var Ws=(e,t)=>{var n={};for(var l in e)xw.call(e,l)&&t.indexOf(l)<0&&(n[l]=e[l]);if(e!=null&&Ag)for(var l of Ag(e))t.indexOf(l)<0&&Fb.call(e,l)&&(n[l]=e[l]);return n};var Ue=(e,t)=>()=>(t||e((t={exports:{}}).exports,t),t.exports),Hb=(e,t)=>{Bb(e);for(var n in t)qc(e,n,{get:t[n],enumerable:!0})},gD=(e,t,n)=>{if(t&&typeof t=="object"||typeof t=="function")for(let l of hD(t))!xw.call(e,l)&&l!=="default"&&qc(e,l,{get:()=>t[l],enumerable:!(n=pD(t,l))||n.enumerable});return e},fe=e=>gD(Bb(qc(e!=null?fD(mD(e)):{},"default",e&&e.__esModule&&"default"in e?{get:()=>e.default,enumerable:!0}:{value:e,enumerable:!0})),e);var Vc=(e,t,n)=>(Sw(e,typeof t!="symbol"?t+"":t,n),n);var Ia=(e,t,n)=>new Promise((l,d)=>{var h=C=>{try{v(n.next(C))}catch(k){d(k)}},c=C=>{try{v(n.throw(C))}catch(k){d(k)}},v=C=>C.done?l(C.value):Promise.resolve(C.value).then(h,c);v((n=n.apply(e,t)).next())});var Cw=Ue((fH,Ub)=>{"use strict";var Wb=Object.getOwnPropertySymbols,vD=Object.prototype.hasOwnProperty,yD=Object.prototype.propertyIsEnumerable;function wD(e){if(e==null)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}o(wD,"toObject");function xD(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de",Object.getOwnPropertyNames(e)[0]==="5")return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;var l=Object.getOwnPropertyNames(t).map(function(h){return t[h]});if(l.join("")!=="0123456789")return!1;var d={};return"abcdefghijklmnopqrst".split("").forEach(function(h){d[h]=h}),Object.keys(Object.assign({},d)).join("")==="abcdefghijklmnopqrst"}catch(h){return!1}}o(xD,"shouldUseNative");Ub.exports=xD()?Object.assign:function(e,t){for(var n,l=wD(e),d,h=1;h{"use strict";var bw=Cw(),Kc=60103,zb=60106;Et.Fragment=60107;Et.StrictMode=60108;Et.Profiler=60114;var $b=60109,jb=60110,qb=60112;Et.Suspense=60113;var Vb=60115,Kb=60116;typeof Symbol=="function"&&Symbol.for&&(Co=Symbol.for,Kc=Co("react.element"),zb=Co("react.portal"),Et.Fragment=Co("react.fragment"),Et.StrictMode=Co("react.strict_mode"),Et.Profiler=Co("react.profiler"),$b=Co("react.provider"),jb=Co("react.context"),qb=Co("react.forward_ref"),Et.Suspense=Co("react.suspense"),Vb=Co("react.memo"),Kb=Co("react.lazy"));var Co,Gb=typeof Symbol=="function"&&Symbol.iterator;function SD(e){return e===null||typeof e!="object"?null:(e=Gb&&e[Gb]||e["@@iterator"],typeof e=="function"?e:null)}o(SD,"y");function Yd(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n{"use strict";iE.exports=nE()});var fE=Ue(Rt=>{"use strict";var Yc,Xd,Ig,Lw;typeof performance=="object"&&typeof performance.now=="function"?(oE=performance,Rt.unstable_now=function(){return oE.now()}):(Pw=Date,sE=Pw.now(),Rt.unstable_now=function(){return Pw.now()-sE});var oE,Pw,sE;typeof window=="undefined"||typeof MessageChannel!="function"?(Xc=null,Ow=null,Mw=o(function(){if(Xc!==null)try{var e=Rt.unstable_now();Xc(!0,e),Xc=null}catch(t){throw setTimeout(Mw,0),t}},"w"),Yc=o(function(e){Xc!==null?setTimeout(Yc,0,e):(Xc=e,setTimeout(Mw,0))},"f"),Xd=o(function(e,t){Ow=setTimeout(e,t)},"g"),Ig=o(function(){clearTimeout(Ow)},"h"),Rt.unstable_shouldYield=function(){return!1},Lw=Rt.unstable_forceFrameRate=function(){}):(lE=window.setTimeout,aE=window.clearTimeout,typeof console!="undefined"&&(uE=window.cancelAnimationFrame,typeof window.requestAnimationFrame!="function"&&console.error("This browser doesn't support requestAnimationFrame. Make sure that you load a polyfill in older browsers. https://reactjs.org/link/react-polyfills"),typeof uE!="function"&&console.error("This browser doesn't support cancelAnimationFrame. Make sure that you load a polyfill in older browsers. https://reactjs.org/link/react-polyfills")),Qd=!1,Zd=null,Fg=-1,Aw=5,Dw=0,Rt.unstable_shouldYield=function(){return Rt.unstable_now()>=Dw},Lw=o(function(){},"k"),Rt.unstable_forceFrameRate=function(e){0>e||125>>1,d=e[l];if(d!==void 0&&0Wg(c,n))C!==void 0&&0>Wg(C,c)?(e[l]=C,e[v]=n,l=v):(e[l]=c,e[h]=n,l=h);else if(C!==void 0&&0>Wg(C,n))e[l]=C,e[v]=n,l=v;else break e}}return t}return null}o(Hg,"K");function Wg(e,t){var n=e.sortIndex-t.sortIndex;return n!==0?n:e.id-t.id}o(Wg,"I");var Us=[],Fa=[],TD=1,bo=null,Yn=3,Ug=!1,df=!1,Jd=!1;function Fw(e){for(var t=os(Fa);t!==null;){if(t.callback===null)Hg(Fa);else if(t.startTime<=e)Hg(Fa),t.sortIndex=t.expirationTime,Iw(Us,t);else break;t=os(Fa)}}o(Fw,"T");function Bw(e){if(Jd=!1,Fw(e),!df)if(os(Us)!==null)df=!0,Yc(Hw);else{var t=os(Fa);t!==null&&Xd(Bw,t.startTime-e)}}o(Bw,"U");function Hw(e,t){df=!1,Jd&&(Jd=!1,Ig()),Ug=!0;var n=Yn;try{for(Fw(t),bo=os(Us);bo!==null&&(!(bo.expirationTime>t)||e&&!Rt.unstable_shouldYield());){var l=bo.callback;if(typeof l=="function"){bo.callback=null,Yn=bo.priorityLevel;var d=l(bo.expirationTime<=t);t=Rt.unstable_now(),typeof d=="function"?bo.callback=d:bo===os(Us)&&Hg(Us),Fw(t)}else Hg(Us);bo=os(Us)}if(bo!==null)var h=!0;else{var c=os(Fa);c!==null&&Xd(Bw,c.startTime-t),h=!1}return h}finally{bo=null,Yn=n,Ug=!1}}o(Hw,"V");var kD=Lw;Rt.unstable_IdlePriority=5;Rt.unstable_ImmediatePriority=1;Rt.unstable_LowPriority=4;Rt.unstable_NormalPriority=3;Rt.unstable_Profiling=null;Rt.unstable_UserBlockingPriority=2;Rt.unstable_cancelCallback=function(e){e.callback=null};Rt.unstable_continueExecution=function(){df||Ug||(df=!0,Yc(Hw))};Rt.unstable_getCurrentPriorityLevel=function(){return Yn};Rt.unstable_getFirstCallbackNode=function(){return os(Us)};Rt.unstable_next=function(e){switch(Yn){case 1:case 2:case 3:var t=3;break;default:t=Yn}var n=Yn;Yn=t;try{return e()}finally{Yn=n}};Rt.unstable_pauseExecution=function(){};Rt.unstable_requestPaint=kD;Rt.unstable_runWithPriority=function(e,t){switch(e){case 1:case 2:case 3:case 4:case 5:break;default:e=3}var n=Yn;Yn=e;try{return t()}finally{Yn=n}};Rt.unstable_scheduleCallback=function(e,t,n){var l=Rt.unstable_now();switch(typeof n=="object"&&n!==null?(n=n.delay,n=typeof n=="number"&&0l?(e.sortIndex=n,Iw(Fa,e),os(Us)===null&&e===os(Fa)&&(Jd?Ig():Jd=!0,Xd(Bw,n-l))):(e.sortIndex=d,Iw(Us,e),df||Ug||(df=!0,Yc(Hw))),e};Rt.unstable_wrapCallback=function(e){var t=Yn;return function(){var n=Yn;Yn=t;try{return e.apply(this,arguments)}finally{Yn=n}}}});var pE=Ue((hH,cE)=>{"use strict";cE.exports=fE()});var ZT=Ue(Lo=>{"use strict";var zg=Oe(),hr=Cw(),Tn=pE();function we(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;nt}return!1}o(OD,"na");function vi(e,t,n,l,d,h,c){this.acceptsBooleans=t===2||t===3||t===4,this.attributeName=l,this.attributeNamespace=d,this.mustUseProperty=n,this.propertyName=e,this.type=t,this.sanitizeURL=h,this.removeEmptyString=c}o(vi,"B");var Rn={};"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style".split(" ").forEach(function(e){Rn[e]=new vi(e,0,!1,e,null,!1,!1)});[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach(function(e){var t=e[0];Rn[t]=new vi(t,1,!1,e[1],null,!1,!1)});["contentEditable","draggable","spellCheck","value"].forEach(function(e){Rn[e]=new vi(e,2,!1,e.toLowerCase(),null,!1,!1)});["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach(function(e){Rn[e]=new vi(e,2,!1,e,null,!1,!1)});"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture disableRemotePlayback formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope".split(" ").forEach(function(e){Rn[e]=new vi(e,3,!1,e.toLowerCase(),null,!1,!1)});["checked","multiple","muted","selected"].forEach(function(e){Rn[e]=new vi(e,3,!0,e,null,!1,!1)});["capture","download"].forEach(function(e){Rn[e]=new vi(e,4,!1,e,null,!1,!1)});["cols","rows","size","span"].forEach(function(e){Rn[e]=new vi(e,6,!1,e,null,!1,!1)});["rowSpan","start"].forEach(function(e){Rn[e]=new vi(e,5,!1,e.toLowerCase(),null,!1,!1)});var Ww=/[\-:]([a-z])/g;function Uw(e){return e[1].toUpperCase()}o(Uw,"pa");"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x image-rendering letter-spacing lighting-color marker-end marker-mid marker-start overline-position overline-thickness paint-order panose-1 pointer-events rendering-intent shape-rendering stop-color stop-opacity strikethrough-position strikethrough-thickness stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width text-anchor text-decoration text-rendering underline-position underline-thickness unicode-bidi unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical vector-effect vert-adv-y vert-origin-x vert-origin-y word-spacing writing-mode xmlns:xlink x-height".split(" ").forEach(function(e){var t=e.replace(Ww,Uw);Rn[t]=new vi(t,1,!1,e,null,!1,!1)});"xlink:actuate xlink:arcrole xlink:role xlink:show xlink:title xlink:type".split(" ").forEach(function(e){var t=e.replace(Ww,Uw);Rn[t]=new vi(t,1,!1,e,"http://www.w3.org/1999/xlink",!1,!1)});["xml:base","xml:lang","xml:space"].forEach(function(e){var t=e.replace(Ww,Uw);Rn[t]=new vi(t,1,!1,e,"http://www.w3.org/XML/1998/namespace",!1,!1)});["tabIndex","crossOrigin"].forEach(function(e){Rn[e]=new vi(e,1,!1,e.toLowerCase(),null,!1,!1)});Rn.xlinkHref=new vi("xlinkHref",1,!1,"xlink:href","http://www.w3.org/1999/xlink",!0,!1);["src","href","action","formAction"].forEach(function(e){Rn[e]=new vi(e,1,!1,e.toLowerCase(),null,!0,!0)});function zw(e,t,n,l){var d=Rn.hasOwnProperty(t)?Rn[t]:null,h=d!==null?d.type===0:l?!1:!(!(2v||d[c]!==h[v])return` -`+d[c].replace(" at new "," at ");while(1<=c&&0<=v);break}}}finally{e1=!1,Error.prepareStackTrace=n}return(e=e?e.displayName||e.name:"")?oh(e):""}o(Vg,"Pa");function MD(e){switch(e.tag){case 5:return oh(e.type);case 16:return oh("Lazy");case 13:return oh("Suspense");case 19:return oh("SuspenseList");case 0:case 2:case 15:return e=Vg(e.type,!1),e;case 11:return e=Vg(e.type.render,!1),e;case 22:return e=Vg(e.type._render,!1),e;case 1:return e=Vg(e.type,!0),e;default:return""}}o(MD,"Qa");function Zc(e){if(e==null)return null;if(typeof e=="function")return e.displayName||e.name||null;if(typeof e=="string")return e;switch(e){case Ba:return"Fragment";case gf:return"Portal";case rh:return"Profiler";case qw:return"StrictMode";case nh:return"Suspense";case jg:return"SuspenseList"}if(typeof e=="object")switch(e.$$typeof){case Kw:return(e.displayName||"Context")+".Consumer";case Vw:return(e._context.displayName||"Context")+".Provider";case $g:var t=e.render;return t=t.displayName||t.name||"",e.displayName||(t!==""?"ForwardRef("+t+")":"ForwardRef");case qg:return Zc(e.type);case Yw:return Zc(e._render);case Gw:t=e._payload,e=e._init;try{return Zc(e(t))}catch(n){}}return null}o(Zc,"Ra");function Ha(e){switch(typeof e){case"boolean":case"number":case"object":case"string":case"undefined":return e;default:return""}}o(Ha,"Sa");function wE(e){var t=e.type;return(e=e.nodeName)&&e.toLowerCase()==="input"&&(t==="checkbox"||t==="radio")}o(wE,"Ta");function AD(e){var t=wE(e)?"checked":"value",n=Object.getOwnPropertyDescriptor(e.constructor.prototype,t),l=""+e[t];if(!e.hasOwnProperty(t)&&typeof n!="undefined"&&typeof n.get=="function"&&typeof n.set=="function"){var d=n.get,h=n.set;return Object.defineProperty(e,t,{configurable:!0,get:function(){return d.call(this)},set:function(c){l=""+c,h.call(this,c)}}),Object.defineProperty(e,t,{enumerable:n.enumerable}),{getValue:function(){return l},setValue:function(c){l=""+c},stopTracking:function(){e._valueTracker=null,delete e[t]}}}}o(AD,"Ua");function Kg(e){e._valueTracker||(e._valueTracker=AD(e))}o(Kg,"Va");function xE(e){if(!e)return!1;var t=e._valueTracker;if(!t)return!0;var n=t.getValue(),l="";return e&&(l=wE(e)?e.checked?"true":"false":e.value),e=l,e!==n?(t.setValue(e),!0):!1}o(xE,"Wa");function Gg(e){if(e=e||(typeof document!="undefined"?document:void 0),typeof e=="undefined")return null;try{return e.activeElement||e.body}catch(t){return e.body}}o(Gg,"Xa");function t1(e,t){var n=t.checked;return hr({},t,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:n??e._wrapperState.initialChecked})}o(t1,"Ya");function SE(e,t){var n=t.defaultValue==null?"":t.defaultValue,l=t.checked!=null?t.checked:t.defaultChecked;n=Ha(t.value!=null?t.value:n),e._wrapperState={initialChecked:l,initialValue:n,controlled:t.type==="checkbox"||t.type==="radio"?t.checked!=null:t.value!=null}}o(SE,"Za");function CE(e,t){t=t.checked,t!=null&&jw(e,"checked",t,!1)}o(CE,"$a");function r1(e,t){CE(e,t);var n=Ha(t.value),l=t.type;if(n!=null)l==="number"?(n===0&&e.value===""||e.value!=n)&&(e.value=""+n):e.value!==""+n&&(e.value=""+n);else if(l==="submit"||l==="reset"){e.removeAttribute("value");return}t.hasOwnProperty("value")?n1(e,t.type,n):t.hasOwnProperty("defaultValue")&&n1(e,t.type,Ha(t.defaultValue)),t.checked==null&&t.defaultChecked!=null&&(e.defaultChecked=!!t.defaultChecked)}o(r1,"ab");function bE(e,t,n){if(t.hasOwnProperty("value")||t.hasOwnProperty("defaultValue")){var l=t.type;if(!(l!=="submit"&&l!=="reset"||t.value!==void 0&&t.value!==null))return;t=""+e._wrapperState.initialValue,n||t===e.value||(e.value=t),e.defaultValue=t}n=e.name,n!==""&&(e.name=""),e.defaultChecked=!!e._wrapperState.initialChecked,n!==""&&(e.name=n)}o(bE,"cb");function n1(e,t,n){(t!=="number"||Gg(e.ownerDocument)!==e)&&(n==null?e.defaultValue=""+e._wrapperState.initialValue:e.defaultValue!==""+n&&(e.defaultValue=""+n))}o(n1,"bb");function DD(e){var t="";return zg.Children.forEach(e,function(n){n!=null&&(t+=n)}),t}o(DD,"db");function i1(e,t){return e=hr({children:void 0},t),(t=DD(t.children))&&(e.children=t),e}o(i1,"eb");function Jc(e,t,n,l){if(e=e.options,t){t={};for(var d=0;d=n.length))throw Error(we(93));n=n[0]}t=n}t==null&&(t=""),n=t}e._wrapperState={initialValue:Ha(n)}}o(EE,"hb");function _E(e,t){var n=Ha(t.value),l=Ha(t.defaultValue);n!=null&&(n=""+n,n!==e.value&&(e.value=n),t.defaultValue==null&&e.defaultValue!==n&&(e.defaultValue=n)),l!=null&&(e.defaultValue=""+l)}o(_E,"ib");function TE(e){var t=e.textContent;t===e._wrapperState.initialValue&&t!==""&&t!==null&&(e.value=t)}o(TE,"jb");var s1={html:"http://www.w3.org/1999/xhtml",mathml:"http://www.w3.org/1998/Math/MathML",svg:"http://www.w3.org/2000/svg"};function kE(e){switch(e){case"svg":return"http://www.w3.org/2000/svg";case"math":return"http://www.w3.org/1998/Math/MathML";default:return"http://www.w3.org/1999/xhtml"}}o(kE,"lb");function l1(e,t){return e==null||e==="http://www.w3.org/1999/xhtml"?kE(t):e==="http://www.w3.org/2000/svg"&&t==="foreignObject"?"http://www.w3.org/1999/xhtml":e}o(l1,"mb");var Yg,NE=function(e){return typeof MSApp!="undefined"&&MSApp.execUnsafeLocalFunction?function(t,n,l,d){MSApp.execUnsafeLocalFunction(function(){return e(t,n,l,d)})}:e}(function(e,t){if(e.namespaceURI!==s1.svg||"innerHTML"in e)e.innerHTML=t;else{for(Yg=Yg||document.createElement("div"),Yg.innerHTML=""+t.valueOf().toString()+"",t=Yg.firstChild;e.firstChild;)e.removeChild(e.firstChild);for(;t.firstChild;)e.appendChild(t.firstChild)}});function sh(e,t){if(t){var n=e.firstChild;if(n&&n===e.lastChild&&n.nodeType===3){n.nodeValue=t;return}}e.textContent=t}o(sh,"pb");var lh={animationIterationCount:!0,borderImageOutset:!0,borderImageSlice:!0,borderImageWidth:!0,boxFlex:!0,boxFlexGroup:!0,boxOrdinalGroup:!0,columnCount:!0,columns:!0,flex:!0,flexGrow:!0,flexPositive:!0,flexShrink:!0,flexNegative:!0,flexOrder:!0,gridArea:!0,gridRow:!0,gridRowEnd:!0,gridRowSpan:!0,gridRowStart:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnSpan:!0,gridColumnStart:!0,fontWeight:!0,lineClamp:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,tabSize:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeDasharray:!0,strokeDashoffset:!0,strokeMiterlimit:!0,strokeOpacity:!0,strokeWidth:!0},RD=["Webkit","ms","Moz","O"];Object.keys(lh).forEach(function(e){RD.forEach(function(t){t=t+e.charAt(0).toUpperCase()+e.substring(1),lh[t]=lh[e]})});function LE(e,t,n){return t==null||typeof t=="boolean"||t===""?"":n||typeof t!="number"||t===0||lh.hasOwnProperty(e)&&lh[e]?(""+t).trim():t+"px"}o(LE,"sb");function PE(e,t){e=e.style;for(var n in t)if(t.hasOwnProperty(n)){var l=n.indexOf("--")===0,d=LE(n,t[n],l);n==="float"&&(n="cssFloat"),l?e.setProperty(n,d):e[n]=d}}o(PE,"tb");var ID=hr({menuitem:!0},{area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,keygen:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0});function a1(e,t){if(t){if(ID[e]&&(t.children!=null||t.dangerouslySetInnerHTML!=null))throw Error(we(137,e));if(t.dangerouslySetInnerHTML!=null){if(t.children!=null)throw Error(we(60));if(!(typeof t.dangerouslySetInnerHTML=="object"&&"__html"in t.dangerouslySetInnerHTML))throw Error(we(61))}if(t.style!=null&&typeof t.style!="object")throw Error(we(62))}}o(a1,"vb");function u1(e,t){if(e.indexOf("-")===-1)return typeof t.is=="string";switch(e){case"annotation-xml":case"color-profile":case"font-face":case"font-face-src":case"font-face-uri":case"font-face-format":case"font-face-name":case"missing-glyph":return!1;default:return!0}}o(u1,"wb");function f1(e){return e=e.target||e.srcElement||window,e.correspondingUseElement&&(e=e.correspondingUseElement),e.nodeType===3?e.parentNode:e}o(f1,"xb");var c1=null,ep=null,tp=null;function OE(e){if(e=_h(e)){if(typeof c1!="function")throw Error(we(280));var t=e.stateNode;t&&(t=gv(t),c1(e.stateNode,e.type,t))}}o(OE,"Bb");function ME(e){ep?tp?tp.push(e):tp=[e]:ep=e}o(ME,"Eb");function AE(){if(ep){var e=ep,t=tp;if(tp=ep=null,OE(e),t)for(e=0;el?0:1<n;n++)t.push(e);return t}o(E1,"Zc");function rv(e,t,n){e.pendingLanes|=t;var l=t-1;e.suspendedLanes&=l,e.pingedLanes&=l,e=e.eventTimes,t=31-$a(t),e[t]=n}o(rv,"$c");var $a=Math.clz32?Math.clz32:ZD,XD=Math.log,QD=Math.LN2;function ZD(e){return e===0?32:31-(XD(e)/QD|0)|0}o(ZD,"ad");var JD=Tn.unstable_UserBlockingPriority,eR=Tn.unstable_runWithPriority,nv=!0;function tR(e,t,n,l){vf||d1();var d=_1,h=vf;vf=!0;try{DE(d,e,t,n,l)}finally{(vf=h)||m1()}}o(tR,"gd");function rR(e,t,n,l){eR(JD,_1.bind(null,e,t,n,l))}o(rR,"id");function _1(e,t,n,l){if(nv){var d;if((d=(t&4)==0)&&0=yh),s_=String.fromCharCode(32),l_=!1;function a_(e,t){switch(e){case"keyup":return _R.indexOf(t.keyCode)!==-1;case"keydown":return t.keyCode!==229;case"keypress":case"mousedown":case"focusout":return!0;default:return!1}}o(a_,"ge");function u_(e){return e=e.detail,typeof e=="object"&&"data"in e?e.data:null}o(u_,"he");var lp=!1;function kR(e,t){switch(e){case"compositionend":return u_(t);case"keypress":return t.which!==32?null:(l_=!0,s_);case"textInput":return e=t.data,e===s_&&l_?null:e;default:return null}}o(kR,"je");function NR(e,t){if(lp)return e==="compositionend"||!A1&&a_(e,t)?(e=e_(),iv=k1=ja=null,lp=!1,e):null;switch(e){case"paste":return null;case"keypress":if(!(t.ctrlKey||t.altKey||t.metaKey)||t.ctrlKey&&t.altKey){if(t.char&&1=t)return{node:n,offset:t-e};e=l}e:{for(;n;){if(n.nextSibling){n=n.nextSibling;break e}n=n.parentNode}n=void 0}n=m_(n)}}o(g_,"Le");function v_(e,t){return e&&t?e===t?!0:e&&e.nodeType===3?!1:t&&t.nodeType===3?v_(e,t.parentNode):"contains"in e?e.contains(t):e.compareDocumentPosition?!!(e.compareDocumentPosition(t)&16):!1:!1}o(v_,"Me");function y_(){for(var e=window,t=Gg();t instanceof e.HTMLIFrameElement;){try{var n=typeof t.contentWindow.location.href=="string"}catch(l){n=!1}if(n)e=t.contentWindow;else break;t=Gg(e.document)}return t}o(y_,"Ne");function R1(e){var t=e&&e.nodeName&&e.nodeName.toLowerCase();return t&&(t==="input"&&(e.type==="text"||e.type==="search"||e.type==="tel"||e.type==="url"||e.type==="password")||t==="textarea"||e.contentEditable==="true")}o(R1,"Oe");var BR=Ll&&"documentMode"in document&&11>=document.documentMode,ap=null,I1=null,Ch=null,F1=!1;function w_(e,t,n){var l=n.window===n?n.document:n.nodeType===9?n:n.ownerDocument;F1||ap==null||ap!==Gg(l)||(l=ap,"selectionStart"in l&&R1(l)?l={start:l.selectionStart,end:l.selectionEnd}:(l=(l.ownerDocument&&l.ownerDocument.defaultView||window).getSelection(),l={anchorNode:l.anchorNode,anchorOffset:l.anchorOffset,focusNode:l.focusNode,focusOffset:l.focusOffset}),Ch&&Sh(Ch,l)||(Ch=l,l=pv(I1,"onSelect"),0dp||(e.current=j1[dp],j1[dp]=null,dp--)}o(fr,"H");function _r(e,t){dp++,j1[dp]=e.current,e.current=t}o(_r,"I");var Ka={},Xn=Va(Ka),Ri=Va(!1),xf=Ka;function hp(e,t){var n=e.type.contextTypes;if(!n)return Ka;var l=e.stateNode;if(l&&l.__reactInternalMemoizedUnmaskedChildContext===t)return l.__reactInternalMemoizedMaskedChildContext;var d={},h;for(h in n)d[h]=t[h];return l&&(e=e.stateNode,e.__reactInternalMemoizedUnmaskedChildContext=t,e.__reactInternalMemoizedMaskedChildContext=d),d}o(hp,"Ef");function Ii(e){return e=e.childContextTypes,e!=null}o(Ii,"Ff");function vv(){fr(Ri),fr(Xn)}o(vv,"Gf");function D_(e,t,n){if(Xn.current!==Ka)throw Error(we(168));_r(Xn,t),_r(Ri,n)}o(D_,"Hf");function R_(e,t,n){var l=e.stateNode;if(e=t.childContextTypes,typeof l.getChildContext!="function")return n;l=l.getChildContext();for(var d in l)if(!(d in e))throw Error(we(108,Zc(t)||"Unknown",d));return hr({},n,l)}o(R_,"If");function yv(e){return e=(e=e.stateNode)&&e.__reactInternalMemoizedMergedChildContext||Ka,xf=Xn.current,_r(Xn,e),_r(Ri,Ri.current),!0}o(yv,"Jf");function I_(e,t,n){var l=e.stateNode;if(!l)throw Error(we(169));n?(e=R_(e,t,xf),l.__reactInternalMemoizedMergedChildContext=e,fr(Ri),fr(Xn),_r(Xn,e)):fr(Ri),_r(Ri,n)}o(I_,"Kf");var q1=null,Sf=null,UR=Tn.unstable_runWithPriority,V1=Tn.unstable_scheduleCallback,K1=Tn.unstable_cancelCallback,zR=Tn.unstable_shouldYield,F_=Tn.unstable_requestPaint,G1=Tn.unstable_now,$R=Tn.unstable_getCurrentPriorityLevel,wv=Tn.unstable_ImmediatePriority,B_=Tn.unstable_UserBlockingPriority,H_=Tn.unstable_NormalPriority,W_=Tn.unstable_LowPriority,U_=Tn.unstable_IdlePriority,Y1={},jR=F_!==void 0?F_:function(){},Pl=null,xv=null,X1=!1,z_=G1(),Qn=1e4>z_?G1:function(){return G1()-z_};function mp(){switch($R()){case wv:return 99;case B_:return 98;case H_:return 97;case W_:return 96;case U_:return 95;default:throw Error(we(332))}}o(mp,"eg");function $_(e){switch(e){case 99:return wv;case 98:return B_;case 97:return H_;case 96:return W_;case 95:return U_;default:throw Error(we(332))}}o($_,"fg");function Cf(e,t){return e=$_(e),UR(e,t)}o(Cf,"gg");function Th(e,t,n){return e=$_(e),V1(e,t,n)}o(Th,"hg");function $s(){if(xv!==null){var e=xv;xv=null,K1(e)}j_()}o($s,"ig");function j_(){if(!X1&&Pl!==null){X1=!0;var e=0;try{var t=Pl;Cf(99,function(){for(;epe?(me=ne,ne=null):me=ne.sibling;var xe=B(R,ne,I[pe],G);if(xe===null){ne===null&&(ne=me);break}e&&ne&&xe.alternate===null&&t(R,ne),A=h(xe,A,pe),se===null?K=xe:se.sibling=xe,se=xe,ne=me}if(pe===I.length)return n(R,ne),K;if(ne===null){for(;pepe?(me=ne,ne=null):me=ne.sibling;var Ve=B(R,ne,xe.value,G);if(Ve===null){ne===null&&(ne=me);break}e&&ne&&Ve.alternate===null&&t(R,ne),A=h(Ve,A,pe),se===null?K=Ve:se.sibling=Ve,se=Ve,ne=me}if(xe.done)return n(R,ne),K;if(ne===null){for(;!xe.done;pe++,xe=I.next())xe=j(R,xe.value,G),xe!==null&&(A=h(xe,A,pe),se===null?K=xe:se.sibling=xe,se=xe);return K}for(ne=l(R,ne);!xe.done;pe++,xe=I.next())xe=X(ne,R,pe,xe.value,G),xe!==null&&(e&&xe.alternate!==null&&ne.delete(xe.key===null?pe:xe.key),A=h(xe,A,pe),se===null?K=xe:se.sibling=xe,se=xe);return e&&ne.forEach(function(tt){return t(R,tt)}),K}return o(Z,"w"),function(R,A,I,G){var K=typeof I=="object"&&I!==null&&I.type===Ba&&I.key===null;K&&(I=I.props.children);var se=typeof I=="object"&&I!==null;if(se)switch(I.$$typeof){case th:e:{for(se=I.key,K=A;K!==null;){if(K.key===se){switch(K.tag){case 7:if(I.type===Ba){n(R,K.sibling),A=d(K,I.props.children),A.return=R,R=A;break e}break;default:if(K.elementType===I.type){n(R,K.sibling),A=d(K,I.props),A.ref=Nh(R,K,I),A.return=R,R=A;break e}}n(R,K);break}else t(R,K);K=K.sibling}I.type===Ba?(A=_p(I.props.children,R.mode,G,I.key),A.return=R,R=A):(G=qv(I.type,I.key,I.props,null,R.mode,G),G.ref=Nh(R,A,I),G.return=R,R=G)}return c(R);case gf:e:{for(K=I.key;A!==null;){if(A.key===K)if(A.tag===4&&A.stateNode.containerInfo===I.containerInfo&&A.stateNode.implementation===I.implementation){n(R,A.sibling),A=d(A,I.children||[]),A.return=R,R=A;break e}else{n(R,A);break}else t(R,A);A=A.sibling}A=Fx(I,R.mode,G),A.return=R,R=A}return c(R)}if(typeof I=="string"||typeof I=="number")return I=""+I,A!==null&&A.tag===6?(n(R,A.sibling),A=d(A,I),A.return=R,R=A):(n(R,A),A=Ix(I,R.mode,G),A.return=R,R=A),c(R);if(Tv(I))return J(R,A,I,G);if(ih(I))return Z(R,A,I,G);if(se&&kv(R,I),typeof I=="undefined"&&!K)switch(R.tag){case 1:case 22:case 0:case 11:case 15:throw Error(we(152,Zc(R.type)||"Component"))}return n(R,A)}}o(J_,"Sg");var Nv=J_(!0),eT=J_(!1),Lh={},js=Va(Lh),Ph=Va(Lh),Oh=Va(Lh);function bf(e){if(e===Lh)throw Error(we(174));return e}o(bf,"dh");function tx(e,t){switch(_r(Oh,t),_r(Ph,e),_r(js,Lh),e=t.nodeType,e){case 9:case 11:t=(t=t.documentElement)?t.namespaceURI:l1(null,"");break;default:e=e===8?t.parentNode:t,t=e.namespaceURI||null,e=e.tagName,t=l1(t,e)}fr(js),_r(js,t)}o(tx,"eh");function yp(){fr(js),fr(Ph),fr(Oh)}o(yp,"fh");function tT(e){bf(Oh.current);var t=bf(js.current),n=l1(t,e.type);t!==n&&(_r(Ph,e),_r(js,n))}o(tT,"gh");function rx(e){Ph.current===e&&(fr(js),fr(Ph))}o(rx,"hh");var Tr=Va(0);function Lv(e){for(var t=e;t!==null;){if(t.tag===13){var n=t.memoizedState;if(n!==null&&(n=n.dehydrated,n===null||n.data==="$?"||n.data==="$!"))return t}else if(t.tag===19&&t.memoizedProps.revealOrder!==void 0){if((t.flags&64)!=0)return t}else if(t.child!==null){t.child.return=t,t=t.child;continue}if(t===e)break;for(;t.sibling===null;){if(t.return===null||t.return===e)return null;t=t.return}t.sibling.return=t.return,t=t.sibling}return null}o(Lv,"ih");var Ol=null,Qa=null,qs=!1;function rT(e,t){var n=No(5,null,null,0);n.elementType="DELETED",n.type="DELETED",n.stateNode=t,n.return=e,n.flags=8,e.lastEffect!==null?(e.lastEffect.nextEffect=n,e.lastEffect=n):e.firstEffect=e.lastEffect=n}o(rT,"mh");function nT(e,t){switch(e.tag){case 5:var n=e.type;return t=t.nodeType!==1||n.toLowerCase()!==t.nodeName.toLowerCase()?null:t,t!==null?(e.stateNode=t,!0):!1;case 6:return t=e.pendingProps===""||t.nodeType!==3?null:t,t!==null?(e.stateNode=t,!0):!1;case 13:return!1;default:return!1}}o(nT,"oh");function nx(e){if(qs){var t=Qa;if(t){var n=t;if(!nT(e,t)){if(t=fp(n.nextSibling),!t||!nT(e,t)){e.flags=e.flags&-1025|2,qs=!1,Ol=e;return}rT(Ol,n)}Ol=e,Qa=fp(t.firstChild)}else e.flags=e.flags&-1025|2,qs=!1,Ol=e}}o(nx,"ph");function iT(e){for(e=e.return;e!==null&&e.tag!==5&&e.tag!==3&&e.tag!==13;)e=e.return;Ol=e}o(iT,"qh");function Pv(e){if(e!==Ol)return!1;if(!qs)return iT(e),qs=!0,!1;var t=e.type;if(e.tag!==5||t!=="head"&&t!=="body"&&!U1(t,e.memoizedProps))for(t=Qa;t;)rT(e,t),t=fp(t.nextSibling);if(iT(e),e.tag===13){if(e=e.memoizedState,e=e!==null?e.dehydrated:null,!e)throw Error(we(317));e:{for(e=e.nextSibling,t=0;e;){if(e.nodeType===8){var n=e.data;if(n==="/$"){if(t===0){Qa=fp(e.nextSibling);break e}t--}else n!=="$"&&n!=="$!"&&n!=="$?"||t++}e=e.nextSibling}Qa=null}}else Qa=Ol?fp(e.stateNode.nextSibling):null;return!0}o(Pv,"rh");function ix(){Qa=Ol=null,qs=!1}o(ix,"sh");var wp=[];function ox(){for(var e=0;eh))throw Error(we(301));h+=1,In=Zn=null,t.updateQueue=null,Mh.current=YR,e=n(l,d)}while(Dh)}if(Mh.current=Rv,t=Zn!==null&&Zn.next!==null,Ah=0,In=Zn=Fr=null,Ov=!1,t)throw Error(we(300));return e}o(lx,"Ch");function Ef(){var e={memoizedState:null,baseState:null,baseQueue:null,queue:null,next:null};return In===null?Fr.memoizedState=In=e:In=In.next=e,In}o(Ef,"Hh");function _f(){if(Zn===null){var e=Fr.alternate;e=e!==null?e.memoizedState:null}else e=Zn.next;var t=In===null?Fr.memoizedState:In.next;if(t!==null)In=t,Zn=e;else{if(e===null)throw Error(we(310));Zn=e,e={memoizedState:Zn.memoizedState,baseState:Zn.baseState,baseQueue:Zn.baseQueue,queue:Zn.queue,next:null},In===null?Fr.memoizedState=In=e:In=In.next=e}return In}o(_f,"Ih");function Vs(e,t){return typeof t=="function"?t(e):t}o(Vs,"Jh");function Rh(e){var t=_f(),n=t.queue;if(n===null)throw Error(we(311));n.lastRenderedReducer=e;var l=Zn,d=l.baseQueue,h=n.pending;if(h!==null){if(d!==null){var c=d.next;d.next=h.next,h.next=c}l.baseQueue=d=h,n.pending=null}if(d!==null){d=d.next,l=l.baseState;var v=c=h=null,C=d;do{var k=C.lane;if((Ah&k)===k)v!==null&&(v=v.next={lane:0,action:C.action,eagerReducer:C.eagerReducer,eagerState:C.eagerState,next:null}),l=C.eagerReducer===e?C.eagerState:e(l,C.action);else{var O={lane:k,action:C.action,eagerReducer:C.eagerReducer,eagerState:C.eagerState,next:null};v===null?(c=v=O,h=l):v=v.next=O,Fr.lanes|=k,Hh|=k}C=C.next}while(C!==null&&C!==d);v===null?h=l:v.next=c,Eo(l,t.memoizedState)||(ls=!0),t.memoizedState=l,t.baseState=h,t.baseQueue=v,n.lastRenderedState=l}return[t.memoizedState,n.dispatch]}o(Rh,"Kh");function Ih(e){var t=_f(),n=t.queue;if(n===null)throw Error(we(311));n.lastRenderedReducer=e;var l=n.dispatch,d=n.pending,h=t.memoizedState;if(d!==null){n.pending=null;var c=d=d.next;do h=e(h,c.action),c=c.next;while(c!==d);Eo(h,t.memoizedState)||(ls=!0),t.memoizedState=h,t.baseQueue===null&&(t.baseState=h),n.lastRenderedState=h}return[h,l]}o(Ih,"Lh");function oT(e,t,n){var l=t._getVersion;l=l(t._source);var d=t._workInProgressVersionPrimary;if(d!==null?e=d===l:(e=e.mutableReadLanes,(e=(Ah&e)===e)&&(t._workInProgressVersionPrimary=l,wp.push(t))),e)return n(t._source);throw wp.push(t),Error(we(350))}o(oT,"Mh");function sT(e,t,n,l){var d=yi;if(d===null)throw Error(we(349));var h=t._getVersion,c=h(t._source),v=Mh.current,C=v.useState(function(){return oT(d,t,n)}),k=C[1],O=C[0];C=In;var j=e.memoizedState,B=j.refs,X=B.getSnapshot,J=j.source;j=j.subscribe;var Z=Fr;return e.memoizedState={refs:B,source:t,subscribe:l},v.useEffect(function(){B.getSnapshot=n,B.setSnapshot=k;var R=h(t._source);if(!Eo(c,R)){R=n(t._source),Eo(O,R)||(k(R),R=Ja(Z),d.mutableReadLanes|=R&d.pendingLanes),R=d.mutableReadLanes,d.entangledLanes|=R;for(var A=d.entanglements,I=R;0n?98:n,function(){e(!0)}),Cf(97<\/script>",e=e.removeChild(e.firstChild)):typeof l.is=="string"?e=c.createElement(n,{is:l.is}):(e=c.createElement(n),n==="select"&&(c=e,l.multiple?c.multiple=!0:l.size&&(c.size=l.size))):e=c.createElementNS(e,n),e[qa]=t,e[mv]=l,kT(e,t,!1,!1),t.stateNode=e,c=u1(n,l),n){case"dialog":ur("cancel",e),ur("close",e),d=l;break;case"iframe":case"object":case"embed":ur("load",e),d=l;break;case"video":case"audio":for(d=0;dkx&&(t.flags|=64,h=!0,Bh(l,!1),t.lanes=33554432)}else{if(!h)if(e=Lv(c),e!==null){if(t.flags|=64,h=!0,n=e.updateQueue,n!==null&&(t.updateQueue=n,t.flags|=4),Bh(l,!0),l.tail===null&&l.tailMode==="hidden"&&!c.alternate&&!qs)return t=t.lastEffect=l.lastEffect,t!==null&&(t.nextEffect=null),null}else 2*Qn()-l.renderingStartTime>kx&&n!==1073741824&&(t.flags|=64,h=!0,Bh(l,!1),t.lanes=33554432);l.isBackwards?(c.sibling=t.child,t.child=c):(n=l.last,n!==null?n.sibling=c:t.child=c,l.last=c)}return l.tail!==null?(n=l.tail,l.rendering=n,l.tail=n.sibling,l.lastEffect=t.lastEffect,l.renderingStartTime=Qn(),n.sibling=null,t=Tr.current,_r(Tr,h?t&1|2:t&1),n):null;case 23:case 24:return Ax(),e!==null&&e.memoizedState!==null!=(t.memoizedState!==null)&&l.mode!=="unstable-defer-without-hiding"&&(t.flags|=4),null}throw Error(we(156,t.tag))}o(QR,"Gi");function ZR(e){switch(e.tag){case 1:Ii(e.type)&&vv();var t=e.flags;return t&4096?(e.flags=t&-4097|64,e):null;case 3:if(yp(),fr(Ri),fr(Xn),ox(),t=e.flags,(t&64)!=0)throw Error(we(285));return e.flags=t&-4097|64,e;case 5:return rx(e),null;case 13:return fr(Tr),t=e.flags,t&4096?(e.flags=t&-4097|64,e):null;case 19:return fr(Tr),null;case 4:return yp(),null;case 10:return Z1(e),null;case 23:case 24:return Ax(),null;default:return null}}o(ZR,"Li");function vx(e,t){try{var n="",l=t;do n+=MD(l),l=l.return;while(l);var d=n}catch(h){d=` +`+d[c].replace(" at new "," at ");while(1<=c&&0<=v);break}}}finally{Zw=!1,Error.prepareStackTrace=n}return(e=e?e.displayName||e.name:"")?oh(e):""}o(Vg,"Pa");function MD(e){switch(e.tag){case 5:return oh(e.type);case 16:return oh("Lazy");case 13:return oh("Suspense");case 19:return oh("SuspenseList");case 0:case 2:case 15:return e=Vg(e.type,!1),e;case 11:return e=Vg(e.type.render,!1),e;case 22:return e=Vg(e.type._render,!1),e;case 1:return e=Vg(e.type,!0),e;default:return""}}o(MD,"Qa");function Zc(e){if(e==null)return null;if(typeof e=="function")return e.displayName||e.name||null;if(typeof e=="string")return e;switch(e){case Ba:return"Fragment";case gf:return"Portal";case rh:return"Profiler";case $w:return"StrictMode";case nh:return"Suspense";case jg:return"SuspenseList"}if(typeof e=="object")switch(e.$$typeof){case qw:return(e.displayName||"Context")+".Consumer";case jw:return(e._context.displayName||"Context")+".Provider";case $g:var t=e.render;return t=t.displayName||t.name||"",e.displayName||(t!==""?"ForwardRef("+t+")":"ForwardRef");case qg:return Zc(e.type);case Kw:return Zc(e._render);case Vw:t=e._payload,e=e._init;try{return Zc(e(t))}catch(n){}}return null}o(Zc,"Ra");function Ha(e){switch(typeof e){case"boolean":case"number":case"object":case"string":case"undefined":return e;default:return""}}o(Ha,"Sa");function wE(e){var t=e.type;return(e=e.nodeName)&&e.toLowerCase()==="input"&&(t==="checkbox"||t==="radio")}o(wE,"Ta");function AD(e){var t=wE(e)?"checked":"value",n=Object.getOwnPropertyDescriptor(e.constructor.prototype,t),l=""+e[t];if(!e.hasOwnProperty(t)&&typeof n!="undefined"&&typeof n.get=="function"&&typeof n.set=="function"){var d=n.get,h=n.set;return Object.defineProperty(e,t,{configurable:!0,get:function(){return d.call(this)},set:function(c){l=""+c,h.call(this,c)}}),Object.defineProperty(e,t,{enumerable:n.enumerable}),{getValue:function(){return l},setValue:function(c){l=""+c},stopTracking:function(){e._valueTracker=null,delete e[t]}}}}o(AD,"Ua");function Kg(e){e._valueTracker||(e._valueTracker=AD(e))}o(Kg,"Va");function xE(e){if(!e)return!1;var t=e._valueTracker;if(!t)return!0;var n=t.getValue(),l="";return e&&(l=wE(e)?e.checked?"true":"false":e.value),e=l,e!==n?(t.setValue(e),!0):!1}o(xE,"Wa");function Gg(e){if(e=e||(typeof document!="undefined"?document:void 0),typeof e=="undefined")return null;try{return e.activeElement||e.body}catch(t){return e.body}}o(Gg,"Xa");function Jw(e,t){var n=t.checked;return hr({},t,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:n??e._wrapperState.initialChecked})}o(Jw,"Ya");function SE(e,t){var n=t.defaultValue==null?"":t.defaultValue,l=t.checked!=null?t.checked:t.defaultChecked;n=Ha(t.value!=null?t.value:n),e._wrapperState={initialChecked:l,initialValue:n,controlled:t.type==="checkbox"||t.type==="radio"?t.checked!=null:t.value!=null}}o(SE,"Za");function CE(e,t){t=t.checked,t!=null&&zw(e,"checked",t,!1)}o(CE,"$a");function e1(e,t){CE(e,t);var n=Ha(t.value),l=t.type;if(n!=null)l==="number"?(n===0&&e.value===""||e.value!=n)&&(e.value=""+n):e.value!==""+n&&(e.value=""+n);else if(l==="submit"||l==="reset"){e.removeAttribute("value");return}t.hasOwnProperty("value")?t1(e,t.type,n):t.hasOwnProperty("defaultValue")&&t1(e,t.type,Ha(t.defaultValue)),t.checked==null&&t.defaultChecked!=null&&(e.defaultChecked=!!t.defaultChecked)}o(e1,"ab");function bE(e,t,n){if(t.hasOwnProperty("value")||t.hasOwnProperty("defaultValue")){var l=t.type;if(!(l!=="submit"&&l!=="reset"||t.value!==void 0&&t.value!==null))return;t=""+e._wrapperState.initialValue,n||t===e.value||(e.value=t),e.defaultValue=t}n=e.name,n!==""&&(e.name=""),e.defaultChecked=!!e._wrapperState.initialChecked,n!==""&&(e.name=n)}o(bE,"cb");function t1(e,t,n){(t!=="number"||Gg(e.ownerDocument)!==e)&&(n==null?e.defaultValue=""+e._wrapperState.initialValue:e.defaultValue!==""+n&&(e.defaultValue=""+n))}o(t1,"bb");function DD(e){var t="";return zg.Children.forEach(e,function(n){n!=null&&(t+=n)}),t}o(DD,"db");function r1(e,t){return e=hr({children:void 0},t),(t=DD(t.children))&&(e.children=t),e}o(r1,"eb");function Jc(e,t,n,l){if(e=e.options,t){t={};for(var d=0;d=n.length))throw Error(we(93));n=n[0]}t=n}t==null&&(t=""),n=t}e._wrapperState={initialValue:Ha(n)}}o(EE,"hb");function _E(e,t){var n=Ha(t.value),l=Ha(t.defaultValue);n!=null&&(n=""+n,n!==e.value&&(e.value=n),t.defaultValue==null&&e.defaultValue!==n&&(e.defaultValue=n)),l!=null&&(e.defaultValue=""+l)}o(_E,"ib");function TE(e){var t=e.textContent;t===e._wrapperState.initialValue&&t!==""&&t!==null&&(e.value=t)}o(TE,"jb");var i1={html:"http://www.w3.org/1999/xhtml",mathml:"http://www.w3.org/1998/Math/MathML",svg:"http://www.w3.org/2000/svg"};function kE(e){switch(e){case"svg":return"http://www.w3.org/2000/svg";case"math":return"http://www.w3.org/1998/Math/MathML";default:return"http://www.w3.org/1999/xhtml"}}o(kE,"lb");function o1(e,t){return e==null||e==="http://www.w3.org/1999/xhtml"?kE(t):e==="http://www.w3.org/2000/svg"&&t==="foreignObject"?"http://www.w3.org/1999/xhtml":e}o(o1,"mb");var Yg,NE=function(e){return typeof MSApp!="undefined"&&MSApp.execUnsafeLocalFunction?function(t,n,l,d){MSApp.execUnsafeLocalFunction(function(){return e(t,n,l,d)})}:e}(function(e,t){if(e.namespaceURI!==i1.svg||"innerHTML"in e)e.innerHTML=t;else{for(Yg=Yg||document.createElement("div"),Yg.innerHTML=""+t.valueOf().toString()+"",t=Yg.firstChild;e.firstChild;)e.removeChild(e.firstChild);for(;t.firstChild;)e.appendChild(t.firstChild)}});function sh(e,t){if(t){var n=e.firstChild;if(n&&n===e.lastChild&&n.nodeType===3){n.nodeValue=t;return}}e.textContent=t}o(sh,"pb");var lh={animationIterationCount:!0,borderImageOutset:!0,borderImageSlice:!0,borderImageWidth:!0,boxFlex:!0,boxFlexGroup:!0,boxOrdinalGroup:!0,columnCount:!0,columns:!0,flex:!0,flexGrow:!0,flexPositive:!0,flexShrink:!0,flexNegative:!0,flexOrder:!0,gridArea:!0,gridRow:!0,gridRowEnd:!0,gridRowSpan:!0,gridRowStart:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnSpan:!0,gridColumnStart:!0,fontWeight:!0,lineClamp:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,tabSize:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeDasharray:!0,strokeDashoffset:!0,strokeMiterlimit:!0,strokeOpacity:!0,strokeWidth:!0},RD=["Webkit","ms","Moz","O"];Object.keys(lh).forEach(function(e){RD.forEach(function(t){t=t+e.charAt(0).toUpperCase()+e.substring(1),lh[t]=lh[e]})});function LE(e,t,n){return t==null||typeof t=="boolean"||t===""?"":n||typeof t!="number"||t===0||lh.hasOwnProperty(e)&&lh[e]?(""+t).trim():t+"px"}o(LE,"sb");function PE(e,t){e=e.style;for(var n in t)if(t.hasOwnProperty(n)){var l=n.indexOf("--")===0,d=LE(n,t[n],l);n==="float"&&(n="cssFloat"),l?e.setProperty(n,d):e[n]=d}}o(PE,"tb");var ID=hr({menuitem:!0},{area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,keygen:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0});function s1(e,t){if(t){if(ID[e]&&(t.children!=null||t.dangerouslySetInnerHTML!=null))throw Error(we(137,e));if(t.dangerouslySetInnerHTML!=null){if(t.children!=null)throw Error(we(60));if(!(typeof t.dangerouslySetInnerHTML=="object"&&"__html"in t.dangerouslySetInnerHTML))throw Error(we(61))}if(t.style!=null&&typeof t.style!="object")throw Error(we(62))}}o(s1,"vb");function l1(e,t){if(e.indexOf("-")===-1)return typeof t.is=="string";switch(e){case"annotation-xml":case"color-profile":case"font-face":case"font-face-src":case"font-face-uri":case"font-face-format":case"font-face-name":case"missing-glyph":return!1;default:return!0}}o(l1,"wb");function a1(e){return e=e.target||e.srcElement||window,e.correspondingUseElement&&(e=e.correspondingUseElement),e.nodeType===3?e.parentNode:e}o(a1,"xb");var u1=null,ep=null,tp=null;function OE(e){if(e=_h(e)){if(typeof u1!="function")throw Error(we(280));var t=e.stateNode;t&&(t=gv(t),u1(e.stateNode,e.type,t))}}o(OE,"Bb");function ME(e){ep?tp?tp.push(e):tp=[e]:ep=e}o(ME,"Eb");function AE(){if(ep){var e=ep,t=tp;if(tp=ep=null,OE(e),t)for(e=0;el?0:1<n;n++)t.push(e);return t}o(C1,"Zc");function rv(e,t,n){e.pendingLanes|=t;var l=t-1;e.suspendedLanes&=l,e.pingedLanes&=l,e=e.eventTimes,t=31-$a(t),e[t]=n}o(rv,"$c");var $a=Math.clz32?Math.clz32:ZD,XD=Math.log,QD=Math.LN2;function ZD(e){return e===0?32:31-(XD(e)/QD|0)|0}o(ZD,"ad");var JD=Tn.unstable_UserBlockingPriority,eR=Tn.unstable_runWithPriority,nv=!0;function tR(e,t,n,l){vf||c1();var d=b1,h=vf;vf=!0;try{DE(d,e,t,n,l)}finally{(vf=h)||d1()}}o(tR,"gd");function rR(e,t,n,l){eR(JD,b1.bind(null,e,t,n,l))}o(rR,"id");function b1(e,t,n,l){if(nv){var d;if((d=(t&4)==0)&&0=yh),s_=String.fromCharCode(32),l_=!1;function a_(e,t){switch(e){case"keyup":return _R.indexOf(t.keyCode)!==-1;case"keydown":return t.keyCode!==229;case"keypress":case"mousedown":case"focusout":return!0;default:return!1}}o(a_,"ge");function u_(e){return e=e.detail,typeof e=="object"&&"data"in e?e.data:null}o(u_,"he");var lp=!1;function kR(e,t){switch(e){case"compositionend":return u_(t);case"keypress":return t.which!==32?null:(l_=!0,s_);case"textInput":return e=t.data,e===s_&&l_?null:e;default:return null}}o(kR,"je");function NR(e,t){if(lp)return e==="compositionend"||!O1&&a_(e,t)?(e=e_(),iv=_1=ja=null,lp=!1,e):null;switch(e){case"paste":return null;case"keypress":if(!(t.ctrlKey||t.altKey||t.metaKey)||t.ctrlKey&&t.altKey){if(t.char&&1=t)return{node:n,offset:t-e};e=l}e:{for(;n;){if(n.nextSibling){n=n.nextSibling;break e}n=n.parentNode}n=void 0}n=m_(n)}}o(g_,"Le");function v_(e,t){return e&&t?e===t?!0:e&&e.nodeType===3?!1:t&&t.nodeType===3?v_(e,t.parentNode):"contains"in e?e.contains(t):e.compareDocumentPosition?!!(e.compareDocumentPosition(t)&16):!1:!1}o(v_,"Me");function y_(){for(var e=window,t=Gg();t instanceof e.HTMLIFrameElement;){try{var n=typeof t.contentWindow.location.href=="string"}catch(l){n=!1}if(n)e=t.contentWindow;else break;t=Gg(e.document)}return t}o(y_,"Ne");function A1(e){var t=e&&e.nodeName&&e.nodeName.toLowerCase();return t&&(t==="input"&&(e.type==="text"||e.type==="search"||e.type==="tel"||e.type==="url"||e.type==="password")||t==="textarea"||e.contentEditable==="true")}o(A1,"Oe");var BR=Ll&&"documentMode"in document&&11>=document.documentMode,ap=null,D1=null,Ch=null,R1=!1;function w_(e,t,n){var l=n.window===n?n.document:n.nodeType===9?n:n.ownerDocument;R1||ap==null||ap!==Gg(l)||(l=ap,"selectionStart"in l&&A1(l)?l={start:l.selectionStart,end:l.selectionEnd}:(l=(l.ownerDocument&&l.ownerDocument.defaultView||window).getSelection(),l={anchorNode:l.anchorNode,anchorOffset:l.anchorOffset,focusNode:l.focusNode,focusOffset:l.focusOffset}),Ch&&Sh(Ch,l)||(Ch=l,l=pv(D1,"onSelect"),0dp||(e.current=z1[dp],z1[dp]=null,dp--)}o(fr,"H");function _r(e,t){dp++,z1[dp]=e.current,e.current=t}o(_r,"I");var Ka={},Xn=Va(Ka),Ri=Va(!1),xf=Ka;function hp(e,t){var n=e.type.contextTypes;if(!n)return Ka;var l=e.stateNode;if(l&&l.__reactInternalMemoizedUnmaskedChildContext===t)return l.__reactInternalMemoizedMaskedChildContext;var d={},h;for(h in n)d[h]=t[h];return l&&(e=e.stateNode,e.__reactInternalMemoizedUnmaskedChildContext=t,e.__reactInternalMemoizedMaskedChildContext=d),d}o(hp,"Ef");function Ii(e){return e=e.childContextTypes,e!=null}o(Ii,"Ff");function vv(){fr(Ri),fr(Xn)}o(vv,"Gf");function D_(e,t,n){if(Xn.current!==Ka)throw Error(we(168));_r(Xn,t),_r(Ri,n)}o(D_,"Hf");function R_(e,t,n){var l=e.stateNode;if(e=t.childContextTypes,typeof l.getChildContext!="function")return n;l=l.getChildContext();for(var d in l)if(!(d in e))throw Error(we(108,Zc(t)||"Unknown",d));return hr({},n,l)}o(R_,"If");function yv(e){return e=(e=e.stateNode)&&e.__reactInternalMemoizedMergedChildContext||Ka,xf=Xn.current,_r(Xn,e),_r(Ri,Ri.current),!0}o(yv,"Jf");function I_(e,t,n){var l=e.stateNode;if(!l)throw Error(we(169));n?(e=R_(e,t,xf),l.__reactInternalMemoizedMergedChildContext=e,fr(Ri),fr(Xn),_r(Xn,e)):fr(Ri),_r(Ri,n)}o(I_,"Kf");var $1=null,Sf=null,UR=Tn.unstable_runWithPriority,j1=Tn.unstable_scheduleCallback,q1=Tn.unstable_cancelCallback,zR=Tn.unstable_shouldYield,F_=Tn.unstable_requestPaint,V1=Tn.unstable_now,$R=Tn.unstable_getCurrentPriorityLevel,wv=Tn.unstable_ImmediatePriority,B_=Tn.unstable_UserBlockingPriority,H_=Tn.unstable_NormalPriority,W_=Tn.unstable_LowPriority,U_=Tn.unstable_IdlePriority,K1={},jR=F_!==void 0?F_:function(){},Pl=null,xv=null,G1=!1,z_=V1(),Qn=1e4>z_?V1:function(){return V1()-z_};function mp(){switch($R()){case wv:return 99;case B_:return 98;case H_:return 97;case W_:return 96;case U_:return 95;default:throw Error(we(332))}}o(mp,"eg");function $_(e){switch(e){case 99:return wv;case 98:return B_;case 97:return H_;case 96:return W_;case 95:return U_;default:throw Error(we(332))}}o($_,"fg");function Cf(e,t){return e=$_(e),UR(e,t)}o(Cf,"gg");function Th(e,t,n){return e=$_(e),j1(e,t,n)}o(Th,"hg");function $s(){if(xv!==null){var e=xv;xv=null,q1(e)}j_()}o($s,"ig");function j_(){if(!G1&&Pl!==null){G1=!0;var e=0;try{var t=Pl;Cf(99,function(){for(;epe?(me=ne,ne=null):me=ne.sibling;var xe=B(R,ne,I[pe],G);if(xe===null){ne===null&&(ne=me);break}e&&ne&&xe.alternate===null&&t(R,ne),A=h(xe,A,pe),se===null?K=xe:se.sibling=xe,se=xe,ne=me}if(pe===I.length)return n(R,ne),K;if(ne===null){for(;pepe?(me=ne,ne=null):me=ne.sibling;var Ve=B(R,ne,xe.value,G);if(Ve===null){ne===null&&(ne=me);break}e&&ne&&Ve.alternate===null&&t(R,ne),A=h(Ve,A,pe),se===null?K=Ve:se.sibling=Ve,se=Ve,ne=me}if(xe.done)return n(R,ne),K;if(ne===null){for(;!xe.done;pe++,xe=I.next())xe=j(R,xe.value,G),xe!==null&&(A=h(xe,A,pe),se===null?K=xe:se.sibling=xe,se=xe);return K}for(ne=l(R,ne);!xe.done;pe++,xe=I.next())xe=X(ne,R,pe,xe.value,G),xe!==null&&(e&&xe.alternate!==null&&ne.delete(xe.key===null?pe:xe.key),A=h(xe,A,pe),se===null?K=xe:se.sibling=xe,se=xe);return e&&ne.forEach(function(tt){return t(R,tt)}),K}return o(Z,"w"),function(R,A,I,G){var K=typeof I=="object"&&I!==null&&I.type===Ba&&I.key===null;K&&(I=I.props.children);var se=typeof I=="object"&&I!==null;if(se)switch(I.$$typeof){case th:e:{for(se=I.key,K=A;K!==null;){if(K.key===se){switch(K.tag){case 7:if(I.type===Ba){n(R,K.sibling),A=d(K,I.props.children),A.return=R,R=A;break e}break;default:if(K.elementType===I.type){n(R,K.sibling),A=d(K,I.props),A.ref=Nh(R,K,I),A.return=R,R=A;break e}}n(R,K);break}else t(R,K);K=K.sibling}I.type===Ba?(A=_p(I.props.children,R.mode,G,I.key),A.return=R,R=A):(G=qv(I.type,I.key,I.props,null,R.mode,G),G.ref=Nh(R,A,I),G.return=R,R=G)}return c(R);case gf:e:{for(K=I.key;A!==null;){if(A.key===K)if(A.tag===4&&A.stateNode.containerInfo===I.containerInfo&&A.stateNode.implementation===I.implementation){n(R,A.sibling),A=d(A,I.children||[]),A.return=R,R=A;break e}else{n(R,A);break}else t(R,A);A=A.sibling}A=Rx(I,R.mode,G),A.return=R,R=A}return c(R)}if(typeof I=="string"||typeof I=="number")return I=""+I,A!==null&&A.tag===6?(n(R,A.sibling),A=d(A,I),A.return=R,R=A):(n(R,A),A=Dx(I,R.mode,G),A.return=R,R=A),c(R);if(Tv(I))return J(R,A,I,G);if(ih(I))return Z(R,A,I,G);if(se&&kv(R,I),typeof I=="undefined"&&!K)switch(R.tag){case 1:case 22:case 0:case 11:case 15:throw Error(we(152,Zc(R.type)||"Component"))}return n(R,A)}}o(J_,"Sg");var Nv=J_(!0),eT=J_(!1),Lh={},js=Va(Lh),Ph=Va(Lh),Oh=Va(Lh);function bf(e){if(e===Lh)throw Error(we(174));return e}o(bf,"dh");function J1(e,t){switch(_r(Oh,t),_r(Ph,e),_r(js,Lh),e=t.nodeType,e){case 9:case 11:t=(t=t.documentElement)?t.namespaceURI:o1(null,"");break;default:e=e===8?t.parentNode:t,t=e.namespaceURI||null,e=e.tagName,t=o1(t,e)}fr(js),_r(js,t)}o(J1,"eh");function yp(){fr(js),fr(Ph),fr(Oh)}o(yp,"fh");function tT(e){bf(Oh.current);var t=bf(js.current),n=o1(t,e.type);t!==n&&(_r(Ph,e),_r(js,n))}o(tT,"gh");function ex(e){Ph.current===e&&(fr(js),fr(Ph))}o(ex,"hh");var Tr=Va(0);function Lv(e){for(var t=e;t!==null;){if(t.tag===13){var n=t.memoizedState;if(n!==null&&(n=n.dehydrated,n===null||n.data==="$?"||n.data==="$!"))return t}else if(t.tag===19&&t.memoizedProps.revealOrder!==void 0){if((t.flags&64)!=0)return t}else if(t.child!==null){t.child.return=t,t=t.child;continue}if(t===e)break;for(;t.sibling===null;){if(t.return===null||t.return===e)return null;t=t.return}t.sibling.return=t.return,t=t.sibling}return null}o(Lv,"ih");var Ol=null,Qa=null,qs=!1;function rT(e,t){var n=No(5,null,null,0);n.elementType="DELETED",n.type="DELETED",n.stateNode=t,n.return=e,n.flags=8,e.lastEffect!==null?(e.lastEffect.nextEffect=n,e.lastEffect=n):e.firstEffect=e.lastEffect=n}o(rT,"mh");function nT(e,t){switch(e.tag){case 5:var n=e.type;return t=t.nodeType!==1||n.toLowerCase()!==t.nodeName.toLowerCase()?null:t,t!==null?(e.stateNode=t,!0):!1;case 6:return t=e.pendingProps===""||t.nodeType!==3?null:t,t!==null?(e.stateNode=t,!0):!1;case 13:return!1;default:return!1}}o(nT,"oh");function tx(e){if(qs){var t=Qa;if(t){var n=t;if(!nT(e,t)){if(t=fp(n.nextSibling),!t||!nT(e,t)){e.flags=e.flags&-1025|2,qs=!1,Ol=e;return}rT(Ol,n)}Ol=e,Qa=fp(t.firstChild)}else e.flags=e.flags&-1025|2,qs=!1,Ol=e}}o(tx,"ph");function iT(e){for(e=e.return;e!==null&&e.tag!==5&&e.tag!==3&&e.tag!==13;)e=e.return;Ol=e}o(iT,"qh");function Pv(e){if(e!==Ol)return!1;if(!qs)return iT(e),qs=!0,!1;var t=e.type;if(e.tag!==5||t!=="head"&&t!=="body"&&!H1(t,e.memoizedProps))for(t=Qa;t;)rT(e,t),t=fp(t.nextSibling);if(iT(e),e.tag===13){if(e=e.memoizedState,e=e!==null?e.dehydrated:null,!e)throw Error(we(317));e:{for(e=e.nextSibling,t=0;e;){if(e.nodeType===8){var n=e.data;if(n==="/$"){if(t===0){Qa=fp(e.nextSibling);break e}t--}else n!=="$"&&n!=="$!"&&n!=="$?"||t++}e=e.nextSibling}Qa=null}}else Qa=Ol?fp(e.stateNode.nextSibling):null;return!0}o(Pv,"rh");function rx(){Qa=Ol=null,qs=!1}o(rx,"sh");var wp=[];function nx(){for(var e=0;eh))throw Error(we(301));h+=1,In=Zn=null,t.updateQueue=null,Mh.current=YR,e=n(l,d)}while(Dh)}if(Mh.current=Rv,t=Zn!==null&&Zn.next!==null,Ah=0,In=Zn=Fr=null,Ov=!1,t)throw Error(we(300));return e}o(ox,"Ch");function Ef(){var e={memoizedState:null,baseState:null,baseQueue:null,queue:null,next:null};return In===null?Fr.memoizedState=In=e:In=In.next=e,In}o(Ef,"Hh");function _f(){if(Zn===null){var e=Fr.alternate;e=e!==null?e.memoizedState:null}else e=Zn.next;var t=In===null?Fr.memoizedState:In.next;if(t!==null)In=t,Zn=e;else{if(e===null)throw Error(we(310));Zn=e,e={memoizedState:Zn.memoizedState,baseState:Zn.baseState,baseQueue:Zn.baseQueue,queue:Zn.queue,next:null},In===null?Fr.memoizedState=In=e:In=In.next=e}return In}o(_f,"Ih");function Vs(e,t){return typeof t=="function"?t(e):t}o(Vs,"Jh");function Rh(e){var t=_f(),n=t.queue;if(n===null)throw Error(we(311));n.lastRenderedReducer=e;var l=Zn,d=l.baseQueue,h=n.pending;if(h!==null){if(d!==null){var c=d.next;d.next=h.next,h.next=c}l.baseQueue=d=h,n.pending=null}if(d!==null){d=d.next,l=l.baseState;var v=c=h=null,C=d;do{var k=C.lane;if((Ah&k)===k)v!==null&&(v=v.next={lane:0,action:C.action,eagerReducer:C.eagerReducer,eagerState:C.eagerState,next:null}),l=C.eagerReducer===e?C.eagerState:e(l,C.action);else{var O={lane:k,action:C.action,eagerReducer:C.eagerReducer,eagerState:C.eagerState,next:null};v===null?(c=v=O,h=l):v=v.next=O,Fr.lanes|=k,Hh|=k}C=C.next}while(C!==null&&C!==d);v===null?h=l:v.next=c,Eo(l,t.memoizedState)||(ls=!0),t.memoizedState=l,t.baseState=h,t.baseQueue=v,n.lastRenderedState=l}return[t.memoizedState,n.dispatch]}o(Rh,"Kh");function Ih(e){var t=_f(),n=t.queue;if(n===null)throw Error(we(311));n.lastRenderedReducer=e;var l=n.dispatch,d=n.pending,h=t.memoizedState;if(d!==null){n.pending=null;var c=d=d.next;do h=e(h,c.action),c=c.next;while(c!==d);Eo(h,t.memoizedState)||(ls=!0),t.memoizedState=h,t.baseQueue===null&&(t.baseState=h),n.lastRenderedState=h}return[h,l]}o(Ih,"Lh");function oT(e,t,n){var l=t._getVersion;l=l(t._source);var d=t._workInProgressVersionPrimary;if(d!==null?e=d===l:(e=e.mutableReadLanes,(e=(Ah&e)===e)&&(t._workInProgressVersionPrimary=l,wp.push(t))),e)return n(t._source);throw wp.push(t),Error(we(350))}o(oT,"Mh");function sT(e,t,n,l){var d=yi;if(d===null)throw Error(we(349));var h=t._getVersion,c=h(t._source),v=Mh.current,C=v.useState(function(){return oT(d,t,n)}),k=C[1],O=C[0];C=In;var j=e.memoizedState,B=j.refs,X=B.getSnapshot,J=j.source;j=j.subscribe;var Z=Fr;return e.memoizedState={refs:B,source:t,subscribe:l},v.useEffect(function(){B.getSnapshot=n,B.setSnapshot=k;var R=h(t._source);if(!Eo(c,R)){R=n(t._source),Eo(O,R)||(k(R),R=Ja(Z),d.mutableReadLanes|=R&d.pendingLanes),R=d.mutableReadLanes,d.entangledLanes|=R;for(var A=d.entanglements,I=R;0n?98:n,function(){e(!0)}),Cf(97<\/script>",e=e.removeChild(e.firstChild)):typeof l.is=="string"?e=c.createElement(n,{is:l.is}):(e=c.createElement(n),n==="select"&&(c=e,l.multiple?c.multiple=!0:l.size&&(c.size=l.size))):e=c.createElementNS(e,n),e[qa]=t,e[mv]=l,kT(e,t,!1,!1),t.stateNode=e,c=l1(n,l),n){case"dialog":ur("cancel",e),ur("close",e),d=l;break;case"iframe":case"object":case"embed":ur("load",e),d=l;break;case"video":case"audio":for(d=0;d_x&&(t.flags|=64,h=!0,Bh(l,!1),t.lanes=33554432)}else{if(!h)if(e=Lv(c),e!==null){if(t.flags|=64,h=!0,n=e.updateQueue,n!==null&&(t.updateQueue=n,t.flags|=4),Bh(l,!0),l.tail===null&&l.tailMode==="hidden"&&!c.alternate&&!qs)return t=t.lastEffect=l.lastEffect,t!==null&&(t.nextEffect=null),null}else 2*Qn()-l.renderingStartTime>_x&&n!==1073741824&&(t.flags|=64,h=!0,Bh(l,!1),t.lanes=33554432);l.isBackwards?(c.sibling=t.child,t.child=c):(n=l.last,n!==null?n.sibling=c:t.child=c,l.last=c)}return l.tail!==null?(n=l.tail,l.rendering=n,l.tail=n.sibling,l.lastEffect=t.lastEffect,l.renderingStartTime=Qn(),n.sibling=null,t=Tr.current,_r(Tr,h?t&1|2:t&1),n):null;case 23:case 24:return Ox(),e!==null&&e.memoizedState!==null!=(t.memoizedState!==null)&&l.mode!=="unstable-defer-without-hiding"&&(t.flags|=4),null}throw Error(we(156,t.tag))}o(QR,"Gi");function ZR(e){switch(e.tag){case 1:Ii(e.type)&&vv();var t=e.flags;return t&4096?(e.flags=t&-4097|64,e):null;case 3:if(yp(),fr(Ri),fr(Xn),nx(),t=e.flags,(t&64)!=0)throw Error(we(285));return e.flags=t&-4097|64,e;case 5:return ex(e),null;case 13:return fr(Tr),t=e.flags,t&4096?(e.flags=t&-4097|64,e):null;case 19:return fr(Tr),null;case 4:return yp(),null;case 10:return X1(e),null;case 23:case 24:return Ox(),null;default:return null}}o(ZR,"Li");function mx(e,t){try{var n="",l=t;do n+=MD(l),l=l.return;while(l);var d=n}catch(h){d=` Error generating stack: `+h.message+` -`+h.stack}return{value:e,source:t,stack:d}}o(vx,"Mi");function yx(e,t){try{console.error(t.value)}catch(n){setTimeout(function(){throw n})}}o(yx,"Ni");var JR=typeof WeakMap=="function"?WeakMap:Map;function PT(e,t,n){n=Ya(-1,n),n.tag=3,n.payload={element:null};var l=t.value;return n.callback=function(){Hv||(Hv=!0,Nx=l),yx(e,t)},n}o(PT,"Pi");function OT(e,t,n){n=Ya(-1,n),n.tag=3;var l=e.type.getDerivedStateFromError;if(typeof l=="function"){var d=t.value;n.payload=function(){return yx(e,t),l(d)}}var h=e.stateNode;return h!==null&&typeof h.componentDidCatch=="function"&&(n.callback=function(){typeof l!="function"&&(Ks===null?Ks=new Set([this]):Ks.add(this),yx(e,t));var c=t.stack;this.componentDidCatch(t.value,{componentStack:c!==null?c:""})}),n}o(OT,"Si");var eI=typeof WeakSet=="function"?WeakSet:Set;function MT(e){var t=e.ref;if(t!==null)if(typeof t=="function")try{t(null)}catch(n){ru(e,n)}else t.current=null}o(MT,"Vi");function tI(e,t){switch(t.tag){case 0:case 11:case 15:case 22:return;case 1:if(t.flags&256&&e!==null){var n=e.memoizedProps,l=e.memoizedState;e=t.stateNode,t=e.getSnapshotBeforeUpdate(t.elementType===t.type?n:ss(t.type,n),l),e.__reactInternalSnapshotBeforeUpdate=t}return;case 3:t.flags&256&&z1(t.stateNode.containerInfo);return;case 5:case 6:case 4:case 17:return}throw Error(we(163))}o(tI,"Xi");function rI(e,t,n){switch(n.tag){case 0:case 11:case 15:case 22:if(t=n.updateQueue,t=t!==null?t.lastEffect:null,t!==null){e=t=t.next;do{if((e.tag&3)==3){var l=e.create;e.destroy=l()}e=e.next}while(e!==t)}if(t=n.updateQueue,t=t!==null?t.lastEffect:null,t!==null){e=t=t.next;do{var d=e;l=d.next,d=d.tag,(d&4)!=0&&(d&1)!=0&&(KT(n,e),fI(n,e)),e=l}while(e!==t)}return;case 1:e=n.stateNode,n.flags&4&&(t===null?e.componentDidMount():(l=n.elementType===n.type?t.memoizedProps:ss(n.type,t.memoizedProps),e.componentDidUpdate(l,t.memoizedState,e.__reactInternalSnapshotBeforeUpdate))),t=n.updateQueue,t!==null&&G_(n,t,e);return;case 3:if(t=n.updateQueue,t!==null){if(e=null,n.child!==null)switch(n.child.tag){case 5:e=n.child.stateNode;break;case 1:e=n.child.stateNode}G_(n,t,e)}return;case 5:e=n.stateNode,t===null&&n.flags&4&&L_(n.type,n.memoizedProps)&&e.focus();return;case 6:return;case 4:return;case 12:return;case 13:n.memoizedState===null&&(n=n.alternate,n!==null&&(n=n.memoizedState,n!==null&&(n=n.dehydrated,n!==null&&VE(n))));return;case 19:case 17:case 20:case 21:case 23:case 24:return}throw Error(we(163))}o(rI,"Yi");function AT(e,t){for(var n=e;;){if(n.tag===5){var l=n.stateNode;if(t)l=l.style,typeof l.setProperty=="function"?l.setProperty("display","none","important"):l.display="none";else{l=n.stateNode;var d=n.memoizedProps.style;d=d!=null&&d.hasOwnProperty("display")?d.display:null,l.style.display=LE("display",d)}}else if(n.tag===6)n.stateNode.nodeValue=t?"":n.memoizedProps;else if((n.tag!==23&&n.tag!==24||n.memoizedState===null||n===e)&&n.child!==null){n.child.return=n,n=n.child;continue}if(n===e)break;for(;n.sibling===null;){if(n.return===null||n.return===e)return;n=n.return}n.sibling.return=n.return,n=n.sibling}}o(AT,"aj");function DT(e,t){if(Sf&&typeof Sf.onCommitFiberUnmount=="function")try{Sf.onCommitFiberUnmount(q1,t)}catch(h){}switch(t.tag){case 0:case 11:case 14:case 15:case 22:if(e=t.updateQueue,e!==null&&(e=e.lastEffect,e!==null)){var n=e=e.next;do{var l=n,d=l.destroy;if(l=l.tag,d!==void 0)if((l&4)!=0)KT(t,n);else{l=t;try{d()}catch(h){ru(l,h)}}n=n.next}while(n!==e)}break;case 1:if(MT(t),e=t.stateNode,typeof e.componentWillUnmount=="function")try{e.props=t.memoizedProps,e.state=t.memoizedState,e.componentWillUnmount()}catch(h){ru(t,h)}break;case 5:MT(t);break;case 4:BT(e,t)}}o(DT,"bj");function RT(e){e.alternate=null,e.child=null,e.dependencies=null,e.firstEffect=null,e.lastEffect=null,e.memoizedProps=null,e.memoizedState=null,e.pendingProps=null,e.return=null,e.updateQueue=null}o(RT,"dj");function IT(e){return e.tag===5||e.tag===3||e.tag===4}o(IT,"ej");function FT(e){e:{for(var t=e.return;t!==null;){if(IT(t))break e;t=t.return}throw Error(we(160))}var n=t;switch(t=n.stateNode,n.tag){case 5:var l=!1;break;case 3:t=t.containerInfo,l=!0;break;case 4:t=t.containerInfo,l=!0;break;default:throw Error(we(161))}n.flags&16&&(sh(t,""),n.flags&=-17);e:t:for(n=e;;){for(;n.sibling===null;){if(n.return===null||IT(n.return)){n=null;break e}n=n.return}for(n.sibling.return=n.return,n=n.sibling;n.tag!==5&&n.tag!==6&&n.tag!==18;){if(n.flags&2||n.child===null||n.tag===4)continue t;n.child.return=n,n=n.child}if(!(n.flags&2)){n=n.stateNode;break e}}l?wx(e,n,t):xx(e,n,t)}o(FT,"fj");function wx(e,t,n){var l=e.tag,d=l===5||l===6;if(d)e=d?e.stateNode:e.stateNode.instance,t?n.nodeType===8?n.parentNode.insertBefore(e,t):n.insertBefore(e,t):(n.nodeType===8?(t=n.parentNode,t.insertBefore(e,n)):(t=n,t.appendChild(e)),n=n._reactRootContainer,n!=null||t.onclick!==null||(t.onclick=dv));else if(l!==4&&(e=e.child,e!==null))for(wx(e,t,n),e=e.sibling;e!==null;)wx(e,t,n),e=e.sibling}o(wx,"gj");function xx(e,t,n){var l=e.tag,d=l===5||l===6;if(d)e=d?e.stateNode:e.stateNode.instance,t?n.insertBefore(e,t):n.appendChild(e);else if(l!==4&&(e=e.child,e!==null))for(xx(e,t,n),e=e.sibling;e!==null;)xx(e,t,n),e=e.sibling}o(xx,"hj");function BT(e,t){for(var n=t,l=!1,d,h;;){if(!l){l=n.return;e:for(;;){if(l===null)throw Error(we(160));switch(d=l.stateNode,l.tag){case 5:h=!1;break e;case 3:d=d.containerInfo,h=!0;break e;case 4:d=d.containerInfo,h=!0;break e}l=l.return}l=!0}if(n.tag===5||n.tag===6){e:for(var c=e,v=n,C=v;;)if(DT(c,C),C.child!==null&&C.tag!==4)C.child.return=C,C=C.child;else{if(C===v)break e;for(;C.sibling===null;){if(C.return===null||C.return===v)break e;C=C.return}C.sibling.return=C.return,C=C.sibling}h?(c=d,v=n.stateNode,c.nodeType===8?c.parentNode.removeChild(v):c.removeChild(v)):d.removeChild(n.stateNode)}else if(n.tag===4){if(n.child!==null){d=n.stateNode.containerInfo,h=!0,n.child.return=n,n=n.child;continue}}else if(DT(e,n),n.child!==null){n.child.return=n,n=n.child;continue}if(n===t)break;for(;n.sibling===null;){if(n.return===null||n.return===t)return;n=n.return,n.tag===4&&(l=!1)}n.sibling.return=n.return,n=n.sibling}}o(BT,"cj");function Sx(e,t){switch(t.tag){case 0:case 11:case 14:case 15:case 22:var n=t.updateQueue;if(n=n!==null?n.lastEffect:null,n!==null){var l=n=n.next;do(l.tag&3)==3&&(e=l.destroy,l.destroy=void 0,e!==void 0&&e()),l=l.next;while(l!==n)}return;case 1:return;case 5:if(n=t.stateNode,n!=null){l=t.memoizedProps;var d=e!==null?e.memoizedProps:l;e=t.type;var h=t.updateQueue;if(t.updateQueue=null,h!==null){for(n[mv]=l,e==="input"&&l.type==="radio"&&l.name!=null&&CE(n,l),u1(e,d),t=u1(e,l),d=0;dd&&(d=c),n&=~h}if(n=d,n=Qn()-n,n=(120>n?120:480>n?480:1080>n?1080:1920>n?1920:3e3>n?3e3:4320>n?4320:1960*iI(n/1960))-n,10d&&(d=c),n&=~h}if(n=d,n=Qn()-n,n=(120>n?120:480>n?480:1080>n?1080:1920>n?1920:3e3>n?3e3:4320>n?4320:1960*iI(n/1960))-n,10 component higher in the tree to provide a loading indicator or placeholder to display.`)}Fn!==5&&(Fn=2),C=vx(C,v),B=c;do{switch(B.tag){case 3:h=C,B.flags|=4096,t&=-t,B.lanes|=t;var se=PT(B,h,t);K_(B,se);break e;case 1:h=C;var ne=B.type,pe=B.stateNode;if((B.flags&64)==0&&(typeof ne.getDerivedStateFromError=="function"||pe!==null&&typeof pe.componentDidCatch=="function"&&(Ks===null||!Ks.has(pe)))){B.flags|=4096,t&=-t,B.lanes|=t;var me=OT(B,h,t);K_(B,me);break e}}B=B.return}while(B!==null)}VT(n)}catch(xe){t=xe,sn===n&&n!==null&&(sn=n=n.return);continue}break}while(1)}o($T,"Sj");function jT(){var e=Fv.current;return Fv.current=Rv,e===null?Rv:e}o(jT,"Pj");function jh(e,t){var n=Ze;Ze|=16;var l=jT();yi===e&&Jn===t||Ep(e,t);do try{sI();break}catch(d){$T(e,d)}while(1);if(Q1(),Ze=n,Fv.current=l,sn!==null)throw Error(we(261));return yi=null,Jn=0,Fn}o(jh,"Tj");function sI(){for(;sn!==null;)qT(sn)}o(sI,"ak");function lI(){for(;sn!==null&&!zR();)qT(sn)}o(lI,"Rj");function qT(e){var t=YT(e.alternate,e,Tf);e.memoizedProps=e.pendingProps,t===null?VT(e):sn=t,Cx.current=null}o(qT,"bk");function VT(e){var t=e;do{var n=t.alternate;if(e=t.return,(t.flags&2048)==0){if(n=QR(n,t,Tf),n!==null){sn=n;return}if(n=t,n.tag!==24&&n.tag!==23||n.memoizedState===null||(Tf&1073741824)!=0||(n.mode&4)==0){for(var l=0,d=n.child;d!==null;)l|=d.lanes|d.childLanes,d=d.sibling;n.childLanes=l}e!==null&&(e.flags&2048)==0&&(e.firstEffect===null&&(e.firstEffect=t.firstEffect),t.lastEffect!==null&&(e.lastEffect!==null&&(e.lastEffect.nextEffect=t.firstEffect),e.lastEffect=t.lastEffect),1c&&(v=c,c=se,se=v),v=g_(I,se),h=g_(I,c),v&&h&&(K.rangeCount!==1||K.anchorNode!==v.node||K.anchorOffset!==v.offset||K.focusNode!==h.node||K.focusOffset!==h.offset)&&(G=G.createRange(),G.setStart(v.node,v.offset),K.removeAllRanges(),se>c?(K.addRange(G),K.extend(h.node,h.offset)):(G.setEnd(h.node,h.offset),K.addRange(G)))))),G=[],K=I;K=K.parentNode;)K.nodeType===1&&G.push({element:K,left:K.scrollLeft,top:K.scrollTop});for(typeof I.focus=="function"&&I.focus(),I=0;IQn()-Tx?Ep(e,0):Ex|=n),ko(e,t)}o(pI,"Yj");function dI(e,t){var n=e.stateNode;n!==null&&n.delete(t),t=0,t===0&&(t=e.mode,(t&2)==0?t=1:(t&4)==0?t=mp()===99?1:2:(Dl===0&&(Dl=xp),t=op(62914560&~Dl),t===0&&(t=4194304))),n=Ji(),e=$v(e,t),e!==null&&(rv(e,t,n),ko(e,n))}o(dI,"lj");var YT;YT=o(function(e,t,n){var l=t.lanes;if(e!==null)if(e.memoizedProps!==t.pendingProps||Ri.current)ls=!0;else if((n&l)!=0)ls=(e.flags&16384)!=0;else{switch(ls=!1,t.tag){case 3:xT(t),ix();break;case 5:tT(t);break;case 1:Ii(t.type)&&yv(t);break;case 4:tx(t,t.stateNode.containerInfo);break;case 10:l=t.memoizedProps.value;var d=t.type._context;_r(Sv,d._currentValue),d._currentValue=l;break;case 13:if(t.memoizedState!==null)return(n&t.child.childLanes)!=0?ST(e,t,n):(_r(Tr,Tr.current&1),t=Ml(e,t,n),t!==null?t.sibling:null);_r(Tr,Tr.current&1);break;case 19:if(l=(n&t.childLanes)!=0,(e.flags&64)!=0){if(l)return TT(e,t,n);t.flags|=64}if(d=t.memoizedState,d!==null&&(d.rendering=null,d.tail=null,d.lastEffect=null),_r(Tr,Tr.current),l)break;return null;case 23:case 24:return t.lanes=0,px(e,t,n)}return Ml(e,t,n)}else ls=!1;switch(t.lanes=0,t.tag){case 2:if(l=t.type,e!==null&&(e.alternate=null,t.alternate=null,t.flags|=2),e=t.pendingProps,d=hp(t,Xn.current),vp(t,n),d=lx(null,t,l,e,d,n),t.flags|=1,typeof d=="object"&&d!==null&&typeof d.render=="function"&&d.$$typeof===void 0){if(t.tag=1,t.memoizedState=null,t.updateQueue=null,Ii(l)){var h=!0;yv(t)}else h=!1;t.memoizedState=d.state!==null&&d.state!==void 0?d.state:null,J1(t);var c=l.getDerivedStateFromProps;typeof c=="function"&&Ev(t,l,c,e),d.updater=_v,t.stateNode=d,d._reactInternals=t,ex(t,l,e,n),t=hx(null,t,l,!0,h,n)}else t.tag=0,Bi(null,t,d,n),t=t.child;return t;case 16:d=t.elementType;e:{switch(e!==null&&(e.alternate=null,t.alternate=null,t.flags|=2),e=t.pendingProps,h=d._init,d=h(d._payload),t.type=d,h=t.tag=mI(d),e=ss(d,e),h){case 0:t=dx(null,t,d,e,n);break e;case 1:t=wT(null,t,d,e,n);break e;case 11:t=mT(null,t,d,e,n);break e;case 14:t=gT(null,t,d,ss(d.type,e),l,n);break e}throw Error(we(306,d,""))}return t;case 0:return l=t.type,d=t.pendingProps,d=t.elementType===l?d:ss(l,d),dx(e,t,l,d,n);case 1:return l=t.type,d=t.pendingProps,d=t.elementType===l?d:ss(l,d),wT(e,t,l,d,n);case 3:if(xT(t),l=t.updateQueue,e===null||l===null)throw Error(we(282));if(l=t.pendingProps,d=t.memoizedState,d=d!==null?d.element:null,V_(e,t),kh(t,l,null,n),l=t.memoizedState.element,l===d)ix(),t=Ml(e,t,n);else{if(d=t.stateNode,(h=d.hydrate)&&(Qa=fp(t.stateNode.containerInfo.firstChild),Ol=t,h=qs=!0),h){if(e=d.mutableSourceEagerHydrationData,e!=null)for(d=0;d{"use strict";function JT(){if(!(typeof __REACT_DEVTOOLS_GLOBAL_HOOK__=="undefined"||typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE!="function"))try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(JT)}catch(e){console.error(e)}}o(JT,"checkDCE");JT(),ek.exports=ZT()});var rk=Ue((v4,tk)=>{"use strict";var CI="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED";tk.exports=CI});var sk=Ue((y4,ok)=>{"use strict";var bI=rk();function nk(){}o(nk,"emptyFunction");function ik(){}o(ik,"emptyFunctionWithReset");ik.resetWarningCache=nk;ok.exports=function(){function e(l,d,h,c,v,C){if(C!==bI){var k=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw k.name="Invariant Violation",k}}o(e,"shim"),e.isRequired=e;function t(){return e}o(t,"getShim");var n={array:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:t,element:e,elementType:e,instanceOf:t,node:e,objectOf:t,oneOf:t,oneOfType:t,shape:t,exact:t,checkPropTypes:ik,resetWarningCache:nk};return n.PropTypes=n,n}});var ak=Ue((S4,lk)=>{lk.exports=sk()();var w4,x4});var mk=Ue(Ht=>{"use strict";var kn=typeof Symbol=="function"&&Symbol.for,zx=kn?Symbol.for("react.element"):60103,$x=kn?Symbol.for("react.portal"):60106,Yv=kn?Symbol.for("react.fragment"):60107,Xv=kn?Symbol.for("react.strict_mode"):60108,Qv=kn?Symbol.for("react.profiler"):60114,Zv=kn?Symbol.for("react.provider"):60109,Jv=kn?Symbol.for("react.context"):60110,jx=kn?Symbol.for("react.async_mode"):60111,e0=kn?Symbol.for("react.concurrent_mode"):60111,t0=kn?Symbol.for("react.forward_ref"):60112,r0=kn?Symbol.for("react.suspense"):60113,kI=kn?Symbol.for("react.suspense_list"):60120,n0=kn?Symbol.for("react.memo"):60115,i0=kn?Symbol.for("react.lazy"):60116,NI=kn?Symbol.for("react.block"):60121,LI=kn?Symbol.for("react.fundamental"):60117,PI=kn?Symbol.for("react.responder"):60118,OI=kn?Symbol.for("react.scope"):60119;function eo(e){if(typeof e=="object"&&e!==null){var t=e.$$typeof;switch(t){case zx:switch(e=e.type,e){case jx:case e0:case Yv:case Qv:case Xv:case r0:return e;default:switch(e=e&&e.$$typeof,e){case Jv:case t0:case i0:case n0:case Zv:return e;default:return t}}case $x:return t}}}o(eo,"z");function hk(e){return eo(e)===e0}o(hk,"A");Ht.AsyncMode=jx;Ht.ConcurrentMode=e0;Ht.ContextConsumer=Jv;Ht.ContextProvider=Zv;Ht.Element=zx;Ht.ForwardRef=t0;Ht.Fragment=Yv;Ht.Lazy=i0;Ht.Memo=n0;Ht.Portal=$x;Ht.Profiler=Qv;Ht.StrictMode=Xv;Ht.Suspense=r0;Ht.isAsyncMode=function(e){return hk(e)||eo(e)===jx};Ht.isConcurrentMode=hk;Ht.isContextConsumer=function(e){return eo(e)===Jv};Ht.isContextProvider=function(e){return eo(e)===Zv};Ht.isElement=function(e){return typeof e=="object"&&e!==null&&e.$$typeof===zx};Ht.isForwardRef=function(e){return eo(e)===t0};Ht.isFragment=function(e){return eo(e)===Yv};Ht.isLazy=function(e){return eo(e)===i0};Ht.isMemo=function(e){return eo(e)===n0};Ht.isPortal=function(e){return eo(e)===$x};Ht.isProfiler=function(e){return eo(e)===Qv};Ht.isStrictMode=function(e){return eo(e)===Xv};Ht.isSuspense=function(e){return eo(e)===r0};Ht.isValidElementType=function(e){return typeof e=="string"||typeof e=="function"||e===Yv||e===e0||e===Qv||e===Xv||e===r0||e===kI||typeof e=="object"&&e!==null&&(e.$$typeof===i0||e.$$typeof===n0||e.$$typeof===Zv||e.$$typeof===Jv||e.$$typeof===t0||e.$$typeof===LI||e.$$typeof===PI||e.$$typeof===OI||e.$$typeof===NI)};Ht.typeOf=eo});var vk=Ue((I4,gk)=>{"use strict";gk.exports=mk()});var Ek=Ue((F4,bk)=>{"use strict";var qx=vk(),MI={childContextTypes:!0,contextType:!0,contextTypes:!0,defaultProps:!0,displayName:!0,getDefaultProps:!0,getDerivedStateFromError:!0,getDerivedStateFromProps:!0,mixins:!0,propTypes:!0,type:!0},AI={name:!0,length:!0,prototype:!0,caller:!0,callee:!0,arguments:!0,arity:!0},DI={$$typeof:!0,render:!0,defaultProps:!0,displayName:!0,propTypes:!0},yk={$$typeof:!0,compare:!0,defaultProps:!0,displayName:!0,propTypes:!0,type:!0},Vx={};Vx[qx.ForwardRef]=DI;Vx[qx.Memo]=yk;function wk(e){return qx.isMemo(e)?yk:Vx[e.$$typeof]||MI}o(wk,"getStatics");var RI=Object.defineProperty,II=Object.getOwnPropertyNames,xk=Object.getOwnPropertySymbols,FI=Object.getOwnPropertyDescriptor,BI=Object.getPrototypeOf,Sk=Object.prototype;function Ck(e,t,n){if(typeof t!="string"){if(Sk){var l=BI(t);l&&l!==Sk&&Ck(e,l,n)}var d=II(t);xk&&(d=d.concat(xk(t)));for(var h=wk(e),c=wk(t),v=0;v{"use strict";var Nn=typeof Symbol=="function"&&Symbol.for,Kx=Nn?Symbol.for("react.element"):60103,Gx=Nn?Symbol.for("react.portal"):60106,o0=Nn?Symbol.for("react.fragment"):60107,s0=Nn?Symbol.for("react.strict_mode"):60108,l0=Nn?Symbol.for("react.profiler"):60114,a0=Nn?Symbol.for("react.provider"):60109,u0=Nn?Symbol.for("react.context"):60110,Yx=Nn?Symbol.for("react.async_mode"):60111,f0=Nn?Symbol.for("react.concurrent_mode"):60111,c0=Nn?Symbol.for("react.forward_ref"):60112,p0=Nn?Symbol.for("react.suspense"):60113,HI=Nn?Symbol.for("react.suspense_list"):60120,d0=Nn?Symbol.for("react.memo"):60115,h0=Nn?Symbol.for("react.lazy"):60116,WI=Nn?Symbol.for("react.block"):60121,UI=Nn?Symbol.for("react.fundamental"):60117,zI=Nn?Symbol.for("react.responder"):60118,$I=Nn?Symbol.for("react.scope"):60119;function to(e){if(typeof e=="object"&&e!==null){var t=e.$$typeof;switch(t){case Kx:switch(e=e.type,e){case Yx:case f0:case o0:case l0:case s0:case p0:return e;default:switch(e=e&&e.$$typeof,e){case u0:case c0:case h0:case d0:case a0:return e;default:return t}}case Gx:return t}}}o(to,"z");function _k(e){return to(e)===f0}o(_k,"A");Wt.AsyncMode=Yx;Wt.ConcurrentMode=f0;Wt.ContextConsumer=u0;Wt.ContextProvider=a0;Wt.Element=Kx;Wt.ForwardRef=c0;Wt.Fragment=o0;Wt.Lazy=h0;Wt.Memo=d0;Wt.Portal=Gx;Wt.Profiler=l0;Wt.StrictMode=s0;Wt.Suspense=p0;Wt.isAsyncMode=function(e){return _k(e)||to(e)===Yx};Wt.isConcurrentMode=_k;Wt.isContextConsumer=function(e){return to(e)===u0};Wt.isContextProvider=function(e){return to(e)===a0};Wt.isElement=function(e){return typeof e=="object"&&e!==null&&e.$$typeof===Kx};Wt.isForwardRef=function(e){return to(e)===c0};Wt.isFragment=function(e){return to(e)===o0};Wt.isLazy=function(e){return to(e)===h0};Wt.isMemo=function(e){return to(e)===d0};Wt.isPortal=function(e){return to(e)===Gx};Wt.isProfiler=function(e){return to(e)===l0};Wt.isStrictMode=function(e){return to(e)===s0};Wt.isSuspense=function(e){return to(e)===p0};Wt.isValidElementType=function(e){return typeof e=="string"||typeof e=="function"||e===o0||e===f0||e===l0||e===s0||e===p0||e===HI||typeof e=="object"&&e!==null&&(e.$$typeof===h0||e.$$typeof===d0||e.$$typeof===a0||e.$$typeof===u0||e.$$typeof===c0||e.$$typeof===UI||e.$$typeof===zI||e.$$typeof===$I||e.$$typeof===WI)};Wt.typeOf=to});var Nk=Ue((H4,kk)=>{"use strict";kk.exports=Tk()});var Qh=Ue((Np,Xh)=>{(function(){var e,t="4.17.21",n=200,l="Unsupported core-js use. Try https://npms.io/search?q=ponyfill.",d="Expected a function",h="Invalid `variable` option passed into `_.template`",c="__lodash_hash_undefined__",v=500,C="__lodash_placeholder__",k=1,O=2,j=4,B=1,X=2,J=1,Z=2,R=4,A=8,I=16,G=32,K=64,se=128,ne=256,pe=512,me=30,xe="...",Ve=800,tt=16,_e=1,St=2,We=3,Ke=1/0,Ge=9007199254740991,Xe=17976931348623157e292,nr=0/0,ct=4294967295,Hr=ct-1,Qt=ct>>>1,_t=[["ary",se],["bind",J],["bindKey",Z],["curry",A],["curryRight",I],["flip",pe],["partial",G],["partialRight",K],["rearg",ne]],Ct="[object Arguments]",ut="[object Array]",Lr="[object AsyncFunction]",zt="[object Boolean]",$t="[object Date]",ie="[object DOMException]",rt="[object Error]",Pr="[object Function]",Gt="[object GeneratorFunction]",Yt="[object Map]",Se="[object Number]",Or="[object Null]",fn="[object Object]",Un="[object Promise]",si="[object Proxy]",cn="[object RegExp]",Zt="[object Set]",gr="[object String]",pt="[object Symbol]",Ho="[object Undefined]",Cr="[object WeakMap]",Ui="[object WeakSet]",pn="[object ArrayBuffer]",zn="[object DataView]",Si="[object Float32Array]",Ci="[object Float64Array]",$n="[object Int8Array]",Mn="[object Int16Array]",Js="[object Int32Array]",H="[object Uint8Array]",ee="[object Uint8ClampedArray]",he="[object Uint16Array]",Te="[object Uint32Array]",ir=/\b__p \+= '';/g,Ul=/\b(__p \+=) '' \+/g,Ft=/(__e\(.*?\)|\b__t\)) \+\n'';/g,Wr=/&(?:amp|lt|gt|quot|#39);/g,or=/[&<>"']/g,li=RegExp(Wr.source),ds=RegExp(or.source),lo=/<%-([\s\S]+?)%>/g,bi=/<%([\s\S]+?)%>/g,el=/<%=([\s\S]+?)%>/g,hs=/\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/,dn=/^\w*$/,id=/[^.[\]]+|\[(?:(-?\d+(?:\.\d+)?)|(["'])((?:(?!\2)[^\\]|\\.)*?)\2)\]|(?=(?:\.|\[\])(?:\.|\[\]|$))/g,tl=/[\\^$.*+?()[\]{}|]/g,Qf=RegExp(tl.source),rl=/^\s+/,od=/\s/,Zf=/\{(?:\n\/\* \[wrapped with .+\] \*\/)?\n?/,wu=/\{\n\/\* \[wrapped with (.+)\] \*/,sd=/,? & /,zl=/[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g,ms=/[()=,{}\[\]\/\s]/,ld=/\\(\\)?/g,Jf=/\$\{([^\\}]*(?:\\.[^\\}]*)*)\}/g,nl=/\w*$/,xu=/^[-+]0x[0-9a-f]+$/i,Wo=/^0b[01]+$/i,ad=/^\[object .+?Constructor\]$/,Uo=/^0o[0-7]+$/i,$l=/^(?:0|[1-9]\d*)$/,ec=/[\xc0-\xd6\xd8-\xf6\xf8-\xff\u0100-\u017f]/g,jt=/($^)/,Me=/['\n\r\u2028\u2029\\]/g,Ei="\\ud800-\\udfff",Su="\\u0300-\\u036f",ai="\\ufe20-\\ufe2f",vt="\\u20d0-\\u20ff",ao=Su+ai+vt,zo="\\u2700-\\u27bf",jl="a-z\\xdf-\\xf6\\xf8-\\xff",ue="\\xac\\xb1\\xd7\\xf7",ze="\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf",Cu="\\u2000-\\u206f",bu=" \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000",gs="A-Z\\xc0-\\xd6\\xd8-\\xde",il="\\ufe0e\\ufe0f",Eu=ue+ze+Cu+bu,He="['\u2019]",ud="["+Ei+"]",ql="["+Eu+"]",uo="["+ao+"]",ui="\\d+",tc="["+zo+"]",_u="["+jl+"]",$o="[^"+Ei+Eu+ui+zo+jl+gs+"]",vs="\\ud83c[\\udffb-\\udfff]",Tu="(?:"+uo+"|"+vs+")",ol="[^"+Ei+"]",Vl="(?:\\ud83c[\\udde6-\\uddff]){2}",Kl="[\\ud800-\\udbff][\\udc00-\\udfff]",fo="["+gs+"]",Gl="\\u200d",Yl="(?:"+_u+"|"+$o+")",rc="(?:"+fo+"|"+$o+")",Xl="(?:"+He+"(?:d|ll|m|re|s|t|ve))?",_i="(?:"+He+"(?:D|LL|M|RE|S|T|VE))?",nc=Tu+"?",Ql="["+il+"]?",co="(?:"+Gl+"(?:"+[ol,Vl,Kl].join("|")+")"+Ql+nc+")*",ys="\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])",ic="\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])",oc=Ql+nc+co,fd="(?:"+[tc,Vl,Kl].join("|")+")"+oc,cd="(?:"+[ol+uo+"?",uo,Vl,Kl,ud].join("|")+")",ku=RegExp(He,"g"),sc=RegExp(uo,"g"),Nu=RegExp(vs+"(?="+vs+")|"+cd+oc,"g"),lc=RegExp([fo+"?"+_u+"+"+Xl+"(?="+[ql,fo,"$"].join("|")+")",rc+"+"+_i+"(?="+[ql,fo+Yl,"$"].join("|")+")",fo+"?"+Yl+"+"+Xl,fo+"+"+_i,ic,ys,ui,fd].join("|"),"g"),ac=RegExp("["+Gl+Ei+ao+il+"]"),Zl=/[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/,Jl=["Array","Buffer","DataView","Date","Error","Float32Array","Float64Array","Function","Int8Array","Int16Array","Int32Array","Map","Math","Object","Promise","RegExp","Set","String","Symbol","TypeError","Uint8Array","Uint8ClampedArray","Uint16Array","Uint32Array","WeakMap","_","clearTimeout","isFinite","parseInt","setTimeout"],Lu=-1,Ot={};Ot[Si]=Ot[Ci]=Ot[$n]=Ot[Mn]=Ot[Js]=Ot[H]=Ot[ee]=Ot[he]=Ot[Te]=!0,Ot[Ct]=Ot[ut]=Ot[pn]=Ot[zt]=Ot[zn]=Ot[$t]=Ot[rt]=Ot[Pr]=Ot[Yt]=Ot[Se]=Ot[fn]=Ot[cn]=Ot[Zt]=Ot[gr]=Ot[Cr]=!1;var Nt={};Nt[Ct]=Nt[ut]=Nt[pn]=Nt[zn]=Nt[zt]=Nt[$t]=Nt[Si]=Nt[Ci]=Nt[$n]=Nt[Mn]=Nt[Js]=Nt[Yt]=Nt[Se]=Nt[fn]=Nt[cn]=Nt[Zt]=Nt[gr]=Nt[pt]=Nt[H]=Nt[ee]=Nt[he]=Nt[Te]=!0,Nt[rt]=Nt[Pr]=Nt[Cr]=!1;var P={\u00C0:"A",\u00C1:"A",\u00C2:"A",\u00C3:"A",\u00C4:"A",\u00C5:"A",\u00E0:"a",\u00E1:"a",\u00E2:"a",\u00E3:"a",\u00E4:"a",\u00E5:"a",\u00C7:"C",\u00E7:"c",\u00D0:"D",\u00F0:"d",\u00C8:"E",\u00C9:"E",\u00CA:"E",\u00CB:"E",\u00E8:"e",\u00E9:"e",\u00EA:"e",\u00EB:"e",\u00CC:"I",\u00CD:"I",\u00CE:"I",\u00CF:"I",\u00EC:"i",\u00ED:"i",\u00EE:"i",\u00EF:"i",\u00D1:"N",\u00F1:"n",\u00D2:"O",\u00D3:"O",\u00D4:"O",\u00D5:"O",\u00D6:"O",\u00D8:"O",\u00F2:"o",\u00F3:"o",\u00F4:"o",\u00F5:"o",\u00F6:"o",\u00F8:"o",\u00D9:"U",\u00DA:"U",\u00DB:"U",\u00DC:"U",\u00F9:"u",\u00FA:"u",\u00FB:"u",\u00FC:"u",\u00DD:"Y",\u00FD:"y",\u00FF:"y",\u00C6:"Ae",\u00E6:"ae",\u00DE:"Th",\u00FE:"th",\u00DF:"ss",\u0100:"A",\u0102:"A",\u0104:"A",\u0101:"a",\u0103:"a",\u0105:"a",\u0106:"C",\u0108:"C",\u010A:"C",\u010C:"C",\u0107:"c",\u0109:"c",\u010B:"c",\u010D:"c",\u010E:"D",\u0110:"D",\u010F:"d",\u0111:"d",\u0112:"E",\u0114:"E",\u0116:"E",\u0118:"E",\u011A:"E",\u0113:"e",\u0115:"e",\u0117:"e",\u0119:"e",\u011B:"e",\u011C:"G",\u011E:"G",\u0120:"G",\u0122:"G",\u011D:"g",\u011F:"g",\u0121:"g",\u0123:"g",\u0124:"H",\u0126:"H",\u0125:"h",\u0127:"h",\u0128:"I",\u012A:"I",\u012C:"I",\u012E:"I",\u0130:"I",\u0129:"i",\u012B:"i",\u012D:"i",\u012F:"i",\u0131:"i",\u0134:"J",\u0135:"j",\u0136:"K",\u0137:"k",\u0138:"k",\u0139:"L",\u013B:"L",\u013D:"L",\u013F:"L",\u0141:"L",\u013A:"l",\u013C:"l",\u013E:"l",\u0140:"l",\u0142:"l",\u0143:"N",\u0145:"N",\u0147:"N",\u014A:"N",\u0144:"n",\u0146:"n",\u0148:"n",\u014B:"n",\u014C:"O",\u014E:"O",\u0150:"O",\u014D:"o",\u014F:"o",\u0151:"o",\u0154:"R",\u0156:"R",\u0158:"R",\u0155:"r",\u0157:"r",\u0159:"r",\u015A:"S",\u015C:"S",\u015E:"S",\u0160:"S",\u015B:"s",\u015D:"s",\u015F:"s",\u0161:"s",\u0162:"T",\u0164:"T",\u0166:"T",\u0163:"t",\u0165:"t",\u0167:"t",\u0168:"U",\u016A:"U",\u016C:"U",\u016E:"U",\u0170:"U",\u0172:"U",\u0169:"u",\u016B:"u",\u016D:"u",\u016F:"u",\u0171:"u",\u0173:"u",\u0174:"W",\u0175:"w",\u0176:"Y",\u0177:"y",\u0178:"Y",\u0179:"Z",\u017B:"Z",\u017D:"Z",\u017A:"z",\u017C:"z",\u017E:"z",\u0132:"IJ",\u0133:"ij",\u0152:"Oe",\u0153:"oe",\u0149:"'n",\u017F:"s"},Re={"&":"&","<":"<",">":">",'"':""","'":"'"},sl={"&":"&","<":"<",">":">",""":'"',"'":"'"},vr={"\\":"\\","'":"'","\n":"n","\r":"r","\u2028":"u2028","\u2029":"u2029"},Pu=parseFloat,ye=parseInt,jo=typeof global=="object"&&global&&global.Object===Object&&global,pd=typeof self=="object"&&self&&self.Object===Object&&self,qt=jo||pd||Function("return this")(),ea=typeof Np=="object"&&Np&&!Np.nodeType&&Np,hn=ea&&typeof Xh=="object"&&Xh&&!Xh.nodeType&&Xh,ws=hn&&hn.exports===ea,zi=ws&&jo.process,Ce=function(){try{var q=hn&&hn.require&&hn.require("util").types;return q||zi&&zi.binding&&zi.binding("util")}catch(re){}}(),ta=Ce&&Ce.isArrayBuffer,Ou=Ce&&Ce.isDate,nt=Ce&&Ce.isMap,uc=Ce&&Ce.isRegExp,fi=Ce&&Ce.isSet,ll=Ce&&Ce.isTypedArray;function Ur(q,re,Q){switch(Q.length){case 0:return q.call(re);case 1:return q.call(re,Q[0]);case 2:return q.call(re,Q[0],Q[1]);case 3:return q.call(re,Q[0],Q[1],Q[2])}return q.apply(re,Q)}o(Ur,"apply");function ra(q,re,Q,Pe){for(var Qe=-1,bt=q==null?0:q.length;++Qe-1}o(xs,"arrayIncludes");function qo(q,re,Q){for(var Pe=-1,Qe=q==null?0:q.length;++Pe-1;);return Q}o(qi,"charsStartIndex");function al(q,re){for(var Q=q.length;Q--&&Go(re,q[Q],0)>-1;);return Q}o(al,"charsEndIndex");function cc(q,re){for(var Q=q.length,Pe=0;Q--;)q[Q]===re&&++Pe;return Pe}o(cc,"countHolders");var Wu=oa(P),gd=oa(Re);function Uu(q){return"\\"+vr[q]}o(Uu,"escapeStringChar");function la(q,re){return q==null?e:q[re]}o(la,"getValue");function qn(q){return ac.test(q)}o(qn,"hasUnicode");function Ti(q){return Zl.test(q)}o(Ti,"hasUnicodeWord");function pc(q){for(var re,Q=[];!(re=q.next()).done;)Q.push(re.value);return Q}o(pc,"iteratorToArray");function aa(q){var re=-1,Q=Array(q.size);return q.forEach(function(Pe,Qe){Q[++re]=[Qe,Pe]}),Q}o(aa,"mapToArray");function dc(q,re){return function(Q){return q(re(Q))}}o(dc,"overArg");function Vi(q,re){for(var Q=-1,Pe=q.length,Qe=0,bt=[];++Q-1}o(Gy,"listCacheHas");function yc(s,f){var m=this.__data__,x=gl(m,s);return x<0?(++this.size,m.push([s,f])):m[x][1]=f,this}o(yc,"listCacheSet"),vo.prototype.clear=Cd,vo.prototype.delete=$m,vo.prototype.get=Yu,vo.prototype.has=Gy,vo.prototype.set=yc;function xr(s){var f=-1,m=s==null?0:s.length;for(this.clear();++f=f?s:f)),s}o(vl,"baseClamp");function Dn(s,f,m,x,_,L){var F,z=f&k,Y=f&O,le=f&j;if(m&&(F=_?m(s,x,_,L):m(s)),F!==e)return F;if(!Sr(s))return s;var ae=ot(s);if(ae){if(F=u(s),!z)return qr(s,F)}else{var ce=xn(s),Le=ce==Pr||ce==Gt;if(Ra(s))return Hd(s,z);if(ce==fn||ce==Ct||Le&&!_){if(F=Y||Le?{}:a(s),!z)return Y?dg(s,Qy(F,s)):ow(s,kd(F,s))}else{if(!Nt[ce])return _?s:{};F=p(s,ce,z)}}L||(L=new Ni);var Fe=L.get(s);if(Fe)return Fe;L.set(s,F),Cb(s)?s.forEach(function(je){F.add(Dn(je,f,m,je,s,L))}):xb(s)&&s.forEach(function(je,ht){F.set(ht,Dn(je,f,m,ht,s,L))});var $e=le?Y?Hc:ff:Y?Ai:_n,ft=ae?e:$e(s);return jn(ft||s,function(je,ht){ft&&(ht=je,je=s[ht]),Qu(F,ht,Dn(je,f,m,ht,s,L))}),F}o(Dn,"baseClone");function Gm(s){var f=_n(s);return function(m){return Ym(m,s,f)}}o(Gm,"baseConforms");function Ym(s,f,m){var x=m.length;if(s==null)return!x;for(s=mt(s);x--;){var _=m[x],L=f[_],F=s[_];if(F===e&&!(_ in s)||!L(F))return!1}return!0}o(Ym,"baseConformsTo");function Xm(s,f,m){if(typeof s!="function")throw new Kn(d);return At(function(){s.apply(e,m)},f)}o(Xm,"baseDelay");function va(s,f,m,x){var _=-1,L=xs,F=!0,z=s.length,Y=[],le=f.length;if(!z)return Y;m&&(f=yt(f,zr(m))),x?(L=qo,F=!1):f.length>=n&&(L=mn,F=!1,f=new tn(f));e:for(;++__?0:_+m),x=x===e||x>_?_:lt(x),x<0&&(x+=_),x=m>x?0:Eb(x);m0&&m(z)?f>1?rn(z,f-1,m,x,_):$i(_,z):x||(_[_.length]=z)}return _}o(rn,"baseFlatten");var bc=mg(),$r=mg(!0);function hi(s,f){return s&&bc(s,f,_n)}o(hi,"baseForOwn");function Ec(s,f){return s&&$r(s,f,_n)}o(Ec,"baseForOwnRight");function Ju(s,f){return Vt(f,function(m){return _l(s[m])})}o(Ju,"baseFunctions");function Ds(s,f){f=Gi(f,s);for(var m=0,x=f.length;s!=null&&mf}o(_c,"baseGt");function Qm(s,f){return s!=null&&et.call(s,f)}o(Qm,"baseHas");function Zm(s,f){return s!=null&&f in mt(s)}o(Zm,"baseHasIn");function ya(s,f,m){return s>=Zr(f,m)&&s=120&&ae.length>=120)?new tn(F&&ae):e}ae=s[0];var ce=-1,Le=z[0];e:for(;++ce<_&&le.length-1;)z!==s&&pa.call(z,Y,1),pa.call(s,Y,1);return s}o(Md,"basePullAll");function Ad(s,f){for(var m=s?f.length:0,x=m-1;m--;){var _=f[m];if(m==x||_!==L){var L=_;S(_)?pa.call(s,_,1):of(s,_)}}return s}o(Ad,"basePullAt");function Pc(s,f){return s+ha(Ns()*(f-s+1))}o(Pc,"baseRandom");function lg(s,f,m,x){for(var _=-1,L=sr(da((f-s)/(m||1)),0),F=Q(L);L--;)F[x?L:++_]=s,s+=m;return F}o(lg,"baseRange");function Dd(s,f){var m="";if(!s||f<1||f>Ge)return m;do f%2&&(m+=s),f=ha(f/2),f&&(s+=s);while(f);return m}o(Dd,"baseRepeat");function st(s,f){return Sn(Ie(s,f,Di),s+"")}o(st,"baseRest");function tw(s){return wc(jc(s))}o(tw,"baseSample");function Fs(s,f){var m=jc(s);return Vr(m,vl(f,0,m.length))}o(Fs,"baseSampleSize");function Zo(s,f,m,x){if(!Sr(s))return s;f=Gi(f,s);for(var _=-1,L=f.length,F=L-1,z=s;z!=null&&++__?0:_+f),m=m>_?_:m,m<0&&(m+=_),_=f>m?0:m-f>>>0,f>>>=0;for(var L=Q(_);++x<_;)L[x]=s[x+f];return L}o(Li,"baseSlice");function rw(s,f){var m;return di(s,function(x,_,L){return m=f(x,_,L),!m}),!!m}o(rw,"baseSome");function es(s,f,m){var x=0,_=s==null?x:s.length;if(typeof f=="number"&&f===f&&_<=Qt){for(;x<_;){var L=x+_>>>1,F=s[L];F!==null&&!Yi(F)&&(m?F<=f:F=n){var le=f?null:uf(s);if(le)return w(le);F=!1,_=mn,Y=new tn}else Y=f?[]:z;e:for(;++x=x?s:Li(s,f,m)}o(Bs,"castSlice");var Ta=Fy||function(s){return qt.clearTimeout(s)};function Hd(s,f){if(f)return s.slice();var m=s.length,x=Hm?Hm(m):new s.constructor(m);return s.copy(x),x}o(Hd,"cloneBuffer");function Dc(s){var f=new s.constructor(s.byteLength);return new cl(f).set(new cl(s)),f}o(Dc,"cloneArrayBuffer");function iw(s,f){var m=f?Dc(s.buffer):s.buffer;return new s.constructor(m,s.byteOffset,s.byteLength)}o(iw,"cloneDataView");function Wd(s){var f=new s.constructor(s.source,nl.exec(s));return f.lastIndex=s.lastIndex,f}o(Wd,"cloneRegExp");function ug(s){return lr?mt(lr.call(s)):{}}o(ug,"cloneSymbol");function fg(s,f){var m=f?Dc(s.buffer):s.buffer;return new s.constructor(m,s.byteOffset,s.length)}o(fg,"cloneTypedArray");function Ud(s,f){if(s!==f){var m=s!==e,x=s===null,_=s===s,L=Yi(s),F=f!==e,z=f===null,Y=f===f,le=Yi(f);if(!z&&!le&&!L&&s>f||L&&F&&Y&&!z&&!le||x&&F&&Y||!m&&Y||!_)return 1;if(!x&&!L&&!le&&s=z)return Y;var le=m[x];return Y*(le=="desc"?-1:1)}}return s.index-f.index}o(cg,"compareMultiple");function pg(s,f,m,x){for(var _=-1,L=s.length,F=m.length,z=-1,Y=f.length,le=sr(L-F,0),ae=Q(Y+le),ce=!x;++z1?m[_-1]:e,F=_>2?m[2]:e;for(L=s.length>3&&typeof L=="function"?(_--,L):e,F&&b(m[0],m[1],F)&&(L=_<3?e:L,_=1),f=mt(f);++x<_;){var z=m[x];z&&s(f,z,x,L)}return f})}o(ka,"createAssigner");function hg(s,f){return function(m,x){if(m==null)return m;if(!Mi(m))return s(m,x);for(var _=m.length,L=f?_:-1,F=mt(m);(f?L--:++L<_)&&x(F[L],L,F)!==!1;);return m}}o(hg,"createBaseEach");function mg(s){return function(f,m,x){for(var _=-1,L=mt(f),F=x(f),z=F.length;z--;){var Y=F[s?z:++_];if(m(L[Y],Y,L)===!1)break}return f}}o(mg,"createBaseFor");function gg(s,f,m){var x=f&J,_=La(s);function L(){var F=this&&this!==qt&&this instanceof L?_:s;return F.apply(x?m:this,arguments)}return o(L,"wrapper"),L}o(gg,"createBind");function vg(s){return function(f){f=Dt(f);var m=qn(f)?Kt(f):e,x=m?m[0]:f.charAt(0),_=m?Bs(m,1).join(""):f.slice(1);return x[s]()+_}}o(vg,"createCaseFirst");function Na(s){return function(f){return Au(Ab(Mb(f).replace(ku,"")),s,"")}}o(Na,"createCompounder");function La(s){return function(){var f=arguments;switch(f.length){case 0:return new s;case 1:return new s(f[0]);case 2:return new s(f[0],f[1]);case 3:return new s(f[0],f[1],f[2]);case 4:return new s(f[0],f[1],f[2],f[3]);case 5:return new s(f[0],f[1],f[2],f[3],f[4]);case 6:return new s(f[0],f[1],f[2],f[3],f[4],f[5]);case 7:return new s(f[0],f[1],f[2],f[3],f[4],f[5],f[6])}var m=go(s.prototype),x=s.apply(m,f);return Sr(x)?x:m}}o(La,"createCtor");function zd(s,f,m){var x=La(s);function _(){for(var L=arguments.length,F=Q(L),z=L,Y=Oa(_);z--;)F[z]=arguments[z];var le=L<3&&F[0]!==Y&&F[L-1]!==Y?[]:Vi(F,Y);if(L-=le.length,L-1?_[L?f[F]:F]:e}}o($d,"createFind");function yg(s){return ts(function(f){var m=f.length,x=m,_=An.prototype.thru;for(s&&f.reverse();x--;){var L=f[x];if(typeof L!="function")throw new Kn(d);if(_&&!F&&cf(L)=="wrapper")var F=new An([],!0)}for(x=F?x:m;++x1&>.reverse(),ae&&Yz))return!1;var le=L.get(s),ae=L.get(f);if(le&&ae)return le==f&&ae==s;var ce=-1,Le=!0,Fe=m&X?new tn:e;for(L.set(s,f),L.set(f,s);++ce1?"& ":"")+f[x],f=f.join(m>2?", ":" "),s.replace(Zf,`{ +Add a component higher in the tree to provide a loading indicator or placeholder to display.`)}Fn!==5&&(Fn=2),C=mx(C,v),B=c;do{switch(B.tag){case 3:h=C,B.flags|=4096,t&=-t,B.lanes|=t;var se=PT(B,h,t);K_(B,se);break e;case 1:h=C;var ne=B.type,pe=B.stateNode;if((B.flags&64)==0&&(typeof ne.getDerivedStateFromError=="function"||pe!==null&&typeof pe.componentDidCatch=="function"&&(Ks===null||!Ks.has(pe)))){B.flags|=4096,t&=-t,B.lanes|=t;var me=OT(B,h,t);K_(B,me);break e}}B=B.return}while(B!==null)}VT(n)}catch(xe){t=xe,sn===n&&n!==null&&(sn=n=n.return);continue}break}while(1)}o($T,"Sj");function jT(){var e=Fv.current;return Fv.current=Rv,e===null?Rv:e}o(jT,"Pj");function jh(e,t){var n=Ze;Ze|=16;var l=jT();yi===e&&Jn===t||Ep(e,t);do try{sI();break}catch(d){$T(e,d)}while(1);if(Y1(),Ze=n,Fv.current=l,sn!==null)throw Error(we(261));return yi=null,Jn=0,Fn}o(jh,"Tj");function sI(){for(;sn!==null;)qT(sn)}o(sI,"ak");function lI(){for(;sn!==null&&!zR();)qT(sn)}o(lI,"Rj");function qT(e){var t=YT(e.alternate,e,Tf);e.memoizedProps=e.pendingProps,t===null?VT(e):sn=t,xx.current=null}o(qT,"bk");function VT(e){var t=e;do{var n=t.alternate;if(e=t.return,(t.flags&2048)==0){if(n=QR(n,t,Tf),n!==null){sn=n;return}if(n=t,n.tag!==24&&n.tag!==23||n.memoizedState===null||(Tf&1073741824)!=0||(n.mode&4)==0){for(var l=0,d=n.child;d!==null;)l|=d.lanes|d.childLanes,d=d.sibling;n.childLanes=l}e!==null&&(e.flags&2048)==0&&(e.firstEffect===null&&(e.firstEffect=t.firstEffect),t.lastEffect!==null&&(e.lastEffect!==null&&(e.lastEffect.nextEffect=t.firstEffect),e.lastEffect=t.lastEffect),1c&&(v=c,c=se,se=v),v=g_(I,se),h=g_(I,c),v&&h&&(K.rangeCount!==1||K.anchorNode!==v.node||K.anchorOffset!==v.offset||K.focusNode!==h.node||K.focusOffset!==h.offset)&&(G=G.createRange(),G.setStart(v.node,v.offset),K.removeAllRanges(),se>c?(K.addRange(G),K.extend(h.node,h.offset)):(G.setEnd(h.node,h.offset),K.addRange(G)))))),G=[],K=I;K=K.parentNode;)K.nodeType===1&&G.push({element:K,left:K.scrollLeft,top:K.scrollTop});for(typeof I.focus=="function"&&I.focus(),I=0;IQn()-Ex?Ep(e,0):Cx|=n),ko(e,t)}o(pI,"Yj");function dI(e,t){var n=e.stateNode;n!==null&&n.delete(t),t=0,t===0&&(t=e.mode,(t&2)==0?t=1:(t&4)==0?t=mp()===99?1:2:(Dl===0&&(Dl=xp),t=op(62914560&~Dl),t===0&&(t=4194304))),n=Ji(),e=$v(e,t),e!==null&&(rv(e,t,n),ko(e,n))}o(dI,"lj");var YT;YT=o(function(e,t,n){var l=t.lanes;if(e!==null)if(e.memoizedProps!==t.pendingProps||Ri.current)ls=!0;else if((n&l)!=0)ls=(e.flags&16384)!=0;else{switch(ls=!1,t.tag){case 3:xT(t),rx();break;case 5:tT(t);break;case 1:Ii(t.type)&&yv(t);break;case 4:J1(t,t.stateNode.containerInfo);break;case 10:l=t.memoizedProps.value;var d=t.type._context;_r(Sv,d._currentValue),d._currentValue=l;break;case 13:if(t.memoizedState!==null)return(n&t.child.childLanes)!=0?ST(e,t,n):(_r(Tr,Tr.current&1),t=Ml(e,t,n),t!==null?t.sibling:null);_r(Tr,Tr.current&1);break;case 19:if(l=(n&t.childLanes)!=0,(e.flags&64)!=0){if(l)return TT(e,t,n);t.flags|=64}if(d=t.memoizedState,d!==null&&(d.rendering=null,d.tail=null,d.lastEffect=null),_r(Tr,Tr.current),l)break;return null;case 23:case 24:return t.lanes=0,fx(e,t,n)}return Ml(e,t,n)}else ls=!1;switch(t.lanes=0,t.tag){case 2:if(l=t.type,e!==null&&(e.alternate=null,t.alternate=null,t.flags|=2),e=t.pendingProps,d=hp(t,Xn.current),vp(t,n),d=ox(null,t,l,e,d,n),t.flags|=1,typeof d=="object"&&d!==null&&typeof d.render=="function"&&d.$$typeof===void 0){if(t.tag=1,t.memoizedState=null,t.updateQueue=null,Ii(l)){var h=!0;yv(t)}else h=!1;t.memoizedState=d.state!==null&&d.state!==void 0?d.state:null,Q1(t);var c=l.getDerivedStateFromProps;typeof c=="function"&&Ev(t,l,c,e),d.updater=_v,t.stateNode=d,d._reactInternals=t,Z1(t,l,e,n),t=px(null,t,l,!0,h,n)}else t.tag=0,Bi(null,t,d,n),t=t.child;return t;case 16:d=t.elementType;e:{switch(e!==null&&(e.alternate=null,t.alternate=null,t.flags|=2),e=t.pendingProps,h=d._init,d=h(d._payload),t.type=d,h=t.tag=mI(d),e=ss(d,e),h){case 0:t=cx(null,t,d,e,n);break e;case 1:t=wT(null,t,d,e,n);break e;case 11:t=mT(null,t,d,e,n);break e;case 14:t=gT(null,t,d,ss(d.type,e),l,n);break e}throw Error(we(306,d,""))}return t;case 0:return l=t.type,d=t.pendingProps,d=t.elementType===l?d:ss(l,d),cx(e,t,l,d,n);case 1:return l=t.type,d=t.pendingProps,d=t.elementType===l?d:ss(l,d),wT(e,t,l,d,n);case 3:if(xT(t),l=t.updateQueue,e===null||l===null)throw Error(we(282));if(l=t.pendingProps,d=t.memoizedState,d=d!==null?d.element:null,V_(e,t),kh(t,l,null,n),l=t.memoizedState.element,l===d)rx(),t=Ml(e,t,n);else{if(d=t.stateNode,(h=d.hydrate)&&(Qa=fp(t.stateNode.containerInfo.firstChild),Ol=t,h=qs=!0),h){if(e=d.mutableSourceEagerHydrationData,e!=null)for(d=0;d{"use strict";function JT(){if(!(typeof __REACT_DEVTOOLS_GLOBAL_HOOK__=="undefined"||typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE!="function"))try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(JT)}catch(e){console.error(e)}}o(JT,"checkDCE");JT(),ek.exports=ZT()});var rk=Ue((vH,tk)=>{"use strict";var CI="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED";tk.exports=CI});var sk=Ue((yH,ok)=>{"use strict";var bI=rk();function nk(){}o(nk,"emptyFunction");function ik(){}o(ik,"emptyFunctionWithReset");ik.resetWarningCache=nk;ok.exports=function(){function e(l,d,h,c,v,C){if(C!==bI){var k=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw k.name="Invariant Violation",k}}o(e,"shim"),e.isRequired=e;function t(){return e}o(t,"getShim");var n={array:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:t,element:e,elementType:e,instanceOf:t,node:e,objectOf:t,oneOf:t,oneOfType:t,shape:t,exact:t,checkPropTypes:ik,resetWarningCache:nk};return n.PropTypes=n,n}});var ak=Ue((SH,lk)=>{lk.exports=sk()();var wH,xH});var mk=Ue(Ht=>{"use strict";var kn=typeof Symbol=="function"&&Symbol.for,Wx=kn?Symbol.for("react.element"):60103,Ux=kn?Symbol.for("react.portal"):60106,Yv=kn?Symbol.for("react.fragment"):60107,Xv=kn?Symbol.for("react.strict_mode"):60108,Qv=kn?Symbol.for("react.profiler"):60114,Zv=kn?Symbol.for("react.provider"):60109,Jv=kn?Symbol.for("react.context"):60110,zx=kn?Symbol.for("react.async_mode"):60111,e0=kn?Symbol.for("react.concurrent_mode"):60111,t0=kn?Symbol.for("react.forward_ref"):60112,r0=kn?Symbol.for("react.suspense"):60113,kI=kn?Symbol.for("react.suspense_list"):60120,n0=kn?Symbol.for("react.memo"):60115,i0=kn?Symbol.for("react.lazy"):60116,NI=kn?Symbol.for("react.block"):60121,LI=kn?Symbol.for("react.fundamental"):60117,PI=kn?Symbol.for("react.responder"):60118,OI=kn?Symbol.for("react.scope"):60119;function eo(e){if(typeof e=="object"&&e!==null){var t=e.$$typeof;switch(t){case Wx:switch(e=e.type,e){case zx:case e0:case Yv:case Qv:case Xv:case r0:return e;default:switch(e=e&&e.$$typeof,e){case Jv:case t0:case i0:case n0:case Zv:return e;default:return t}}case Ux:return t}}}o(eo,"z");function hk(e){return eo(e)===e0}o(hk,"A");Ht.AsyncMode=zx;Ht.ConcurrentMode=e0;Ht.ContextConsumer=Jv;Ht.ContextProvider=Zv;Ht.Element=Wx;Ht.ForwardRef=t0;Ht.Fragment=Yv;Ht.Lazy=i0;Ht.Memo=n0;Ht.Portal=Ux;Ht.Profiler=Qv;Ht.StrictMode=Xv;Ht.Suspense=r0;Ht.isAsyncMode=function(e){return hk(e)||eo(e)===zx};Ht.isConcurrentMode=hk;Ht.isContextConsumer=function(e){return eo(e)===Jv};Ht.isContextProvider=function(e){return eo(e)===Zv};Ht.isElement=function(e){return typeof e=="object"&&e!==null&&e.$$typeof===Wx};Ht.isForwardRef=function(e){return eo(e)===t0};Ht.isFragment=function(e){return eo(e)===Yv};Ht.isLazy=function(e){return eo(e)===i0};Ht.isMemo=function(e){return eo(e)===n0};Ht.isPortal=function(e){return eo(e)===Ux};Ht.isProfiler=function(e){return eo(e)===Qv};Ht.isStrictMode=function(e){return eo(e)===Xv};Ht.isSuspense=function(e){return eo(e)===r0};Ht.isValidElementType=function(e){return typeof e=="string"||typeof e=="function"||e===Yv||e===e0||e===Qv||e===Xv||e===r0||e===kI||typeof e=="object"&&e!==null&&(e.$$typeof===i0||e.$$typeof===n0||e.$$typeof===Zv||e.$$typeof===Jv||e.$$typeof===t0||e.$$typeof===LI||e.$$typeof===PI||e.$$typeof===OI||e.$$typeof===NI)};Ht.typeOf=eo});var vk=Ue((IH,gk)=>{"use strict";gk.exports=mk()});var Ek=Ue((FH,bk)=>{"use strict";var $x=vk(),MI={childContextTypes:!0,contextType:!0,contextTypes:!0,defaultProps:!0,displayName:!0,getDefaultProps:!0,getDerivedStateFromError:!0,getDerivedStateFromProps:!0,mixins:!0,propTypes:!0,type:!0},AI={name:!0,length:!0,prototype:!0,caller:!0,callee:!0,arguments:!0,arity:!0},DI={$$typeof:!0,render:!0,defaultProps:!0,displayName:!0,propTypes:!0},yk={$$typeof:!0,compare:!0,defaultProps:!0,displayName:!0,propTypes:!0,type:!0},jx={};jx[$x.ForwardRef]=DI;jx[$x.Memo]=yk;function wk(e){return $x.isMemo(e)?yk:jx[e.$$typeof]||MI}o(wk,"getStatics");var RI=Object.defineProperty,II=Object.getOwnPropertyNames,xk=Object.getOwnPropertySymbols,FI=Object.getOwnPropertyDescriptor,BI=Object.getPrototypeOf,Sk=Object.prototype;function Ck(e,t,n){if(typeof t!="string"){if(Sk){var l=BI(t);l&&l!==Sk&&Ck(e,l,n)}var d=II(t);xk&&(d=d.concat(xk(t)));for(var h=wk(e),c=wk(t),v=0;v{"use strict";var Nn=typeof Symbol=="function"&&Symbol.for,qx=Nn?Symbol.for("react.element"):60103,Vx=Nn?Symbol.for("react.portal"):60106,o0=Nn?Symbol.for("react.fragment"):60107,s0=Nn?Symbol.for("react.strict_mode"):60108,l0=Nn?Symbol.for("react.profiler"):60114,a0=Nn?Symbol.for("react.provider"):60109,u0=Nn?Symbol.for("react.context"):60110,Kx=Nn?Symbol.for("react.async_mode"):60111,f0=Nn?Symbol.for("react.concurrent_mode"):60111,c0=Nn?Symbol.for("react.forward_ref"):60112,p0=Nn?Symbol.for("react.suspense"):60113,HI=Nn?Symbol.for("react.suspense_list"):60120,d0=Nn?Symbol.for("react.memo"):60115,h0=Nn?Symbol.for("react.lazy"):60116,WI=Nn?Symbol.for("react.block"):60121,UI=Nn?Symbol.for("react.fundamental"):60117,zI=Nn?Symbol.for("react.responder"):60118,$I=Nn?Symbol.for("react.scope"):60119;function to(e){if(typeof e=="object"&&e!==null){var t=e.$$typeof;switch(t){case qx:switch(e=e.type,e){case Kx:case f0:case o0:case l0:case s0:case p0:return e;default:switch(e=e&&e.$$typeof,e){case u0:case c0:case h0:case d0:case a0:return e;default:return t}}case Vx:return t}}}o(to,"z");function _k(e){return to(e)===f0}o(_k,"A");Wt.AsyncMode=Kx;Wt.ConcurrentMode=f0;Wt.ContextConsumer=u0;Wt.ContextProvider=a0;Wt.Element=qx;Wt.ForwardRef=c0;Wt.Fragment=o0;Wt.Lazy=h0;Wt.Memo=d0;Wt.Portal=Vx;Wt.Profiler=l0;Wt.StrictMode=s0;Wt.Suspense=p0;Wt.isAsyncMode=function(e){return _k(e)||to(e)===Kx};Wt.isConcurrentMode=_k;Wt.isContextConsumer=function(e){return to(e)===u0};Wt.isContextProvider=function(e){return to(e)===a0};Wt.isElement=function(e){return typeof e=="object"&&e!==null&&e.$$typeof===qx};Wt.isForwardRef=function(e){return to(e)===c0};Wt.isFragment=function(e){return to(e)===o0};Wt.isLazy=function(e){return to(e)===h0};Wt.isMemo=function(e){return to(e)===d0};Wt.isPortal=function(e){return to(e)===Vx};Wt.isProfiler=function(e){return to(e)===l0};Wt.isStrictMode=function(e){return to(e)===s0};Wt.isSuspense=function(e){return to(e)===p0};Wt.isValidElementType=function(e){return typeof e=="string"||typeof e=="function"||e===o0||e===f0||e===l0||e===s0||e===p0||e===HI||typeof e=="object"&&e!==null&&(e.$$typeof===h0||e.$$typeof===d0||e.$$typeof===a0||e.$$typeof===u0||e.$$typeof===c0||e.$$typeof===UI||e.$$typeof===zI||e.$$typeof===$I||e.$$typeof===WI)};Wt.typeOf=to});var Nk=Ue((HH,kk)=>{"use strict";kk.exports=Tk()});var Qh=Ue((Np,Xh)=>{(function(){var e,t="4.17.21",n=200,l="Unsupported core-js use. Try https://npms.io/search?q=ponyfill.",d="Expected a function",h="Invalid `variable` option passed into `_.template`",c="__lodash_hash_undefined__",v=500,C="__lodash_placeholder__",k=1,O=2,j=4,B=1,X=2,J=1,Z=2,R=4,A=8,I=16,G=32,K=64,se=128,ne=256,pe=512,me=30,xe="...",Ve=800,tt=16,_e=1,St=2,We=3,Ke=1/0,Ge=9007199254740991,Xe=17976931348623157e292,nr=0/0,ct=4294967295,Hr=ct-1,Qt=ct>>>1,_t=[["ary",se],["bind",J],["bindKey",Z],["curry",A],["curryRight",I],["flip",pe],["partial",G],["partialRight",K],["rearg",ne]],Ct="[object Arguments]",ut="[object Array]",Lr="[object AsyncFunction]",zt="[object Boolean]",$t="[object Date]",ie="[object DOMException]",rt="[object Error]",Pr="[object Function]",Gt="[object GeneratorFunction]",Yt="[object Map]",Se="[object Number]",Or="[object Null]",fn="[object Object]",Un="[object Promise]",si="[object Proxy]",cn="[object RegExp]",Zt="[object Set]",gr="[object String]",pt="[object Symbol]",Ho="[object Undefined]",Cr="[object WeakMap]",Ui="[object WeakSet]",pn="[object ArrayBuffer]",zn="[object DataView]",Si="[object Float32Array]",Ci="[object Float64Array]",$n="[object Int8Array]",Mn="[object Int16Array]",Js="[object Int32Array]",H="[object Uint8Array]",ee="[object Uint8ClampedArray]",he="[object Uint16Array]",Te="[object Uint32Array]",ir=/\b__p \+= '';/g,Ul=/\b(__p \+=) '' \+/g,Ft=/(__e\(.*?\)|\b__t\)) \+\n'';/g,Wr=/&(?:amp|lt|gt|quot|#39);/g,or=/[&<>"']/g,li=RegExp(Wr.source),ds=RegExp(or.source),lo=/<%-([\s\S]+?)%>/g,bi=/<%([\s\S]+?)%>/g,el=/<%=([\s\S]+?)%>/g,hs=/\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/,dn=/^\w*$/,id=/[^.[\]]+|\[(?:(-?\d+(?:\.\d+)?)|(["'])((?:(?!\2)[^\\]|\\.)*?)\2)\]|(?=(?:\.|\[\])(?:\.|\[\]|$))/g,tl=/[\\^$.*+?()[\]{}|]/g,Qf=RegExp(tl.source),rl=/^\s+/,od=/\s/,Zf=/\{(?:\n\/\* \[wrapped with .+\] \*\/)?\n?/,wu=/\{\n\/\* \[wrapped with (.+)\] \*/,sd=/,? & /,zl=/[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g,ms=/[()=,{}\[\]\/\s]/,ld=/\\(\\)?/g,Jf=/\$\{([^\\}]*(?:\\.[^\\}]*)*)\}/g,nl=/\w*$/,xu=/^[-+]0x[0-9a-f]+$/i,Wo=/^0b[01]+$/i,ad=/^\[object .+?Constructor\]$/,Uo=/^0o[0-7]+$/i,$l=/^(?:0|[1-9]\d*)$/,ec=/[\xc0-\xd6\xd8-\xf6\xf8-\xff\u0100-\u017f]/g,jt=/($^)/,Me=/['\n\r\u2028\u2029\\]/g,Ei="\\ud800-\\udfff",Su="\\u0300-\\u036f",ai="\\ufe20-\\ufe2f",vt="\\u20d0-\\u20ff",ao=Su+ai+vt,zo="\\u2700-\\u27bf",jl="a-z\\xdf-\\xf6\\xf8-\\xff",ue="\\xac\\xb1\\xd7\\xf7",ze="\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf",Cu="\\u2000-\\u206f",bu=" \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000",gs="A-Z\\xc0-\\xd6\\xd8-\\xde",il="\\ufe0e\\ufe0f",Eu=ue+ze+Cu+bu,He="['\u2019]",ud="["+Ei+"]",ql="["+Eu+"]",uo="["+ao+"]",ui="\\d+",tc="["+zo+"]",_u="["+jl+"]",$o="[^"+Ei+Eu+ui+zo+jl+gs+"]",vs="\\ud83c[\\udffb-\\udfff]",Tu="(?:"+uo+"|"+vs+")",ol="[^"+Ei+"]",Vl="(?:\\ud83c[\\udde6-\\uddff]){2}",Kl="[\\ud800-\\udbff][\\udc00-\\udfff]",fo="["+gs+"]",Gl="\\u200d",Yl="(?:"+_u+"|"+$o+")",rc="(?:"+fo+"|"+$o+")",Xl="(?:"+He+"(?:d|ll|m|re|s|t|ve))?",_i="(?:"+He+"(?:D|LL|M|RE|S|T|VE))?",nc=Tu+"?",Ql="["+il+"]?",co="(?:"+Gl+"(?:"+[ol,Vl,Kl].join("|")+")"+Ql+nc+")*",ys="\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])",ic="\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])",oc=Ql+nc+co,fd="(?:"+[tc,Vl,Kl].join("|")+")"+oc,cd="(?:"+[ol+uo+"?",uo,Vl,Kl,ud].join("|")+")",ku=RegExp(He,"g"),sc=RegExp(uo,"g"),Nu=RegExp(vs+"(?="+vs+")|"+cd+oc,"g"),lc=RegExp([fo+"?"+_u+"+"+Xl+"(?="+[ql,fo,"$"].join("|")+")",rc+"+"+_i+"(?="+[ql,fo+Yl,"$"].join("|")+")",fo+"?"+Yl+"+"+Xl,fo+"+"+_i,ic,ys,ui,fd].join("|"),"g"),ac=RegExp("["+Gl+Ei+ao+il+"]"),Zl=/[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/,Jl=["Array","Buffer","DataView","Date","Error","Float32Array","Float64Array","Function","Int8Array","Int16Array","Int32Array","Map","Math","Object","Promise","RegExp","Set","String","Symbol","TypeError","Uint8Array","Uint8ClampedArray","Uint16Array","Uint32Array","WeakMap","_","clearTimeout","isFinite","parseInt","setTimeout"],Lu=-1,Ot={};Ot[Si]=Ot[Ci]=Ot[$n]=Ot[Mn]=Ot[Js]=Ot[H]=Ot[ee]=Ot[he]=Ot[Te]=!0,Ot[Ct]=Ot[ut]=Ot[pn]=Ot[zt]=Ot[zn]=Ot[$t]=Ot[rt]=Ot[Pr]=Ot[Yt]=Ot[Se]=Ot[fn]=Ot[cn]=Ot[Zt]=Ot[gr]=Ot[Cr]=!1;var Nt={};Nt[Ct]=Nt[ut]=Nt[pn]=Nt[zn]=Nt[zt]=Nt[$t]=Nt[Si]=Nt[Ci]=Nt[$n]=Nt[Mn]=Nt[Js]=Nt[Yt]=Nt[Se]=Nt[fn]=Nt[cn]=Nt[Zt]=Nt[gr]=Nt[pt]=Nt[H]=Nt[ee]=Nt[he]=Nt[Te]=!0,Nt[rt]=Nt[Pr]=Nt[Cr]=!1;var P={\u00C0:"A",\u00C1:"A",\u00C2:"A",\u00C3:"A",\u00C4:"A",\u00C5:"A",\u00E0:"a",\u00E1:"a",\u00E2:"a",\u00E3:"a",\u00E4:"a",\u00E5:"a",\u00C7:"C",\u00E7:"c",\u00D0:"D",\u00F0:"d",\u00C8:"E",\u00C9:"E",\u00CA:"E",\u00CB:"E",\u00E8:"e",\u00E9:"e",\u00EA:"e",\u00EB:"e",\u00CC:"I",\u00CD:"I",\u00CE:"I",\u00CF:"I",\u00EC:"i",\u00ED:"i",\u00EE:"i",\u00EF:"i",\u00D1:"N",\u00F1:"n",\u00D2:"O",\u00D3:"O",\u00D4:"O",\u00D5:"O",\u00D6:"O",\u00D8:"O",\u00F2:"o",\u00F3:"o",\u00F4:"o",\u00F5:"o",\u00F6:"o",\u00F8:"o",\u00D9:"U",\u00DA:"U",\u00DB:"U",\u00DC:"U",\u00F9:"u",\u00FA:"u",\u00FB:"u",\u00FC:"u",\u00DD:"Y",\u00FD:"y",\u00FF:"y",\u00C6:"Ae",\u00E6:"ae",\u00DE:"Th",\u00FE:"th",\u00DF:"ss",\u0100:"A",\u0102:"A",\u0104:"A",\u0101:"a",\u0103:"a",\u0105:"a",\u0106:"C",\u0108:"C",\u010A:"C",\u010C:"C",\u0107:"c",\u0109:"c",\u010B:"c",\u010D:"c",\u010E:"D",\u0110:"D",\u010F:"d",\u0111:"d",\u0112:"E",\u0114:"E",\u0116:"E",\u0118:"E",\u011A:"E",\u0113:"e",\u0115:"e",\u0117:"e",\u0119:"e",\u011B:"e",\u011C:"G",\u011E:"G",\u0120:"G",\u0122:"G",\u011D:"g",\u011F:"g",\u0121:"g",\u0123:"g",\u0124:"H",\u0126:"H",\u0125:"h",\u0127:"h",\u0128:"I",\u012A:"I",\u012C:"I",\u012E:"I",\u0130:"I",\u0129:"i",\u012B:"i",\u012D:"i",\u012F:"i",\u0131:"i",\u0134:"J",\u0135:"j",\u0136:"K",\u0137:"k",\u0138:"k",\u0139:"L",\u013B:"L",\u013D:"L",\u013F:"L",\u0141:"L",\u013A:"l",\u013C:"l",\u013E:"l",\u0140:"l",\u0142:"l",\u0143:"N",\u0145:"N",\u0147:"N",\u014A:"N",\u0144:"n",\u0146:"n",\u0148:"n",\u014B:"n",\u014C:"O",\u014E:"O",\u0150:"O",\u014D:"o",\u014F:"o",\u0151:"o",\u0154:"R",\u0156:"R",\u0158:"R",\u0155:"r",\u0157:"r",\u0159:"r",\u015A:"S",\u015C:"S",\u015E:"S",\u0160:"S",\u015B:"s",\u015D:"s",\u015F:"s",\u0161:"s",\u0162:"T",\u0164:"T",\u0166:"T",\u0163:"t",\u0165:"t",\u0167:"t",\u0168:"U",\u016A:"U",\u016C:"U",\u016E:"U",\u0170:"U",\u0172:"U",\u0169:"u",\u016B:"u",\u016D:"u",\u016F:"u",\u0171:"u",\u0173:"u",\u0174:"W",\u0175:"w",\u0176:"Y",\u0177:"y",\u0178:"Y",\u0179:"Z",\u017B:"Z",\u017D:"Z",\u017A:"z",\u017C:"z",\u017E:"z",\u0132:"IJ",\u0133:"ij",\u0152:"Oe",\u0153:"oe",\u0149:"'n",\u017F:"s"},Re={"&":"&","<":"<",">":">",'"':""","'":"'"},sl={"&":"&","<":"<",">":">",""":'"',"'":"'"},vr={"\\":"\\","'":"'","\n":"n","\r":"r","\u2028":"u2028","\u2029":"u2029"},Pu=parseFloat,ye=parseInt,jo=typeof global=="object"&&global&&global.Object===Object&&global,pd=typeof self=="object"&&self&&self.Object===Object&&self,qt=jo||pd||Function("return this")(),ea=typeof Np=="object"&&Np&&!Np.nodeType&&Np,hn=ea&&typeof Xh=="object"&&Xh&&!Xh.nodeType&&Xh,ws=hn&&hn.exports===ea,zi=ws&&jo.process,Ce=function(){try{var q=hn&&hn.require&&hn.require("util").types;return q||zi&&zi.binding&&zi.binding("util")}catch(re){}}(),ta=Ce&&Ce.isArrayBuffer,Ou=Ce&&Ce.isDate,nt=Ce&&Ce.isMap,uc=Ce&&Ce.isRegExp,fi=Ce&&Ce.isSet,ll=Ce&&Ce.isTypedArray;function Ur(q,re,Q){switch(Q.length){case 0:return q.call(re);case 1:return q.call(re,Q[0]);case 2:return q.call(re,Q[0],Q[1]);case 3:return q.call(re,Q[0],Q[1],Q[2])}return q.apply(re,Q)}o(Ur,"apply");function ra(q,re,Q,Pe){for(var Qe=-1,bt=q==null?0:q.length;++Qe-1}o(xs,"arrayIncludes");function qo(q,re,Q){for(var Pe=-1,Qe=q==null?0:q.length;++Pe-1;);return Q}o(qi,"charsStartIndex");function al(q,re){for(var Q=q.length;Q--&&Go(re,q[Q],0)>-1;);return Q}o(al,"charsEndIndex");function cc(q,re){for(var Q=q.length,Pe=0;Q--;)q[Q]===re&&++Pe;return Pe}o(cc,"countHolders");var Wu=oa(P),gd=oa(Re);function Uu(q){return"\\"+vr[q]}o(Uu,"escapeStringChar");function la(q,re){return q==null?e:q[re]}o(la,"getValue");function qn(q){return ac.test(q)}o(qn,"hasUnicode");function Ti(q){return Zl.test(q)}o(Ti,"hasUnicodeWord");function pc(q){for(var re,Q=[];!(re=q.next()).done;)Q.push(re.value);return Q}o(pc,"iteratorToArray");function aa(q){var re=-1,Q=Array(q.size);return q.forEach(function(Pe,Qe){Q[++re]=[Qe,Pe]}),Q}o(aa,"mapToArray");function dc(q,re){return function(Q){return q(re(Q))}}o(dc,"overArg");function Vi(q,re){for(var Q=-1,Pe=q.length,Qe=0,bt=[];++Q-1}o(Vy,"listCacheHas");function yc(s,f){var m=this.__data__,x=gl(m,s);return x<0?(++this.size,m.push([s,f])):m[x][1]=f,this}o(yc,"listCacheSet"),vo.prototype.clear=Cd,vo.prototype.delete=$m,vo.prototype.get=Yu,vo.prototype.has=Vy,vo.prototype.set=yc;function xr(s){var f=-1,m=s==null?0:s.length;for(this.clear();++f=f?s:f)),s}o(vl,"baseClamp");function Dn(s,f,m,x,_,L){var F,z=f&k,Y=f&O,le=f&j;if(m&&(F=_?m(s,x,_,L):m(s)),F!==e)return F;if(!Sr(s))return s;var ae=ot(s);if(ae){if(F=u(s),!z)return qr(s,F)}else{var ce=xn(s),Le=ce==Pr||ce==Gt;if(Ra(s))return Hd(s,z);if(ce==fn||ce==Ct||Le&&!_){if(F=Y||Le?{}:a(s),!z)return Y?dg(s,Yy(F,s)):nw(s,kd(F,s))}else{if(!Nt[ce])return _?s:{};F=p(s,ce,z)}}L||(L=new Ni);var Fe=L.get(s);if(Fe)return Fe;L.set(s,F),Cb(s)?s.forEach(function(je){F.add(Dn(je,f,m,je,s,L))}):xb(s)&&s.forEach(function(je,ht){F.set(ht,Dn(je,f,m,ht,s,L))});var $e=le?Y?Hc:ff:Y?Ai:_n,ft=ae?e:$e(s);return jn(ft||s,function(je,ht){ft&&(ht=je,je=s[ht]),Qu(F,ht,Dn(je,f,m,ht,s,L))}),F}o(Dn,"baseClone");function Gm(s){var f=_n(s);return function(m){return Ym(m,s,f)}}o(Gm,"baseConforms");function Ym(s,f,m){var x=m.length;if(s==null)return!x;for(s=mt(s);x--;){var _=m[x],L=f[_],F=s[_];if(F===e&&!(_ in s)||!L(F))return!1}return!0}o(Ym,"baseConformsTo");function Xm(s,f,m){if(typeof s!="function")throw new Kn(d);return At(function(){s.apply(e,m)},f)}o(Xm,"baseDelay");function va(s,f,m,x){var _=-1,L=xs,F=!0,z=s.length,Y=[],le=f.length;if(!z)return Y;m&&(f=yt(f,zr(m))),x?(L=qo,F=!1):f.length>=n&&(L=mn,F=!1,f=new tn(f));e:for(;++__?0:_+m),x=x===e||x>_?_:lt(x),x<0&&(x+=_),x=m>x?0:Eb(x);m0&&m(z)?f>1?rn(z,f-1,m,x,_):$i(_,z):x||(_[_.length]=z)}return _}o(rn,"baseFlatten");var bc=mg(),$r=mg(!0);function hi(s,f){return s&&bc(s,f,_n)}o(hi,"baseForOwn");function Ec(s,f){return s&&$r(s,f,_n)}o(Ec,"baseForOwnRight");function Ju(s,f){return Vt(f,function(m){return _l(s[m])})}o(Ju,"baseFunctions");function Ds(s,f){f=Gi(f,s);for(var m=0,x=f.length;s!=null&&mf}o(_c,"baseGt");function Qm(s,f){return s!=null&&et.call(s,f)}o(Qm,"baseHas");function Zm(s,f){return s!=null&&f in mt(s)}o(Zm,"baseHasIn");function ya(s,f,m){return s>=Zr(f,m)&&s=120&&ae.length>=120)?new tn(F&&ae):e}ae=s[0];var ce=-1,Le=z[0];e:for(;++ce<_&&le.length-1;)z!==s&&pa.call(z,Y,1),pa.call(s,Y,1);return s}o(Md,"basePullAll");function Ad(s,f){for(var m=s?f.length:0,x=m-1;m--;){var _=f[m];if(m==x||_!==L){var L=_;S(_)?pa.call(s,_,1):of(s,_)}}return s}o(Ad,"basePullAt");function Pc(s,f){return s+ha(Ns()*(f-s+1))}o(Pc,"baseRandom");function lg(s,f,m,x){for(var _=-1,L=sr(da((f-s)/(m||1)),0),F=Q(L);L--;)F[x?L:++_]=s,s+=m;return F}o(lg,"baseRange");function Dd(s,f){var m="";if(!s||f<1||f>Ge)return m;do f%2&&(m+=s),f=ha(f/2),f&&(s+=s);while(f);return m}o(Dd,"baseRepeat");function st(s,f){return Sn(Ie(s,f,Di),s+"")}o(st,"baseRest");function Jy(s){return wc(jc(s))}o(Jy,"baseSample");function Fs(s,f){var m=jc(s);return Vr(m,vl(f,0,m.length))}o(Fs,"baseSampleSize");function Zo(s,f,m,x){if(!Sr(s))return s;f=Gi(f,s);for(var _=-1,L=f.length,F=L-1,z=s;z!=null&&++__?0:_+f),m=m>_?_:m,m<0&&(m+=_),_=f>m?0:m-f>>>0,f>>>=0;for(var L=Q(_);++x<_;)L[x]=s[x+f];return L}o(Li,"baseSlice");function ew(s,f){var m;return di(s,function(x,_,L){return m=f(x,_,L),!m}),!!m}o(ew,"baseSome");function es(s,f,m){var x=0,_=s==null?x:s.length;if(typeof f=="number"&&f===f&&_<=Qt){for(;x<_;){var L=x+_>>>1,F=s[L];F!==null&&!Yi(F)&&(m?F<=f:F=n){var le=f?null:uf(s);if(le)return w(le);F=!1,_=mn,Y=new tn}else Y=f?[]:z;e:for(;++x=x?s:Li(s,f,m)}o(Bs,"castSlice");var Ta=Ry||function(s){return qt.clearTimeout(s)};function Hd(s,f){if(f)return s.slice();var m=s.length,x=Hm?Hm(m):new s.constructor(m);return s.copy(x),x}o(Hd,"cloneBuffer");function Dc(s){var f=new s.constructor(s.byteLength);return new cl(f).set(new cl(s)),f}o(Dc,"cloneArrayBuffer");function rw(s,f){var m=f?Dc(s.buffer):s.buffer;return new s.constructor(m,s.byteOffset,s.byteLength)}o(rw,"cloneDataView");function Wd(s){var f=new s.constructor(s.source,nl.exec(s));return f.lastIndex=s.lastIndex,f}o(Wd,"cloneRegExp");function ug(s){return lr?mt(lr.call(s)):{}}o(ug,"cloneSymbol");function fg(s,f){var m=f?Dc(s.buffer):s.buffer;return new s.constructor(m,s.byteOffset,s.length)}o(fg,"cloneTypedArray");function Ud(s,f){if(s!==f){var m=s!==e,x=s===null,_=s===s,L=Yi(s),F=f!==e,z=f===null,Y=f===f,le=Yi(f);if(!z&&!le&&!L&&s>f||L&&F&&Y&&!z&&!le||x&&F&&Y||!m&&Y||!_)return 1;if(!x&&!L&&!le&&s=z)return Y;var le=m[x];return Y*(le=="desc"?-1:1)}}return s.index-f.index}o(cg,"compareMultiple");function pg(s,f,m,x){for(var _=-1,L=s.length,F=m.length,z=-1,Y=f.length,le=sr(L-F,0),ae=Q(Y+le),ce=!x;++z1?m[_-1]:e,F=_>2?m[2]:e;for(L=s.length>3&&typeof L=="function"?(_--,L):e,F&&b(m[0],m[1],F)&&(L=_<3?e:L,_=1),f=mt(f);++x<_;){var z=m[x];z&&s(f,z,x,L)}return f})}o(ka,"createAssigner");function hg(s,f){return function(m,x){if(m==null)return m;if(!Mi(m))return s(m,x);for(var _=m.length,L=f?_:-1,F=mt(m);(f?L--:++L<_)&&x(F[L],L,F)!==!1;);return m}}o(hg,"createBaseEach");function mg(s){return function(f,m,x){for(var _=-1,L=mt(f),F=x(f),z=F.length;z--;){var Y=F[s?z:++_];if(m(L[Y],Y,L)===!1)break}return f}}o(mg,"createBaseFor");function gg(s,f,m){var x=f&J,_=La(s);function L(){var F=this&&this!==qt&&this instanceof L?_:s;return F.apply(x?m:this,arguments)}return o(L,"wrapper"),L}o(gg,"createBind");function vg(s){return function(f){f=Dt(f);var m=qn(f)?Kt(f):e,x=m?m[0]:f.charAt(0),_=m?Bs(m,1).join(""):f.slice(1);return x[s]()+_}}o(vg,"createCaseFirst");function Na(s){return function(f){return Au(Ab(Mb(f).replace(ku,"")),s,"")}}o(Na,"createCompounder");function La(s){return function(){var f=arguments;switch(f.length){case 0:return new s;case 1:return new s(f[0]);case 2:return new s(f[0],f[1]);case 3:return new s(f[0],f[1],f[2]);case 4:return new s(f[0],f[1],f[2],f[3]);case 5:return new s(f[0],f[1],f[2],f[3],f[4]);case 6:return new s(f[0],f[1],f[2],f[3],f[4],f[5]);case 7:return new s(f[0],f[1],f[2],f[3],f[4],f[5],f[6])}var m=go(s.prototype),x=s.apply(m,f);return Sr(x)?x:m}}o(La,"createCtor");function zd(s,f,m){var x=La(s);function _(){for(var L=arguments.length,F=Q(L),z=L,Y=Oa(_);z--;)F[z]=arguments[z];var le=L<3&&F[0]!==Y&&F[L-1]!==Y?[]:Vi(F,Y);if(L-=le.length,L-1?_[L?f[F]:F]:e}}o($d,"createFind");function yg(s){return ts(function(f){var m=f.length,x=m,_=An.prototype.thru;for(s&&f.reverse();x--;){var L=f[x];if(typeof L!="function")throw new Kn(d);if(_&&!F&&cf(L)=="wrapper")var F=new An([],!0)}for(x=F?x:m;++x1&>.reverse(),ae&&Yz))return!1;var le=L.get(s),ae=L.get(f);if(le&&ae)return le==f&&ae==s;var ce=-1,Le=!0,Fe=m&X?new tn:e;for(L.set(s,f),L.set(f,s);++ce1?"& ":"")+f[x],f=f.join(m>2?", ":" "),s.replace(Zf,`{ /* [wrapped with `+f+`] */ -`)}o(g,"insertWrapDetails");function y(s){return ot(s)||pf(s)||!!(pl&&s&&s[pl])}o(y,"isFlattenable");function S(s,f){var m=typeof s;return f=f??Ge,!!f&&(m=="number"||m!="symbol"&&$l.test(s))&&s>-1&&s%1==0&&s0){if(++f>=Ve)return arguments[0]}else f=0;return s.apply(e,arguments)}}o(dr,"shortOut");function Vr(s,f){var m=-1,x=s.length,_=x-1;for(f=f===e?x:f;++m1?s[f-1]:e;return m=typeof m=="function"?(s.pop(),m):e,ab(s,m)});function ub(s){var f=N(s);return f.__chain__=!0,f}o(ub,"chain");function JO(s,f){return f(s),s}o(JO,"tap");function _g(s,f){return f(s)}o(_g,"thru");var eM=ts(function(s){var f=s.length,m=f?s[0]:0,x=this.__wrapped__,_=o(function(L){return Nd(L,s)},"interceptor");return f>1||this.__actions__.length||!(x instanceof dt)||!S(m)?this.thru(_):(x=x.slice(m,+m+(f?1:0)),x.__actions__.push({func:_g,args:[_],thisArg:e}),new An(x,this.__chain__).thru(function(L){return f&&!L.length&&L.push(e),L}))});function tM(){return ub(this)}o(tM,"wrapperChain");function rM(){return new An(this.value(),this.__chain__)}o(rM,"wrapperCommit");function nM(){this.__values__===e&&(this.__values__=bb(this.value()));var s=this.__index__>=this.__values__.length,f=s?e:this.__values__[this.__index__++];return{done:s,value:f}}o(nM,"wrapperNext");function iM(){return this}o(iM,"wrapperToIterator");function oM(s){for(var f,m=this;m instanceof vc;){var x=bn(m);x.__index__=0,x.__values__=e,f?_.__wrapped__=x:f=x;var _=x;m=m.__wrapped__}return _.__wrapped__=s,f}o(oM,"wrapperPlant");function sM(){var s=this.__wrapped__;if(s instanceof dt){var f=s;return this.__actions__.length&&(f=new dt(this)),f=f.reverse(),f.__actions__.push({func:_g,args:[lw],thisArg:e}),new An(f,this.__chain__)}return this.thru(lw)}o(sM,"wrapperReverse");function lM(){return ag(this.__wrapped__,this.__actions__)}o(lM,"wrapperValue");var aM=Ic(function(s,f,m){et.call(s,m)?++s[m]:yo(s,m,1)});function uM(s,f,m){var x=ot(s)?Mu:Cc;return m&&b(s,f,m)&&(f=e),x(s,Be(f,3))}o(uM,"every");function fM(s,f){var m=ot(s)?Vt:Pd;return m(s,Be(f,3))}o(fM,"filter");var cM=$d(zc),pM=$d(ib);function dM(s,f){return rn(Tg(s,f),1)}o(dM,"flatMap");function hM(s,f){return rn(Tg(s,f),Ke)}o(hM,"flatMapDeep");function mM(s,f,m){return m=m===e?1:lt(m),rn(Tg(s,f),m)}o(mM,"flatMapDepth");function fb(s,f){var m=ot(s)?jn:di;return m(s,Be(f,3))}o(fb,"forEach");function cb(s,f){var m=ot(s)?dd:Sc;return m(s,Be(f,3))}o(cb,"forEachRight");var gM=Ic(function(s,f,m){et.call(s,m)?s[m].push(f):yo(s,m,[f])});function vM(s,f,m,x){s=Mi(s)?s:jc(s),m=m&&!x?lt(m):0;var _=s.length;return m<0&&(m=sr(_+m,0)),Og(s)?m<=_&&s.indexOf(f,m)>-1:!!_&&Go(s,f,m)>-1}o(vM,"includes");var yM=st(function(s,f,m){var x=-1,_=typeof f=="function",L=Mi(s)?Q(s.length):[];return di(s,function(F){L[++x]=_?Ur(f,F,m):wa(F,f,m)}),L}),wM=Ic(function(s,f,m){yo(s,m,f)});function Tg(s,f){var m=ot(s)?yt:Ea;return m(s,Be(f,3))}o(Tg,"map");function xM(s,f,m,x){return s==null?[]:(ot(f)||(f=f==null?[]:[f]),m=x?e:m,ot(m)||(m=m==null?[]:[m]),yn(s,f,m))}o(xM,"orderBy");var SM=Ic(function(s,f,m){s[m?0:1].push(f)},function(){return[[],[]]});function CM(s,f,m){var x=ot(s)?Au:Fu,_=arguments.length<3;return x(s,Be(f,4),m,_,di)}o(CM,"reduce");function bM(s,f,m){var x=ot(s)?hd:Fu,_=arguments.length<3;return x(s,Be(f,4),m,_,Sc)}o(bM,"reduceRight");function EM(s,f){var m=ot(s)?Vt:Pd;return m(s,Lg(Be(f,3)))}o(EM,"reject");function _M(s){var f=ot(s)?wc:tw;return f(s)}o(_M,"sample");function TM(s,f,m){(m?b(s,f,m):f===e)?f=1:f=lt(f);var x=ot(s)?As:Fs;return x(s,f)}o(TM,"sampleSize");function kM(s){var f=ot(s)?Km:Jo;return f(s)}o(kM,"shuffle");function NM(s){if(s==null)return 0;if(Mi(s))return Og(s)?wr(s):s.length;var f=xn(s);return f==Yt||f==Zt?s.size:Nc(s).length}o(NM,"size");function LM(s,f,m){var x=ot(s)?Vo:rw;return m&&b(s,f,m)&&(f=e),x(s,Be(f,3))}o(LM,"some");var PM=st(function(s,f){if(s==null)return[];var m=f.length;return m>1&&b(s,f[0],f[1])?f=[]:m>2&&b(f[0],f[1],f[2])&&(f=[f[0]]),yn(s,rn(f,1),[])}),kg=By||function(){return qt.Date.now()};function OM(s,f){if(typeof f!="function")throw new Kn(d);return s=lt(s),function(){if(--s<1)return f.apply(this,arguments)}}o(OM,"after");function pb(s,f,m){return f=m?e:f,f=s&&f==null?s.length:f,Oi(s,se,e,e,e,e,f)}o(pb,"ary");function db(s,f){var m;if(typeof f!="function")throw new Kn(d);return s=lt(s),function(){return--s>0&&(m=f.apply(this,arguments)),s<=1&&(f=e),m}}o(db,"before");var uw=st(function(s,f,m){var x=J;if(m.length){var _=Vi(m,Oa(uw));x|=G}return Oi(s,x,f,m,_)}),hb=st(function(s,f,m){var x=J|Z;if(m.length){var _=Vi(m,Oa(hb));x|=G}return Oi(f,x,s,m,_)});function mb(s,f,m){f=m?e:f;var x=Oi(s,A,e,e,e,e,e,f);return x.placeholder=mb.placeholder,x}o(mb,"curry");function gb(s,f,m){f=m?e:f;var x=Oi(s,I,e,e,e,e,e,f);return x.placeholder=gb.placeholder,x}o(gb,"curryRight");function vb(s,f,m){var x,_,L,F,z,Y,le=0,ae=!1,ce=!1,Le=!0;if(typeof s!="function")throw new Kn(d);f=So(f)||0,Sr(m)&&(ae=!!m.leading,ce="maxWait"in m,L=ce?sr(So(m.maxWait)||0,f):L,Le="trailing"in m?!!m.trailing:Le);function Fe(Ir){var is=x,kl=_;return x=_=e,le=Ir,F=s.apply(kl,is),F}o(Fe,"invokeFunc");function $e(Ir){return le=Ir,z=At(ht,f),ae?Fe(Ir):F}o($e,"leadingEdge");function ft(Ir){var is=Ir-Y,kl=Ir-le,Ib=f-is;return ce?Zr(Ib,L-kl):Ib}o(ft,"remainingWait");function je(Ir){var is=Ir-Y,kl=Ir-le;return Y===e||is>=f||is<0||ce&&kl>=L}o(je,"shouldInvoke");function ht(){var Ir=kg();if(je(Ir))return gt(Ir);z=At(ht,ft(Ir))}o(ht,"timerExpired");function gt(Ir){return z=e,Le&&x?Fe(Ir):(x=_=e,F)}o(gt,"trailingEdge");function Xi(){z!==e&&Ta(z),le=0,x=Y=_=z=e}o(Xi,"cancel");function gi(){return z===e?F:gt(kg())}o(gi,"flush");function Qi(){var Ir=kg(),is=je(Ir);if(x=arguments,_=this,Y=Ir,is){if(z===e)return $e(Y);if(ce)return Ta(z),z=At(ht,f),Fe(Y)}return z===e&&(z=At(ht,f)),F}return o(Qi,"debounced"),Qi.cancel=Xi,Qi.flush=gi,Qi}o(vb,"debounce");var MM=st(function(s,f){return Xm(s,1,f)}),AM=st(function(s,f,m){return Xm(s,So(f)||0,m)});function DM(s){return Oi(s,pe)}o(DM,"flip");function Ng(s,f){if(typeof s!="function"||f!=null&&typeof f!="function")throw new Kn(d);var m=o(function(){var x=arguments,_=f?f.apply(this,x):x[0],L=m.cache;if(L.has(_))return L.get(_);var F=s.apply(this,x);return m.cache=L.set(_,F)||L,F},"memoized");return m.cache=new(Ng.Cache||xr),m}o(Ng,"memoize"),Ng.Cache=xr;function Lg(s){if(typeof s!="function")throw new Kn(d);return function(){var f=arguments;switch(f.length){case 0:return!s.call(this);case 1:return!s.call(this,f[0]);case 2:return!s.call(this,f[0],f[1]);case 3:return!s.call(this,f[0],f[1],f[2])}return!s.apply(this,f)}}o(Lg,"negate");function RM(s){return db(2,s)}o(RM,"once");var IM=nw(function(s,f){f=f.length==1&&ot(f[0])?yt(f[0],zr(Be())):yt(rn(f,1),zr(Be()));var m=f.length;return st(function(x){for(var _=-1,L=Zr(x.length,m);++_=f}),pf=xa(function(){return arguments}())?xa:function(s){return Er(s)&&et.call(s,"callee")&&!hc.call(s,"callee")},ot=Q.isArray,QM=ta?zr(ta):Zy;function Mi(s){return s!=null&&Pg(s.length)&&!_l(s)}o(Mi,"isArrayLike");function Rr(s){return Er(s)&&Mi(s)}o(Rr,"isArrayLikeObject");function ZM(s){return s===!0||s===!1||Er(s)&&jr(s)==zt}o(ZM,"isBoolean");var Ra=qu||Sw,JM=Ou?zr(Ou):Sa;function eA(s){return Er(s)&&s.nodeType===1&&!Gd(s)}o(eA,"isElement");function tA(s){if(s==null)return!0;if(Mi(s)&&(ot(s)||typeof s=="string"||typeof s.splice=="function"||Ra(s)||$c(s)||pf(s)))return!s.length;var f=xn(s);if(f==Yt||f==Zt)return!s.size;if(te(s))return!Nc(s).length;for(var m in s)if(et.call(s,m))return!1;return!0}o(tA,"isEmpty");function rA(s,f){return Ca(s,f)}o(rA,"isEqual");function nA(s,f,m){m=typeof m=="function"?m:e;var x=m?m(s,f):e;return x===e?Ca(s,f,e,m):!!x}o(nA,"isEqualWith");function cw(s){if(!Er(s))return!1;var f=jr(s);return f==rt||f==ie||typeof s.message=="string"&&typeof s.name=="string"&&!Gd(s)}o(cw,"isError");function iA(s){return typeof s=="number"&&Wm(s)}o(iA,"isFinite");function _l(s){if(!Sr(s))return!1;var f=jr(s);return f==Pr||f==Gt||f==Lr||f==si}o(_l,"isFunction");function wb(s){return typeof s=="number"&&s==lt(s)}o(wb,"isInteger");function Pg(s){return typeof s=="number"&&s>-1&&s%1==0&&s<=Ge}o(Pg,"isLength");function Sr(s){var f=typeof s;return s!=null&&(f=="object"||f=="function")}o(Sr,"isObject");function Er(s){return s!=null&&typeof s=="object"}o(Er,"isObjectLike");var xb=nt?zr(nt):eg;function oA(s,f){return s===f||wl(s,f,Ma(f))}o(oA,"isMatch");function sA(s,f,m){return m=typeof m=="function"?m:e,wl(s,f,Ma(f),m)}o(sA,"isMatchWith");function lA(s){return Sb(s)&&s!=+s}o(lA,"isNaN");function aA(s){if($(s))throw new Qe(l);return ba(s)}o(aA,"isNative");function uA(s){return s===null}o(uA,"isNull");function fA(s){return s==null}o(fA,"isNil");function Sb(s){return typeof s=="number"||Er(s)&&jr(s)==Se}o(Sb,"isNumber");function Gd(s){if(!Er(s)||jr(s)!=fn)return!1;var f=ca(s);if(f===null)return!0;var m=et.call(f,"constructor")&&f.constructor;return typeof m=="function"&&m instanceof m&&ho.call(m)==Iy}o(Gd,"isPlainObject");var pw=uc?zr(uc):ef;function cA(s){return wb(s)&&s>=-Ge&&s<=Ge}o(cA,"isSafeInteger");var Cb=fi?zr(fi):tf;function Og(s){return typeof s=="string"||!ot(s)&&Er(s)&&jr(s)==gr}o(Og,"isString");function Yi(s){return typeof s=="symbol"||Er(s)&&jr(s)==pt}o(Yi,"isSymbol");var $c=ll?zr(ll):tg;function pA(s){return s===e}o(pA,"isUndefined");function dA(s){return Er(s)&&xn(s)==Cr}o(dA,"isWeakMap");function hA(s){return Er(s)&&jr(s)==Ui}o(hA,"isWeakSet");var mA=Mt(Is),gA=Mt(function(s,f){return s<=f});function bb(s){if(!s)return[];if(Mi(s))return Og(s)?Kt(s):qr(s);if(_s&&s[_s])return pc(s[_s]());var f=xn(s),m=f==Yt?aa:f==Zt?w:jc;return m(s)}o(bb,"toArray");function Tl(s){if(!s)return s===0?s:0;if(s=So(s),s===Ke||s===-Ke){var f=s<0?-1:1;return f*Xe}return s===s?s:0}o(Tl,"toFinite");function lt(s){var f=Tl(s),m=f%1;return f===f?m?f-m:f:0}o(lt,"toInteger");function Eb(s){return s?vl(lt(s),0,ct):0}o(Eb,"toLength");function So(s){if(typeof s=="number")return s;if(Yi(s))return nr;if(Sr(s)){var f=typeof s.valueOf=="function"?s.valueOf():s;s=Sr(f)?f+"":f}if(typeof s!="string")return s===0?s:+s;s=Ss(s);var m=Wo.test(s);return m||Uo.test(s)?ye(s.slice(2),m?2:8):xu.test(s)?nr:+s}o(So,"toNumber");function _b(s){return Gn(s,Ai(s))}o(_b,"toPlainObject");function vA(s){return s?vl(lt(s),-Ge,Ge):s===0?s:0}o(vA,"toSafeInteger");function Dt(s){return s==null?"":wn(s)}o(Dt,"toString");var yA=ka(function(s,f){if(te(f)||Mi(f)){Gn(f,_n(f),s);return}for(var m in f)et.call(f,m)&&Qu(s,m,f[m])}),Tb=ka(function(s,f){Gn(f,Ai(f),s)}),Mg=ka(function(s,f,m,x){Gn(f,Ai(f),s,x)}),wA=ka(function(s,f,m,x){Gn(f,_n(f),s,x)}),xA=ts(Nd);function SA(s,f){var m=go(s);return f==null?m:kd(m,f)}o(SA,"create");var CA=st(function(s,f){s=mt(s);var m=-1,x=f.length,_=x>2?f[2]:e;for(_&&b(f[0],f[1],_)&&(x=1);++m1),L}),Gn(s,Hc(s),m),x&&(m=Dn(m,k|O|j,Sg));for(var _=f.length;_--;)of(m,f[_]);return m});function WA(s,f){return Nb(s,Lg(Be(f)))}o(WA,"omitBy");var UA=ts(function(s,f){return s==null?{}:og(s,f)});function Nb(s,f){if(s==null)return{};var m=yt(Hc(s),function(x){return[x]});return f=Be(f),sg(s,m,function(x,_){return f(x,_[0])})}o(Nb,"pickBy");function zA(s,f,m){f=Gi(f,s);var x=-1,_=f.length;for(_||(_=1,s=e);++x<_;){var L=s==null?e:s[Lt(f[x])];L===e&&(x=_,L=m),s=_l(L)?L.call(s):L}return s}o(zA,"result");function $A(s,f,m){return s==null?s:Zo(s,f,m)}o($A,"set");function jA(s,f,m,x){return x=typeof x=="function"?x:e,s==null?s:Zo(s,f,m,x)}o(jA,"setWith");var Lb=Pi(_n),Pb=Pi(Ai);function qA(s,f,m){var x=ot(s),_=x||Ra(s)||$c(s);if(f=Be(f,4),m==null){var L=s&&s.constructor;_?m=x?new L:[]:Sr(s)?m=_l(L)?go(ca(s)):{}:m={}}return(_?jn:hi)(s,function(F,z,Y){return f(m,F,z,Y)}),m}o(qA,"transform");function VA(s,f){return s==null?!0:of(s,f)}o(VA,"unset");function KA(s,f,m){return s==null?s:Mc(s,f,Ac(m))}o(KA,"update");function GA(s,f,m,x){return x=typeof x=="function"?x:e,s==null?s:Mc(s,f,Ac(m),x)}o(GA,"updateWith");function jc(s){return s==null?[]:sa(s,_n(s))}o(jc,"values");function YA(s){return s==null?[]:sa(s,Ai(s))}o(YA,"valuesIn");function XA(s,f,m){return m===e&&(m=f,f=e),m!==e&&(m=So(m),m=m===m?m:0),f!==e&&(f=So(f),f=f===f?f:0),vl(So(s),f,m)}o(XA,"clamp");function QA(s,f,m){return f=Tl(f),m===e?(m=f,f=0):m=Tl(m),s=So(s),ya(s,f,m)}o(QA,"inRange");function ZA(s,f,m){if(m&&typeof m!="boolean"&&b(s,f,m)&&(f=m=e),m===e&&(typeof f=="boolean"?(m=f,f=e):typeof s=="boolean"&&(m=s,s=e)),s===e&&f===e?(s=0,f=1):(s=Tl(s),f===e?(f=s,s=0):f=Tl(f)),s>f){var x=s;s=f,f=x}if(m||s%1||f%1){var _=Ns();return Zr(s+_*(f-s+Pu("1e-"+((_+"").length-1))),f)}return Pc(s,f)}o(ZA,"random");var JA=Na(function(s,f,m){return f=f.toLowerCase(),s+(m?Ob(f):f)});function Ob(s){return mw(Dt(s).toLowerCase())}o(Ob,"capitalize");function Mb(s){return s=Dt(s),s&&s.replace(ec,Wu).replace(sc,"")}o(Mb,"deburr");function e2(s,f,m){s=Dt(s),f=wn(f);var x=s.length;m=m===e?x:vl(lt(m),0,x);var _=m;return m-=f.length,m>=0&&s.slice(m,_)==f}o(e2,"endsWith");function t2(s){return s=Dt(s),s&&ds.test(s)?s.replace(or,gd):s}o(t2,"escape");function r2(s){return s=Dt(s),s&&Qf.test(s)?s.replace(tl,"\\$&"):s}o(r2,"escapeRegExp");var n2=Na(function(s,f,m){return s+(m?"-":"")+f.toLowerCase()}),i2=Na(function(s,f,m){return s+(m?" ":"")+f.toLowerCase()}),o2=vg("toLowerCase");function s2(s,f,m){s=Dt(s),f=lt(f);var x=f?wr(s):0;if(!f||x>=f)return s;var _=(f-x)/2;return Fc(ha(_),m)+s+Fc(da(_),m)}o(s2,"pad");function l2(s,f,m){s=Dt(s),f=lt(f);var x=f?wr(s):0;return f&&x>>0,m?(s=Dt(s),s&&(typeof f=="string"||f!=null&&!pw(f))&&(f=wn(f),!f&&qn(s))?Bs(Kt(s),0,m):s.split(f,m)):[]}o(d2,"split");var h2=Na(function(s,f,m){return s+(m?" ":"")+mw(f)});function m2(s,f,m){return s=Dt(s),m=m==null?0:vl(lt(m),0,s.length),f=wn(f),s.slice(m,m+f.length)==f}o(m2,"startsWith");function g2(s,f,m){var x=N.templateSettings;m&&b(s,f,m)&&(f=e),s=Dt(s),f=Mg({},f,x,Bc);var _=Mg({},f.imports,x.imports,Bc),L=_n(_),F=sa(_,L),z,Y,le=0,ae=f.interpolate||jt,ce="__p += '",Le=Cs((f.escape||jt).source+"|"+ae.source+"|"+(ae===el?Jf:jt).source+"|"+(f.evaluate||jt).source+"|$","g"),Fe="//# sourceURL="+(et.call(f,"sourceURL")?(f.sourceURL+"").replace(/\s/g," "):"lodash.templateSources["+ ++Lu+"]")+` +`)}o(g,"insertWrapDetails");function y(s){return ot(s)||pf(s)||!!(pl&&s&&s[pl])}o(y,"isFlattenable");function S(s,f){var m=typeof s;return f=f??Ge,!!f&&(m=="number"||m!="symbol"&&$l.test(s))&&s>-1&&s%1==0&&s0){if(++f>=Ve)return arguments[0]}else f=0;return s.apply(e,arguments)}}o(dr,"shortOut");function Vr(s,f){var m=-1,x=s.length,_=x-1;for(f=f===e?x:f;++m1?s[f-1]:e;return m=typeof m=="function"?(s.pop(),m):e,ab(s,m)});function ub(s){var f=N(s);return f.__chain__=!0,f}o(ub,"chain");function JO(s,f){return f(s),s}o(JO,"tap");function _g(s,f){return f(s)}o(_g,"thru");var eM=ts(function(s){var f=s.length,m=f?s[0]:0,x=this.__wrapped__,_=o(function(L){return Nd(L,s)},"interceptor");return f>1||this.__actions__.length||!(x instanceof dt)||!S(m)?this.thru(_):(x=x.slice(m,+m+(f?1:0)),x.__actions__.push({func:_g,args:[_],thisArg:e}),new An(x,this.__chain__).thru(function(L){return f&&!L.length&&L.push(e),L}))});function tM(){return ub(this)}o(tM,"wrapperChain");function rM(){return new An(this.value(),this.__chain__)}o(rM,"wrapperCommit");function nM(){this.__values__===e&&(this.__values__=bb(this.value()));var s=this.__index__>=this.__values__.length,f=s?e:this.__values__[this.__index__++];return{done:s,value:f}}o(nM,"wrapperNext");function iM(){return this}o(iM,"wrapperToIterator");function oM(s){for(var f,m=this;m instanceof vc;){var x=bn(m);x.__index__=0,x.__values__=e,f?_.__wrapped__=x:f=x;var _=x;m=m.__wrapped__}return _.__wrapped__=s,f}o(oM,"wrapperPlant");function sM(){var s=this.__wrapped__;if(s instanceof dt){var f=s;return this.__actions__.length&&(f=new dt(this)),f=f.reverse(),f.__actions__.push({func:_g,args:[ow],thisArg:e}),new An(f,this.__chain__)}return this.thru(ow)}o(sM,"wrapperReverse");function lM(){return ag(this.__wrapped__,this.__actions__)}o(lM,"wrapperValue");var aM=Ic(function(s,f,m){et.call(s,m)?++s[m]:yo(s,m,1)});function uM(s,f,m){var x=ot(s)?Mu:Cc;return m&&b(s,f,m)&&(f=e),x(s,Be(f,3))}o(uM,"every");function fM(s,f){var m=ot(s)?Vt:Pd;return m(s,Be(f,3))}o(fM,"filter");var cM=$d(zc),pM=$d(ib);function dM(s,f){return rn(Tg(s,f),1)}o(dM,"flatMap");function hM(s,f){return rn(Tg(s,f),Ke)}o(hM,"flatMapDeep");function mM(s,f,m){return m=m===e?1:lt(m),rn(Tg(s,f),m)}o(mM,"flatMapDepth");function fb(s,f){var m=ot(s)?jn:di;return m(s,Be(f,3))}o(fb,"forEach");function cb(s,f){var m=ot(s)?dd:Sc;return m(s,Be(f,3))}o(cb,"forEachRight");var gM=Ic(function(s,f,m){et.call(s,m)?s[m].push(f):yo(s,m,[f])});function vM(s,f,m,x){s=Mi(s)?s:jc(s),m=m&&!x?lt(m):0;var _=s.length;return m<0&&(m=sr(_+m,0)),Og(s)?m<=_&&s.indexOf(f,m)>-1:!!_&&Go(s,f,m)>-1}o(vM,"includes");var yM=st(function(s,f,m){var x=-1,_=typeof f=="function",L=Mi(s)?Q(s.length):[];return di(s,function(F){L[++x]=_?Ur(f,F,m):wa(F,f,m)}),L}),wM=Ic(function(s,f,m){yo(s,m,f)});function Tg(s,f){var m=ot(s)?yt:Ea;return m(s,Be(f,3))}o(Tg,"map");function xM(s,f,m,x){return s==null?[]:(ot(f)||(f=f==null?[]:[f]),m=x?e:m,ot(m)||(m=m==null?[]:[m]),yn(s,f,m))}o(xM,"orderBy");var SM=Ic(function(s,f,m){s[m?0:1].push(f)},function(){return[[],[]]});function CM(s,f,m){var x=ot(s)?Au:Fu,_=arguments.length<3;return x(s,Be(f,4),m,_,di)}o(CM,"reduce");function bM(s,f,m){var x=ot(s)?hd:Fu,_=arguments.length<3;return x(s,Be(f,4),m,_,Sc)}o(bM,"reduceRight");function EM(s,f){var m=ot(s)?Vt:Pd;return m(s,Lg(Be(f,3)))}o(EM,"reject");function _M(s){var f=ot(s)?wc:Jy;return f(s)}o(_M,"sample");function TM(s,f,m){(m?b(s,f,m):f===e)?f=1:f=lt(f);var x=ot(s)?As:Fs;return x(s,f)}o(TM,"sampleSize");function kM(s){var f=ot(s)?Km:Jo;return f(s)}o(kM,"shuffle");function NM(s){if(s==null)return 0;if(Mi(s))return Og(s)?wr(s):s.length;var f=xn(s);return f==Yt||f==Zt?s.size:Nc(s).length}o(NM,"size");function LM(s,f,m){var x=ot(s)?Vo:ew;return m&&b(s,f,m)&&(f=e),x(s,Be(f,3))}o(LM,"some");var PM=st(function(s,f){if(s==null)return[];var m=f.length;return m>1&&b(s,f[0],f[1])?f=[]:m>2&&b(f[0],f[1],f[2])&&(f=[f[0]]),yn(s,rn(f,1),[])}),kg=Iy||function(){return qt.Date.now()};function OM(s,f){if(typeof f!="function")throw new Kn(d);return s=lt(s),function(){if(--s<1)return f.apply(this,arguments)}}o(OM,"after");function pb(s,f,m){return f=m?e:f,f=s&&f==null?s.length:f,Oi(s,se,e,e,e,e,f)}o(pb,"ary");function db(s,f){var m;if(typeof f!="function")throw new Kn(d);return s=lt(s),function(){return--s>0&&(m=f.apply(this,arguments)),s<=1&&(f=e),m}}o(db,"before");var lw=st(function(s,f,m){var x=J;if(m.length){var _=Vi(m,Oa(lw));x|=G}return Oi(s,x,f,m,_)}),hb=st(function(s,f,m){var x=J|Z;if(m.length){var _=Vi(m,Oa(hb));x|=G}return Oi(f,x,s,m,_)});function mb(s,f,m){f=m?e:f;var x=Oi(s,A,e,e,e,e,e,f);return x.placeholder=mb.placeholder,x}o(mb,"curry");function gb(s,f,m){f=m?e:f;var x=Oi(s,I,e,e,e,e,e,f);return x.placeholder=gb.placeholder,x}o(gb,"curryRight");function vb(s,f,m){var x,_,L,F,z,Y,le=0,ae=!1,ce=!1,Le=!0;if(typeof s!="function")throw new Kn(d);f=So(f)||0,Sr(m)&&(ae=!!m.leading,ce="maxWait"in m,L=ce?sr(So(m.maxWait)||0,f):L,Le="trailing"in m?!!m.trailing:Le);function Fe(Ir){var is=x,kl=_;return x=_=e,le=Ir,F=s.apply(kl,is),F}o(Fe,"invokeFunc");function $e(Ir){return le=Ir,z=At(ht,f),ae?Fe(Ir):F}o($e,"leadingEdge");function ft(Ir){var is=Ir-Y,kl=Ir-le,Ib=f-is;return ce?Zr(Ib,L-kl):Ib}o(ft,"remainingWait");function je(Ir){var is=Ir-Y,kl=Ir-le;return Y===e||is>=f||is<0||ce&&kl>=L}o(je,"shouldInvoke");function ht(){var Ir=kg();if(je(Ir))return gt(Ir);z=At(ht,ft(Ir))}o(ht,"timerExpired");function gt(Ir){return z=e,Le&&x?Fe(Ir):(x=_=e,F)}o(gt,"trailingEdge");function Xi(){z!==e&&Ta(z),le=0,x=Y=_=z=e}o(Xi,"cancel");function gi(){return z===e?F:gt(kg())}o(gi,"flush");function Qi(){var Ir=kg(),is=je(Ir);if(x=arguments,_=this,Y=Ir,is){if(z===e)return $e(Y);if(ce)return Ta(z),z=At(ht,f),Fe(Y)}return z===e&&(z=At(ht,f)),F}return o(Qi,"debounced"),Qi.cancel=Xi,Qi.flush=gi,Qi}o(vb,"debounce");var MM=st(function(s,f){return Xm(s,1,f)}),AM=st(function(s,f,m){return Xm(s,So(f)||0,m)});function DM(s){return Oi(s,pe)}o(DM,"flip");function Ng(s,f){if(typeof s!="function"||f!=null&&typeof f!="function")throw new Kn(d);var m=o(function(){var x=arguments,_=f?f.apply(this,x):x[0],L=m.cache;if(L.has(_))return L.get(_);var F=s.apply(this,x);return m.cache=L.set(_,F)||L,F},"memoized");return m.cache=new(Ng.Cache||xr),m}o(Ng,"memoize"),Ng.Cache=xr;function Lg(s){if(typeof s!="function")throw new Kn(d);return function(){var f=arguments;switch(f.length){case 0:return!s.call(this);case 1:return!s.call(this,f[0]);case 2:return!s.call(this,f[0],f[1]);case 3:return!s.call(this,f[0],f[1],f[2])}return!s.apply(this,f)}}o(Lg,"negate");function RM(s){return db(2,s)}o(RM,"once");var IM=tw(function(s,f){f=f.length==1&&ot(f[0])?yt(f[0],zr(Be())):yt(rn(f,1),zr(Be()));var m=f.length;return st(function(x){for(var _=-1,L=Zr(x.length,m);++_=f}),pf=xa(function(){return arguments}())?xa:function(s){return Er(s)&&et.call(s,"callee")&&!hc.call(s,"callee")},ot=Q.isArray,QM=ta?zr(ta):Xy;function Mi(s){return s!=null&&Pg(s.length)&&!_l(s)}o(Mi,"isArrayLike");function Rr(s){return Er(s)&&Mi(s)}o(Rr,"isArrayLikeObject");function ZM(s){return s===!0||s===!1||Er(s)&&jr(s)==zt}o(ZM,"isBoolean");var Ra=qu||ww,JM=Ou?zr(Ou):Sa;function eA(s){return Er(s)&&s.nodeType===1&&!Gd(s)}o(eA,"isElement");function tA(s){if(s==null)return!0;if(Mi(s)&&(ot(s)||typeof s=="string"||typeof s.splice=="function"||Ra(s)||$c(s)||pf(s)))return!s.length;var f=xn(s);if(f==Yt||f==Zt)return!s.size;if(te(s))return!Nc(s).length;for(var m in s)if(et.call(s,m))return!1;return!0}o(tA,"isEmpty");function rA(s,f){return Ca(s,f)}o(rA,"isEqual");function nA(s,f,m){m=typeof m=="function"?m:e;var x=m?m(s,f):e;return x===e?Ca(s,f,e,m):!!x}o(nA,"isEqualWith");function uw(s){if(!Er(s))return!1;var f=jr(s);return f==rt||f==ie||typeof s.message=="string"&&typeof s.name=="string"&&!Gd(s)}o(uw,"isError");function iA(s){return typeof s=="number"&&Wm(s)}o(iA,"isFinite");function _l(s){if(!Sr(s))return!1;var f=jr(s);return f==Pr||f==Gt||f==Lr||f==si}o(_l,"isFunction");function wb(s){return typeof s=="number"&&s==lt(s)}o(wb,"isInteger");function Pg(s){return typeof s=="number"&&s>-1&&s%1==0&&s<=Ge}o(Pg,"isLength");function Sr(s){var f=typeof s;return s!=null&&(f=="object"||f=="function")}o(Sr,"isObject");function Er(s){return s!=null&&typeof s=="object"}o(Er,"isObjectLike");var xb=nt?zr(nt):eg;function oA(s,f){return s===f||wl(s,f,Ma(f))}o(oA,"isMatch");function sA(s,f,m){return m=typeof m=="function"?m:e,wl(s,f,Ma(f),m)}o(sA,"isMatchWith");function lA(s){return Sb(s)&&s!=+s}o(lA,"isNaN");function aA(s){if($(s))throw new Qe(l);return ba(s)}o(aA,"isNative");function uA(s){return s===null}o(uA,"isNull");function fA(s){return s==null}o(fA,"isNil");function Sb(s){return typeof s=="number"||Er(s)&&jr(s)==Se}o(Sb,"isNumber");function Gd(s){if(!Er(s)||jr(s)!=fn)return!1;var f=ca(s);if(f===null)return!0;var m=et.call(f,"constructor")&&f.constructor;return typeof m=="function"&&m instanceof m&&ho.call(m)==Dy}o(Gd,"isPlainObject");var fw=uc?zr(uc):ef;function cA(s){return wb(s)&&s>=-Ge&&s<=Ge}o(cA,"isSafeInteger");var Cb=fi?zr(fi):tf;function Og(s){return typeof s=="string"||!ot(s)&&Er(s)&&jr(s)==gr}o(Og,"isString");function Yi(s){return typeof s=="symbol"||Er(s)&&jr(s)==pt}o(Yi,"isSymbol");var $c=ll?zr(ll):tg;function pA(s){return s===e}o(pA,"isUndefined");function dA(s){return Er(s)&&xn(s)==Cr}o(dA,"isWeakMap");function hA(s){return Er(s)&&jr(s)==Ui}o(hA,"isWeakSet");var mA=Mt(Is),gA=Mt(function(s,f){return s<=f});function bb(s){if(!s)return[];if(Mi(s))return Og(s)?Kt(s):qr(s);if(_s&&s[_s])return pc(s[_s]());var f=xn(s),m=f==Yt?aa:f==Zt?w:jc;return m(s)}o(bb,"toArray");function Tl(s){if(!s)return s===0?s:0;if(s=So(s),s===Ke||s===-Ke){var f=s<0?-1:1;return f*Xe}return s===s?s:0}o(Tl,"toFinite");function lt(s){var f=Tl(s),m=f%1;return f===f?m?f-m:f:0}o(lt,"toInteger");function Eb(s){return s?vl(lt(s),0,ct):0}o(Eb,"toLength");function So(s){if(typeof s=="number")return s;if(Yi(s))return nr;if(Sr(s)){var f=typeof s.valueOf=="function"?s.valueOf():s;s=Sr(f)?f+"":f}if(typeof s!="string")return s===0?s:+s;s=Ss(s);var m=Wo.test(s);return m||Uo.test(s)?ye(s.slice(2),m?2:8):xu.test(s)?nr:+s}o(So,"toNumber");function _b(s){return Gn(s,Ai(s))}o(_b,"toPlainObject");function vA(s){return s?vl(lt(s),-Ge,Ge):s===0?s:0}o(vA,"toSafeInteger");function Dt(s){return s==null?"":wn(s)}o(Dt,"toString");var yA=ka(function(s,f){if(te(f)||Mi(f)){Gn(f,_n(f),s);return}for(var m in f)et.call(f,m)&&Qu(s,m,f[m])}),Tb=ka(function(s,f){Gn(f,Ai(f),s)}),Mg=ka(function(s,f,m,x){Gn(f,Ai(f),s,x)}),wA=ka(function(s,f,m,x){Gn(f,_n(f),s,x)}),xA=ts(Nd);function SA(s,f){var m=go(s);return f==null?m:kd(m,f)}o(SA,"create");var CA=st(function(s,f){s=mt(s);var m=-1,x=f.length,_=x>2?f[2]:e;for(_&&b(f[0],f[1],_)&&(x=1);++m1),L}),Gn(s,Hc(s),m),x&&(m=Dn(m,k|O|j,Sg));for(var _=f.length;_--;)of(m,f[_]);return m});function WA(s,f){return Nb(s,Lg(Be(f)))}o(WA,"omitBy");var UA=ts(function(s,f){return s==null?{}:og(s,f)});function Nb(s,f){if(s==null)return{};var m=yt(Hc(s),function(x){return[x]});return f=Be(f),sg(s,m,function(x,_){return f(x,_[0])})}o(Nb,"pickBy");function zA(s,f,m){f=Gi(f,s);var x=-1,_=f.length;for(_||(_=1,s=e);++x<_;){var L=s==null?e:s[Lt(f[x])];L===e&&(x=_,L=m),s=_l(L)?L.call(s):L}return s}o(zA,"result");function $A(s,f,m){return s==null?s:Zo(s,f,m)}o($A,"set");function jA(s,f,m,x){return x=typeof x=="function"?x:e,s==null?s:Zo(s,f,m,x)}o(jA,"setWith");var Lb=Pi(_n),Pb=Pi(Ai);function qA(s,f,m){var x=ot(s),_=x||Ra(s)||$c(s);if(f=Be(f,4),m==null){var L=s&&s.constructor;_?m=x?new L:[]:Sr(s)?m=_l(L)?go(ca(s)):{}:m={}}return(_?jn:hi)(s,function(F,z,Y){return f(m,F,z,Y)}),m}o(qA,"transform");function VA(s,f){return s==null?!0:of(s,f)}o(VA,"unset");function KA(s,f,m){return s==null?s:Mc(s,f,Ac(m))}o(KA,"update");function GA(s,f,m,x){return x=typeof x=="function"?x:e,s==null?s:Mc(s,f,Ac(m),x)}o(GA,"updateWith");function jc(s){return s==null?[]:sa(s,_n(s))}o(jc,"values");function YA(s){return s==null?[]:sa(s,Ai(s))}o(YA,"valuesIn");function XA(s,f,m){return m===e&&(m=f,f=e),m!==e&&(m=So(m),m=m===m?m:0),f!==e&&(f=So(f),f=f===f?f:0),vl(So(s),f,m)}o(XA,"clamp");function QA(s,f,m){return f=Tl(f),m===e?(m=f,f=0):m=Tl(m),s=So(s),ya(s,f,m)}o(QA,"inRange");function ZA(s,f,m){if(m&&typeof m!="boolean"&&b(s,f,m)&&(f=m=e),m===e&&(typeof f=="boolean"?(m=f,f=e):typeof s=="boolean"&&(m=s,s=e)),s===e&&f===e?(s=0,f=1):(s=Tl(s),f===e?(f=s,s=0):f=Tl(f)),s>f){var x=s;s=f,f=x}if(m||s%1||f%1){var _=Ns();return Zr(s+_*(f-s+Pu("1e-"+((_+"").length-1))),f)}return Pc(s,f)}o(ZA,"random");var JA=Na(function(s,f,m){return f=f.toLowerCase(),s+(m?Ob(f):f)});function Ob(s){return dw(Dt(s).toLowerCase())}o(Ob,"capitalize");function Mb(s){return s=Dt(s),s&&s.replace(ec,Wu).replace(sc,"")}o(Mb,"deburr");function e2(s,f,m){s=Dt(s),f=wn(f);var x=s.length;m=m===e?x:vl(lt(m),0,x);var _=m;return m-=f.length,m>=0&&s.slice(m,_)==f}o(e2,"endsWith");function t2(s){return s=Dt(s),s&&ds.test(s)?s.replace(or,gd):s}o(t2,"escape");function r2(s){return s=Dt(s),s&&Qf.test(s)?s.replace(tl,"\\$&"):s}o(r2,"escapeRegExp");var n2=Na(function(s,f,m){return s+(m?"-":"")+f.toLowerCase()}),i2=Na(function(s,f,m){return s+(m?" ":"")+f.toLowerCase()}),o2=vg("toLowerCase");function s2(s,f,m){s=Dt(s),f=lt(f);var x=f?wr(s):0;if(!f||x>=f)return s;var _=(f-x)/2;return Fc(ha(_),m)+s+Fc(da(_),m)}o(s2,"pad");function l2(s,f,m){s=Dt(s),f=lt(f);var x=f?wr(s):0;return f&&x>>0,m?(s=Dt(s),s&&(typeof f=="string"||f!=null&&!fw(f))&&(f=wn(f),!f&&qn(s))?Bs(Kt(s),0,m):s.split(f,m)):[]}o(d2,"split");var h2=Na(function(s,f,m){return s+(m?" ":"")+dw(f)});function m2(s,f,m){return s=Dt(s),m=m==null?0:vl(lt(m),0,s.length),f=wn(f),s.slice(m,m+f.length)==f}o(m2,"startsWith");function g2(s,f,m){var x=N.templateSettings;m&&b(s,f,m)&&(f=e),s=Dt(s),f=Mg({},f,x,Bc);var _=Mg({},f.imports,x.imports,Bc),L=_n(_),F=sa(_,L),z,Y,le=0,ae=f.interpolate||jt,ce="__p += '",Le=Cs((f.escape||jt).source+"|"+ae.source+"|"+(ae===el?Jf:jt).source+"|"+(f.evaluate||jt).source+"|$","g"),Fe="//# sourceURL="+(et.call(f,"sourceURL")?(f.sourceURL+"").replace(/\s/g," "):"lodash.templateSources["+ ++Lu+"]")+` `;s.replace(Le,function(je,ht,gt,Xi,gi,Qi){return gt||(gt=Xi),ce+=s.slice(le,Qi).replace(Me,Uu),ht&&(z=!0,ce+=`' + __e(`+ht+`) + '`),gi&&(Y=!0,ce+=`'; @@ -25,21 +25,21 @@ __p += '`),gt&&(ce+=`' + function print() { __p += __j.call(arguments, '') } `:`; `)+ce+`return __p -}`;var ft=Db(function(){return bt(L,Fe+"return "+ce).apply(e,F)});if(ft.source=ce,cw(ft))throw ft;return ft}o(g2,"template");function v2(s){return Dt(s).toLowerCase()}o(v2,"toLower");function y2(s){return Dt(s).toUpperCase()}o(y2,"toUpper");function w2(s,f,m){if(s=Dt(s),s&&(m||f===e))return Ss(s);if(!s||!(f=wn(f)))return s;var x=Kt(s),_=Kt(f),L=qi(x,_),F=al(x,_)+1;return Bs(x,L,F).join("")}o(w2,"trim");function x2(s,f,m){if(s=Dt(s),s&&(m||f===e))return s.slice(0,gn(s)+1);if(!s||!(f=wn(f)))return s;var x=Kt(s),_=al(x,Kt(f))+1;return Bs(x,0,_).join("")}o(x2,"trimEnd");function S2(s,f,m){if(s=Dt(s),s&&(m||f===e))return s.replace(rl,"");if(!s||!(f=wn(f)))return s;var x=Kt(s),_=qi(x,Kt(f));return Bs(x,_).join("")}o(S2,"trimStart");function C2(s,f){var m=me,x=xe;if(Sr(f)){var _="separator"in f?f.separator:_;m="length"in f?lt(f.length):m,x="omission"in f?wn(f.omission):x}s=Dt(s);var L=s.length;if(qn(s)){var F=Kt(s);L=F.length}if(m>=L)return s;var z=m-wr(x);if(z<1)return x;var Y=F?Bs(F,0,z).join(""):s.slice(0,z);if(_===e)return Y+x;if(F&&(z+=Y.length-z),pw(_)){if(s.slice(z).search(_)){var le,ae=Y;for(_.global||(_=Cs(_.source,Dt(nl.exec(_))+"g")),_.lastIndex=0;le=_.exec(ae);)var ce=le.index;Y=Y.slice(0,ce===e?z:ce)}}else if(s.indexOf(wn(_),z)!=z){var Le=Y.lastIndexOf(_);Le>-1&&(Y=Y.slice(0,Le))}return Y+x}o(C2,"truncate");function b2(s){return s=Dt(s),s&&li.test(s)?s.replace(Wr,ci):s}o(b2,"unescape");var E2=Na(function(s,f,m){return s+(m?" ":"")+f.toUpperCase()}),mw=vg("toUpperCase");function Ab(s,f,m){return s=Dt(s),f=m?e:f,f===e?Ti(s)?Vn(s):Du(s):s.match(f)||[]}o(Ab,"words");var Db=st(function(s,f){try{return Ur(s,e,f)}catch(m){return cw(m)?m:new Qe(m)}}),_2=ts(function(s,f){return jn(f,function(m){m=Lt(m),yo(s,m,uw(s[m],s))}),s});function T2(s){var f=s==null?0:s.length,m=Be();return s=f?yt(s,function(x){if(typeof x[1]!="function")throw new Kn(d);return[m(x[0]),x[1]]}):[],st(function(x){for(var _=-1;++_Ge)return[];var m=ct,x=Zr(s,ct);f=Be(f),s-=ct;for(var _=Yo(x,f);++m0||f<0)?new dt(m):(s<0?m=m.takeRight(-s):s&&(m=m.drop(s)),f!==e&&(f=lt(f),m=f<0?m.dropRight(-f):m.take(f-s)),m)},dt.prototype.takeRightWhile=function(s){return this.reverse().takeWhile(s).reverse()},dt.prototype.toArray=function(){return this.take(ct)},hi(dt.prototype,function(s,f){var m=/^(?:filter|find|map|reject)|While$/.test(f),x=/^(?:head|last)$/.test(f),_=N[x?"take"+(f=="last"?"Right":""):f],L=x||/^find/.test(f);!_||(N.prototype[f]=function(){var F=this.__wrapped__,z=x?[1]:arguments,Y=F instanceof dt,le=z[0],ae=Y||ot(F),ce=o(function(ht){var gt=_.apply(N,$i([ht],z));return x&&Le?gt[0]:gt},"interceptor");ae&&m&&typeof le=="function"&&le.length!=1&&(Y=ae=!1);var Le=this.__chain__,Fe=!!this.__actions__.length,$e=L&&!Le,ft=Y&&!Fe;if(!L&&ae){F=ft?F:new dt(this);var je=s.apply(F,z);return je.__actions__.push({func:_g,args:[ce],thisArg:e}),new An(je,Le)}return $e&&ft?s.apply(this,z):(je=this.thru(ce),$e?x?je.value()[0]:je.value():je)})}),jn(["pop","push","shift","sort","splice","unshift"],function(s){var f=fa[s],m=/^(?:push|sort|unshift)$/.test(s)?"tap":"thru",x=/^(?:pop|shift)$/.test(s);N.prototype[s]=function(){var _=arguments;if(x&&!this.__chain__){var L=this.value();return f.apply(ot(L)?L:[],_)}return this[m](function(F){return f.apply(ot(F)?F:[],_)})}}),hi(dt.prototype,function(s,f){var m=N[f];if(m){var x=m.name+"";et.call(ga,x)||(ga[x]=[]),ga[x].push({name:f,func:m})}}),ga[sf(e,Z).name]=[{name:"wrapper",func:e}],dt.prototype.clone=qy,dt.prototype.reverse=Vy,dt.prototype.value=yd,N.prototype.at=eM,N.prototype.chain=tM,N.prototype.commit=rM,N.prototype.next=nM,N.prototype.plant=oM,N.prototype.reverse=sM,N.prototype.toJSON=N.prototype.valueOf=N.prototype.value=lM,N.prototype.first=N.prototype.head,_s&&(N.prototype[_s]=iM),N},"runInContext"),vn=zu();typeof define=="function"&&typeof define.amd=="object"&&define.amd?(qt._=vn,define(function(){return vn})):hn?((hn.exports=vn)._=vn,ea._=vn):qt._=vn}).call(Np)});var $k=Ue((oS,sS)=>{(function(e,t){typeof oS=="object"&&typeof sS!="undefined"?sS.exports=t():typeof define=="function"&&define.amd?define(t):e.stable=t()})(oS,function(){"use strict";var e=o(function(l,d){return t(l.slice(),d)},"stable");e.inplace=function(l,d){var h=t(l,d);return h!==l&&n(h,null,l.length,l),l};function t(l,d){typeof d!="function"&&(d=o(function(k,O){return String(k).localeCompare(O)},"comp"));var h=l.length;if(h<=1)return l;for(var c=new Array(h),v=1;vv&&(j=v),B>v&&(B=v),X=O,J=j;;)if(X{(function(){"use strict";var e={}.hasOwnProperty;function t(){for(var n=[],l=0;l{(function(e,t){typeof FS=="object"&&typeof BS!="undefined"?BS.exports=t():typeof define=="function"&&define.amd?define(t):(e=e||self,e.CodeMirror=t())})(FS,function(){"use strict";var e=navigator.userAgent,t=navigator.platform,n=/gecko\/\d/i.test(e),l=/MSIE \d/.test(e),d=/Trident\/(?:[7-9]|\d{2,})\..*rv:(\d+)/.exec(e),h=/Edge\/(\d+)/.exec(e),c=l||d||h,v=c&&(l?document.documentMode||6:+(h||d)[1]),C=!h&&/WebKit\//.test(e),k=C&&/Qt\/\d+\.\d+/.test(e),O=!h&&/Chrome\//.test(e),j=/Opera\//.test(e),B=/Apple Computer/.test(navigator.vendor),X=/Mac OS X 1\d\D([8-9]|\d\d)\D/.test(e),J=/PhantomJS/.test(e),Z=B&&(/Mobile\/\w+/.test(e)||navigator.maxTouchPoints>2),R=/Android/.test(e),A=Z||R||/webOS|BlackBerry|Opera Mini|Opera Mobi|IEMobile/i.test(e),I=Z||/Mac/.test(t),G=/\bCrOS\b/.test(e),K=/win/i.test(t),se=j&&e.match(/Version\/(\d*\.\d*)/);se&&(se=Number(se[1])),se&&se>=15&&(j=!1,C=!0);var ne=I&&(k||j&&(se==null||se<12.11)),pe=n||c&&v>=9;function me(r){return new RegExp("(^|\\s)"+r+"(?:$|\\s)\\s*")}o(me,"classTest");var xe=o(function(r,i){var u=r.className,a=me(i).exec(u);if(a){var p=u.slice(a.index+a[0].length);r.className=u.slice(0,a.index)+(p?a[1]+p:"")}},"rmClass");function Ve(r){for(var i=r.childNodes.length;i>0;--i)r.removeChild(r.firstChild);return r}o(Ve,"removeChildren");function tt(r,i){return Ve(r).appendChild(i)}o(tt,"removeChildrenAndAdd");function _e(r,i,u,a){var p=document.createElement(r);if(u&&(p.className=u),a&&(p.style.cssText=a),typeof i=="string")p.appendChild(document.createTextNode(i));else if(i)for(var g=0;g=i)return y+(i-g);y+=S-g,y+=u-y%u,g=S+1}}o(_t,"countColumn");var Ct=o(function(){this.id=null,this.f=null,this.time=0,this.handler=Hr(this.onTimeout,this)},"Delayed");Ct.prototype.onTimeout=function(r){r.id=0,r.time<=+new Date?r.f():setTimeout(r.handler,r.time-+new Date)},Ct.prototype.set=function(r,i){this.f=i;var u=+new Date+r;(!this.id||u=i)return a+Math.min(y,i-p);if(p+=g-a,p+=u-p%u,a=g+1,p>=i)return a}}o(Pr,"findColumn");var Gt=[""];function Yt(r){for(;Gt.length<=r;)Gt.push(Se(Gt)+" ");return Gt[r]}o(Yt,"spaceStr");function Se(r){return r[r.length-1]}o(Se,"lst");function Or(r,i){for(var u=[],a=0;a"\x80"&&(r.toUpperCase()!=r.toLowerCase()||cn.test(r))}o(Zt,"isWordCharBasic");function gr(r,i){return i?i.source.indexOf("\\w")>-1&&Zt(r)?!0:i.test(r):Zt(r)}o(gr,"isWordChar");function pt(r){for(var i in r)if(r.hasOwnProperty(i)&&r[i])return!1;return!0}o(pt,"isEmpty");var Ho=/[\u0300-\u036f\u0483-\u0489\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u064b-\u065e\u0670\u06d6-\u06dc\u06de-\u06e4\u06e7\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0900-\u0902\u093c\u0941-\u0948\u094d\u0951-\u0955\u0962\u0963\u0981\u09bc\u09be\u09c1-\u09c4\u09cd\u09d7\u09e2\u09e3\u0a01\u0a02\u0a3c\u0a41\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a70\u0a71\u0a75\u0a81\u0a82\u0abc\u0ac1-\u0ac5\u0ac7\u0ac8\u0acd\u0ae2\u0ae3\u0b01\u0b3c\u0b3e\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b57\u0b62\u0b63\u0b82\u0bbe\u0bc0\u0bcd\u0bd7\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62\u0c63\u0cbc\u0cbf\u0cc2\u0cc6\u0ccc\u0ccd\u0cd5\u0cd6\u0ce2\u0ce3\u0d3e\u0d41-\u0d44\u0d4d\u0d57\u0d62\u0d63\u0dca\u0dcf\u0dd2-\u0dd4\u0dd6\u0ddf\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb\u0ebc\u0ec8-\u0ecd\u0f18\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86\u0f87\u0f90-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039\u103a\u103d\u103e\u1058\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085\u1086\u108d\u109d\u135f\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927\u1928\u1932\u1939-\u193b\u1a17\u1a18\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80\u1b81\u1ba2-\u1ba5\u1ba8\u1ba9\u1c2c-\u1c33\u1c36\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1dc0-\u1de6\u1dfd-\u1dff\u200c\u200d\u20d0-\u20f0\u2cef-\u2cf1\u2de0-\u2dff\u302a-\u302f\u3099\u309a\ua66f-\ua672\ua67c\ua67d\ua6f0\ua6f1\ua802\ua806\ua80b\ua825\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31\uaa32\uaa35\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7\uaab8\uaabe\uaabf\uaac1\uabe5\uabe8\uabed\udc00-\udfff\ufb1e\ufe00-\ufe0f\ufe20-\ufe26\uff9e\uff9f]/;function Cr(r){return r.charCodeAt(0)>=768&&Ho.test(r)}o(Cr,"isExtendingChar");function Ui(r,i,u){for(;(u<0?i>0:iu?-1:1;;){if(i==u)return i;var p=(i+u)/2,g=a<0?Math.ceil(p):Math.floor(p);if(g==i)return r(g)?i:u;r(g)?u=g:i=g+a}}o(pn,"findFirst");function zn(r,i,u,a){if(!r)return a(i,u,"ltr",0);for(var p=!1,g=0;gi||i==u&&y.to==i)&&(a(Math.max(y.from,i),Math.min(y.to,u),y.level==1?"rtl":"ltr",g),p=!0)}p||a(i,u,"ltr")}o(zn,"iterateBidiSections");var Si=null;function Ci(r,i,u){var a;Si=null;for(var p=0;pi)return p;g.to==i&&(g.from!=g.to&&u=="before"?a=p:Si=p),g.from==i&&(g.from!=g.to&&u!="before"?a=p:Si=p)}return a??Si}o(Ci,"getBidiPartAt");var $n=function(){var r="bbbbbbbbbtstwsbbbbbbbbbbbbbbssstwNN%%%NNNNNN,N,N1111111111NNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNbbbbbbsbbbbbbbbbbbbbbbbbbbbbbbbbb,N%%%%NNNNLNNNNN%%11NLNNN1LNNNNNLLLLLLLLLLLLLLLLLLLLLLLNLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLN",i="nnnnnnNNr%%r,rNNmmmmmmmmmmmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmmmmmmmmmmmmmmmnnnnnnnnnn%nnrrrmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmnNmmmmmmrrmmNmmmmrr1111111111";function u(E){return E<=247?r.charAt(E):1424<=E&&E<=1524?"R":1536<=E&&E<=1785?i.charAt(E-1536):1774<=E&&E<=2220?"r":8192<=E&&E<=8203?"w":E==8204?"b":"L"}o(u,"charType");var a=/[\u0590-\u05f4\u0600-\u06ff\u0700-\u08ac]/,p=/[stwN]/,g=/[LRr]/,y=/[Lb1n]/,S=/[1n]/;function b(E,M,D){this.level=E,this.from=M,this.to=D}return o(b,"BidiSpan"),function(E,M){var D=M=="ltr"?"L":"R";if(E.length==0||M=="ltr"&&!a.test(E))return!1;for(var V=E.length,$=[],te=0;te-1&&(a[i]=p.slice(0,g).concat(p.slice(g+1)))}}}o(he,"off");function Te(r,i){var u=ee(r,i);if(!!u.length)for(var a=Array.prototype.slice.call(arguments,2),p=0;p0}o(Ft,"hasHandler");function Wr(r){r.prototype.on=function(i,u){H(this,i,u)},r.prototype.off=function(i,u){he(this,i,u)}}o(Wr,"eventMixin");function or(r){r.preventDefault?r.preventDefault():r.returnValue=!1}o(or,"e_preventDefault");function li(r){r.stopPropagation?r.stopPropagation():r.cancelBubble=!0}o(li,"e_stopPropagation");function ds(r){return r.defaultPrevented!=null?r.defaultPrevented:r.returnValue==!1}o(ds,"e_defaultPrevented");function lo(r){or(r),li(r)}o(lo,"e_stop");function bi(r){return r.target||r.srcElement}o(bi,"e_target");function el(r){var i=r.which;return i==null&&(r.button&1?i=1:r.button&2?i=3:r.button&4&&(i=2)),I&&r.ctrlKey&&i==1&&(i=3),i}o(el,"e_button");var hs=function(){if(c&&v<9)return!1;var r=_e("div");return"draggable"in r||"dragDrop"in r}(),dn;function id(r){if(dn==null){var i=_e("span","\u200B");tt(r,_e("span",[i,document.createTextNode("x")])),r.firstChild.offsetHeight!=0&&(dn=i.offsetWidth<=1&&i.offsetHeight>2&&!(c&&v<8))}var u=dn?_e("span","\u200B"):_e("span","\xA0",null,"display: inline-block; width: 1px; margin-right: -1px");return u.setAttribute("cm-text",""),u}o(id,"zeroWidthElement");var tl;function Qf(r){if(tl!=null)return tl;var i=tt(r,document.createTextNode("A\u062EA")),u=We(i,0,1).getBoundingClientRect(),a=We(i,1,2).getBoundingClientRect();return Ve(r),!u||u.left==u.right?!1:tl=a.right-u.right<3}o(Qf,"hasBadBidiRects");var rl=` +}`;var ft=Db(function(){return bt(L,Fe+"return "+ce).apply(e,F)});if(ft.source=ce,uw(ft))throw ft;return ft}o(g2,"template");function v2(s){return Dt(s).toLowerCase()}o(v2,"toLower");function y2(s){return Dt(s).toUpperCase()}o(y2,"toUpper");function w2(s,f,m){if(s=Dt(s),s&&(m||f===e))return Ss(s);if(!s||!(f=wn(f)))return s;var x=Kt(s),_=Kt(f),L=qi(x,_),F=al(x,_)+1;return Bs(x,L,F).join("")}o(w2,"trim");function x2(s,f,m){if(s=Dt(s),s&&(m||f===e))return s.slice(0,gn(s)+1);if(!s||!(f=wn(f)))return s;var x=Kt(s),_=al(x,Kt(f))+1;return Bs(x,0,_).join("")}o(x2,"trimEnd");function S2(s,f,m){if(s=Dt(s),s&&(m||f===e))return s.replace(rl,"");if(!s||!(f=wn(f)))return s;var x=Kt(s),_=qi(x,Kt(f));return Bs(x,_).join("")}o(S2,"trimStart");function C2(s,f){var m=me,x=xe;if(Sr(f)){var _="separator"in f?f.separator:_;m="length"in f?lt(f.length):m,x="omission"in f?wn(f.omission):x}s=Dt(s);var L=s.length;if(qn(s)){var F=Kt(s);L=F.length}if(m>=L)return s;var z=m-wr(x);if(z<1)return x;var Y=F?Bs(F,0,z).join(""):s.slice(0,z);if(_===e)return Y+x;if(F&&(z+=Y.length-z),fw(_)){if(s.slice(z).search(_)){var le,ae=Y;for(_.global||(_=Cs(_.source,Dt(nl.exec(_))+"g")),_.lastIndex=0;le=_.exec(ae);)var ce=le.index;Y=Y.slice(0,ce===e?z:ce)}}else if(s.indexOf(wn(_),z)!=z){var Le=Y.lastIndexOf(_);Le>-1&&(Y=Y.slice(0,Le))}return Y+x}o(C2,"truncate");function b2(s){return s=Dt(s),s&&li.test(s)?s.replace(Wr,ci):s}o(b2,"unescape");var E2=Na(function(s,f,m){return s+(m?" ":"")+f.toUpperCase()}),dw=vg("toUpperCase");function Ab(s,f,m){return s=Dt(s),f=m?e:f,f===e?Ti(s)?Vn(s):Du(s):s.match(f)||[]}o(Ab,"words");var Db=st(function(s,f){try{return Ur(s,e,f)}catch(m){return uw(m)?m:new Qe(m)}}),_2=ts(function(s,f){return jn(f,function(m){m=Lt(m),yo(s,m,lw(s[m],s))}),s});function T2(s){var f=s==null?0:s.length,m=Be();return s=f?yt(s,function(x){if(typeof x[1]!="function")throw new Kn(d);return[m(x[0]),x[1]]}):[],st(function(x){for(var _=-1;++_Ge)return[];var m=ct,x=Zr(s,ct);f=Be(f),s-=ct;for(var _=Yo(x,f);++m0||f<0)?new dt(m):(s<0?m=m.takeRight(-s):s&&(m=m.drop(s)),f!==e&&(f=lt(f),m=f<0?m.dropRight(-f):m.take(f-s)),m)},dt.prototype.takeRightWhile=function(s){return this.reverse().takeWhile(s).reverse()},dt.prototype.toArray=function(){return this.take(ct)},hi(dt.prototype,function(s,f){var m=/^(?:filter|find|map|reject)|While$/.test(f),x=/^(?:head|last)$/.test(f),_=N[x?"take"+(f=="last"?"Right":""):f],L=x||/^find/.test(f);!_||(N.prototype[f]=function(){var F=this.__wrapped__,z=x?[1]:arguments,Y=F instanceof dt,le=z[0],ae=Y||ot(F),ce=o(function(ht){var gt=_.apply(N,$i([ht],z));return x&&Le?gt[0]:gt},"interceptor");ae&&m&&typeof le=="function"&&le.length!=1&&(Y=ae=!1);var Le=this.__chain__,Fe=!!this.__actions__.length,$e=L&&!Le,ft=Y&&!Fe;if(!L&&ae){F=ft?F:new dt(this);var je=s.apply(F,z);return je.__actions__.push({func:_g,args:[ce],thisArg:e}),new An(je,Le)}return $e&&ft?s.apply(this,z):(je=this.thru(ce),$e?x?je.value()[0]:je.value():je)})}),jn(["pop","push","shift","sort","splice","unshift"],function(s){var f=fa[s],m=/^(?:push|sort|unshift)$/.test(s)?"tap":"thru",x=/^(?:pop|shift)$/.test(s);N.prototype[s]=function(){var _=arguments;if(x&&!this.__chain__){var L=this.value();return f.apply(ot(L)?L:[],_)}return this[m](function(F){return f.apply(ot(F)?F:[],_)})}}),hi(dt.prototype,function(s,f){var m=N[f];if(m){var x=m.name+"";et.call(ga,x)||(ga[x]=[]),ga[x].push({name:f,func:m})}}),ga[sf(e,Z).name]=[{name:"wrapper",func:e}],dt.prototype.clone=$y,dt.prototype.reverse=jy,dt.prototype.value=yd,N.prototype.at=eM,N.prototype.chain=tM,N.prototype.commit=rM,N.prototype.next=nM,N.prototype.plant=oM,N.prototype.reverse=sM,N.prototype.toJSON=N.prototype.valueOf=N.prototype.value=lM,N.prototype.first=N.prototype.head,_s&&(N.prototype[_s]=iM),N},"runInContext"),vn=zu();typeof define=="function"&&typeof define.amd=="object"&&define.amd?(qt._=vn,define(function(){return vn})):hn?((hn.exports=vn)._=vn,ea._=vn):qt._=vn}).call(Np)});var $k=Ue((nS,iS)=>{(function(e,t){typeof nS=="object"&&typeof iS!="undefined"?iS.exports=t():typeof define=="function"&&define.amd?define(t):e.stable=t()})(nS,function(){"use strict";var e=o(function(l,d){return t(l.slice(),d)},"stable");e.inplace=function(l,d){var h=t(l,d);return h!==l&&n(h,null,l.length,l),l};function t(l,d){typeof d!="function"&&(d=o(function(k,O){return String(k).localeCompare(O)},"comp"));var h=l.length;if(h<=1)return l;for(var c=new Array(h),v=1;vv&&(j=v),B>v&&(B=v),X=O,J=j;;)if(X{(function(){"use strict";var e={}.hasOwnProperty;function t(){for(var n=[],l=0;l{(function(e,t){typeof RS=="object"&&typeof IS!="undefined"?IS.exports=t():typeof define=="function"&&define.amd?define(t):(e=e||self,e.CodeMirror=t())})(RS,function(){"use strict";var e=navigator.userAgent,t=navigator.platform,n=/gecko\/\d/i.test(e),l=/MSIE \d/.test(e),d=/Trident\/(?:[7-9]|\d{2,})\..*rv:(\d+)/.exec(e),h=/Edge\/(\d+)/.exec(e),c=l||d||h,v=c&&(l?document.documentMode||6:+(h||d)[1]),C=!h&&/WebKit\//.test(e),k=C&&/Qt\/\d+\.\d+/.test(e),O=!h&&/Chrome\//.test(e),j=/Opera\//.test(e),B=/Apple Computer/.test(navigator.vendor),X=/Mac OS X 1\d\D([8-9]|\d\d)\D/.test(e),J=/PhantomJS/.test(e),Z=B&&(/Mobile\/\w+/.test(e)||navigator.maxTouchPoints>2),R=/Android/.test(e),A=Z||R||/webOS|BlackBerry|Opera Mini|Opera Mobi|IEMobile/i.test(e),I=Z||/Mac/.test(t),G=/\bCrOS\b/.test(e),K=/win/i.test(t),se=j&&e.match(/Version\/(\d*\.\d*)/);se&&(se=Number(se[1])),se&&se>=15&&(j=!1,C=!0);var ne=I&&(k||j&&(se==null||se<12.11)),pe=n||c&&v>=9;function me(r){return new RegExp("(^|\\s)"+r+"(?:$|\\s)\\s*")}o(me,"classTest");var xe=o(function(r,i){var u=r.className,a=me(i).exec(u);if(a){var p=u.slice(a.index+a[0].length);r.className=u.slice(0,a.index)+(p?a[1]+p:"")}},"rmClass");function Ve(r){for(var i=r.childNodes.length;i>0;--i)r.removeChild(r.firstChild);return r}o(Ve,"removeChildren");function tt(r,i){return Ve(r).appendChild(i)}o(tt,"removeChildrenAndAdd");function _e(r,i,u,a){var p=document.createElement(r);if(u&&(p.className=u),a&&(p.style.cssText=a),typeof i=="string")p.appendChild(document.createTextNode(i));else if(i)for(var g=0;g=i)return y+(i-g);y+=S-g,y+=u-y%u,g=S+1}}o(_t,"countColumn");var Ct=o(function(){this.id=null,this.f=null,this.time=0,this.handler=Hr(this.onTimeout,this)},"Delayed");Ct.prototype.onTimeout=function(r){r.id=0,r.time<=+new Date?r.f():setTimeout(r.handler,r.time-+new Date)},Ct.prototype.set=function(r,i){this.f=i;var u=+new Date+r;(!this.id||u=i)return a+Math.min(y,i-p);if(p+=g-a,p+=u-p%u,a=g+1,p>=i)return a}}o(Pr,"findColumn");var Gt=[""];function Yt(r){for(;Gt.length<=r;)Gt.push(Se(Gt)+" ");return Gt[r]}o(Yt,"spaceStr");function Se(r){return r[r.length-1]}o(Se,"lst");function Or(r,i){for(var u=[],a=0;a"\x80"&&(r.toUpperCase()!=r.toLowerCase()||cn.test(r))}o(Zt,"isWordCharBasic");function gr(r,i){return i?i.source.indexOf("\\w")>-1&&Zt(r)?!0:i.test(r):Zt(r)}o(gr,"isWordChar");function pt(r){for(var i in r)if(r.hasOwnProperty(i)&&r[i])return!1;return!0}o(pt,"isEmpty");var Ho=/[\u0300-\u036f\u0483-\u0489\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u064b-\u065e\u0670\u06d6-\u06dc\u06de-\u06e4\u06e7\u06e8\u06ea-\u06ed\u0711\u0730-\u074a\u07a6-\u07b0\u07eb-\u07f3\u0816-\u0819\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0900-\u0902\u093c\u0941-\u0948\u094d\u0951-\u0955\u0962\u0963\u0981\u09bc\u09be\u09c1-\u09c4\u09cd\u09d7\u09e2\u09e3\u0a01\u0a02\u0a3c\u0a41\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a70\u0a71\u0a75\u0a81\u0a82\u0abc\u0ac1-\u0ac5\u0ac7\u0ac8\u0acd\u0ae2\u0ae3\u0b01\u0b3c\u0b3e\u0b3f\u0b41-\u0b44\u0b4d\u0b56\u0b57\u0b62\u0b63\u0b82\u0bbe\u0bc0\u0bcd\u0bd7\u0c3e-\u0c40\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62\u0c63\u0cbc\u0cbf\u0cc2\u0cc6\u0ccc\u0ccd\u0cd5\u0cd6\u0ce2\u0ce3\u0d3e\u0d41-\u0d44\u0d4d\u0d57\u0d62\u0d63\u0dca\u0dcf\u0dd2-\u0dd4\u0dd6\u0ddf\u0e31\u0e34-\u0e3a\u0e47-\u0e4e\u0eb1\u0eb4-\u0eb9\u0ebb\u0ebc\u0ec8-\u0ecd\u0f18\u0f19\u0f35\u0f37\u0f39\u0f71-\u0f7e\u0f80-\u0f84\u0f86\u0f87\u0f90-\u0f97\u0f99-\u0fbc\u0fc6\u102d-\u1030\u1032-\u1037\u1039\u103a\u103d\u103e\u1058\u1059\u105e-\u1060\u1071-\u1074\u1082\u1085\u1086\u108d\u109d\u135f\u1712-\u1714\u1732-\u1734\u1752\u1753\u1772\u1773\u17b7-\u17bd\u17c6\u17c9-\u17d3\u17dd\u180b-\u180d\u18a9\u1920-\u1922\u1927\u1928\u1932\u1939-\u193b\u1a17\u1a18\u1a56\u1a58-\u1a5e\u1a60\u1a62\u1a65-\u1a6c\u1a73-\u1a7c\u1a7f\u1b00-\u1b03\u1b34\u1b36-\u1b3a\u1b3c\u1b42\u1b6b-\u1b73\u1b80\u1b81\u1ba2-\u1ba5\u1ba8\u1ba9\u1c2c-\u1c33\u1c36\u1c37\u1cd0-\u1cd2\u1cd4-\u1ce0\u1ce2-\u1ce8\u1ced\u1dc0-\u1de6\u1dfd-\u1dff\u200c\u200d\u20d0-\u20f0\u2cef-\u2cf1\u2de0-\u2dff\u302a-\u302f\u3099\u309a\ua66f-\ua672\ua67c\ua67d\ua6f0\ua6f1\ua802\ua806\ua80b\ua825\ua826\ua8c4\ua8e0-\ua8f1\ua926-\ua92d\ua947-\ua951\ua980-\ua982\ua9b3\ua9b6-\ua9b9\ua9bc\uaa29-\uaa2e\uaa31\uaa32\uaa35\uaa36\uaa43\uaa4c\uaab0\uaab2-\uaab4\uaab7\uaab8\uaabe\uaabf\uaac1\uabe5\uabe8\uabed\udc00-\udfff\ufb1e\ufe00-\ufe0f\ufe20-\ufe26\uff9e\uff9f]/;function Cr(r){return r.charCodeAt(0)>=768&&Ho.test(r)}o(Cr,"isExtendingChar");function Ui(r,i,u){for(;(u<0?i>0:iu?-1:1;;){if(i==u)return i;var p=(i+u)/2,g=a<0?Math.ceil(p):Math.floor(p);if(g==i)return r(g)?i:u;r(g)?u=g:i=g+a}}o(pn,"findFirst");function zn(r,i,u,a){if(!r)return a(i,u,"ltr",0);for(var p=!1,g=0;gi||i==u&&y.to==i)&&(a(Math.max(y.from,i),Math.min(y.to,u),y.level==1?"rtl":"ltr",g),p=!0)}p||a(i,u,"ltr")}o(zn,"iterateBidiSections");var Si=null;function Ci(r,i,u){var a;Si=null;for(var p=0;pi)return p;g.to==i&&(g.from!=g.to&&u=="before"?a=p:Si=p),g.from==i&&(g.from!=g.to&&u!="before"?a=p:Si=p)}return a??Si}o(Ci,"getBidiPartAt");var $n=function(){var r="bbbbbbbbbtstwsbbbbbbbbbbbbbbssstwNN%%%NNNNNN,N,N1111111111NNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNNNLLLLLLLLLLLLLLLLLLLLLLLLLLNNNNbbbbbbsbbbbbbbbbbbbbbbbbbbbbbbbbb,N%%%%NNNNLNNNNN%%11NLNNN1LNNNNNLLLLLLLLLLLLLLLLLLLLLLLNLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLLN",i="nnnnnnNNr%%r,rNNmmmmmmmmmmmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmmmmmmmmmmmmmmmnnnnnnnnnn%nnrrrmrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrmmmmmmmnNmmmmmmrrmmNmmmmrr1111111111";function u(E){return E<=247?r.charAt(E):1424<=E&&E<=1524?"R":1536<=E&&E<=1785?i.charAt(E-1536):1774<=E&&E<=2220?"r":8192<=E&&E<=8203?"w":E==8204?"b":"L"}o(u,"charType");var a=/[\u0590-\u05f4\u0600-\u06ff\u0700-\u08ac]/,p=/[stwN]/,g=/[LRr]/,y=/[Lb1n]/,S=/[1n]/;function b(E,M,D){this.level=E,this.from=M,this.to=D}return o(b,"BidiSpan"),function(E,M){var D=M=="ltr"?"L":"R";if(E.length==0||M=="ltr"&&!a.test(E))return!1;for(var V=E.length,$=[],te=0;te-1&&(a[i]=p.slice(0,g).concat(p.slice(g+1)))}}}o(he,"off");function Te(r,i){var u=ee(r,i);if(!!u.length)for(var a=Array.prototype.slice.call(arguments,2),p=0;p0}o(Ft,"hasHandler");function Wr(r){r.prototype.on=function(i,u){H(this,i,u)},r.prototype.off=function(i,u){he(this,i,u)}}o(Wr,"eventMixin");function or(r){r.preventDefault?r.preventDefault():r.returnValue=!1}o(or,"e_preventDefault");function li(r){r.stopPropagation?r.stopPropagation():r.cancelBubble=!0}o(li,"e_stopPropagation");function ds(r){return r.defaultPrevented!=null?r.defaultPrevented:r.returnValue==!1}o(ds,"e_defaultPrevented");function lo(r){or(r),li(r)}o(lo,"e_stop");function bi(r){return r.target||r.srcElement}o(bi,"e_target");function el(r){var i=r.which;return i==null&&(r.button&1?i=1:r.button&2?i=3:r.button&4&&(i=2)),I&&r.ctrlKey&&i==1&&(i=3),i}o(el,"e_button");var hs=function(){if(c&&v<9)return!1;var r=_e("div");return"draggable"in r||"dragDrop"in r}(),dn;function id(r){if(dn==null){var i=_e("span","\u200B");tt(r,_e("span",[i,document.createTextNode("x")])),r.firstChild.offsetHeight!=0&&(dn=i.offsetWidth<=1&&i.offsetHeight>2&&!(c&&v<8))}var u=dn?_e("span","\u200B"):_e("span","\xA0",null,"display: inline-block; width: 1px; margin-right: -1px");return u.setAttribute("cm-text",""),u}o(id,"zeroWidthElement");var tl;function Qf(r){if(tl!=null)return tl;var i=tt(r,document.createTextNode("A\u062EA")),u=We(i,0,1).getBoundingClientRect(),a=We(i,1,2).getBoundingClientRect();return Ve(r),!u||u.left==u.right?!1:tl=a.right-u.right<3}o(Qf,"hasBadBidiRects");var rl=` b`.split(/\n/).length!=3?function(r){for(var i=0,u=[],a=r.length;i<=a;){var p=r.indexOf(` `,i);p==-1&&(p=r.length);var g=r.slice(i,r.charAt(p-1)=="\r"?p-1:p),y=g.indexOf("\r");y!=-1?(u.push(g.slice(0,y)),i+=y+1):(u.push(g),i=p+1)}return u}:function(r){return r.split(/\r\n?|\n/)},od=window.getSelection?function(r){try{return r.selectionStart!=r.selectionEnd}catch(i){return!1}}:function(r){var i;try{i=r.ownerDocument.selection.createRange()}catch(u){}return!i||i.parentElement()!=r?!1:i.compareEndPoints("StartToEnd",i)!=0},Zf=function(){var r=_e("div");return"oncopy"in r?!0:(r.setAttribute("oncopy","return;"),typeof r.oncopy=="function")}(),wu=null;function sd(r){if(wu!=null)return wu;var i=tt(r,_e("span","x")),u=i.getBoundingClientRect(),a=We(i,0,1).getBoundingClientRect();return wu=Math.abs(u.left-a.left)>1}o(sd,"hasBadZoomedRects");var zl={},ms={};function ld(r,i){arguments.length>2&&(i.dependencies=Array.prototype.slice.call(arguments,2)),zl[r]=i}o(ld,"defineMode");function Jf(r,i){ms[r]=i}o(Jf,"defineMIME");function nl(r){if(typeof r=="string"&&ms.hasOwnProperty(r))r=ms[r];else if(r&&typeof r.name=="string"&&ms.hasOwnProperty(r.name)){var i=ms[r.name];typeof i=="string"&&(i={name:i}),r=si(i,r),r.name=i.name}else{if(typeof r=="string"&&/^[\w\-]+\/[\w\-]+\+xml$/.test(r))return nl("application/xml");if(typeof r=="string"&&/^[\w\-]+\/[\w\-]+\+json$/.test(r))return nl("application/json")}return typeof r=="string"?{name:r}:r||{name:"null"}}o(nl,"resolveMode");function xu(r,i){i=nl(i);var u=zl[i.name];if(!u)return xu(r,"text/plain");var a=u(r,i);if(Wo.hasOwnProperty(i.name)){var p=Wo[i.name];for(var g in p)!p.hasOwnProperty(g)||(a.hasOwnProperty(g)&&(a["_"+g]=a[g]),a[g]=p[g])}if(a.name=i.name,i.helperType&&(a.helperType=i.helperType),i.modeProps)for(var y in i.modeProps)a[y]=i.modeProps[y];return a}o(xu,"getMode");var Wo={};function ad(r,i){var u=Wo.hasOwnProperty(r)?Wo[r]:Wo[r]={};Qt(i,u)}o(ad,"extendMode");function Uo(r,i){if(i===!0)return i;if(r.copyState)return r.copyState(i);var u={};for(var a in i){var p=i[a];p instanceof Array&&(p=p.concat([])),u[a]=p}return u}o(Uo,"copyState");function $l(r,i){for(var u;r.innerMode&&(u=r.innerMode(i),!(!u||u.mode==r));)i=u.state,r=u.mode;return u||{mode:r,state:i}}o($l,"innerMode");function ec(r,i,u){return r.startState?r.startState(i,u):!0}o(ec,"startState");var jt=o(function(r,i,u){this.pos=this.start=0,this.string=r,this.tabSize=i||8,this.lastColumnPos=this.lastColumnValue=0,this.lineStart=0,this.lineOracle=u},"StringStream");jt.prototype.eol=function(){return this.pos>=this.string.length},jt.prototype.sol=function(){return this.pos==this.lineStart},jt.prototype.peek=function(){return this.string.charAt(this.pos)||void 0},jt.prototype.next=function(){if(this.posi},jt.prototype.eatSpace=function(){for(var r=this.pos;/[\s\u00a0]/.test(this.string.charAt(this.pos));)++this.pos;return this.pos>r},jt.prototype.skipToEnd=function(){this.pos=this.string.length},jt.prototype.skipTo=function(r){var i=this.string.indexOf(r,this.pos);if(i>-1)return this.pos=i,!0},jt.prototype.backUp=function(r){this.pos-=r},jt.prototype.column=function(){return this.lastColumnPos0?null:(g&&i!==!1&&(this.pos+=g[0].length),g)}},jt.prototype.current=function(){return this.string.slice(this.start,this.pos)},jt.prototype.hideFirstChars=function(r,i){this.lineStart+=r;try{return i()}finally{this.lineStart-=r}},jt.prototype.lookAhead=function(r){var i=this.lineOracle;return i&&i.lookAhead(r)},jt.prototype.baseToken=function(){var r=this.lineOracle;return r&&r.baseToken(this.pos)};function Me(r,i){if(i-=r.first,i<0||i>=r.size)throw new Error("There is no line "+(i+r.first)+" in the document.");for(var u=r;!u.lines;)for(var a=0;;++a){var p=u.children[a],g=p.chunkSize();if(i=r.first&&iu?ue(u,Me(r,u).text.length):ud(i,Me(r,i.line).text.length)}o(He,"clipPos");function ud(r,i){var u=r.ch;return u==null||u>i?ue(r.line,i):u<0?ue(r.line,0):r}o(ud,"clipToLen");function ql(r,i){for(var u=[],a=0;athis.maxLookAhead&&(this.maxLookAhead=r),i},ui.prototype.baseToken=function(r){if(!this.baseTokens)return null;for(;this.baseTokens[this.baseTokenPos]<=r;)this.baseTokenPos+=2;var i=this.baseTokens[this.baseTokenPos+1];return{type:i&&i.replace(/( |^)overlay .*/,""),size:this.baseTokens[this.baseTokenPos]-r}},ui.prototype.nextLine=function(){this.line++,this.maxLookAhead>0&&this.maxLookAhead--},ui.fromSaved=function(r,i,u){return i instanceof uo?new ui(r,Uo(r.mode,i.state),u,i.lookAhead):new ui(r,Uo(r.mode,i),u)},ui.prototype.save=function(r){var i=r!==!1?Uo(this.doc.mode,this.state):this.state;return this.maxLookAhead>0?new uo(i,this.maxLookAhead):i};function tc(r,i,u,a){var p=[r.state.modeGen],g={};Gl(r,i.text,r.doc.mode,u,function(E,M){return p.push(E,M)},g,a);for(var y=u.state,S=o(function(E){u.baseTokens=p;var M=r.state.overlays[E],D=1,V=0;u.state=!0,Gl(r,i.text,M.mode,u,function($,te){for(var oe=D;V<$;){var de=p[D];de>$&&p.splice(D,1,$,p[D+1],de),D+=2,V=Math.min($,de)}if(!!te)if(M.opaque)p.splice(oe,D-oe,$,"overlay "+te),D=oe+2;else for(;oer.options.maxHighlightLength&&Uo(r.doc.mode,a.state),g=tc(r,i,a);p&&(a.state=p),i.stateAfter=a.save(!p),i.styles=g.styles,g.classes?i.styleClasses=g.classes:i.styleClasses&&(i.styleClasses=null),u===r.doc.highlightFrontier&&(r.doc.modeFrontier=Math.max(r.doc.modeFrontier,++r.doc.highlightFrontier))}return i.styles}o(_u,"getLineStyles");function $o(r,i,u){var a=r.doc,p=r.display;if(!a.mode.startState)return new ui(a,!0,i);var g=Yl(r,i,u),y=g>a.first&&Me(a,g-1).stateAfter,S=y?ui.fromSaved(a,y,g):new ui(a,ec(a.mode),g);return a.iter(g,i,function(b){vs(r,b.text,S);var E=S.line;b.stateAfter=E==i-1||E%5==0||E>=p.viewFrom&&Ei.start)return g}throw new Error("Mode "+r.name+" failed to advance stream.")}o(ol,"readToken");var Vl=o(function(r,i,u){this.start=r.start,this.end=r.pos,this.string=r.current(),this.type=i||null,this.state=u},"Token");function Kl(r,i,u,a){var p=r.doc,g=p.mode,y;i=He(p,i);var S=Me(p,i.line),b=$o(r,i.line,u),E=new jt(S.text,r.options.tabSize,b),M;for(a&&(M=[]);(a||E.posr.options.maxHighlightLength?(S=!1,y&&vs(r,i,a,M.pos),M.pos=i.length,D=null):D=fo(ol(u,M,a.state,V),g),V){var $=V[0].name;$&&(D="m-"+(D?$+" "+D:$))}if(!S||E!=D){for(;by;--S){if(S<=g.first)return g.first;var b=Me(g,S-1),E=b.stateAfter;if(E&&(!u||S+(E instanceof uo?E.lookAhead:0)<=g.modeFrontier))return S;var M=_t(b.text,null,r.options.tabSize);(p==null||a>M)&&(p=S-1,a=M)}return p}o(Yl,"findStartLine");function rc(r,i){if(r.modeFrontier=Math.min(r.modeFrontier,i),!(r.highlightFrontieru;a--){var p=Me(r,a).stateAfter;if(p&&(!(p instanceof uo)||a+p.lookAhead=i:g.to>i);(a||(a=[])).push(new co(y,g.from,b?null:g.to))}}return a}o(fd,"markedSpansBefore");function cd(r,i,u){var a;if(r)for(var p=0;p=i:g.to>i);if(S||g.from==i&&y.type=="bookmark"&&(!u||g.marker.insertLeft)){var b=g.from==null||(y.inclusiveLeft?g.from<=i:g.from0&&S)for(var Ne=0;Ne0)){var M=[b,1],D=ze(E.from,S.from),V=ze(E.to,S.to);(D<0||!y.inclusiveLeft&&!D)&&M.push({from:E.from,to:S.from}),(V>0||!y.inclusiveRight&&!V)&&M.push({from:S.to,to:E.to}),p.splice.apply(p,M),b+=M.length-3}}return p}o(Nu,"removeReadOnlyRanges");function lc(r){var i=r.markedSpans;if(!!i){for(var u=0;ui)&&(!a||Lu(a,g.marker)<0)&&(a=g.marker)}return a}o(Re,"collapsedSpanAround");function sl(r,i,u,a,p){var g=Me(r,i),y=_i&&g.markedSpans;if(y)for(var S=0;S=0&&D<=0||M<=0&&D>=0)&&(M<=0&&(b.marker.inclusiveRight&&p.inclusiveLeft?ze(E.to,u)>=0:ze(E.to,u)>0)||M>=0&&(b.marker.inclusiveRight&&p.inclusiveLeft?ze(E.from,a)<=0:ze(E.from,a)<0)))return!0}}}o(sl,"conflictingCollapsedRange");function vr(r){for(var i;i=Nt(r);)r=i.find(-1,!0).line;return r}o(vr,"visualLine");function Pu(r){for(var i;i=P(r);)r=i.find(1,!0).line;return r}o(Pu,"visualLineEnd");function ye(r){for(var i,u;i=P(r);)r=i.find(1,!0).line,(u||(u=[])).push(r);return u}o(ye,"visualLineContinued");function jo(r,i){var u=Me(r,i),a=vr(u);return u==a?i:vt(a)}o(jo,"visualLineNo");function pd(r,i){if(i>r.lastLine())return i;var u=Me(r,i),a;if(!qt(r,u))return i;for(;a=P(u);)u=a.find(1,!0).line;return vt(u)+1}o(pd,"visualLineEndNo");function qt(r,i){var u=_i&&i.markedSpans;if(u){for(var a=void 0,p=0;pi.maxLineLength&&(i.maxLineLength=p,i.maxLine=a)})}o(zi,"findMaxLine");var Ce=o(function(r,i,u){this.text=r,ac(this,i),this.height=u?u(this):1},"Line");Ce.prototype.lineNo=function(){return vt(this)},Wr(Ce);function ta(r,i,u,a){r.text=i,r.stateAfter&&(r.stateAfter=null),r.styles&&(r.styles=null),r.order!=null&&(r.order=null),lc(r),ac(r,u);var p=a?a(r):1;p!=r.height&&ai(r,p)}o(ta,"updateLine");function Ou(r){r.parent=null,lc(r)}o(Ou,"cleanUpLine");var nt={},uc={};function fi(r,i){if(!r||/^\s*$/.test(r))return null;var u=i.addModeClass?uc:nt;return u[r]||(u[r]=r.replace(/\S+/g,"cm-$&"))}o(fi,"interpretTokenStyle");function ll(r,i){var u=St("span",null,null,C?"padding-right: .1px":null),a={pre:St("pre",[u],"CodeMirror-line"),content:u,col:0,pos:0,cm:r,trailingSpace:!1,splitSpaces:r.getOption("lineWrapping")};i.measure={};for(var p=0;p<=(i.rest?i.rest.length:0);p++){var g=p?i.rest[p-1]:i.line,y=void 0;a.pos=0,a.addToken=ra,Qf(r.display.measure)&&(y=Mn(g,r.doc.direction))&&(a.addToken=dd(a.addToken,y)),a.map=[];var S=i!=r.display.externalMeasured&&vt(g);Vt(g,a,_u(r,g,S)),g.styleClasses&&(g.styleClasses.bgClass&&(a.bgClass=nr(g.styleClasses.bgClass,a.bgClass||"")),g.styleClasses.textClass&&(a.textClass=nr(g.styleClasses.textClass,a.textClass||""))),a.map.length==0&&a.map.push(0,0,a.content.appendChild(id(r.display.measure))),p==0?(i.measure.map=a.map,i.measure.cache={}):((i.measure.maps||(i.measure.maps=[])).push(a.map),(i.measure.caches||(i.measure.caches=[])).push({}))}if(C){var b=a.content.lastChild;(/\bcm-tab\b/.test(b.className)||b.querySelector&&b.querySelector(".cm-tab"))&&(a.content.className="cm-tab-wrap-hack")}return Te(r,"renderLine",r,i.line,a.pre),a.pre.className&&(a.textClass=nr(a.pre.className,a.textClass||"")),a}o(ll,"buildLineContent");function Ur(r){var i=_e("span","\u2022","cm-invalidchar");return i.title="\\u"+r.charCodeAt(0).toString(16),i.setAttribute("aria-label",i.title),i}o(Ur,"defaultSpecialCharPlaceholder");function ra(r,i,u,a,p,g,y){if(!!i){var S=r.splitSpaces?jn(i,r.trailingSpace):i,b=r.cm.state.specialChars,E=!1,M;if(!b.test(i))r.col+=i.length,M=document.createTextNode(S),r.map.push(r.pos,r.pos+i.length,M),c&&v<9&&(E=!0),r.pos+=i.length;else{M=document.createDocumentFragment();for(var D=0;;){b.lastIndex=D;var V=b.exec(i),$=V?V.index-D:i.length-D;if($){var te=document.createTextNode(S.slice(D,D+$));c&&v<9?M.appendChild(_e("span",[te])):M.appendChild(te),r.map.push(r.pos,r.pos+$,te),r.col+=$,r.pos+=$}if(!V)break;D+=$+1;var oe=void 0;if(V[0]==" "){var de=r.cm.options.tabSize,ge=de-r.col%de;oe=M.appendChild(_e("span",Yt(ge),"cm-tab")),oe.setAttribute("role","presentation"),oe.setAttribute("cm-text"," "),r.col+=ge}else V[0]=="\r"||V[0]==` -`?(oe=M.appendChild(_e("span",V[0]=="\r"?"\u240D":"\u2424","cm-invalidchar")),oe.setAttribute("cm-text",V[0]),r.col+=1):(oe=r.cm.options.specialCharPlaceholder(V[0]),oe.setAttribute("cm-text",V[0]),c&&v<9?M.appendChild(_e("span",[oe])):M.appendChild(oe),r.col+=1);r.map.push(r.pos,r.pos+1,oe),r.pos++}}if(r.trailingSpace=S.charCodeAt(i.length-1)==32,u||a||p||E||g||y){var be=u||"";a&&(be+=a),p&&(be+=p);var ve=_e("span",[M],be,g);if(y)for(var Ne in y)y.hasOwnProperty(Ne)&&Ne!="style"&&Ne!="class"&&ve.setAttribute(Ne,y[Ne]);return r.content.appendChild(ve)}r.content.appendChild(M)}}o(ra,"buildToken");function jn(r,i){if(r.length>1&&!/ /.test(r))return r;for(var u=i,a="",p=0;pE&&D.from<=E));V++);if(D.to>=M)return r(u,a,p,g,y,S,b);r(u,a.slice(0,D.to-E),p,g,null,S,b),g=null,a=a.slice(D.to-E),E=D.to}}}o(dd,"buildTokenBadBidi");function Mu(r,i,u,a){var p=!a&&u.widgetNode;p&&r.map.push(r.pos,r.pos+i,p),!a&&r.cm.display.input.needsContentAttribute&&(p||(p=r.content.appendChild(document.createElement("span"))),p.setAttribute("cm-marker",u.id)),p&&(r.cm.display.input.setUneditable(p),r.content.appendChild(p)),r.pos+=i,r.trailingSpace=!1}o(Mu,"buildCollapsedSpan");function Vt(r,i,u){var a=r.markedSpans,p=r.text,g=0;if(!a){for(var y=1;yb||it.collapsed&&De.to==b&&De.from==b)){if(De.to!=null&&De.to!=b&&$>De.to&&($=De.to,oe=""),it.className&&(te+=" "+it.className),it.css&&(V=(V?V+";":"")+it.css),it.startStyle&&De.from==b&&(de+=" "+it.startStyle),it.endStyle&&De.to==$&&(Ne||(Ne=[])).push(it.endStyle,De.to),it.title&&((be||(be={})).title=it.title),it.attributes)for(var Tt in it.attributes)(be||(be={}))[Tt]=it.attributes[Tt];it.collapsed&&(!ge||Lu(ge.marker,it)<0)&&(ge=De)}else De.from>b&&$>De.from&&($=De.from)}if(Ne)for(var br=0;br=S)break;for(var Sn=Math.min(S,$);;){if(M){var Cn=b+M.length;if(!ge){var dr=Cn>Sn?M.slice(0,Sn-b):M;i.addToken(i,dr,D?D+te:te,de,b+dr.length==$?oe:"",V,be)}if(Cn>=Sn){M=M.slice(Sn-b),b=Sn;break}b=Cn,de=""}M=p.slice(g,g=u[E++]),D=fi(u[E++],i.cm.options)}}}o(Vt,"insertLineContent");function xs(r,i,u){this.line=i,this.rest=ye(i),this.size=this.rest?vt(Se(this.rest))-u+1:1,this.node=this.text=null,this.hidden=qt(r,i)}o(xs,"LineView");function qo(r,i,u){for(var a=[],p,g=i;g2&&g.push((b.bottom+E.top)/2-u.top)}}g.push(u.bottom-u.top)}}o(cc,"ensureLineHeights");function Wu(r,i,u){if(r.line==i)return{map:r.measure.map,cache:r.measure.cache};for(var a=0;au)return{map:r.measure.maps[p],cache:r.measure.caches[p],before:!0}}o(Wu,"mapFromLineView");function gd(r,i){i=vr(i);var u=vt(i),a=r.display.externalMeasured=new xs(r.doc,i,u);a.lineN=u;var p=a.built=ll(r,a);return a.text=p.pre,tt(r.display.lineMeasure,p.pre),a}o(gd,"updateExternalMeasurement");function Uu(r,i,u,a){return Ti(r,qn(r,i),u,a)}o(Uu,"measureChar");function la(r,i){if(i>=r.display.viewFrom&&i=u.lineN&&ii)&&(g=b-S,p=g-1,i>=b&&(y="right")),p!=null){if(a=r[E+2],S==b&&u==(a.insertLeft?"left":"right")&&(y=u),u=="left"&&p==0)for(;E&&r[E-2]==r[E-3]&&r[E-1].insertLeft;)a=r[(E-=3)+2],y="left";if(u=="right"&&p==b-S)for(;E=0&&(u=r[p]).left==u.right;p--);return u}o(dc,"getUsefulRect");function Vi(r,i,u,a){var p=aa(i.map,u,a),g=p.node,y=p.start,S=p.end,b=p.collapse,E;if(g.nodeType==3){for(var M=0;M<4;M++){for(;y&&Cr(i.line.text.charAt(p.coverStart+y));)--y;for(;p.coverStart+S0&&(b=a="right");var D;r.options.lineWrapping&&(D=g.getClientRects()).length>1?E=D[a=="right"?D.length-1:0]:E=g.getBoundingClientRect()}if(c&&v<9&&!y&&(!E||!E.left&&!E.right)){var V=g.parentNode.getClientRects()[0];V?E={left:V.left,right:V.left+ua(r.display),top:V.top,bottom:V.bottom}:E=pc}for(var $=E.top-i.rect.top,te=E.bottom-i.rect.top,oe=($+te)/2,de=i.view.measure.heights,ge=0;ge=a.text.length?(b=a.text.length,E="before"):b<=0&&(b=0,E="after"),!S)return y(E=="before"?b-1:b,E=="before");function M(te,oe,de){var ge=S[oe],be=ge.level==1;return y(de?te-1:te,be!=de)}o(M,"getBidi");var D=Ci(S,b,E),V=Si,$=M(b,D,E=="before");return V!=null&&($.other=M(b,V,E!="before")),$}o(Vn,"cursorCoords");function zu(r,i){var u=0;i=He(r.doc,i),r.options.lineWrapping||(u=ua(r.display)*i.ch);var a=Me(r.doc,i.line),p=hn(a)+Ss(r.display);return{left:u,right:u,top:p,bottom:p+a.height}}o(zu,"estimateCoords");function vn(r,i,u,a,p){var g=ue(r,i,u);return g.xRel=p,a&&(g.outside=a),g}o(vn,"PosWithInfo");function q(r,i,u){var a=r.doc;if(u+=r.display.viewOffset,u<0)return vn(a.first,0,null,-1,-1);var p=ao(a,u),g=a.first+a.size-1;if(p>g)return vn(a.first+a.size-1,Me(a,g).text.length,null,1,1);i<0&&(i=0);for(var y=Me(a,p);;){var S=Qe(r,y,p,i,u),b=Re(y,S.ch+(S.xRel>0||S.outside>0?1:0));if(!b)return S;var E=b.find(1);if(E.line==p)return E;y=Me(a,p=E.line)}}o(q,"coordsChar");function re(r,i,u,a){a-=gn(i);var p=i.text.length,g=pn(function(y){return Ti(r,u,y-1).bottom<=a},p,0);return p=pn(function(y){return Ti(r,u,y).top>a},g,p),{begin:g,end:p}}o(re,"wrappedLineExtent");function Q(r,i,u,a){u||(u=qn(r,i));var p=ci(r,i,Ti(r,u,a),"line").top;return re(r,i,u,p)}o(Q,"wrappedLineExtentChar");function Pe(r,i,u,a){return r.bottom<=u?!1:r.top>u?!0:(a?r.left:r.right)>i}o(Pe,"boxIsAfter");function Qe(r,i,u,a,p){p-=hn(i);var g=qn(r,i),y=gn(i),S=0,b=i.text.length,E=!0,M=Mn(i,r.doc.direction);if(M){var D=(r.options.lineWrapping?Mr:bt)(r,i,u,g,M,a,p);E=D.level!=1,S=E?D.from:D.to-1,b=E?D.to:D.from-1}var V=null,$=null,te=pn(function(Ie){var De=Ti(r,g,Ie);return De.top+=y,De.bottom+=y,Pe(De,a,p,!1)?(De.top<=p&&De.left<=a&&(V=Ie,$=De),!0):!1},S,b),oe,de,ge=!1;if($){var be=a-$.left<$.right-a,ve=be==E;te=V+(ve?0:1),de=ve?"after":"before",oe=be?$.left:$.right}else{!E&&(te==b||te==S)&&te++,de=te==0?"after":te==i.text.length?"before":Ti(r,g,te-(E?1:0)).bottom+y<=p==E?"after":"before";var Ne=Vn(r,ue(u,te,de),"line",i,g);oe=Ne.left,ge=p=Ne.bottom?1:0}return te=Ui(i.text,te,1),vn(u,te,de,ge,a-oe)}o(Qe,"coordsCharInner");function bt(r,i,u,a,p,g,y){var S=pn(function(D){var V=p[D],$=V.level!=1;return Pe(Vn(r,ue(u,$?V.to:V.from,$?"before":"after"),"line",i,a),g,y,!0)},0,p.length-1),b=p[S];if(S>0){var E=b.level!=1,M=Vn(r,ue(u,E?b.from:b.to,E?"after":"before"),"line",i,a);Pe(M,g,y,!0)&&M.top>y&&(b=p[S-1])}return b}o(bt,"coordsBidiPart");function Mr(r,i,u,a,p,g,y){var S=re(r,i,a,y),b=S.begin,E=S.end;/\s/.test(i.text.charAt(E-1))&&E--;for(var M=null,D=null,V=0;V=E||$.to<=b)){var te=$.level!=1,oe=Ti(r,a,te?Math.min(E,$.to)-1:Math.max(b,$.from)).right,de=oede)&&(M=$,D=de)}}return M||(M=p[p.length-1]),M.fromE&&(M={from:M.from,to:E,level:M.level}),M}o(Mr,"coordsBidiPartWrapped");var mt;function Cs(r){if(r.cachedTextHeight!=null)return r.cachedTextHeight;if(mt==null){mt=_e("pre",null,"CodeMirror-line-like");for(var i=0;i<49;++i)mt.appendChild(document.createTextNode("x")),mt.appendChild(_e("br"));mt.appendChild(document.createTextNode("x"))}tt(r.measure,mt);var u=mt.offsetHeight/50;return u>3&&(r.cachedTextHeight=u),Ve(r.measure),u||1}o(Cs,"textHeight");function ua(r){if(r.cachedCharWidth!=null)return r.cachedCharWidth;var i=_e("span","xxxxxxxxxx"),u=_e("pre",[i],"CodeMirror-line-like");tt(r.measure,u);var a=i.getBoundingClientRect(),p=(a.right-a.left)/10;return p>2&&(r.cachedCharWidth=p),p||10}o(ua,"charWidth");function Kn(r){for(var i=r.display,u={},a={},p=i.gutters.clientLeft,g=i.gutters.firstChild,y=0;g;g=g.nextSibling,++y){var S=r.display.gutterSpecs[y].className;u[S]=g.offsetLeft+g.clientLeft+p,a[S]=g.clientWidth}return{fixedPos:fa(i),gutterTotalWidth:i.gutters.offsetWidth,gutterLeft:u,gutterWidth:a,wrapperWidth:i.wrapper.clientWidth}}o(Kn,"getDimensions");function fa(r){return r.scroller.getBoundingClientRect().left-r.sizer.getBoundingClientRect().left}o(fa,"compensateForHScroll");function Fm(r){var i=Cs(r.display),u=r.options.lineWrapping,a=u&&Math.max(5,r.display.scroller.clientWidth/ua(r.display)-3);return function(p){if(qt(r.doc,p))return 0;var g=0;if(p.widgets)for(var y=0;y0&&(E=Me(r.doc,b.line).text).length==b.ch){var M=_t(E,E.length,r.options.tabSize)-E.length;b=ue(b.line,Math.max(0,Math.round((g-sa(r.display).left)/ua(r.display))-M))}return b}o(po,"posFromMouse");function ho(r,i){if(i>=r.display.viewTo||(i-=r.display.viewFrom,i<0))return null;for(var u=r.display.view,a=0;ai)&&(p.updateLineNumbers=i),r.curOp.viewChanged=!0,i>=p.viewTo)_i&&jo(r.doc,i)p.viewFrom?Xo(r):(p.viewFrom+=a,p.viewTo+=a);else if(i<=p.viewFrom&&u>=p.viewTo)Xo(r);else if(i<=p.viewFrom){var g=fl(r,u,u+a,1);g?(p.view=p.view.slice(g.index),p.viewFrom=g.lineN,p.viewTo+=a):Xo(r)}else if(u>=p.viewTo){var y=fl(r,i,i,-1);y?(p.view=p.view.slice(0,y.index),p.viewTo=y.lineN):Xo(r)}else{var S=fl(r,i,i,-1),b=fl(r,u,u+a,1);S&&b?(p.view=p.view.slice(0,S.index).concat(qo(r,S.lineN,b.lineN)).concat(p.view.slice(b.index)),p.viewTo+=a):Xo(r)}var E=p.externalMeasured;E&&(u=p.lineN&&i=a.viewTo)){var g=a.view[ho(r,i)];if(g.node!=null){var y=g.changes||(g.changes=[]);ut(y,u)==-1&&y.push(u)}}}o(Es,"regLineChange");function Xo(r){r.display.viewFrom=r.display.viewTo=r.doc.first,r.display.view=[],r.display.viewOffset=0}o(Xo,"resetView");function fl(r,i,u,a){var p=ho(r,i),g,y=r.display.view;if(!_i||u==r.doc.first+r.doc.size)return{index:p,lineN:u};for(var S=r.display.viewFrom,b=0;b0){if(p==y.length-1)return null;g=S+y[p].size-i,p++}else g=S-i;i+=g,u+=g}for(;jo(r.doc,u)!=u;){if(p==(a<0?0:y.length-1))return null;u+=a*y[p-(a<0?1:0)].size,p+=a}return{index:p,lineN:u}}o(fl,"viewCuttingPoint");function Iy(r,i,u){var a=r.display,p=a.view;p.length==0||i>=a.viewTo||u<=a.viewFrom?(a.view=qo(r,i,u),a.viewFrom=i):(a.viewFrom>i?a.view=qo(r,i,a.viewFrom).concat(a.view):a.viewFromu&&(a.view=a.view.slice(0,ho(r,u)))),a.viewTo=u}o(Iy,"adjustView");function Bm(r){for(var i=r.display.view,u=0,a=0;a=r.display.viewTo||S.to().line1&&!/ /.test(r))return r;for(var u=i,a="",p=0;pE&&D.from<=E));V++);if(D.to>=M)return r(u,a,p,g,y,S,b);r(u,a.slice(0,D.to-E),p,g,null,S,b),g=null,a=a.slice(D.to-E),E=D.to}}}o(dd,"buildTokenBadBidi");function Mu(r,i,u,a){var p=!a&&u.widgetNode;p&&r.map.push(r.pos,r.pos+i,p),!a&&r.cm.display.input.needsContentAttribute&&(p||(p=r.content.appendChild(document.createElement("span"))),p.setAttribute("cm-marker",u.id)),p&&(r.cm.display.input.setUneditable(p),r.content.appendChild(p)),r.pos+=i,r.trailingSpace=!1}o(Mu,"buildCollapsedSpan");function Vt(r,i,u){var a=r.markedSpans,p=r.text,g=0;if(!a){for(var y=1;yb||it.collapsed&&De.to==b&&De.from==b)){if(De.to!=null&&De.to!=b&&$>De.to&&($=De.to,oe=""),it.className&&(te+=" "+it.className),it.css&&(V=(V?V+";":"")+it.css),it.startStyle&&De.from==b&&(de+=" "+it.startStyle),it.endStyle&&De.to==$&&(Ne||(Ne=[])).push(it.endStyle,De.to),it.title&&((be||(be={})).title=it.title),it.attributes)for(var Tt in it.attributes)(be||(be={}))[Tt]=it.attributes[Tt];it.collapsed&&(!ge||Lu(ge.marker,it)<0)&&(ge=De)}else De.from>b&&$>De.from&&($=De.from)}if(Ne)for(var br=0;br=S)break;for(var Sn=Math.min(S,$);;){if(M){var Cn=b+M.length;if(!ge){var dr=Cn>Sn?M.slice(0,Sn-b):M;i.addToken(i,dr,D?D+te:te,de,b+dr.length==$?oe:"",V,be)}if(Cn>=Sn){M=M.slice(Sn-b),b=Sn;break}b=Cn,de=""}M=p.slice(g,g=u[E++]),D=fi(u[E++],i.cm.options)}}}o(Vt,"insertLineContent");function xs(r,i,u){this.line=i,this.rest=ye(i),this.size=this.rest?vt(Se(this.rest))-u+1:1,this.node=this.text=null,this.hidden=qt(r,i)}o(xs,"LineView");function qo(r,i,u){for(var a=[],p,g=i;g2&&g.push((b.bottom+E.top)/2-u.top)}}g.push(u.bottom-u.top)}}o(cc,"ensureLineHeights");function Wu(r,i,u){if(r.line==i)return{map:r.measure.map,cache:r.measure.cache};for(var a=0;au)return{map:r.measure.maps[p],cache:r.measure.caches[p],before:!0}}o(Wu,"mapFromLineView");function gd(r,i){i=vr(i);var u=vt(i),a=r.display.externalMeasured=new xs(r.doc,i,u);a.lineN=u;var p=a.built=ll(r,a);return a.text=p.pre,tt(r.display.lineMeasure,p.pre),a}o(gd,"updateExternalMeasurement");function Uu(r,i,u,a){return Ti(r,qn(r,i),u,a)}o(Uu,"measureChar");function la(r,i){if(i>=r.display.viewFrom&&i=u.lineN&&ii)&&(g=b-S,p=g-1,i>=b&&(y="right")),p!=null){if(a=r[E+2],S==b&&u==(a.insertLeft?"left":"right")&&(y=u),u=="left"&&p==0)for(;E&&r[E-2]==r[E-3]&&r[E-1].insertLeft;)a=r[(E-=3)+2],y="left";if(u=="right"&&p==b-S)for(;E=0&&(u=r[p]).left==u.right;p--);return u}o(dc,"getUsefulRect");function Vi(r,i,u,a){var p=aa(i.map,u,a),g=p.node,y=p.start,S=p.end,b=p.collapse,E;if(g.nodeType==3){for(var M=0;M<4;M++){for(;y&&Cr(i.line.text.charAt(p.coverStart+y));)--y;for(;p.coverStart+S0&&(b=a="right");var D;r.options.lineWrapping&&(D=g.getClientRects()).length>1?E=D[a=="right"?D.length-1:0]:E=g.getBoundingClientRect()}if(c&&v<9&&!y&&(!E||!E.left&&!E.right)){var V=g.parentNode.getClientRects()[0];V?E={left:V.left,right:V.left+ua(r.display),top:V.top,bottom:V.bottom}:E=pc}for(var $=E.top-i.rect.top,te=E.bottom-i.rect.top,oe=($+te)/2,de=i.view.measure.heights,ge=0;ge=a.text.length?(b=a.text.length,E="before"):b<=0&&(b=0,E="after"),!S)return y(E=="before"?b-1:b,E=="before");function M(te,oe,de){var ge=S[oe],be=ge.level==1;return y(de?te-1:te,be!=de)}o(M,"getBidi");var D=Ci(S,b,E),V=Si,$=M(b,D,E=="before");return V!=null&&($.other=M(b,V,E!="before")),$}o(Vn,"cursorCoords");function zu(r,i){var u=0;i=He(r.doc,i),r.options.lineWrapping||(u=ua(r.display)*i.ch);var a=Me(r.doc,i.line),p=hn(a)+Ss(r.display);return{left:u,right:u,top:p,bottom:p+a.height}}o(zu,"estimateCoords");function vn(r,i,u,a,p){var g=ue(r,i,u);return g.xRel=p,a&&(g.outside=a),g}o(vn,"PosWithInfo");function q(r,i,u){var a=r.doc;if(u+=r.display.viewOffset,u<0)return vn(a.first,0,null,-1,-1);var p=ao(a,u),g=a.first+a.size-1;if(p>g)return vn(a.first+a.size-1,Me(a,g).text.length,null,1,1);i<0&&(i=0);for(var y=Me(a,p);;){var S=Qe(r,y,p,i,u),b=Re(y,S.ch+(S.xRel>0||S.outside>0?1:0));if(!b)return S;var E=b.find(1);if(E.line==p)return E;y=Me(a,p=E.line)}}o(q,"coordsChar");function re(r,i,u,a){a-=gn(i);var p=i.text.length,g=pn(function(y){return Ti(r,u,y-1).bottom<=a},p,0);return p=pn(function(y){return Ti(r,u,y).top>a},g,p),{begin:g,end:p}}o(re,"wrappedLineExtent");function Q(r,i,u,a){u||(u=qn(r,i));var p=ci(r,i,Ti(r,u,a),"line").top;return re(r,i,u,p)}o(Q,"wrappedLineExtentChar");function Pe(r,i,u,a){return r.bottom<=u?!1:r.top>u?!0:(a?r.left:r.right)>i}o(Pe,"boxIsAfter");function Qe(r,i,u,a,p){p-=hn(i);var g=qn(r,i),y=gn(i),S=0,b=i.text.length,E=!0,M=Mn(i,r.doc.direction);if(M){var D=(r.options.lineWrapping?Mr:bt)(r,i,u,g,M,a,p);E=D.level!=1,S=E?D.from:D.to-1,b=E?D.to:D.from-1}var V=null,$=null,te=pn(function(Ie){var De=Ti(r,g,Ie);return De.top+=y,De.bottom+=y,Pe(De,a,p,!1)?(De.top<=p&&De.left<=a&&(V=Ie,$=De),!0):!1},S,b),oe,de,ge=!1;if($){var be=a-$.left<$.right-a,ve=be==E;te=V+(ve?0:1),de=ve?"after":"before",oe=be?$.left:$.right}else{!E&&(te==b||te==S)&&te++,de=te==0?"after":te==i.text.length?"before":Ti(r,g,te-(E?1:0)).bottom+y<=p==E?"after":"before";var Ne=Vn(r,ue(u,te,de),"line",i,g);oe=Ne.left,ge=p=Ne.bottom?1:0}return te=Ui(i.text,te,1),vn(u,te,de,ge,a-oe)}o(Qe,"coordsCharInner");function bt(r,i,u,a,p,g,y){var S=pn(function(D){var V=p[D],$=V.level!=1;return Pe(Vn(r,ue(u,$?V.to:V.from,$?"before":"after"),"line",i,a),g,y,!0)},0,p.length-1),b=p[S];if(S>0){var E=b.level!=1,M=Vn(r,ue(u,E?b.from:b.to,E?"after":"before"),"line",i,a);Pe(M,g,y,!0)&&M.top>y&&(b=p[S-1])}return b}o(bt,"coordsBidiPart");function Mr(r,i,u,a,p,g,y){var S=re(r,i,a,y),b=S.begin,E=S.end;/\s/.test(i.text.charAt(E-1))&&E--;for(var M=null,D=null,V=0;V=E||$.to<=b)){var te=$.level!=1,oe=Ti(r,a,te?Math.min(E,$.to)-1:Math.max(b,$.from)).right,de=oede)&&(M=$,D=de)}}return M||(M=p[p.length-1]),M.fromE&&(M={from:M.from,to:E,level:M.level}),M}o(Mr,"coordsBidiPartWrapped");var mt;function Cs(r){if(r.cachedTextHeight!=null)return r.cachedTextHeight;if(mt==null){mt=_e("pre",null,"CodeMirror-line-like");for(var i=0;i<49;++i)mt.appendChild(document.createTextNode("x")),mt.appendChild(_e("br"));mt.appendChild(document.createTextNode("x"))}tt(r.measure,mt);var u=mt.offsetHeight/50;return u>3&&(r.cachedTextHeight=u),Ve(r.measure),u||1}o(Cs,"textHeight");function ua(r){if(r.cachedCharWidth!=null)return r.cachedCharWidth;var i=_e("span","xxxxxxxxxx"),u=_e("pre",[i],"CodeMirror-line-like");tt(r.measure,u);var a=i.getBoundingClientRect(),p=(a.right-a.left)/10;return p>2&&(r.cachedCharWidth=p),p||10}o(ua,"charWidth");function Kn(r){for(var i=r.display,u={},a={},p=i.gutters.clientLeft,g=i.gutters.firstChild,y=0;g;g=g.nextSibling,++y){var S=r.display.gutterSpecs[y].className;u[S]=g.offsetLeft+g.clientLeft+p,a[S]=g.clientWidth}return{fixedPos:fa(i),gutterTotalWidth:i.gutters.offsetWidth,gutterLeft:u,gutterWidth:a,wrapperWidth:i.wrapper.clientWidth}}o(Kn,"getDimensions");function fa(r){return r.scroller.getBoundingClientRect().left-r.sizer.getBoundingClientRect().left}o(fa,"compensateForHScroll");function Fm(r){var i=Cs(r.display),u=r.options.lineWrapping,a=u&&Math.max(5,r.display.scroller.clientWidth/ua(r.display)-3);return function(p){if(qt(r.doc,p))return 0;var g=0;if(p.widgets)for(var y=0;y0&&(E=Me(r.doc,b.line).text).length==b.ch){var M=_t(E,E.length,r.options.tabSize)-E.length;b=ue(b.line,Math.max(0,Math.round((g-sa(r.display).left)/ua(r.display))-M))}return b}o(po,"posFromMouse");function ho(r,i){if(i>=r.display.viewTo||(i-=r.display.viewFrom,i<0))return null;for(var u=r.display.view,a=0;ai)&&(p.updateLineNumbers=i),r.curOp.viewChanged=!0,i>=p.viewTo)_i&&jo(r.doc,i)p.viewFrom?Xo(r):(p.viewFrom+=a,p.viewTo+=a);else if(i<=p.viewFrom&&u>=p.viewTo)Xo(r);else if(i<=p.viewFrom){var g=fl(r,u,u+a,1);g?(p.view=p.view.slice(g.index),p.viewFrom=g.lineN,p.viewTo+=a):Xo(r)}else if(u>=p.viewTo){var y=fl(r,i,i,-1);y?(p.view=p.view.slice(0,y.index),p.viewTo=y.lineN):Xo(r)}else{var S=fl(r,i,i,-1),b=fl(r,u,u+a,1);S&&b?(p.view=p.view.slice(0,S.index).concat(qo(r,S.lineN,b.lineN)).concat(p.view.slice(b.index)),p.viewTo+=a):Xo(r)}var E=p.externalMeasured;E&&(u=p.lineN&&i=a.viewTo)){var g=a.view[ho(r,i)];if(g.node!=null){var y=g.changes||(g.changes=[]);ut(y,u)==-1&&y.push(u)}}}o(Es,"regLineChange");function Xo(r){r.display.viewFrom=r.display.viewTo=r.doc.first,r.display.view=[],r.display.viewOffset=0}o(Xo,"resetView");function fl(r,i,u,a){var p=ho(r,i),g,y=r.display.view;if(!_i||u==r.doc.first+r.doc.size)return{index:p,lineN:u};for(var S=r.display.viewFrom,b=0;b0){if(p==y.length-1)return null;g=S+y[p].size-i,p++}else g=S-i;i+=g,u+=g}for(;jo(r.doc,u)!=u;){if(p==(a<0?0:y.length-1))return null;u+=a*y[p-(a<0?1:0)].size,p+=a}return{index:p,lineN:u}}o(fl,"viewCuttingPoint");function Dy(r,i,u){var a=r.display,p=a.view;p.length==0||i>=a.viewTo||u<=a.viewFrom?(a.view=qo(r,i,u),a.viewFrom=i):(a.viewFrom>i?a.view=qo(r,i,a.viewFrom).concat(a.view):a.viewFromu&&(a.view=a.view.slice(0,ho(r,u)))),a.viewTo=u}o(Dy,"adjustView");function Bm(r){for(var i=r.display.view,u=0,a=0;a=r.display.viewTo||S.to().line0?i.blinker=setInterval(function(){r.hasFocus()||pl(r),i.cursorDiv.style.visibility=(u=!u)?"":"hidden"},r.options.cursorBlinkRate):r.options.cursorBlinkRate<0&&(i.cursorDiv.style.visibility="hidden")}}o(ca,"restartBlink");function vd(r){r.hasFocus()||(r.display.input.focus(),r.state.focused||pa(r))}o(vd,"ensureFocus");function hc(r){r.state.delayingBlurEvent=!0,setTimeout(function(){r.state.delayingBlurEvent&&(r.state.delayingBlurEvent=!1,r.state.focused&&pl(r))},100)}o(hc,"delayBlurEvent");function pa(r,i){r.state.delayingBlurEvent&&!r.state.draggingText&&(r.state.delayingBlurEvent=!1),r.options.readOnly!="nocursor"&&(r.state.focused||(Te(r,"focus",r,i),r.state.focused=!0,Xe(r.display.wrapper,"CodeMirror-focused"),!r.curOp&&r.display.selForContextMenu!=r.doc.sel&&(r.display.input.reset(),C&&setTimeout(function(){return r.display.input.reset(!0)},20)),r.display.input.receivedFocus()),ca(r))}o(pa,"onFocus");function pl(r,i){r.state.delayingBlurEvent||(r.state.focused&&(Te(r,"blur",r,i),r.state.focused=!1,xe(r.display.wrapper,"CodeMirror-focused")),clearInterval(r.display.blinker),setTimeout(function(){r.state.focused||(r.display.shift=!1)},150))}o(pl,"onBlur");function _s(r){for(var i=r.display,u=i.lineDiv.offsetTop,a=0;a.005||M<-.005)&&(ai(p.line,y),Ts(p.line),p.rest))for(var D=0;Dr.display.sizerWidth){var V=Math.ceil(S/ua(r.display));V>r.display.maxLineLength&&(r.display.maxLineLength=V,r.display.maxLine=p.line,r.display.maxLineChanged=!0)}}}}o(_s,"updateHeightsInViewport");function Ts(r){if(r.widgets)for(var i=0;i=y&&(g=ao(i,hn(Me(i,b))-r.wrapper.clientHeight),y=b)}return{from:g,to:Math.max(y,g+1)}}o(dl,"visibleLines");function Fy(r,i){if(!ir(r,"scrollCursorIntoView")){var u=r.display,a=u.sizer.getBoundingClientRect(),p=null;if(i.top+a.top<0?p=!0:i.bottom+a.top>(window.innerHeight||document.documentElement.clientHeight)&&(p=!1),p!=null&&!J){var g=_e("div","\u200B",null,`position: absolute; + height: `+(De-Ne)+"px"))}o(M,"add");function D(ve,Ne,Ie){var De=Me(p,ve),it=De.text.length,Tt,br;function At(dr,Vr){return ki(r,ue(ve,dr),"div",De,Vr)}o(At,"coords");function Sn(dr,Vr,Ar){var Lt=Q(r,De,null,dr),Bt=Vr=="ltr"==(Ar=="after")?"left":"right",ar=Ar=="after"?Lt.begin:Lt.end-(/\s/.test(De.text.charAt(Lt.end-1))?2:1);return At(ar,Bt)[Bt]}o(Sn,"wrapX");var Cn=Mn(De,p.direction);return zn(Cn,Ne||0,Ie??it,function(dr,Vr,Ar,Lt){var Bt=Ar=="ltr",ar=At(dr,Bt?"left":"right"),bn=At(Vr-1,Bt?"right":"left"),Aa=Ne==null&&dr==0,Hs=Ie==null&&Vr==it,nn=Lt==0,wo=!Cn||Lt==Cn.length-1;if(bn.top-ar.top<=3){var Dr=(E?Aa:Hs)&&nn,Kd=(E?Hs:Aa)&&wo,rs=Dr?S:(Bt?ar:bn).left,bl=Kd?b:(Bt?bn:ar).right;M(rs,ar.top,bl-rs,ar.bottom)}else{var El,En,Da,zc;Bt?(El=E&&Aa&&nn?S:ar.left,En=E?b:Sn(dr,Ar,"before"),Da=E?S:Sn(Vr,Ar,"after"),zc=E&&Hs&&wo?b:bn.right):(El=E?Sn(dr,Ar,"before"):S,En=!E&&Aa&&nn?b:ar.right,Da=!E&&Hs&&wo?S:bn.left,zc=E?Sn(Vr,Ar,"after"):b),M(El,ar.top,En-El,ar.bottom),ar.bottom0?i.blinker=setInterval(function(){r.hasFocus()||pl(r),i.cursorDiv.style.visibility=(u=!u)?"":"hidden"},r.options.cursorBlinkRate):r.options.cursorBlinkRate<0&&(i.cursorDiv.style.visibility="hidden")}}o(ca,"restartBlink");function vd(r){r.hasFocus()||(r.display.input.focus(),r.state.focused||pa(r))}o(vd,"ensureFocus");function hc(r){r.state.delayingBlurEvent=!0,setTimeout(function(){r.state.delayingBlurEvent&&(r.state.delayingBlurEvent=!1,r.state.focused&&pl(r))},100)}o(hc,"delayBlurEvent");function pa(r,i){r.state.delayingBlurEvent&&!r.state.draggingText&&(r.state.delayingBlurEvent=!1),r.options.readOnly!="nocursor"&&(r.state.focused||(Te(r,"focus",r,i),r.state.focused=!0,Xe(r.display.wrapper,"CodeMirror-focused"),!r.curOp&&r.display.selForContextMenu!=r.doc.sel&&(r.display.input.reset(),C&&setTimeout(function(){return r.display.input.reset(!0)},20)),r.display.input.receivedFocus()),ca(r))}o(pa,"onFocus");function pl(r,i){r.state.delayingBlurEvent||(r.state.focused&&(Te(r,"blur",r,i),r.state.focused=!1,xe(r.display.wrapper,"CodeMirror-focused")),clearInterval(r.display.blinker),setTimeout(function(){r.state.focused||(r.display.shift=!1)},150))}o(pl,"onBlur");function _s(r){for(var i=r.display,u=i.lineDiv.offsetTop,a=0;a.005||M<-.005)&&(ai(p.line,y),Ts(p.line),p.rest))for(var D=0;Dr.display.sizerWidth){var V=Math.ceil(S/ua(r.display));V>r.display.maxLineLength&&(r.display.maxLineLength=V,r.display.maxLine=p.line,r.display.maxLineChanged=!0)}}}}o(_s,"updateHeightsInViewport");function Ts(r){if(r.widgets)for(var i=0;i=y&&(g=ao(i,hn(Me(i,b))-r.wrapper.clientHeight),y=b)}return{from:g,to:Math.max(y,g+1)}}o(dl,"visibleLines");function Ry(r,i){if(!ir(r,"scrollCursorIntoView")){var u=r.display,a=u.sizer.getBoundingClientRect(),p=null;if(i.top+a.top<0?p=!0:i.bottom+a.top>(window.innerHeight||document.documentElement.clientHeight)&&(p=!1),p!=null&&!J){var g=_e("div","\u200B",null,`position: absolute; top: `+(i.top-u.viewOffset-Ss(r.display))+`px; height: `+(i.bottom-i.top+mn(r)+u.barHeight)+`px; - left: `+i.left+"px; width: "+Math.max(2,i.right-i.left)+"px;");r.display.lineSpace.appendChild(g),g.scrollIntoView(p),r.display.lineSpace.removeChild(g)}}}o(Fy,"maybeScrollWindow");function By(r,i,u,a){a==null&&(a=0);var p;!r.options.lineWrapping&&i==u&&(u=i.sticky=="before"?ue(i.line,i.ch+1,"before"):i,i=i.ch?ue(i.line,i.sticky=="before"?i.ch-1:i.ch,"after"):i);for(var g=0;g<5;g++){var y=!1,S=Vn(r,i),b=!u||u==i?S:Vn(r,u);p={left:Math.min(S.left,b.left),top:Math.min(S.top,b.top)-a,right:Math.max(S.left,b.left),bottom:Math.max(S.bottom,b.bottom)+a};var E=da(r,p),M=r.doc.scrollTop,D=r.doc.scrollLeft;if(E.scrollTop!=null&&(sr(r,E.scrollTop),Math.abs(r.doc.scrollTop-M)>1&&(y=!0)),E.scrollLeft!=null&&(hl(r,E.scrollLeft),Math.abs(r.doc.scrollLeft-D)>1&&(y=!0)),!y)break}return p}o(By,"scrollPosIntoView");function Hy(r,i){var u=da(r,i);u.scrollTop!=null&&sr(r,u.scrollTop),u.scrollLeft!=null&&hl(r,u.scrollLeft)}o(Hy,"scrollIntoView");function da(r,i){var u=r.display,a=Cs(r.display);i.top<0&&(i.top=0);var p=r.curOp&&r.curOp.scrollTop!=null?r.curOp.scrollTop:u.scroller.scrollTop,g=al(r),y={};i.bottom-i.top>g&&(i.bottom=i.top+g);var S=r.doc.height+zr(u),b=i.topS-a;if(i.topp+g){var M=Math.min(i.top,(E?S:i.bottom)-g);M!=p&&(y.scrollTop=M)}var D=r.options.fixedGutter?0:u.gutters.offsetWidth,V=r.curOp&&r.curOp.scrollLeft!=null?r.curOp.scrollLeft:u.scroller.scrollLeft-D,$=qi(r)-u.gutters.offsetWidth,te=i.right-i.left>$;return te&&(i.right=i.left+$),i.left<10?y.scrollLeft=0:i.left$+V-3&&(y.scrollLeft=i.right+(te?0:10)-$),y}o(da,"calculateScrollPos");function ha(r,i){i!=null&&(mc(r),r.curOp.scrollTop=(r.curOp.scrollTop==null?r.doc.scrollTop:r.curOp.scrollTop)+i)}o(ha,"addToScrollTop");function ks(r){mc(r);var i=r.getCursor();r.curOp.scrollToPos={from:i,to:i,margin:r.options.cursorScrollMargin}}o(ks,"ensureCursorVisible");function qu(r,i,u){(i!=null||u!=null)&&mc(r),i!=null&&(r.curOp.scrollLeft=i),u!=null&&(r.curOp.scrollTop=u)}o(qu,"scrollToCoords");function Wm(r,i){mc(r),r.curOp.scrollToPos=i}o(Wm,"scrollToRange");function mc(r){var i=r.curOp.scrollToPos;if(i){r.curOp.scrollToPos=null;var u=zu(r,i.from),a=zu(r,i.to);Um(r,u,a,i.margin)}}o(mc,"resolveScrollToPos");function Um(r,i,u,a){var p=da(r,{left:Math.min(i.left,u.left),top:Math.min(i.top,u.top)-a,right:Math.max(i.right,u.right),bottom:Math.max(i.bottom,u.bottom)+a});qu(r,p.scrollLeft,p.scrollTop)}o(Um,"scrollToCoordsRange");function sr(r,i){Math.abs(r.doc.scrollTop-i)<2||(n||wd(r,{top:i}),Zr(r,i,!0),n&&wd(r),go(r,100))}o(sr,"updateScrollTop");function Zr(r,i,u){i=Math.max(0,Math.min(r.display.scroller.scrollHeight-r.display.scroller.clientHeight,i)),!(r.display.scroller.scrollTop==i&&!u)&&(r.doc.scrollTop=i,r.display.scrollbars.setScrollTop(i),r.display.scroller.scrollTop!=i&&(r.display.scroller.scrollTop=i))}o(Zr,"setScrollTop");function hl(r,i,u,a){i=Math.max(0,Math.min(i,r.display.scroller.scrollWidth-r.display.scroller.clientWidth)),!((u?i==r.doc.scrollLeft:Math.abs(r.doc.scrollLeft-i)<2)&&!a)&&(r.doc.scrollLeft=i,zm(r),r.display.scroller.scrollLeft!=i&&(r.display.scroller.scrollLeft=i),r.display.scrollbars.setScrollLeft(i))}o(hl,"setScrollLeft");function Vu(r){var i=r.display,u=i.gutters.offsetWidth,a=Math.round(r.doc.height+zr(r.display));return{clientHeight:i.scroller.clientHeight,viewHeight:i.wrapper.clientHeight,scrollWidth:i.scroller.scrollWidth,clientWidth:i.scroller.clientWidth,viewWidth:i.wrapper.clientWidth,barLeft:r.options.fixedGutter?u:0,docHeight:a,scrollHeight:a+mn(r)+i.barHeight,nativeBarWidth:i.nativeBarWidth,gutterWidth:u}}o(Vu,"measureForScrollbars");var Ns=o(function(r,i,u){this.cm=u;var a=this.vert=_e("div",[_e("div",null,null,"min-width: 1px")],"CodeMirror-vscrollbar"),p=this.horiz=_e("div",[_e("div",null,null,"height: 100%; min-height: 1px")],"CodeMirror-hscrollbar");a.tabIndex=p.tabIndex=-1,r(a),r(p),H(a,"scroll",function(){a.clientHeight&&i(a.scrollTop,"vertical")}),H(p,"scroll",function(){p.clientWidth&&i(p.scrollLeft,"horizontal")}),this.checkedZeroWidth=!1,c&&v<8&&(this.horiz.style.minHeight=this.vert.style.minWidth="18px")},"NativeScrollbars");Ns.prototype.update=function(r){var i=r.scrollWidth>r.clientWidth+1,u=r.scrollHeight>r.clientHeight+1,a=r.nativeBarWidth;if(u){this.vert.style.display="block",this.vert.style.bottom=i?a+"px":"0";var p=r.viewHeight-(i?a:0);this.vert.firstChild.style.height=Math.max(0,r.scrollHeight-r.clientHeight+p)+"px"}else this.vert.style.display="",this.vert.firstChild.style.height="0";if(i){this.horiz.style.display="block",this.horiz.style.right=u?a+"px":"0",this.horiz.style.left=r.barLeft+"px";var g=r.viewWidth-r.barLeft-(u?a:0);this.horiz.firstChild.style.width=Math.max(0,r.scrollWidth-r.clientWidth+g)+"px"}else this.horiz.style.display="",this.horiz.firstChild.style.width="0";return!this.checkedZeroWidth&&r.clientHeight>0&&(a==0&&this.zeroWidthHack(),this.checkedZeroWidth=!0),{right:u?a:0,bottom:i?a:0}},Ns.prototype.setScrollLeft=function(r){this.horiz.scrollLeft!=r&&(this.horiz.scrollLeft=r),this.disableHoriz&&this.enableZeroWidthBar(this.horiz,this.disableHoriz,"horiz")},Ns.prototype.setScrollTop=function(r){this.vert.scrollTop!=r&&(this.vert.scrollTop=r),this.disableVert&&this.enableZeroWidthBar(this.vert,this.disableVert,"vert")},Ns.prototype.zeroWidthHack=function(){var r=I&&!X?"12px":"18px";this.horiz.style.height=this.vert.style.width=r,this.horiz.style.pointerEvents=this.vert.style.pointerEvents="none",this.disableHoriz=new Ct,this.disableVert=new Ct},Ns.prototype.enableZeroWidthBar=function(r,i,u){r.style.pointerEvents="auto";function a(){var p=r.getBoundingClientRect(),g=u=="vert"?document.elementFromPoint(p.right-1,(p.top+p.bottom)/2):document.elementFromPoint((p.right+p.left)/2,p.bottom-1);g!=r?r.style.pointerEvents="none":i.set(1e3,a)}o(a,"maybeDisable"),i.set(1e3,a)},Ns.prototype.clear=function(){var r=this.horiz.parentNode;r.removeChild(this.horiz),r.removeChild(this.vert)};var Ku=o(function(){},"NullScrollbars");Ku.prototype.update=function(){return{bottom:0,right:0}},Ku.prototype.setScrollLeft=function(){},Ku.prototype.setScrollTop=function(){},Ku.prototype.clear=function(){};function Ls(r,i){i||(i=Vu(r));var u=r.display.barWidth,a=r.display.barHeight;ma(r,i);for(var p=0;p<4&&u!=r.display.barWidth||a!=r.display.barHeight;p++)u!=r.display.barWidth&&r.options.lineWrapping&&_s(r),ma(r,Vu(r)),u=r.display.barWidth,a=r.display.barHeight}o(Ls,"updateScrollbars");function ma(r,i){var u=r.display,a=u.scrollbars.update(i);u.sizer.style.paddingRight=(u.barWidth=a.right)+"px",u.sizer.style.paddingBottom=(u.barHeight=a.bottom)+"px",u.heightForcer.style.borderBottom=a.bottom+"px solid transparent",a.right&&a.bottom?(u.scrollbarFiller.style.display="block",u.scrollbarFiller.style.height=a.bottom+"px",u.scrollbarFiller.style.width=a.right+"px"):u.scrollbarFiller.style.display="",a.bottom&&r.options.coverGutterNextToScrollbar&&r.options.fixedGutter?(u.gutterFiller.style.display="block",u.gutterFiller.style.height=a.bottom+"px",u.gutterFiller.style.width=i.gutterWidth+"px"):u.gutterFiller.style.display=""}o(ma,"updateScrollbarsInner");var gc={native:Ns,null:Ku};function ml(r){r.display.scrollbars&&(r.display.scrollbars.clear(),r.display.scrollbars.addClass&&xe(r.display.wrapper,r.display.scrollbars.addClass)),r.display.scrollbars=new gc[r.options.scrollbarStyle](function(i){r.display.wrapper.insertBefore(i,r.display.scrollbarFiller),H(i,"mousedown",function(){r.state.focused&&setTimeout(function(){return r.display.input.focus()},0)}),i.setAttribute("cm-not-content","true")},function(i,u){u=="horizontal"?hl(r,i):sr(r,i)},r),r.display.scrollbars.addClass&&Xe(r.display.wrapper,r.display.scrollbars.addClass)}o(ml,"initScrollbars");var Gu=0;function Ki(r){r.curOp={cm:r,viewChanged:!1,startHeight:r.doc.height,forceUpdate:!1,updateInput:0,typing:!1,changeObjs:null,cursorActivityHandlers:null,cursorActivityCalled:0,selectionChanged:!1,updateMaxLine:!1,scrollLeft:null,scrollTop:null,scrollToPos:null,focus:!1,id:++Gu,markArrays:null},$i(r.curOp)}o(Ki,"startOperation");function mo(r){var i=r.curOp;i&&hd(i,function(u){for(var a=0;a=u.viewTo)||u.maxLineChanged&&i.options.lineWrapping,r.update=r.mustUpdate&&new An(i,r.mustUpdate&&{top:r.scrollTop,ensure:r.scrollToPos},r.forceUpdate)}o(Wy,"endOperation_R1");function Uy(r){r.updatedDisplay=r.mustUpdate&&yd(r.cm,r.update)}o(Uy,"endOperation_W1");function zy(r){var i=r.cm,u=i.display;r.updatedDisplay&&_s(i),r.barMeasure=Vu(i),u.maxLineChanged&&!i.options.lineWrapping&&(r.adjustWidthTo=Uu(i,u.maxLine,u.maxLine.text.length).left+3,i.display.sizerWidth=r.adjustWidthTo,r.barMeasure.scrollWidth=Math.max(u.scroller.clientWidth,u.sizer.offsetLeft+r.adjustWidthTo+mn(i)+i.display.barWidth),r.maxScrollLeft=Math.max(0,u.sizer.offsetLeft+r.adjustWidthTo-qi(i))),(r.updatedDisplay||r.selectionChanged)&&(r.preparedSelection=u.input.prepareSelection())}o(zy,"endOperation_R2");function $y(r){var i=r.cm;r.adjustWidthTo!=null&&(i.display.sizer.style.minWidth=r.adjustWidthTo+"px",r.maxScrollLeft=r.display.viewTo)){var u=+new Date+r.options.workTime,a=$o(r,i.highlightFrontier),p=[];i.iter(a.line,Math.min(i.first+i.size,r.display.viewTo+500),function(g){if(a.line>=r.display.viewFrom){var y=g.styles,S=g.text.length>r.options.maxHighlightLength?Uo(i.mode,a.state):null,b=tc(r,g,a,!0);S&&(a.state=S),g.styles=b.styles;var E=g.styleClasses,M=b.classes;M?g.styleClasses=M:E&&(g.styleClasses=null);for(var D=!y||y.length!=g.styles.length||E!=M&&(!E||!M||E.bgClass!=M.bgClass||E.textClass!=M.textClass),V=0;!D&&Vu)return go(r,r.options.workDelay),!0}),i.highlightFrontier=a.line,i.modeFrontier=Math.max(i.modeFrontier,a.line),p.length&&Jr(r,function(){for(var g=0;g=u.viewFrom&&i.visible.to<=u.viewTo&&(u.updateLineNumbers==null||u.updateLineNumbers>=u.viewTo)&&u.renderedView==u.view&&Bm(r)==0)return!1;vo(r)&&(Xo(r),i.dims=Kn(r));var p=a.first+a.size,g=Math.max(i.visible.from-r.options.viewportMargin,a.first),y=Math.min(p,i.visible.to+r.options.viewportMargin);u.viewFromy&&u.viewTo-y<20&&(y=Math.min(p,u.viewTo)),_i&&(g=jo(r.doc,g),y=pd(r.doc,y));var S=g!=u.viewFrom||y!=u.viewTo||u.lastWrapHeight!=i.wrapperHeight||u.lastWrapWidth!=i.wrapperWidth;Iy(r,g,y),u.viewOffset=hn(Me(r.doc,u.viewFrom)),r.display.mover.style.top=u.viewOffset+"px";var b=Bm(r);if(!S&&b==0&&!i.force&&u.renderedView==u.view&&(u.updateLineNumbers==null||u.updateLineNumbers>=u.viewTo))return!1;var E=qy(r);return b>4&&(u.lineDiv.style.display="none"),Ky(r,u.updateLineNumbers,i.dims),b>4&&(u.lineDiv.style.display=""),u.renderedView=u.view,Vy(E),Ve(u.cursorDiv),Ve(u.selectionDiv),u.gutters.style.height=u.sizer.style.minHeight=0,S&&(u.lastWrapHeight=i.wrapperHeight,u.lastWrapWidth=i.wrapperWidth,go(r,400)),u.updateLineNumbers=null,!0}o(yd,"updateDisplayIfNeeded");function Ps(r,i){for(var u=i.viewport,a=!0;;a=!1){if(!a||!r.options.lineWrapping||i.oldDisplayWidth==qi(r)){if(u&&u.top!=null&&(u={top:Math.min(r.doc.height+zr(r.display)-al(r),u.top)}),i.visible=dl(r.display,r.doc,u),i.visible.from>=r.display.viewFrom&&i.visible.to<=r.display.viewTo)break}else a&&(i.visible=dl(r.display,r.doc,u));if(!yd(r,i))break;_s(r);var p=Vu(r);$u(r),Ls(r,p),Sd(r,p),i.force=!1}i.signal(r,"update",r),(r.display.viewFrom!=r.display.reportedViewFrom||r.display.viewTo!=r.display.reportedViewTo)&&(i.signal(r,"viewportChange",r,r.display.viewFrom,r.display.viewTo),r.display.reportedViewFrom=r.display.viewFrom,r.display.reportedViewTo=r.display.viewTo)}o(Ps,"postUpdateDisplay");function wd(r,i){var u=new An(r,i);if(yd(r,u)){_s(r),Ps(r,u);var a=Vu(r);$u(r),Ls(r,a),Sd(r,a),u.finish()}}o(wd,"updateDisplaySimple");function Ky(r,i,u){var a=r.display,p=r.options.lineNumbers,g=a.lineDiv,y=g.firstChild;function S(te){var oe=te.nextSibling;return C&&I&&r.display.currentWheelTarget==te?te.style.display="none":te.parentNode.removeChild(te),oe}o(S,"rm");for(var b=a.view,E=a.viewFrom,M=0;M-1&&($=!1),Du(r,D,E,u)),$&&(Ve(D.lineNumber),D.lineNumber.appendChild(document.createTextNode(jl(r.options,E)))),y=D.node.nextSibling}E+=D.size}for(;y;)y=S(y)}o(Ky,"patchDisplay");function xd(r){var i=r.gutters.offsetWidth;r.sizer.style.marginLeft=i+"px",yr(r,"gutterChanged",r)}o(xd,"updateGutterSpace");function Sd(r,i){r.display.sizer.style.minHeight=i.docHeight+"px",r.display.heightForcer.style.top=i.docHeight+"px",r.display.gutters.style.height=i.docHeight+r.display.barHeight+mn(r)+"px"}o(Sd,"setDocumentHeight");function zm(r){var i=r.display,u=i.view;if(!(!i.alignWidgets&&(!i.gutters.firstChild||!r.options.fixedGutter))){for(var a=fa(i)-i.scroller.scrollLeft+r.doc.scrollLeft,p=i.gutters.offsetWidth,g=a+"px",y=0;yy.clientWidth,b=y.scrollHeight>y.clientHeight;if(!!(a&&S||p&&b)){if(p&&I&&C){e:for(var E=i.target,M=g.view;E!=y;E=E.parentNode)for(var D=0;D=0&&ze(r,a.to())<=0)return u}return-1};var wt=o(function(r,i){this.anchor=r,this.head=i},"Range");wt.prototype.from=function(){return il(this.anchor,this.head)},wt.prototype.to=function(){return gs(this.anchor,this.head)},wt.prototype.empty=function(){return this.head.line==this.anchor.line&&this.head.ch==this.anchor.ch};function tn(r,i,u){var a=r&&r.options.selectionsMayTouch,p=i[u];i.sort(function(V,$){return ze(V.from(),$.from())}),u=ut(i,p);for(var g=1;g0:b>=0){var E=il(S.from(),y.from()),M=gs(S.to(),y.to()),D=S.empty()?y.from()==y.head:S.from()==S.head;g<=u&&--u,i.splice(--g,2,new wt(D?M:E,D?E:M))}}return new pi(i,u)}o(tn,"normalizeSelection");function Os(r,i){return new pi([new wt(r,i||r)],0)}o(Os,"simpleSelection");function Ms(r){return r.text?ue(r.from.line+r.text.length-1,Se(r.text).length+(r.text.length==1?r.from.ch:0)):r.to}o(Ms,"changeEnd");function Ni(r,i){if(ze(r,i.from)<0)return r;if(ze(r,i.to)<=0)return Ms(i);var u=r.line+i.text.length-(i.to.line-i.from.line)-1,a=r.ch;return r.line==i.to.line&&(a+=Ms(i).ch-i.to.ch),ue(u,a)}o(Ni,"adjustForChange");function bd(r,i){for(var u=[],a=0;a1&&r.remove(S.line+1,te-1),r.insert(S.line+1,ge)}yr(r,"change",r,i)}o(wc,"updateDoc");function As(r,i,u){function a(p,g,y){if(p.linked)for(var S=0;S1&&!r.done[r.done.length-2].ranges)return r.done.pop(),Se(r.done)}o(Qy,"lastChangeEvent");function yo(r,i,u,a){var p=r.history;p.undone.length=0;var g=+new Date,y,S;if((p.lastOp==a||p.lastOrigin==i.origin&&i.origin&&(i.origin.charAt(0)=="+"&&p.lastModTime>g-(r.cm?r.cm.options.historyEventDelay:500)||i.origin.charAt(0)=="*"))&&(y=Qy(p,p.lastOp==a)))S=Se(y.changes),ze(i.from,i.to)==0&&ze(i.from,S.to)==0?S.to=Ms(i):y.changes.push(Td(r,i));else{var b=Se(p.done);for((!b||!b.ranges)&&Dn(r.sel,p.done),y={changes:[Td(r,i)],generation:p.generation},p.done.push(y);p.done.length>p.undoDepth;)p.done.shift(),p.done[0].ranges||p.done.shift()}p.done.push(u),p.generation=++p.maxGeneration,p.lastModTime=p.lastSelTime=g,p.lastOp=p.lastSelOp=a,p.lastOrigin=p.lastSelOrigin=i.origin,S||Te(r,"historyAdded")}o(yo,"addChangeToHistory");function Nd(r,i,u,a){var p=i.charAt(0);return p=="*"||p=="+"&&u.ranges.length==a.ranges.length&&u.somethingSelected()==a.somethingSelected()&&new Date-r.history.lastSelTime<=(r.cm?r.cm.options.historyEventDelay:500)}o(Nd,"selectionEventCanBeMerged");function vl(r,i,u,a){var p=r.history,g=a&&a.origin;u==p.lastSelOp||g&&p.lastSelOrigin==g&&(p.lastModTime==p.lastSelTime&&p.lastOrigin==g||Nd(r,g,Se(p.done),i))?p.done[p.done.length-1]=i:Dn(i,p.done),p.lastSelTime=+new Date,p.lastSelOrigin=g,p.lastSelOp=u,a&&a.clearRedo!==!1&&kd(p.undone)}o(vl,"addSelectionToHistory");function Dn(r,i){var u=Se(i);u&&u.ranges&&u.equals(r)||i.push(r)}o(Dn,"pushSelectionToHistory");function Gm(r,i,u,a){var p=i["spans_"+r.id],g=0;r.iter(Math.max(r.first,u),Math.min(r.first+r.size,a),function(y){y.markedSpans&&((p||(p=i["spans_"+r.id]={}))[g]=y.markedSpans),++g})}o(Gm,"attachLocalSpans");function Ym(r){if(!r)return null;for(var i,u=0;u-1&&(Se(S)[D]=E[D],delete E[D])}}return a}o(di,"copyHistoryArray");function Sc(r,i,u,a){if(a){var p=r.anchor;if(u){var g=ze(i,p)<0;g!=ze(u,p)<0?(p=i,i=u):g!=ze(i,u)<0&&(i=u)}return new wt(p,i)}else return new wt(u||i,i)}o(Sc,"extendRange");function Cc(r,i,u,a,p){p==null&&(p=r.cm&&(r.cm.display.shift||r.extend)),$r(r,new pi([Sc(r.sel.primary(),i,u,p)],0),a)}o(Cc,"extendSelection");function Zu(r,i,u){for(var a=[],p=r.cm&&(r.cm.display.shift||r.extend),g=0;g=i.ch:S.to>i.ch))){if(p&&(Te(b,"beforeCursorEnter"),b.explicitlyCleared))if(g.markedSpans){--y;continue}else break;if(!b.atomic)continue;if(u){var D=b.find(a<0?1:-1),V=void 0;if((a<0?M:E)&&(D=_c(r,D,-a,D&&D.line==i.line?g:null)),D&&D.line==i.line&&(V=ze(D,u))&&(a<0?V<0:V>0))return yl(r,D,i,a,p)}var $=b.find(a<0?-1:1);return(a<0?E:M)&&($=_c(r,$,a,$.line==i.line?g:null)),$?yl(r,$,i,a,p):null}}return i}o(yl,"skipAtomicInner");function jr(r,i,u,a,p){var g=a||1,y=yl(r,i,u,g,p)||!p&&yl(r,i,u,g,!0)||yl(r,i,u,-g,p)||!p&&yl(r,i,u,-g,!0);return y||(r.cantEdit=!0,ue(r.first,0))}o(jr,"skipAtomic");function _c(r,i,u,a){return u<0&&i.ch==0?i.line>r.first?He(r,ue(i.line-1)):null:u>0&&i.ch==(a||Me(r,i.line)).text.length?i.line=0;--p)Tc(r,{from:a[p].from,to:a[p].to,text:p?[""]:i.text,origin:i.origin});else Tc(r,i)}}o(ya,"makeChange");function Tc(r,i){if(!(i.text.length==1&&i.text[0]==""&&ze(i.from,i.to)==0)){var u=bd(r,i);yo(r,i,u,r.cm?r.cm.curOp.id:NaN),xa(r,i,u,ku(r,i));var a=[];As(r,function(p,g){!g&&ut(a,p.history)==-1&&(eg(p.history,i),a.push(p.history)),xa(p,i,null,ku(p,i))})}}o(Tc,"makeChangeInner");function kc(r,i,u){var a=r.cm&&r.cm.state.suppressEdits;if(!(a&&!u)){for(var p=r.history,g,y=r.sel,S=i=="undo"?p.done:p.undone,b=i=="undo"?p.undone:p.done,E=0;E=0;--$){var te=V($);if(te)return te.v}}}}o(kc,"makeChangeFromHistory");function wa(r,i){if(i!=0&&(r.first+=i,r.sel=new pi(Or(r.sel.ranges,function(p){return new wt(ue(p.anchor.line+i,p.anchor.ch),ue(p.head.line+i,p.head.ch))}),r.sel.primIndex),r.cm)){et(r.cm,r.first,r.first-i,i);for(var u=r.cm.display,a=u.viewFrom;ar.lastLine())){if(i.from.lineg&&(i={from:i.from,to:ue(g,Me(r,g).text.length),text:[i.text[0]],origin:i.origin}),i.removed=Ei(r,i.from,i.to),u||(u=bd(r,i)),r.cm?Zy(r.cm,i,a):wc(r,i,a),hi(r,u,$t),r.cantEdit&&jr(r,ue(r.firstLine(),0))&&(r.cantEdit=!1)}}o(xa,"makeChangeSingleDoc");function Zy(r,i,u){var a=r.doc,p=r.display,g=i.from,y=i.to,S=!1,b=g.line;r.options.lineWrapping||(b=vt(vr(Me(a,g.line))),a.iter(b,y.line+1,function($){if($==p.maxLine)return S=!0,!0})),a.sel.contains(i.from,i.to)>-1&&Ul(r),wc(a,i,u,Fm(r)),r.options.lineWrapping||(a.iter(b,g.line+i.text.length,function($){var te=ws($);te>p.maxLineLength&&(p.maxLine=$,p.maxLineLength=te,p.maxLineChanged=!0,S=!1)}),S&&(r.curOp.updateMaxLine=!0)),rc(a,g.line),go(r,400);var E=i.text.length-(y.line-g.line)-1;i.full?et(r):g.line==y.line&&i.text.length==1&&!_d(r.doc,i)?Es(r,g.line,"text"):et(r,g.line,y.line+1,E);var M=Ft(r,"changes"),D=Ft(r,"change");if(D||M){var V={from:g,to:y,text:i.text,removed:i.removed,origin:i.origin};D&&yr(r,"change",r,V),M&&(r.curOp.changeObjs||(r.curOp.changeObjs=[])).push(V)}r.display.selForContextMenu=null}o(Zy,"makeChangeSingleDocInEditor");function Sa(r,i,u,a,p){var g;a||(a=u),ze(a,u)<0&&(g=[a,u],u=g[0],a=g[1]),typeof i=="string"&&(i=r.splitLines(i)),ya(r,{from:u,to:a,text:i,origin:p})}o(Sa,"replaceRange");function Ca(r,i,u,a){u1||!(this.children[0]instanceof ba))){var S=[];this.collapse(S),this.children=[new ba(S)],this.children[0].parent=this}},collapse:function(r){for(var i=0;i50){for(var y=p.lines.length%25+25,S=y;S10);r.parent.maybeSpill()}},iterN:function(r,i,u){for(var a=0;ar.display.maxLineLength&&(r.display.maxLine=E,r.display.maxLineLength=M,r.display.maxLineChanged=!0)}a!=null&&r&&this.collapsed&&et(r,a,p+1),this.lines.length=0,this.explicitlyCleared=!0,this.atomic&&this.doc.cantEdit&&(this.doc.cantEdit=!1,r&&Ju(r.doc)),r&&yr(r,"markerCleared",r,this,a,p),i&&mo(r),this.parent&&this.parent.clear()}},Rs.prototype.find=function(r,i){r==null&&this.type=="bookmark"&&(r=1);for(var u,a,p=0;p0||y==0&&g.clearWhenEmpty!==!1)return g;if(g.replacedWith&&(g.collapsed=!0,g.widgetNode=St("span",[g.replacedWith],"CodeMirror-widget"),a.handleMouseEvents||g.widgetNode.setAttribute("cm-ignore-events","true"),a.insertLeft&&(g.widgetNode.insertLeft=!0)),g.collapsed){if(sl(r,i.line,i,u,g)||i.line!=u.line&&sl(r,u.line,i,u,g))throw new Error("Inserting collapsed marker partially overlapping an existing one");Ql()}g.addToHistory&&yo(r,{from:i,to:u,origin:"markText"},r.sel,NaN);var S=i.line,b=r.cm,E;if(r.iter(S,u.line+1,function(D){b&&g.collapsed&&!b.options.lineWrapping&&vr(D)==b.display.maxLine&&(E=!0),g.collapsed&&S!=i.line&&ai(D,0),oc(D,new co(g,S==i.line?i.ch:null,S==u.line?u.ch:null),r.cm&&r.cm.curOp),++S}),g.collapsed&&r.iter(i.line,u.line+1,function(D){qt(r,D)&&ai(D,0)}),g.clearOnEnter&&H(g,"beforeCursorEnter",function(){return g.clear()}),g.readOnly&&(nc(),(r.history.done.length||r.history.undone.length)&&r.clearHistory()),g.collapsed&&(g.id=++Nc,g.atomic=!0),b){if(E&&(b.curOp.updateMaxLine=!0),g.collapsed)et(b,i.line,u.line+1);else if(g.className||g.startStyle||g.endStyle||g.css||g.attributes||g.title)for(var M=i.line;M<=u.line;M++)Es(b,M,"text");g.atomic&&Ju(b.doc),yr(b,"markerAdded",b,g)}return g}o(Is,"markText");var Ea=o(function(r,i){this.markers=r,this.primary=i;for(var u=0;u=0;b--)ya(this,a[b]);S?bc(this,S):this.cm&&ks(this.cm)}),undo:N(function(){kc(this,"undo")}),redo:N(function(){kc(this,"redo")}),undoSelection:N(function(){kc(this,"undo",!0)}),redoSelection:N(function(){kc(this,"redo",!0)}),setExtending:function(r){this.extend=r},getExtending:function(){return this.extend},historySize:function(){for(var r=this.history,i=0,u=0,a=0;a=r.ch)&&i.push(p.marker.parent||p.marker)}return i},findMarks:function(r,i,u){r=He(this,r),i=He(this,i);var a=[],p=r.line;return this.iter(r.line,i.line+1,function(g){var y=g.markedSpans;if(y)for(var S=0;S=b.to||b.from==null&&p!=r.line||b.from!=null&&p==i.line&&b.from>=i.ch)&&(!u||u(b.marker))&&a.push(b.marker.parent||b.marker)}++p}),a},getAllMarks:function(){var r=[];return this.iter(function(i){var u=i.markedSpans;if(u)for(var a=0;ar)return i=r,!0;r-=g,++u}),He(this,ue(u,i))},indexFromPos:function(r){r=He(this,r);var i=r.ch;if(r.linei&&(i=r.from),r.to!=null&&r.to-1){i.state.draggingText(r),setTimeout(function(){return i.display.input.focus()},20);return}try{var M=r.dataTransfer.getData("Text");if(M){var D;if(i.state.draggingText&&!i.state.draggingText.copy&&(D=i.listSelections()),hi(i.doc,Os(u,u)),D)for(var V=0;V=0;S--)Sa(r.doc,"",a[S].from,a[S].to,"+delete");ks(r)})}o(mi,"deleteNearSelection");function of(r,i,u){var a=Ui(r.text,i+u,u);return a<0||a>r.text.length?null:a}o(of,"moveCharLogically");function Mc(r,i,u){var a=of(r,i.ch,u);return a==null?null:new ue(i.line,a,u<0?"after":"before")}o(Mc,"moveLogically");function _a(r,i,u,a,p){if(r){i.doc.direction=="rtl"&&(p=-p);var g=Mn(u,i.doc.direction);if(g){var y=p<0?Se(g):g[0],S=p<0==(y.level==1),b=S?"after":"before",E;if(y.level>0||i.doc.direction=="rtl"){var M=qn(i,u);E=p<0?u.text.length-1:0;var D=Ti(i,M,E).top;E=pn(function(V){return Ti(i,M,V).top==D},p<0==(y.level==1)?y.from:y.to-1,E),b=="before"&&(E=of(u,E,1))}else E=p<0?y.to:y.from;return new ue(a,E,b)}}return new ue(a,p<0?u.text.length:0,p<0?"before":"after")}o(_a,"endOfLine");function ag(r,i,u,a){var p=Mn(i,r.doc.direction);if(!p)return Mc(i,u,a);u.ch>=i.text.length?(u.ch=i.text.length,u.sticky="before"):u.ch<=0&&(u.ch=0,u.sticky="after");var g=Ci(p,u.ch,u.sticky),y=p[g];if(r.doc.direction=="ltr"&&y.level%2==0&&(a>0?y.to>u.ch:y.from=y.from&&V>=M.begin)){var $=D?"before":"after";return new ue(u.line,V,$)}}var te=o(function(ge,be,ve){for(var Ne=o(function(Tt,br){return br?new ue(u.line,S(Tt,1),"before"):new ue(u.line,Tt,"after")},"getRes");ge>=0&&ge0==(Ie.level!=1),it=De?ve.begin:S(ve.end,-1);if(Ie.from<=it&&it0?M.end:S(M.begin,-1);return de!=null&&!(a>0&&de==i.text.length)&&(oe=te(a>0?0:p.length-1,a,E(de)),oe)?oe:null}o(ag,"moveVisually");var xl={selectAll:Qm,singleSelection:function(r){return r.setSelection(r.getCursor("anchor"),r.getCursor("head"),$t)},killLine:function(r){return mi(r,function(i){if(i.empty()){var u=Me(r.doc,i.head.line).text.length;return i.head.ch==u&&i.head.line0)p=new ue(p.line,p.ch+1),r.replaceRange(g.charAt(p.ch-1)+g.charAt(p.ch-2),ue(p.line,p.ch-2),p,"+transpose");else if(p.line>r.doc.first){var y=Me(r.doc,p.line-1).text;y&&(p=new ue(p.line,1),r.replaceRange(g.charAt(0)+r.doc.lineSeparator()+y.charAt(y.length-1),ue(p.line-1,y.length-1),p,"+transpose"))}}u.push(new wt(p,p))}r.setSelections(u)})},newlineAndIndent:function(r){return Jr(r,function(){for(var i=r.listSelections(),u=i.length-1;u>=0;u--)r.replaceRange(r.doc.lineSeparator(),i[u].anchor,i[u].head,"+input");i=r.listSelections();for(var a=0;ar&&ze(i,this.pos)==0&&u==this.button};var qr,Gn;function ow(r,i){var u=+new Date;return Gn&&Gn.compare(u,r,i)?(qr=Gn=null,"triple"):qr&&qr.compare(u,r,i)?(Gn=new Rc(u,r,i),qr=null,"double"):(qr=new Rc(u,r,i),Gn=null,"single")}o(ow,"clickRepeat");function dg(r){var i=this,u=i.display;if(!(ir(i,r)||u.activeTouch&&u.input.supportsTouch())){if(u.input.ensurePolled(),u.shift=r.shiftKey,ji(u,r)){C||(u.scroller.draggable=!1,setTimeout(function(){return u.scroller.draggable=!0},100));return}if(!zd(i,r)){var a=po(i,r),p=el(r),g=a?ow(a,p):"single";window.focus(),p==1&&i.state.selectingText&&i.state.selectingText(r),!(a&&Ic(i,p,a,g,r))&&(p==1?a?hg(i,a,g,r):bi(r)==u.scroller&&or(r):p==2?(a&&Cc(i.doc,a),setTimeout(function(){return u.input.focus()},20)):p==3&&(pe?i.display.input.onContextMenu(r):hc(i)))}}}o(dg,"onMouseDown");function Ic(r,i,u,a,p){var g="Click";return a=="double"?g="Double"+g:a=="triple"&&(g="Triple"+g),g=(i==1?"Left":i==2?"Middle":"Right")+g,Ta(r,Rd(g,p),p,function(y){if(typeof y=="string"&&(y=xl[y]),!y)return!1;var S=!1;try{r.isReadOnly()&&(r.state.suppressEdits=!0),S=y(r,u)!=zt}finally{r.state.suppressEdits=!1}return S})}o(Ic,"handleMappedButton");function ka(r,i,u){var a=r.getOption("configureMouse"),p=a?a(r,i,u):{};if(p.unit==null){var g=G?u.shiftKey&&u.metaKey:u.altKey;p.unit=g?"rectangle":i=="single"?"char":i=="double"?"word":"line"}return(p.extend==null||r.doc.extend)&&(p.extend=r.doc.extend||u.shiftKey),p.addNew==null&&(p.addNew=I?u.metaKey:u.ctrlKey),p.moveOnDrag==null&&(p.moveOnDrag=!(I?u.altKey:u.ctrlKey)),p}o(ka,"configureMouse");function hg(r,i,u,a){c?setTimeout(Hr(vd,r),0):r.curOp.focus=Ge();var p=ka(r,u,a),g=r.doc.sel,y;r.options.dragDrop&&hs&&!r.isReadOnly()&&u=="single"&&(y=g.contains(i))>-1&&(ze((y=g.ranges[y]).from(),i)<0||i.xRel>0)&&(ze(y.to(),i)>0||i.xRel<0)?mg(r,a,i,p):vg(r,a,i,p)}o(hg,"leftButtonDown");function mg(r,i,u,a){var p=r.display,g=!1,y=lr(r,function(E){C&&(p.scroller.draggable=!1),r.state.draggingText=!1,r.state.delayingBlurEvent&&(r.hasFocus()?r.state.delayingBlurEvent=!1:hc(r)),he(p.wrapper.ownerDocument,"mouseup",y),he(p.wrapper.ownerDocument,"mousemove",S),he(p.scroller,"dragstart",b),he(p.scroller,"drop",y),g||(or(E),a.addNew||Cc(r.doc,u,null,null,a.extend),C&&!B||c&&v==9?setTimeout(function(){p.wrapper.ownerDocument.body.focus({preventScroll:!0}),p.input.focus()},20):p.input.focus())}),S=o(function(E){g=g||Math.abs(i.clientX-E.clientX)+Math.abs(i.clientY-E.clientY)>=10},"mouseMove"),b=o(function(){return g=!0},"dragStart");C&&(p.scroller.draggable=!0),r.state.draggingText=y,y.copy=!a.moveOnDrag,H(p.wrapper.ownerDocument,"mouseup",y),H(p.wrapper.ownerDocument,"mousemove",S),H(p.scroller,"dragstart",b),H(p.scroller,"drop",y),r.state.delayingBlurEvent=!0,setTimeout(function(){return p.input.focus()},20),p.scroller.dragDrop&&p.scroller.dragDrop()}o(mg,"leftButtonStartDrag");function gg(r,i,u){if(u=="char")return new wt(i,i);if(u=="word")return r.findWordAt(i);if(u=="line")return new wt(ue(i.line,0),He(r.doc,ue(i.line+1,0)));var a=u(r,i);return new wt(a.from,a.to)}o(gg,"rangeForUnit");function vg(r,i,u,a){c&&hc(r);var p=r.display,g=r.doc;or(i);var y,S,b=g.sel,E=b.ranges;if(a.addNew&&!a.extend?(S=g.sel.contains(u),S>-1?y=E[S]:y=new wt(u,u)):(y=g.sel.primary(),S=g.sel.primIndex),a.unit=="rectangle")a.addNew||(y=new wt(u,u)),u=po(r,i,!0,!0),S=-1;else{var M=gg(r,u,a.unit);a.extend?y=Sc(y,M.anchor,M.head,a.extend):y=M}a.addNew?S==-1?(S=E.length,$r(g,tn(r,E.concat([y]),S),{scroll:!1,origin:"*mouse"})):E.length>1&&E[S].empty()&&a.unit=="char"&&!a.extend?($r(g,tn(r,E.slice(0,S).concat(E.slice(S+1)),0),{scroll:!1,origin:"*mouse"}),b=g.sel):Ld(g,S,y,ie):(S=0,$r(g,new pi([y],0),ie),b=g.sel);var D=u;function V(ve){if(ze(D,ve)!=0)if(D=ve,a.unit=="rectangle"){for(var Ne=[],Ie=r.options.tabSize,De=_t(Me(g,u.line).text,u.ch,Ie),it=_t(Me(g,ve.line).text,ve.ch,Ie),Tt=Math.min(De,it),br=Math.max(De,it),At=Math.min(u.line,ve.line),Sn=Math.min(r.lastLine(),Math.max(u.line,ve.line));At<=Sn;At++){var Cn=Me(g,At).text,dr=Pr(Cn,Tt,Ie);Tt==br?Ne.push(new wt(ue(At,dr),ue(At,dr))):Cn.length>dr&&Ne.push(new wt(ue(At,dr),ue(At,Pr(Cn,br,Ie))))}Ne.length||Ne.push(new wt(u,u)),$r(g,tn(r,b.ranges.slice(0,S).concat(Ne),S),{origin:"*mouse",scroll:!1}),r.scrollIntoView(ve)}else{var Vr=y,Ar=gg(r,ve,a.unit),Lt=Vr.anchor,Bt;ze(Ar.anchor,Lt)>0?(Bt=Ar.head,Lt=il(Vr.from(),Ar.anchor)):(Bt=Ar.anchor,Lt=gs(Vr.to(),Ar.head));var ar=b.ranges.slice(0);ar[S]=Na(r,new wt(He(g,Lt),Bt)),$r(g,tn(r,ar,S),ie)}}o(V,"extendTo");var $=p.wrapper.getBoundingClientRect(),te=0;function oe(ve){var Ne=++te,Ie=po(r,ve,!0,a.unit=="rectangle");if(!!Ie)if(ze(Ie,D)!=0){r.curOp.focus=Ge(),V(Ie);var De=dl(p,g);(Ie.line>=De.to||Ie.line$.bottom?20:0;it&&setTimeout(lr(r,function(){te==Ne&&(p.scroller.scrollTop+=it,oe(ve))}),50)}}o(oe,"extend");function de(ve){r.state.selectingText=!1,te=1/0,ve&&(or(ve),p.input.focus()),he(p.wrapper.ownerDocument,"mousemove",ge),he(p.wrapper.ownerDocument,"mouseup",be),g.history.lastSelOrigin=null}o(de,"done");var ge=lr(r,function(ve){ve.buttons===0||!el(ve)?de(ve):oe(ve)}),be=lr(r,de);r.state.selectingText=be,H(p.wrapper.ownerDocument,"mousemove",ge),H(p.wrapper.ownerDocument,"mouseup",be)}o(vg,"leftButtonSelect");function Na(r,i){var u=i.anchor,a=i.head,p=Me(r.doc,u.line);if(ze(u,a)==0&&u.sticky==a.sticky)return i;var g=Mn(p);if(!g)return i;var y=Ci(g,u.ch,u.sticky),S=g[y];if(S.from!=u.ch&&S.to!=u.ch)return i;var b=y+(S.from==u.ch==(S.level!=1)?0:1);if(b==0||b==g.length)return i;var E;if(a.line!=u.line)E=(a.line-u.line)*(r.doc.direction=="ltr"?1:-1)>0;else{var M=Ci(g,a.ch,a.sticky),D=M-y||(a.ch-u.ch)*(S.level==1?-1:1);M==b-1||M==b?E=D<0:E=D>0}var V=g[b+(E?-1:0)],$=E==(V.level==1),te=$?V.from:V.to,oe=$?"after":"before";return u.ch==te&&u.sticky==oe?i:new wt(new ue(u.line,te,oe),a)}o(Na,"bidiSimplify");function La(r,i,u,a){var p,g;if(i.touches)p=i.touches[0].clientX,g=i.touches[0].clientY;else try{p=i.clientX,g=i.clientY}catch(V){return!1}if(p>=Math.floor(r.display.gutters.getBoundingClientRect().right))return!1;a&&or(i);var y=r.display,S=y.lineDiv.getBoundingClientRect();if(g>S.bottom||!Ft(r,u))return ds(i);g-=S.top-y.viewOffset;for(var b=0;b=p){var M=ao(r.doc,g),D=r.display.gutterSpecs[b];return Te(r,u,r,M,D.className,i),ds(i)}}}o(La,"gutterEvent");function zd(r,i){return La(r,i,"gutterClick",!0)}o(zd,"clickInGutter");function $d(r,i){ji(r.display,i)||yg(r,i)||ir(r,i,"contextmenu")||pe||r.display.input.onContextMenu(i)}o($d,"onContextMenu");function yg(r,i){return Ft(r,"gutterContextMenu")?La(r,i,"gutterContextMenu",!1):!1}o(yg,"contextMenuInGutter");function sf(r){r.display.wrapper.className=r.display.wrapper.className.replace(/\s*cm-s-\S+/g,"")+r.options.theme.replace(/(^|\s)\s*/g," cm-s-"),U(r)}o(sf,"themeChanged");var Sl={toString:function(){return"CodeMirror.Init"}},lf={},Pa={};function Fc(r){var i=r.optionHandlers;function u(a,p,g,y){r.defaults[a]=p,g&&(i[a]=y?function(S,b,E){E!=Sl&&g(S,b,E)}:g)}o(u,"option"),r.defineOption=u,r.Init=Sl,u("value","",function(a,p){return a.setValue(p)},!0),u("mode",null,function(a,p){a.doc.modeOption=p,Ed(a)},!0),u("indentUnit",2,Ed,!0),u("indentWithTabs",!1),u("smartIndent",!0),u("tabSize",4,function(a){Xu(a),U(a),et(a)},!0),u("lineSeparator",null,function(a,p){if(a.doc.lineSep=p,!!p){var g=[],y=a.doc.first;a.doc.iter(function(b){for(var E=0;;){var M=b.text.indexOf(p,E);if(M==-1)break;E=M+p.length,g.push(ue(y,M))}y++});for(var S=g.length-1;S>=0;S--)Sa(a.doc,p,g[S],ue(g[S].line,g[S].ch+p.length))}}),u("specialChars",/[\u0000-\u001f\u007f-\u009f\u00ad\u061c\u200b\u200e\u200f\u2028\u2029\ufeff\ufff9-\ufffc]/g,function(a,p,g){a.state.specialChars=new RegExp(p.source+(p.test(" ")?"":"| "),"g"),g!=Sl&&a.refresh()}),u("specialCharPlaceholder",Ur,function(a){return a.refresh()},!0),u("electricChars",!0),u("inputStyle",A?"contenteditable":"textarea",function(){throw new Error("inputStyle can not (yet) be changed in a running editor")},!0),u("spellcheck",!1,function(a,p){return a.getInputField().spellcheck=p},!0),u("autocorrect",!1,function(a,p){return a.getInputField().autocorrect=p},!0),u("autocapitalize",!1,function(a,p){return a.getInputField().autocapitalize=p},!0),u("rtlMoveVisually",!K),u("wholeLineUpdateBefore",!0),u("theme","default",function(a){sf(a),Yu(a)},!0),u("keyMap","default",function(a,p,g){var y=wn(p),S=g!=Sl&&wn(g);S&&S.detach&&S.detach(a,y),y.attach&&y.attach(a,S||null)}),u("extraKeys",null),u("configureMouse",null),u("lineWrapping",!1,wg,!0),u("gutters",[],function(a,p){a.display.gutterSpecs=Cd(p,a.options.lineNumbers),Yu(a)},!0),u("fixedGutter",!0,function(a,p){a.display.gutters.style.left=p?fa(a.display)+"px":"0",a.refresh()},!0),u("coverGutterNextToScrollbar",!1,function(a){return Ls(a)},!0),u("scrollbarStyle","native",function(a){ml(a),Ls(a),a.display.scrollbars.setScrollTop(a.doc.scrollTop),a.display.scrollbars.setScrollLeft(a.doc.scrollLeft)},!0),u("lineNumbers",!1,function(a,p){a.display.gutterSpecs=Cd(a.options.gutters,p),Yu(a)},!0),u("firstLineNumber",1,Yu,!0),u("lineNumberFormatter",function(a){return a},Yu,!0),u("showCursorWhenSelecting",!1,$u,!0),u("resetSelectionOnContextMenu",!0),u("lineWiseCopyCut",!0),u("pasteLinesPerSelection",!0),u("selectionsMayTouch",!1),u("readOnly",!1,function(a,p){p=="nocursor"&&(pl(a),a.display.input.blur()),a.display.input.readOnlyChanged(p)}),u("screenReaderLabel",null,function(a,p){p=p===""?null:p,a.display.input.screenReaderLabelChanged(p)}),u("disableInput",!1,function(a,p){p||a.display.input.reset()},!0),u("dragDrop",!0,sw),u("allowDropFileTypes",null),u("cursorBlinkRate",530),u("cursorScrollMargin",0),u("cursorHeight",1,$u,!0),u("singleCursorHeightPerLine",!0,$u,!0),u("workTime",100),u("workDelay",100),u("flattenSpans",!0,Xu,!0),u("addModeClass",!1,Xu,!0),u("pollInterval",100),u("undoDepth",200,function(a,p){return a.doc.history.undoDepth=p}),u("historyEventDelay",1250),u("viewportMargin",10,function(a){return a.refresh()},!0),u("maxHighlightLength",1e4,Xu,!0),u("moveInputWithCursor",!0,function(a,p){p||a.display.input.resetPosition()}),u("tabindex",null,function(a,p){return a.display.input.getField().tabIndex=p||""}),u("autofocus",null),u("direction","ltr",function(a,p){return a.doc.setDirection(p)},!0),u("phrases",null)}o(Fc,"defineOptions");function sw(r,i,u){var a=u&&u!=Sl;if(!i!=!a){var p=r.display.dragFunctions,g=i?H:he;g(r.display.scroller,"dragstart",p.start),g(r.display.scroller,"dragenter",p.enter),g(r.display.scroller,"dragover",p.over),g(r.display.scroller,"dragleave",p.leave),g(r.display.scroller,"drop",p.drop)}}o(sw,"dragDropChanged");function wg(r){r.options.lineWrapping?(Xe(r.display.wrapper,"CodeMirror-wrap"),r.display.sizer.style.minWidth="",r.display.sizerWidth=null):(xe(r.display.wrapper,"CodeMirror-wrap"),zi(r)),bs(r),et(r),U(r),setTimeout(function(){return Ls(r)},100)}o(wg,"wrappingChanged");function Mt(r,i){var u=this;if(!(this instanceof Mt))return new Mt(r,i);this.options=i=i?Qt(i):{},Qt(lf,i,!1);var a=i.value;typeof a=="string"?a=new yn(a,i.mode,null,i.lineSeparator,i.direction):i.mode&&(a.modeOption=i.mode),this.doc=a;var p=new Mt.inputStyles[i.inputStyle](this),g=this.display=new Gy(r,a,p,i);g.wrapper.CodeMirror=this,sf(this),i.lineWrapping&&(this.display.wrapper.className+=" CodeMirror-wrap"),ml(this),this.state={keyMaps:[],overlays:[],modeGen:0,overwrite:!1,delayingBlurEvent:!1,focused:!1,suppressEdits:!1,pasteIncoming:-1,cutIncoming:-1,selectingText:!1,draggingText:!1,highlight:new Ct,keySeq:null,specialChars:null},i.autofocus&&!A&&g.input.focus(),c&&v<11&&setTimeout(function(){return u.display.input.reset(!0)},20),xg(this),Dd(),Ki(this),this.curOp.forceUpdate=!0,Km(this,a),i.autofocus&&!A||this.hasFocus()?setTimeout(function(){u.hasFocus()&&!u.state.focused&&pa(u)},20):pl(this);for(var y in Pa)Pa.hasOwnProperty(y)&&Pa[y](this,i[y],Sl);vo(this),i.finishInit&&i.finishInit(this);for(var S=0;S20*20}o(y,"farAway"),H(i.scroller,"touchstart",function(b){if(!ir(r,b)&&!g(b)&&!zd(r,b)){i.input.ensurePolled(),clearTimeout(u);var E=+new Date;i.activeTouch={start:E,moved:!1,prev:E-a.end<=300?a:null},b.touches.length==1&&(i.activeTouch.left=b.touches[0].pageX,i.activeTouch.top=b.touches[0].pageY)}}),H(i.scroller,"touchmove",function(){i.activeTouch&&(i.activeTouch.moved=!0)}),H(i.scroller,"touchend",function(b){var E=i.activeTouch;if(E&&!ji(i,b)&&E.left!=null&&!E.moved&&new Date-E.start<300){var M=r.coordsChar(i.activeTouch,"page"),D;!E.prev||y(E,E.prev)?D=new wt(M,M):!E.prev.prev||y(E,E.prev.prev)?D=r.findWordAt(M):D=new wt(ue(M.line,0),He(r.doc,ue(M.line+1,0))),r.setSelection(D.anchor,D.head),r.focus(),or(b)}p()}),H(i.scroller,"touchcancel",p),H(i.scroller,"scroll",function(){i.scroller.clientHeight&&(sr(r,i.scroller.scrollTop),hl(r,i.scroller.scrollLeft,!0),Te(r,"scroll",r))}),H(i.scroller,"mousewheel",function(b){return qm(r,b)}),H(i.scroller,"DOMMouseScroll",function(b){return qm(r,b)}),H(i.wrapper,"scroll",function(){return i.wrapper.scrollTop=i.wrapper.scrollLeft=0}),i.dragFunctions={enter:function(b){ir(r,b)||lo(b)},over:function(b){ir(r,b)||(Md(r,b),lo(b))},start:function(b){return ew(r,b)},drop:lr(r,sg),leave:function(b){ir(r,b)||Ad(r)}};var S=i.input.getField();H(S,"keyup",function(b){return Ud.call(r,b)}),H(S,"keydown",lr(r,ug)),H(S,"keypress",lr(r,cg)),H(S,"focus",function(b){return pa(r,b)}),H(S,"blur",function(b){return pl(r,b)})}o(xg,"registerEventHandlers");var af=[];Mt.defineInitHook=function(r){return af.push(r)};function uf(r,i,u,a){var p=r.doc,g;u==null&&(u="add"),u=="smart"&&(p.mode.indent?g=$o(r,i).state:u="prev");var y=r.options.tabSize,S=Me(p,i),b=_t(S.text,null,y);S.stateAfter&&(S.stateAfter=null);var E=S.text.match(/^\s*/)[0],M;if(!a&&!/\S/.test(S.text))M=0,u="not";else if(u=="smart"&&(M=p.mode.indent(g,S.text.slice(E.length),S.text),M==zt||M>150)){if(!a)return;u="prev"}u=="prev"?i>p.first?M=_t(Me(p,i-1).text,null,y):M=0:u=="add"?M=b+r.options.indentUnit:u=="subtract"?M=b-r.options.indentUnit:typeof u=="number"&&(M=b+u),M=Math.max(0,M);var D="",V=0;if(r.options.indentWithTabs)for(var $=Math.floor(M/y);$;--$)V+=y,D+=" ";if(Vy,b=rl(i),E=null;if(S&&a.ranges.length>1)if(Pi&&Pi.text.join(` + left: `+i.left+"px; width: "+Math.max(2,i.right-i.left)+"px;");r.display.lineSpace.appendChild(g),g.scrollIntoView(p),r.display.lineSpace.removeChild(g)}}}o(Ry,"maybeScrollWindow");function Iy(r,i,u,a){a==null&&(a=0);var p;!r.options.lineWrapping&&i==u&&(u=i.sticky=="before"?ue(i.line,i.ch+1,"before"):i,i=i.ch?ue(i.line,i.sticky=="before"?i.ch-1:i.ch,"after"):i);for(var g=0;g<5;g++){var y=!1,S=Vn(r,i),b=!u||u==i?S:Vn(r,u);p={left:Math.min(S.left,b.left),top:Math.min(S.top,b.top)-a,right:Math.max(S.left,b.left),bottom:Math.max(S.bottom,b.bottom)+a};var E=da(r,p),M=r.doc.scrollTop,D=r.doc.scrollLeft;if(E.scrollTop!=null&&(sr(r,E.scrollTop),Math.abs(r.doc.scrollTop-M)>1&&(y=!0)),E.scrollLeft!=null&&(hl(r,E.scrollLeft),Math.abs(r.doc.scrollLeft-D)>1&&(y=!0)),!y)break}return p}o(Iy,"scrollPosIntoView");function Fy(r,i){var u=da(r,i);u.scrollTop!=null&&sr(r,u.scrollTop),u.scrollLeft!=null&&hl(r,u.scrollLeft)}o(Fy,"scrollIntoView");function da(r,i){var u=r.display,a=Cs(r.display);i.top<0&&(i.top=0);var p=r.curOp&&r.curOp.scrollTop!=null?r.curOp.scrollTop:u.scroller.scrollTop,g=al(r),y={};i.bottom-i.top>g&&(i.bottom=i.top+g);var S=r.doc.height+zr(u),b=i.topS-a;if(i.topp+g){var M=Math.min(i.top,(E?S:i.bottom)-g);M!=p&&(y.scrollTop=M)}var D=r.options.fixedGutter?0:u.gutters.offsetWidth,V=r.curOp&&r.curOp.scrollLeft!=null?r.curOp.scrollLeft:u.scroller.scrollLeft-D,$=qi(r)-u.gutters.offsetWidth,te=i.right-i.left>$;return te&&(i.right=i.left+$),i.left<10?y.scrollLeft=0:i.left$+V-3&&(y.scrollLeft=i.right+(te?0:10)-$),y}o(da,"calculateScrollPos");function ha(r,i){i!=null&&(mc(r),r.curOp.scrollTop=(r.curOp.scrollTop==null?r.doc.scrollTop:r.curOp.scrollTop)+i)}o(ha,"addToScrollTop");function ks(r){mc(r);var i=r.getCursor();r.curOp.scrollToPos={from:i,to:i,margin:r.options.cursorScrollMargin}}o(ks,"ensureCursorVisible");function qu(r,i,u){(i!=null||u!=null)&&mc(r),i!=null&&(r.curOp.scrollLeft=i),u!=null&&(r.curOp.scrollTop=u)}o(qu,"scrollToCoords");function Wm(r,i){mc(r),r.curOp.scrollToPos=i}o(Wm,"scrollToRange");function mc(r){var i=r.curOp.scrollToPos;if(i){r.curOp.scrollToPos=null;var u=zu(r,i.from),a=zu(r,i.to);Um(r,u,a,i.margin)}}o(mc,"resolveScrollToPos");function Um(r,i,u,a){var p=da(r,{left:Math.min(i.left,u.left),top:Math.min(i.top,u.top)-a,right:Math.max(i.right,u.right),bottom:Math.max(i.bottom,u.bottom)+a});qu(r,p.scrollLeft,p.scrollTop)}o(Um,"scrollToCoordsRange");function sr(r,i){Math.abs(r.doc.scrollTop-i)<2||(n||wd(r,{top:i}),Zr(r,i,!0),n&&wd(r),go(r,100))}o(sr,"updateScrollTop");function Zr(r,i,u){i=Math.max(0,Math.min(r.display.scroller.scrollHeight-r.display.scroller.clientHeight,i)),!(r.display.scroller.scrollTop==i&&!u)&&(r.doc.scrollTop=i,r.display.scrollbars.setScrollTop(i),r.display.scroller.scrollTop!=i&&(r.display.scroller.scrollTop=i))}o(Zr,"setScrollTop");function hl(r,i,u,a){i=Math.max(0,Math.min(i,r.display.scroller.scrollWidth-r.display.scroller.clientWidth)),!((u?i==r.doc.scrollLeft:Math.abs(r.doc.scrollLeft-i)<2)&&!a)&&(r.doc.scrollLeft=i,zm(r),r.display.scroller.scrollLeft!=i&&(r.display.scroller.scrollLeft=i),r.display.scrollbars.setScrollLeft(i))}o(hl,"setScrollLeft");function Vu(r){var i=r.display,u=i.gutters.offsetWidth,a=Math.round(r.doc.height+zr(r.display));return{clientHeight:i.scroller.clientHeight,viewHeight:i.wrapper.clientHeight,scrollWidth:i.scroller.scrollWidth,clientWidth:i.scroller.clientWidth,viewWidth:i.wrapper.clientWidth,barLeft:r.options.fixedGutter?u:0,docHeight:a,scrollHeight:a+mn(r)+i.barHeight,nativeBarWidth:i.nativeBarWidth,gutterWidth:u}}o(Vu,"measureForScrollbars");var Ns=o(function(r,i,u){this.cm=u;var a=this.vert=_e("div",[_e("div",null,null,"min-width: 1px")],"CodeMirror-vscrollbar"),p=this.horiz=_e("div",[_e("div",null,null,"height: 100%; min-height: 1px")],"CodeMirror-hscrollbar");a.tabIndex=p.tabIndex=-1,r(a),r(p),H(a,"scroll",function(){a.clientHeight&&i(a.scrollTop,"vertical")}),H(p,"scroll",function(){p.clientWidth&&i(p.scrollLeft,"horizontal")}),this.checkedZeroWidth=!1,c&&v<8&&(this.horiz.style.minHeight=this.vert.style.minWidth="18px")},"NativeScrollbars");Ns.prototype.update=function(r){var i=r.scrollWidth>r.clientWidth+1,u=r.scrollHeight>r.clientHeight+1,a=r.nativeBarWidth;if(u){this.vert.style.display="block",this.vert.style.bottom=i?a+"px":"0";var p=r.viewHeight-(i?a:0);this.vert.firstChild.style.height=Math.max(0,r.scrollHeight-r.clientHeight+p)+"px"}else this.vert.style.display="",this.vert.firstChild.style.height="0";if(i){this.horiz.style.display="block",this.horiz.style.right=u?a+"px":"0",this.horiz.style.left=r.barLeft+"px";var g=r.viewWidth-r.barLeft-(u?a:0);this.horiz.firstChild.style.width=Math.max(0,r.scrollWidth-r.clientWidth+g)+"px"}else this.horiz.style.display="",this.horiz.firstChild.style.width="0";return!this.checkedZeroWidth&&r.clientHeight>0&&(a==0&&this.zeroWidthHack(),this.checkedZeroWidth=!0),{right:u?a:0,bottom:i?a:0}},Ns.prototype.setScrollLeft=function(r){this.horiz.scrollLeft!=r&&(this.horiz.scrollLeft=r),this.disableHoriz&&this.enableZeroWidthBar(this.horiz,this.disableHoriz,"horiz")},Ns.prototype.setScrollTop=function(r){this.vert.scrollTop!=r&&(this.vert.scrollTop=r),this.disableVert&&this.enableZeroWidthBar(this.vert,this.disableVert,"vert")},Ns.prototype.zeroWidthHack=function(){var r=I&&!X?"12px":"18px";this.horiz.style.height=this.vert.style.width=r,this.horiz.style.pointerEvents=this.vert.style.pointerEvents="none",this.disableHoriz=new Ct,this.disableVert=new Ct},Ns.prototype.enableZeroWidthBar=function(r,i,u){r.style.pointerEvents="auto";function a(){var p=r.getBoundingClientRect(),g=u=="vert"?document.elementFromPoint(p.right-1,(p.top+p.bottom)/2):document.elementFromPoint((p.right+p.left)/2,p.bottom-1);g!=r?r.style.pointerEvents="none":i.set(1e3,a)}o(a,"maybeDisable"),i.set(1e3,a)},Ns.prototype.clear=function(){var r=this.horiz.parentNode;r.removeChild(this.horiz),r.removeChild(this.vert)};var Ku=o(function(){},"NullScrollbars");Ku.prototype.update=function(){return{bottom:0,right:0}},Ku.prototype.setScrollLeft=function(){},Ku.prototype.setScrollTop=function(){},Ku.prototype.clear=function(){};function Ls(r,i){i||(i=Vu(r));var u=r.display.barWidth,a=r.display.barHeight;ma(r,i);for(var p=0;p<4&&u!=r.display.barWidth||a!=r.display.barHeight;p++)u!=r.display.barWidth&&r.options.lineWrapping&&_s(r),ma(r,Vu(r)),u=r.display.barWidth,a=r.display.barHeight}o(Ls,"updateScrollbars");function ma(r,i){var u=r.display,a=u.scrollbars.update(i);u.sizer.style.paddingRight=(u.barWidth=a.right)+"px",u.sizer.style.paddingBottom=(u.barHeight=a.bottom)+"px",u.heightForcer.style.borderBottom=a.bottom+"px solid transparent",a.right&&a.bottom?(u.scrollbarFiller.style.display="block",u.scrollbarFiller.style.height=a.bottom+"px",u.scrollbarFiller.style.width=a.right+"px"):u.scrollbarFiller.style.display="",a.bottom&&r.options.coverGutterNextToScrollbar&&r.options.fixedGutter?(u.gutterFiller.style.display="block",u.gutterFiller.style.height=a.bottom+"px",u.gutterFiller.style.width=i.gutterWidth+"px"):u.gutterFiller.style.display=""}o(ma,"updateScrollbarsInner");var gc={native:Ns,null:Ku};function ml(r){r.display.scrollbars&&(r.display.scrollbars.clear(),r.display.scrollbars.addClass&&xe(r.display.wrapper,r.display.scrollbars.addClass)),r.display.scrollbars=new gc[r.options.scrollbarStyle](function(i){r.display.wrapper.insertBefore(i,r.display.scrollbarFiller),H(i,"mousedown",function(){r.state.focused&&setTimeout(function(){return r.display.input.focus()},0)}),i.setAttribute("cm-not-content","true")},function(i,u){u=="horizontal"?hl(r,i):sr(r,i)},r),r.display.scrollbars.addClass&&Xe(r.display.wrapper,r.display.scrollbars.addClass)}o(ml,"initScrollbars");var Gu=0;function Ki(r){r.curOp={cm:r,viewChanged:!1,startHeight:r.doc.height,forceUpdate:!1,updateInput:0,typing:!1,changeObjs:null,cursorActivityHandlers:null,cursorActivityCalled:0,selectionChanged:!1,updateMaxLine:!1,scrollLeft:null,scrollTop:null,scrollToPos:null,focus:!1,id:++Gu,markArrays:null},$i(r.curOp)}o(Ki,"startOperation");function mo(r){var i=r.curOp;i&&hd(i,function(u){for(var a=0;a=u.viewTo)||u.maxLineChanged&&i.options.lineWrapping,r.update=r.mustUpdate&&new An(i,r.mustUpdate&&{top:r.scrollTop,ensure:r.scrollToPos},r.forceUpdate)}o(By,"endOperation_R1");function Hy(r){r.updatedDisplay=r.mustUpdate&&yd(r.cm,r.update)}o(Hy,"endOperation_W1");function Wy(r){var i=r.cm,u=i.display;r.updatedDisplay&&_s(i),r.barMeasure=Vu(i),u.maxLineChanged&&!i.options.lineWrapping&&(r.adjustWidthTo=Uu(i,u.maxLine,u.maxLine.text.length).left+3,i.display.sizerWidth=r.adjustWidthTo,r.barMeasure.scrollWidth=Math.max(u.scroller.clientWidth,u.sizer.offsetLeft+r.adjustWidthTo+mn(i)+i.display.barWidth),r.maxScrollLeft=Math.max(0,u.sizer.offsetLeft+r.adjustWidthTo-qi(i))),(r.updatedDisplay||r.selectionChanged)&&(r.preparedSelection=u.input.prepareSelection())}o(Wy,"endOperation_R2");function Uy(r){var i=r.cm;r.adjustWidthTo!=null&&(i.display.sizer.style.minWidth=r.adjustWidthTo+"px",r.maxScrollLeft=r.display.viewTo)){var u=+new Date+r.options.workTime,a=$o(r,i.highlightFrontier),p=[];i.iter(a.line,Math.min(i.first+i.size,r.display.viewTo+500),function(g){if(a.line>=r.display.viewFrom){var y=g.styles,S=g.text.length>r.options.maxHighlightLength?Uo(i.mode,a.state):null,b=tc(r,g,a,!0);S&&(a.state=S),g.styles=b.styles;var E=g.styleClasses,M=b.classes;M?g.styleClasses=M:E&&(g.styleClasses=null);for(var D=!y||y.length!=g.styles.length||E!=M&&(!E||!M||E.bgClass!=M.bgClass||E.textClass!=M.textClass),V=0;!D&&Vu)return go(r,r.options.workDelay),!0}),i.highlightFrontier=a.line,i.modeFrontier=Math.max(i.modeFrontier,a.line),p.length&&Jr(r,function(){for(var g=0;g=u.viewFrom&&i.visible.to<=u.viewTo&&(u.updateLineNumbers==null||u.updateLineNumbers>=u.viewTo)&&u.renderedView==u.view&&Bm(r)==0)return!1;vo(r)&&(Xo(r),i.dims=Kn(r));var p=a.first+a.size,g=Math.max(i.visible.from-r.options.viewportMargin,a.first),y=Math.min(p,i.visible.to+r.options.viewportMargin);u.viewFromy&&u.viewTo-y<20&&(y=Math.min(p,u.viewTo)),_i&&(g=jo(r.doc,g),y=pd(r.doc,y));var S=g!=u.viewFrom||y!=u.viewTo||u.lastWrapHeight!=i.wrapperHeight||u.lastWrapWidth!=i.wrapperWidth;Dy(r,g,y),u.viewOffset=hn(Me(r.doc,u.viewFrom)),r.display.mover.style.top=u.viewOffset+"px";var b=Bm(r);if(!S&&b==0&&!i.force&&u.renderedView==u.view&&(u.updateLineNumbers==null||u.updateLineNumbers>=u.viewTo))return!1;var E=$y(r);return b>4&&(u.lineDiv.style.display="none"),qy(r,u.updateLineNumbers,i.dims),b>4&&(u.lineDiv.style.display=""),u.renderedView=u.view,jy(E),Ve(u.cursorDiv),Ve(u.selectionDiv),u.gutters.style.height=u.sizer.style.minHeight=0,S&&(u.lastWrapHeight=i.wrapperHeight,u.lastWrapWidth=i.wrapperWidth,go(r,400)),u.updateLineNumbers=null,!0}o(yd,"updateDisplayIfNeeded");function Ps(r,i){for(var u=i.viewport,a=!0;;a=!1){if(!a||!r.options.lineWrapping||i.oldDisplayWidth==qi(r)){if(u&&u.top!=null&&(u={top:Math.min(r.doc.height+zr(r.display)-al(r),u.top)}),i.visible=dl(r.display,r.doc,u),i.visible.from>=r.display.viewFrom&&i.visible.to<=r.display.viewTo)break}else a&&(i.visible=dl(r.display,r.doc,u));if(!yd(r,i))break;_s(r);var p=Vu(r);$u(r),Ls(r,p),Sd(r,p),i.force=!1}i.signal(r,"update",r),(r.display.viewFrom!=r.display.reportedViewFrom||r.display.viewTo!=r.display.reportedViewTo)&&(i.signal(r,"viewportChange",r,r.display.viewFrom,r.display.viewTo),r.display.reportedViewFrom=r.display.viewFrom,r.display.reportedViewTo=r.display.viewTo)}o(Ps,"postUpdateDisplay");function wd(r,i){var u=new An(r,i);if(yd(r,u)){_s(r),Ps(r,u);var a=Vu(r);$u(r),Ls(r,a),Sd(r,a),u.finish()}}o(wd,"updateDisplaySimple");function qy(r,i,u){var a=r.display,p=r.options.lineNumbers,g=a.lineDiv,y=g.firstChild;function S(te){var oe=te.nextSibling;return C&&I&&r.display.currentWheelTarget==te?te.style.display="none":te.parentNode.removeChild(te),oe}o(S,"rm");for(var b=a.view,E=a.viewFrom,M=0;M-1&&($=!1),Du(r,D,E,u)),$&&(Ve(D.lineNumber),D.lineNumber.appendChild(document.createTextNode(jl(r.options,E)))),y=D.node.nextSibling}E+=D.size}for(;y;)y=S(y)}o(qy,"patchDisplay");function xd(r){var i=r.gutters.offsetWidth;r.sizer.style.marginLeft=i+"px",yr(r,"gutterChanged",r)}o(xd,"updateGutterSpace");function Sd(r,i){r.display.sizer.style.minHeight=i.docHeight+"px",r.display.heightForcer.style.top=i.docHeight+"px",r.display.gutters.style.height=i.docHeight+r.display.barHeight+mn(r)+"px"}o(Sd,"setDocumentHeight");function zm(r){var i=r.display,u=i.view;if(!(!i.alignWidgets&&(!i.gutters.firstChild||!r.options.fixedGutter))){for(var a=fa(i)-i.scroller.scrollLeft+r.doc.scrollLeft,p=i.gutters.offsetWidth,g=a+"px",y=0;yy.clientWidth,b=y.scrollHeight>y.clientHeight;if(!!(a&&S||p&&b)){if(p&&I&&C){e:for(var E=i.target,M=g.view;E!=y;E=E.parentNode)for(var D=0;D=0&&ze(r,a.to())<=0)return u}return-1};var wt=o(function(r,i){this.anchor=r,this.head=i},"Range");wt.prototype.from=function(){return il(this.anchor,this.head)},wt.prototype.to=function(){return gs(this.anchor,this.head)},wt.prototype.empty=function(){return this.head.line==this.anchor.line&&this.head.ch==this.anchor.ch};function tn(r,i,u){var a=r&&r.options.selectionsMayTouch,p=i[u];i.sort(function(V,$){return ze(V.from(),$.from())}),u=ut(i,p);for(var g=1;g0:b>=0){var E=il(S.from(),y.from()),M=gs(S.to(),y.to()),D=S.empty()?y.from()==y.head:S.from()==S.head;g<=u&&--u,i.splice(--g,2,new wt(D?M:E,D?E:M))}}return new pi(i,u)}o(tn,"normalizeSelection");function Os(r,i){return new pi([new wt(r,i||r)],0)}o(Os,"simpleSelection");function Ms(r){return r.text?ue(r.from.line+r.text.length-1,Se(r.text).length+(r.text.length==1?r.from.ch:0)):r.to}o(Ms,"changeEnd");function Ni(r,i){if(ze(r,i.from)<0)return r;if(ze(r,i.to)<=0)return Ms(i);var u=r.line+i.text.length-(i.to.line-i.from.line)-1,a=r.ch;return r.line==i.to.line&&(a+=Ms(i).ch-i.to.ch),ue(u,a)}o(Ni,"adjustForChange");function bd(r,i){for(var u=[],a=0;a1&&r.remove(S.line+1,te-1),r.insert(S.line+1,ge)}yr(r,"change",r,i)}o(wc,"updateDoc");function As(r,i,u){function a(p,g,y){if(p.linked)for(var S=0;S1&&!r.done[r.done.length-2].ranges)return r.done.pop(),Se(r.done)}o(Yy,"lastChangeEvent");function yo(r,i,u,a){var p=r.history;p.undone.length=0;var g=+new Date,y,S;if((p.lastOp==a||p.lastOrigin==i.origin&&i.origin&&(i.origin.charAt(0)=="+"&&p.lastModTime>g-(r.cm?r.cm.options.historyEventDelay:500)||i.origin.charAt(0)=="*"))&&(y=Yy(p,p.lastOp==a)))S=Se(y.changes),ze(i.from,i.to)==0&&ze(i.from,S.to)==0?S.to=Ms(i):y.changes.push(Td(r,i));else{var b=Se(p.done);for((!b||!b.ranges)&&Dn(r.sel,p.done),y={changes:[Td(r,i)],generation:p.generation},p.done.push(y);p.done.length>p.undoDepth;)p.done.shift(),p.done[0].ranges||p.done.shift()}p.done.push(u),p.generation=++p.maxGeneration,p.lastModTime=p.lastSelTime=g,p.lastOp=p.lastSelOp=a,p.lastOrigin=p.lastSelOrigin=i.origin,S||Te(r,"historyAdded")}o(yo,"addChangeToHistory");function Nd(r,i,u,a){var p=i.charAt(0);return p=="*"||p=="+"&&u.ranges.length==a.ranges.length&&u.somethingSelected()==a.somethingSelected()&&new Date-r.history.lastSelTime<=(r.cm?r.cm.options.historyEventDelay:500)}o(Nd,"selectionEventCanBeMerged");function vl(r,i,u,a){var p=r.history,g=a&&a.origin;u==p.lastSelOp||g&&p.lastSelOrigin==g&&(p.lastModTime==p.lastSelTime&&p.lastOrigin==g||Nd(r,g,Se(p.done),i))?p.done[p.done.length-1]=i:Dn(i,p.done),p.lastSelTime=+new Date,p.lastSelOrigin=g,p.lastSelOp=u,a&&a.clearRedo!==!1&&kd(p.undone)}o(vl,"addSelectionToHistory");function Dn(r,i){var u=Se(i);u&&u.ranges&&u.equals(r)||i.push(r)}o(Dn,"pushSelectionToHistory");function Gm(r,i,u,a){var p=i["spans_"+r.id],g=0;r.iter(Math.max(r.first,u),Math.min(r.first+r.size,a),function(y){y.markedSpans&&((p||(p=i["spans_"+r.id]={}))[g]=y.markedSpans),++g})}o(Gm,"attachLocalSpans");function Ym(r){if(!r)return null;for(var i,u=0;u-1&&(Se(S)[D]=E[D],delete E[D])}}return a}o(di,"copyHistoryArray");function Sc(r,i,u,a){if(a){var p=r.anchor;if(u){var g=ze(i,p)<0;g!=ze(u,p)<0?(p=i,i=u):g!=ze(i,u)<0&&(i=u)}return new wt(p,i)}else return new wt(u||i,i)}o(Sc,"extendRange");function Cc(r,i,u,a,p){p==null&&(p=r.cm&&(r.cm.display.shift||r.extend)),$r(r,new pi([Sc(r.sel.primary(),i,u,p)],0),a)}o(Cc,"extendSelection");function Zu(r,i,u){for(var a=[],p=r.cm&&(r.cm.display.shift||r.extend),g=0;g=i.ch:S.to>i.ch))){if(p&&(Te(b,"beforeCursorEnter"),b.explicitlyCleared))if(g.markedSpans){--y;continue}else break;if(!b.atomic)continue;if(u){var D=b.find(a<0?1:-1),V=void 0;if((a<0?M:E)&&(D=_c(r,D,-a,D&&D.line==i.line?g:null)),D&&D.line==i.line&&(V=ze(D,u))&&(a<0?V<0:V>0))return yl(r,D,i,a,p)}var $=b.find(a<0?-1:1);return(a<0?E:M)&&($=_c(r,$,a,$.line==i.line?g:null)),$?yl(r,$,i,a,p):null}}return i}o(yl,"skipAtomicInner");function jr(r,i,u,a,p){var g=a||1,y=yl(r,i,u,g,p)||!p&&yl(r,i,u,g,!0)||yl(r,i,u,-g,p)||!p&&yl(r,i,u,-g,!0);return y||(r.cantEdit=!0,ue(r.first,0))}o(jr,"skipAtomic");function _c(r,i,u,a){return u<0&&i.ch==0?i.line>r.first?He(r,ue(i.line-1)):null:u>0&&i.ch==(a||Me(r,i.line)).text.length?i.line=0;--p)Tc(r,{from:a[p].from,to:a[p].to,text:p?[""]:i.text,origin:i.origin});else Tc(r,i)}}o(ya,"makeChange");function Tc(r,i){if(!(i.text.length==1&&i.text[0]==""&&ze(i.from,i.to)==0)){var u=bd(r,i);yo(r,i,u,r.cm?r.cm.curOp.id:NaN),xa(r,i,u,ku(r,i));var a=[];As(r,function(p,g){!g&&ut(a,p.history)==-1&&(eg(p.history,i),a.push(p.history)),xa(p,i,null,ku(p,i))})}}o(Tc,"makeChangeInner");function kc(r,i,u){var a=r.cm&&r.cm.state.suppressEdits;if(!(a&&!u)){for(var p=r.history,g,y=r.sel,S=i=="undo"?p.done:p.undone,b=i=="undo"?p.undone:p.done,E=0;E=0;--$){var te=V($);if(te)return te.v}}}}o(kc,"makeChangeFromHistory");function wa(r,i){if(i!=0&&(r.first+=i,r.sel=new pi(Or(r.sel.ranges,function(p){return new wt(ue(p.anchor.line+i,p.anchor.ch),ue(p.head.line+i,p.head.ch))}),r.sel.primIndex),r.cm)){et(r.cm,r.first,r.first-i,i);for(var u=r.cm.display,a=u.viewFrom;ar.lastLine())){if(i.from.lineg&&(i={from:i.from,to:ue(g,Me(r,g).text.length),text:[i.text[0]],origin:i.origin}),i.removed=Ei(r,i.from,i.to),u||(u=bd(r,i)),r.cm?Xy(r.cm,i,a):wc(r,i,a),hi(r,u,$t),r.cantEdit&&jr(r,ue(r.firstLine(),0))&&(r.cantEdit=!1)}}o(xa,"makeChangeSingleDoc");function Xy(r,i,u){var a=r.doc,p=r.display,g=i.from,y=i.to,S=!1,b=g.line;r.options.lineWrapping||(b=vt(vr(Me(a,g.line))),a.iter(b,y.line+1,function($){if($==p.maxLine)return S=!0,!0})),a.sel.contains(i.from,i.to)>-1&&Ul(r),wc(a,i,u,Fm(r)),r.options.lineWrapping||(a.iter(b,g.line+i.text.length,function($){var te=ws($);te>p.maxLineLength&&(p.maxLine=$,p.maxLineLength=te,p.maxLineChanged=!0,S=!1)}),S&&(r.curOp.updateMaxLine=!0)),rc(a,g.line),go(r,400);var E=i.text.length-(y.line-g.line)-1;i.full?et(r):g.line==y.line&&i.text.length==1&&!_d(r.doc,i)?Es(r,g.line,"text"):et(r,g.line,y.line+1,E);var M=Ft(r,"changes"),D=Ft(r,"change");if(D||M){var V={from:g,to:y,text:i.text,removed:i.removed,origin:i.origin};D&&yr(r,"change",r,V),M&&(r.curOp.changeObjs||(r.curOp.changeObjs=[])).push(V)}r.display.selForContextMenu=null}o(Xy,"makeChangeSingleDocInEditor");function Sa(r,i,u,a,p){var g;a||(a=u),ze(a,u)<0&&(g=[a,u],u=g[0],a=g[1]),typeof i=="string"&&(i=r.splitLines(i)),ya(r,{from:u,to:a,text:i,origin:p})}o(Sa,"replaceRange");function Ca(r,i,u,a){u1||!(this.children[0]instanceof ba))){var S=[];this.collapse(S),this.children=[new ba(S)],this.children[0].parent=this}},collapse:function(r){for(var i=0;i50){for(var y=p.lines.length%25+25,S=y;S10);r.parent.maybeSpill()}},iterN:function(r,i,u){for(var a=0;ar.display.maxLineLength&&(r.display.maxLine=E,r.display.maxLineLength=M,r.display.maxLineChanged=!0)}a!=null&&r&&this.collapsed&&et(r,a,p+1),this.lines.length=0,this.explicitlyCleared=!0,this.atomic&&this.doc.cantEdit&&(this.doc.cantEdit=!1,r&&Ju(r.doc)),r&&yr(r,"markerCleared",r,this,a,p),i&&mo(r),this.parent&&this.parent.clear()}},Rs.prototype.find=function(r,i){r==null&&this.type=="bookmark"&&(r=1);for(var u,a,p=0;p0||y==0&&g.clearWhenEmpty!==!1)return g;if(g.replacedWith&&(g.collapsed=!0,g.widgetNode=St("span",[g.replacedWith],"CodeMirror-widget"),a.handleMouseEvents||g.widgetNode.setAttribute("cm-ignore-events","true"),a.insertLeft&&(g.widgetNode.insertLeft=!0)),g.collapsed){if(sl(r,i.line,i,u,g)||i.line!=u.line&&sl(r,u.line,i,u,g))throw new Error("Inserting collapsed marker partially overlapping an existing one");Ql()}g.addToHistory&&yo(r,{from:i,to:u,origin:"markText"},r.sel,NaN);var S=i.line,b=r.cm,E;if(r.iter(S,u.line+1,function(D){b&&g.collapsed&&!b.options.lineWrapping&&vr(D)==b.display.maxLine&&(E=!0),g.collapsed&&S!=i.line&&ai(D,0),oc(D,new co(g,S==i.line?i.ch:null,S==u.line?u.ch:null),r.cm&&r.cm.curOp),++S}),g.collapsed&&r.iter(i.line,u.line+1,function(D){qt(r,D)&&ai(D,0)}),g.clearOnEnter&&H(g,"beforeCursorEnter",function(){return g.clear()}),g.readOnly&&(nc(),(r.history.done.length||r.history.undone.length)&&r.clearHistory()),g.collapsed&&(g.id=++Nc,g.atomic=!0),b){if(E&&(b.curOp.updateMaxLine=!0),g.collapsed)et(b,i.line,u.line+1);else if(g.className||g.startStyle||g.endStyle||g.css||g.attributes||g.title)for(var M=i.line;M<=u.line;M++)Es(b,M,"text");g.atomic&&Ju(b.doc),yr(b,"markerAdded",b,g)}return g}o(Is,"markText");var Ea=o(function(r,i){this.markers=r,this.primary=i;for(var u=0;u=0;b--)ya(this,a[b]);S?bc(this,S):this.cm&&ks(this.cm)}),undo:N(function(){kc(this,"undo")}),redo:N(function(){kc(this,"redo")}),undoSelection:N(function(){kc(this,"undo",!0)}),redoSelection:N(function(){kc(this,"redo",!0)}),setExtending:function(r){this.extend=r},getExtending:function(){return this.extend},historySize:function(){for(var r=this.history,i=0,u=0,a=0;a=r.ch)&&i.push(p.marker.parent||p.marker)}return i},findMarks:function(r,i,u){r=He(this,r),i=He(this,i);var a=[],p=r.line;return this.iter(r.line,i.line+1,function(g){var y=g.markedSpans;if(y)for(var S=0;S=b.to||b.from==null&&p!=r.line||b.from!=null&&p==i.line&&b.from>=i.ch)&&(!u||u(b.marker))&&a.push(b.marker.parent||b.marker)}++p}),a},getAllMarks:function(){var r=[];return this.iter(function(i){var u=i.markedSpans;if(u)for(var a=0;ar)return i=r,!0;r-=g,++u}),He(this,ue(u,i))},indexFromPos:function(r){r=He(this,r);var i=r.ch;if(r.linei&&(i=r.from),r.to!=null&&r.to-1){i.state.draggingText(r),setTimeout(function(){return i.display.input.focus()},20);return}try{var M=r.dataTransfer.getData("Text");if(M){var D;if(i.state.draggingText&&!i.state.draggingText.copy&&(D=i.listSelections()),hi(i.doc,Os(u,u)),D)for(var V=0;V=0;S--)Sa(r.doc,"",a[S].from,a[S].to,"+delete");ks(r)})}o(mi,"deleteNearSelection");function of(r,i,u){var a=Ui(r.text,i+u,u);return a<0||a>r.text.length?null:a}o(of,"moveCharLogically");function Mc(r,i,u){var a=of(r,i.ch,u);return a==null?null:new ue(i.line,a,u<0?"after":"before")}o(Mc,"moveLogically");function _a(r,i,u,a,p){if(r){i.doc.direction=="rtl"&&(p=-p);var g=Mn(u,i.doc.direction);if(g){var y=p<0?Se(g):g[0],S=p<0==(y.level==1),b=S?"after":"before",E;if(y.level>0||i.doc.direction=="rtl"){var M=qn(i,u);E=p<0?u.text.length-1:0;var D=Ti(i,M,E).top;E=pn(function(V){return Ti(i,M,V).top==D},p<0==(y.level==1)?y.from:y.to-1,E),b=="before"&&(E=of(u,E,1))}else E=p<0?y.to:y.from;return new ue(a,E,b)}}return new ue(a,p<0?u.text.length:0,p<0?"before":"after")}o(_a,"endOfLine");function ag(r,i,u,a){var p=Mn(i,r.doc.direction);if(!p)return Mc(i,u,a);u.ch>=i.text.length?(u.ch=i.text.length,u.sticky="before"):u.ch<=0&&(u.ch=0,u.sticky="after");var g=Ci(p,u.ch,u.sticky),y=p[g];if(r.doc.direction=="ltr"&&y.level%2==0&&(a>0?y.to>u.ch:y.from=y.from&&V>=M.begin)){var $=D?"before":"after";return new ue(u.line,V,$)}}var te=o(function(ge,be,ve){for(var Ne=o(function(Tt,br){return br?new ue(u.line,S(Tt,1),"before"):new ue(u.line,Tt,"after")},"getRes");ge>=0&&ge0==(Ie.level!=1),it=De?ve.begin:S(ve.end,-1);if(Ie.from<=it&&it0?M.end:S(M.begin,-1);return de!=null&&!(a>0&&de==i.text.length)&&(oe=te(a>0?0:p.length-1,a,E(de)),oe)?oe:null}o(ag,"moveVisually");var xl={selectAll:Qm,singleSelection:function(r){return r.setSelection(r.getCursor("anchor"),r.getCursor("head"),$t)},killLine:function(r){return mi(r,function(i){if(i.empty()){var u=Me(r.doc,i.head.line).text.length;return i.head.ch==u&&i.head.line0)p=new ue(p.line,p.ch+1),r.replaceRange(g.charAt(p.ch-1)+g.charAt(p.ch-2),ue(p.line,p.ch-2),p,"+transpose");else if(p.line>r.doc.first){var y=Me(r.doc,p.line-1).text;y&&(p=new ue(p.line,1),r.replaceRange(g.charAt(0)+r.doc.lineSeparator()+y.charAt(y.length-1),ue(p.line-1,y.length-1),p,"+transpose"))}}u.push(new wt(p,p))}r.setSelections(u)})},newlineAndIndent:function(r){return Jr(r,function(){for(var i=r.listSelections(),u=i.length-1;u>=0;u--)r.replaceRange(r.doc.lineSeparator(),i[u].anchor,i[u].head,"+input");i=r.listSelections();for(var a=0;ar&&ze(i,this.pos)==0&&u==this.button};var qr,Gn;function nw(r,i){var u=+new Date;return Gn&&Gn.compare(u,r,i)?(qr=Gn=null,"triple"):qr&&qr.compare(u,r,i)?(Gn=new Rc(u,r,i),qr=null,"double"):(qr=new Rc(u,r,i),Gn=null,"single")}o(nw,"clickRepeat");function dg(r){var i=this,u=i.display;if(!(ir(i,r)||u.activeTouch&&u.input.supportsTouch())){if(u.input.ensurePolled(),u.shift=r.shiftKey,ji(u,r)){C||(u.scroller.draggable=!1,setTimeout(function(){return u.scroller.draggable=!0},100));return}if(!zd(i,r)){var a=po(i,r),p=el(r),g=a?nw(a,p):"single";window.focus(),p==1&&i.state.selectingText&&i.state.selectingText(r),!(a&&Ic(i,p,a,g,r))&&(p==1?a?hg(i,a,g,r):bi(r)==u.scroller&&or(r):p==2?(a&&Cc(i.doc,a),setTimeout(function(){return u.input.focus()},20)):p==3&&(pe?i.display.input.onContextMenu(r):hc(i)))}}}o(dg,"onMouseDown");function Ic(r,i,u,a,p){var g="Click";return a=="double"?g="Double"+g:a=="triple"&&(g="Triple"+g),g=(i==1?"Left":i==2?"Middle":"Right")+g,Ta(r,Rd(g,p),p,function(y){if(typeof y=="string"&&(y=xl[y]),!y)return!1;var S=!1;try{r.isReadOnly()&&(r.state.suppressEdits=!0),S=y(r,u)!=zt}finally{r.state.suppressEdits=!1}return S})}o(Ic,"handleMappedButton");function ka(r,i,u){var a=r.getOption("configureMouse"),p=a?a(r,i,u):{};if(p.unit==null){var g=G?u.shiftKey&&u.metaKey:u.altKey;p.unit=g?"rectangle":i=="single"?"char":i=="double"?"word":"line"}return(p.extend==null||r.doc.extend)&&(p.extend=r.doc.extend||u.shiftKey),p.addNew==null&&(p.addNew=I?u.metaKey:u.ctrlKey),p.moveOnDrag==null&&(p.moveOnDrag=!(I?u.altKey:u.ctrlKey)),p}o(ka,"configureMouse");function hg(r,i,u,a){c?setTimeout(Hr(vd,r),0):r.curOp.focus=Ge();var p=ka(r,u,a),g=r.doc.sel,y;r.options.dragDrop&&hs&&!r.isReadOnly()&&u=="single"&&(y=g.contains(i))>-1&&(ze((y=g.ranges[y]).from(),i)<0||i.xRel>0)&&(ze(y.to(),i)>0||i.xRel<0)?mg(r,a,i,p):vg(r,a,i,p)}o(hg,"leftButtonDown");function mg(r,i,u,a){var p=r.display,g=!1,y=lr(r,function(E){C&&(p.scroller.draggable=!1),r.state.draggingText=!1,r.state.delayingBlurEvent&&(r.hasFocus()?r.state.delayingBlurEvent=!1:hc(r)),he(p.wrapper.ownerDocument,"mouseup",y),he(p.wrapper.ownerDocument,"mousemove",S),he(p.scroller,"dragstart",b),he(p.scroller,"drop",y),g||(or(E),a.addNew||Cc(r.doc,u,null,null,a.extend),C&&!B||c&&v==9?setTimeout(function(){p.wrapper.ownerDocument.body.focus({preventScroll:!0}),p.input.focus()},20):p.input.focus())}),S=o(function(E){g=g||Math.abs(i.clientX-E.clientX)+Math.abs(i.clientY-E.clientY)>=10},"mouseMove"),b=o(function(){return g=!0},"dragStart");C&&(p.scroller.draggable=!0),r.state.draggingText=y,y.copy=!a.moveOnDrag,H(p.wrapper.ownerDocument,"mouseup",y),H(p.wrapper.ownerDocument,"mousemove",S),H(p.scroller,"dragstart",b),H(p.scroller,"drop",y),r.state.delayingBlurEvent=!0,setTimeout(function(){return p.input.focus()},20),p.scroller.dragDrop&&p.scroller.dragDrop()}o(mg,"leftButtonStartDrag");function gg(r,i,u){if(u=="char")return new wt(i,i);if(u=="word")return r.findWordAt(i);if(u=="line")return new wt(ue(i.line,0),He(r.doc,ue(i.line+1,0)));var a=u(r,i);return new wt(a.from,a.to)}o(gg,"rangeForUnit");function vg(r,i,u,a){c&&hc(r);var p=r.display,g=r.doc;or(i);var y,S,b=g.sel,E=b.ranges;if(a.addNew&&!a.extend?(S=g.sel.contains(u),S>-1?y=E[S]:y=new wt(u,u)):(y=g.sel.primary(),S=g.sel.primIndex),a.unit=="rectangle")a.addNew||(y=new wt(u,u)),u=po(r,i,!0,!0),S=-1;else{var M=gg(r,u,a.unit);a.extend?y=Sc(y,M.anchor,M.head,a.extend):y=M}a.addNew?S==-1?(S=E.length,$r(g,tn(r,E.concat([y]),S),{scroll:!1,origin:"*mouse"})):E.length>1&&E[S].empty()&&a.unit=="char"&&!a.extend?($r(g,tn(r,E.slice(0,S).concat(E.slice(S+1)),0),{scroll:!1,origin:"*mouse"}),b=g.sel):Ld(g,S,y,ie):(S=0,$r(g,new pi([y],0),ie),b=g.sel);var D=u;function V(ve){if(ze(D,ve)!=0)if(D=ve,a.unit=="rectangle"){for(var Ne=[],Ie=r.options.tabSize,De=_t(Me(g,u.line).text,u.ch,Ie),it=_t(Me(g,ve.line).text,ve.ch,Ie),Tt=Math.min(De,it),br=Math.max(De,it),At=Math.min(u.line,ve.line),Sn=Math.min(r.lastLine(),Math.max(u.line,ve.line));At<=Sn;At++){var Cn=Me(g,At).text,dr=Pr(Cn,Tt,Ie);Tt==br?Ne.push(new wt(ue(At,dr),ue(At,dr))):Cn.length>dr&&Ne.push(new wt(ue(At,dr),ue(At,Pr(Cn,br,Ie))))}Ne.length||Ne.push(new wt(u,u)),$r(g,tn(r,b.ranges.slice(0,S).concat(Ne),S),{origin:"*mouse",scroll:!1}),r.scrollIntoView(ve)}else{var Vr=y,Ar=gg(r,ve,a.unit),Lt=Vr.anchor,Bt;ze(Ar.anchor,Lt)>0?(Bt=Ar.head,Lt=il(Vr.from(),Ar.anchor)):(Bt=Ar.anchor,Lt=gs(Vr.to(),Ar.head));var ar=b.ranges.slice(0);ar[S]=Na(r,new wt(He(g,Lt),Bt)),$r(g,tn(r,ar,S),ie)}}o(V,"extendTo");var $=p.wrapper.getBoundingClientRect(),te=0;function oe(ve){var Ne=++te,Ie=po(r,ve,!0,a.unit=="rectangle");if(!!Ie)if(ze(Ie,D)!=0){r.curOp.focus=Ge(),V(Ie);var De=dl(p,g);(Ie.line>=De.to||Ie.line$.bottom?20:0;it&&setTimeout(lr(r,function(){te==Ne&&(p.scroller.scrollTop+=it,oe(ve))}),50)}}o(oe,"extend");function de(ve){r.state.selectingText=!1,te=1/0,ve&&(or(ve),p.input.focus()),he(p.wrapper.ownerDocument,"mousemove",ge),he(p.wrapper.ownerDocument,"mouseup",be),g.history.lastSelOrigin=null}o(de,"done");var ge=lr(r,function(ve){ve.buttons===0||!el(ve)?de(ve):oe(ve)}),be=lr(r,de);r.state.selectingText=be,H(p.wrapper.ownerDocument,"mousemove",ge),H(p.wrapper.ownerDocument,"mouseup",be)}o(vg,"leftButtonSelect");function Na(r,i){var u=i.anchor,a=i.head,p=Me(r.doc,u.line);if(ze(u,a)==0&&u.sticky==a.sticky)return i;var g=Mn(p);if(!g)return i;var y=Ci(g,u.ch,u.sticky),S=g[y];if(S.from!=u.ch&&S.to!=u.ch)return i;var b=y+(S.from==u.ch==(S.level!=1)?0:1);if(b==0||b==g.length)return i;var E;if(a.line!=u.line)E=(a.line-u.line)*(r.doc.direction=="ltr"?1:-1)>0;else{var M=Ci(g,a.ch,a.sticky),D=M-y||(a.ch-u.ch)*(S.level==1?-1:1);M==b-1||M==b?E=D<0:E=D>0}var V=g[b+(E?-1:0)],$=E==(V.level==1),te=$?V.from:V.to,oe=$?"after":"before";return u.ch==te&&u.sticky==oe?i:new wt(new ue(u.line,te,oe),a)}o(Na,"bidiSimplify");function La(r,i,u,a){var p,g;if(i.touches)p=i.touches[0].clientX,g=i.touches[0].clientY;else try{p=i.clientX,g=i.clientY}catch(V){return!1}if(p>=Math.floor(r.display.gutters.getBoundingClientRect().right))return!1;a&&or(i);var y=r.display,S=y.lineDiv.getBoundingClientRect();if(g>S.bottom||!Ft(r,u))return ds(i);g-=S.top-y.viewOffset;for(var b=0;b=p){var M=ao(r.doc,g),D=r.display.gutterSpecs[b];return Te(r,u,r,M,D.className,i),ds(i)}}}o(La,"gutterEvent");function zd(r,i){return La(r,i,"gutterClick",!0)}o(zd,"clickInGutter");function $d(r,i){ji(r.display,i)||yg(r,i)||ir(r,i,"contextmenu")||pe||r.display.input.onContextMenu(i)}o($d,"onContextMenu");function yg(r,i){return Ft(r,"gutterContextMenu")?La(r,i,"gutterContextMenu",!1):!1}o(yg,"contextMenuInGutter");function sf(r){r.display.wrapper.className=r.display.wrapper.className.replace(/\s*cm-s-\S+/g,"")+r.options.theme.replace(/(^|\s)\s*/g," cm-s-"),U(r)}o(sf,"themeChanged");var Sl={toString:function(){return"CodeMirror.Init"}},lf={},Pa={};function Fc(r){var i=r.optionHandlers;function u(a,p,g,y){r.defaults[a]=p,g&&(i[a]=y?function(S,b,E){E!=Sl&&g(S,b,E)}:g)}o(u,"option"),r.defineOption=u,r.Init=Sl,u("value","",function(a,p){return a.setValue(p)},!0),u("mode",null,function(a,p){a.doc.modeOption=p,Ed(a)},!0),u("indentUnit",2,Ed,!0),u("indentWithTabs",!1),u("smartIndent",!0),u("tabSize",4,function(a){Xu(a),U(a),et(a)},!0),u("lineSeparator",null,function(a,p){if(a.doc.lineSep=p,!!p){var g=[],y=a.doc.first;a.doc.iter(function(b){for(var E=0;;){var M=b.text.indexOf(p,E);if(M==-1)break;E=M+p.length,g.push(ue(y,M))}y++});for(var S=g.length-1;S>=0;S--)Sa(a.doc,p,g[S],ue(g[S].line,g[S].ch+p.length))}}),u("specialChars",/[\u0000-\u001f\u007f-\u009f\u00ad\u061c\u200b\u200e\u200f\u2028\u2029\ufeff\ufff9-\ufffc]/g,function(a,p,g){a.state.specialChars=new RegExp(p.source+(p.test(" ")?"":"| "),"g"),g!=Sl&&a.refresh()}),u("specialCharPlaceholder",Ur,function(a){return a.refresh()},!0),u("electricChars",!0),u("inputStyle",A?"contenteditable":"textarea",function(){throw new Error("inputStyle can not (yet) be changed in a running editor")},!0),u("spellcheck",!1,function(a,p){return a.getInputField().spellcheck=p},!0),u("autocorrect",!1,function(a,p){return a.getInputField().autocorrect=p},!0),u("autocapitalize",!1,function(a,p){return a.getInputField().autocapitalize=p},!0),u("rtlMoveVisually",!K),u("wholeLineUpdateBefore",!0),u("theme","default",function(a){sf(a),Yu(a)},!0),u("keyMap","default",function(a,p,g){var y=wn(p),S=g!=Sl&&wn(g);S&&S.detach&&S.detach(a,y),y.attach&&y.attach(a,S||null)}),u("extraKeys",null),u("configureMouse",null),u("lineWrapping",!1,wg,!0),u("gutters",[],function(a,p){a.display.gutterSpecs=Cd(p,a.options.lineNumbers),Yu(a)},!0),u("fixedGutter",!0,function(a,p){a.display.gutters.style.left=p?fa(a.display)+"px":"0",a.refresh()},!0),u("coverGutterNextToScrollbar",!1,function(a){return Ls(a)},!0),u("scrollbarStyle","native",function(a){ml(a),Ls(a),a.display.scrollbars.setScrollTop(a.doc.scrollTop),a.display.scrollbars.setScrollLeft(a.doc.scrollLeft)},!0),u("lineNumbers",!1,function(a,p){a.display.gutterSpecs=Cd(a.options.gutters,p),Yu(a)},!0),u("firstLineNumber",1,Yu,!0),u("lineNumberFormatter",function(a){return a},Yu,!0),u("showCursorWhenSelecting",!1,$u,!0),u("resetSelectionOnContextMenu",!0),u("lineWiseCopyCut",!0),u("pasteLinesPerSelection",!0),u("selectionsMayTouch",!1),u("readOnly",!1,function(a,p){p=="nocursor"&&(pl(a),a.display.input.blur()),a.display.input.readOnlyChanged(p)}),u("screenReaderLabel",null,function(a,p){p=p===""?null:p,a.display.input.screenReaderLabelChanged(p)}),u("disableInput",!1,function(a,p){p||a.display.input.reset()},!0),u("dragDrop",!0,iw),u("allowDropFileTypes",null),u("cursorBlinkRate",530),u("cursorScrollMargin",0),u("cursorHeight",1,$u,!0),u("singleCursorHeightPerLine",!0,$u,!0),u("workTime",100),u("workDelay",100),u("flattenSpans",!0,Xu,!0),u("addModeClass",!1,Xu,!0),u("pollInterval",100),u("undoDepth",200,function(a,p){return a.doc.history.undoDepth=p}),u("historyEventDelay",1250),u("viewportMargin",10,function(a){return a.refresh()},!0),u("maxHighlightLength",1e4,Xu,!0),u("moveInputWithCursor",!0,function(a,p){p||a.display.input.resetPosition()}),u("tabindex",null,function(a,p){return a.display.input.getField().tabIndex=p||""}),u("autofocus",null),u("direction","ltr",function(a,p){return a.doc.setDirection(p)},!0),u("phrases",null)}o(Fc,"defineOptions");function iw(r,i,u){var a=u&&u!=Sl;if(!i!=!a){var p=r.display.dragFunctions,g=i?H:he;g(r.display.scroller,"dragstart",p.start),g(r.display.scroller,"dragenter",p.enter),g(r.display.scroller,"dragover",p.over),g(r.display.scroller,"dragleave",p.leave),g(r.display.scroller,"drop",p.drop)}}o(iw,"dragDropChanged");function wg(r){r.options.lineWrapping?(Xe(r.display.wrapper,"CodeMirror-wrap"),r.display.sizer.style.minWidth="",r.display.sizerWidth=null):(xe(r.display.wrapper,"CodeMirror-wrap"),zi(r)),bs(r),et(r),U(r),setTimeout(function(){return Ls(r)},100)}o(wg,"wrappingChanged");function Mt(r,i){var u=this;if(!(this instanceof Mt))return new Mt(r,i);this.options=i=i?Qt(i):{},Qt(lf,i,!1);var a=i.value;typeof a=="string"?a=new yn(a,i.mode,null,i.lineSeparator,i.direction):i.mode&&(a.modeOption=i.mode),this.doc=a;var p=new Mt.inputStyles[i.inputStyle](this),g=this.display=new Vy(r,a,p,i);g.wrapper.CodeMirror=this,sf(this),i.lineWrapping&&(this.display.wrapper.className+=" CodeMirror-wrap"),ml(this),this.state={keyMaps:[],overlays:[],modeGen:0,overwrite:!1,delayingBlurEvent:!1,focused:!1,suppressEdits:!1,pasteIncoming:-1,cutIncoming:-1,selectingText:!1,draggingText:!1,highlight:new Ct,keySeq:null,specialChars:null},i.autofocus&&!A&&g.input.focus(),c&&v<11&&setTimeout(function(){return u.display.input.reset(!0)},20),xg(this),Dd(),Ki(this),this.curOp.forceUpdate=!0,Km(this,a),i.autofocus&&!A||this.hasFocus()?setTimeout(function(){u.hasFocus()&&!u.state.focused&&pa(u)},20):pl(this);for(var y in Pa)Pa.hasOwnProperty(y)&&Pa[y](this,i[y],Sl);vo(this),i.finishInit&&i.finishInit(this);for(var S=0;S20*20}o(y,"farAway"),H(i.scroller,"touchstart",function(b){if(!ir(r,b)&&!g(b)&&!zd(r,b)){i.input.ensurePolled(),clearTimeout(u);var E=+new Date;i.activeTouch={start:E,moved:!1,prev:E-a.end<=300?a:null},b.touches.length==1&&(i.activeTouch.left=b.touches[0].pageX,i.activeTouch.top=b.touches[0].pageY)}}),H(i.scroller,"touchmove",function(){i.activeTouch&&(i.activeTouch.moved=!0)}),H(i.scroller,"touchend",function(b){var E=i.activeTouch;if(E&&!ji(i,b)&&E.left!=null&&!E.moved&&new Date-E.start<300){var M=r.coordsChar(i.activeTouch,"page"),D;!E.prev||y(E,E.prev)?D=new wt(M,M):!E.prev.prev||y(E,E.prev.prev)?D=r.findWordAt(M):D=new wt(ue(M.line,0),He(r.doc,ue(M.line+1,0))),r.setSelection(D.anchor,D.head),r.focus(),or(b)}p()}),H(i.scroller,"touchcancel",p),H(i.scroller,"scroll",function(){i.scroller.clientHeight&&(sr(r,i.scroller.scrollTop),hl(r,i.scroller.scrollLeft,!0),Te(r,"scroll",r))}),H(i.scroller,"mousewheel",function(b){return qm(r,b)}),H(i.scroller,"DOMMouseScroll",function(b){return qm(r,b)}),H(i.wrapper,"scroll",function(){return i.wrapper.scrollTop=i.wrapper.scrollLeft=0}),i.dragFunctions={enter:function(b){ir(r,b)||lo(b)},over:function(b){ir(r,b)||(Md(r,b),lo(b))},start:function(b){return Zy(r,b)},drop:lr(r,sg),leave:function(b){ir(r,b)||Ad(r)}};var S=i.input.getField();H(S,"keyup",function(b){return Ud.call(r,b)}),H(S,"keydown",lr(r,ug)),H(S,"keypress",lr(r,cg)),H(S,"focus",function(b){return pa(r,b)}),H(S,"blur",function(b){return pl(r,b)})}o(xg,"registerEventHandlers");var af=[];Mt.defineInitHook=function(r){return af.push(r)};function uf(r,i,u,a){var p=r.doc,g;u==null&&(u="add"),u=="smart"&&(p.mode.indent?g=$o(r,i).state:u="prev");var y=r.options.tabSize,S=Me(p,i),b=_t(S.text,null,y);S.stateAfter&&(S.stateAfter=null);var E=S.text.match(/^\s*/)[0],M;if(!a&&!/\S/.test(S.text))M=0,u="not";else if(u=="smart"&&(M=p.mode.indent(g,S.text.slice(E.length),S.text),M==zt||M>150)){if(!a)return;u="prev"}u=="prev"?i>p.first?M=_t(Me(p,i-1).text,null,y):M=0:u=="add"?M=b+r.options.indentUnit:u=="subtract"?M=b-r.options.indentUnit:typeof u=="number"&&(M=b+u),M=Math.max(0,M);var D="",V=0;if(r.options.indentWithTabs)for(var $=Math.floor(M/y);$;--$)V+=y,D+=" ";if(Vy,b=rl(i),E=null;if(S&&a.ranges.length>1)if(Pi&&Pi.text.join(` `)==i){if(a.ranges.length%Pi.text.length==0){E=[];for(var M=0;M=0;V--){var $=a.ranges[V],te=$.from(),oe=$.to();$.empty()&&(u&&u>0?te=ue(te.line,te.ch-u):r.state.overwrite&&!S?oe=ue(oe.line,Math.min(Me(g,oe.line).text.length,oe.ch+Se(b).length)):S&&Pi&&Pi.lineWise&&Pi.text.join(` `)==b.join(` -`)&&(te=oe=ue(te.line,0)));var de={from:te,to:oe,text:E?E[V%E.length]:b,origin:p||(S?"paste":r.state.cutIncoming>y?"cut":"+input")};ya(r.doc,de),yr(r,"inputRead",r,de)}i&&!S&&Sg(r,i),ks(r),r.curOp.updateInput<2&&(r.curOp.updateInput=D),r.curOp.typing=!0,r.state.pasteIncoming=r.state.cutIncoming=-1}o(Bc,"applyTextInput");function jd(r,i){var u=r.clipboardData&&r.clipboardData.getData("Text");if(u)return r.preventDefault(),!i.isReadOnly()&&!i.options.disableInput&&Jr(i,function(){return Bc(i,u,0,null,"paste")}),!0}o(jd,"handlePaste");function Sg(r,i){if(!(!r.options.electricChars||!r.options.smartIndent))for(var u=r.doc.sel,a=u.ranges.length-1;a>=0;a--){var p=u.ranges[a];if(!(p.head.ch>100||a&&u.ranges[a-1].head.line==p.head.line)){var g=r.getModeAt(p.head),y=!1;if(g.electricChars){for(var S=0;S-1){y=uf(r,p.head.line,"smart");break}}else g.electricInput&&g.electricInput.test(Me(r.doc,p.head.line).text.slice(0,p.head.ch))&&(y=uf(r,p.head.line,"smart"));y&&yr(r,"electricInput",r,p.head.line)}}}o(Sg,"triggerElectric");function qd(r){for(var i=[],u=[],a=0;ag&&(uf(this,S.head.line,a,!0),g=S.head.line,y==this.doc.sel.primIndex&&ks(this));else{var b=S.from(),E=S.to(),M=Math.max(g,b.line);g=Math.min(this.lastLine(),E.line-(E.ch?0:1))+1;for(var D=M;D0&&Ld(this.doc,y,new wt(b,V[y].to()),$t)}}}),getTokenAt:function(a,p){return Kl(this,a,p)},getLineTokens:function(a,p){return Kl(this,ue(a),p,!0)},getTokenTypeAt:function(a){a=He(this.doc,a);var p=_u(this,Me(this.doc,a.line)),g=0,y=(p.length-1)/2,S=a.ch,b;if(S==0)b=p[2];else for(;;){var E=g+y>>1;if((E?p[E*2-1]:0)>=S)y=E;else if(p[E*2+1]b&&(a=b,y=!0),S=Me(this.doc,a)}else S=a;return ci(this,S,{top:0,left:0},p||"page",g||y).top+(y?this.doc.height-hn(S):0)},defaultTextHeight:function(){return Cs(this.display)},defaultCharWidth:function(){return ua(this.display)},getViewport:function(){return{from:this.display.viewFrom,to:this.display.viewTo}},addWidget:function(a,p,g,y,S){var b=this.display;a=Vn(this,He(this.doc,a));var E=a.bottom,M=a.left;if(p.style.position="absolute",p.setAttribute("cm-ignore-events","true"),this.display.input.setUneditable(p),b.sizer.appendChild(p),y=="over")E=a.top;else if(y=="above"||y=="near"){var D=Math.max(b.wrapper.clientHeight,this.doc.height),V=Math.max(b.sizer.clientWidth,b.lineSpace.clientWidth);(y=="above"||a.bottom+p.offsetHeight>D)&&a.top>p.offsetHeight?E=a.top-p.offsetHeight:a.bottom+p.offsetHeight<=D&&(E=a.bottom),M+p.offsetWidth>V&&(M=V-p.offsetWidth)}p.style.top=E+"px",p.style.left=p.style.right="",S=="right"?(M=b.sizer.clientWidth-p.offsetWidth,p.style.right="0px"):(S=="left"?M=0:S=="middle"&&(M=(b.sizer.clientWidth-p.offsetWidth)/2),p.style.left=M+"px"),g&&Hy(this,{left:M,top:E,right:M+p.offsetWidth,bottom:E+p.offsetHeight})},triggerOnKeyDown:en(ug),triggerOnKeyPress:en(cg),triggerOnKeyUp:Ud,triggerOnMouseDown:en(dg),execCommand:function(a){if(xl.hasOwnProperty(a))return xl[a].call(null,this)},triggerElectric:en(function(a){Sg(this,a)}),findPosH:function(a,p,g,y){var S=1;p<0&&(S=-1,p=-p);for(var b=He(this.doc,a),E=0;E0&&M(g.charAt(y-1));)--y;for(;S.5||this.options.lineWrapping)&&bs(this),Te(this,"refresh",this)}),swapDoc:en(function(a){var p=this.doc;return p.cm=null,this.state.selectingText&&this.state.selectingText(),Km(this,a),U(this),this.display.input.reset(),qu(this,a.scrollLeft,a.scrollTop),this.curOp.forceScroll=!0,yr(this,"swapDoc",this,p),p}),phrase:function(a){var p=this.options.phrases;return p&&Object.prototype.hasOwnProperty.call(p,a)?p[a]:a},getInputField:function(){return this.display.input.getField()},getWrapperElement:function(){return this.display.wrapper},getScrollerElement:function(){return this.display.scroller},getGutterElement:function(){return this.display.gutters}},Wr(r),r.registerHelper=function(a,p,g){u.hasOwnProperty(a)||(u[a]=r[a]={_global:[]}),u[a][p]=g},r.registerGlobalHelper=function(a,p,g,y){r.registerHelper(a,p,y),u[a]._global.push({pred:g,val:y})}}o(ts,"addEditorMethods");function ff(r,i,u,a,p){var g=i,y=u,S=Me(r,i.line),b=p&&r.direction=="rtl"?-u:u;function E(){var be=i.line+b;return be=r.first+r.size?!1:(i=new ue(be,i.ch,i.sticky),S=Me(r,be))}o(E,"findNextLine");function M(be){var ve;if(a=="codepoint"){var Ne=S.text.charCodeAt(i.ch+(u>0?0:-1));if(isNaN(Ne))ve=null;else{var Ie=u>0?Ne>=55296&&Ne<56320:Ne>=56320&&Ne<57343;ve=new ue(i.line,Math.max(0,Math.min(S.text.length,i.ch+u*(Ie?2:1))),-u)}}else p?ve=ag(r.cm,S,i,u):ve=Mc(S,i,u);if(ve==null)if(!be&&E())i=_a(p,r.cm,S,i.line,b);else return!1;else i=ve;return!0}if(o(M,"moveOnce"),a=="char"||a=="codepoint")M();else if(a=="column")M(!0);else if(a=="word"||a=="group")for(var D=null,V=a=="group",$=r.cm&&r.cm.getHelper(i,"wordChars"),te=!0;!(u<0&&!M(!te));te=!1){var oe=S.text.charAt(i.ch)||` +`)&&(te=oe=ue(te.line,0)));var de={from:te,to:oe,text:E?E[V%E.length]:b,origin:p||(S?"paste":r.state.cutIncoming>y?"cut":"+input")};ya(r.doc,de),yr(r,"inputRead",r,de)}i&&!S&&Sg(r,i),ks(r),r.curOp.updateInput<2&&(r.curOp.updateInput=D),r.curOp.typing=!0,r.state.pasteIncoming=r.state.cutIncoming=-1}o(Bc,"applyTextInput");function jd(r,i){var u=r.clipboardData&&r.clipboardData.getData("Text");if(u)return r.preventDefault(),!i.isReadOnly()&&!i.options.disableInput&&Jr(i,function(){return Bc(i,u,0,null,"paste")}),!0}o(jd,"handlePaste");function Sg(r,i){if(!(!r.options.electricChars||!r.options.smartIndent))for(var u=r.doc.sel,a=u.ranges.length-1;a>=0;a--){var p=u.ranges[a];if(!(p.head.ch>100||a&&u.ranges[a-1].head.line==p.head.line)){var g=r.getModeAt(p.head),y=!1;if(g.electricChars){for(var S=0;S-1){y=uf(r,p.head.line,"smart");break}}else g.electricInput&&g.electricInput.test(Me(r.doc,p.head.line).text.slice(0,p.head.ch))&&(y=uf(r,p.head.line,"smart"));y&&yr(r,"electricInput",r,p.head.line)}}}o(Sg,"triggerElectric");function qd(r){for(var i=[],u=[],a=0;ag&&(uf(this,S.head.line,a,!0),g=S.head.line,y==this.doc.sel.primIndex&&ks(this));else{var b=S.from(),E=S.to(),M=Math.max(g,b.line);g=Math.min(this.lastLine(),E.line-(E.ch?0:1))+1;for(var D=M;D0&&Ld(this.doc,y,new wt(b,V[y].to()),$t)}}}),getTokenAt:function(a,p){return Kl(this,a,p)},getLineTokens:function(a,p){return Kl(this,ue(a),p,!0)},getTokenTypeAt:function(a){a=He(this.doc,a);var p=_u(this,Me(this.doc,a.line)),g=0,y=(p.length-1)/2,S=a.ch,b;if(S==0)b=p[2];else for(;;){var E=g+y>>1;if((E?p[E*2-1]:0)>=S)y=E;else if(p[E*2+1]b&&(a=b,y=!0),S=Me(this.doc,a)}else S=a;return ci(this,S,{top:0,left:0},p||"page",g||y).top+(y?this.doc.height-hn(S):0)},defaultTextHeight:function(){return Cs(this.display)},defaultCharWidth:function(){return ua(this.display)},getViewport:function(){return{from:this.display.viewFrom,to:this.display.viewTo}},addWidget:function(a,p,g,y,S){var b=this.display;a=Vn(this,He(this.doc,a));var E=a.bottom,M=a.left;if(p.style.position="absolute",p.setAttribute("cm-ignore-events","true"),this.display.input.setUneditable(p),b.sizer.appendChild(p),y=="over")E=a.top;else if(y=="above"||y=="near"){var D=Math.max(b.wrapper.clientHeight,this.doc.height),V=Math.max(b.sizer.clientWidth,b.lineSpace.clientWidth);(y=="above"||a.bottom+p.offsetHeight>D)&&a.top>p.offsetHeight?E=a.top-p.offsetHeight:a.bottom+p.offsetHeight<=D&&(E=a.bottom),M+p.offsetWidth>V&&(M=V-p.offsetWidth)}p.style.top=E+"px",p.style.left=p.style.right="",S=="right"?(M=b.sizer.clientWidth-p.offsetWidth,p.style.right="0px"):(S=="left"?M=0:S=="middle"&&(M=(b.sizer.clientWidth-p.offsetWidth)/2),p.style.left=M+"px"),g&&Fy(this,{left:M,top:E,right:M+p.offsetWidth,bottom:E+p.offsetHeight})},triggerOnKeyDown:en(ug),triggerOnKeyPress:en(cg),triggerOnKeyUp:Ud,triggerOnMouseDown:en(dg),execCommand:function(a){if(xl.hasOwnProperty(a))return xl[a].call(null,this)},triggerElectric:en(function(a){Sg(this,a)}),findPosH:function(a,p,g,y){var S=1;p<0&&(S=-1,p=-p);for(var b=He(this.doc,a),E=0;E0&&M(g.charAt(y-1));)--y;for(;S.5||this.options.lineWrapping)&&bs(this),Te(this,"refresh",this)}),swapDoc:en(function(a){var p=this.doc;return p.cm=null,this.state.selectingText&&this.state.selectingText(),Km(this,a),U(this),this.display.input.reset(),qu(this,a.scrollLeft,a.scrollTop),this.curOp.forceScroll=!0,yr(this,"swapDoc",this,p),p}),phrase:function(a){var p=this.options.phrases;return p&&Object.prototype.hasOwnProperty.call(p,a)?p[a]:a},getInputField:function(){return this.display.input.getField()},getWrapperElement:function(){return this.display.wrapper},getScrollerElement:function(){return this.display.scroller},getGutterElement:function(){return this.display.gutters}},Wr(r),r.registerHelper=function(a,p,g){u.hasOwnProperty(a)||(u[a]=r[a]={_global:[]}),u[a][p]=g},r.registerGlobalHelper=function(a,p,g,y){r.registerHelper(a,p,y),u[a]._global.push({pred:g,val:y})}}o(ts,"addEditorMethods");function ff(r,i,u,a,p){var g=i,y=u,S=Me(r,i.line),b=p&&r.direction=="rtl"?-u:u;function E(){var be=i.line+b;return be=r.first+r.size?!1:(i=new ue(be,i.ch,i.sticky),S=Me(r,be))}o(E,"findNextLine");function M(be){var ve;if(a=="codepoint"){var Ne=S.text.charCodeAt(i.ch+(u>0?0:-1));if(isNaN(Ne))ve=null;else{var Ie=u>0?Ne>=55296&&Ne<56320:Ne>=56320&&Ne<57343;ve=new ue(i.line,Math.max(0,Math.min(S.text.length,i.ch+u*(Ie?2:1))),-u)}}else p?ve=ag(r.cm,S,i,u):ve=Mc(S,i,u);if(ve==null)if(!be&&E())i=_a(p,r.cm,S,i.line,b);else return!1;else i=ve;return!0}if(o(M,"moveOnce"),a=="char"||a=="codepoint")M();else if(a=="column")M(!0);else if(a=="word"||a=="group")for(var D=null,V=a=="group",$=r.cm&&r.cm.getHelper(i,"wordChars"),te=!0;!(u<0&&!M(!te));te=!1){var oe=S.text.charAt(i.ch)||` `,de=gr(oe,$)?"w":V&&oe==` `?"n":!V||/\s/.test(oe)?null:"p";if(V&&!te&&!de&&(de="s"),D&&D!=de){u<0&&(u=1,M(),i.sticky="after");break}if(de&&(D=de),u>0&&!M(!te))break}var ge=jr(r,i,g,y,!0);return Cu(g,ge)&&(ge.hitSide=!0),ge}o(ff,"findPosH");function Hc(r,i,u,a){var p=r.doc,g=i.left,y;if(a=="page"){var S=Math.min(r.display.wrapper.clientHeight,window.innerHeight||document.documentElement.clientHeight),b=Math.max(S-.5*Cs(r.display),3);y=(u>0?i.bottom:i.top)+u*b}else a=="line"&&(y=u>0?i.bottom+3:i.top-3);for(var E;E=q(r,g,y),!!E.outside;){if(u<0?y<=0:y>=p.height){E.hitSide=!0;break}y+=u*5}return E}o(Hc,"findPosV");var xt=o(function(r){this.cm=r,this.lastAnchorNode=this.lastAnchorOffset=this.lastFocusNode=this.lastFocusOffset=null,this.polling=new Ct,this.composing=null,this.gracePeriod=!1,this.readDOMTimeout=null},"ContentEditableInput");xt.prototype.init=function(r){var i=this,u=this,a=u.cm,p=u.div=r.lineDiv;p.contentEditable=!0,Cg(p,a.options.spellcheck,a.options.autocorrect,a.options.autocapitalize);function g(S){for(var b=S.target;b;b=b.parentNode){if(b==p)return!0;if(/\bCodeMirror-(?:line)?widget\b/.test(b.className))break}return!1}o(g,"belongsToInput"),H(p,"paste",function(S){!g(S)||ir(a,S)||jd(S,a)||v<=11&&setTimeout(lr(a,function(){return i.updateFromDOM()}),20)}),H(p,"compositionstart",function(S){i.composing={data:S.data,done:!1}}),H(p,"compositionupdate",function(S){i.composing||(i.composing={data:S.data,done:!1})}),H(p,"compositionend",function(S){i.composing&&(S.data!=i.composing.data&&i.readFromDOMSoon(),i.composing.done=!0)}),H(p,"touchstart",function(){return u.forceCompositionEnd()}),H(p,"input",function(){i.composing||i.readFromDOMSoon()});function y(S){if(!(!g(S)||ir(a,S))){if(a.somethingSelected())Oi({lineWise:!1,text:a.getSelections()}),S.type=="cut"&&a.replaceSelection("",null,"cut");else if(a.options.lineWiseCopyCut){var b=qd(a);Oi({lineWise:!0,text:b.text}),S.type=="cut"&&a.operation(function(){a.setSelections(b.ranges,0,$t),a.replaceSelection("",null,"cut")})}else return;if(S.clipboardData){S.clipboardData.clearData();var E=Pi.text.join(` `);if(S.clipboardData.setData("Text",E),S.clipboardData.getData("Text")==E){S.preventDefault();return}}var M=bg(),D=M.firstChild;a.display.lineSpace.insertBefore(M,a.display.lineSpace.firstChild),D.value=Pi.text.join(` @@ -48,26 +48,26 @@ b`.split(/\n/).length!=3?function(r){for(var i=0,u=[],a=r.length;i<=a;){var p=r. `)>-1?u.value=r.prevInput="":r.prevInput=p,r.composing&&(r.composing.range.clear(),r.composing.range=i.markText(r.composing.start,i.getCursor("to"),{className:"CodeMirror-composing"}))}),!0},pr.prototype.ensurePolled=function(){this.pollingFast&&this.poll()&&(this.pollingFast=!1)},pr.prototype.onKeyPress=function(){c&&v>=9&&(this.hasSelection=null),this.fastPoll()},pr.prototype.onContextMenu=function(r){var i=this,u=i.cm,a=u.display,p=i.textarea;i.contextMenuPending&&i.contextMenuPending();var g=po(u,r),y=a.scroller.scrollTop;if(!g||j)return;var S=u.options.resetSelectionOnContextMenu;S&&u.doc.sel.contains(g)==-1&&lr(u,$r)(u.doc,Os(g),$t);var b=p.style.cssText,E=i.wrapper.style.cssText,M=i.wrapper.offsetParent.getBoundingClientRect();i.wrapper.style.cssText="position: static",p.style.cssText=`position: absolute; width: 30px; height: 30px; top: `+(r.clientY-M.top-5)+"px; left: "+(r.clientX-M.left-5)+`px; z-index: 1000; background: `+(c?"rgba(255, 255, 255, .05)":"transparent")+`; - outline: none; border-width: 0; outline: none; overflow: hidden; opacity: .05; filter: alpha(opacity=5);`;var D;C&&(D=window.scrollY),a.input.focus(),C&&window.scrollTo(null,D),a.input.reset(),u.somethingSelected()||(p.value=i.prevInput=" "),i.contextMenuPending=$,a.selForContextMenu=u.doc.sel,clearTimeout(a.detectingSelectAll);function V(){if(p.selectionStart!=null){var oe=u.somethingSelected(),de="\u200B"+(oe?p.value:"");p.value="\u21DA",p.value=de,i.prevInput=oe?"":"\u200B",p.selectionStart=1,p.selectionEnd=de.length,a.selForContextMenu=u.doc.sel}}o(V,"prepareSelectAllHack");function $(){if(i.contextMenuPending==$&&(i.contextMenuPending=!1,i.wrapper.style.cssText=E,p.style.cssText=b,c&&v<9&&a.scrollbars.setScrollTop(a.scroller.scrollTop=y),p.selectionStart!=null)){(!c||c&&v<9)&&V();var oe=0,de=o(function(){a.selForContextMenu==u.doc.sel&&p.selectionStart==0&&p.selectionEnd>0&&i.prevInput=="\u200B"?lr(u,Qm)(u):oe++<10?a.detectingSelectAll=setTimeout(de,500):(a.selForContextMenu=null,a.input.reset())},"poll");a.detectingSelectAll=setTimeout(de,200)}}if(o($,"rehide"),c&&v>=9&&V(),pe){lo(r);var te=o(function(){he(window,"mouseup",te),setTimeout($,20)},"mouseup");H(window,"mouseup",te)}else setTimeout($,50)},pr.prototype.readOnlyChanged=function(r){r||this.reset(),this.textarea.disabled=r=="nocursor",this.textarea.readOnly=!!r},pr.prototype.setUneditable=function(){},pr.prototype.needsContentAttribute=!1;function Vd(r,i){if(i=i?Qt(i):{},i.value=r.value,!i.tabindex&&r.tabIndex&&(i.tabindex=r.tabIndex),!i.placeholder&&r.placeholder&&(i.placeholder=r.placeholder),i.autofocus==null){var u=Ge();i.autofocus=u==r||r.getAttribute("autofocus")!=null&&u==document.body}function a(){r.value=S.getValue()}o(a,"save");var p;if(r.form&&(H(r.form,"submit",a),!i.leaveSubmitMethodAlone)){var g=r.form;p=g.submit;try{var y=g.submit=function(){a(),g.submit=p,g.submit(),g.submit=y}}catch(b){}}i.finishInit=function(b){b.save=a,b.getTextArea=function(){return r},b.toTextArea=function(){b.toTextArea=isNaN,a(),r.parentNode.removeChild(b.getWrapperElement()),r.style.display="",r.form&&(he(r.form,"submit",a),!i.leaveSubmitMethodAlone&&typeof r.form.submit=="function"&&(r.form.submit=p))}},r.style.display="none";var S=Mt(function(b){return r.parentNode.insertBefore(b,r.nextSibling)},i);return S}o(Vd,"fromTextArea");function Eg(r){r.off=he,r.on=H,r.wheelEventPixels=Yy,r.Doc=yn,r.splitLines=rl,r.countColumn=_t,r.findColumn=Pr,r.isWordChar=Zt,r.Pass=zt,r.signal=Te,r.Line=Ce,r.changeEnd=Ms,r.scrollbarModel=gc,r.Pos=ue,r.cmpPos=ze,r.modes=zl,r.mimeModes=ms,r.resolveMode=nl,r.getMode=xu,r.modeExtensions=Wo,r.extendMode=ad,r.copyState=Uo,r.startState=ec,r.innerMode=$l,r.commands=xl,r.keyMap=Jo,r.keyName=Id,r.isModifierKey=Oc,r.lookupKey=es,r.normalizeKeyMap=rw,r.StringStream=jt,r.SharedTextMarker=Ea,r.TextMarker=Rs,r.LineWidget=tf,r.e_preventDefault=or,r.e_stopPropagation=li,r.e_stop=lo,r.addClass=Xe,r.contains=Ke,r.rmClass=xe,r.keyNames=Fs}o(Eg,"addLegacyProps"),Fc(Mt),ts(Mt);var xn="iter insert remove copy getEditor constructor".split(" ");for(var Uc in yn.prototype)yn.prototype.hasOwnProperty(Uc)&&ut(xn,Uc)<0&&(Mt.prototype[Uc]=function(r){return function(){return r.apply(this.doc,arguments)}}(yn.prototype[Uc]));return Wr(yn),Mt.inputStyles={textarea:pr,contenteditable:xt},Mt.defineMode=function(r){!Mt.defaults.mode&&r!="null"&&(Mt.defaults.mode=r),ld.apply(this,arguments)},Mt.defineMIME=Jf,Mt.defineMode("null",function(){return{token:function(r){return r.skipToEnd()}}}),Mt.defineMIME("text/plain","null"),Mt.defineExtension=function(r,i){Mt.prototype[r]=i},Mt.defineDocExtension=function(r,i){yn.prototype[r]=i},Mt.fromTextArea=Vd,Eg(Mt),Mt.version="5.62.3",Mt})});var cL=Ue((oz,fL)=>{var T3=typeof Element!="undefined",k3=typeof Map=="function",N3=typeof Set=="function",L3=typeof ArrayBuffer=="function"&&!!ArrayBuffer.isView;function oy(e,t){if(e===t)return!0;if(e&&t&&typeof e=="object"&&typeof t=="object"){if(e.constructor!==t.constructor)return!1;var n,l,d;if(Array.isArray(e)){if(n=e.length,n!=t.length)return!1;for(l=n;l--!=0;)if(!oy(e[l],t[l]))return!1;return!0}var h;if(k3&&e instanceof Map&&t instanceof Map){if(e.size!==t.size)return!1;for(h=e.entries();!(l=h.next()).done;)if(!t.has(l.value[0]))return!1;for(h=e.entries();!(l=h.next()).done;)if(!oy(l.value[1],t.get(l.value[0])))return!1;return!0}if(N3&&e instanceof Set&&t instanceof Set){if(e.size!==t.size)return!1;for(h=e.entries();!(l=h.next()).done;)if(!t.has(l.value[0]))return!1;return!0}if(L3&&ArrayBuffer.isView(e)&&ArrayBuffer.isView(t)){if(n=e.length,n!=t.length)return!1;for(l=n;l--!=0;)if(e[l]!==t[l])return!1;return!0}if(e.constructor===RegExp)return e.source===t.source&&e.flags===t.flags;if(e.valueOf!==Object.prototype.valueOf)return e.valueOf()===t.valueOf();if(e.toString!==Object.prototype.toString)return e.toString()===t.toString();if(d=Object.keys(e),n=d.length,n!==Object.keys(t).length)return!1;for(l=n;l--!=0;)if(!Object.prototype.hasOwnProperty.call(t,d[l]))return!1;if(T3&&e instanceof Element)return!1;for(l=n;l--!=0;)if(!((d[l]==="_owner"||d[l]==="__v"||d[l]==="__o")&&e.$$typeof)&&!oy(e[d[l]],t[d[l]]))return!1;return!0}return e!==e&&t!==t}o(oy,"equal");fL.exports=o(function(t,n){try{return oy(t,n)}catch(l){if((l.message||"").match(/stack|recursion/i))return console.warn("react-fast-compare cannot handle circular refs"),!1;throw l}},"isEqual")});var EL=Ue((_9,bL)=>{"use strict";var U3="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED";bL.exports=U3});var NL=Ue((T9,kL)=>{"use strict";var z3=EL();function _L(){}o(_L,"emptyFunction");function TL(){}o(TL,"emptyFunctionWithReset");TL.resetWarningCache=_L;kL.exports=function(){function e(l,d,h,c,v,C){if(C!==z3){var k=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw k.name="Invariant Violation",k}}o(e,"shim"),e.isRequired=e;function t(){return e}o(t,"getShim");var n={array:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:t,element:e,elementType:e,instanceOf:t,node:e,objectOf:t,oneOf:t,oneOfType:t,shape:t,exact:t,checkPropTypes:TL,resetWarningCache:_L};return n.PropTypes=n,n}});var Sm=Ue((L9,LL)=>{LL.exports=NL()();var k9,N9});var fC=Ue((P9,PL)=>{PL.exports=o(function(t,n,l,d){var h=l?l.call(d,t,n):void 0;if(h!==void 0)return!!h;if(t===n)return!0;if(typeof t!="object"||!t||typeof n!="object"||!n)return!1;var c=Object.keys(t),v=Object.keys(n);if(c.length!==v.length)return!1;for(var C=Object.prototype.hasOwnProperty.bind(n),k=0;k{HL.exports=function(){return typeof Promise=="function"&&Promise.prototype&&Promise.prototype.then}});var du=Ue(qf=>{var pC,j3=[0,26,44,70,100,134,172,196,242,292,346,404,466,532,581,655,733,815,901,991,1085,1156,1258,1364,1474,1588,1706,1828,1921,2051,2185,2323,2465,2611,2761,2876,3034,3196,3362,3532,3706];qf.getSymbolSize=o(function(t){if(!t)throw new Error('"version" cannot be null or undefined');if(t<1||t>40)throw new Error('"version" should be in range from 1 to 40');return t*4+17},"getSymbolSize");qf.getSymbolTotalCodewords=o(function(t){return j3[t]},"getSymbolTotalCodewords");qf.getBCHDigit=function(e){let t=0;for(;e!==0;)t++,e>>>=1;return t};qf.setToSJISFunction=o(function(t){if(typeof t!="function")throw new Error('"toSJISFunc" is not a valid function.');pC=t},"setToSJISFunction");qf.isKanjiModeEnabled=function(){return typeof pC!="undefined"};qf.toSJIS=o(function(t){return pC(t)},"toSJIS")});var vy=Ue(Do=>{Do.L={bit:1};Do.M={bit:0};Do.Q={bit:3};Do.H={bit:2};function q3(e){if(typeof e!="string")throw new Error("Param is not a string");switch(e.toLowerCase()){case"l":case"low":return Do.L;case"m":case"medium":return Do.M;case"q":case"quartile":return Do.Q;case"h":case"high":return Do.H;default:throw new Error("Unknown EC Level: "+e)}}o(q3,"fromString");Do.isValid=o(function(t){return t&&typeof t.bit!="undefined"&&t.bit>=0&&t.bit<4},"isValid");Do.from=o(function(t,n){if(Do.isValid(t))return t;try{return q3(t)}catch(l){return n}},"from")});var $L=Ue((X9,zL)=>{function UL(){this.buffer=[],this.length=0}o(UL,"BitBuffer");UL.prototype={get:function(e){let t=Math.floor(e/8);return(this.buffer[t]>>>7-e%8&1)==1},put:function(e,t){for(let n=0;n>>t-n-1&1)==1)},getLengthInBits:function(){return this.length},putBit:function(e){let t=Math.floor(this.length/8);this.buffer.length<=t&&this.buffer.push(0),e&&(this.buffer[t]|=128>>>this.length%8),this.length++}};zL.exports=UL});var qL=Ue((Q9,jL)=>{function _m(e){if(!e||e<1)throw new Error("BitMatrix size must be defined and greater than 0");this.size=e,this.data=new Uint8Array(e*e),this.reservedBit=new Uint8Array(e*e)}o(_m,"BitMatrix");_m.prototype.set=function(e,t,n,l){let d=e*this.size+t;this.data[d]=n,l&&(this.reservedBit[d]=!0)};_m.prototype.get=function(e,t){return this.data[e*this.size+t]};_m.prototype.xor=function(e,t,n){this.data[e*this.size+t]^=n};_m.prototype.isReserved=function(e,t){return this.reservedBit[e*this.size+t]};jL.exports=_m});var VL=Ue(yy=>{var V3=du().getSymbolSize;yy.getRowColCoords=o(function(t){if(t===1)return[];let n=Math.floor(t/7)+2,l=V3(t),d=l===145?26:Math.ceil((l-13)/(2*n-2))*2,h=[l-7];for(let c=1;c{var K3=du().getSymbolSize,KL=7;GL.getPositions=o(function(t){let n=K3(t);return[[0,0],[n-KL,0],[0,n-KL]]},"getPositions")});var XL=Ue(rr=>{rr.Patterns={PATTERN000:0,PATTERN001:1,PATTERN010:2,PATTERN011:3,PATTERN100:4,PATTERN101:5,PATTERN110:6,PATTERN111:7};var Vf={N1:3,N2:3,N3:40,N4:10};rr.isValid=o(function(t){return t!=null&&t!==""&&!isNaN(t)&&t>=0&&t<=7},"isValid");rr.from=o(function(t){return rr.isValid(t)?parseInt(t,10):void 0},"from");rr.getPenaltyN1=o(function(t){let n=t.size,l=0,d=0,h=0,c=null,v=null;for(let C=0;C=5&&(l+=Vf.N1+(d-5)),c=O,d=1),O=t.get(k,C),O===v?h++:(h>=5&&(l+=Vf.N1+(h-5)),v=O,h=1)}d>=5&&(l+=Vf.N1+(d-5)),h>=5&&(l+=Vf.N1+(h-5))}return l},"getPenaltyN1");rr.getPenaltyN2=o(function(t){let n=t.size,l=0;for(let d=0;d=10&&(d===1488||d===93)&&l++,h=h<<1&2047|t.get(v,c),v>=10&&(h===1488||h===93)&&l++}return l*Vf.N3},"getPenaltyN3");rr.getPenaltyN4=o(function(t){let n=0,l=t.data.length;for(let h=0;h{var hu=vy(),wy=[1,1,1,1,1,1,1,1,1,1,2,2,1,2,2,4,1,2,4,4,2,4,4,4,2,4,6,5,2,4,6,6,2,5,8,8,4,5,8,8,4,5,8,11,4,8,10,11,4,9,12,16,4,9,16,16,6,10,12,18,6,10,17,16,6,11,16,19,6,13,18,21,7,14,21,25,8,16,20,25,8,17,23,25,9,17,23,34,9,18,25,30,10,20,27,32,12,21,29,35,12,23,34,37,12,25,34,40,13,26,35,42,14,28,38,45,15,29,40,48,16,31,43,51,17,33,45,54,18,35,48,57,19,37,51,60,19,38,53,63,20,40,56,66,21,43,59,70,22,45,62,74,24,47,65,77,25,49,68,81],xy=[7,10,13,17,10,16,22,28,15,26,36,44,20,36,52,64,26,48,72,88,36,64,96,112,40,72,108,130,48,88,132,156,60,110,160,192,72,130,192,224,80,150,224,264,96,176,260,308,104,198,288,352,120,216,320,384,132,240,360,432,144,280,408,480,168,308,448,532,180,338,504,588,196,364,546,650,224,416,600,700,224,442,644,750,252,476,690,816,270,504,750,900,300,560,810,960,312,588,870,1050,336,644,952,1110,360,700,1020,1200,390,728,1050,1260,420,784,1140,1350,450,812,1200,1440,480,868,1290,1530,510,924,1350,1620,540,980,1440,1710,570,1036,1530,1800,570,1064,1590,1890,600,1120,1680,1980,630,1204,1770,2100,660,1260,1860,2220,720,1316,1950,2310,750,1372,2040,2430];dC.getBlocksCount=o(function(t,n){switch(n){case hu.L:return wy[(t-1)*4+0];case hu.M:return wy[(t-1)*4+1];case hu.Q:return wy[(t-1)*4+2];case hu.H:return wy[(t-1)*4+3];default:return}},"getBlocksCount");dC.getTotalCodewordsCount=o(function(t,n){switch(n){case hu.L:return xy[(t-1)*4+0];case hu.M:return xy[(t-1)*4+1];case hu.Q:return xy[(t-1)*4+2];case hu.H:return xy[(t-1)*4+3];default:return}},"getTotalCodewordsCount")});var QL=Ue(Cy=>{var Tm=new Uint8Array(512),Sy=new Uint8Array(256);o(function(){let t=1;for(let n=0;n<255;n++)Tm[n]=t,Sy[t]=n,t<<=1,t&256&&(t^=285);for(let n=255;n<512;n++)Tm[n]=Tm[n-255]},"initTables")();Cy.log=o(function(t){if(t<1)throw new Error("log("+t+")");return Sy[t]},"log");Cy.exp=o(function(t){return Tm[t]},"exp");Cy.mul=o(function(t,n){return t===0||n===0?0:Tm[Sy[t]+Sy[n]]},"mul")});var ZL=Ue(km=>{var mC=QL();km.mul=o(function(t,n){let l=new Uint8Array(t.length+n.length-1);for(let d=0;d=0;){let d=l[0];for(let c=0;c{var JL=ZL();function gC(e){this.genPoly=void 0,this.degree=e,this.degree&&this.initialize(this.degree)}o(gC,"ReedSolomonEncoder");gC.prototype.initialize=o(function(t){this.degree=t,this.genPoly=JL.generateECPolynomial(this.degree)},"initialize");gC.prototype.encode=o(function(t){if(!this.genPoly)throw new Error("Encoder not initialized");let n=new Uint8Array(t.length+this.degree);n.set(t);let l=JL.mod(n,this.genPoly),d=this.degree-l.length;if(d>0){let h=new Uint8Array(this.degree);return h.set(l,d),h}return l},"encode");eP.exports=gC});var vC=Ue(rP=>{rP.isValid=o(function(t){return!isNaN(t)&&t>=1&&t<=40},"isValid")});var yC=Ue(Bl=>{var nP="[0-9]+",Y3="[A-Z $%*+\\-./:]+",Nm="(?:[u3000-u303F]|[u3040-u309F]|[u30A0-u30FF]|[uFF00-uFFEF]|[u4E00-u9FAF]|[u2605-u2606]|[u2190-u2195]|u203B|[u2010u2015u2018u2019u2025u2026u201Cu201Du2225u2260]|[u0391-u0451]|[u00A7u00A8u00B1u00B4u00D7u00F7])+";Nm=Nm.replace(/u/g,"\\u");var X3="(?:(?![A-Z0-9 $%*+\\-./:]|"+Nm+`)(?:.|[\r -]))+`;Bl.KANJI=new RegExp(Nm,"g");Bl.BYTE_KANJI=new RegExp("[^A-Z0-9 $%*+\\-./:]+","g");Bl.BYTE=new RegExp(X3,"g");Bl.NUMERIC=new RegExp(nP,"g");Bl.ALPHANUMERIC=new RegExp(Y3,"g");var Q3=new RegExp("^"+Nm+"$"),Z3=new RegExp("^"+nP+"$"),J3=new RegExp("^[A-Z0-9 $%*+\\-./:]+$");Bl.testKanji=o(function(t){return Q3.test(t)},"testKanji");Bl.testNumeric=o(function(t){return Z3.test(t)},"testNumeric");Bl.testAlphanumeric=o(function(t){return J3.test(t)},"testAlphanumeric")});var mu=Ue(Xr=>{var eB=vC(),wC=yC();Xr.NUMERIC={id:"Numeric",bit:1<<0,ccBits:[10,12,14]};Xr.ALPHANUMERIC={id:"Alphanumeric",bit:1<<1,ccBits:[9,11,13]};Xr.BYTE={id:"Byte",bit:1<<2,ccBits:[8,16,16]};Xr.KANJI={id:"Kanji",bit:1<<3,ccBits:[8,10,12]};Xr.MIXED={bit:-1};Xr.getCharCountIndicator=o(function(t,n){if(!t.ccBits)throw new Error("Invalid mode: "+t);if(!eB.isValid(n))throw new Error("Invalid version: "+n);return n>=1&&n<10?t.ccBits[0]:n<27?t.ccBits[1]:t.ccBits[2]},"getCharCountIndicator");Xr.getBestModeForData=o(function(t){return wC.testNumeric(t)?Xr.NUMERIC:wC.testAlphanumeric(t)?Xr.ALPHANUMERIC:wC.testKanji(t)?Xr.KANJI:Xr.BYTE},"getBestModeForData");Xr.toString=o(function(t){if(t&&t.id)return t.id;throw new Error("Invalid mode")},"toString");Xr.isValid=o(function(t){return t&&t.bit&&t.ccBits},"isValid");function tB(e){if(typeof e!="string")throw new Error("Param is not a string");switch(e.toLowerCase()){case"numeric":return Xr.NUMERIC;case"alphanumeric":return Xr.ALPHANUMERIC;case"kanji":return Xr.KANJI;case"byte":return Xr.BYTE;default:throw new Error("Unknown mode: "+e)}}o(tB,"fromString");Xr.from=o(function(t,n){if(Xr.isValid(t))return t;try{return tB(t)}catch(l){return n}},"from")});var aP=Ue(Kf=>{var by=du(),rB=hC(),iP=vy(),gu=mu(),xC=vC(),oP=1<<12|1<<11|1<<10|1<<9|1<<8|1<<5|1<<2|1<<0,sP=by.getBCHDigit(oP);function nB(e,t,n){for(let l=1;l<=40;l++)if(t<=Kf.getCapacity(l,n,e))return l}o(nB,"getBestVersionForDataLength");function lP(e,t){return gu.getCharCountIndicator(e,t)+4}o(lP,"getReservedBitsCount");function iB(e,t){let n=0;return e.forEach(function(l){n+=lP(l.mode,t)+l.getBitsLength()}),n}o(iB,"getTotalBitsFromDataArray");function oB(e,t){for(let n=1;n<=40;n++)if(iB(e,n)<=Kf.getCapacity(n,t,gu.MIXED))return n}o(oB,"getBestVersionForMixedData");Kf.from=o(function(t,n){return xC.isValid(t)?parseInt(t,10):n},"from");Kf.getCapacity=o(function(t,n,l){if(!xC.isValid(t))throw new Error("Invalid QR Code version");typeof l=="undefined"&&(l=gu.BYTE);let d=by.getSymbolTotalCodewords(t),h=rB.getTotalCodewordsCount(t,n),c=(d-h)*8;if(l===gu.MIXED)return c;let v=c-lP(l,t);switch(l){case gu.NUMERIC:return Math.floor(v/10*3);case gu.ALPHANUMERIC:return Math.floor(v/11*2);case gu.KANJI:return Math.floor(v/13);case gu.BYTE:default:return Math.floor(v/8)}},"getCapacity");Kf.getBestVersionForData=o(function(t,n){let l,d=iP.from(n,iP.M);if(Array.isArray(t)){if(t.length>1)return oB(t,d);if(t.length===0)return 1;l=t[0]}else l=t;return nB(l.mode,l.getLength(),d)},"getBestVersionForData");Kf.getEncodedBits=o(function(t){if(!xC.isValid(t)||t<7)throw new Error("Invalid QR Code version");let n=t<<12;for(;by.getBCHDigit(n)-sP>=0;)n^=oP<{var SC=du(),uP=1<<10|1<<8|1<<5|1<<4|1<<2|1<<1|1<<0,sB=1<<14|1<<12|1<<10|1<<4|1<<1,fP=SC.getBCHDigit(uP);cP.getEncodedBits=o(function(t,n){let l=t.bit<<3|n,d=l<<10;for(;SC.getBCHDigit(d)-fP>=0;)d^=uP<{var lB=mu();function Kp(e){this.mode=lB.NUMERIC,this.data=e.toString()}o(Kp,"NumericData");Kp.getBitsLength=o(function(t){return 10*Math.floor(t/3)+(t%3?t%3*3+1:0)},"getBitsLength");Kp.prototype.getLength=o(function(){return this.data.length},"getLength");Kp.prototype.getBitsLength=o(function(){return Kp.getBitsLength(this.data.length)},"getBitsLength");Kp.prototype.write=o(function(t){let n,l,d;for(n=0;n+3<=this.data.length;n+=3)l=this.data.substr(n,3),d=parseInt(l,10),t.put(d,10);let h=this.data.length-n;h>0&&(l=this.data.substr(n),d=parseInt(l,10),t.put(d,h*3+1))},"write");dP.exports=Kp});var gP=Ue((c$,mP)=>{var aB=mu(),CC=["0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z"," ","$","%","*","+","-",".","/",":"];function Gp(e){this.mode=aB.ALPHANUMERIC,this.data=e}o(Gp,"AlphanumericData");Gp.getBitsLength=o(function(t){return 11*Math.floor(t/2)+6*(t%2)},"getBitsLength");Gp.prototype.getLength=o(function(){return this.data.length},"getLength");Gp.prototype.getBitsLength=o(function(){return Gp.getBitsLength(this.data.length)},"getBitsLength");Gp.prototype.write=o(function(t){let n;for(n=0;n+2<=this.data.length;n+=2){let l=CC.indexOf(this.data[n])*45;l+=CC.indexOf(this.data[n+1]),t.put(l,11)}this.data.length%2&&t.put(CC.indexOf(this.data[n]),6)},"write");mP.exports=Gp});var yP=Ue((p$,vP)=>{"use strict";vP.exports=o(function(t){for(var n=[],l=t.length,d=0;d=55296&&h<=56319&&l>d+1){var c=t.charCodeAt(d+1);c>=56320&&c<=57343&&(h=(h-55296)*1024+c-56320+65536,d+=1)}if(h<128){n.push(h);continue}if(h<2048){n.push(h>>6|192),n.push(h&63|128);continue}if(h<55296||h>=57344&&h<65536){n.push(h>>12|224),n.push(h>>6&63|128),n.push(h&63|128);continue}if(h>=65536&&h<=1114111){n.push(h>>18|240),n.push(h>>12&63|128),n.push(h>>6&63|128),n.push(h&63|128);continue}n.push(239,191,189)}return new Uint8Array(n).buffer},"encodeUtf8")});var xP=Ue((d$,wP)=>{var uB=yP(),fB=mu();function Yp(e){this.mode=fB.BYTE,typeof e=="string"&&(e=uB(e)),this.data=new Uint8Array(e)}o(Yp,"ByteData");Yp.getBitsLength=o(function(t){return t*8},"getBitsLength");Yp.prototype.getLength=o(function(){return this.data.length},"getLength");Yp.prototype.getBitsLength=o(function(){return Yp.getBitsLength(this.data.length)},"getBitsLength");Yp.prototype.write=function(e){for(let t=0,n=this.data.length;t{var cB=mu(),pB=du();function Xp(e){this.mode=cB.KANJI,this.data=e}o(Xp,"KanjiData");Xp.getBitsLength=o(function(t){return t*13},"getBitsLength");Xp.prototype.getLength=o(function(){return this.data.length},"getLength");Xp.prototype.getBitsLength=o(function(){return Xp.getBitsLength(this.data.length)},"getBitsLength");Xp.prototype.write=function(e){let t;for(t=0;t=33088&&n<=40956)n-=33088;else if(n>=57408&&n<=60351)n-=49472;else throw new Error("Invalid SJIS character: "+this.data[t]+` -Make sure your charset is UTF-8`);n=(n>>>8&255)*192+(n&255),e.put(n,13)}};SP.exports=Xp});var bP=Ue((m$,bC)=>{"use strict";var Lm={single_source_shortest_paths:function(e,t,n){var l={},d={};d[t]=0;var h=Lm.PriorityQueue.make();h.push(t,0);for(var c,v,C,k,O,j,B,X,J;!h.empty();){c=h.pop(),v=c.value,k=c.cost,O=e[v]||{};for(C in O)O.hasOwnProperty(C)&&(j=O[C],B=k+j,X=d[C],J=typeof d[C]=="undefined",(J||X>B)&&(d[C]=B,h.push(C,B),l[C]=v))}if(typeof n!="undefined"&&typeof d[n]=="undefined"){var Z=["Could not find a path from ",t," to ",n,"."].join("");throw new Error(Z)}return l},extract_shortest_path_from_predecessor_list:function(e,t){for(var n=[],l=t,d;l;)n.push(l),d=e[l],l=e[l];return n.reverse(),n},find_path:function(e,t,n){var l=Lm.single_source_shortest_paths(e,t,n);return Lm.extract_shortest_path_from_predecessor_list(l,n)},PriorityQueue:{make:function(e){var t=Lm.PriorityQueue,n={},l;e=e||{};for(l in t)t.hasOwnProperty(l)&&(n[l]=t[l]);return n.queue=[],n.sorter=e.sorter||t.default_sorter,n},default_sorter:function(e,t){return e.cost-t.cost},push:function(e,t){var n={value:e,cost:t};this.queue.push(n),this.queue.sort(this.sorter)},pop:function(){return this.queue.shift()},empty:function(){return this.queue.length===0}}};typeof bC!="undefined"&&(bC.exports=Lm)});var OP=Ue(Qp=>{var It=mu(),EP=hP(),_P=gP(),TP=xP(),kP=CP(),Pm=yC(),Ey=du(),dB=bP();function NP(e){return unescape(encodeURIComponent(e)).length}o(NP,"getStringByteLength");function Om(e,t,n){let l=[],d;for(;(d=e.exec(n))!==null;)l.push({data:d[0],index:d.index,mode:t,length:d[0].length});return l}o(Om,"getSegments");function LP(e){let t=Om(Pm.NUMERIC,It.NUMERIC,e),n=Om(Pm.ALPHANUMERIC,It.ALPHANUMERIC,e),l,d;return Ey.isKanjiModeEnabled()?(l=Om(Pm.BYTE,It.BYTE,e),d=Om(Pm.KANJI,It.KANJI,e)):(l=Om(Pm.BYTE_KANJI,It.BYTE,e),d=[]),t.concat(n,l,d).sort(function(c,v){return c.index-v.index}).map(function(c){return{data:c.data,mode:c.mode,length:c.length}})}o(LP,"getSegmentsFromString");function EC(e,t){switch(t){case It.NUMERIC:return EP.getBitsLength(e);case It.ALPHANUMERIC:return _P.getBitsLength(e);case It.KANJI:return kP.getBitsLength(e);case It.BYTE:return TP.getBitsLength(e)}}o(EC,"getSegmentBitsLength");function hB(e){return e.reduce(function(t,n){let l=t.length-1>=0?t[t.length-1]:null;return l&&l.mode===n.mode?(t[t.length-1].data+=n.data,t):(t.push(n),t)},[])}o(hB,"mergeSegments");function mB(e){let t=[];for(let n=0;n{var _y=du(),_C=vy(),vB=$L(),yB=qL(),wB=VL(),xB=YL(),TC=XL(),kC=hC(),SB=tP(),Ty=aP(),CB=pP(),bB=mu(),NC=OP();function EB(e,t){let n=e.size,l=xB.getPositions(t);for(let d=0;d=0&&v<=6&&(C===0||C===6)||C>=0&&C<=6&&(v===0||v===6)||v>=2&&v<=4&&C>=2&&C<=4?e.set(h+v,c+C,!0,!0):e.set(h+v,c+C,!1,!0))}}o(EB,"setupFinderPattern");function _B(e){let t=e.size;for(let n=8;n>v&1)==1,e.set(d,h,c,!0),e.set(h,d,c,!0)}o(kB,"setupVersionInfo");function LC(e,t,n){let l=e.size,d=CB.getEncodedBits(t,n),h,c;for(h=0;h<15;h++)c=(d>>h&1)==1,h<6?e.set(h,8,c,!0):h<8?e.set(h+1,8,c,!0):e.set(l-15+h,8,c,!0),h<8?e.set(8,l-h-1,c,!0):h<9?e.set(8,15-h-1+1,c,!0):e.set(8,15-h-1,c,!0);e.set(l-8,8,1,!0)}o(LC,"setupFormatInfo");function NB(e,t){let n=e.size,l=-1,d=n-1,h=7,c=0;for(let v=n-1;v>0;v-=2)for(v===6&&v--;;){for(let C=0;C<2;C++)if(!e.isReserved(d,v-C)){let k=!1;c>>h&1)==1),e.set(d,v-C,k),h--,h===-1&&(c++,h=7)}if(d+=l,d<0||n<=d){d-=l,l=-l;break}}}o(NB,"setupData");function LB(e,t,n){let l=new vB;n.forEach(function(C){l.put(C.mode.bit,4),l.put(C.getLength(),bB.getCharCountIndicator(C.mode,e)),C.write(l)});let d=_y.getSymbolTotalCodewords(e),h=kC.getTotalCodewordsCount(e,t),c=(d-h)*8;for(l.getLengthInBits()+4<=c&&l.put(0,4);l.getLengthInBits()%8!=0;)l.putBit(0);let v=(c-l.getLengthInBits())/8;for(let C=0;C0&&i.prevInput=="\u200B"?lr(u,Qm)(u):oe++<10?a.detectingSelectAll=setTimeout(de,500):(a.selForContextMenu=null,a.input.reset())},"poll");a.detectingSelectAll=setTimeout(de,200)}}if(o($,"rehide"),c&&v>=9&&V(),pe){lo(r);var te=o(function(){he(window,"mouseup",te),setTimeout($,20)},"mouseup");H(window,"mouseup",te)}else setTimeout($,50)},pr.prototype.readOnlyChanged=function(r){r||this.reset(),this.textarea.disabled=r=="nocursor",this.textarea.readOnly=!!r},pr.prototype.setUneditable=function(){},pr.prototype.needsContentAttribute=!1;function Vd(r,i){if(i=i?Qt(i):{},i.value=r.value,!i.tabindex&&r.tabIndex&&(i.tabindex=r.tabIndex),!i.placeholder&&r.placeholder&&(i.placeholder=r.placeholder),i.autofocus==null){var u=Ge();i.autofocus=u==r||r.getAttribute("autofocus")!=null&&u==document.body}function a(){r.value=S.getValue()}o(a,"save");var p;if(r.form&&(H(r.form,"submit",a),!i.leaveSubmitMethodAlone)){var g=r.form;p=g.submit;try{var y=g.submit=function(){a(),g.submit=p,g.submit(),g.submit=y}}catch(b){}}i.finishInit=function(b){b.save=a,b.getTextArea=function(){return r},b.toTextArea=function(){b.toTextArea=isNaN,a(),r.parentNode.removeChild(b.getWrapperElement()),r.style.display="",r.form&&(he(r.form,"submit",a),!i.leaveSubmitMethodAlone&&typeof r.form.submit=="function"&&(r.form.submit=p))}},r.style.display="none";var S=Mt(function(b){return r.parentNode.insertBefore(b,r.nextSibling)},i);return S}o(Vd,"fromTextArea");function Eg(r){r.off=he,r.on=H,r.wheelEventPixels=Ky,r.Doc=yn,r.splitLines=rl,r.countColumn=_t,r.findColumn=Pr,r.isWordChar=Zt,r.Pass=zt,r.signal=Te,r.Line=Ce,r.changeEnd=Ms,r.scrollbarModel=gc,r.Pos=ue,r.cmpPos=ze,r.modes=zl,r.mimeModes=ms,r.resolveMode=nl,r.getMode=xu,r.modeExtensions=Wo,r.extendMode=ad,r.copyState=Uo,r.startState=ec,r.innerMode=$l,r.commands=xl,r.keyMap=Jo,r.keyName=Id,r.isModifierKey=Oc,r.lookupKey=es,r.normalizeKeyMap=ew,r.StringStream=jt,r.SharedTextMarker=Ea,r.TextMarker=Rs,r.LineWidget=tf,r.e_preventDefault=or,r.e_stopPropagation=li,r.e_stop=lo,r.addClass=Xe,r.contains=Ke,r.rmClass=xe,r.keyNames=Fs}o(Eg,"addLegacyProps"),Fc(Mt),ts(Mt);var xn="iter insert remove copy getEditor constructor".split(" ");for(var Uc in yn.prototype)yn.prototype.hasOwnProperty(Uc)&&ut(xn,Uc)<0&&(Mt.prototype[Uc]=function(r){return function(){return r.apply(this.doc,arguments)}}(yn.prototype[Uc]));return Wr(yn),Mt.inputStyles={textarea:pr,contenteditable:xt},Mt.defineMode=function(r){!Mt.defaults.mode&&r!="null"&&(Mt.defaults.mode=r),ld.apply(this,arguments)},Mt.defineMIME=Jf,Mt.defineMode("null",function(){return{token:function(r){return r.skipToEnd()}}}),Mt.defineMIME("text/plain","null"),Mt.defineExtension=function(r,i){Mt.prototype[r]=i},Mt.defineDocExtension=function(r,i){yn.prototype[r]=i},Mt.fromTextArea=Vd,Eg(Mt),Mt.version="5.62.3",Mt})});var cL=Ue((oz,fL)=>{var T3=typeof Element!="undefined",k3=typeof Map=="function",N3=typeof Set=="function",L3=typeof ArrayBuffer=="function"&&!!ArrayBuffer.isView;function oy(e,t){if(e===t)return!0;if(e&&t&&typeof e=="object"&&typeof t=="object"){if(e.constructor!==t.constructor)return!1;var n,l,d;if(Array.isArray(e)){if(n=e.length,n!=t.length)return!1;for(l=n;l--!=0;)if(!oy(e[l],t[l]))return!1;return!0}var h;if(k3&&e instanceof Map&&t instanceof Map){if(e.size!==t.size)return!1;for(h=e.entries();!(l=h.next()).done;)if(!t.has(l.value[0]))return!1;for(h=e.entries();!(l=h.next()).done;)if(!oy(l.value[1],t.get(l.value[0])))return!1;return!0}if(N3&&e instanceof Set&&t instanceof Set){if(e.size!==t.size)return!1;for(h=e.entries();!(l=h.next()).done;)if(!t.has(l.value[0]))return!1;return!0}if(L3&&ArrayBuffer.isView(e)&&ArrayBuffer.isView(t)){if(n=e.length,n!=t.length)return!1;for(l=n;l--!=0;)if(e[l]!==t[l])return!1;return!0}if(e.constructor===RegExp)return e.source===t.source&&e.flags===t.flags;if(e.valueOf!==Object.prototype.valueOf)return e.valueOf()===t.valueOf();if(e.toString!==Object.prototype.toString)return e.toString()===t.toString();if(d=Object.keys(e),n=d.length,n!==Object.keys(t).length)return!1;for(l=n;l--!=0;)if(!Object.prototype.hasOwnProperty.call(t,d[l]))return!1;if(T3&&e instanceof Element)return!1;for(l=n;l--!=0;)if(!((d[l]==="_owner"||d[l]==="__v"||d[l]==="__o")&&e.$$typeof)&&!oy(e[d[l]],t[d[l]]))return!1;return!0}return e!==e&&t!==t}o(oy,"equal");fL.exports=o(function(t,n){try{return oy(t,n)}catch(l){if((l.message||"").match(/stack|recursion/i))return console.warn("react-fast-compare cannot handle circular refs"),!1;throw l}},"isEqual")});var EL=Ue((_9,bL)=>{"use strict";var U3="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED";bL.exports=U3});var NL=Ue((T9,kL)=>{"use strict";var z3=EL();function _L(){}o(_L,"emptyFunction");function TL(){}o(TL,"emptyFunctionWithReset");TL.resetWarningCache=_L;kL.exports=function(){function e(l,d,h,c,v,C){if(C!==z3){var k=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw k.name="Invariant Violation",k}}o(e,"shim"),e.isRequired=e;function t(){return e}o(t,"getShim");var n={array:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:t,element:e,elementType:e,instanceOf:t,node:e,objectOf:t,oneOf:t,oneOfType:t,shape:t,exact:t,checkPropTypes:TL,resetWarningCache:_L};return n.PropTypes=n,n}});var Sm=Ue((L9,LL)=>{LL.exports=NL()();var k9,N9});var fC=Ue((P9,PL)=>{PL.exports=o(function(t,n,l,d){var h=l?l.call(d,t,n):void 0;if(h!==void 0)return!!h;if(t===n)return!0;if(typeof t!="object"||!t||typeof n!="object"||!n)return!1;var c=Object.keys(t),v=Object.keys(n);if(c.length!==v.length)return!1;for(var C=Object.prototype.hasOwnProperty.bind(n),k=0;k{HL.exports=function(){return typeof Promise=="function"&&Promise.prototype&&Promise.prototype.then}});var du=Ue(qf=>{var pC,j3=[0,26,44,70,100,134,172,196,242,292,346,404,466,532,581,655,733,815,901,991,1085,1156,1258,1364,1474,1588,1706,1828,1921,2051,2185,2323,2465,2611,2761,2876,3034,3196,3362,3532,3706];qf.getSymbolSize=o(function(t){if(!t)throw new Error('"version" cannot be null or undefined');if(t<1||t>40)throw new Error('"version" should be in range from 1 to 40');return t*4+17},"getSymbolSize");qf.getSymbolTotalCodewords=o(function(t){return j3[t]},"getSymbolTotalCodewords");qf.getBCHDigit=function(e){let t=0;for(;e!==0;)t++,e>>>=1;return t};qf.setToSJISFunction=o(function(t){if(typeof t!="function")throw new Error('"toSJISFunc" is not a valid function.');pC=t},"setToSJISFunction");qf.isKanjiModeEnabled=function(){return typeof pC!="undefined"};qf.toSJIS=o(function(t){return pC(t)},"toSJIS")});var my=Ue(Do=>{Do.L={bit:1};Do.M={bit:0};Do.Q={bit:3};Do.H={bit:2};function q3(e){if(typeof e!="string")throw new Error("Param is not a string");switch(e.toLowerCase()){case"l":case"low":return Do.L;case"m":case"medium":return Do.M;case"q":case"quartile":return Do.Q;case"h":case"high":return Do.H;default:throw new Error("Unknown EC Level: "+e)}}o(q3,"fromString");Do.isValid=o(function(t){return t&&typeof t.bit!="undefined"&&t.bit>=0&&t.bit<4},"isValid");Do.from=o(function(t,n){if(Do.isValid(t))return t;try{return q3(t)}catch(l){return n}},"from")});var $L=Ue((X9,zL)=>{function UL(){this.buffer=[],this.length=0}o(UL,"BitBuffer");UL.prototype={get:function(e){let t=Math.floor(e/8);return(this.buffer[t]>>>7-e%8&1)==1},put:function(e,t){for(let n=0;n>>t-n-1&1)==1)},getLengthInBits:function(){return this.length},putBit:function(e){let t=Math.floor(this.length/8);this.buffer.length<=t&&this.buffer.push(0),e&&(this.buffer[t]|=128>>>this.length%8),this.length++}};zL.exports=UL});var qL=Ue((Q9,jL)=>{function _m(e){if(!e||e<1)throw new Error("BitMatrix size must be defined and greater than 0");this.size=e,this.data=new Uint8Array(e*e),this.reservedBit=new Uint8Array(e*e)}o(_m,"BitMatrix");_m.prototype.set=function(e,t,n,l){let d=e*this.size+t;this.data[d]=n,l&&(this.reservedBit[d]=!0)};_m.prototype.get=function(e,t){return this.data[e*this.size+t]};_m.prototype.xor=function(e,t,n){this.data[e*this.size+t]^=n};_m.prototype.isReserved=function(e,t){return this.reservedBit[e*this.size+t]};jL.exports=_m});var VL=Ue(gy=>{var V3=du().getSymbolSize;gy.getRowColCoords=o(function(t){if(t===1)return[];let n=Math.floor(t/7)+2,l=V3(t),d=l===145?26:Math.ceil((l-13)/(2*n-2))*2,h=[l-7];for(let c=1;c{var K3=du().getSymbolSize,KL=7;GL.getPositions=o(function(t){let n=K3(t);return[[0,0],[n-KL,0],[0,n-KL]]},"getPositions")});var XL=Ue(rr=>{rr.Patterns={PATTERN000:0,PATTERN001:1,PATTERN010:2,PATTERN011:3,PATTERN100:4,PATTERN101:5,PATTERN110:6,PATTERN111:7};var Vf={N1:3,N2:3,N3:40,N4:10};rr.isValid=o(function(t){return t!=null&&t!==""&&!isNaN(t)&&t>=0&&t<=7},"isValid");rr.from=o(function(t){return rr.isValid(t)?parseInt(t,10):void 0},"from");rr.getPenaltyN1=o(function(t){let n=t.size,l=0,d=0,h=0,c=null,v=null;for(let C=0;C=5&&(l+=Vf.N1+(d-5)),c=O,d=1),O=t.get(k,C),O===v?h++:(h>=5&&(l+=Vf.N1+(h-5)),v=O,h=1)}d>=5&&(l+=Vf.N1+(d-5)),h>=5&&(l+=Vf.N1+(h-5))}return l},"getPenaltyN1");rr.getPenaltyN2=o(function(t){let n=t.size,l=0;for(let d=0;d=10&&(d===1488||d===93)&&l++,h=h<<1&2047|t.get(v,c),v>=10&&(h===1488||h===93)&&l++}return l*Vf.N3},"getPenaltyN3");rr.getPenaltyN4=o(function(t){let n=0,l=t.data.length;for(let h=0;h{var hu=my(),vy=[1,1,1,1,1,1,1,1,1,1,2,2,1,2,2,4,1,2,4,4,2,4,4,4,2,4,6,5,2,4,6,6,2,5,8,8,4,5,8,8,4,5,8,11,4,8,10,11,4,9,12,16,4,9,16,16,6,10,12,18,6,10,17,16,6,11,16,19,6,13,18,21,7,14,21,25,8,16,20,25,8,17,23,25,9,17,23,34,9,18,25,30,10,20,27,32,12,21,29,35,12,23,34,37,12,25,34,40,13,26,35,42,14,28,38,45,15,29,40,48,16,31,43,51,17,33,45,54,18,35,48,57,19,37,51,60,19,38,53,63,20,40,56,66,21,43,59,70,22,45,62,74,24,47,65,77,25,49,68,81],yy=[7,10,13,17,10,16,22,28,15,26,36,44,20,36,52,64,26,48,72,88,36,64,96,112,40,72,108,130,48,88,132,156,60,110,160,192,72,130,192,224,80,150,224,264,96,176,260,308,104,198,288,352,120,216,320,384,132,240,360,432,144,280,408,480,168,308,448,532,180,338,504,588,196,364,546,650,224,416,600,700,224,442,644,750,252,476,690,816,270,504,750,900,300,560,810,960,312,588,870,1050,336,644,952,1110,360,700,1020,1200,390,728,1050,1260,420,784,1140,1350,450,812,1200,1440,480,868,1290,1530,510,924,1350,1620,540,980,1440,1710,570,1036,1530,1800,570,1064,1590,1890,600,1120,1680,1980,630,1204,1770,2100,660,1260,1860,2220,720,1316,1950,2310,750,1372,2040,2430];dC.getBlocksCount=o(function(t,n){switch(n){case hu.L:return vy[(t-1)*4+0];case hu.M:return vy[(t-1)*4+1];case hu.Q:return vy[(t-1)*4+2];case hu.H:return vy[(t-1)*4+3];default:return}},"getBlocksCount");dC.getTotalCodewordsCount=o(function(t,n){switch(n){case hu.L:return yy[(t-1)*4+0];case hu.M:return yy[(t-1)*4+1];case hu.Q:return yy[(t-1)*4+2];case hu.H:return yy[(t-1)*4+3];default:return}},"getTotalCodewordsCount")});var QL=Ue(xy=>{var Tm=new Uint8Array(512),wy=new Uint8Array(256);o(function(){let t=1;for(let n=0;n<255;n++)Tm[n]=t,wy[t]=n,t<<=1,t&256&&(t^=285);for(let n=255;n<512;n++)Tm[n]=Tm[n-255]},"initTables")();xy.log=o(function(t){if(t<1)throw new Error("log("+t+")");return wy[t]},"log");xy.exp=o(function(t){return Tm[t]},"exp");xy.mul=o(function(t,n){return t===0||n===0?0:Tm[wy[t]+wy[n]]},"mul")});var ZL=Ue(km=>{var mC=QL();km.mul=o(function(t,n){let l=new Uint8Array(t.length+n.length-1);for(let d=0;d=0;){let d=l[0];for(let c=0;c{var JL=ZL();function gC(e){this.genPoly=void 0,this.degree=e,this.degree&&this.initialize(this.degree)}o(gC,"ReedSolomonEncoder");gC.prototype.initialize=o(function(t){this.degree=t,this.genPoly=JL.generateECPolynomial(this.degree)},"initialize");gC.prototype.encode=o(function(t){if(!this.genPoly)throw new Error("Encoder not initialized");let n=new Uint8Array(t.length+this.degree);n.set(t);let l=JL.mod(n,this.genPoly),d=this.degree-l.length;if(d>0){let h=new Uint8Array(this.degree);return h.set(l,d),h}return l},"encode");eP.exports=gC});var vC=Ue(rP=>{rP.isValid=o(function(t){return!isNaN(t)&&t>=1&&t<=40},"isValid")});var yC=Ue(Bl=>{var nP="[0-9]+",Y3="[A-Z $%*+\\-./:]+",Nm="(?:[u3000-u303F]|[u3040-u309F]|[u30A0-u30FF]|[uFF00-uFFEF]|[u4E00-u9FAF]|[u2605-u2606]|[u2190-u2195]|u203B|[u2010u2015u2018u2019u2025u2026u201Cu201Du2225u2260]|[u0391-u0451]|[u00A7u00A8u00B1u00B4u00D7u00F7])+";Nm=Nm.replace(/u/g,"\\u");var X3="(?:(?![A-Z0-9 $%*+\\-./:]|"+Nm+`)(?:.|[\r +]))+`;Bl.KANJI=new RegExp(Nm,"g");Bl.BYTE_KANJI=new RegExp("[^A-Z0-9 $%*+\\-./:]+","g");Bl.BYTE=new RegExp(X3,"g");Bl.NUMERIC=new RegExp(nP,"g");Bl.ALPHANUMERIC=new RegExp(Y3,"g");var Q3=new RegExp("^"+Nm+"$"),Z3=new RegExp("^"+nP+"$"),J3=new RegExp("^[A-Z0-9 $%*+\\-./:]+$");Bl.testKanji=o(function(t){return Q3.test(t)},"testKanji");Bl.testNumeric=o(function(t){return Z3.test(t)},"testNumeric");Bl.testAlphanumeric=o(function(t){return J3.test(t)},"testAlphanumeric")});var mu=Ue(Xr=>{var eB=vC(),wC=yC();Xr.NUMERIC={id:"Numeric",bit:1<<0,ccBits:[10,12,14]};Xr.ALPHANUMERIC={id:"Alphanumeric",bit:1<<1,ccBits:[9,11,13]};Xr.BYTE={id:"Byte",bit:1<<2,ccBits:[8,16,16]};Xr.KANJI={id:"Kanji",bit:1<<3,ccBits:[8,10,12]};Xr.MIXED={bit:-1};Xr.getCharCountIndicator=o(function(t,n){if(!t.ccBits)throw new Error("Invalid mode: "+t);if(!eB.isValid(n))throw new Error("Invalid version: "+n);return n>=1&&n<10?t.ccBits[0]:n<27?t.ccBits[1]:t.ccBits[2]},"getCharCountIndicator");Xr.getBestModeForData=o(function(t){return wC.testNumeric(t)?Xr.NUMERIC:wC.testAlphanumeric(t)?Xr.ALPHANUMERIC:wC.testKanji(t)?Xr.KANJI:Xr.BYTE},"getBestModeForData");Xr.toString=o(function(t){if(t&&t.id)return t.id;throw new Error("Invalid mode")},"toString");Xr.isValid=o(function(t){return t&&t.bit&&t.ccBits},"isValid");function tB(e){if(typeof e!="string")throw new Error("Param is not a string");switch(e.toLowerCase()){case"numeric":return Xr.NUMERIC;case"alphanumeric":return Xr.ALPHANUMERIC;case"kanji":return Xr.KANJI;case"byte":return Xr.BYTE;default:throw new Error("Unknown mode: "+e)}}o(tB,"fromString");Xr.from=o(function(t,n){if(Xr.isValid(t))return t;try{return tB(t)}catch(l){return n}},"from")});var aP=Ue(Kf=>{var Sy=du(),rB=hC(),iP=my(),gu=mu(),xC=vC(),oP=1<<12|1<<11|1<<10|1<<9|1<<8|1<<5|1<<2|1<<0,sP=Sy.getBCHDigit(oP);function nB(e,t,n){for(let l=1;l<=40;l++)if(t<=Kf.getCapacity(l,n,e))return l}o(nB,"getBestVersionForDataLength");function lP(e,t){return gu.getCharCountIndicator(e,t)+4}o(lP,"getReservedBitsCount");function iB(e,t){let n=0;return e.forEach(function(l){n+=lP(l.mode,t)+l.getBitsLength()}),n}o(iB,"getTotalBitsFromDataArray");function oB(e,t){for(let n=1;n<=40;n++)if(iB(e,n)<=Kf.getCapacity(n,t,gu.MIXED))return n}o(oB,"getBestVersionForMixedData");Kf.from=o(function(t,n){return xC.isValid(t)?parseInt(t,10):n},"from");Kf.getCapacity=o(function(t,n,l){if(!xC.isValid(t))throw new Error("Invalid QR Code version");typeof l=="undefined"&&(l=gu.BYTE);let d=Sy.getSymbolTotalCodewords(t),h=rB.getTotalCodewordsCount(t,n),c=(d-h)*8;if(l===gu.MIXED)return c;let v=c-lP(l,t);switch(l){case gu.NUMERIC:return Math.floor(v/10*3);case gu.ALPHANUMERIC:return Math.floor(v/11*2);case gu.KANJI:return Math.floor(v/13);case gu.BYTE:default:return Math.floor(v/8)}},"getCapacity");Kf.getBestVersionForData=o(function(t,n){let l,d=iP.from(n,iP.M);if(Array.isArray(t)){if(t.length>1)return oB(t,d);if(t.length===0)return 1;l=t[0]}else l=t;return nB(l.mode,l.getLength(),d)},"getBestVersionForData");Kf.getEncodedBits=o(function(t){if(!xC.isValid(t)||t<7)throw new Error("Invalid QR Code version");let n=t<<12;for(;Sy.getBCHDigit(n)-sP>=0;)n^=oP<{var SC=du(),uP=1<<10|1<<8|1<<5|1<<4|1<<2|1<<1|1<<0,sB=1<<14|1<<12|1<<10|1<<4|1<<1,fP=SC.getBCHDigit(uP);cP.getEncodedBits=o(function(t,n){let l=t.bit<<3|n,d=l<<10;for(;SC.getBCHDigit(d)-fP>=0;)d^=uP<{var lB=mu();function Kp(e){this.mode=lB.NUMERIC,this.data=e.toString()}o(Kp,"NumericData");Kp.getBitsLength=o(function(t){return 10*Math.floor(t/3)+(t%3?t%3*3+1:0)},"getBitsLength");Kp.prototype.getLength=o(function(){return this.data.length},"getLength");Kp.prototype.getBitsLength=o(function(){return Kp.getBitsLength(this.data.length)},"getBitsLength");Kp.prototype.write=o(function(t){let n,l,d;for(n=0;n+3<=this.data.length;n+=3)l=this.data.substr(n,3),d=parseInt(l,10),t.put(d,10);let h=this.data.length-n;h>0&&(l=this.data.substr(n),d=parseInt(l,10),t.put(d,h*3+1))},"write");dP.exports=Kp});var gP=Ue((c$,mP)=>{var aB=mu(),CC=["0","1","2","3","4","5","6","7","8","9","A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z"," ","$","%","*","+","-",".","/",":"];function Gp(e){this.mode=aB.ALPHANUMERIC,this.data=e}o(Gp,"AlphanumericData");Gp.getBitsLength=o(function(t){return 11*Math.floor(t/2)+6*(t%2)},"getBitsLength");Gp.prototype.getLength=o(function(){return this.data.length},"getLength");Gp.prototype.getBitsLength=o(function(){return Gp.getBitsLength(this.data.length)},"getBitsLength");Gp.prototype.write=o(function(t){let n;for(n=0;n+2<=this.data.length;n+=2){let l=CC.indexOf(this.data[n])*45;l+=CC.indexOf(this.data[n+1]),t.put(l,11)}this.data.length%2&&t.put(CC.indexOf(this.data[n]),6)},"write");mP.exports=Gp});var yP=Ue((p$,vP)=>{"use strict";vP.exports=o(function(t){for(var n=[],l=t.length,d=0;d=55296&&h<=56319&&l>d+1){var c=t.charCodeAt(d+1);c>=56320&&c<=57343&&(h=(h-55296)*1024+c-56320+65536,d+=1)}if(h<128){n.push(h);continue}if(h<2048){n.push(h>>6|192),n.push(h&63|128);continue}if(h<55296||h>=57344&&h<65536){n.push(h>>12|224),n.push(h>>6&63|128),n.push(h&63|128);continue}if(h>=65536&&h<=1114111){n.push(h>>18|240),n.push(h>>12&63|128),n.push(h>>6&63|128),n.push(h&63|128);continue}n.push(239,191,189)}return new Uint8Array(n).buffer},"encodeUtf8")});var xP=Ue((d$,wP)=>{var uB=yP(),fB=mu();function Yp(e){this.mode=fB.BYTE,typeof e=="string"&&(e=uB(e)),this.data=new Uint8Array(e)}o(Yp,"ByteData");Yp.getBitsLength=o(function(t){return t*8},"getBitsLength");Yp.prototype.getLength=o(function(){return this.data.length},"getLength");Yp.prototype.getBitsLength=o(function(){return Yp.getBitsLength(this.data.length)},"getBitsLength");Yp.prototype.write=function(e){for(let t=0,n=this.data.length;t{var cB=mu(),pB=du();function Xp(e){this.mode=cB.KANJI,this.data=e}o(Xp,"KanjiData");Xp.getBitsLength=o(function(t){return t*13},"getBitsLength");Xp.prototype.getLength=o(function(){return this.data.length},"getLength");Xp.prototype.getBitsLength=o(function(){return Xp.getBitsLength(this.data.length)},"getBitsLength");Xp.prototype.write=function(e){let t;for(t=0;t=33088&&n<=40956)n-=33088;else if(n>=57408&&n<=60351)n-=49472;else throw new Error("Invalid SJIS character: "+this.data[t]+` +Make sure your charset is UTF-8`);n=(n>>>8&255)*192+(n&255),e.put(n,13)}};SP.exports=Xp});var bP=Ue((m$,bC)=>{"use strict";var Lm={single_source_shortest_paths:function(e,t,n){var l={},d={};d[t]=0;var h=Lm.PriorityQueue.make();h.push(t,0);for(var c,v,C,k,O,j,B,X,J;!h.empty();){c=h.pop(),v=c.value,k=c.cost,O=e[v]||{};for(C in O)O.hasOwnProperty(C)&&(j=O[C],B=k+j,X=d[C],J=typeof d[C]=="undefined",(J||X>B)&&(d[C]=B,h.push(C,B),l[C]=v))}if(typeof n!="undefined"&&typeof d[n]=="undefined"){var Z=["Could not find a path from ",t," to ",n,"."].join("");throw new Error(Z)}return l},extract_shortest_path_from_predecessor_list:function(e,t){for(var n=[],l=t,d;l;)n.push(l),d=e[l],l=e[l];return n.reverse(),n},find_path:function(e,t,n){var l=Lm.single_source_shortest_paths(e,t,n);return Lm.extract_shortest_path_from_predecessor_list(l,n)},PriorityQueue:{make:function(e){var t=Lm.PriorityQueue,n={},l;e=e||{};for(l in t)t.hasOwnProperty(l)&&(n[l]=t[l]);return n.queue=[],n.sorter=e.sorter||t.default_sorter,n},default_sorter:function(e,t){return e.cost-t.cost},push:function(e,t){var n={value:e,cost:t};this.queue.push(n),this.queue.sort(this.sorter)},pop:function(){return this.queue.shift()},empty:function(){return this.queue.length===0}}};typeof bC!="undefined"&&(bC.exports=Lm)});var OP=Ue(Qp=>{var It=mu(),EP=hP(),_P=gP(),TP=xP(),kP=CP(),Pm=yC(),Cy=du(),dB=bP();function NP(e){return unescape(encodeURIComponent(e)).length}o(NP,"getStringByteLength");function Om(e,t,n){let l=[],d;for(;(d=e.exec(n))!==null;)l.push({data:d[0],index:d.index,mode:t,length:d[0].length});return l}o(Om,"getSegments");function LP(e){let t=Om(Pm.NUMERIC,It.NUMERIC,e),n=Om(Pm.ALPHANUMERIC,It.ALPHANUMERIC,e),l,d;return Cy.isKanjiModeEnabled()?(l=Om(Pm.BYTE,It.BYTE,e),d=Om(Pm.KANJI,It.KANJI,e)):(l=Om(Pm.BYTE_KANJI,It.BYTE,e),d=[]),t.concat(n,l,d).sort(function(c,v){return c.index-v.index}).map(function(c){return{data:c.data,mode:c.mode,length:c.length}})}o(LP,"getSegmentsFromString");function EC(e,t){switch(t){case It.NUMERIC:return EP.getBitsLength(e);case It.ALPHANUMERIC:return _P.getBitsLength(e);case It.KANJI:return kP.getBitsLength(e);case It.BYTE:return TP.getBitsLength(e)}}o(EC,"getSegmentBitsLength");function hB(e){return e.reduce(function(t,n){let l=t.length-1>=0?t[t.length-1]:null;return l&&l.mode===n.mode?(t[t.length-1].data+=n.data,t):(t.push(n),t)},[])}o(hB,"mergeSegments");function mB(e){let t=[];for(let n=0;n{var by=du(),_C=my(),vB=$L(),yB=qL(),wB=VL(),xB=YL(),TC=XL(),kC=hC(),SB=tP(),Ey=aP(),CB=pP(),bB=mu(),NC=OP();function EB(e,t){let n=e.size,l=xB.getPositions(t);for(let d=0;d=0&&v<=6&&(C===0||C===6)||C>=0&&C<=6&&(v===0||v===6)||v>=2&&v<=4&&C>=2&&C<=4?e.set(h+v,c+C,!0,!0):e.set(h+v,c+C,!1,!0))}}o(EB,"setupFinderPattern");function _B(e){let t=e.size;for(let n=8;n>v&1)==1,e.set(d,h,c,!0),e.set(h,d,c,!0)}o(kB,"setupVersionInfo");function LC(e,t,n){let l=e.size,d=CB.getEncodedBits(t,n),h,c;for(h=0;h<15;h++)c=(d>>h&1)==1,h<6?e.set(h,8,c,!0):h<8?e.set(h+1,8,c,!0):e.set(l-15+h,8,c,!0),h<8?e.set(8,l-h-1,c,!0):h<9?e.set(8,15-h-1+1,c,!0):e.set(8,15-h-1,c,!0);e.set(l-8,8,1,!0)}o(LC,"setupFormatInfo");function NB(e,t){let n=e.size,l=-1,d=n-1,h=7,c=0;for(let v=n-1;v>0;v-=2)for(v===6&&v--;;){for(let C=0;C<2;C++)if(!e.isReserved(d,v-C)){let k=!1;c>>h&1)==1),e.set(d,v-C,k),h--,h===-1&&(c++,h=7)}if(d+=l,d<0||n<=d){d-=l,l=-l;break}}}o(NB,"setupData");function LB(e,t,n){let l=new vB;n.forEach(function(C){l.put(C.mode.bit,4),l.put(C.getLength(),bB.getCharCountIndicator(C.mode,e)),C.write(l)});let d=by.getSymbolTotalCodewords(e),h=kC.getTotalCodewordsCount(e,t),c=(d-h)*8;for(l.getLengthInBits()+4<=c&&l.put(0,4);l.getLengthInBits()%8!=0;)l.putBit(0);let v=(c-l.getLengthInBits())/8;for(let C=0;C=7&&kB(C,t),NB(C,c),isNaN(l)&&(l=TC.getBestMask(C,LC.bind(null,C,n))),TC.applyMask(l,C),LC(C,n,l),{modules:C,version:t,errorCorrectionLevel:n,maskPattern:l,segments:d}}o(OB,"createSymbol");MP.create=o(function(t,n){if(typeof t=="undefined"||t==="")throw new Error("No input text");let l=_C.M,d,h;return typeof n!="undefined"&&(l=_C.from(n.errorCorrectionLevel,_C.M),d=Ty.from(n.version),h=TC.from(n.maskPattern),n.toSJISFunc&&_y.setToSJISFunction(n.toSJISFunc)),OB(t,d,l,h)},"create")});var PC=Ue(Gf=>{function DP(e){if(typeof e=="number"&&(e=e.toString()),typeof e!="string")throw new Error("Color should be defined as hex string");let t=e.slice().replace("#","").split("");if(t.length<3||t.length===5||t.length>8)throw new Error("Invalid hex color: "+e);(t.length===3||t.length===4)&&(t=Array.prototype.concat.apply([],t.map(function(l){return[l,l]}))),t.length===6&&t.push("F","F");let n=parseInt(t.join(""),16);return{r:n>>24&255,g:n>>16&255,b:n>>8&255,a:n&255,hex:"#"+t.slice(0,6).join("")}}o(DP,"hex2rgba");Gf.getOptions=o(function(t){t||(t={}),t.color||(t.color={});let n=typeof t.margin=="undefined"||t.margin===null||t.margin<0?4:t.margin,l=t.width&&t.width>=21?t.width:void 0,d=t.scale||4;return{width:l,scale:l?4:d,margin:n,color:{dark:DP(t.color.dark||"#000000ff"),light:DP(t.color.light||"#ffffffff")},type:t.type,rendererOpts:t.rendererOpts||{}}},"getOptions");Gf.getScale=o(function(t,n){return n.width&&n.width>=t+n.margin*2?n.width/(t+n.margin*2):n.scale},"getScale");Gf.getImageWidth=o(function(t,n){let l=Gf.getScale(t,n);return Math.floor((t+n.margin*2)*l)},"getImageWidth");Gf.qrToImageData=o(function(t,n,l){let d=n.modules.size,h=n.modules.data,c=Gf.getScale(d,l),v=Math.floor((d+l.margin*2)*c),C=l.margin*c,k=[l.color.light,l.color.dark];for(let O=0;O=C&&j>=C&&O{var OC=PC();function MB(e,t,n){e.clearRect(0,0,t.width,t.height),t.style||(t.style={}),t.height=n,t.width=n,t.style.height=n+"px",t.style.width=n+"px"}o(MB,"clearCanvas");function AB(){try{return document.createElement("canvas")}catch(e){throw new Error("You need to specify a canvas element")}}o(AB,"getCanvasElement");ky.render=o(function(t,n,l){let d=l,h=n;typeof d=="undefined"&&(!n||!n.getContext)&&(d=n,n=void 0),n||(h=AB()),d=OC.getOptions(d);let c=OC.getImageWidth(t.modules.size,d),v=h.getContext("2d"),C=v.createImageData(c,c);return OC.qrToImageData(C.data,t,d),MB(v,h,c),v.putImageData(C,0,0),h},"render");ky.renderToDataURL=o(function(t,n,l){let d=l;typeof d=="undefined"&&(!n||!n.getContext)&&(d=n,n=void 0),d||(d={});let h=ky.render(t,n,d),c=d.type||"image/png",v=d.rendererOpts||{};return h.toDataURL(c,v.quality)},"renderToDataURL")});var BP=Ue(FP=>{var DB=PC();function IP(e,t){let n=e.a/255,l=t+'="'+e.hex+'"';return n<1?l+" "+t+'-opacity="'+n.toFixed(2).slice(1)+'"':l}o(IP,"getColorAttrib");function MC(e,t,n){let l=e+t;return typeof n!="undefined"&&(l+=" "+n),l}o(MC,"svgCmd");function RB(e,t,n){let l="",d=0,h=!1,c=0;for(let v=0;v0&&C>0&&e[v-1]||(l+=h?MC("M",C+n,.5+k+n):MC("m",d,0),d=0,h=!1),C+1':"",k="',O='viewBox="0 0 '+v+" "+v+'"',j=d.width?'width="'+d.width+'" height="'+d.width+'" ':"",B=''+C+k+` -`;return typeof l=="function"&&l(null,B),B},"render")});var WP=Ue(Mm=>{var IB=WL(),AC=AP(),HP=RP(),FB=BP();function DC(e,t,n,l,d){let h=[].slice.call(arguments,1),c=h.length,v=typeof h[c-1]=="function";if(!v&&!IB())throw new Error("Callback required as last argument");if(v){if(c<2)throw new Error("Too few arguments provided");c===2?(d=n,n=t,t=l=void 0):c===3&&(t.getContext&&typeof d=="undefined"?(d=l,l=void 0):(d=l,l=n,n=t,t=void 0))}else{if(c<1)throw new Error("Too few arguments provided");return c===1?(n=t,t=l=void 0):c===2&&!t.getContext&&(l=n,n=t,t=void 0),new Promise(function(C,k){try{let O=AC.create(n,l);C(e(O,t,l))}catch(O){k(O)}})}try{let C=AC.create(n,l);d(null,e(C,t,l))}catch(C){d(C)}}o(DC,"renderCanvas");Mm.create=AC.create;Mm.toCanvas=DC.bind(null,HP.render);Mm.toDataURL=DC.bind(null,HP.renderToDataURL);Mm.toString=DC.bind(null,function(e,t,n){return FB.render(e,n)})});var nb=fe(Oe()),aO=fe(iu());var Gh=fe(Oe()),k4=fe(ak());var uk=fe(Oe()),ei=uk.default.createContext(null);function EI(e){e()}o(EI,"defaultNoopBatch");var fk=EI,ck=o(function(t){return fk=t},"setBatch"),pk=o(function(){return fk},"getBatch");var dk={notify:o(function(){},"notify")};function _I(){var e=pk(),t=null,n=null;return{clear:o(function(){t=null,n=null},"clear"),notify:o(function(){e(function(){for(var d=t;d;)d.callback(),d=d.next})},"notify"),get:o(function(){for(var d=[],h=t;h;)d.push(h),h=h.next;return d},"get"),subscribe:o(function(d){var h=!0,c=n={callback:d,next:null,prev:n};return c.prev?c.prev.next=c:t=c,o(function(){!h||t===null||(h=!1,c.next?c.next.prev=c.prev:n=c.prev,c.prev?c.prev.next=c.next:t=c.next)},"unsubscribe")},"subscribe")}}o(_I,"createListenerCollection");var Tp=function(){function e(n,l){this.store=n,this.parentSub=l,this.unsubscribe=null,this.listeners=dk,this.handleChangeWrapper=this.handleChangeWrapper.bind(this)}o(e,"Subscription");var t=e.prototype;return t.addNestedSub=o(function(l){return this.trySubscribe(),this.listeners.subscribe(l)},"addNestedSub"),t.notifyNestedSubs=o(function(){this.listeners.notify()},"notifyNestedSubs"),t.handleChangeWrapper=o(function(){this.onStateChange&&this.onStateChange()},"handleChangeWrapper"),t.isSubscribed=o(function(){return Boolean(this.unsubscribe)},"isSubscribed"),t.trySubscribe=o(function(){this.unsubscribe||(this.unsubscribe=this.parentSub?this.parentSub.addNestedSub(this.handleChangeWrapper):this.store.subscribe(this.handleChangeWrapper),this.listeners=_I())},"trySubscribe"),t.tryUnsubscribe=o(function(){this.unsubscribe&&(this.unsubscribe(),this.unsubscribe=null,this.listeners.clear(),this.listeners=dk)},"tryUnsubscribe"),e}();var Gv=fe(Oe()),Nf=typeof window!="undefined"&&typeof window.document!="undefined"&&typeof window.document.createElement!="undefined"?Gv.useLayoutEffect:Gv.useEffect;function TI(e){var t=e.store,n=e.context,l=e.children,d=(0,Gh.useMemo)(function(){var v=new Tp(t);return v.onStateChange=v.notifyNestedSubs,{store:t,subscription:v}},[t]),h=(0,Gh.useMemo)(function(){return t.getState()},[t]);Nf(function(){var v=d.subscription;return v.trySubscribe(),h!==t.getState()&&v.notifyNestedSubs(),function(){v.tryUnsubscribe(),v.onStateChange=null}},[d,h]);var c=n||ei;return Gh.default.createElement(c.Provider,{value:d},l)}o(TI,"Provider");var Ux=TI;function Po(){return Po=Object.assign||function(e){for(var t=1;t=0)&&(n[d]=e[d]);return n}o(ou,"_objectWithoutPropertiesLoose");var Xx=fe(Ek()),cr=fe(Oe()),Lk=fe(Nk());var jI=[],qI=[null,null];function VI(e,t){var n=e[1];return[t.payload,n+1]}o(VI,"storeStateUpdatesReducer");function Pk(e,t,n){Nf(function(){return e.apply(void 0,t)},n)}o(Pk,"useIsomorphicLayoutEffectWithArgs");function KI(e,t,n,l,d,h,c){e.current=l,t.current=d,n.current=!1,h.current&&(h.current=null,c())}o(KI,"captureWrapperProps");function GI(e,t,n,l,d,h,c,v,C,k){if(!!e){var O=!1,j=null,B=o(function(){if(!O){var Z=t.getState(),R,A;try{R=l(Z,d.current)}catch(I){A=I,j=I}A||(j=null),R===h.current?c.current||C():(h.current=R,v.current=R,c.current=!0,k({type:"STORE_UPDATED",payload:{error:A}}))}},"checkForUpdates");n.onStateChange=B,n.trySubscribe(),B();var X=o(function(){if(O=!0,n.tryUnsubscribe(),n.onStateChange=null,j)throw j},"unsubscribeWrapper");return X}}o(GI,"subscribeUpdates");var YI=o(function(){return[null,0]},"initStateUpdates");function m0(e,t){t===void 0&&(t={});var n=t,l=n.getDisplayName,d=l===void 0?function(ne){return"ConnectAdvanced("+ne+")"}:l,h=n.methodName,c=h===void 0?"connectAdvanced":h,v=n.renderCountProp,C=v===void 0?void 0:v,k=n.shouldHandleStateChanges,O=k===void 0?!0:k,j=n.storeKey,B=j===void 0?"store":j,X=n.withRef,J=X===void 0?!1:X,Z=n.forwardRef,R=Z===void 0?!1:Z,A=n.context,I=A===void 0?ei:A,G=ou(n,["getDisplayName","methodName","renderCountProp","shouldHandleStateChanges","storeKey","withRef","forwardRef","context"]);if(!1)var K;var se=I;return o(function(pe){var me=pe.displayName||pe.name||"Component",xe=d(me),Ve=Po({},G,{getDisplayName:d,methodName:c,renderCountProp:C,shouldHandleStateChanges:O,storeKey:B,displayName:xe,wrappedComponentName:me,WrappedComponent:pe}),tt=G.pure;function _e(Xe){return e(Xe.dispatch,Ve)}o(_e,"createChildSelector");var St=tt?cr.useMemo:function(Xe){return Xe()};function We(Xe){var nr=(0,cr.useMemo)(function(){var Cr=Xe.reactReduxForwardedRef,Ui=ou(Xe,["reactReduxForwardedRef"]);return[Xe.context,Cr,Ui]},[Xe]),ct=nr[0],Hr=nr[1],Qt=nr[2],_t=(0,cr.useMemo)(function(){return ct&&ct.Consumer&&(0,Lk.isContextConsumer)(cr.default.createElement(ct.Consumer,null))?ct:se},[ct,se]),Ct=(0,cr.useContext)(_t),ut=Boolean(Xe.store)&&Boolean(Xe.store.getState)&&Boolean(Xe.store.dispatch),Lr=Boolean(Ct)&&Boolean(Ct.store),zt=ut?Xe.store:Ct.store,$t=(0,cr.useMemo)(function(){return _e(zt)},[zt]),ie=(0,cr.useMemo)(function(){if(!O)return qI;var Cr=new Tp(zt,ut?null:Ct.subscription),Ui=Cr.notifyNestedSubs.bind(Cr);return[Cr,Ui]},[zt,ut,Ct]),rt=ie[0],Pr=ie[1],Gt=(0,cr.useMemo)(function(){return ut?Ct:Po({},Ct,{subscription:rt})},[ut,Ct,rt]),Yt=(0,cr.useReducer)(VI,jI,YI),Se=Yt[0],Or=Se[0],fn=Yt[1];if(Or&&Or.error)throw Or.error;var Un=(0,cr.useRef)(),si=(0,cr.useRef)(Qt),cn=(0,cr.useRef)(),Zt=(0,cr.useRef)(!1),gr=St(function(){return cn.current&&Qt===si.current?cn.current:$t(zt.getState(),Qt)},[zt,Or,Qt]);Pk(KI,[si,Un,Zt,Qt,gr,cn,Pr]),Pk(GI,[O,zt,rt,$t,si,Un,Zt,cn,Pr,fn],[zt,rt,$t]);var pt=(0,cr.useMemo)(function(){return cr.default.createElement(pe,Po({},gr,{ref:Hr}))},[Hr,pe,gr]),Ho=(0,cr.useMemo)(function(){return O?cr.default.createElement(_t.Provider,{value:Gt},pt):pt},[_t,pt,Gt]);return Ho}o(We,"ConnectFunction");var Ke=tt?cr.default.memo(We):We;if(Ke.WrappedComponent=pe,Ke.displayName=We.displayName=xe,R){var Ge=cr.default.forwardRef(o(function(nr,ct){return cr.default.createElement(Ke,Po({},nr,{reactReduxForwardedRef:ct}))},"forwardConnectRef"));return Ge.displayName=xe,Ge.WrappedComponent=pe,(0,Xx.default)(Ge,pe)}return(0,Xx.default)(Ke,pe)},"wrapWithConnect")}o(m0,"connectAdvanced");function Ok(e,t){return e===t?e!==0||t!==0||1/e==1/t:e!==e&&t!==t}o(Ok,"is");function kp(e,t){if(Ok(e,t))return!0;if(typeof e!="object"||e===null||typeof t!="object"||t===null)return!1;var n=Object.keys(e),l=Object.keys(t);if(n.length!==l.length)return!1;for(var d=0;d=0;l--){var d=t[l](e);if(d)return d}return function(h,c){throw new Error("Invalid value of type "+typeof e+" for "+n+" argument when connecting component "+c.wrappedComponentName+".")}}o(Jx,"match");function aF(e,t){return e===t}o(aF,"strictEqual");function uF(e){var t=e===void 0?{}:e,n=t.connectHOC,l=n===void 0?m0:n,d=t.mapStateToPropsFactories,h=d===void 0?Dk:d,c=t.mapDispatchToPropsFactories,v=c===void 0?Ak:c,C=t.mergePropsFactories,k=C===void 0?Rk:C,O=t.selectorFactory,j=O===void 0?Zx:O;return o(function(X,J,Z,R){R===void 0&&(R={});var A=R,I=A.pure,G=I===void 0?!0:I,K=A.areStatesEqual,se=K===void 0?aF:K,ne=A.areOwnPropsEqual,pe=ne===void 0?kp:ne,me=A.areStatePropsEqual,xe=me===void 0?kp:me,Ve=A.areMergedPropsEqual,tt=Ve===void 0?kp:Ve,_e=ou(A,["pure","areStatesEqual","areOwnPropsEqual","areStatePropsEqual","areMergedPropsEqual"]),St=Jx(X,h,"mapStateToProps"),We=Jx(J,v,"mapDispatchToProps"),Ke=Jx(Z,k,"mergeProps");return l(j,Po({methodName:"connect",getDisplayName:o(function(Xe){return"Connect("+Xe+")"},"getDisplayName"),shouldHandleStateChanges:Boolean(X),initMapStateToProps:St,initMapDispatchToProps:We,initMergeProps:Ke,pure:G,areStatesEqual:se,areOwnPropsEqual:pe,areStatePropsEqual:xe,areMergedPropsEqual:tt},_e))},"connect")}o(uF,"createConnect");var Hi=uF();var Fk=fe(Oe());var Ik=fe(Oe());function v0(){var e=(0,Ik.useContext)(ei);return e}o(v0,"useReduxContext");function y0(e){e===void 0&&(e=ei);var t=e===ei?v0:function(){return(0,Fk.useContext)(e)};return o(function(){var l=t(),d=l.store;return d},"useStore")}o(y0,"createStoreHook");var eS=y0();function Bk(e){e===void 0&&(e=ei);var t=e===ei?eS:y0(e);return o(function(){var l=t();return l.dispatch},"useDispatch")}o(Bk,"createDispatchHook");var Gs=Bk();var ro=fe(Oe());var fF=o(function(t,n){return t===n},"refEquality");function cF(e,t,n,l){var d=(0,ro.useReducer)(function(J){return J+1},0),h=d[1],c=(0,ro.useMemo)(function(){return new Tp(n,l)},[n,l]),v=(0,ro.useRef)(),C=(0,ro.useRef)(),k=(0,ro.useRef)(),O=(0,ro.useRef)(),j=n.getState(),B;try{if(e!==C.current||j!==k.current||v.current){var X=e(j);O.current===void 0||!t(X,O.current)?B=X:B=O.current}else B=O.current}catch(J){throw v.current&&(J.message+=` +`);let c=LB(t,n,d),v=by.getSymbolSize(t),C=new yB(v);return EB(C,t),_B(C),TB(C,t),LC(C,n,0),t>=7&&kB(C,t),NB(C,c),isNaN(l)&&(l=TC.getBestMask(C,LC.bind(null,C,n))),TC.applyMask(l,C),LC(C,n,l),{modules:C,version:t,errorCorrectionLevel:n,maskPattern:l,segments:d}}o(OB,"createSymbol");MP.create=o(function(t,n){if(typeof t=="undefined"||t==="")throw new Error("No input text");let l=_C.M,d,h;return typeof n!="undefined"&&(l=_C.from(n.errorCorrectionLevel,_C.M),d=Ey.from(n.version),h=TC.from(n.maskPattern),n.toSJISFunc&&by.setToSJISFunction(n.toSJISFunc)),OB(t,d,l,h)},"create")});var PC=Ue(Gf=>{function DP(e){if(typeof e=="number"&&(e=e.toString()),typeof e!="string")throw new Error("Color should be defined as hex string");let t=e.slice().replace("#","").split("");if(t.length<3||t.length===5||t.length>8)throw new Error("Invalid hex color: "+e);(t.length===3||t.length===4)&&(t=Array.prototype.concat.apply([],t.map(function(l){return[l,l]}))),t.length===6&&t.push("F","F");let n=parseInt(t.join(""),16);return{r:n>>24&255,g:n>>16&255,b:n>>8&255,a:n&255,hex:"#"+t.slice(0,6).join("")}}o(DP,"hex2rgba");Gf.getOptions=o(function(t){t||(t={}),t.color||(t.color={});let n=typeof t.margin=="undefined"||t.margin===null||t.margin<0?4:t.margin,l=t.width&&t.width>=21?t.width:void 0,d=t.scale||4;return{width:l,scale:l?4:d,margin:n,color:{dark:DP(t.color.dark||"#000000ff"),light:DP(t.color.light||"#ffffffff")},type:t.type,rendererOpts:t.rendererOpts||{}}},"getOptions");Gf.getScale=o(function(t,n){return n.width&&n.width>=t+n.margin*2?n.width/(t+n.margin*2):n.scale},"getScale");Gf.getImageWidth=o(function(t,n){let l=Gf.getScale(t,n);return Math.floor((t+n.margin*2)*l)},"getImageWidth");Gf.qrToImageData=o(function(t,n,l){let d=n.modules.size,h=n.modules.data,c=Gf.getScale(d,l),v=Math.floor((d+l.margin*2)*c),C=l.margin*c,k=[l.color.light,l.color.dark];for(let O=0;O=C&&j>=C&&O{var OC=PC();function MB(e,t,n){e.clearRect(0,0,t.width,t.height),t.style||(t.style={}),t.height=n,t.width=n,t.style.height=n+"px",t.style.width=n+"px"}o(MB,"clearCanvas");function AB(){try{return document.createElement("canvas")}catch(e){throw new Error("You need to specify a canvas element")}}o(AB,"getCanvasElement");_y.render=o(function(t,n,l){let d=l,h=n;typeof d=="undefined"&&(!n||!n.getContext)&&(d=n,n=void 0),n||(h=AB()),d=OC.getOptions(d);let c=OC.getImageWidth(t.modules.size,d),v=h.getContext("2d"),C=v.createImageData(c,c);return OC.qrToImageData(C.data,t,d),MB(v,h,c),v.putImageData(C,0,0),h},"render");_y.renderToDataURL=o(function(t,n,l){let d=l;typeof d=="undefined"&&(!n||!n.getContext)&&(d=n,n=void 0),d||(d={});let h=_y.render(t,n,d),c=d.type||"image/png",v=d.rendererOpts||{};return h.toDataURL(c,v.quality)},"renderToDataURL")});var BP=Ue(FP=>{var DB=PC();function IP(e,t){let n=e.a/255,l=t+'="'+e.hex+'"';return n<1?l+" "+t+'-opacity="'+n.toFixed(2).slice(1)+'"':l}o(IP,"getColorAttrib");function MC(e,t,n){let l=e+t;return typeof n!="undefined"&&(l+=" "+n),l}o(MC,"svgCmd");function RB(e,t,n){let l="",d=0,h=!1,c=0;for(let v=0;v0&&C>0&&e[v-1]||(l+=h?MC("M",C+n,.5+k+n):MC("m",d,0),d=0,h=!1),C+1':"",k="',O='viewBox="0 0 '+v+" "+v+'"',j=d.width?'width="'+d.width+'" height="'+d.width+'" ':"",B=''+C+k+` +`;return typeof l=="function"&&l(null,B),B},"render")});var WP=Ue(Mm=>{var IB=WL(),AC=AP(),HP=RP(),FB=BP();function DC(e,t,n,l,d){let h=[].slice.call(arguments,1),c=h.length,v=typeof h[c-1]=="function";if(!v&&!IB())throw new Error("Callback required as last argument");if(v){if(c<2)throw new Error("Too few arguments provided");c===2?(d=n,n=t,t=l=void 0):c===3&&(t.getContext&&typeof d=="undefined"?(d=l,l=void 0):(d=l,l=n,n=t,t=void 0))}else{if(c<1)throw new Error("Too few arguments provided");return c===1?(n=t,t=l=void 0):c===2&&!t.getContext&&(l=n,n=t,t=void 0),new Promise(function(C,k){try{let O=AC.create(n,l);C(e(O,t,l))}catch(O){k(O)}})}try{let C=AC.create(n,l);d(null,e(C,t,l))}catch(C){d(C)}}o(DC,"renderCanvas");Mm.create=AC.create;Mm.toCanvas=DC.bind(null,HP.render);Mm.toDataURL=DC.bind(null,HP.renderToDataURL);Mm.toString=DC.bind(null,function(e,t,n){return FB.render(e,n)})});var nb=fe(Oe()),aO=fe(iu());var Gh=fe(Oe()),kH=fe(ak());var uk=fe(Oe()),ei=uk.default.createContext(null);function EI(e){e()}o(EI,"defaultNoopBatch");var fk=EI,ck=o(function(t){return fk=t},"setBatch"),pk=o(function(){return fk},"getBatch");var dk={notify:o(function(){},"notify")};function _I(){var e=pk(),t=null,n=null;return{clear:o(function(){t=null,n=null},"clear"),notify:o(function(){e(function(){for(var d=t;d;)d.callback(),d=d.next})},"notify"),get:o(function(){for(var d=[],h=t;h;)d.push(h),h=h.next;return d},"get"),subscribe:o(function(d){var h=!0,c=n={callback:d,next:null,prev:n};return c.prev?c.prev.next=c:t=c,o(function(){!h||t===null||(h=!1,c.next?c.next.prev=c.prev:n=c.prev,c.prev?c.prev.next=c.next:t=c.next)},"unsubscribe")},"subscribe")}}o(_I,"createListenerCollection");var Tp=function(){function e(n,l){this.store=n,this.parentSub=l,this.unsubscribe=null,this.listeners=dk,this.handleChangeWrapper=this.handleChangeWrapper.bind(this)}o(e,"Subscription");var t=e.prototype;return t.addNestedSub=o(function(l){return this.trySubscribe(),this.listeners.subscribe(l)},"addNestedSub"),t.notifyNestedSubs=o(function(){this.listeners.notify()},"notifyNestedSubs"),t.handleChangeWrapper=o(function(){this.onStateChange&&this.onStateChange()},"handleChangeWrapper"),t.isSubscribed=o(function(){return Boolean(this.unsubscribe)},"isSubscribed"),t.trySubscribe=o(function(){this.unsubscribe||(this.unsubscribe=this.parentSub?this.parentSub.addNestedSub(this.handleChangeWrapper):this.store.subscribe(this.handleChangeWrapper),this.listeners=_I())},"trySubscribe"),t.tryUnsubscribe=o(function(){this.unsubscribe&&(this.unsubscribe(),this.unsubscribe=null,this.listeners.clear(),this.listeners=dk)},"tryUnsubscribe"),e}();var Gv=fe(Oe()),Nf=typeof window!="undefined"&&typeof window.document!="undefined"&&typeof window.document.createElement!="undefined"?Gv.useLayoutEffect:Gv.useEffect;function TI(e){var t=e.store,n=e.context,l=e.children,d=(0,Gh.useMemo)(function(){var v=new Tp(t);return v.onStateChange=v.notifyNestedSubs,{store:t,subscription:v}},[t]),h=(0,Gh.useMemo)(function(){return t.getState()},[t]);Nf(function(){var v=d.subscription;return v.trySubscribe(),h!==t.getState()&&v.notifyNestedSubs(),function(){v.tryUnsubscribe(),v.onStateChange=null}},[d,h]);var c=n||ei;return Gh.default.createElement(c.Provider,{value:d},l)}o(TI,"Provider");var Hx=TI;function Po(){return Po=Object.assign||function(e){for(var t=1;t=0)&&(n[d]=e[d]);return n}o(ou,"_objectWithoutPropertiesLoose");var Gx=fe(Ek()),cr=fe(Oe()),Lk=fe(Nk());var jI=[],qI=[null,null];function VI(e,t){var n=e[1];return[t.payload,n+1]}o(VI,"storeStateUpdatesReducer");function Pk(e,t,n){Nf(function(){return e.apply(void 0,t)},n)}o(Pk,"useIsomorphicLayoutEffectWithArgs");function KI(e,t,n,l,d,h,c){e.current=l,t.current=d,n.current=!1,h.current&&(h.current=null,c())}o(KI,"captureWrapperProps");function GI(e,t,n,l,d,h,c,v,C,k){if(!!e){var O=!1,j=null,B=o(function(){if(!O){var Z=t.getState(),R,A;try{R=l(Z,d.current)}catch(I){A=I,j=I}A||(j=null),R===h.current?c.current||C():(h.current=R,v.current=R,c.current=!0,k({type:"STORE_UPDATED",payload:{error:A}}))}},"checkForUpdates");n.onStateChange=B,n.trySubscribe(),B();var X=o(function(){if(O=!0,n.tryUnsubscribe(),n.onStateChange=null,j)throw j},"unsubscribeWrapper");return X}}o(GI,"subscribeUpdates");var YI=o(function(){return[null,0]},"initStateUpdates");function m0(e,t){t===void 0&&(t={});var n=t,l=n.getDisplayName,d=l===void 0?function(ne){return"ConnectAdvanced("+ne+")"}:l,h=n.methodName,c=h===void 0?"connectAdvanced":h,v=n.renderCountProp,C=v===void 0?void 0:v,k=n.shouldHandleStateChanges,O=k===void 0?!0:k,j=n.storeKey,B=j===void 0?"store":j,X=n.withRef,J=X===void 0?!1:X,Z=n.forwardRef,R=Z===void 0?!1:Z,A=n.context,I=A===void 0?ei:A,G=ou(n,["getDisplayName","methodName","renderCountProp","shouldHandleStateChanges","storeKey","withRef","forwardRef","context"]);if(!1)var K;var se=I;return o(function(pe){var me=pe.displayName||pe.name||"Component",xe=d(me),Ve=Po({},G,{getDisplayName:d,methodName:c,renderCountProp:C,shouldHandleStateChanges:O,storeKey:B,displayName:xe,wrappedComponentName:me,WrappedComponent:pe}),tt=G.pure;function _e(Xe){return e(Xe.dispatch,Ve)}o(_e,"createChildSelector");var St=tt?cr.useMemo:function(Xe){return Xe()};function We(Xe){var nr=(0,cr.useMemo)(function(){var Cr=Xe.reactReduxForwardedRef,Ui=ou(Xe,["reactReduxForwardedRef"]);return[Xe.context,Cr,Ui]},[Xe]),ct=nr[0],Hr=nr[1],Qt=nr[2],_t=(0,cr.useMemo)(function(){return ct&&ct.Consumer&&(0,Lk.isContextConsumer)(cr.default.createElement(ct.Consumer,null))?ct:se},[ct,se]),Ct=(0,cr.useContext)(_t),ut=Boolean(Xe.store)&&Boolean(Xe.store.getState)&&Boolean(Xe.store.dispatch),Lr=Boolean(Ct)&&Boolean(Ct.store),zt=ut?Xe.store:Ct.store,$t=(0,cr.useMemo)(function(){return _e(zt)},[zt]),ie=(0,cr.useMemo)(function(){if(!O)return qI;var Cr=new Tp(zt,ut?null:Ct.subscription),Ui=Cr.notifyNestedSubs.bind(Cr);return[Cr,Ui]},[zt,ut,Ct]),rt=ie[0],Pr=ie[1],Gt=(0,cr.useMemo)(function(){return ut?Ct:Po({},Ct,{subscription:rt})},[ut,Ct,rt]),Yt=(0,cr.useReducer)(VI,jI,YI),Se=Yt[0],Or=Se[0],fn=Yt[1];if(Or&&Or.error)throw Or.error;var Un=(0,cr.useRef)(),si=(0,cr.useRef)(Qt),cn=(0,cr.useRef)(),Zt=(0,cr.useRef)(!1),gr=St(function(){return cn.current&&Qt===si.current?cn.current:$t(zt.getState(),Qt)},[zt,Or,Qt]);Pk(KI,[si,Un,Zt,Qt,gr,cn,Pr]),Pk(GI,[O,zt,rt,$t,si,Un,Zt,cn,Pr,fn],[zt,rt,$t]);var pt=(0,cr.useMemo)(function(){return cr.default.createElement(pe,Po({},gr,{ref:Hr}))},[Hr,pe,gr]),Ho=(0,cr.useMemo)(function(){return O?cr.default.createElement(_t.Provider,{value:Gt},pt):pt},[_t,pt,Gt]);return Ho}o(We,"ConnectFunction");var Ke=tt?cr.default.memo(We):We;if(Ke.WrappedComponent=pe,Ke.displayName=We.displayName=xe,R){var Ge=cr.default.forwardRef(o(function(nr,ct){return cr.default.createElement(Ke,Po({},nr,{reactReduxForwardedRef:ct}))},"forwardConnectRef"));return Ge.displayName=xe,Ge.WrappedComponent=pe,(0,Gx.default)(Ge,pe)}return(0,Gx.default)(Ke,pe)},"wrapWithConnect")}o(m0,"connectAdvanced");function Ok(e,t){return e===t?e!==0||t!==0||1/e==1/t:e!==e&&t!==t}o(Ok,"is");function kp(e,t){if(Ok(e,t))return!0;if(typeof e!="object"||e===null||typeof t!="object"||t===null)return!1;var n=Object.keys(e),l=Object.keys(t);if(n.length!==l.length)return!1;for(var d=0;d=0;l--){var d=t[l](e);if(d)return d}return function(h,c){throw new Error("Invalid value of type "+typeof e+" for "+n+" argument when connecting component "+c.wrappedComponentName+".")}}o(Qx,"match");function aF(e,t){return e===t}o(aF,"strictEqual");function uF(e){var t=e===void 0?{}:e,n=t.connectHOC,l=n===void 0?m0:n,d=t.mapStateToPropsFactories,h=d===void 0?Dk:d,c=t.mapDispatchToPropsFactories,v=c===void 0?Ak:c,C=t.mergePropsFactories,k=C===void 0?Rk:C,O=t.selectorFactory,j=O===void 0?Xx:O;return o(function(X,J,Z,R){R===void 0&&(R={});var A=R,I=A.pure,G=I===void 0?!0:I,K=A.areStatesEqual,se=K===void 0?aF:K,ne=A.areOwnPropsEqual,pe=ne===void 0?kp:ne,me=A.areStatePropsEqual,xe=me===void 0?kp:me,Ve=A.areMergedPropsEqual,tt=Ve===void 0?kp:Ve,_e=ou(A,["pure","areStatesEqual","areOwnPropsEqual","areStatePropsEqual","areMergedPropsEqual"]),St=Qx(X,h,"mapStateToProps"),We=Qx(J,v,"mapDispatchToProps"),Ke=Qx(Z,k,"mergeProps");return l(j,Po({methodName:"connect",getDisplayName:o(function(Xe){return"Connect("+Xe+")"},"getDisplayName"),shouldHandleStateChanges:Boolean(X),initMapStateToProps:St,initMapDispatchToProps:We,initMergeProps:Ke,pure:G,areStatesEqual:se,areOwnPropsEqual:pe,areStatePropsEqual:xe,areMergedPropsEqual:tt},_e))},"connect")}o(uF,"createConnect");var Hi=uF();var Fk=fe(Oe());var Ik=fe(Oe());function v0(){var e=(0,Ik.useContext)(ei);return e}o(v0,"useReduxContext");function y0(e){e===void 0&&(e=ei);var t=e===ei?v0:function(){return(0,Fk.useContext)(e)};return o(function(){var l=t(),d=l.store;return d},"useStore")}o(y0,"createStoreHook");var Zx=y0();function Bk(e){e===void 0&&(e=ei);var t=e===ei?Zx:y0(e);return o(function(){var l=t();return l.dispatch},"useDispatch")}o(Bk,"createDispatchHook");var Gs=Bk();var ro=fe(Oe());var fF=o(function(t,n){return t===n},"refEquality");function cF(e,t,n,l){var d=(0,ro.useReducer)(function(J){return J+1},0),h=d[1],c=(0,ro.useMemo)(function(){return new Tp(n,l)},[n,l]),v=(0,ro.useRef)(),C=(0,ro.useRef)(),k=(0,ro.useRef)(),O=(0,ro.useRef)(),j=n.getState(),B;try{if(e!==C.current||j!==k.current||v.current){var X=e(j);O.current===void 0||!t(X,O.current)?B=X:B=O.current}else B=O.current}catch(J){throw v.current&&(J.message+=` The error may be correlated with this previous error: `+v.current.stack+` -`),J}return Nf(function(){C.current=e,k.current=j,O.current=B,v.current=void 0}),Nf(function(){function J(){try{var Z=n.getState(),R=C.current(Z);if(t(R,O.current))return;O.current=R,k.current=Z}catch(A){v.current=A}h()}return o(J,"checkForUpdates"),c.onStateChange=J,c.trySubscribe(),J(),function(){return c.tryUnsubscribe()}},[n,c]),B}o(cF,"useSelectorWithStoreAndSubscription");function Hk(e){e===void 0&&(e=ei);var t=e===ei?v0:function(){return(0,ro.useContext)(e)};return o(function(l,d){d===void 0&&(d=fF);var h=t(),c=h.store,v=h.subscription,C=cF(l,d,c,v);return(0,ro.useDebugValue)(C),C},"useSelector")}o(Hk,"createSelectorHook");var tS=Hk();var rS=fe(iu());ck(rS.unstable_batchedUpdates);var Wn=fe(Oe());var Wk="UI_FLOWVIEW_SET_TAB",Uk="SET_CONTENT_VIEW_FOR",pF={tab:"request",contentViewFor:{}};function nS(e=pF,t){switch(t.type){case Uk:return Pt(ke({},e),{contentViewFor:Pt(ke({},e.contentViewFor),{[t.messageId]:t.contentView})});case Wk:return Pt(ke({},e),{tab:t.tab?t.tab:"request"});default:return e}}o(nS,"reducer");function Lf(e){return{type:Wk,tab:e}}o(Lf,"selectTab");function w0(e,t){return{type:Uk,messageId:e,contentView:t}}o(w0,"setContentViewFor");var zk=fe(Qh()),dF=fe(Oe());window._=zk.default;window.React=dF;var x0=o(function(e){if(e===0)return"0";for(var t=["b","kb","mb","gb","tb"],n=0;ne);n++);var l;return e%Math.pow(1024,n)==0?l=0:l=1,(e/Math.pow(1024,n)).toFixed(l)+t[n]},"formatSize"),S0=o(function(e){for(var t=e,n=["ms","s","min","h"],l=[1e3,60,60],d=0;Math.abs(t)>=l[d]&&dkt(e,ke({method:"PUT",headers:{"Content-Type":"application/json"},body:JSON.stringify(t)},n));kt.post=(e,t,n={})=>kt(e,ke({method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(t)},n));function Pf(e,...t){return Ia(this,null,function*(){return yield(yield kt(`/commands/${e}`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({arguments:t})})).json()})}o(Pf,"runCommand");var Jh={};Hb(Jh,{ADD:()=>fS,RECEIVE:()=>dS,REMOVE:()=>pS,SET_FILTER:()=>aS,SET_SORT:()=>uS,UPDATE:()=>cS,add:()=>gF,defaultState:()=>C0,receive:()=>wF,reduce:()=>Lp,remove:()=>yF,setFilter:()=>hS,setSort:()=>jk,update:()=>vF});var lS=fe($k()),aS="LIST_SET_FILTER",uS="LIST_SET_SORT",fS="LIST_ADD",cS="LIST_UPDATE",pS="LIST_REMOVE",dS="LIST_RECEIVE",C0={byId:{},list:[],listIndex:{},view:[],viewIndex:{}};function Lp(e=C0,t){let{byId:n,list:l,listIndex:d,view:h,viewIndex:c}=e;switch(t.type){case aS:h=(0,lS.default)(l.filter(t.filter),t.sort),c={},h.forEach((k,O)=>{c[k.id]=O});break;case uS:h=(0,lS.default)([...h],t.sort),c={},h.forEach((k,O)=>{c[k.id]=O});break;case fS:if(t.item.id in n)break;n=Pt(ke({},n),{[t.item.id]:t.item}),d=Pt(ke({},d),{[t.item.id]:l.length}),l=[...l,t.item],t.filter(t.item)&&({view:h,viewIndex:c}=qk(e,t.item,t.sort));break;case cS:n=Pt(ke({},n),{[t.item.id]:t.item}),l=[...l],l[d[t.item.id]]=t.item;let v=t.item.id in c,C=t.filter(t.item);C&&!v?{view:h,viewIndex:c}=qk(e,t.item,t.sort):!C&&v?{data:h,dataIndex:c}=mS(h,c,t.item.id):C&&v&&({view:h,viewIndex:c}=xF(e,t.item,t.sort));break;case pS:if(!(t.id in n))break;n=ke({},n),delete n[t.id],{data:l,dataIndex:d}=mS(l,d,t.id),t.id in c&&({data:h,dataIndex:c}=mS(h,c,t.id));break;case dS:l=t.list,d={},n={},l.forEach((k,O)=>{n[k.id]=k,d[k.id]=O}),h=l.filter(t.filter).sort(t.sort),c={},h.forEach((k,O)=>{c[k.id]=O});break}return{byId:n,list:l,listIndex:d,view:h,viewIndex:c}}o(Lp,"reduce");function hS(e=b0,t=Zh){return{type:aS,filter:e,sort:t}}o(hS,"setFilter");function jk(e=Zh){return{type:uS,sort:e}}o(jk,"setSort");function gF(e,t=b0,n=Zh){return{type:fS,item:e,filter:t,sort:n}}o(gF,"add");function vF(e,t=b0,n=Zh){return{type:cS,item:e,filter:t,sort:n}}o(vF,"update");function yF(e){return{type:pS,id:e}}o(yF,"remove");function wF(e,t=b0,n=Zh){return{type:dS,list:e,filter:t,sort:n}}o(wF,"receive");function qk(e,t,n){let l=SF(e.view,t,n),d=[...e.view],h=ke({},e.viewIndex);d.splice(l,0,t);for(let c=d.length-1;c>=l;c--)h[d[c].id]=c;return{view:d,viewIndex:h}}o(qk,"sortedInsert");function mS(e,t,n){let l=t[n],d=[...e],h=ke({},t);delete h[n],d.splice(l,1);for(let c=d.length-1;c>=l;c--)h[d[c].id]=c;return{data:d,dataIndex:h}}o(mS,"removeData");function xF(e,t,n){let l=[...e.view],d=ke({},e.viewIndex),h=d[t.id];for(l[h]=t;h+10;)l[h]=l[h+1],l[h+1]=t,d[t.id]=h+1,d[l[h].id]=h,++h;for(;h>0&&n(l[h],l[h-1])<0;)l[h]=l[h-1],l[h-1]=t,d[t.id]=h-1,d[l[h].id]=h,--h;return{view:l,viewIndex:d}}o(xF,"sortedUpdate");function SF(e,t,n){let l=0,d=e.length;for(;l>>1;n(t,e[h])>=0?l=h+1:d=h}return l}o(SF,"sortedIndex");function b0(){return!0}o(b0,"defaultFilter");function Zh(e,t){return 0}o(Zh,"defaultSort");var Vk={http:80,https:443},Kr=class{static getContentType(t){var n=Kr.get_first_header(t,/^Content-Type$/i);if(n)return n.split(";")[0].trim()}static get_first_header(t,n){let l=t;l._headerLookups||Object.defineProperty(l,"_headerLookups",{value:{},configurable:!1,enumerable:!1,writable:!1});let d=n.toString();if(!(d in l._headerLookups)){let h;for(let c=0;c{var t,n;switch(e.type){case"http":let l=e.request.contentLength||0;return e.response&&(l+=e.response.contentLength||0),e.websocket&&(l+=e.websocket.messages_meta.contentLength||0),l;case"tcp":case"udp":return e.messages_meta.contentLength||0;case"dns":return(n=(t=e.response)==null?void 0:t.size)!=null?n:0}},"getTotalSize"),E0=o(e=>e.type==="http"&&!e.websocket,"canReplay");var Of=function(){"use strict";function e(l,d){function h(){this.constructor=l}o(h,"ctor"),h.prototype=d.prototype,l.prototype=new h}o(e,"peg$subclass");function t(l,d,h,c){this.message=l,this.expected=d,this.found=h,this.location=c,this.name="SyntaxError",typeof Error.captureStackTrace=="function"&&Error.captureStackTrace(this,t)}o(t,"peg$SyntaxError"),e(t,Error);function n(l){var d=arguments.length>1?arguments[1]:{},h=this,c={},v={start:Ou},C=Ou,k={type:"other",description:"filter expression"},O=o(function(w){return w},"peg$c1"),j={type:"other",description:"whitespace"},B=/^[ \t\n\r]/,X={type:"class",value:"[ \\t\\n\\r]",description:"[ \\t\\n\\r]"},J={type:"other",description:"control character"},Z=/^[|&!()~"]/,R={type:"class",value:'[|&!()~"]',description:'[|&!()~"]'},A={type:"other",description:"optional whitespace"},I="|",G={type:"literal",value:"|",description:'"|"'},K=o(function(w,T){return Au(w,T)},"peg$c11"),se="&",ne={type:"literal",value:"&",description:'"&"'},pe=o(function(w,T){return hd(w,T)},"peg$c14"),me="!",xe={type:"literal",value:"!",description:'"!"'},Ve=o(function(w){return Vo(w)},"peg$c17"),tt="(",_e={type:"literal",value:"(",description:'"("'},St=")",We={type:"literal",value:")",description:'")"'},Ke=o(function(w){return yr(w)},"peg$c22"),Ge="~all",Xe={type:"literal",value:"~all",description:'"~all"'},nr=o(function(){return fc},"peg$c25"),ct="~a",Hr={type:"literal",value:"~a",description:'"~a"'},Qt=o(function(){return Ko},"peg$c28"),_t="~b",Ct={type:"literal",value:"~b",description:'"~b"'},ut=o(function(w){return na(w)},"peg$c31"),Lr="~bq",zt={type:"literal",value:"~bq",description:'"~bq"'},$t=o(function(w){return Go(w)},"peg$c34"),ie="~bs",rt={type:"literal",value:"~bs",description:'"~bs"'},Pr=o(function(w){return md(w)},"peg$c37"),Gt="~c",Yt={type:"literal",value:"~c",description:'"~c"'},Se=o(function(w){return ia(w)},"peg$c40"),Or="~comment",fn={type:"literal",value:"~comment",description:'"~comment"'},Un=o(function(w){return Ru(w)},"peg$c43"),si="~d",cn={type:"literal",value:"~d",description:'"~d"'},Zt=o(function(w){return Iu(w)},"peg$c46"),gr="~dns",pt={type:"literal",value:"~dns",description:'"~dns"'},Ho=o(function(){return oa},"peg$c49"),Cr="~dst",Ui={type:"literal",value:"~dst",description:'"~dst"'},pn=o(function(w){return Fu(w)},"peg$c52"),zn="~e",Si={type:"literal",value:"~e",description:'"~e"'},Ci=o(function(){return Bu},"peg$c55"),$n="~h",Mn={type:"literal",value:"~h",description:'"~h"'},Js=o(function(w){return Hu(w)},"peg$c58"),H="~hq",ee={type:"literal",value:"~hq",description:'"~hq"'},he=o(function(w){return Yo(w)},"peg$c61"),Te="~hs",ir={type:"literal",value:"~hs",description:'"~hs"'},Ul=o(function(w){return ji(w)},"peg$c64"),Ft="~http",Wr={type:"literal",value:"~http",description:'"~http"'},or=o(function(){return Ss},"peg$c67"),li="~marked",ds={type:"literal",value:"~marked",description:'"~marked"'},lo=o(function(){return zr},"peg$c70"),bi="~marker",el={type:"literal",value:"~marker",description:'"~marker"'},hs=o(function(w){return sa(w)},"peg$c73"),dn="~m",id={type:"literal",value:"~m",description:'"~m"'},tl=o(function(w){return mn(w)},"peg$c76"),Qf="~q",rl={type:"literal",value:"~q",description:'"~q"'},od=o(function(){return qi},"peg$c79"),Zf="~replayq",wu={type:"literal",value:"~replayq",description:'"~replayq"'},sd=o(function(){return al},"peg$c82"),zl="~replays",ms={type:"literal",value:"~replays",description:'"~replays"'},ld=o(function(){return cc},"peg$c85"),Jf="~replay",nl={type:"literal",value:"~replay",description:'"~replay"'},xu=o(function(){return Wu},"peg$c88"),Wo="~src",ad={type:"literal",value:"~src",description:'"~src"'},Uo=o(function(w){return gd(w)},"peg$c91"),$l="~s",ec={type:"literal",value:"~s",description:'"~s"'},jt=o(function(){return Uu},"peg$c94"),Me="~tcp",Ei={type:"literal",value:"~tcp",description:'"~tcp"'},Su=o(function(){return la},"peg$c97"),ai="~udp",vt={type:"literal",value:"~udp",description:'"~udp"'},ao=o(function(){return qn},"peg$c100"),zo="~tq",jl={type:"literal",value:"~tq",description:'"~tq"'},ue=o(function(w){return Ti(w)},"peg$c103"),ze="~ts",Cu={type:"literal",value:"~ts",description:'"~ts"'},bu=o(function(w){return pc(w)},"peg$c106"),gs="~t",il={type:"literal",value:"~t",description:'"~t"'},Eu=o(function(w){return aa(w)},"peg$c109"),He="~u",ud={type:"literal",value:"~u",description:'"~u"'},ql=o(function(w){return dc(w)},"peg$c112"),uo="~websocket",ui={type:"literal",value:"~websocket",description:'"~websocket"'},tc=o(function(){return Vi},"peg$c115"),_u={type:"other",description:"integer"},$o=/^['"]/,vs={type:"class",value:`['"]`,description:`['"]`},Tu=/^[0-9]/,ol={type:"class",value:"[0-9]",description:"[0-9]"},Vl=o(function(w){return parseInt(w.join(""),10)},"peg$c121"),Kl={type:"other",description:"string"},fo='"',Gl={type:"literal",value:'"',description:'"\\""'},Yl=o(function(w){return w.join("")},"peg$c125"),rc="'",Xl={type:"literal",value:"'",description:`"'"`},_i=/^["\\]/,nc={type:"class",value:'["\\\\]',description:'["\\\\]'},Ql={type:"any",description:"any character"},co=o(function(w){return w},"peg$c131"),ys="\\",ic={type:"literal",value:"\\",description:'"\\\\"'},oc=/^['\\]/,fd={type:"class",value:"['\\\\]",description:"['\\\\]"},cd=/^['"\\]/,ku={type:"class",value:`['"\\\\]`,description:`['"\\\\]`},sc="n",Nu={type:"literal",value:"n",description:'"n"'},lc=o(function(){return` +`),J}return Nf(function(){C.current=e,k.current=j,O.current=B,v.current=void 0}),Nf(function(){function J(){try{var Z=n.getState(),R=C.current(Z);if(t(R,O.current))return;O.current=R,k.current=Z}catch(A){v.current=A}h()}return o(J,"checkForUpdates"),c.onStateChange=J,c.trySubscribe(),J(),function(){return c.tryUnsubscribe()}},[n,c]),B}o(cF,"useSelectorWithStoreAndSubscription");function Hk(e){e===void 0&&(e=ei);var t=e===ei?v0:function(){return(0,ro.useContext)(e)};return o(function(l,d){d===void 0&&(d=fF);var h=t(),c=h.store,v=h.subscription,C=cF(l,d,c,v);return(0,ro.useDebugValue)(C),C},"useSelector")}o(Hk,"createSelectorHook");var Jx=Hk();var eS=fe(iu());ck(eS.unstable_batchedUpdates);var Wn=fe(Oe());var Wk="UI_FLOWVIEW_SET_TAB",Uk="SET_CONTENT_VIEW_FOR",pF={tab:"request",contentViewFor:{}};function tS(e=pF,t){switch(t.type){case Uk:return Pt(ke({},e),{contentViewFor:Pt(ke({},e.contentViewFor),{[t.messageId]:t.contentView})});case Wk:return Pt(ke({},e),{tab:t.tab?t.tab:"request"});default:return e}}o(tS,"reducer");function Lf(e){return{type:Wk,tab:e}}o(Lf,"selectTab");function w0(e,t){return{type:Uk,messageId:e,contentView:t}}o(w0,"setContentViewFor");var zk=fe(Qh()),dF=fe(Oe());window._=zk.default;window.React=dF;var x0=o(function(e){if(e===0)return"0";for(var t=["b","kb","mb","gb","tb"],n=0;ne);n++);var l;return e%Math.pow(1024,n)==0?l=0:l=1,(e/Math.pow(1024,n)).toFixed(l)+t[n]},"formatSize"),S0=o(function(e){for(var t=e,n=["ms","s","min","h"],l=[1e3,60,60],d=0;Math.abs(t)>=l[d]&&dkt(e,ke({method:"PUT",headers:{"Content-Type":"application/json"},body:JSON.stringify(t)},n));kt.post=(e,t,n={})=>kt(e,ke({method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(t)},n));function Pf(e,...t){return Ia(this,null,function*(){return yield(yield kt(`/commands/${e}`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({arguments:t})})).json()})}o(Pf,"runCommand");var Jh={};Hb(Jh,{ADD:()=>aS,RECEIVE:()=>cS,REMOVE:()=>fS,SET_FILTER:()=>sS,SET_SORT:()=>lS,UPDATE:()=>uS,add:()=>gF,defaultState:()=>C0,receive:()=>wF,reduce:()=>Lp,remove:()=>yF,setFilter:()=>pS,setSort:()=>jk,update:()=>vF});var oS=fe($k()),sS="LIST_SET_FILTER",lS="LIST_SET_SORT",aS="LIST_ADD",uS="LIST_UPDATE",fS="LIST_REMOVE",cS="LIST_RECEIVE",C0={byId:{},list:[],listIndex:{},view:[],viewIndex:{}};function Lp(e=C0,t){let{byId:n,list:l,listIndex:d,view:h,viewIndex:c}=e;switch(t.type){case sS:h=(0,oS.default)(l.filter(t.filter),t.sort),c={},h.forEach((k,O)=>{c[k.id]=O});break;case lS:h=(0,oS.default)([...h],t.sort),c={},h.forEach((k,O)=>{c[k.id]=O});break;case aS:if(t.item.id in n)break;n=Pt(ke({},n),{[t.item.id]:t.item}),d=Pt(ke({},d),{[t.item.id]:l.length}),l=[...l,t.item],t.filter(t.item)&&({view:h,viewIndex:c}=qk(e,t.item,t.sort));break;case uS:n=Pt(ke({},n),{[t.item.id]:t.item}),l=[...l],l[d[t.item.id]]=t.item;let v=t.item.id in c,C=t.filter(t.item);C&&!v?{view:h,viewIndex:c}=qk(e,t.item,t.sort):!C&&v?{data:h,dataIndex:c}=dS(h,c,t.item.id):C&&v&&({view:h,viewIndex:c}=xF(e,t.item,t.sort));break;case fS:if(!(t.id in n))break;n=ke({},n),delete n[t.id],{data:l,dataIndex:d}=dS(l,d,t.id),t.id in c&&({data:h,dataIndex:c}=dS(h,c,t.id));break;case cS:l=t.list,d={},n={},l.forEach((k,O)=>{n[k.id]=k,d[k.id]=O}),h=l.filter(t.filter).sort(t.sort),c={},h.forEach((k,O)=>{c[k.id]=O});break}return{byId:n,list:l,listIndex:d,view:h,viewIndex:c}}o(Lp,"reduce");function pS(e=b0,t=Zh){return{type:sS,filter:e,sort:t}}o(pS,"setFilter");function jk(e=Zh){return{type:lS,sort:e}}o(jk,"setSort");function gF(e,t=b0,n=Zh){return{type:aS,item:e,filter:t,sort:n}}o(gF,"add");function vF(e,t=b0,n=Zh){return{type:uS,item:e,filter:t,sort:n}}o(vF,"update");function yF(e){return{type:fS,id:e}}o(yF,"remove");function wF(e,t=b0,n=Zh){return{type:cS,list:e,filter:t,sort:n}}o(wF,"receive");function qk(e,t,n){let l=SF(e.view,t,n),d=[...e.view],h=ke({},e.viewIndex);d.splice(l,0,t);for(let c=d.length-1;c>=l;c--)h[d[c].id]=c;return{view:d,viewIndex:h}}o(qk,"sortedInsert");function dS(e,t,n){let l=t[n],d=[...e],h=ke({},t);delete h[n],d.splice(l,1);for(let c=d.length-1;c>=l;c--)h[d[c].id]=c;return{data:d,dataIndex:h}}o(dS,"removeData");function xF(e,t,n){let l=[...e.view],d=ke({},e.viewIndex),h=d[t.id];for(l[h]=t;h+10;)l[h]=l[h+1],l[h+1]=t,d[t.id]=h+1,d[l[h].id]=h,++h;for(;h>0&&n(l[h],l[h-1])<0;)l[h]=l[h-1],l[h-1]=t,d[t.id]=h-1,d[l[h].id]=h,--h;return{view:l,viewIndex:d}}o(xF,"sortedUpdate");function SF(e,t,n){let l=0,d=e.length;for(;l>>1;n(t,e[h])>=0?l=h+1:d=h}return l}o(SF,"sortedIndex");function b0(){return!0}o(b0,"defaultFilter");function Zh(e,t){return 0}o(Zh,"defaultSort");var Vk={http:80,https:443},Kr=class{static getContentType(t){var n=Kr.get_first_header(t,/^Content-Type$/i);if(n)return n.split(";")[0].trim()}static get_first_header(t,n){let l=t;l._headerLookups||Object.defineProperty(l,"_headerLookups",{value:{},configurable:!1,enumerable:!1,writable:!1});let d=n.toString();if(!(d in l._headerLookups)){let h;for(let c=0;c{var t,n;switch(e.type){case"http":let l=e.request.contentLength||0;return e.response&&(l+=e.response.contentLength||0),e.websocket&&(l+=e.websocket.messages_meta.contentLength||0),l;case"tcp":case"udp":return e.messages_meta.contentLength||0;case"dns":return(n=(t=e.response)==null?void 0:t.size)!=null?n:0}},"getTotalSize"),E0=o(e=>e.type==="http"&&!e.websocket,"canReplay");var Of=function(){"use strict";function e(l,d){function h(){this.constructor=l}o(h,"ctor"),h.prototype=d.prototype,l.prototype=new h}o(e,"peg$subclass");function t(l,d,h,c){this.message=l,this.expected=d,this.found=h,this.location=c,this.name="SyntaxError",typeof Error.captureStackTrace=="function"&&Error.captureStackTrace(this,t)}o(t,"peg$SyntaxError"),e(t,Error);function n(l){var d=arguments.length>1?arguments[1]:{},h=this,c={},v={start:Ou},C=Ou,k={type:"other",description:"filter expression"},O=o(function(w){return w},"peg$c1"),j={type:"other",description:"whitespace"},B=/^[ \t\n\r]/,X={type:"class",value:"[ \\t\\n\\r]",description:"[ \\t\\n\\r]"},J={type:"other",description:"control character"},Z=/^[|&!()~"]/,R={type:"class",value:'[|&!()~"]',description:'[|&!()~"]'},A={type:"other",description:"optional whitespace"},I="|",G={type:"literal",value:"|",description:'"|"'},K=o(function(w,T){return Au(w,T)},"peg$c11"),se="&",ne={type:"literal",value:"&",description:'"&"'},pe=o(function(w,T){return hd(w,T)},"peg$c14"),me="!",xe={type:"literal",value:"!",description:'"!"'},Ve=o(function(w){return Vo(w)},"peg$c17"),tt="(",_e={type:"literal",value:"(",description:'"("'},St=")",We={type:"literal",value:")",description:'")"'},Ke=o(function(w){return yr(w)},"peg$c22"),Ge="~all",Xe={type:"literal",value:"~all",description:'"~all"'},nr=o(function(){return fc},"peg$c25"),ct="~a",Hr={type:"literal",value:"~a",description:'"~a"'},Qt=o(function(){return Ko},"peg$c28"),_t="~b",Ct={type:"literal",value:"~b",description:'"~b"'},ut=o(function(w){return na(w)},"peg$c31"),Lr="~bq",zt={type:"literal",value:"~bq",description:'"~bq"'},$t=o(function(w){return Go(w)},"peg$c34"),ie="~bs",rt={type:"literal",value:"~bs",description:'"~bs"'},Pr=o(function(w){return md(w)},"peg$c37"),Gt="~c",Yt={type:"literal",value:"~c",description:'"~c"'},Se=o(function(w){return ia(w)},"peg$c40"),Or="~comment",fn={type:"literal",value:"~comment",description:'"~comment"'},Un=o(function(w){return Ru(w)},"peg$c43"),si="~d",cn={type:"literal",value:"~d",description:'"~d"'},Zt=o(function(w){return Iu(w)},"peg$c46"),gr="~dns",pt={type:"literal",value:"~dns",description:'"~dns"'},Ho=o(function(){return oa},"peg$c49"),Cr="~dst",Ui={type:"literal",value:"~dst",description:'"~dst"'},pn=o(function(w){return Fu(w)},"peg$c52"),zn="~e",Si={type:"literal",value:"~e",description:'"~e"'},Ci=o(function(){return Bu},"peg$c55"),$n="~h",Mn={type:"literal",value:"~h",description:'"~h"'},Js=o(function(w){return Hu(w)},"peg$c58"),H="~hq",ee={type:"literal",value:"~hq",description:'"~hq"'},he=o(function(w){return Yo(w)},"peg$c61"),Te="~hs",ir={type:"literal",value:"~hs",description:'"~hs"'},Ul=o(function(w){return ji(w)},"peg$c64"),Ft="~http",Wr={type:"literal",value:"~http",description:'"~http"'},or=o(function(){return Ss},"peg$c67"),li="~marked",ds={type:"literal",value:"~marked",description:'"~marked"'},lo=o(function(){return zr},"peg$c70"),bi="~marker",el={type:"literal",value:"~marker",description:'"~marker"'},hs=o(function(w){return sa(w)},"peg$c73"),dn="~m",id={type:"literal",value:"~m",description:'"~m"'},tl=o(function(w){return mn(w)},"peg$c76"),Qf="~q",rl={type:"literal",value:"~q",description:'"~q"'},od=o(function(){return qi},"peg$c79"),Zf="~replayq",wu={type:"literal",value:"~replayq",description:'"~replayq"'},sd=o(function(){return al},"peg$c82"),zl="~replays",ms={type:"literal",value:"~replays",description:'"~replays"'},ld=o(function(){return cc},"peg$c85"),Jf="~replay",nl={type:"literal",value:"~replay",description:'"~replay"'},xu=o(function(){return Wu},"peg$c88"),Wo="~src",ad={type:"literal",value:"~src",description:'"~src"'},Uo=o(function(w){return gd(w)},"peg$c91"),$l="~s",ec={type:"literal",value:"~s",description:'"~s"'},jt=o(function(){return Uu},"peg$c94"),Me="~tcp",Ei={type:"literal",value:"~tcp",description:'"~tcp"'},Su=o(function(){return la},"peg$c97"),ai="~udp",vt={type:"literal",value:"~udp",description:'"~udp"'},ao=o(function(){return qn},"peg$c100"),zo="~tq",jl={type:"literal",value:"~tq",description:'"~tq"'},ue=o(function(w){return Ti(w)},"peg$c103"),ze="~ts",Cu={type:"literal",value:"~ts",description:'"~ts"'},bu=o(function(w){return pc(w)},"peg$c106"),gs="~t",il={type:"literal",value:"~t",description:'"~t"'},Eu=o(function(w){return aa(w)},"peg$c109"),He="~u",ud={type:"literal",value:"~u",description:'"~u"'},ql=o(function(w){return dc(w)},"peg$c112"),uo="~websocket",ui={type:"literal",value:"~websocket",description:'"~websocket"'},tc=o(function(){return Vi},"peg$c115"),_u={type:"other",description:"integer"},$o=/^['"]/,vs={type:"class",value:`['"]`,description:`['"]`},Tu=/^[0-9]/,ol={type:"class",value:"[0-9]",description:"[0-9]"},Vl=o(function(w){return parseInt(w.join(""),10)},"peg$c121"),Kl={type:"other",description:"string"},fo='"',Gl={type:"literal",value:'"',description:'"\\""'},Yl=o(function(w){return w.join("")},"peg$c125"),rc="'",Xl={type:"literal",value:"'",description:`"'"`},_i=/^["\\]/,nc={type:"class",value:'["\\\\]',description:'["\\\\]'},Ql={type:"any",description:"any character"},co=o(function(w){return w},"peg$c131"),ys="\\",ic={type:"literal",value:"\\",description:'"\\\\"'},oc=/^['\\]/,fd={type:"class",value:"['\\\\]",description:"['\\\\]"},cd=/^['"\\]/,ku={type:"class",value:`['"\\\\]`,description:`['"\\\\]`},sc="n",Nu={type:"literal",value:"n",description:'"n"'},lc=o(function(){return` `},"peg$c140"),ac="r",Zl={type:"literal",value:"r",description:'"r"'},Jl=o(function(){return"\r"},"peg$c143"),Lu="t",Ot={type:"literal",value:"t",description:'"t"'},Nt=o(function(){return" "},"peg$c146"),P=0,Re=0,sl=[{line:1,column:1,seenCR:!1}],vr=0,Pu=[],ye=0,jo;if("startRule"in d){if(!(d.startRule in v))throw new Error(`Can't start parsing from rule "`+d.startRule+'".');C=v[d.startRule]}function pd(){return l.substring(Re,P)}o(pd,"text");function qt(){return zi(Re,P)}o(qt,"location");function ea(w){throw ta(null,[{type:"other",description:w}],l.substring(Re,P),zi(Re,P))}o(ea,"expected");function hn(w){throw ta(w,null,l.substring(Re,P),zi(Re,P))}o(hn,"error");function ws(w){var T=sl[w],W,U;if(T)return T;for(W=w-1;!sl[W];)W--;for(T=sl[W],T={line:T.line,column:T.column,seenCR:T.seenCR};Wvr&&(vr=P,Pu=[]),Pu.push(w))}o(Ce,"peg$fail");function ta(w,T,W,U){function wr(gn){var ci=1;for(gn.sort(function(ul,ki){return ul.descriptionki.description?1:0});ci1?ki.slice(0,-1).join(", ")+" or "+ki[gn.length-1]:ki[0],zu=ci?'"'+ul(ci)+'"':"end of input","Expected "+Vn+" but "+zu+" found."}return o(Kt,"buildMessage"),T!==null&&wr(T),new t(w!==null?w:Kt(T,W),T,W,U)}o(ta,"peg$buildException");function Ou(){var w,T,W,U;return ye++,w=P,T=fi(),T!==c?(W=ll(),W!==c?(U=fi(),U!==c?(Re=w,T=O(W),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),ye--,w===c&&(T=c,ye===0&&Ce(k)),w}o(Ou,"peg$parsestart");function nt(){var w,T;return ye++,B.test(l.charAt(P))?(w=l.charAt(P),P++):(w=c,ye===0&&Ce(X)),ye--,w===c&&(T=c,ye===0&&Ce(j)),w}o(nt,"peg$parsews");function uc(){var w,T;return ye++,Z.test(l.charAt(P))?(w=l.charAt(P),P++):(w=c,ye===0&&Ce(R)),ye--,w===c&&(T=c,ye===0&&Ce(J)),w}o(uc,"peg$parsecc");function fi(){var w,T;for(ye++,w=[],T=nt();T!==c;)w.push(T),T=nt();return ye--,w===c&&(T=c,ye===0&&Ce(A)),w}o(fi,"peg$parse__");function ll(){var w,T,W,U,wr,Kt;return w=P,T=Ur(),T!==c?(W=fi(),W!==c?(l.charCodeAt(P)===124?(U=I,P++):(U=c,ye===0&&Ce(G)),U!==c?(wr=fi(),wr!==c?(Kt=ll(),Kt!==c?(Re=w,T=K(T,Kt),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c&&(w=Ur()),w}o(ll,"peg$parseOrExpr");function Ur(){var w,T,W,U,wr,Kt;if(w=P,T=ra(),T!==c?(W=fi(),W!==c?(l.charCodeAt(P)===38?(U=se,P++):(U=c,ye===0&&Ce(ne)),U!==c?(wr=fi(),wr!==c?(Kt=Ur(),Kt!==c?(Re=w,T=pe(T,Kt),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c){if(w=P,T=ra(),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Ur(),U!==c?(Re=w,T=pe(T,U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;w===c&&(w=ra())}return w}o(Ur,"peg$parseAndExpr");function ra(){var w,T,W,U;return w=P,l.charCodeAt(P)===33?(T=me,P++):(T=c,ye===0&&Ce(xe)),T!==c?(W=fi(),W!==c?(U=ra(),U!==c?(Re=w,T=Ve(U),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c&&(w=jn()),w}o(ra,"peg$parseNotExpr");function jn(){var w,T,W,U,wr,Kt;return w=P,l.charCodeAt(P)===40?(T=tt,P++):(T=c,ye===0&&Ce(_e)),T!==c?(W=fi(),W!==c?(U=ll(),U!==c?(wr=fi(),wr!==c?(l.charCodeAt(P)===41?(Kt=St,P++):(Kt=c,ye===0&&Ce(We)),Kt!==c?(Re=w,T=Ke(U),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c&&(w=dd()),w}o(jn,"peg$parseBindingExpr");function dd(){var w,T,W,U;if(w=P,l.substr(P,4)===Ge?(T=Ge,P+=4):(T=c,ye===0&&Ce(Xe)),T!==c&&(Re=w,T=nr()),w=T,w===c&&(w=P,l.substr(P,2)===ct?(T=ct,P+=2):(T=c,ye===0&&Ce(Hr)),T!==c&&(Re=w,T=Qt()),w=T,w===c)){if(w=P,l.substr(P,2)===_t?(T=_t,P+=2):(T=c,ye===0&&Ce(Ct)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=ut(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===Lr?(T=Lr,P+=3):(T=c,ye===0&&Ce(zt)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=$t(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===ie?(T=ie,P+=3):(T=c,ye===0&&Ce(rt)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Pr(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===Gt?(T=Gt,P+=2):(T=c,ye===0&&Ce(Yt)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Mu(),U!==c?(Re=w,T=Se(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,8)===Or?(T=Or,P+=8):(T=c,ye===0&&Ce(fn)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Un(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===si?(T=si,P+=2):(T=c,ye===0&&Ce(cn)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Zt(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,4)===gr?(T=gr,P+=4):(T=c,ye===0&&Ce(pt)),T!==c&&(Re=w,T=Ho()),w=T,w===c)){if(w=P,l.substr(P,4)===Cr?(T=Cr,P+=4):(T=c,ye===0&&Ce(Ui)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=pn(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,2)===zn?(T=zn,P+=2):(T=c,ye===0&&Ce(Si)),T!==c&&(Re=w,T=Ci()),w=T,w===c)){if(w=P,l.substr(P,2)===$n?(T=$n,P+=2):(T=c,ye===0&&Ce(Mn)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Js(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===H?(T=H,P+=3):(T=c,ye===0&&Ce(ee)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=he(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===Te?(T=Te,P+=3):(T=c,ye===0&&Ce(ir)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Ul(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,5)===Ft?(T=Ft,P+=5):(T=c,ye===0&&Ce(Wr)),T!==c&&(Re=w,T=or()),w=T,w===c&&(w=P,l.substr(P,7)===li?(T=li,P+=7):(T=c,ye===0&&Ce(ds)),T!==c&&(Re=w,T=lo()),w=T,w===c))){if(w=P,l.substr(P,7)===bi?(T=bi,P+=7):(T=c,ye===0&&Ce(el)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=hs(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===dn?(T=dn,P+=2):(T=c,ye===0&&Ce(id)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=tl(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,2)===Qf?(T=Qf,P+=2):(T=c,ye===0&&Ce(rl)),T!==c&&(Re=w,T=od()),w=T,w===c&&(w=P,l.substr(P,8)===Zf?(T=Zf,P+=8):(T=c,ye===0&&Ce(wu)),T!==c&&(Re=w,T=sd()),w=T,w===c&&(w=P,l.substr(P,8)===zl?(T=zl,P+=8):(T=c,ye===0&&Ce(ms)),T!==c&&(Re=w,T=ld()),w=T,w===c&&(w=P,l.substr(P,7)===Jf?(T=Jf,P+=7):(T=c,ye===0&&Ce(nl)),T!==c&&(Re=w,T=xu()),w=T,w===c))))){if(w=P,l.substr(P,4)===Wo?(T=Wo,P+=4):(T=c,ye===0&&Ce(ad)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Uo(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,2)===$l?(T=$l,P+=2):(T=c,ye===0&&Ce(ec)),T!==c&&(Re=w,T=jt()),w=T,w===c&&(w=P,l.substr(P,4)===Me?(T=Me,P+=4):(T=c,ye===0&&Ce(Ei)),T!==c&&(Re=w,T=Su()),w=T,w===c&&(w=P,l.substr(P,4)===ai?(T=ai,P+=4):(T=c,ye===0&&Ce(vt)),T!==c&&(Re=w,T=ao()),w=T,w===c)))){if(w=P,l.substr(P,3)===zo?(T=zo,P+=3):(T=c,ye===0&&Ce(jl)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=ue(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===ze?(T=ze,P+=3):(T=c,ye===0&&Ce(Cu)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=bu(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===gs?(T=gs,P+=2):(T=c,ye===0&&Ce(il)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Eu(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===He?(T=He,P+=2):(T=c,ye===0&&Ce(ud)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=ql(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;w===c&&(w=P,l.substr(P,10)===uo?(T=uo,P+=10):(T=c,ye===0&&Ce(ui)),T!==c&&(Re=w,T=tc()),w=T,w===c&&(w=P,T=Vt(),T!==c&&(Re=w,T=ql(T)),w=T))}}}}}}}}}}}}}}}}}return w}o(dd,"peg$parseExpr");function Mu(){var w,T,W,U;if(ye++,w=P,$o.test(l.charAt(P))?(T=l.charAt(P),P++):(T=c,ye===0&&Ce(vs)),T===c&&(T=null),T!==c){if(W=[],Tu.test(l.charAt(P))?(U=l.charAt(P),P++):(U=c,ye===0&&Ce(ol)),U!==c)for(;U!==c;)W.push(U),Tu.test(l.charAt(P))?(U=l.charAt(P),P++):(U=c,ye===0&&Ce(ol));else W=c;W!==c?($o.test(l.charAt(P))?(U=l.charAt(P),P++):(U=c,ye===0&&Ce(vs)),U===c&&(U=null),U!==c?(Re=w,T=Vl(W),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;return ye--,w===c&&(T=c,ye===0&&Ce(_u)),w}o(Mu,"peg$parseIntegerLiteral");function Vt(){var w,T,W,U;if(ye++,w=P,l.charCodeAt(P)===34?(T=fo,P++):(T=c,ye===0&&Ce(Gl)),T!==c){for(W=[],U=xs();U!==c;)W.push(U),U=xs();W!==c?(l.charCodeAt(P)===34?(U=fo,P++):(U=c,ye===0&&Ce(Gl)),U!==c?(Re=w,T=Yl(W),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.charCodeAt(P)===39?(T=rc,P++):(T=c,ye===0&&Ce(Xl)),T!==c){for(W=[],U=qo();U!==c;)W.push(U),U=qo();W!==c?(l.charCodeAt(P)===39?(U=rc,P++):(U=c,ye===0&&Ce(Xl)),U!==c?(Re=w,T=Yl(W),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c)if(w=P,T=P,ye++,W=uc(),ye--,W===c?T=void 0:(P=T,T=c),T!==c){if(W=[],U=yt(),U!==c)for(;U!==c;)W.push(U),U=yt();else W=c;W!==c?(Re=w,T=Yl(W),w=T):(P=w,w=c)}else P=w,w=c}return ye--,w===c&&(T=c,ye===0&&Ce(Kl)),w}o(Vt,"peg$parseStringLiteral");function xs(){var w,T,W;return w=P,T=P,ye++,_i.test(l.charAt(P))?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(nc)),ye--,W===c?T=void 0:(P=T,T=c),T!==c?(l.length>P?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(Ql)),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c),w===c&&(w=P,l.charCodeAt(P)===92?(T=ys,P++):(T=c,ye===0&&Ce(ic)),T!==c?(W=$i(),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c)),w}o(xs,"peg$parseDoubleStringChar");function qo(){var w,T,W;return w=P,T=P,ye++,oc.test(l.charAt(P))?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(fd)),ye--,W===c?T=void 0:(P=T,T=c),T!==c?(l.length>P?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(Ql)),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c),w===c&&(w=P,l.charCodeAt(P)===92?(T=ys,P++):(T=c,ye===0&&Ce(ic)),T!==c?(W=$i(),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c)),w}o(qo,"peg$parseSingleStringChar");function yt(){var w,T,W;return w=P,T=P,ye++,W=nt(),ye--,W===c?T=void 0:(P=T,T=c),T!==c?(l.length>P?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(Ql)),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c),w}o(yt,"peg$parseUnquotedStringChar");function $i(){var w,T;return cd.test(l.charAt(P))?(w=l.charAt(P),P++):(w=c,ye===0&&Ce(ku)),w===c&&(w=P,l.charCodeAt(P)===110?(T=sc,P++):(T=c,ye===0&&Ce(Nu)),T!==c&&(Re=w,T=lc()),w=T,w===c&&(w=P,l.charCodeAt(P)===114?(T=ac,P++):(T=c,ye===0&&Ce(Zl)),T!==c&&(Re=w,T=Jl()),w=T,w===c&&(w=P,l.charCodeAt(P)===116?(T=Lu,P++):(T=c,ye===0&&Ce(Ot)),T!==c&&(Re=w,T=Nt()),w=T))),w}o($i,"peg$parseEscapeSequence");function Au(w,T){function W(){return w.apply(this,arguments)||T.apply(this,arguments)}return o(W,"orFilter"),W.desc=w.desc+" or "+T.desc,W}o(Au,"or");function hd(w,T){function W(){return w.apply(this,arguments)&&T.apply(this,arguments)}return o(W,"andFilter"),W.desc=w.desc+" and "+T.desc,W}o(hd,"and");function Vo(w){function T(){return!w.apply(this,arguments)}return o(T,"notFilter"),T.desc="not "+w.desc,T}o(Vo,"not");function yr(w){function T(){return w.apply(this,arguments)}return o(T,"bindingFilter"),T.desc="("+w.desc+")",T}o(yr,"binding");function fc(w){return!0}o(fc,"allFilter"),fc.desc="all flows";var Du=[new RegExp("text/javascript"),new RegExp("application/x-javascript"),new RegExp("application/javascript"),new RegExp("text/css"),new RegExp("image/.*"),new RegExp("font/.*"),new RegExp("application/font.*")];function Ko(w){if(w.response){for(var T=Ys.getContentType(w.response),W=Du.length;W--;)if(Du[W].test(T))return!0}return!1}o(Ko,"assetFilter"),Ko.desc="is asset";function na(w){w=new RegExp(w,"i");function T(W){return!0}return o(T,"bodyFilter"),T.desc="body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10",T}o(na,"body");function Go(w){w=new RegExp(w,"i");function T(W){return!0}return o(T,"requestBodyFilter"),T.desc="body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10",T}o(Go,"requestBody");function md(w){w=new RegExp(w,"i");function T(W){return!0}return o(T,"responseBodyFilter"),T.desc="body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10",T}o(md,"responseBody");function ia(w){function T(W){return W.response&&W.response.status_code===w}return o(T,"responseCodeFilter"),T.desc="resp. code is "+w,T}o(ia,"responseCode");function Ru(w){w=new RegExp(w,"i");function T(W){return w.test(W.comment)}return o(T,"commentFilter"),T.desc="comment matches "+w,T}o(Ru,"comment");function Iu(w){w=new RegExp(w,"i");function T(W){return W.request&&(w.test(W.request.host)||w.test(W.request.pretty_host))}return o(T,"domainFilter"),T.desc="domain matches "+w,T}o(Iu,"domain");function oa(w){return w.type==="dns"}o(oa,"dnsFilter"),oa.desc="is a DNS Flow";function Fu(w){w=new RegExp(w,"i");function T(W){return!!W.server_conn.address&&w.test(W.server_conn.address[0]+":"+W.server_conn.address[1])}return o(T,"destinationFilter"),T.desc="destination address matches "+w,T}o(Fu,"destination");function Bu(w){return!!w.error}o(Bu,"errorFilter"),Bu.desc="has error";function Hu(w){w=new RegExp(w,"i");function T(W){return W.request&&Oo.match_header(W.request,w)||W.response&&Ys.match_header(W.response,w)}return o(T,"headerFilter"),T.desc="header matches "+w,T}o(Hu,"header");function Yo(w){w=new RegExp(w,"i");function T(W){return W.request&&Oo.match_header(W.request,w)}return o(T,"requestHeaderFilter"),T.desc="req. header matches "+w,T}o(Yo,"requestHeader");function ji(w){w=new RegExp(w,"i");function T(W){return W.response&&Ys.match_header(W.response,w)}return o(T,"responseHeaderFilter"),T.desc="resp. header matches "+w,T}o(ji,"responseHeader");function Ss(w){return w.type==="http"}o(Ss,"httpFilter"),Ss.desc="is an HTTP Flow";function zr(w){return w.marked}o(zr,"markedFilter"),zr.desc="is marked";function sa(w){w=new RegExp(w,"i");function T(W){return w.test(W.marked)}return o(T,"markerFilter"),T.desc="marker matches "+w,T}o(sa,"marker");function mn(w){w=new RegExp(w,"i");function T(W){return W.request&&w.test(W.request.method)}return o(T,"methodFilter"),T.desc="method matches "+w,T}o(mn,"method");function qi(w){return w.request&&!w.response}o(qi,"noResponseFilter"),qi.desc="has no response";function al(w){return w.is_replay==="request"}o(al,"clientReplayFilter"),al.desc="request has been replayed";function cc(w){return w.is_replay==="response"}o(cc,"serverReplayFilter"),cc.desc="response has been replayed";function Wu(w){return!!w.is_replay}o(Wu,"replayFilter"),Wu.desc="flow has been replayed";function gd(w){w=new RegExp(w,"i");function T(W){return!!W.client_conn.peername&&w.test(W.client_conn.peername[0]+":"+W.client_conn.peername[1])}return o(T,"sourceFilter"),T.desc="source address matches "+w,T}o(gd,"source");function Uu(w){return!!w.response}o(Uu,"responseFilter"),Uu.desc="has response";function la(w){return w.type==="tcp"}o(la,"tcpFilter"),la.desc="is a TCP Flow";function qn(w){return w.type==="udp"}o(qn,"udpFilter"),qn.desc="is a UDP Flow";function Ti(w){w=new RegExp(w,"i");function T(W){return W.request&&w.test(Oo.getContentType(W.request))}return o(T,"requestContentTypeFilter"),T.desc="req. content type matches "+w,T}o(Ti,"requestContentType");function pc(w){w=new RegExp(w,"i");function T(W){return W.response&&w.test(Ys.getContentType(W.response))}return o(T,"responseContentTypeFilter"),T.desc="resp. content type matches "+w,T}o(pc,"responseContentType");function aa(w){w=new RegExp(w,"i");function T(W){return W.request&&w.test(Oo.getContentType(W.request))||W.response&&w.test(Ys.getContentType(W.response))}return o(T,"contentTypeFilter"),T.desc="content type matches "+w,T}o(aa,"contentType");function dc(w){w=new RegExp(w,"i");function T(W){var U;if(W.type==="dns"){let wr=(U=W.request)==null?void 0:U.questions[0];return wr&&w.test(wr.name)}return W.request&&w.test(Oo.pretty_url(W.request))}return o(T,"urlFilter"),T.desc="url matches "+w,T}o(dc,"url");function Vi(w){return!!w.websocket}if(o(Vi,"websocketFilter"),Vi.desc="is a Websocket Flow",jo=C(),jo!==c&&P===l.length)return jo;throw jo!==c&&PxS,icon:()=>N0,method:()=>tm,path:()=>L0,quickactions:()=>Pp,size:()=>P0,status:()=>rm,time:()=>O0,timestamp:()=>M0,tls:()=>k0});var er=fe(Oe());var T0=fe(ti());var k0=o(({flow:e})=>er.default.createElement("td",{className:(0,T0.default)("col-tls",e.client_conn.tls_established?"col-tls-https":"col-tls-http")}),"tls");k0.headerName="";k0.sortKey=e=>e.type==="http"&&e.request.scheme;var N0=o(({flow:e})=>er.default.createElement("td",{className:"col-icon"},er.default.createElement("div",{className:(0,T0.default)("resource-icon",Kk(e))})),"icon");N0.headerName="";N0.sortKey=e=>Kk(e);var Kk=o(e=>{if(e.type!=="http")return`resource-icon-${e.type}`;if(e.websocket)return"resource-icon-websocket";if(!e.response)return"resource-icon-plain";var t=Ys.getContentType(e.response)||"";return e.response.status_code===304?"resource-icon-not-modified":300<=e.response.status_code&&e.response.status_code<400?"resource-icon-redirect":t.indexOf("image")>=0?"resource-icon-image":t.indexOf("javascript")>=0?"resource-icon-js":t.indexOf("css")>=0?"resource-icon-css":t.indexOf("html")>=0?"resource-icon-document":"resource-icon-plain"},"getIcon"),Gk=o(e=>{var t,n,l,d;switch(e.type){case"http":return Oo.pretty_url(e.request);case"tcp":case"udp":return`${e.client_conn.peername.join(":")} \u2194 ${(n=(t=e.server_conn)==null?void 0:t.address)==null?void 0:n.join(":")}`;case"dns":return`${e.request.questions.map(h=>`${h.name} ${h.type}`).join(", ")} = ${((d=(l=e.response)==null?void 0:l.answers.map(h=>h.data).join(", "))!=null?d:"...")||"?"}`}},"mainPath"),L0=o(({flow:e})=>{let t;return e.error&&(e.error.msg==="Connection killed."?t=er.default.createElement("i",{className:"fa fa-fw fa-times pull-right"}):t=er.default.createElement("i",{className:"fa fa-fw fa-exclamation pull-right"})),er.default.createElement("td",{className:"col-path"},e.is_replay==="request"&&er.default.createElement("i",{className:"fa fa-fw fa-repeat pull-right"}),e.intercepted&&er.default.createElement("i",{className:"fa fa-fw fa-pause pull-right"}),t,er.default.createElement("span",{className:"marker pull-right"},e.marked),Gk(e))},"path");L0.headerName="Path";L0.sortKey=e=>Gk(e);var tm=o(({flow:e})=>er.default.createElement("td",{className:"col-method"},tm.sortKey(e)),"method");tm.headerName="Method";tm.sortKey=e=>{switch(e.type){case"http":return e.websocket?e.client_conn.tls_established?"WSS":"WS":e.request.method;case"dns":return e.request.op_code;default:return e.type.toUpperCase()}};var rm=o(({flow:e})=>{let t="darkred";return e.type!=="http"&&e.type!="dns"||!e.response?er.default.createElement("td",{className:"col-status"}):(100<=e.response.status_code&&e.response.status_code<200?t="green":200<=e.response.status_code&&e.response.status_code<300?t="darkgreen":300<=e.response.status_code&&e.response.status_code<400?t="lightblue":(400<=e.response.status_code&&e.response.status_code<500||500<=e.response.status_code&&e.response.status_code<600)&&(t="red"),er.default.createElement("td",{className:"col-status",style:{color:t}},rm.sortKey(e)))},"status");rm.headerName="Status";rm.sortKey=e=>{var t,n;switch(e.type){case"http":return(t=e.response)==null?void 0:t.status_code;case"dns":return(n=e.response)==null?void 0:n.response_code;default:return}};var P0=o(({flow:e})=>er.default.createElement("td",{className:"col-size"},x0(wS(e))),"size");P0.headerName="Size";P0.sortKey=e=>wS(e);var O0=o(({flow:e})=>{let t=em(e),n=yS(e);return er.default.createElement("td",{className:"col-time"},t&&n?S0(1e3*(n-t)):"...")},"time");O0.headerName="Time";O0.sortKey=e=>{let t=em(e),n=yS(e);return t&&n&&n-t};var M0=o(({flow:e})=>{let t=em(e);return er.default.createElement("td",{className:"col-timestamp"},t?no(t):"...")},"timestamp");M0.headerName="Start time";M0.sortKey=e=>em(e);var Pp=o(({flow:e})=>{let t=Gs(),[n,l]=(0,er.useState)(!1),d=null;return e.intercepted?d=er.default.createElement("a",{href:"#",className:"quickaction",onClick:()=>t(Op(e))},er.default.createElement("i",{className:"fa fa-fw fa-play text-success"})):E0(e)&&(d=er.default.createElement("a",{href:"#",className:"quickaction",onClick:()=>t(Mp(e))},er.default.createElement("i",{className:"fa fa-fw fa-repeat text-primary"}))),er.default.createElement("td",{className:(0,T0.default)("col-quickactions",{hover:n}),onClick:()=>0},d?er.default.createElement("div",null,d):er.default.createElement(er.default.Fragment,null))},"quickactions");Pp.headerName="";Pp.sortKey=e=>0;var xS={icon:N0,method:tm,path:L0,quickactions:Pp,size:P0,status:rm,time:O0,timestamp:M0,tls:k0};var EF="FLOWS_ADD",_F="FLOWS_UPDATE",Yk="FLOWS_REMOVE",TF="FLOWS_RECEIVE",Xk="FLOWS_SELECT",Qk="FLOWS_SET_FILTER",Zk="FLOWS_SET_SORT",Jk="FLOWS_SET_HIGHLIGHT",kF=ke({highlight:void 0,filter:void 0,sort:{column:void 0,desc:!1},selected:[]},C0);function SS(e=kF,t){switch(t.type){case EF:case _F:case Yk:case TF:let n=Jh[t.cmd](t.data,eN(e.filter),CS(e.sort)),l=e.selected;if(t.type===Yk&&e.selected.includes(t.data)){if(e.selected.length>1)l=l.filter(d=>d!==t.data);else if(l=[],t.data in e.viewIndex&&e.view.length>1){let d=e.viewIndex[t.data],h;d===e.view.length-1?h=e.view[d-1]:h=e.view[d+1],l.push(h.id)}}return ke(Pt(ke({},e),{selected:l}),Lp(e,n));case Qk:return ke(Pt(ke({},e),{filter:t.filter}),Lp(e,hS(eN(t.filter),CS(e.sort))));case Jk:return Pt(ke({},e),{highlight:t.highlight});case Zk:return ke(Pt(ke({},e),{sort:t.sort}),Lp(e,jk(CS(t.sort))));case Xk:return Pt(ke({},e),{selected:t.flowIds});default:return e}}o(SS,"reducer");function eN(e){if(!!e)return Of.parse(e)}o(eN,"makeFilter");function CS({column:e,desc:t}){if(!e)return(l,d)=>0;let n=xS[e].sortKey;return(l,d)=>{let h=n(l),c=n(d);return h>c?t?-1:1:hkt(`/flows/${e.id}/resume`,{method:"POST"})}o(Op,"resume");function R0(){return e=>kt("/flows/resume",{method:"POST"})}o(R0,"resumeAll");function I0(e){return t=>kt(`/flows/${e.id}/kill`,{method:"POST"})}o(I0,"kill");function rN(){return e=>kt("/flows/kill",{method:"POST"})}o(rN,"killAll");function F0(e){return t=>kt(`/flows/${e.id}`,{method:"DELETE"})}o(F0,"remove");function B0(e){return t=>kt(`/flows/${e.id}/duplicate`,{method:"POST"})}o(B0,"duplicate");function Mp(e){return t=>kt(`/flows/${e.id}/replay`,{method:"POST"})}o(Mp,"replay");function H0(e){return t=>kt(`/flows/${e.id}/revert`,{method:"POST"})}o(H0,"revert");function Wi(e,t){return n=>kt.put(`/flows/${e.id}`,t)}o(Wi,"update");function nN(e,t,n){let l=new FormData;return t=new window.Blob([t],{type:"plain/text"}),l.append("file",t),d=>kt(`/flows/${e.id}/${n}/content.data`,{method:"POST",body:l})}o(nN,"uploadContent");function W0(){return e=>kt("/clear",{method:"POST"})}o(W0,"clear");function iN(e){let t=new FormData;return t.append("file",e),n=>kt("/flows/dump",{method:"POST",body:t})}o(iN,"upload");function Af(e){return{type:Xk,flowIds:e?[e]:[]}}o(Af,"select");var U0="UI_HIDE_MODAL",oN="UI_SET_ACTIVE_MODAL",NF={activeModal:void 0};function bS(e=NF,t){switch(t.type){case oN:return Pt(ke({},e),{activeModal:t.activeModal});case U0:return Pt(ke({},e),{activeModal:void 0});default:return e}}o(bS,"reducer");function sN(e){return{type:oN,activeModal:e}}o(sN,"setActiveModal");function z0(){return{type:U0}}o(z0,"hideModal");var ym=fe(Oe());var Ut=fe(Oe());var Dp=fe(Oe());var im=fe(Oe()),lN=fe(ti()),aN=(()=>{let e=document.createElement("div");return e.setAttribute("contenteditable","PLAINTEXT-ONLY"),e.contentEditable==="plaintext-only"?"plaintext-only":"true"})(),Ap=!1,Xs=class extends im.Component{constructor(){super(...arguments);this.input=im.default.createRef();this.isEditing=o(()=>{var t;return((t=this.input.current)==null?void 0:t.contentEditable)===aN},"isEditing");this.startEditing=o(()=>{if(!this.input.current)return console.error("unreachable");this.isEditing()||(this.suppress_events=!0,this.input.current.blur(),this.input.current.contentEditable=aN,window.requestAnimationFrame(()=>{var l,d;if(!this.input.current)return;this.input.current.focus(),this.suppress_events=!1;let t=document.createRange();t.selectNodeContents(this.input.current);let n=window.getSelection();n==null||n.removeAllRanges(),n==null||n.addRange(t),(d=(l=this.props).onEditStart)==null||d.call(l)}))},"startEditing");this.resetValue=o(()=>{var t,n;if(!this.input.current)return console.error("unreachable");this.input.current.textContent=this.props.content,(n=(t=this.props).onInput)==null||n.call(t,this.props.content)},"resetValue");this.finishEditing=o(()=>{if(!this.input.current)return console.error("unreachable");this.props.onEditDone(this.input.current.textContent||""),this.input.current.blur(),this.input.current.contentEditable="inherit"},"finishEditing");this.onPaste=o(t=>{t.preventDefault();let n=t.clipboardData.getData("text/plain");document.execCommand("insertHTML",!1,n)},"onPaste");this.suppress_events=!1;this.onMouseDown=o(t=>{Ap&&console.debug("onMouseDown",this.suppress_events),this.suppress_events=!0,window.addEventListener("mouseup",this.onMouseUp,{once:!0})},"onMouseDown");this.onMouseUp=o(t=>{var d;let n=t.target===this.input.current,l=!((d=window.getSelection())==null?void 0:d.toString());Ap&&console.warn("mouseUp",this.suppress_events,n,l),n&&l&&this.startEditing(),this.suppress_events=!1},"onMouseUp");this.onClick=o(t=>{Ap&&console.debug("onClick",this.suppress_events)},"onClick");this.onFocus=o(t=>{if(Ap&&console.debug("onFocus",this.props.content,this.suppress_events),!this.input.current)throw"unreachable";this.suppress_events||this.startEditing()},"onFocus");this.onInput=o(t=>{var n,l,d;(d=(l=this.props).onInput)==null||d.call(l,((n=this.input.current)==null?void 0:n.textContent)||"")},"onInput");this.onBlur=o(t=>{Ap&&console.debug("onBlur",this.props.content,this.suppress_events),!this.suppress_events&&this.finishEditing()},"onBlur");this.onKeyDown=o(t=>{var n,l;switch(Ap&&console.debug("keydown",t),t.stopPropagation(),t.key){case"Escape":t.preventDefault(),this.resetValue(),this.finishEditing();break;case"Enter":t.shiftKey||(t.preventDefault(),this.finishEditing());break;default:break}(l=(n=this.props).onKeyDown)==null||l.call(n,t)},"onKeyDown")}render(){let t=(0,lN.default)("inline-input",this.props.className);return im.default.createElement("span",{ref:this.input,tabIndex:0,className:t,placeholder:this.props.placeholder,onFocus:this.onFocus,onBlur:this.onBlur,onKeyDown:this.onKeyDown,onInput:this.onInput,onPaste:this.onPaste,onMouseDown:this.onMouseDown,onClick:this.onClick},this.props.content)}componentDidUpdate(t){var n,l;t.content!==this.props.content&&((l=(n=this.props).onInput)==null||l.call(n,this.props.content))}};o(Xs,"ValueEditor");var uN=fe(ti());function Df(e){let[t,n]=(0,Dp.useState)(e.isValid(e.content)),l=(0,Dp.useRef)(null),d=o(c=>{var v;e.isValid(c)?e.onEditDone(c):(v=l.current)==null||v.resetValue()},"onEditDone"),h=(0,uN.default)(e.className,t?"has-success":"has-warning");return Dp.default.createElement(Xs,Pt(ke({},e),{className:h,onInput:c=>n(e.isValid(c)),onEditDone:d,ref:l}))}o(Df,"ValidateEditor");function ES(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}o(ES,"_defineProperty");function fN(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);t&&(l=l.filter(function(d){return Object.getOwnPropertyDescriptor(e,d).enumerable})),n.push.apply(n,l)}return n}o(fN,"ownKeys");function $0(e){for(var t=1;tn[l.level])));case dN:case OF:return ke(ke({},e),Lp(e,Jh[t.cmd](t.data,l=>e.filters[l.level])));default:return e}}o(NS,"reduce");function gN(e){return{type:mN,filter:e}}o(gN,"toggleFilter");function Rp(){return{type:hN}}o(Rp,"toggleVisibility");function vN(e,t="web"){let n={id:Math.random().toString(),message:e,level:t};return{type:dN,cmd:"add",data:n}}o(vN,"add");var yN="UI_OPTION_UPDATE_START",wN="UI_OPTION_UPDATE_SUCCESS",xN="UI_OPTION_UPDATE_ERROR",AF={};function LS(e=AF,t){switch(t.type){case yN:return Pt(ke({},e),{[t.option]:{isUpdating:!0,value:t.value,error:!1}});case wN:return Pt(ke({},e),{[t.option]:void 0});case xN:let n=e[t.option].value;return typeof n=="boolean"&&(n=!n),Pt(ke({},e),{[t.option]:{value:n,isUpdating:!1,error:t.error}});case U0:return{};default:return e}}o(LS,"reducer");function SN(e,t){return{type:yN,option:e,value:t}}o(SN,"startUpdate");function CN(e){return{type:wN,option:e}}o(CN,"updateSuccess");function bN(e,t){return{type:xN,option:e,error:t}}o(bN,"updateError");var EN=q0({flow:nS,modal:bS,optionsEditor:LS});var ni;(function(h){h.INIT="CONNECTION_INIT",h.FETCHING="CONNECTION_FETCHING",h.ESTABLISHED="CONNECTION_ESTABLISHED",h.ERROR="CONNECTION_ERROR",h.OFFLINE="CONNECTION_OFFLINE"})(ni||(ni={}));var DF={state:ni.INIT,message:void 0};function PS(e=DF,t){switch(t.type){case ni.ESTABLISHED:case ni.FETCHING:case ni.ERROR:case ni.OFFLINE:return{state:t.type,message:t.message};default:return e}}o(PS,"reducer");function _N(){return{type:ni.FETCHING}}o(_N,"startFetching");function TN(){return{type:ni.ESTABLISHED}}o(TN,"connectionEstablished");function kN(e){return{type:ni.ERROR,message:e}}o(kN,"connectionError");var NN={add_upstream_certs_to_client_chain:!1,allow_hosts:[],anticache:!1,anticomp:!1,block_global:!0,block_list:[],block_private:!1,body_size_limit:void 0,cert_passphrase:void 0,certs:[],ciphers_client:void 0,ciphers_server:void 0,client_certs:void 0,client_replay:[],client_replay_concurrency:1,command_history:!0,confdir:"~/.mitmproxy",connect_addr:void 0,connection_strategy:"eager",console_focus_follow:!1,content_view_lines_cutoff:512,export_preserve_original_ip:!1,http2:!0,http2_ping_keepalive:58,ignore_hosts:[],intercept:void 0,intercept_active:!1,keep_host_header:!1,key_size:2048,listen_host:"",listen_port:void 0,map_local:[],map_remote:[],mode:["regular"],modify_body:[],modify_headers:[],normalize_outbound_headers:!0,onboarding:!0,onboarding_host:"mitm.it",onboarding_port:80,proxy_debug:!1,proxyauth:void 0,rawtcp:!0,rawudp:!0,readfile_filter:void 0,rfile:void 0,save_stream_file:void 0,save_stream_filter:void 0,scripts:[],server:!0,server_replay:[],server_replay_ignore_content:!1,server_replay_ignore_host:!1,server_replay_ignore_params:[],server_replay_ignore_payload_params:[],server_replay_ignore_port:!1,server_replay_kill_extra:!1,server_replay_nopop:!1,server_replay_refresh:!0,server_replay_use_headers:[],showhost:!1,ssl_insecure:!1,ssl_verify_upstream_trusted_ca:void 0,ssl_verify_upstream_trusted_confdir:void 0,stickyauth:void 0,stickycookie:void 0,stream_large_bodies:void 0,tcp_hosts:[],termlog_verbosity:"info",tls_version_client_max:"UNBOUNDED",tls_version_client_min:"TLS1_2",tls_version_server_max:"UNBOUNDED",tls_version_server_min:"TLS1_2",udp_hosts:[],upstream_auth:void 0,upstream_cert:!0,validate_inbound_headers:!0,view_filter:void 0,view_order:"time",view_order_reversed:!1,web_columns:["tls","icon","path","method","status","size","time"],web_debug:!1,web_host:"127.0.0.1",web_open_browser:!0,web_port:8081,web_static_viewer:"",websocket:!0};var OS="OPTIONS_RECEIVE",MS="OPTIONS_UPDATE";function AS(e=NN,t){switch(t.type){case OS:let n={};for(let[d,{value:h}]of Object.entries(t.data))n[d]=h;return n;case MS:let l=ke({},e);for(let[d,{value:h}]of Object.entries(t.data))l[d]=h;return l;default:return e}}o(AS,"reducer");function RF(e,t,n){return Ia(this,null,function*(){try{let l=yield kt.put("/options",{[e]:t});if(l.status===200)n(CN(e));else throw yield l.text()}catch(l){n(bN(e,l))}})}o(RF,"pureSendUpdate");var IF=RF;function Ip(e,t){return n=>{n(SN(e,t)),IF(e,t,n)}}o(Ip,"update");function LN(){return e=>kt("/options/save",{method:"POST"})}o(LN,"save");var PN="COMMANDBAR_TOGGLE_VISIBILITY",FF={visible:!1};function DS(e=FF,t){switch(t.type){case PN:return Pt(ke({},e),{visible:!e.visible});default:return e}}o(DS,"reducer");function V0(){return{type:PN}}o(V0,"toggleVisibility");function ON(e){return function(t){var n=t.dispatch,l=t.getState;return function(d){return function(h){return typeof h=="function"?h(n,l,e):d(h)}}}}o(ON,"createThunkMiddleware");var MN=ON();MN.withExtraArgument=ON;var AN=MN;var BF="STATE_RECEIVE",HF="STATE_UPDATE",WF={available:!1,version:"",contentViews:[],servers:[]};function RS(e=WF,t){switch(t.type){case BF:case HF:return ke(Pt(ke({},e),{available:!0}),t.data);default:return e}}o(RS,"reducer");var UF={},zF=o((e=UF,t)=>{switch(t.type){case OS:return t.data;case MS:return ke(ke({},e),t.data);default:return e}},"reducer"),DN=zF;var $F=window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__||kS,jF=q0({commandBar:DS,eventLog:NS,flows:SS,connection:PS,ui:EN,options:AS,options_meta:DN,backendState:RS}),qF=o(e=>TS(jF,e,$F(pN(AN))),"createAppStore"),Fp=qF(void 0),Xt=o(()=>Gs(),"useAppDispatch"),qe=tS;var io=fe(Oe());var RN=fe(Qh()),IN=fe(ti()),IS=class extends io.Component{constructor(){super(...arguments);this.container=io.default.createRef();this.nameInput=io.default.createRef();this.valueInput=io.default.createRef();this.render=o(()=>{let[t,n]=this.props.item;return io.default.createElement("div",{ref:this.container,className:"kv-row",onClick:this.onClick,onKeyDownCapture:this.onKeyDown},io.default.createElement(Xs,{ref:this.nameInput,className:"kv-key",content:t,onEditStart:this.props.onEditStart,onEditDone:l=>this.props.onEditDone([l,n])}),":\xA0",io.default.createElement(Xs,{ref:this.valueInput,className:"kv-value",content:n,onEditStart:this.props.onEditStart,onEditDone:l=>this.props.onEditDone([t,l]),placeholder:"empty"}))},"render");this.onClick=o(t=>{t.target===this.container.current&&this.props.onClickEmptyArea()},"onClick");this.onKeyDown=o(t=>{var n;t.target===((n=this.valueInput.current)==null?void 0:n.input.current)&&t.key==="Tab"&&this.props.onTabNext()},"onKeyDown")}};o(IS,"Row");var Bp=class extends io.Component{constructor(){super(...arguments);this.rowRefs={};this.state={currentList:this.props.data||[],initialList:this.props.data};this.render=o(()=>{this.rowRefs={};let t=this.state.currentList.map((n,l)=>io.default.createElement(IS,{key:l,item:n,onEditStart:()=>this.currentlyEditing=l,onEditDone:d=>this.onEditDone(l,d),onClickEmptyArea:()=>this.onClickEmptyArea(l),onTabNext:()=>this.onTabNext(l),ref:d=>this.rowRefs[l]=d}));return io.default.createElement("div",{className:(0,IN.default)("kv-editor",this.props.className),onMouseDown:this.onMouseDown},t,io.default.createElement("div",{onClick:n=>{n.preventDefault(),this.onClickEmptyArea(this.state.currentList.length-1)},className:"kv-add-row fa fa-plus-square-o",role:"button","aria-label":"Add"}))},"render");this.onEditDone=o((t,n)=>{let l=[...this.state.currentList];n[0]?l[t]=n:l.splice(t,1),this.currentlyEditing=void 0,(0,RN.isEqual)(this.state.currentList,l)||this.props.onChange(l),this.setState({currentList:l})},"onEditDone");this.onClickEmptyArea=o(t=>{if(this.justFinishedEditing)return;let n=[...this.state.currentList];n.splice(t+1,0,["",""]),this.setState({currentList:n},()=>{var l,d;return(d=(l=this.rowRefs[t+1])==null?void 0:l.nameInput.current)==null?void 0:d.startEditing()})},"onClickEmptyArea");this.onTabNext=o(t=>{t==this.state.currentList.length-1&&this.onClickEmptyArea(t)},"onTabNext");this.onMouseDown=o(t=>{this.justFinishedEditing=this.currentlyEditing},"onMouseDown")}static getDerivedStateFromProps(t,n){return t.data!==n.initialList?{currentList:t.data||[],initialList:t.data}:null}};o(Bp,"KeyValueListEditor");var tr=fe(Oe());var om=fe(Oe());function K0(e,t){let[n,l]=(0,om.useState)(),[d,h]=(0,om.useState)();return(0,om.useEffect)(()=>{d&&d.abort();let c=new AbortController;return kt(e,{signal:c.signal}).then(v=>{if(!v.ok)throw`${v.status} ${v.statusText}`.trim();return v.text()}).then(v=>{l(v)}).catch(v=>{c.signal.aborted||l(`Error getting content: ${v}.`)}),h(c),()=>{c.signal.aborted||c.abort()}},[e,t]),n}o(K0,"useContent");var sm=fe(Oe()),G0=sm.default.memo(o(function({icon:t,text:n,className:l,title:d,onOpenFile:h,onClick:c}){let v;return sm.default.createElement("a",{href:"#",onClick:C=>{v.click(),c&&c(C)},className:l,title:d},sm.default.createElement("i",{className:"fa fa-fw "+t}),n,sm.default.createElement("input",{ref:C=>v=C,className:"hidden",type:"file",onChange:C=>{C.preventDefault(),C.target.files&&C.target.files.length>0&&h(C.target.files[0]),v.value=""}}))},"FileChooser"));var Hp=fe(Oe()),FN=fe(ti());function kr({onClick:e,children:t,icon:n,disabled:l,className:d,title:h}){return Hp.createElement("button",{className:(0,FN.default)(d,"btn btn-default"),onClick:l?void 0:e,disabled:l,title:h},n&&Hp.createElement(Hp.Fragment,null,Hp.createElement("i",{className:"fa "+n}),"\xA0"),t)}o(kr,"Button");var am=fe(Oe()),$N=fe(Oe());var lm=fe(Oe()),HN=fe(ti()),WN=fe(BN()),UN=fe(Qh());function zN(e){return e&&e.replace(/\r\n|\r/g,` -`)}o(zN,"normalizeLineEndings");var Wp=class extends lm.Component{constructor(t){super(t);this.state={isFocused:!1}}getCodeMirrorInstance(){return this.props.codeMirrorInstance||WN.default}UNSAFE_componentWillMount(){this.props.path&&console.error("Warning: react-codemirror: the `path` prop has been changed to `name`")}componentDidMount(){let t=this.getCodeMirrorInstance();this.codeMirror=t.fromTextArea(this.textareaNode,this.props.options),this.codeMirror.on("change",this.codemirrorValueChanged.bind(this)),this.codeMirror.on("cursorActivity",this.cursorActivity.bind(this)),this.codeMirror.on("focus",this.focusChanged.bind(this,!0)),this.codeMirror.on("blur",this.focusChanged.bind(this,!1)),this.codeMirror.on("scroll",this.scrollChanged.bind(this)),this.codeMirror.setValue(this.props.defaultValue||this.props.value||"")}componentWillUnmount(){this.codeMirror&&this.codeMirror.toTextArea()}UNSAFE_componentWillReceiveProps(t){if(this.codeMirror&&t.value!==void 0&&t.value!==this.props.value&&zN(this.codeMirror.getValue())!==zN(t.value))if(this.props.preserveScrollPosition){var n=this.codeMirror.getScrollInfo();this.codeMirror.setValue(t.value),this.codeMirror.scrollTo(n.left,n.top)}else this.codeMirror.setValue(t.value);if(typeof t.options=="object")for(let l in t.options)t.options.hasOwnProperty(l)&&this.setOptionIfChanged(l,t.options[l])}setOptionIfChanged(t,n){let l=this.codeMirror.getOption(t);UN.default.isEqual(l,n)||this.codeMirror.setOption(t,n)}getCodeMirror(){return this.codeMirror}focus(){this.codeMirror&&this.codeMirror.focus()}focusChanged(t){this.setState({isFocused:t}),this.props.onFocusChange&&this.props.onFocusChange(t)}cursorActivity(t){this.props.onCursorActivity&&this.props.onCursorActivity(t)}scrollChanged(t){this.props.onScroll&&this.props.onScroll(t.getScrollInfo())}codemirrorValueChanged(t,n){this.props.onChange&&n.origin!=="setValue"&&this.props.onChange(t.getValue(),n)}render(){let t=(0,HN.default)("ReactCodeMirror",this.state.isFocused?"ReactCodeMirror--focused":null,this.props.className);return lm.createElement("div",{className:t},lm.createElement("textarea",{ref:n=>this.textareaNode=n,name:this.props.name||this.props.path,defaultValue:this.props.value,autoComplete:"off",autoFocus:this.props.autoFocus}))}};o(Wp,"CodeMirror"),Wp.defaultProps={preserveScrollPosition:!1};var um=class extends $N.Component{constructor(){super(...arguments);this.editor=am.createRef();this.getContent=o(()=>{var t;return(t=this.editor.current)==null?void 0:t.codeMirror.getValue()},"getContent");this.render=o(()=>{let t={lineNumbers:!0};return am.createElement("div",{className:"codeeditor",onKeyDown:n=>n.stopPropagation()},am.createElement(Wp,{ref:this.editor,value:this.props.initialContent,onChange:()=>0,options:t}))},"render")}};o(um,"CodeEditor");var Rf=fe(Oe()),VF=Rf.default.memo(o(function({lines:t,maxLines:n,showMore:l}){return t.length===0?null:Rf.default.createElement("pre",null,t.map((d,h)=>h===n?Rf.default.createElement("button",{key:"showmore",onClick:l,className:"btn btn-xs btn-info"},Rf.default.createElement("i",{className:"fa fa-angle-double-down","aria-hidden":"true"})," Show more"):Rf.default.createElement("div",{key:h},d.map(([c,v],C)=>Rf.default.createElement("span",{key:C,className:c},v)))))},"LineRenderer")),Y0=VF;var zf=fe(Oe());var xi=fe(Oe());var X0=fe(Oe());var HS=o(function(t){return t.reduce(function(n,l){var d=l[0],h=l[1];return n[d]=h,n},{})},"fromEntries"),WS=typeof window!="undefined"&&window.document&&window.document.createElement?X0.useLayoutEffect:X0.useEffect;var fu=fe(Oe());var Gr="top",Ln="bottom",ln="right",an="left",Q0="auto",su=[Gr,Ln,ln,an],Rl="start",Z0="end",jN="clippingParents",J0="viewport",Up="popper",qN="reference",US=su.reduce(function(e,t){return e.concat([t+"-"+Rl,t+"-"+Z0])},[]),ey=[].concat(su,[Q0]).reduce(function(e,t){return e.concat([t,t+"-"+Rl,t+"-"+Z0])},[]),KF="beforeRead",GF="read",YF="afterRead",XF="beforeMain",QF="main",ZF="afterMain",JF="beforeWrite",e3="write",t3="afterWrite",VN=[KF,GF,YF,XF,QF,ZF,JF,e3,t3];function Pn(e){return e?(e.nodeName||"").toLowerCase():null}o(Pn,"getNodeName");function Br(e){if(e==null)return window;if(e.toString()!=="[object Window]"){var t=e.ownerDocument;return t&&t.defaultView||window}return e}o(Br,"getWindow");function Il(e){var t=Br(e).Element;return e instanceof t||e instanceof Element}o(Il,"isElement");function Yr(e){var t=Br(e).HTMLElement;return e instanceof t||e instanceof HTMLElement}o(Yr,"isHTMLElement");function ty(e){if(typeof ShadowRoot=="undefined")return!1;var t=Br(e).ShadowRoot;return e instanceof t||e instanceof ShadowRoot}o(ty,"isShadowRoot");function r3(e){var t=e.state;Object.keys(t.elements).forEach(function(n){var l=t.styles[n]||{},d=t.attributes[n]||{},h=t.elements[n];!Yr(h)||!Pn(h)||(Object.assign(h.style,l),Object.keys(d).forEach(function(c){var v=d[c];v===!1?h.removeAttribute(c):h.setAttribute(c,v===!0?"":v)}))})}o(r3,"applyStyles");function n3(e){var t=e.state,n={popper:{position:t.options.strategy,left:"0",top:"0",margin:"0"},arrow:{position:"absolute"},reference:{}};return Object.assign(t.elements.popper.style,n.popper),t.styles=n,t.elements.arrow&&Object.assign(t.elements.arrow.style,n.arrow),function(){Object.keys(t.elements).forEach(function(l){var d=t.elements[l],h=t.attributes[l]||{},c=Object.keys(t.styles.hasOwnProperty(l)?t.styles[l]:n[l]),v=c.reduce(function(C,k){return C[k]="",C},{});!Yr(d)||!Pn(d)||(Object.assign(d.style,v),Object.keys(h).forEach(function(C){d.removeAttribute(C)}))})}}o(n3,"effect");var KN={name:"applyStyles",enabled:!0,phase:"write",fn:r3,effect:n3,requires:["computeStyles"]};function On(e){return e.split("-")[0]}o(On,"getBasePlacement");var lu=Math.round;function Mo(e,t){t===void 0&&(t=!1);var n=e.getBoundingClientRect(),l=1,d=1;return Yr(e)&&t&&(l=n.width/e.offsetWidth||1,d=n.height/e.offsetHeight||1),{width:lu(n.width/l),height:lu(n.height/d),top:lu(n.top/d),right:lu(n.right/l),bottom:lu(n.bottom/d),left:lu(n.left/l),x:lu(n.left/l),y:lu(n.top/d)}}o(Mo,"getBoundingClientRect");function If(e){var t=Mo(e),n=e.offsetWidth,l=e.offsetHeight;return Math.abs(t.width-n)<=1&&(n=t.width),Math.abs(t.height-l)<=1&&(l=t.height),{x:e.offsetLeft,y:e.offsetTop,width:n,height:l}}o(If,"getLayoutRect");function fm(e,t){var n=t.getRootNode&&t.getRootNode();if(e.contains(t))return!0;if(n&&ty(n)){var l=t;do{if(l&&e.isSameNode(l))return!0;l=l.parentNode||l.host}while(l)}return!1}o(fm,"contains");function wi(e){return Br(e).getComputedStyle(e)}o(wi,"getComputedStyle");function zS(e){return["table","td","th"].indexOf(Pn(e))>=0}o(zS,"isTableElement");function Bn(e){return((Il(e)?e.ownerDocument:e.document)||window.document).documentElement}o(Bn,"getDocumentElement");function Fl(e){return Pn(e)==="html"?e:e.assignedSlot||e.parentNode||(ty(e)?e.host:null)||Bn(e)}o(Fl,"getParentNode");function GN(e){return!Yr(e)||wi(e).position==="fixed"?null:e.offsetParent}o(GN,"getTrueOffsetParent");function i3(e){var t=navigator.userAgent.toLowerCase().indexOf("firefox")!==-1,n=navigator.userAgent.indexOf("Trident")!==-1;if(n&&Yr(e)){var l=wi(e);if(l.position==="fixed")return null}for(var d=Fl(e);Yr(d)&&["html","body"].indexOf(Pn(d))<0;){var h=wi(d);if(h.transform!=="none"||h.perspective!=="none"||h.contain==="paint"||["transform","perspective"].indexOf(h.willChange)!==-1||t&&h.willChange==="filter"||t&&h.filter&&h.filter!=="none")return d;d=d.parentNode}return null}o(i3,"getContainingBlock");function as(e){for(var t=Br(e),n=GN(e);n&&zS(n)&&wi(n).position==="static";)n=GN(n);return n&&(Pn(n)==="html"||Pn(n)==="body"&&wi(n).position==="static")?t:n||i3(e)||t}o(as,"getOffsetParent");function Ff(e){return["top","bottom"].indexOf(e)>=0?"x":"y"}o(Ff,"getMainAxisFromPlacement");var Ao=Math.max,au=Math.min,cm=Math.round;function Bf(e,t,n){return Ao(e,au(t,n))}o(Bf,"within");function pm(){return{top:0,right:0,bottom:0,left:0}}o(pm,"getFreshSideObject");function dm(e){return Object.assign({},pm(),e)}o(dm,"mergePaddingObject");function hm(e,t){return t.reduce(function(n,l){return n[l]=e,n},{})}o(hm,"expandToHashMap");var o3=o(function(t,n){return t=typeof t=="function"?t(Object.assign({},n.rects,{placement:n.placement})):t,dm(typeof t!="number"?t:hm(t,su))},"toPaddingObject");function s3(e){var t,n=e.state,l=e.name,d=e.options,h=n.elements.arrow,c=n.modifiersData.popperOffsets,v=On(n.placement),C=Ff(v),k=[an,ln].indexOf(v)>=0,O=k?"height":"width";if(!(!h||!c)){var j=o3(d.padding,n),B=If(h),X=C==="y"?Gr:an,J=C==="y"?Ln:ln,Z=n.rects.reference[O]+n.rects.reference[C]-c[C]-n.rects.popper[O],R=c[C]-n.rects.reference[C],A=as(h),I=A?C==="y"?A.clientHeight||0:A.clientWidth||0:0,G=Z/2-R/2,K=j[X],se=I-B[O]-j[J],ne=I/2-B[O]/2+G,pe=Bf(K,ne,se),me=C;n.modifiersData[l]=(t={},t[me]=pe,t.centerOffset=pe-ne,t)}}o(s3,"arrow");function l3(e){var t=e.state,n=e.options,l=n.element,d=l===void 0?"[data-popper-arrow]":l;d!=null&&(typeof d=="string"&&(d=t.elements.popper.querySelector(d),!d)||!fm(t.elements.popper,d)||(t.elements.arrow=d))}o(l3,"effect");var YN={name:"arrow",enabled:!0,phase:"main",fn:s3,effect:l3,requires:["popperOffsets"],requiresIfExists:["preventOverflow"]};var a3={top:"auto",right:"auto",bottom:"auto",left:"auto"};function u3(e){var t=e.x,n=e.y,l=window,d=l.devicePixelRatio||1;return{x:cm(cm(t*d)/d)||0,y:cm(cm(n*d)/d)||0}}o(u3,"roundOffsetsByDPR");function XN(e){var t,n=e.popper,l=e.popperRect,d=e.placement,h=e.offsets,c=e.position,v=e.gpuAcceleration,C=e.adaptive,k=e.roundOffsets,O=k===!0?u3(h):typeof k=="function"?k(h):h,j=O.x,B=j===void 0?0:j,X=O.y,J=X===void 0?0:X,Z=h.hasOwnProperty("x"),R=h.hasOwnProperty("y"),A=an,I=Gr,G=window;if(C){var K=as(n),se="clientHeight",ne="clientWidth";K===Br(n)&&(K=Bn(n),wi(K).position!=="static"&&(se="scrollHeight",ne="scrollWidth")),K=K,d===Gr&&(I=Ln,J-=K[se]-l.height,J*=v?1:-1),d===an&&(A=ln,B-=K[ne]-l.width,B*=v?1:-1)}var pe=Object.assign({position:c},C&&a3);if(v){var me;return Object.assign({},pe,(me={},me[I]=R?"0":"",me[A]=Z?"0":"",me.transform=(G.devicePixelRatio||1)<2?"translate("+B+"px, "+J+"px)":"translate3d("+B+"px, "+J+"px, 0)",me))}return Object.assign({},pe,(t={},t[I]=R?J+"px":"",t[A]=Z?B+"px":"",t.transform="",t))}o(XN,"mapToStyles");function f3(e){var t=e.state,n=e.options,l=n.gpuAcceleration,d=l===void 0?!0:l,h=n.adaptive,c=h===void 0?!0:h,v=n.roundOffsets,C=v===void 0?!0:v;if(!1)var k;var O={placement:On(t.placement),popper:t.elements.popper,popperRect:t.rects.popper,gpuAcceleration:d};t.modifiersData.popperOffsets!=null&&(t.styles.popper=Object.assign({},t.styles.popper,XN(Object.assign({},O,{offsets:t.modifiersData.popperOffsets,position:t.options.strategy,adaptive:c,roundOffsets:C})))),t.modifiersData.arrow!=null&&(t.styles.arrow=Object.assign({},t.styles.arrow,XN(Object.assign({},O,{offsets:t.modifiersData.arrow,position:"absolute",adaptive:!1,roundOffsets:C})))),t.attributes.popper=Object.assign({},t.attributes.popper,{"data-popper-placement":t.placement})}o(f3,"computeStyles");var QN={name:"computeStyles",enabled:!0,phase:"beforeWrite",fn:f3,data:{}};var ry={passive:!0};function c3(e){var t=e.state,n=e.instance,l=e.options,d=l.scroll,h=d===void 0?!0:d,c=l.resize,v=c===void 0?!0:c,C=Br(t.elements.popper),k=[].concat(t.scrollParents.reference,t.scrollParents.popper);return h&&k.forEach(function(O){O.addEventListener("scroll",n.update,ry)}),v&&C.addEventListener("resize",n.update,ry),function(){h&&k.forEach(function(O){O.removeEventListener("scroll",n.update,ry)}),v&&C.removeEventListener("resize",n.update,ry)}}o(c3,"effect");var ZN={name:"eventListeners",enabled:!0,phase:"write",fn:o(function(){},"fn"),effect:c3,data:{}};var p3={left:"right",right:"left",bottom:"top",top:"bottom"};function zp(e){return e.replace(/left|right|bottom|top/g,function(t){return p3[t]})}o(zp,"getOppositePlacement");var d3={start:"end",end:"start"};function ny(e){return e.replace(/start|end/g,function(t){return d3[t]})}o(ny,"getOppositeVariationPlacement");function Hf(e){var t=Br(e),n=t.pageXOffset,l=t.pageYOffset;return{scrollLeft:n,scrollTop:l}}o(Hf,"getWindowScroll");function Wf(e){return Mo(Bn(e)).left+Hf(e).scrollLeft}o(Wf,"getWindowScrollBarX");function $S(e){var t=Br(e),n=Bn(e),l=t.visualViewport,d=n.clientWidth,h=n.clientHeight,c=0,v=0;return l&&(d=l.width,h=l.height,/^((?!chrome|android).)*safari/i.test(navigator.userAgent)||(c=l.offsetLeft,v=l.offsetTop)),{width:d,height:h,x:c+Wf(e),y:v}}o($S,"getViewportRect");function jS(e){var t,n=Bn(e),l=Hf(e),d=(t=e.ownerDocument)==null?void 0:t.body,h=Ao(n.scrollWidth,n.clientWidth,d?d.scrollWidth:0,d?d.clientWidth:0),c=Ao(n.scrollHeight,n.clientHeight,d?d.scrollHeight:0,d?d.clientHeight:0),v=-l.scrollLeft+Wf(e),C=-l.scrollTop;return wi(d||n).direction==="rtl"&&(v+=Ao(n.clientWidth,d?d.clientWidth:0)-h),{width:h,height:c,x:v,y:C}}o(jS,"getDocumentRect");function Uf(e){var t=wi(e),n=t.overflow,l=t.overflowX,d=t.overflowY;return/auto|scroll|overlay|hidden/.test(n+d+l)}o(Uf,"isScrollParent");function iy(e){return["html","body","#document"].indexOf(Pn(e))>=0?e.ownerDocument.body:Yr(e)&&Uf(e)?e:iy(Fl(e))}o(iy,"getScrollParent");function uu(e,t){var n;t===void 0&&(t=[]);var l=iy(e),d=l===((n=e.ownerDocument)==null?void 0:n.body),h=Br(l),c=d?[h].concat(h.visualViewport||[],Uf(l)?l:[]):l,v=t.concat(c);return d?v:v.concat(uu(Fl(c)))}o(uu,"listScrollParents");function $p(e){return Object.assign({},e,{left:e.x,top:e.y,right:e.x+e.width,bottom:e.y+e.height})}o($p,"rectToClientRect");function h3(e){var t=Mo(e);return t.top=t.top+e.clientTop,t.left=t.left+e.clientLeft,t.bottom=t.top+e.clientHeight,t.right=t.left+e.clientWidth,t.width=e.clientWidth,t.height=e.clientHeight,t.x=t.left,t.y=t.top,t}o(h3,"getInnerBoundingClientRect");function JN(e,t){return t===J0?$p($S(e)):Yr(t)?h3(t):$p(jS(Bn(e)))}o(JN,"getClientRectFromMixedType");function m3(e){var t=uu(Fl(e)),n=["absolute","fixed"].indexOf(wi(e).position)>=0,l=n&&Yr(e)?as(e):e;return Il(l)?t.filter(function(d){return Il(d)&&fm(d,l)&&Pn(d)!=="body"}):[]}o(m3,"getClippingParents");function qS(e,t,n){var l=t==="clippingParents"?m3(e):[].concat(t),d=[].concat(l,[n]),h=d[0],c=d.reduce(function(v,C){var k=JN(e,C);return v.top=Ao(k.top,v.top),v.right=au(k.right,v.right),v.bottom=au(k.bottom,v.bottom),v.left=Ao(k.left,v.left),v},JN(e,h));return c.width=c.right-c.left,c.height=c.bottom-c.top,c.x=c.left,c.y=c.top,c}o(qS,"getClippingRect");function Qs(e){return e.split("-")[1]}o(Qs,"getVariation");function mm(e){var t=e.reference,n=e.element,l=e.placement,d=l?On(l):null,h=l?Qs(l):null,c=t.x+t.width/2-n.width/2,v=t.y+t.height/2-n.height/2,C;switch(d){case Gr:C={x:c,y:t.y-n.height};break;case Ln:C={x:c,y:t.y+t.height};break;case ln:C={x:t.x+t.width,y:v};break;case an:C={x:t.x-n.width,y:v};break;default:C={x:t.x,y:t.y}}var k=d?Ff(d):null;if(k!=null){var O=k==="y"?"height":"width";switch(h){case Rl:C[k]=C[k]-(t[O]/2-n[O]/2);break;case Z0:C[k]=C[k]+(t[O]/2-n[O]/2);break;default:}}return C}o(mm,"computeOffsets");function us(e,t){t===void 0&&(t={});var n=t,l=n.placement,d=l===void 0?e.placement:l,h=n.boundary,c=h===void 0?jN:h,v=n.rootBoundary,C=v===void 0?J0:v,k=n.elementContext,O=k===void 0?Up:k,j=n.altBoundary,B=j===void 0?!1:j,X=n.padding,J=X===void 0?0:X,Z=dm(typeof J!="number"?J:hm(J,su)),R=O===Up?qN:Up,A=e.elements.reference,I=e.rects.popper,G=e.elements[B?R:O],K=qS(Il(G)?G:G.contextElement||Bn(e.elements.popper),c,C),se=Mo(A),ne=mm({reference:se,element:I,strategy:"absolute",placement:d}),pe=$p(Object.assign({},I,ne)),me=O===Up?pe:se,xe={top:K.top-me.top+Z.top,bottom:me.bottom-K.bottom+Z.bottom,left:K.left-me.left+Z.left,right:me.right-K.right+Z.right},Ve=e.modifiersData.offset;if(O===Up&&Ve){var tt=Ve[d];Object.keys(xe).forEach(function(_e){var St=[ln,Ln].indexOf(_e)>=0?1:-1,We=[Gr,Ln].indexOf(_e)>=0?"y":"x";xe[_e]+=tt[We]*St})}return xe}o(us,"detectOverflow");function VS(e,t){t===void 0&&(t={});var n=t,l=n.placement,d=n.boundary,h=n.rootBoundary,c=n.padding,v=n.flipVariations,C=n.allowedAutoPlacements,k=C===void 0?ey:C,O=Qs(l),j=O?v?US:US.filter(function(J){return Qs(J)===O}):su,B=j.filter(function(J){return k.indexOf(J)>=0});B.length===0&&(B=j);var X=B.reduce(function(J,Z){return J[Z]=us(e,{placement:Z,boundary:d,rootBoundary:h,padding:c})[On(Z)],J},{});return Object.keys(X).sort(function(J,Z){return X[J]-X[Z]})}o(VS,"computeAutoPlacement");function g3(e){if(On(e)===Q0)return[];var t=zp(e);return[ny(e),t,ny(t)]}o(g3,"getExpandedFallbackPlacements");function v3(e){var t=e.state,n=e.options,l=e.name;if(!t.modifiersData[l]._skip){for(var d=n.mainAxis,h=d===void 0?!0:d,c=n.altAxis,v=c===void 0?!0:c,C=n.fallbackPlacements,k=n.padding,O=n.boundary,j=n.rootBoundary,B=n.altBoundary,X=n.flipVariations,J=X===void 0?!0:X,Z=n.allowedAutoPlacements,R=t.options.placement,A=On(R),I=A===R,G=C||(I||!J?[zp(R)]:g3(R)),K=[R].concat(G).reduce(function(ut,Lr){return ut.concat(On(Lr)===Q0?VS(t,{placement:Lr,boundary:O,rootBoundary:j,padding:k,flipVariations:J,allowedAutoPlacements:Z}):Lr)},[]),se=t.rects.reference,ne=t.rects.popper,pe=new Map,me=!0,xe=K[0],Ve=0;Ve=0,Ke=We?"width":"height",Ge=us(t,{placement:tt,boundary:O,rootBoundary:j,altBoundary:B,padding:k}),Xe=We?St?ln:an:St?Ln:Gr;se[Ke]>ne[Ke]&&(Xe=zp(Xe));var nr=zp(Xe),ct=[];if(h&&ct.push(Ge[_e]<=0),v&&ct.push(Ge[Xe]<=0,Ge[nr]<=0),ct.every(function(ut){return ut})){xe=tt,me=!1;break}pe.set(tt,ct)}if(me)for(var Hr=J?3:1,Qt=o(function(Lr){var zt=K.find(function($t){var ie=pe.get($t);if(ie)return ie.slice(0,Lr).every(function(rt){return rt})});if(zt)return xe=zt,"break"},"_loop"),_t=Hr;_t>0;_t--){var Ct=Qt(_t);if(Ct==="break")break}t.placement!==xe&&(t.modifiersData[l]._skip=!0,t.placement=xe,t.reset=!0)}}o(v3,"flip");var eL={name:"flip",enabled:!0,phase:"main",fn:v3,requiresIfExists:["offset"],data:{_skip:!1}};function tL(e,t,n){return n===void 0&&(n={x:0,y:0}),{top:e.top-t.height-n.y,right:e.right-t.width+n.x,bottom:e.bottom-t.height+n.y,left:e.left-t.width-n.x}}o(tL,"getSideOffsets");function rL(e){return[Gr,ln,Ln,an].some(function(t){return e[t]>=0})}o(rL,"isAnySideFullyClipped");function y3(e){var t=e.state,n=e.name,l=t.rects.reference,d=t.rects.popper,h=t.modifiersData.preventOverflow,c=us(t,{elementContext:"reference"}),v=us(t,{altBoundary:!0}),C=tL(c,l),k=tL(v,d,h),O=rL(C),j=rL(k);t.modifiersData[n]={referenceClippingOffsets:C,popperEscapeOffsets:k,isReferenceHidden:O,hasPopperEscaped:j},t.attributes.popper=Object.assign({},t.attributes.popper,{"data-popper-reference-hidden":O,"data-popper-escaped":j})}o(y3,"hide");var nL={name:"hide",enabled:!0,phase:"main",requiresIfExists:["preventOverflow"],fn:y3};function w3(e,t,n){var l=On(e),d=[an,Gr].indexOf(l)>=0?-1:1,h=typeof n=="function"?n(Object.assign({},t,{placement:e})):n,c=h[0],v=h[1];return c=c||0,v=(v||0)*d,[an,ln].indexOf(l)>=0?{x:v,y:c}:{x:c,y:v}}o(w3,"distanceAndSkiddingToXY");function x3(e){var t=e.state,n=e.options,l=e.name,d=n.offset,h=d===void 0?[0,0]:d,c=ey.reduce(function(O,j){return O[j]=w3(j,t.rects,h),O},{}),v=c[t.placement],C=v.x,k=v.y;t.modifiersData.popperOffsets!=null&&(t.modifiersData.popperOffsets.x+=C,t.modifiersData.popperOffsets.y+=k),t.modifiersData[l]=c}o(x3,"offset");var iL={name:"offset",enabled:!0,phase:"main",requires:["popperOffsets"],fn:x3};function S3(e){var t=e.state,n=e.name;t.modifiersData[n]=mm({reference:t.rects.reference,element:t.rects.popper,strategy:"absolute",placement:t.placement})}o(S3,"popperOffsets");var oL={name:"popperOffsets",enabled:!0,phase:"read",fn:S3,data:{}};function KS(e){return e==="x"?"y":"x"}o(KS,"getAltAxis");function C3(e){var t=e.state,n=e.options,l=e.name,d=n.mainAxis,h=d===void 0?!0:d,c=n.altAxis,v=c===void 0?!1:c,C=n.boundary,k=n.rootBoundary,O=n.altBoundary,j=n.padding,B=n.tether,X=B===void 0?!0:B,J=n.tetherOffset,Z=J===void 0?0:J,R=us(t,{boundary:C,rootBoundary:k,padding:j,altBoundary:O}),A=On(t.placement),I=Qs(t.placement),G=!I,K=Ff(A),se=KS(K),ne=t.modifiersData.popperOffsets,pe=t.rects.reference,me=t.rects.popper,xe=typeof Z=="function"?Z(Object.assign({},t.rects,{placement:t.placement})):Z,Ve={x:0,y:0};if(!!ne){if(h||v){var tt=K==="y"?Gr:an,_e=K==="y"?Ln:ln,St=K==="y"?"height":"width",We=ne[K],Ke=ne[K]+R[tt],Ge=ne[K]-R[_e],Xe=X?-me[St]/2:0,nr=I===Rl?pe[St]:me[St],ct=I===Rl?-me[St]:-pe[St],Hr=t.elements.arrow,Qt=X&&Hr?If(Hr):{width:0,height:0},_t=t.modifiersData["arrow#persistent"]?t.modifiersData["arrow#persistent"].padding:pm(),Ct=_t[tt],ut=_t[_e],Lr=Bf(0,pe[St],Qt[St]),zt=G?pe[St]/2-Xe-Lr-Ct-xe:nr-Lr-Ct-xe,$t=G?-pe[St]/2+Xe+Lr+ut+xe:ct+Lr+ut+xe,ie=t.elements.arrow&&as(t.elements.arrow),rt=ie?K==="y"?ie.clientTop||0:ie.clientLeft||0:0,Pr=t.modifiersData.offset?t.modifiersData.offset[t.placement][K]:0,Gt=ne[K]+zt-Pr-rt,Yt=ne[K]+$t-Pr;if(h){var Se=Bf(X?au(Ke,Gt):Ke,We,X?Ao(Ge,Yt):Ge);ne[K]=Se,Ve[K]=Se-We}if(v){var Or=K==="x"?Gr:an,fn=K==="x"?Ln:ln,Un=ne[se],si=Un+R[Or],cn=Un-R[fn],Zt=Bf(X?au(si,Gt):si,Un,X?Ao(cn,Yt):cn);ne[se]=Zt,Ve[se]=Zt-Un}}t.modifiersData[l]=Ve}}o(C3,"preventOverflow");var sL={name:"preventOverflow",enabled:!0,phase:"main",fn:C3,requiresIfExists:["offset"]};function GS(e){return{scrollLeft:e.scrollLeft,scrollTop:e.scrollTop}}o(GS,"getHTMLElementScroll");function YS(e){return e===Br(e)||!Yr(e)?Hf(e):GS(e)}o(YS,"getNodeScroll");function b3(e){var t=e.getBoundingClientRect(),n=t.width/e.offsetWidth||1,l=t.height/e.offsetHeight||1;return n!==1||l!==1}o(b3,"isElementScaled");function XS(e,t,n){n===void 0&&(n=!1);var l=Yr(t),d=Yr(t)&&b3(t),h=Bn(t),c=Mo(e,d),v={scrollLeft:0,scrollTop:0},C={x:0,y:0};return(l||!l&&!n)&&((Pn(t)!=="body"||Uf(h))&&(v=YS(t)),Yr(t)?(C=Mo(t,!0),C.x+=t.clientLeft,C.y+=t.clientTop):h&&(C.x=Wf(h))),{x:c.left+v.scrollLeft-C.x,y:c.top+v.scrollTop-C.y,width:c.width,height:c.height}}o(XS,"getCompositeRect");function E3(e){var t=new Map,n=new Set,l=[];e.forEach(function(h){t.set(h.name,h)});function d(h){n.add(h.name);var c=[].concat(h.requires||[],h.requiresIfExists||[]);c.forEach(function(v){if(!n.has(v)){var C=t.get(v);C&&d(C)}}),l.push(h)}return o(d,"sort"),e.forEach(function(h){n.has(h.name)||d(h)}),l}o(E3,"order");function QS(e){var t=E3(e);return VN.reduce(function(n,l){return n.concat(t.filter(function(d){return d.phase===l}))},[])}o(QS,"orderModifiers");function ZS(e){var t;return function(){return t||(t=new Promise(function(n){Promise.resolve().then(function(){t=void 0,n(e())})})),t}}o(ZS,"debounce");function JS(e){var t=e.reduce(function(n,l){var d=n[l.name];return n[l.name]=d?Object.assign({},d,l,{options:Object.assign({},d.options,l.options),data:Object.assign({},d.data,l.data)}):l,n},{});return Object.keys(t).map(function(n){return t[n]})}o(JS,"mergeByName");var lL={placement:"bottom",modifiers:[],strategy:"absolute"};function aL(){for(var e=arguments.length,t=new Array(e),n=0;nxi.default.createElement("li",{role:"separator",className:"divider"}),"Divider");function ii(l){var d=l,{onClick:e,children:t}=d,n=Ws(d,["onClick","children"]);return xi.default.createElement("li",null,xi.default.createElement("a",ke({href:"#",onClick:o(c=>{c.preventDefault(),e()},"click")},n),t))}o(ii,"MenuItem");var cu=xi.default.memo(o(function(v){var C=v,{text:t,children:n,options:l,className:d,onOpen:h}=C,c=Ws(C,["text","children","options","className","onOpen"]);let[k,O]=(0,xi.useState)(null),[j,B]=(0,xi.useState)(!1),[X,J]=(0,xi.useState)(null),{styles:Z,attributes:R}=tC(k,X,ke({},l)),A=o(G=>{B(G),h&&h(G)},"setOpen");(0,xi.useEffect)(()=>{!X||document.addEventListener("click",G=>{X.contains(G.target)?document.addEventListener("click",()=>A(!1),{once:!0}):(G.preventDefault(),G.stopPropagation(),A(!1))},{once:!0,capture:!0})},[X]);let I;return j?I=xi.default.createElement("ul",ke({className:"dropdown-menu show",ref:J,style:Z.popper},R.popper),n):I=null,xi.default.createElement(xi.default.Fragment,null,xi.default.createElement("a",ke({href:"#",ref:O,className:(0,dL.default)(d,{open:j}),onClick:G=>{G.preventDefault(),A(!0)}},c),t),I)},"Dropdown"));function gm({value:e,onChange:t}){let n=qe(d=>d.backendState.contentViews||[]),l=zf.default.createElement("span",null,zf.default.createElement("i",{className:"fa fa-fw fa-files-o"}),"\xA0",zf.default.createElement("b",null,"View:")," ",e.toLowerCase()," ",zf.default.createElement("span",{className:"caret"}));return zf.default.createElement(cu,{text:l,className:"btn btn-default btn-xs",options:{placement:"top-start"}},n.map(d=>zf.default.createElement(ii,{key:d,onClick:()=>t(d)},d.toLowerCase().replace("_"," "))))}o(gm,"ViewSelector");function rC({flow:e,message:t}){let n=Xt(),l=e.request===t?"request":"response",d=qe(J=>J.ui.flow.contentViewFor[e.id+l]||"Auto"),h=(0,tr.useRef)(null),[c,v]=(0,tr.useState)(qe(J=>J.options.content_view_lines_cutoff)),C=(0,tr.useCallback)(()=>v(Math.max(1024,c*2)),[c]),[k,O]=(0,tr.useState)(!1),j;k?j=Kr.getContentURL(e,t):j=Kr.getContentURL(e,t,d,c+1);let B=K0(j,t.contentHash),X=(0,tr.useMemo)(()=>{if(B&&!k)try{return JSON.parse(B)}catch(J){return{description:"Network Error",lines:[[["error",`${B}`]]]}}else return},[B]);if(k)return tr.default.createElement("div",{className:"contentview",key:"edit"},tr.default.createElement("div",{className:"controls"},tr.default.createElement("h5",null,"[Editing]"),tr.default.createElement(kr,{onClick:o(()=>Ia(this,null,function*(){var R;let Z=(R=h.current)==null?void 0:R.getContent();yield n(Wi(e,{[l]:{content:Z}})),O(!1)}),"save"),icon:"fa-check text-success",className:"btn-xs"},"Done"),"\xA0",tr.default.createElement(kr,{onClick:()=>O(!1),icon:"fa-times text-danger",className:"btn-xs"},"Cancel")),tr.default.createElement(um,{ref:h,initialContent:B||""}));{let J=X?X.description:"Loading...";return tr.default.createElement("div",{className:"contentview",key:"view"},tr.default.createElement("div",{className:"controls"},tr.default.createElement("h5",null,J),tr.default.createElement(kr,{onClick:()=>O(!0),icon:"fa-edit",className:"btn-xs"},"Edit"),"\xA0",tr.default.createElement(G0,{icon:"fa-upload",text:"Replace",title:"Upload a file to replace the content.",onOpenFile:Z=>n(nN(e,Z,l)),className:"btn btn-default btn-xs"}),"\xA0",tr.default.createElement(gm,{value:d,onChange:Z=>n(w0(e.id+l,Z))})),nC.matches(t)&&tr.default.createElement(nC,{flow:e,message:t}),tr.default.createElement(Y0,{lines:(X==null?void 0:X.lines)||[],maxLines:c,showMore:C}))}}o(rC,"HttpMessage");var O3=/^image\/(png|jpe?g|gif|webp|vnc.microsoft.icon|x-icon)$/i;nC.matches=e=>O3.test(Kr.getContentType(e)||"");function nC({flow:e,message:t}){return tr.default.createElement("div",{className:"flowview-image"},tr.default.createElement("img",{src:Kr.getContentURL(e,t),alt:"preview",className:"img-thumbnail"}))}o(nC,"ViewImage");function M3({flow:e}){let t=Xt();return Ut.createElement("div",{className:"first-line request-line"},Ut.createElement("div",null,Ut.createElement(Df,{content:e.request.method,onEditDone:n=>t(Wi(e,{request:{method:n}})),isValid:n=>n.length>0}),"\xA0",Ut.createElement(Df,{content:Oo.pretty_url(e.request),onEditDone:n=>t(Wi(e,{request:ke({path:""},gS(n))})),isValid:n=>{var l;return!!((l=gS(n))==null?void 0:l.host)}}),"\xA0",Ut.createElement(Df,{content:e.request.http_version,onEditDone:n=>t(Wi(e,{request:{http_version:n}})),isValid:vS})))}o(M3,"RequestLine");function A3({flow:e}){let t=Xt();return Ut.createElement("div",{className:"first-line response-line"},Ut.createElement(Df,{content:e.response.http_version,onEditDone:n=>t(Wi(e,{response:{http_version:n}})),isValid:vS}),"\xA0",Ut.createElement(Df,{content:e.response.status_code+"",onEditDone:n=>t(Wi(e,{response:{code:parseInt(n)}})),isValid:n=>/^\d+$/.test(n)}),e.response.http_version!=="HTTP/2.0"&&Ut.createElement(Ut.Fragment,null,"\xA0",Ut.createElement(Xs,{content:e.response.reason,onEditDone:n=>t(Wi(e,{response:{msg:n}}))})))}o(A3,"ResponseLine");function D3({flow:e,message:t}){let n=Xt(),l=e.request===t?"request":"response";return Ut.createElement(Bp,{className:"headers",data:t.headers,onChange:d=>n(Wi(e,{[l]:{headers:d}}))})}o(D3,"Headers");function R3({flow:e,message:t}){let n=Xt(),l=e.request===t?"request":"response";return!Kr.get_first_header(t,/^trailer$/i)?null:Ut.createElement(Ut.Fragment,null,Ut.createElement("hr",null),Ut.createElement("h5",null,"HTTP Trailers"),Ut.createElement(Bp,{className:"trailers",data:t.trailers,onChange:h=>n(Wi(e,{[l]:{trailers:h}}))}))}o(R3,"Trailers");var mL=Ut.memo(o(function({flow:t,message:n}){let l=t.request===n?"request":"response",d=t.request===n?M3:A3;return Ut.createElement("section",{className:l},Ut.createElement(d,{flow:t}),Ut.createElement(D3,{flow:t,message:n}),Ut.createElement("hr",null),Ut.createElement(rC,{key:t.id+l,flow:t,message:n}),Ut.createElement(R3,{flow:t,message:n}))},"Message"));function iC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ut.createElement(mL,{flow:e,message:e.request})}o(iC,"Request");iC.displayName="Request";function oC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ut.createElement(mL,{flow:e,message:e.response})}o(oC,"Response");oC.displayName="Response";var Ye=fe(Oe());var I3=o(({message:e})=>Ye.createElement("div",null,e.query?e.op_code:e.response_code,"\xA0",e.truncation?"(Truncated)":""),"Summary"),F3=o(({message:e})=>Ye.createElement(Ye.Fragment,null,Ye.createElement("h5",null,e.recursion_desired?"Recursive ":"","Question"),Ye.createElement("table",null,Ye.createElement("thead",null,Ye.createElement("tr",null,Ye.createElement("th",null,"Name"),Ye.createElement("th",null,"Type"),Ye.createElement("th",null,"Class"))),Ye.createElement("tbody",null,e.questions.map((t,n)=>Ye.createElement("tr",{key:n},Ye.createElement("td",null,t.name),Ye.createElement("td",null,t.type),Ye.createElement("td",null,t.class)))))),"Questions"),sC=o(({name:e,values:t})=>Ye.createElement(Ye.Fragment,null,Ye.createElement("h5",null,e),t.length>0?Ye.createElement("table",null,Ye.createElement("thead",null,Ye.createElement("tr",null,Ye.createElement("th",null,"Name"),Ye.createElement("th",null,"Type"),Ye.createElement("th",null,"Class"),Ye.createElement("th",null,"TTL"),Ye.createElement("th",null,"Data"))),Ye.createElement("tbody",null,t.map((n,l)=>Ye.createElement("tr",{key:l},Ye.createElement("td",null,n.name),Ye.createElement("td",null,n.type),Ye.createElement("td",null,n.class),Ye.createElement("td",null,n.ttl),Ye.createElement("td",null,n.data))))):"\u2014"),"ResourceRecords"),gL=o(({type:e,message:t})=>Ye.createElement("section",{className:"dns-"+e},Ye.createElement("div",{className:`first-line ${e}-line`},Ye.createElement(I3,{message:t})),Ye.createElement(F3,{message:t}),Ye.createElement("hr",null),Ye.createElement(sC,{name:`${t.authoritative_answer?"Authoritative ":""}${t.recursion_available?"Recursive ":""}Answer`,values:t.answers}),Ye.createElement("hr",null),Ye.createElement(sC,{name:"Authority",values:t.authorities}),Ye.createElement("hr",null),Ye.createElement(sC,{name:"Additional",values:t.additionals})),"Message");function lC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ye.createElement(gL,{type:"request",message:e.request})}o(lC,"Request");lC.displayName="Request";function aC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ye.createElement(gL,{type:"response",message:e.response})}o(aC,"Response");aC.displayName="Response";var Ee=fe(Oe());function vL({conn:e}){var n,l,d;let t=null;return"address"in e?t=Ee.createElement(Ee.Fragment,null,Ee.createElement("tr",null,Ee.createElement("td",null,"Address:"),Ee.createElement("td",null,(n=e.address)==null?void 0:n.join(":"))),e.peername&&Ee.createElement("tr",null,Ee.createElement("td",null,"Resolved address:"),Ee.createElement("td",null,e.peername.join(":"))),e.sockname&&Ee.createElement("tr",null,Ee.createElement("td",null,"Source address:"),Ee.createElement("td",null,e.sockname.join(":")))):((l=e.peername)==null?void 0:l[0])&&(t=Ee.createElement(Ee.Fragment,null,Ee.createElement("tr",null,Ee.createElement("td",null,"Address:"),Ee.createElement("td",null,(d=e.peername)==null?void 0:d.join(":"))))),Ee.createElement("table",{className:"connection-table"},Ee.createElement("tbody",null,t,e.sni?Ee.createElement("tr",null,Ee.createElement("td",null,Ee.createElement("abbr",{title:"TLS Server Name Indication"},"SNI"),":"),Ee.createElement("td",null,e.sni)):null,e.alpn?Ee.createElement("tr",null,Ee.createElement("td",null,Ee.createElement("abbr",{title:"ALPN protocol negotiated"},"ALPN"),":"),Ee.createElement("td",null,e.alpn)):null,e.tls_version?Ee.createElement("tr",null,Ee.createElement("td",null,"TLS Version:"),Ee.createElement("td",null,e.tls_version)):null,e.cipher?Ee.createElement("tr",null,Ee.createElement("td",null,"TLS Cipher:"),Ee.createElement("td",null,e.cipher)):null))}o(vL,"ConnectionInfo");function yL(e){return Ee.createElement("dl",{className:"cert-attributes"},e.map(([t,n])=>Ee.createElement(Ee.Fragment,{key:t},Ee.createElement("dt",null,t),Ee.createElement("dd",null,n))))}o(yL,"attrList");function B3({flow:e}){var n;let t=(n=e.server_conn)==null?void 0:n.cert;return t?Ee.createElement(Ee.Fragment,null,Ee.createElement("h4",{key:"name"},"Server Certificate"),Ee.createElement("table",{className:"certificate-table"},Ee.createElement("tbody",null,Ee.createElement("tr",null,Ee.createElement("td",null,"Type"),Ee.createElement("td",null,t.keyinfo[0],", ",t.keyinfo[1]," bits")),Ee.createElement("tr",null,Ee.createElement("td",null,"SHA256 digest"),Ee.createElement("td",null,t.sha256)),Ee.createElement("tr",null,Ee.createElement("td",null,"Valid from"),Ee.createElement("td",null,no(t.notbefore,{milliseconds:!1}))),Ee.createElement("tr",null,Ee.createElement("td",null,"Valid to"),Ee.createElement("td",null,no(t.notafter,{milliseconds:!1}))),Ee.createElement("tr",null,Ee.createElement("td",null,"Subject Alternative Names"),Ee.createElement("td",null,t.altnames.join(", "))),Ee.createElement("tr",null,Ee.createElement("td",null,"Subject"),Ee.createElement("td",null,yL(t.subject))),Ee.createElement("tr",null,Ee.createElement("td",null,"Issuer"),Ee.createElement("td",null,yL(t.issuer))),Ee.createElement("tr",null,Ee.createElement("td",null,"Serial"),Ee.createElement("td",null,t.serial))))):Ee.createElement(Ee.Fragment,null)}o(B3,"CertificateInfo");function sy({flow:e}){var t;return Ee.createElement("section",{className:"detail"},Ee.createElement("h4",null,"Client Connection"),Ee.createElement(vL,{conn:e.client_conn}),((t=e.server_conn)==null?void 0:t.address)&&Ee.createElement(Ee.Fragment,null,Ee.createElement("h4",null,"Server Connection"),Ee.createElement(vL,{conn:e.server_conn})),Ee.createElement(B3,{flow:e}))}o(sy,"Connection");sy.displayName="Connection";var vm=fe(Oe());function ly({flow:e}){return vm.createElement("section",{className:"error"},vm.createElement("div",{className:"alert alert-warning"},e.error.msg,vm.createElement("div",null,vm.createElement("small",null,no(e.error.timestamp)))))}o(ly,"Error");ly.displayName="Error";var fs=fe(Oe());function H3({t:e,deltaTo:t,title:n}){return e?fs.createElement("tr",null,fs.createElement("td",null,n,":"),fs.createElement("td",null,no(e),t&&fs.createElement("span",{className:"text-muted"},"(",S0(1e3*(e-t)),")"))):fs.createElement("tr",null)}o(H3,"TimeStamp");function ay({flow:e}){var l,d,h,c,v,C;let t;e.type==="http"?t=e.request.timestamp_start:t=e.client_conn.timestamp_start;let n=[{title:"Server conn. initiated",t:(l=e.server_conn)==null?void 0:l.timestamp_start,deltaTo:t},{title:"Server conn. TCP handshake",t:(d=e.server_conn)==null?void 0:d.timestamp_tcp_setup,deltaTo:t},{title:"Server conn. TLS handshake",t:(h=e.server_conn)==null?void 0:h.timestamp_tls_setup,deltaTo:t},{title:"Server conn. closed",t:(c=e.server_conn)==null?void 0:c.timestamp_end,deltaTo:t},{title:"Client conn. established",t:e.client_conn.timestamp_start,deltaTo:e.type==="http"?t:void 0},{title:"Client conn. TLS handshake",t:e.client_conn.timestamp_tls_setup,deltaTo:t},{title:"Client conn. closed",t:e.client_conn.timestamp_end,deltaTo:t}];return e.type==="http"&&n.push({title:"First request byte",t:e.request.timestamp_start},{title:"Request complete",t:e.request.timestamp_end,deltaTo:t},{title:"First response byte",t:(v=e.response)==null?void 0:v.timestamp_start,deltaTo:t},{title:"Response complete",t:(C=e.response)==null?void 0:C.timestamp_end,deltaTo:t}),fs.createElement("section",{className:"timing"},fs.createElement("h4",null,"Timing"),fs.createElement("table",{className:"timing-table"},fs.createElement("tbody",null,n.filter(k=>!!k.t).sort((k,O)=>k.t-O.t).map(k=>fs.createElement(H3,ke({key:k.title},k))))))}o(ay,"Timing");ay.displayName="Timing";var pu=fe(Oe());var Zs=fe(Oe()),jp=fe(Oe());function $f({flow:e,messages_meta:t}){let n=Xt(),l=qe(k=>k.ui.flow.contentViewFor[e.id+"messages"]||"Auto"),[d,h]=(0,jp.useState)(qe(k=>k.options.content_view_lines_cutoff)),c=(0,jp.useCallback)(()=>h(Math.max(1024,d*2)),[d]),v=K0(Kr.getContentURL(e,"messages",l,d+1),e.id+t.count),C=(0,jp.useMemo)(()=>{if(v)try{return JSON.parse(v)}catch(k){return{description:"Network Error",lines:[[["error",`${v}`]]]}}},[v])||[];return Zs.createElement("div",{className:"contentview"},Zs.createElement("div",{className:"controls"},Zs.createElement("h5",null,t.count," Messages"),Zs.createElement(gm,{value:l,onChange:k=>n(w0(e.id+"messages",k))})),C.map((k,O)=>{let j=`fa fa-fw fa-arrow-${k.from_client?"right text-primary":"left text-danger"}`,B=Zs.createElement("div",{key:O},Zs.createElement("small",null,Zs.createElement("i",{className:j}),Zs.createElement("span",{className:"pull-right"},k.timestamp&&no(k.timestamp))),Zs.createElement(Y0,{lines:k.lines,maxLines:d,showMore:c}));return d-=k.lines.length,B}))}o($f,"Messages");function uy({flow:e}){return pu.createElement("section",{className:"websocket"},pu.createElement("h4",null,"WebSocket"),pu.createElement($f,{flow:e,messages_meta:e.websocket.messages_meta}),pu.createElement(W3,{websocket:e.websocket}))}o(uy,"WebSocket");uy.displayName="WebSocket";function W3({websocket:e}){if(!e.timestamp_end)return null;let t=e.close_reason?`(${e.close_reason})`:"";return pu.createElement("div",null,pu.createElement("i",{className:"fa fa-fw fa-window-close text-muted"}),"\xA0 Closed by ",e.closed_by_client?"client":"server"," with code ",e.close_code," ",t,".",pu.createElement("small",{className:"pull-right"},no(e.timestamp_end)))}o(W3,"CloseSummary");var wL=fe(ti());var fy=fe(Oe());function cy({flow:e}){return fy.createElement("section",{className:"tcp"},fy.createElement("h4",null,"TCP Data"),fy.createElement($f,{flow:e,messages_meta:e.messages_meta}))}o(cy,"TcpMessages");cy.displayName="TCP Messages";var py=fe(Oe());function dy({flow:e}){return py.createElement("section",{className:"udp"},py.createElement("h4",null,"UDP Data"),py.createElement($f,{flow:e,messages_meta:e.messages_meta}))}o(dy,"UdpMessages");dy.displayName="UDP Messages";var xL={request:iC,response:oC,error:ly,connection:sy,timing:ay,websocket:uy,tcpmessages:cy,udpmessages:dy,dnsrequest:lC,dnsresponse:aC};function hy(e){let t;switch(e.type){case"http":t=["request","response","websocket"].filter(n=>e[n]);break;case"tcp":t=["tcpmessages"];break;case"udp":t=["udpmessages"];break;case"dns":t=["request","response"].filter(n=>e[n]).map(n=>"dns"+n);break}return e.error&&t.push("error"),t.push("connection"),t.push("timing"),t}o(hy,"tabsForFlow");function uC(){let e=Xt(),t=qe(h=>h.flows.byId[h.flows.selected[0]]),n=hy(t),l=qe(h=>h.ui.flow.tab);n.indexOf(l)<0&&(l==="response"&&t.error?l="error":l==="error"&&"response"in t?l="response":l=n[0]);let d=xL[l];return ym.createElement("div",{className:"flow-detail"},ym.createElement("nav",{className:"nav-tabs nav-tabs-sm"},n.map(h=>ym.createElement("a",{key:h,href:"#",className:(0,wL.default)({active:l===h}),onClick:c=>{c.preventDefault(),e(Lf(h))}},xL[h].displayName))),ym.createElement(d,{flow:t}))}o(uC,"FlowView");function SL(e){if(e.ctrlKey||e.metaKey)return()=>{};let t=e.key;return e.preventDefault(),(n,l)=>{let d=l().flows,h=d.byId[l().flows.selected[0]];switch(t){case"k":case"ArrowUp":n(Mf(d,-1));break;case"j":case"ArrowDown":n(Mf(d,1));break;case" ":case"PageDown":n(Mf(d,10));break;case"PageUp":n(Mf(d,-10));break;case"End":n(Mf(d,1e10));break;case"Home":n(Mf(d,-1e10));break;case"Escape":l().ui.modal.activeModal?n(z0()):n(Af(void 0));break;case"ArrowLeft":{if(!h)break;let c=hy(h),v=l().ui.flow.tab,C=c[(Math.max(0,c.indexOf(v))-1+c.length)%c.length];n(Lf(C));break}case"Tab":case"ArrowRight":{if(!h)break;let c=hy(h),v=l().ui.flow.tab,C=c[(Math.max(0,c.indexOf(v))+1)%c.length];n(Lf(C));break}case"d":{if(!h)return;n(F0(h));break}case"n":{Pf("view.flows.create","get","https://example.com/");break}case"D":{if(!h)return;n(B0(h));break}case"a":{h&&h.intercepted&&n(Op(h));break}case"A":{n(R0());break}case"r":{h&&n(Mp(h));break}case"v":{h&&h.modified&&n(H0(h));break}case"x":{h&&h.intercepted&&n(I0(h));break}case"X":{n(rN());break}case"z":{n(W0());break}default:return}}}o(SL,"onKeyDown");var Zp=fe(Oe());var wm=fe(Oe()),xm=fe(iu()),CL=fe(ti()),qp=class extends wm.Component{constructor(t,n){super(t,n);this.state={applied:!1,startX:0,startY:0},this.onMouseMove=this.onMouseMove.bind(this),this.onMouseDown=this.onMouseDown.bind(this),this.onMouseUp=this.onMouseUp.bind(this),this.onDragEnd=this.onDragEnd.bind(this)}onMouseDown(t){this.setState({startX:t.pageX,startY:t.pageY}),window.addEventListener("mousemove",this.onMouseMove),window.addEventListener("mouseup",this.onMouseUp),window.addEventListener("dragend",this.onDragEnd)}onDragEnd(){xm.default.findDOMNode(this).style.transform="",window.removeEventListener("dragend",this.onDragEnd),window.removeEventListener("mouseup",this.onMouseUp),window.removeEventListener("mousemove",this.onMouseMove)}onMouseUp(t){this.onDragEnd();let n=xm.default.findDOMNode(this),l=n.previousElementSibling,d=l.offsetHeight+t.pageY-this.state.startY;this.props.axis==="x"&&(d=l.offsetWidth+t.pageX-this.state.startX),l.style.flex=`0 0 ${Math.max(0,d)}px`,n.nextElementSibling.style.flex="1 1 auto",this.setState({applied:!0}),this.onResize()}onMouseMove(t){let n=0,l=0;this.props.axis==="x"?n=t.pageX-this.state.startX:l=t.pageY-this.state.startY,xm.default.findDOMNode(this).style.transform=`translate(${n}px, ${l}px)`}onResize(){window.setTimeout(()=>window.dispatchEvent(new CustomEvent("resize")),1)}reset(t){if(!this.state.applied)return;let n=xm.default.findDOMNode(this);n.previousElementSibling.style.flex="",n.nextElementSibling.style.flex="",t||this.setState({applied:!1}),this.onResize()}componentWillUnmount(){this.reset(!0)}render(){return wm.default.createElement("div",{className:(0,CL.default)("splitter",this.props.axis==="x"?"splitter-x":"splitter-y")},wm.default.createElement("div",{onMouseDown:this.onMouseDown,draggable:"true"}))}};o(qp,"Splitter"),qp.defaultProps={axis:"x"};var cs=fe(Oe()),bm=fe(Sm()),gy=fe(iu());var FL=fe(fC());var cC=fe(iu()),OL=Symbol("shouldStick"),ML=o(e=>e.scrollTop+e.clientHeight===e.scrollHeight,"isAtBottom"),my=o(e=>{var t;return Object.assign((o(t=class extends e{UNSAFE_componentWillUpdate(){let l=cC.default.findDOMNode(this);this[OL]=l.scrollTop&&ML(l),super.UNSAFE_componentWillUpdate&&super.UNSAFE_componentWillUpdate(),super.componentWillUpdate&&super.componentWillUpdate()}componentDidUpdate(){let l=cC.default.findDOMNode(this);this[OL]&&!ML(l)&&(l.scrollTop=l.scrollHeight),super.componentDidUpdate&&super.componentDidUpdate()}},"AutoScrollWrapper"),t.displayName=e.name,t),e)},"default");function jf(e=void 0){if(!e)return{start:0,end:0,paddingTop:0,paddingBottom:0};let{itemCount:t,rowHeight:n,viewportTop:l,viewportHeight:d,itemHeights:h}=e,c=l+d,v=0,C=0,k=0,O=0;if(h){let j=0;for(let B=0;B0&&jv.flows.sort.desc),l=qe(v=>v.flows.sort.column),d=qe(v=>v.options.web_columns),h=n?"sort-desc":"sort-asc",c=d.map(v=>nm[v]).filter(v=>v).concat(Pp);return Cm.createElement("tr",null,c.map(v=>Cm.createElement("th",{className:(0,AL.default)(`col-${v.name}`,l===v.name&&h),key:v.name,onClick:()=>t(tN(v.name===l&&n?void 0:v.name,v.name!==l?!1:!n))},v.headerName)))},"FlowTableHead"));var Vp=fe(Oe()),RL=fe(ti());var IL=Vp.default.memo(o(function({flow:t,selected:n,highlighted:l}){let d=Xt(),h=qe(k=>k.options.web_columns),c=(0,RL.default)({selected:n,highlighted:l,intercepted:t.intercepted,"has-request":t.type==="http"&&t.request,"has-response":t.type==="http"&&t.response}),v=(0,Vp.useCallback)(k=>{let O=k.target;for(;O.parentNode;){if(O.classList.contains("col-quickactions"))return;O=O.parentNode}d(Af(t.id))},[t]),C=h.map(k=>nm[k]).filter(k=>k).concat(Pp);return Vp.default.createElement("tr",{className:c,onClick:v},C.map(k=>Vp.default.createElement(k,{key:k.name,flow:t})))},"FlowRow"));var Em=class extends cs.Component{constructor(t,n){super(t,n);this.state={vScroll:jf()},this.onViewportUpdate=this.onViewportUpdate.bind(this)}UNSAFE_componentWillMount(){window.addEventListener("resize",this.onViewportUpdate)}componentDidMount(){this.onViewportUpdate()}UNSAFE_componentWillUnmount(){window.removeEventListener("resize",this.onViewportUpdate)}componentDidUpdate(){if(this.onViewportUpdate(),!this.shouldScrollIntoView)return;this.shouldScrollIntoView=!1;let{rowHeight:t,flows:n,selected:l}=this.props,d=gy.default.findDOMNode(this),h=gy.default.findDOMNode(this.refs.head),c=h?h.offsetHeight:0,v=n.indexOf(l)*t+c,C=v+t,k=d.scrollTop,O=d.offsetHeight;v-ck+O&&(d.scrollTop=C-O)}UNSAFE_componentWillReceiveProps(t){t.selected&&t.selected!==this.props.selected&&(this.shouldScrollIntoView=!0)}onViewportUpdate(){let t=gy.default.findDOMNode(this),n=t.scrollTop||0,l=jf({viewportTop:n,viewportHeight:t.offsetHeight||0,itemCount:this.props.flows.length,rowHeight:this.props.rowHeight});if(this.state.viewportTop!==n||!(0,FL.default)(this.state.vScroll,l)){let d=Math.min(n,l.end*this.props.rowHeight);this.setState({vScroll:l,viewportTop:d})}}render(){let{vScroll:t,viewportTop:n}=this.state,{flows:l,selected:d,highlight:h}=this.props,c=h?Of.parse(h):()=>!1;return cs.createElement("div",{className:"flow-table",onScroll:this.onViewportUpdate},cs.createElement("table",null,cs.createElement("thead",{ref:"head",style:{transform:`translateY(${n}px)`}},cs.createElement(DL,null)),cs.createElement("tbody",null,cs.createElement("tr",{style:{height:t.paddingTop}}),l.slice(t.start,t.end).map(v=>cs.createElement(IL,{key:v.id,flow:v,selected:v===d,highlighted:c(v)})),cs.createElement("tr",{style:{height:t.paddingBottom}}))))}};o(Em,"FlowTable"),Vc(Em,"propTypes",{flows:bm.default.array.isRequired,rowHeight:bm.default.number,highlight:bm.default.string,selected:bm.default.object}),Vc(Em,"defaultProps",{rowHeight:32});var $3=my(Em),BL=Hi(e=>({flows:e.flows.view,highlight:e.flows.highlight,selected:e.flows.byId[e.flows.selected[0]]}))($3);var Nr=fe(Oe()),Ny=fe(Oe());var UP=fe(WP());function RC(){let e=qe(n=>n.backendState.servers),t;return e.length===0?t="":e.length===1?t="Configure your client to use the following proxy server:":t="Configure your client to use one of the following proxy servers:",Nr.createElement("div",{style:{padding:"1em 2em"}},Nr.createElement("h3",null,"mitmproxy is running."),Nr.createElement("p",null,"No flows have been recorded yet.",Nr.createElement("br",null),t),Nr.createElement("ul",{className:"fa-ul"},e.map((n,l)=>Nr.createElement("li",{key:n.full_spec},Nr.createElement(BB,ke({},n))))))}o(RC,"CaptureSetup");function BB({description:e,listen_addrs:t,last_exception:n,is_running:l,full_spec:d,wireguard_conf:h}){let c=(0,Ny.useRef)(null);(0,Ny.useEffect)(()=>{h&&c.current&&UP.default.toCanvas(c.current,h,{margin:0,scale:3})},[h]);let v,C=t.length===1||t.length===2&&t[0][1]===t[1][1],k=t.every(B=>["::","0.0.0.0"].includes(B[0]));C&&k?v=iS(["*",t[0][1]]):v=t.map(iS).join(" and "),e=e[0].toUpperCase()+e.substr(1);let O,j;return n?(j="fa-exclamation text-error",O=Nr.createElement(Nr.Fragment,null,e," (",d,"):",Nr.createElement("br",null),n)):l?(j="fa-check text-success",O=`${e} listening at ${v}.`,h&&(O=Nr.createElement(Nr.Fragment,null,O,Nr.createElement("div",{className:"wireguard-config"},Nr.createElement("pre",null,h),Nr.createElement("canvas",{ref:c}))))):(j="fa-pause text-warning",O=Nr.createElement(Nr.Fragment,null,e," (",d,")")),Nr.createElement(Nr.Fragment,null,Nr.createElement("i",{className:`fa fa-li ${j}`}),O)}o(BB,"ServerDescription");function IC(){let e=qe(n=>!!n.flows.byId[n.flows.selected[0]]),t=qe(n=>n.flows.list.length>0);return Zp.createElement("div",{className:"main-view"},t?Zp.createElement(BL,null):Zp.createElement(RC,null),e&&Zp.createElement(qp,{key:"splitter"}),e&&Zp.createElement(uC,{key:"flowDetails"}))}o(IC,"MainView");var Io=fe(Oe()),GP=fe(ti());var oi=fe(Oe());var ps=fe(Oe()),Ly=fe(iu()),zP=fe(ti());var oo=fe(Oe());var so=class extends oo.Component{constructor(t,n){super(t,n);this.state={doc:so.doc}}componentDidMount(){so.xhr||(so.xhr=kt("/filter-help").then(t=>t.json()),so.xhr.catch(()=>{so.xhr=null})),this.state.doc||so.xhr.then(t=>{so.doc=t,this.setState({doc:t})})}render(){let{doc:t}=this.state;return t?oo.default.createElement("table",{className:"table table-condensed"},oo.default.createElement("tbody",null,t.commands.map(n=>oo.default.createElement("tr",{key:n[1],onClick:l=>this.props.selectHandler(n[0].split(" ")[0]+" ")},oo.default.createElement("td",null,n[0].replace(" ","\xA0")),oo.default.createElement("td",null,n[1]))),oo.default.createElement("tr",{key:"docs-link"},oo.default.createElement("td",{colSpan:2},oo.default.createElement("a",{href:"https://mitmproxy.org/docs/latest/concepts-filters/",target:"_blank"},oo.default.createElement("i",{className:"fa fa-external-link"}),"\xA0 mitmproxy docs"))))):oo.default.createElement("i",{className:"fa fa-spinner fa-spin"})}};o(so,"FilterDocs");var Yf=class extends ps.Component{constructor(t,n){super(t,n);this.state={value:this.props.value,focus:!1,mousefocus:!1},this.onChange=this.onChange.bind(this),this.onFocus=this.onFocus.bind(this),this.onBlur=this.onBlur.bind(this),this.onKeyDown=this.onKeyDown.bind(this),this.onMouseEnter=this.onMouseEnter.bind(this),this.onMouseLeave=this.onMouseLeave.bind(this),this.selectFilter=this.selectFilter.bind(this)}UNSAFE_componentWillReceiveProps(t){this.setState({value:t.value})}isValid(t){try{return t&&Of.parse(t),!0}catch(n){return!1}}getDesc(){if(!this.state.value)return ps.default.createElement(so,{selectHandler:this.selectFilter});try{return Of.parse(this.state.value).desc}catch(t){return""+t}}onChange(t){let n=t.target.value;this.setState({value:n}),this.isValid(n)&&this.props.onChange(n)}onFocus(){this.setState({focus:!0})}onBlur(){this.setState({focus:!1})}onMouseEnter(){this.setState({mousefocus:!0})}onMouseLeave(){this.setState({mousefocus:!1})}onKeyDown(t){(t.key==="Escape"||t.key==="Enter")&&(this.blur(),this.setState({mousefocus:!1})),t.stopPropagation()}selectFilter(t){this.setState({value:t}),Ly.default.findDOMNode(this.refs.input).focus()}blur(){Ly.default.findDOMNode(this.refs.input).blur()}select(){Ly.default.findDOMNode(this.refs.input).select()}render(){let{type:t,color:n,placeholder:l}=this.props,{value:d,focus:h,mousefocus:c}=this.state;return ps.default.createElement("div",{className:(0,zP.default)("filter-input input-group",{"has-error":!this.isValid(d)})},ps.default.createElement("span",{className:"input-group-addon"},ps.default.createElement("i",{className:"fa fa-fw fa-"+t,style:{color:n}})),ps.default.createElement("input",{type:"text",ref:"input",placeholder:l,className:"form-control",value:d,onChange:this.onChange,onFocus:this.onFocus,onBlur:this.onBlur,onKeyDown:this.onKeyDown}),(h||c)&&ps.default.createElement("div",{className:"popover bottom",onMouseEnter:this.onMouseEnter,onMouseLeave:this.onMouseLeave},ps.default.createElement("div",{className:"arrow"}),ps.default.createElement("div",{className:"popover-content"},this.getDesc())))}};o(Yf,"FilterInput");Jp.title="Start";function Jp(){return oi.createElement("div",{className:"main-menu"},oi.createElement("div",{className:"menu-group"},oi.createElement("div",{className:"menu-content"},oi.createElement(WB,null),oi.createElement(UB,null)),oi.createElement("div",{className:"menu-legend"},"Find")),oi.createElement("div",{className:"menu-group"},oi.createElement("div",{className:"menu-content"},oi.createElement(HB,null),oi.createElement(zB,null)),oi.createElement("div",{className:"menu-legend"},"Intercept")))}o(Jp,"StartMenu");function HB(){let e=Xt(),t=qe(n=>n.options.intercept);return oi.createElement(Yf,{value:t||"",placeholder:"Intercept",type:"pause",color:"hsl(208, 56%, 53%)",onChange:n=>e(Ip("intercept",n))})}o(HB,"InterceptInput");function WB(){let e=Xt(),t=qe(n=>n.flows.filter);return oi.createElement(Yf,{value:t||"",placeholder:"Search",type:"search",color:"black",onChange:n=>e(A0(n))})}o(WB,"FlowFilterInput");function UB(){let e=Xt(),t=qe(n=>n.flows.highlight);return oi.createElement(Yf,{value:t||"",placeholder:"Highlight",type:"tag",color:"hsl(48, 100%, 50%)",onChange:n=>e(D0(n))})}o(UB,"HighlightInput");function zB(){let e=Xt();return oi.createElement(kr,{className:"btn-sm",title:"[a]ccept all",icon:"fa-forward text-success",onClick:()=>e(R0())},"Resume All")}o(zB,"ResumeAll");var Qr=fe(Oe());var Xf=fe(Oe());function FC({value:e,onChange:t,children:n}){return Xf.createElement("div",{className:"menu-entry"},Xf.createElement("label",null,Xf.createElement("input",{type:"checkbox",checked:e,onChange:t}),n))}o(FC,"MenuToggle");function Py({name:e,children:t}){let n=Xt(),l=qe(d=>d.options[e]);return Xf.createElement(FC,{value:!!l,onChange:()=>n(Ip(e,!l))},t)}o(Py,"OptionsToggle");function $P(){let e=Gs(),t=qe(n=>n.eventLog.visible);return Xf.createElement(FC,{value:t,onChange:()=>e(Rp())},"Display Event Log")}o($P,"EventlogToggle");function jP(){let e=Gs(),t=qe(n=>n.commandBar.visible);return Xf.createElement(FC,{value:t,onChange:()=>e(V0())},"Display Command Bar")}o(jP,"CommandBarToggle");var BC=fe(Oe());function HC({children:e,resource:t}){let n=`https://docs.mitmproxy.org/stable/${t}`;return BC.createElement("a",{target:"_blank",href:n},e||BC.createElement("i",{className:"fa fa-question-circle"}))}o(HC,"DocsLink");var Oy=fe(Oe());function Ro({children:e}){return window.MITMWEB_STATIC?null:Oy.createElement(Oy.Fragment,null,e)}o(Ro,"HideInStatic");My.title="Options";function My(){let e=Xt(),t=o(()=>sN("OptionModal"),"openOptions");return Qr.createElement("div",null,Qr.createElement(Ro,null,Qr.createElement("div",{className:"menu-group"},Qr.createElement("div",{className:"menu-content"},Qr.createElement(kr,{title:"Open Options",icon:"fa-cogs text-primary",onClick:()=>e(t())},"Edit Options ",Qr.createElement("sup",null,"alpha"))),Qr.createElement("div",{className:"menu-legend"},"Options Editor")),Qr.createElement("div",{className:"menu-group"},Qr.createElement("div",{className:"menu-content"},Qr.createElement(Py,{name:"anticache"},"Strip cache headers ",Qr.createElement(HC,{resource:"overview-features/#anticache"})),Qr.createElement(Py,{name:"showhost"},"Use host header for display"),Qr.createElement(Py,{name:"ssl_insecure"},"Don't verify server certificates")),Qr.createElement("div",{className:"menu-legend"},"Quick Options"))),Qr.createElement("div",{className:"menu-group"},Qr.createElement("div",{className:"menu-content"},Qr.createElement($P,null),Qr.createElement(jP,null)),Qr.createElement("div",{className:"menu-legend"},"View Options")))}o(My,"OptionMenu");var Hn=fe(Oe());var qP=Hn.memo(o(function(){let t=Gs(),n=qe(l=>l.flows.filter);return Hn.createElement(cu,{className:"pull-left special",text:"File",options:{placement:"bottom-start"}},Hn.createElement("li",null,Hn.createElement(G0,{icon:"fa-folder-open",text:"\xA0Open...",onClick:l=>l.stopPropagation(),onOpenFile:l=>{t(iN(l)),document.body.click()}})),Hn.createElement(ii,{onClick:()=>location.replace("/flows/dump")},Hn.createElement("i",{className:"fa fa-fw fa-floppy-o"}),"\xA0Save"),Hn.createElement(ii,{onClick:()=>location.replace("/flows/dump?filter="+n)},Hn.createElement("i",{className:"fa fa-fw fa-floppy-o"}),"\xA0Save filtered"),Hn.createElement(ii,{onClick:()=>confirm("Delete all flows?")&&t(W0())},Hn.createElement("i",{className:"fa fa-fw fa-trash"}),"\xA0Clear All"),Hn.createElement(Ro,null,Hn.createElement(hL,null),Hn.createElement("li",null,Hn.createElement("a",{href:"http://mitm.it/",target:"_blank"},Hn.createElement("i",{className:"fa fa-fw fa-external-link"}),"\xA0Install Certificates..."))))},"FileMenu"));var at=fe(Oe());function VP(e){if(navigator.clipboard&&window.isSecureContext)return navigator.clipboard.writeText(e);{let t=document.createElement("textarea");t.value=e,t.style.position="absolute",t.style.opacity="0",document.body.appendChild(t);try{return t.focus(),t.select(),document.execCommand("copy"),Promise.resolve()}catch(n){return alert(e),Promise.reject(n)}finally{t.remove()}}}o(VP,"copyToClipboard");var ed=o((e,t)=>Ia(void 0,null,function*(){let n=yield Pf("export",t,`@${e.id}`);n.value?yield VP(n.value):n.error?alert(n.error):console.error(n)}),"copy");td.title="Flow";function td(){let e=Xt(),t=qe(n=>n.flows.byId[n.flows.selected[0]]);return t?at.createElement("div",{className:"flow-menu"},at.createElement(Ro,null,at.createElement("div",{className:"menu-group"},at.createElement("div",{className:"menu-content"},at.createElement(kr,{title:"[r]eplay flow",icon:"fa-repeat text-primary",onClick:()=>e(Mp(t)),disabled:!E0(t)},"Replay"),at.createElement(kr,{title:"[D]uplicate flow",icon:"fa-copy text-info",onClick:()=>e(B0(t))},"Duplicate"),at.createElement(kr,{disabled:!t||!t.modified,title:"revert changes to flow [V]",icon:"fa-history text-warning",onClick:()=>e(H0(t))},"Revert"),at.createElement(kr,{title:"[d]elete flow",icon:"fa-trash text-danger",onClick:()=>e(F0(t))},"Delete"),at.createElement(VB,{flow:t})),at.createElement("div",{className:"menu-legend"},"Flow Modification"))),at.createElement("div",{className:"menu-group"},at.createElement("div",{className:"menu-content"},at.createElement($B,{flow:t}),at.createElement(jB,{flow:t})),at.createElement("div",{className:"menu-legend"},"Export")),at.createElement(Ro,null,at.createElement("div",{className:"menu-group"},at.createElement("div",{className:"menu-content"},at.createElement(kr,{disabled:!t||!t.intercepted,title:"[a]ccept intercepted flow",icon:"fa-play text-success",onClick:()=>e(Op(t))},"Resume"),at.createElement(kr,{disabled:!t||!t.intercepted,title:"kill intercepted flow [x]",icon:"fa-times text-danger",onClick:()=>e(I0(t))},"Abort")),at.createElement("div",{className:"menu-legend"},"Interception")))):at.createElement("div",null)}o(td,"FlowMenu");var Ay=o(e=>{let t=window.open(e,"_blank","noopener,noreferrer");t&&(t.opener=null)},"openInNewTab");function $B({flow:e}){var t;if(e.type!=="http")return at.createElement(kr,{icon:"fa-download",onClick:()=>0,disabled:!0},"Download");if(e.request.contentLength&&!((t=e.response)==null?void 0:t.contentLength))return at.createElement(kr,{icon:"fa-download",onClick:()=>Ay(Kr.getContentURL(e,e.request))},"Download");if(e.response){let n=e.response;if(!e.request.contentLength&&e.response.contentLength)return at.createElement(kr,{icon:"fa-download",onClick:()=>Ay(Kr.getContentURL(e,n))},"Download");if(e.request.contentLength&&e.response.contentLength)return at.createElement(cu,{text:at.createElement(kr,{icon:"fa-download",onClick:()=>1},"Download\u25BE"),options:{placement:"bottom-start"}},at.createElement(ii,{onClick:()=>Ay(Kr.getContentURL(e,e.request))},"Download request"),at.createElement(ii,{onClick:()=>Ay(Kr.getContentURL(e,n))},"Download response"))}return null}o($B,"DownloadButton");function jB({flow:e}){return at.createElement(cu,{className:"",text:at.createElement(kr,{title:"Export flow.",icon:"fa-clone",onClick:()=>1,disabled:e.type!=="http"},"Export\u25BE"),options:{placement:"bottom-start"}},at.createElement(ii,{onClick:()=>ed(e,"raw_request")},"Copy raw request"),at.createElement(ii,{onClick:()=>ed(e,"raw_response")},"Copy raw response"),at.createElement(ii,{onClick:()=>ed(e,"raw")},"Copy raw request and response"),at.createElement(ii,{onClick:()=>ed(e,"curl")},"Copy as cURL"),at.createElement(ii,{onClick:()=>ed(e,"httpie")},"Copy as HTTPie"))}o(jB,"ExportButton");var qB={":red_circle:":"\u{1F534}",":orange_circle:":"\u{1F7E0}",":yellow_circle:":"\u{1F7E1}",":green_circle:":"\u{1F7E2}",":large_blue_circle:":"\u{1F535}",":purple_circle:":"\u{1F7E3}",":brown_circle:":"\u{1F7E4}"};function VB({flow:e}){let t=Xt();return at.createElement(cu,{className:"",text:at.createElement(kr,{title:"mark flow",icon:"fa-paint-brush text-success",onClick:()=>1},"Mark\u25BE"),options:{placement:"bottom-start"}},at.createElement(ii,{onClick:()=>t(Wi(e,{marked:""}))},"\u26AA (no marker)"),Object.entries(qB).map(([n,l])=>at.createElement(ii,{key:n,onClick:()=>t(Wi(e,{marked:n}))},l," ",n.replace(/[:_]/g," "))))}o(VB,"MarkButton");var vu=fe(Oe());var KP=vu.memo(o(function(){let t=qe(l=>l.connection.state),n=qe(l=>l.connection.message);switch(t){case ni.INIT:return vu.createElement("span",{className:"connection-indicator init"},"connecting\u2026");case ni.FETCHING:return vu.createElement("span",{className:"connection-indicator fetching"},"fetching data\u2026");case ni.ESTABLISHED:return vu.createElement("span",{className:"connection-indicator established"},"connected");case ni.ERROR:return vu.createElement("span",{className:"connection-indicator error",title:n},"connection lost");case ni.OFFLINE:return vu.createElement("span",{className:"connection-indicator offline"},"offline");default:let l=t;throw"unknown connection state"}},"ConnectionIndicator"));function WC(){let e=qe(v=>v.flows.selected.filter(C=>C in v.flows.byId)),[t,n]=(0,Io.useState)(()=>Jp),[l,d]=(0,Io.useState)(!1),h=[Jp,My];e.length>0?(l||(n(()=>td),d(!0)),h.push(td)):(l&&d(!1),t===td&&n(()=>Jp));function c(v,C){C.preventDefault(),n(()=>v)}return o(c,"handleClick"),Io.default.createElement("header",null,Io.default.createElement("nav",{className:"nav-tabs nav-tabs-lg"},Io.default.createElement(qP,null),h.map(v=>Io.default.createElement("a",{key:v.title,href:"#",className:(0,GP.default)({active:v===t}),onClick:C=>c(v,C)},v.title)),Io.default.createElement(Ro,null,Io.default.createElement(KP,null))),Io.default.createElement("div",null,Io.default.createElement(t,null)))}o(WC,"Header");var Je=fe(Oe()),YP=fe(ti());var Dy=function(){"use strict";function e(l,d){function h(){this.constructor=l}o(h,"ctor"),h.prototype=d.prototype,l.prototype=new h}o(e,"peg$subclass");function t(l,d,h,c){this.message=l,this.expected=d,this.found=h,this.location=c,this.name="SyntaxError",typeof Error.captureStackTrace=="function"&&Error.captureStackTrace(this,t)}o(t,"peg$SyntaxError"),e(t,Error);function n(l){var d=arguments.length>1?arguments[1]:{},h=this,c={},v={Expr:Cr},C=Cr,k=o(function(H,ee){return[H,...ee]},"peg$c0"),O=o(function(H){return[H]},"peg$c1"),j=o(function(){return""},"peg$c2"),B={type:"other",description:"string"},X='"',J={type:"literal",value:'"',description:'"\\""'},Z=o(function(H){return H.join("")},"peg$c6"),R="'",A={type:"literal",value:"'",description:`"'"`},I=/^["\\]/,G={type:"class",value:'["\\\\]',description:'["\\\\]'},K={type:"any",description:"any character"},se=o(function(H){return H},"peg$c12"),ne="\\",pe={type:"literal",value:"\\",description:'"\\\\"'},me=/^['\\]/,xe={type:"class",value:"['\\\\]",description:"['\\\\]"},Ve=/^['"\\]/,tt={type:"class",value:`['"\\\\]`,description:`['"\\\\]`},_e="n",St={type:"literal",value:"n",description:'"n"'},We=o(function(){return` +`?(T.seenCR||T.line++,T.column=1,T.seenCR=!1):U==="\r"||U==="\u2028"||U==="\u2029"?(T.line++,T.column=1,T.seenCR=!0):(T.column++,T.seenCR=!1),W++;return sl[w]=T,T}o(ws,"peg$computePosDetails");function zi(w,T){var W=ws(w),U=ws(T);return{start:{offset:w,line:W.line,column:W.column},end:{offset:T,line:U.line,column:U.column}}}o(zi,"peg$computeLocation");function Ce(w){Pvr&&(vr=P,Pu=[]),Pu.push(w))}o(Ce,"peg$fail");function ta(w,T,W,U){function wr(gn){var ci=1;for(gn.sort(function(ul,ki){return ul.descriptionki.description?1:0});ci1?ki.slice(0,-1).join(", ")+" or "+ki[gn.length-1]:ki[0],zu=ci?'"'+ul(ci)+'"':"end of input","Expected "+Vn+" but "+zu+" found."}return o(Kt,"buildMessage"),T!==null&&wr(T),new t(w!==null?w:Kt(T,W),T,W,U)}o(ta,"peg$buildException");function Ou(){var w,T,W,U;return ye++,w=P,T=fi(),T!==c?(W=ll(),W!==c?(U=fi(),U!==c?(Re=w,T=O(W),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),ye--,w===c&&(T=c,ye===0&&Ce(k)),w}o(Ou,"peg$parsestart");function nt(){var w,T;return ye++,B.test(l.charAt(P))?(w=l.charAt(P),P++):(w=c,ye===0&&Ce(X)),ye--,w===c&&(T=c,ye===0&&Ce(j)),w}o(nt,"peg$parsews");function uc(){var w,T;return ye++,Z.test(l.charAt(P))?(w=l.charAt(P),P++):(w=c,ye===0&&Ce(R)),ye--,w===c&&(T=c,ye===0&&Ce(J)),w}o(uc,"peg$parsecc");function fi(){var w,T;for(ye++,w=[],T=nt();T!==c;)w.push(T),T=nt();return ye--,w===c&&(T=c,ye===0&&Ce(A)),w}o(fi,"peg$parse__");function ll(){var w,T,W,U,wr,Kt;return w=P,T=Ur(),T!==c?(W=fi(),W!==c?(l.charCodeAt(P)===124?(U=I,P++):(U=c,ye===0&&Ce(G)),U!==c?(wr=fi(),wr!==c?(Kt=ll(),Kt!==c?(Re=w,T=K(T,Kt),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c&&(w=Ur()),w}o(ll,"peg$parseOrExpr");function Ur(){var w,T,W,U,wr,Kt;if(w=P,T=ra(),T!==c?(W=fi(),W!==c?(l.charCodeAt(P)===38?(U=se,P++):(U=c,ye===0&&Ce(ne)),U!==c?(wr=fi(),wr!==c?(Kt=Ur(),Kt!==c?(Re=w,T=pe(T,Kt),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c){if(w=P,T=ra(),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Ur(),U!==c?(Re=w,T=pe(T,U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;w===c&&(w=ra())}return w}o(Ur,"peg$parseAndExpr");function ra(){var w,T,W,U;return w=P,l.charCodeAt(P)===33?(T=me,P++):(T=c,ye===0&&Ce(xe)),T!==c?(W=fi(),W!==c?(U=ra(),U!==c?(Re=w,T=Ve(U),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c&&(w=jn()),w}o(ra,"peg$parseNotExpr");function jn(){var w,T,W,U,wr,Kt;return w=P,l.charCodeAt(P)===40?(T=tt,P++):(T=c,ye===0&&Ce(_e)),T!==c?(W=fi(),W!==c?(U=ll(),U!==c?(wr=fi(),wr!==c?(l.charCodeAt(P)===41?(Kt=St,P++):(Kt=c,ye===0&&Ce(We)),Kt!==c?(Re=w,T=Ke(U),w=T):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c)):(P=w,w=c),w===c&&(w=dd()),w}o(jn,"peg$parseBindingExpr");function dd(){var w,T,W,U;if(w=P,l.substr(P,4)===Ge?(T=Ge,P+=4):(T=c,ye===0&&Ce(Xe)),T!==c&&(Re=w,T=nr()),w=T,w===c&&(w=P,l.substr(P,2)===ct?(T=ct,P+=2):(T=c,ye===0&&Ce(Hr)),T!==c&&(Re=w,T=Qt()),w=T,w===c)){if(w=P,l.substr(P,2)===_t?(T=_t,P+=2):(T=c,ye===0&&Ce(Ct)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=ut(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===Lr?(T=Lr,P+=3):(T=c,ye===0&&Ce(zt)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=$t(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===ie?(T=ie,P+=3):(T=c,ye===0&&Ce(rt)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Pr(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===Gt?(T=Gt,P+=2):(T=c,ye===0&&Ce(Yt)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Mu(),U!==c?(Re=w,T=Se(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,8)===Or?(T=Or,P+=8):(T=c,ye===0&&Ce(fn)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Un(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===si?(T=si,P+=2):(T=c,ye===0&&Ce(cn)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Zt(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,4)===gr?(T=gr,P+=4):(T=c,ye===0&&Ce(pt)),T!==c&&(Re=w,T=Ho()),w=T,w===c)){if(w=P,l.substr(P,4)===Cr?(T=Cr,P+=4):(T=c,ye===0&&Ce(Ui)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=pn(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,2)===zn?(T=zn,P+=2):(T=c,ye===0&&Ce(Si)),T!==c&&(Re=w,T=Ci()),w=T,w===c)){if(w=P,l.substr(P,2)===$n?(T=$n,P+=2):(T=c,ye===0&&Ce(Mn)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Js(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===H?(T=H,P+=3):(T=c,ye===0&&Ce(ee)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=he(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===Te?(T=Te,P+=3):(T=c,ye===0&&Ce(ir)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Ul(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,5)===Ft?(T=Ft,P+=5):(T=c,ye===0&&Ce(Wr)),T!==c&&(Re=w,T=or()),w=T,w===c&&(w=P,l.substr(P,7)===li?(T=li,P+=7):(T=c,ye===0&&Ce(ds)),T!==c&&(Re=w,T=lo()),w=T,w===c))){if(w=P,l.substr(P,7)===bi?(T=bi,P+=7):(T=c,ye===0&&Ce(el)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=hs(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===dn?(T=dn,P+=2):(T=c,ye===0&&Ce(id)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=tl(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,2)===Qf?(T=Qf,P+=2):(T=c,ye===0&&Ce(rl)),T!==c&&(Re=w,T=od()),w=T,w===c&&(w=P,l.substr(P,8)===Zf?(T=Zf,P+=8):(T=c,ye===0&&Ce(wu)),T!==c&&(Re=w,T=sd()),w=T,w===c&&(w=P,l.substr(P,8)===zl?(T=zl,P+=8):(T=c,ye===0&&Ce(ms)),T!==c&&(Re=w,T=ld()),w=T,w===c&&(w=P,l.substr(P,7)===Jf?(T=Jf,P+=7):(T=c,ye===0&&Ce(nl)),T!==c&&(Re=w,T=xu()),w=T,w===c))))){if(w=P,l.substr(P,4)===Wo?(T=Wo,P+=4):(T=c,ye===0&&Ce(ad)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Uo(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c&&(w=P,l.substr(P,2)===$l?(T=$l,P+=2):(T=c,ye===0&&Ce(ec)),T!==c&&(Re=w,T=jt()),w=T,w===c&&(w=P,l.substr(P,4)===Me?(T=Me,P+=4):(T=c,ye===0&&Ce(Ei)),T!==c&&(Re=w,T=Su()),w=T,w===c&&(w=P,l.substr(P,4)===ai?(T=ai,P+=4):(T=c,ye===0&&Ce(vt)),T!==c&&(Re=w,T=ao()),w=T,w===c)))){if(w=P,l.substr(P,3)===zo?(T=zo,P+=3):(T=c,ye===0&&Ce(jl)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=ue(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,3)===ze?(T=ze,P+=3):(T=c,ye===0&&Ce(Cu)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=bu(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===gs?(T=gs,P+=2):(T=c,ye===0&&Ce(il)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=Eu(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.substr(P,2)===He?(T=He,P+=2):(T=c,ye===0&&Ce(ud)),T!==c){if(W=[],U=nt(),U!==c)for(;U!==c;)W.push(U),U=nt();else W=c;W!==c?(U=Vt(),U!==c?(Re=w,T=ql(U),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;w===c&&(w=P,l.substr(P,10)===uo?(T=uo,P+=10):(T=c,ye===0&&Ce(ui)),T!==c&&(Re=w,T=tc()),w=T,w===c&&(w=P,T=Vt(),T!==c&&(Re=w,T=ql(T)),w=T))}}}}}}}}}}}}}}}}}return w}o(dd,"peg$parseExpr");function Mu(){var w,T,W,U;if(ye++,w=P,$o.test(l.charAt(P))?(T=l.charAt(P),P++):(T=c,ye===0&&Ce(vs)),T===c&&(T=null),T!==c){if(W=[],Tu.test(l.charAt(P))?(U=l.charAt(P),P++):(U=c,ye===0&&Ce(ol)),U!==c)for(;U!==c;)W.push(U),Tu.test(l.charAt(P))?(U=l.charAt(P),P++):(U=c,ye===0&&Ce(ol));else W=c;W!==c?($o.test(l.charAt(P))?(U=l.charAt(P),P++):(U=c,ye===0&&Ce(vs)),U===c&&(U=null),U!==c?(Re=w,T=Vl(W),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;return ye--,w===c&&(T=c,ye===0&&Ce(_u)),w}o(Mu,"peg$parseIntegerLiteral");function Vt(){var w,T,W,U;if(ye++,w=P,l.charCodeAt(P)===34?(T=fo,P++):(T=c,ye===0&&Ce(Gl)),T!==c){for(W=[],U=xs();U!==c;)W.push(U),U=xs();W!==c?(l.charCodeAt(P)===34?(U=fo,P++):(U=c,ye===0&&Ce(Gl)),U!==c?(Re=w,T=Yl(W),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c){if(w=P,l.charCodeAt(P)===39?(T=rc,P++):(T=c,ye===0&&Ce(Xl)),T!==c){for(W=[],U=qo();U!==c;)W.push(U),U=qo();W!==c?(l.charCodeAt(P)===39?(U=rc,P++):(U=c,ye===0&&Ce(Xl)),U!==c?(Re=w,T=Yl(W),w=T):(P=w,w=c)):(P=w,w=c)}else P=w,w=c;if(w===c)if(w=P,T=P,ye++,W=uc(),ye--,W===c?T=void 0:(P=T,T=c),T!==c){if(W=[],U=yt(),U!==c)for(;U!==c;)W.push(U),U=yt();else W=c;W!==c?(Re=w,T=Yl(W),w=T):(P=w,w=c)}else P=w,w=c}return ye--,w===c&&(T=c,ye===0&&Ce(Kl)),w}o(Vt,"peg$parseStringLiteral");function xs(){var w,T,W;return w=P,T=P,ye++,_i.test(l.charAt(P))?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(nc)),ye--,W===c?T=void 0:(P=T,T=c),T!==c?(l.length>P?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(Ql)),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c),w===c&&(w=P,l.charCodeAt(P)===92?(T=ys,P++):(T=c,ye===0&&Ce(ic)),T!==c?(W=$i(),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c)),w}o(xs,"peg$parseDoubleStringChar");function qo(){var w,T,W;return w=P,T=P,ye++,oc.test(l.charAt(P))?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(fd)),ye--,W===c?T=void 0:(P=T,T=c),T!==c?(l.length>P?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(Ql)),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c),w===c&&(w=P,l.charCodeAt(P)===92?(T=ys,P++):(T=c,ye===0&&Ce(ic)),T!==c?(W=$i(),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c)),w}o(qo,"peg$parseSingleStringChar");function yt(){var w,T,W;return w=P,T=P,ye++,W=nt(),ye--,W===c?T=void 0:(P=T,T=c),T!==c?(l.length>P?(W=l.charAt(P),P++):(W=c,ye===0&&Ce(Ql)),W!==c?(Re=w,T=co(W),w=T):(P=w,w=c)):(P=w,w=c),w}o(yt,"peg$parseUnquotedStringChar");function $i(){var w,T;return cd.test(l.charAt(P))?(w=l.charAt(P),P++):(w=c,ye===0&&Ce(ku)),w===c&&(w=P,l.charCodeAt(P)===110?(T=sc,P++):(T=c,ye===0&&Ce(Nu)),T!==c&&(Re=w,T=lc()),w=T,w===c&&(w=P,l.charCodeAt(P)===114?(T=ac,P++):(T=c,ye===0&&Ce(Zl)),T!==c&&(Re=w,T=Jl()),w=T,w===c&&(w=P,l.charCodeAt(P)===116?(T=Lu,P++):(T=c,ye===0&&Ce(Ot)),T!==c&&(Re=w,T=Nt()),w=T))),w}o($i,"peg$parseEscapeSequence");function Au(w,T){function W(){return w.apply(this,arguments)||T.apply(this,arguments)}return o(W,"orFilter"),W.desc=w.desc+" or "+T.desc,W}o(Au,"or");function hd(w,T){function W(){return w.apply(this,arguments)&&T.apply(this,arguments)}return o(W,"andFilter"),W.desc=w.desc+" and "+T.desc,W}o(hd,"and");function Vo(w){function T(){return!w.apply(this,arguments)}return o(T,"notFilter"),T.desc="not "+w.desc,T}o(Vo,"not");function yr(w){function T(){return w.apply(this,arguments)}return o(T,"bindingFilter"),T.desc="("+w.desc+")",T}o(yr,"binding");function fc(w){return!0}o(fc,"allFilter"),fc.desc="all flows";var Du=[new RegExp("text/javascript"),new RegExp("application/x-javascript"),new RegExp("application/javascript"),new RegExp("text/css"),new RegExp("image/.*"),new RegExp("font/.*"),new RegExp("application/font.*")];function Ko(w){if(w.response){for(var T=Ys.getContentType(w.response),W=Du.length;W--;)if(Du[W].test(T))return!0}return!1}o(Ko,"assetFilter"),Ko.desc="is asset";function na(w){w=new RegExp(w,"i");function T(W){return!0}return o(T,"bodyFilter"),T.desc="body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10",T}o(na,"body");function Go(w){w=new RegExp(w,"i");function T(W){return!0}return o(T,"requestBodyFilter"),T.desc="body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10",T}o(Go,"requestBody");function md(w){w=new RegExp(w,"i");function T(W){return!0}return o(T,"responseBodyFilter"),T.desc="body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10",T}o(md,"responseBody");function ia(w){function T(W){return W.response&&W.response.status_code===w}return o(T,"responseCodeFilter"),T.desc="resp. code is "+w,T}o(ia,"responseCode");function Ru(w){w=new RegExp(w,"i");function T(W){return w.test(W.comment)}return o(T,"commentFilter"),T.desc="comment matches "+w,T}o(Ru,"comment");function Iu(w){w=new RegExp(w,"i");function T(W){return W.request&&(w.test(W.request.host)||w.test(W.request.pretty_host))}return o(T,"domainFilter"),T.desc="domain matches "+w,T}o(Iu,"domain");function oa(w){return w.type==="dns"}o(oa,"dnsFilter"),oa.desc="is a DNS Flow";function Fu(w){w=new RegExp(w,"i");function T(W){return!!W.server_conn.address&&w.test(W.server_conn.address[0]+":"+W.server_conn.address[1])}return o(T,"destinationFilter"),T.desc="destination address matches "+w,T}o(Fu,"destination");function Bu(w){return!!w.error}o(Bu,"errorFilter"),Bu.desc="has error";function Hu(w){w=new RegExp(w,"i");function T(W){return W.request&&Oo.match_header(W.request,w)||W.response&&Ys.match_header(W.response,w)}return o(T,"headerFilter"),T.desc="header matches "+w,T}o(Hu,"header");function Yo(w){w=new RegExp(w,"i");function T(W){return W.request&&Oo.match_header(W.request,w)}return o(T,"requestHeaderFilter"),T.desc="req. header matches "+w,T}o(Yo,"requestHeader");function ji(w){w=new RegExp(w,"i");function T(W){return W.response&&Ys.match_header(W.response,w)}return o(T,"responseHeaderFilter"),T.desc="resp. header matches "+w,T}o(ji,"responseHeader");function Ss(w){return w.type==="http"}o(Ss,"httpFilter"),Ss.desc="is an HTTP Flow";function zr(w){return w.marked}o(zr,"markedFilter"),zr.desc="is marked";function sa(w){w=new RegExp(w,"i");function T(W){return w.test(W.marked)}return o(T,"markerFilter"),T.desc="marker matches "+w,T}o(sa,"marker");function mn(w){w=new RegExp(w,"i");function T(W){return W.request&&w.test(W.request.method)}return o(T,"methodFilter"),T.desc="method matches "+w,T}o(mn,"method");function qi(w){return w.request&&!w.response}o(qi,"noResponseFilter"),qi.desc="has no response";function al(w){return w.is_replay==="request"}o(al,"clientReplayFilter"),al.desc="request has been replayed";function cc(w){return w.is_replay==="response"}o(cc,"serverReplayFilter"),cc.desc="response has been replayed";function Wu(w){return!!w.is_replay}o(Wu,"replayFilter"),Wu.desc="flow has been replayed";function gd(w){w=new RegExp(w,"i");function T(W){return!!W.client_conn.peername&&w.test(W.client_conn.peername[0]+":"+W.client_conn.peername[1])}return o(T,"sourceFilter"),T.desc="source address matches "+w,T}o(gd,"source");function Uu(w){return!!w.response}o(Uu,"responseFilter"),Uu.desc="has response";function la(w){return w.type==="tcp"}o(la,"tcpFilter"),la.desc="is a TCP Flow";function qn(w){return w.type==="udp"}o(qn,"udpFilter"),qn.desc="is a UDP Flow";function Ti(w){w=new RegExp(w,"i");function T(W){return W.request&&w.test(Oo.getContentType(W.request))}return o(T,"requestContentTypeFilter"),T.desc="req. content type matches "+w,T}o(Ti,"requestContentType");function pc(w){w=new RegExp(w,"i");function T(W){return W.response&&w.test(Ys.getContentType(W.response))}return o(T,"responseContentTypeFilter"),T.desc="resp. content type matches "+w,T}o(pc,"responseContentType");function aa(w){w=new RegExp(w,"i");function T(W){return W.request&&w.test(Oo.getContentType(W.request))||W.response&&w.test(Ys.getContentType(W.response))}return o(T,"contentTypeFilter"),T.desc="content type matches "+w,T}o(aa,"contentType");function dc(w){w=new RegExp(w,"i");function T(W){var U;if(W.type==="dns"){let wr=(U=W.request)==null?void 0:U.questions[0];return wr&&w.test(wr.name)}return W.request&&w.test(Oo.pretty_url(W.request))}return o(T,"urlFilter"),T.desc="url matches "+w,T}o(dc,"url");function Vi(w){return!!w.websocket}if(o(Vi,"websocketFilter"),Vi.desc="is a Websocket Flow",jo=C(),jo!==c&&P===l.length)return jo;throw jo!==c&&PyS,icon:()=>N0,method:()=>tm,path:()=>L0,quickactions:()=>Pp,size:()=>P0,status:()=>rm,time:()=>O0,timestamp:()=>M0,tls:()=>k0});var er=fe(Oe());var T0=fe(ti());var k0=o(({flow:e})=>er.default.createElement("td",{className:(0,T0.default)("col-tls",e.client_conn.tls_established?"col-tls-https":"col-tls-http")}),"tls");k0.headerName="";k0.sortKey=e=>e.type==="http"&&e.request.scheme;var N0=o(({flow:e})=>er.default.createElement("td",{className:"col-icon"},er.default.createElement("div",{className:(0,T0.default)("resource-icon",Kk(e))})),"icon");N0.headerName="";N0.sortKey=e=>Kk(e);var Kk=o(e=>{if(e.type!=="http")return e.client_conn.tls_version==="QUIC"?"resource-icon-quic":`resource-icon-${e.type}`;if(e.websocket)return"resource-icon-websocket";if(!e.response)return"resource-icon-plain";var t=Ys.getContentType(e.response)||"";return e.response.status_code===304?"resource-icon-not-modified":300<=e.response.status_code&&e.response.status_code<400?"resource-icon-redirect":t.indexOf("image")>=0?"resource-icon-image":t.indexOf("javascript")>=0?"resource-icon-js":t.indexOf("css")>=0?"resource-icon-css":t.indexOf("html")>=0?"resource-icon-document":"resource-icon-plain"},"getIcon"),Gk=o(e=>{var t,n,l,d;switch(e.type){case"http":return Oo.pretty_url(e.request);case"tcp":case"udp":return`${e.client_conn.peername.join(":")} \u2194 ${(n=(t=e.server_conn)==null?void 0:t.address)==null?void 0:n.join(":")}`;case"dns":return`${e.request.questions.map(h=>`${h.name} ${h.type}`).join(", ")} = ${((d=(l=e.response)==null?void 0:l.answers.map(h=>h.data).join(", "))!=null?d:"...")||"?"}`}},"mainPath"),L0=o(({flow:e})=>{let t;return e.error&&(e.error.msg==="Connection killed."?t=er.default.createElement("i",{className:"fa fa-fw fa-times pull-right"}):t=er.default.createElement("i",{className:"fa fa-fw fa-exclamation pull-right"})),er.default.createElement("td",{className:"col-path"},e.is_replay==="request"&&er.default.createElement("i",{className:"fa fa-fw fa-repeat pull-right"}),e.intercepted&&er.default.createElement("i",{className:"fa fa-fw fa-pause pull-right"}),t,er.default.createElement("span",{className:"marker pull-right"},e.marked),Gk(e))},"path");L0.headerName="Path";L0.sortKey=e=>Gk(e);var tm=o(({flow:e})=>er.default.createElement("td",{className:"col-method"},tm.sortKey(e)),"method");tm.headerName="Method";tm.sortKey=e=>{switch(e.type){case"http":return e.websocket?e.client_conn.tls_established?"WSS":"WS":e.request.method;case"dns":return e.request.op_code;default:return e.type.toUpperCase()}};var rm=o(({flow:e})=>{let t="darkred";return e.type!=="http"&&e.type!="dns"||!e.response?er.default.createElement("td",{className:"col-status"}):(100<=e.response.status_code&&e.response.status_code<200?t="green":200<=e.response.status_code&&e.response.status_code<300?t="darkgreen":300<=e.response.status_code&&e.response.status_code<400?t="lightblue":(400<=e.response.status_code&&e.response.status_code<500||500<=e.response.status_code&&e.response.status_code<600)&&(t="red"),er.default.createElement("td",{className:"col-status",style:{color:t}},rm.sortKey(e)))},"status");rm.headerName="Status";rm.sortKey=e=>{var t,n;switch(e.type){case"http":return(t=e.response)==null?void 0:t.status_code;case"dns":return(n=e.response)==null?void 0:n.response_code;default:return}};var P0=o(({flow:e})=>er.default.createElement("td",{className:"col-size"},x0(vS(e))),"size");P0.headerName="Size";P0.sortKey=e=>vS(e);var O0=o(({flow:e})=>{let t=em(e),n=gS(e);return er.default.createElement("td",{className:"col-time"},t&&n?S0(1e3*(n-t)):"...")},"time");O0.headerName="Time";O0.sortKey=e=>{let t=em(e),n=gS(e);return t&&n&&n-t};var M0=o(({flow:e})=>{let t=em(e);return er.default.createElement("td",{className:"col-timestamp"},t?no(t):"...")},"timestamp");M0.headerName="Start time";M0.sortKey=e=>em(e);var Pp=o(({flow:e})=>{let t=Gs(),[n,l]=(0,er.useState)(!1),d=null;return e.intercepted?d=er.default.createElement("a",{href:"#",className:"quickaction",onClick:()=>t(Op(e))},er.default.createElement("i",{className:"fa fa-fw fa-play text-success"})):E0(e)&&(d=er.default.createElement("a",{href:"#",className:"quickaction",onClick:()=>t(Mp(e))},er.default.createElement("i",{className:"fa fa-fw fa-repeat text-primary"}))),er.default.createElement("td",{className:(0,T0.default)("col-quickactions",{hover:n}),onClick:()=>0},d?er.default.createElement("div",null,d):er.default.createElement(er.default.Fragment,null))},"quickactions");Pp.headerName="";Pp.sortKey=e=>0;var yS={icon:N0,method:tm,path:L0,quickactions:Pp,size:P0,status:rm,time:O0,timestamp:M0,tls:k0};var EF="FLOWS_ADD",_F="FLOWS_UPDATE",Yk="FLOWS_REMOVE",TF="FLOWS_RECEIVE",Xk="FLOWS_SELECT",Qk="FLOWS_SET_FILTER",Zk="FLOWS_SET_SORT",Jk="FLOWS_SET_HIGHLIGHT",kF=ke({highlight:void 0,filter:void 0,sort:{column:void 0,desc:!1},selected:[]},C0);function wS(e=kF,t){switch(t.type){case EF:case _F:case Yk:case TF:let n=Jh[t.cmd](t.data,eN(e.filter),xS(e.sort)),l=e.selected;if(t.type===Yk&&e.selected.includes(t.data)){if(e.selected.length>1)l=l.filter(d=>d!==t.data);else if(l=[],t.data in e.viewIndex&&e.view.length>1){let d=e.viewIndex[t.data],h;d===e.view.length-1?h=e.view[d-1]:h=e.view[d+1],l.push(h.id)}}return ke(Pt(ke({},e),{selected:l}),Lp(e,n));case Qk:return ke(Pt(ke({},e),{filter:t.filter}),Lp(e,pS(eN(t.filter),xS(e.sort))));case Jk:return Pt(ke({},e),{highlight:t.highlight});case Zk:return ke(Pt(ke({},e),{sort:t.sort}),Lp(e,jk(xS(t.sort))));case Xk:return Pt(ke({},e),{selected:t.flowIds});default:return e}}o(wS,"reducer");function eN(e){if(!!e)return Of.parse(e)}o(eN,"makeFilter");function xS({column:e,desc:t}){if(!e)return(l,d)=>0;let n=yS[e].sortKey;return(l,d)=>{let h=n(l),c=n(d);return h>c?t?-1:1:hkt(`/flows/${e.id}/resume`,{method:"POST"})}o(Op,"resume");function R0(){return e=>kt("/flows/resume",{method:"POST"})}o(R0,"resumeAll");function I0(e){return t=>kt(`/flows/${e.id}/kill`,{method:"POST"})}o(I0,"kill");function rN(){return e=>kt("/flows/kill",{method:"POST"})}o(rN,"killAll");function F0(e){return t=>kt(`/flows/${e.id}`,{method:"DELETE"})}o(F0,"remove");function B0(e){return t=>kt(`/flows/${e.id}/duplicate`,{method:"POST"})}o(B0,"duplicate");function Mp(e){return t=>kt(`/flows/${e.id}/replay`,{method:"POST"})}o(Mp,"replay");function H0(e){return t=>kt(`/flows/${e.id}/revert`,{method:"POST"})}o(H0,"revert");function Wi(e,t){return n=>kt.put(`/flows/${e.id}`,t)}o(Wi,"update");function nN(e,t,n){let l=new FormData;return t=new window.Blob([t],{type:"plain/text"}),l.append("file",t),d=>kt(`/flows/${e.id}/${n}/content.data`,{method:"POST",body:l})}o(nN,"uploadContent");function W0(){return e=>kt("/clear",{method:"POST"})}o(W0,"clear");function iN(e){let t=new FormData;return t.append("file",e),n=>kt("/flows/dump",{method:"POST",body:t})}o(iN,"upload");function Af(e){return{type:Xk,flowIds:e?[e]:[]}}o(Af,"select");var U0="UI_HIDE_MODAL",oN="UI_SET_ACTIVE_MODAL",NF={activeModal:void 0};function SS(e=NF,t){switch(t.type){case oN:return Pt(ke({},e),{activeModal:t.activeModal});case U0:return Pt(ke({},e),{activeModal:void 0});default:return e}}o(SS,"reducer");function sN(e){return{type:oN,activeModal:e}}o(sN,"setActiveModal");function z0(){return{type:U0}}o(z0,"hideModal");var ym=fe(Oe());var Ut=fe(Oe());var Dp=fe(Oe());var im=fe(Oe()),lN=fe(ti()),aN=(()=>{let e=document.createElement("div");return e.setAttribute("contenteditable","PLAINTEXT-ONLY"),e.contentEditable==="plaintext-only"?"plaintext-only":"true"})(),Ap=!1,Xs=class extends im.Component{constructor(){super(...arguments);this.input=im.default.createRef();this.isEditing=o(()=>{var t;return((t=this.input.current)==null?void 0:t.contentEditable)===aN},"isEditing");this.startEditing=o(()=>{if(!this.input.current)return console.error("unreachable");this.isEditing()||(this.suppress_events=!0,this.input.current.blur(),this.input.current.contentEditable=aN,window.requestAnimationFrame(()=>{var l,d;if(!this.input.current)return;this.input.current.focus(),this.suppress_events=!1;let t=document.createRange();t.selectNodeContents(this.input.current);let n=window.getSelection();n==null||n.removeAllRanges(),n==null||n.addRange(t),(d=(l=this.props).onEditStart)==null||d.call(l)}))},"startEditing");this.resetValue=o(()=>{var t,n;if(!this.input.current)return console.error("unreachable");this.input.current.textContent=this.props.content,(n=(t=this.props).onInput)==null||n.call(t,this.props.content)},"resetValue");this.finishEditing=o(()=>{if(!this.input.current)return console.error("unreachable");this.props.onEditDone(this.input.current.textContent||""),this.input.current.blur(),this.input.current.contentEditable="inherit"},"finishEditing");this.onPaste=o(t=>{t.preventDefault();let n=t.clipboardData.getData("text/plain");document.execCommand("insertHTML",!1,n)},"onPaste");this.suppress_events=!1;this.onMouseDown=o(t=>{Ap&&console.debug("onMouseDown",this.suppress_events),this.suppress_events=!0,window.addEventListener("mouseup",this.onMouseUp,{once:!0})},"onMouseDown");this.onMouseUp=o(t=>{var d;let n=t.target===this.input.current,l=!((d=window.getSelection())==null?void 0:d.toString());Ap&&console.warn("mouseUp",this.suppress_events,n,l),n&&l&&this.startEditing(),this.suppress_events=!1},"onMouseUp");this.onClick=o(t=>{Ap&&console.debug("onClick",this.suppress_events)},"onClick");this.onFocus=o(t=>{if(Ap&&console.debug("onFocus",this.props.content,this.suppress_events),!this.input.current)throw"unreachable";this.suppress_events||this.startEditing()},"onFocus");this.onInput=o(t=>{var n,l,d;(d=(l=this.props).onInput)==null||d.call(l,((n=this.input.current)==null?void 0:n.textContent)||"")},"onInput");this.onBlur=o(t=>{Ap&&console.debug("onBlur",this.props.content,this.suppress_events),!this.suppress_events&&this.finishEditing()},"onBlur");this.onKeyDown=o(t=>{var n,l;switch(Ap&&console.debug("keydown",t),t.stopPropagation(),t.key){case"Escape":t.preventDefault(),this.resetValue(),this.finishEditing();break;case"Enter":t.shiftKey||(t.preventDefault(),this.finishEditing());break;default:break}(l=(n=this.props).onKeyDown)==null||l.call(n,t)},"onKeyDown")}render(){let t=(0,lN.default)("inline-input",this.props.className);return im.default.createElement("span",{ref:this.input,tabIndex:0,className:t,placeholder:this.props.placeholder,onFocus:this.onFocus,onBlur:this.onBlur,onKeyDown:this.onKeyDown,onInput:this.onInput,onPaste:this.onPaste,onMouseDown:this.onMouseDown,onClick:this.onClick},this.props.content)}componentDidUpdate(t){var n,l;t.content!==this.props.content&&((l=(n=this.props).onInput)==null||l.call(n,this.props.content))}};o(Xs,"ValueEditor");var uN=fe(ti());function Df(e){let[t,n]=(0,Dp.useState)(e.isValid(e.content)),l=(0,Dp.useRef)(null),d=o(c=>{var v;e.isValid(c)?e.onEditDone(c):(v=l.current)==null||v.resetValue()},"onEditDone"),h=(0,uN.default)(e.className,t?"has-success":"has-warning");return Dp.default.createElement(Xs,Pt(ke({},e),{className:h,onInput:c=>n(e.isValid(c)),onEditDone:d,ref:l}))}o(Df,"ValidateEditor");function CS(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}o(CS,"_defineProperty");function fN(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);t&&(l=l.filter(function(d){return Object.getOwnPropertyDescriptor(e,d).enumerable})),n.push.apply(n,l)}return n}o(fN,"ownKeys");function $0(e){for(var t=1;tn[l.level])));case dN:case OF:return ke(ke({},e),Lp(e,Jh[t.cmd](t.data,l=>e.filters[l.level])));default:return e}}o(TS,"reduce");function gN(e){return{type:mN,filter:e}}o(gN,"toggleFilter");function Rp(){return{type:hN}}o(Rp,"toggleVisibility");function vN(e,t="web"){let n={id:Math.random().toString(),message:e,level:t};return{type:dN,cmd:"add",data:n}}o(vN,"add");var yN="UI_OPTION_UPDATE_START",wN="UI_OPTION_UPDATE_SUCCESS",xN="UI_OPTION_UPDATE_ERROR",AF={};function kS(e=AF,t){switch(t.type){case yN:return Pt(ke({},e),{[t.option]:{isUpdating:!0,value:t.value,error:!1}});case wN:return Pt(ke({},e),{[t.option]:void 0});case xN:let n=e[t.option].value;return typeof n=="boolean"&&(n=!n),Pt(ke({},e),{[t.option]:{value:n,isUpdating:!1,error:t.error}});case U0:return{};default:return e}}o(kS,"reducer");function SN(e,t){return{type:yN,option:e,value:t}}o(SN,"startUpdate");function CN(e){return{type:wN,option:e}}o(CN,"updateSuccess");function bN(e,t){return{type:xN,option:e,error:t}}o(bN,"updateError");var EN=q0({flow:tS,modal:SS,optionsEditor:kS});var ni;(function(h){h.INIT="CONNECTION_INIT",h.FETCHING="CONNECTION_FETCHING",h.ESTABLISHED="CONNECTION_ESTABLISHED",h.ERROR="CONNECTION_ERROR",h.OFFLINE="CONNECTION_OFFLINE"})(ni||(ni={}));var DF={state:ni.INIT,message:void 0};function NS(e=DF,t){switch(t.type){case ni.ESTABLISHED:case ni.FETCHING:case ni.ERROR:case ni.OFFLINE:return{state:t.type,message:t.message};default:return e}}o(NS,"reducer");function _N(){return{type:ni.FETCHING}}o(_N,"startFetching");function TN(){return{type:ni.ESTABLISHED}}o(TN,"connectionEstablished");function kN(e){return{type:ni.ERROR,message:e}}o(kN,"connectionError");var NN={add_upstream_certs_to_client_chain:!1,allow_hosts:[],anticache:!1,anticomp:!1,block_global:!0,block_list:[],block_private:!1,body_size_limit:void 0,cert_passphrase:void 0,certs:[],ciphers_client:void 0,ciphers_server:void 0,client_certs:void 0,client_replay:[],client_replay_concurrency:1,command_history:!0,confdir:"~/.mitmproxy",connect_addr:void 0,connection_strategy:"eager",console_focus_follow:!1,content_view_lines_cutoff:512,export_preserve_original_ip:!1,http2:!0,http2_ping_keepalive:58,ignore_hosts:[],intercept:void 0,intercept_active:!1,keep_host_header:!1,key_size:2048,listen_host:"",listen_port:void 0,map_local:[],map_remote:[],mode:["regular"],modify_body:[],modify_headers:[],normalize_outbound_headers:!0,onboarding:!0,onboarding_host:"mitm.it",proxy_debug:!1,proxyauth:void 0,rawtcp:!0,readfile_filter:void 0,rfile:void 0,save_stream_file:void 0,save_stream_filter:void 0,scripts:[],server:!0,server_replay:[],server_replay_ignore_content:!1,server_replay_ignore_host:!1,server_replay_ignore_params:[],server_replay_ignore_payload_params:[],server_replay_ignore_port:!1,server_replay_kill_extra:!1,server_replay_nopop:!1,server_replay_refresh:!0,server_replay_use_headers:[],showhost:!1,ssl_insecure:!1,ssl_verify_upstream_trusted_ca:void 0,ssl_verify_upstream_trusted_confdir:void 0,stickyauth:void 0,stickycookie:void 0,stream_large_bodies:void 0,tcp_hosts:[],termlog_verbosity:"info",tls_version_client_max:"UNBOUNDED",tls_version_client_min:"TLS1_2",tls_version_server_max:"UNBOUNDED",tls_version_server_min:"TLS1_2",udp_hosts:[],upstream_auth:void 0,upstream_cert:!0,validate_inbound_headers:!0,view_filter:void 0,view_order:"time",view_order_reversed:!1,web_columns:["tls","icon","path","method","status","size","time"],web_debug:!1,web_host:"127.0.0.1",web_open_browser:!0,web_port:8081,web_static_viewer:"",websocket:!0};var LS="OPTIONS_RECEIVE",PS="OPTIONS_UPDATE";function OS(e=NN,t){switch(t.type){case LS:let n={};for(let[d,{value:h}]of Object.entries(t.data))n[d]=h;return n;case PS:let l=ke({},e);for(let[d,{value:h}]of Object.entries(t.data))l[d]=h;return l;default:return e}}o(OS,"reducer");function RF(e,t,n){return Ia(this,null,function*(){try{let l=yield kt.put("/options",{[e]:t});if(l.status===200)n(CN(e));else throw yield l.text()}catch(l){n(bN(e,l))}})}o(RF,"pureSendUpdate");var IF=RF;function Ip(e,t){return n=>{n(SN(e,t)),IF(e,t,n)}}o(Ip,"update");function LN(){return e=>kt("/options/save",{method:"POST"})}o(LN,"save");var PN="COMMANDBAR_TOGGLE_VISIBILITY",FF={visible:!1};function MS(e=FF,t){switch(t.type){case PN:return Pt(ke({},e),{visible:!e.visible});default:return e}}o(MS,"reducer");function V0(){return{type:PN}}o(V0,"toggleVisibility");function ON(e){return function(t){var n=t.dispatch,l=t.getState;return function(d){return function(h){return typeof h=="function"?h(n,l,e):d(h)}}}}o(ON,"createThunkMiddleware");var MN=ON();MN.withExtraArgument=ON;var AN=MN;var BF="STATE_RECEIVE",HF="STATE_UPDATE",WF={available:!1,version:"",contentViews:[],servers:[]};function AS(e=WF,t){switch(t.type){case BF:case HF:return ke(Pt(ke({},e),{available:!0}),t.data);default:return e}}o(AS,"reducer");var UF={},zF=o((e=UF,t)=>{switch(t.type){case LS:return t.data;case PS:return ke(ke({},e),t.data);default:return e}},"reducer"),DN=zF;var $F=window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__||_S,jF=q0({commandBar:MS,eventLog:TS,flows:wS,connection:NS,ui:EN,options:OS,options_meta:DN,backendState:AS}),qF=o(e=>ES(jF,e,$F(pN(AN))),"createAppStore"),Fp=qF(void 0),Xt=o(()=>Gs(),"useAppDispatch"),qe=Jx;var io=fe(Oe());var RN=fe(Qh()),IN=fe(ti()),DS=class extends io.Component{constructor(){super(...arguments);this.container=io.default.createRef();this.nameInput=io.default.createRef();this.valueInput=io.default.createRef();this.render=o(()=>{let[t,n]=this.props.item;return io.default.createElement("div",{ref:this.container,className:"kv-row",onClick:this.onClick,onKeyDownCapture:this.onKeyDown},io.default.createElement(Xs,{ref:this.nameInput,className:"kv-key",content:t,onEditStart:this.props.onEditStart,onEditDone:l=>this.props.onEditDone([l,n])}),":\xA0",io.default.createElement(Xs,{ref:this.valueInput,className:"kv-value",content:n,onEditStart:this.props.onEditStart,onEditDone:l=>this.props.onEditDone([t,l]),placeholder:"empty"}))},"render");this.onClick=o(t=>{t.target===this.container.current&&this.props.onClickEmptyArea()},"onClick");this.onKeyDown=o(t=>{var n;t.target===((n=this.valueInput.current)==null?void 0:n.input.current)&&t.key==="Tab"&&this.props.onTabNext()},"onKeyDown")}};o(DS,"Row");var Bp=class extends io.Component{constructor(){super(...arguments);this.rowRefs={};this.state={currentList:this.props.data||[],initialList:this.props.data};this.render=o(()=>{this.rowRefs={};let t=this.state.currentList.map((n,l)=>io.default.createElement(DS,{key:l,item:n,onEditStart:()=>this.currentlyEditing=l,onEditDone:d=>this.onEditDone(l,d),onClickEmptyArea:()=>this.onClickEmptyArea(l),onTabNext:()=>this.onTabNext(l),ref:d=>this.rowRefs[l]=d}));return io.default.createElement("div",{className:(0,IN.default)("kv-editor",this.props.className),onMouseDown:this.onMouseDown},t,io.default.createElement("div",{onClick:n=>{n.preventDefault(),this.onClickEmptyArea(this.state.currentList.length-1)},className:"kv-add-row fa fa-plus-square-o",role:"button","aria-label":"Add"}))},"render");this.onEditDone=o((t,n)=>{let l=[...this.state.currentList];n[0]?l[t]=n:l.splice(t,1),this.currentlyEditing=void 0,(0,RN.isEqual)(this.state.currentList,l)||this.props.onChange(l),this.setState({currentList:l})},"onEditDone");this.onClickEmptyArea=o(t=>{if(this.justFinishedEditing)return;let n=[...this.state.currentList];n.splice(t+1,0,["",""]),this.setState({currentList:n},()=>{var l,d;return(d=(l=this.rowRefs[t+1])==null?void 0:l.nameInput.current)==null?void 0:d.startEditing()})},"onClickEmptyArea");this.onTabNext=o(t=>{t==this.state.currentList.length-1&&this.onClickEmptyArea(t)},"onTabNext");this.onMouseDown=o(t=>{this.justFinishedEditing=this.currentlyEditing},"onMouseDown")}static getDerivedStateFromProps(t,n){return t.data!==n.initialList?{currentList:t.data||[],initialList:t.data}:null}};o(Bp,"KeyValueListEditor");var tr=fe(Oe());var om=fe(Oe());function K0(e,t){let[n,l]=(0,om.useState)(),[d,h]=(0,om.useState)();return(0,om.useEffect)(()=>{d&&d.abort();let c=new AbortController;return kt(e,{signal:c.signal}).then(v=>{if(!v.ok)throw`${v.status} ${v.statusText}`.trim();return v.text()}).then(v=>{l(v)}).catch(v=>{c.signal.aborted||l(`Error getting content: ${v}.`)}),h(c),()=>{c.signal.aborted||c.abort()}},[e,t]),n}o(K0,"useContent");var sm=fe(Oe()),G0=sm.default.memo(o(function({icon:t,text:n,className:l,title:d,onOpenFile:h,onClick:c}){let v;return sm.default.createElement("a",{href:"#",onClick:C=>{v.click(),c&&c(C)},className:l,title:d},sm.default.createElement("i",{className:"fa fa-fw "+t}),n,sm.default.createElement("input",{ref:C=>v=C,className:"hidden",type:"file",onChange:C=>{C.preventDefault(),C.target.files&&C.target.files.length>0&&h(C.target.files[0]),v.value=""}}))},"FileChooser"));var Hp=fe(Oe()),FN=fe(ti());function kr({onClick:e,children:t,icon:n,disabled:l,className:d,title:h}){return Hp.createElement("button",{className:(0,FN.default)(d,"btn btn-default"),onClick:l?void 0:e,disabled:l,title:h},n&&Hp.createElement(Hp.Fragment,null,Hp.createElement("i",{className:"fa "+n}),"\xA0"),t)}o(kr,"Button");var am=fe(Oe()),$N=fe(Oe());var lm=fe(Oe()),HN=fe(ti()),WN=fe(BN()),UN=fe(Qh());function zN(e){return e&&e.replace(/\r\n|\r/g,` +`)}o(zN,"normalizeLineEndings");var Wp=class extends lm.Component{constructor(t){super(t);this.state={isFocused:!1}}getCodeMirrorInstance(){return this.props.codeMirrorInstance||WN.default}UNSAFE_componentWillMount(){this.props.path&&console.error("Warning: react-codemirror: the `path` prop has been changed to `name`")}componentDidMount(){let t=this.getCodeMirrorInstance();this.codeMirror=t.fromTextArea(this.textareaNode,this.props.options),this.codeMirror.on("change",this.codemirrorValueChanged.bind(this)),this.codeMirror.on("cursorActivity",this.cursorActivity.bind(this)),this.codeMirror.on("focus",this.focusChanged.bind(this,!0)),this.codeMirror.on("blur",this.focusChanged.bind(this,!1)),this.codeMirror.on("scroll",this.scrollChanged.bind(this)),this.codeMirror.setValue(this.props.defaultValue||this.props.value||"")}componentWillUnmount(){this.codeMirror&&this.codeMirror.toTextArea()}UNSAFE_componentWillReceiveProps(t){if(this.codeMirror&&t.value!==void 0&&t.value!==this.props.value&&zN(this.codeMirror.getValue())!==zN(t.value))if(this.props.preserveScrollPosition){var n=this.codeMirror.getScrollInfo();this.codeMirror.setValue(t.value),this.codeMirror.scrollTo(n.left,n.top)}else this.codeMirror.setValue(t.value);if(typeof t.options=="object")for(let l in t.options)t.options.hasOwnProperty(l)&&this.setOptionIfChanged(l,t.options[l])}setOptionIfChanged(t,n){let l=this.codeMirror.getOption(t);UN.default.isEqual(l,n)||this.codeMirror.setOption(t,n)}getCodeMirror(){return this.codeMirror}focus(){this.codeMirror&&this.codeMirror.focus()}focusChanged(t){this.setState({isFocused:t}),this.props.onFocusChange&&this.props.onFocusChange(t)}cursorActivity(t){this.props.onCursorActivity&&this.props.onCursorActivity(t)}scrollChanged(t){this.props.onScroll&&this.props.onScroll(t.getScrollInfo())}codemirrorValueChanged(t,n){this.props.onChange&&n.origin!=="setValue"&&this.props.onChange(t.getValue(),n)}render(){let t=(0,HN.default)("ReactCodeMirror",this.state.isFocused?"ReactCodeMirror--focused":null,this.props.className);return lm.createElement("div",{className:t},lm.createElement("textarea",{ref:n=>this.textareaNode=n,name:this.props.name||this.props.path,defaultValue:this.props.value,autoComplete:"off",autoFocus:this.props.autoFocus}))}};o(Wp,"CodeMirror"),Wp.defaultProps={preserveScrollPosition:!1};var um=class extends $N.Component{constructor(){super(...arguments);this.editor=am.createRef();this.getContent=o(()=>{var t;return(t=this.editor.current)==null?void 0:t.codeMirror.getValue()},"getContent");this.render=o(()=>{let t={lineNumbers:!0};return am.createElement("div",{className:"codeeditor",onKeyDown:n=>n.stopPropagation()},am.createElement(Wp,{ref:this.editor,value:this.props.initialContent,onChange:()=>0,options:t}))},"render")}};o(um,"CodeEditor");var Rf=fe(Oe()),VF=Rf.default.memo(o(function({lines:t,maxLines:n,showMore:l}){return t.length===0?null:Rf.default.createElement("pre",null,t.map((d,h)=>h===n?Rf.default.createElement("button",{key:"showmore",onClick:l,className:"btn btn-xs btn-info"},Rf.default.createElement("i",{className:"fa fa-angle-double-down","aria-hidden":"true"})," Show more"):Rf.default.createElement("div",{key:h},d.map(([c,v],C)=>Rf.default.createElement("span",{key:C,className:c},v)))))},"LineRenderer")),Y0=VF;var zf=fe(Oe());var xi=fe(Oe());var X0=fe(Oe());var FS=o(function(t){return t.reduce(function(n,l){var d=l[0],h=l[1];return n[d]=h,n},{})},"fromEntries"),BS=typeof window!="undefined"&&window.document&&window.document.createElement?X0.useLayoutEffect:X0.useEffect;var fu=fe(Oe());var Gr="top",Ln="bottom",ln="right",an="left",Q0="auto",su=[Gr,Ln,ln,an],Rl="start",Z0="end",jN="clippingParents",J0="viewport",Up="popper",qN="reference",HS=su.reduce(function(e,t){return e.concat([t+"-"+Rl,t+"-"+Z0])},[]),ey=[].concat(su,[Q0]).reduce(function(e,t){return e.concat([t,t+"-"+Rl,t+"-"+Z0])},[]),KF="beforeRead",GF="read",YF="afterRead",XF="beforeMain",QF="main",ZF="afterMain",JF="beforeWrite",e3="write",t3="afterWrite",VN=[KF,GF,YF,XF,QF,ZF,JF,e3,t3];function Pn(e){return e?(e.nodeName||"").toLowerCase():null}o(Pn,"getNodeName");function Br(e){if(e==null)return window;if(e.toString()!=="[object Window]"){var t=e.ownerDocument;return t&&t.defaultView||window}return e}o(Br,"getWindow");function Il(e){var t=Br(e).Element;return e instanceof t||e instanceof Element}o(Il,"isElement");function Yr(e){var t=Br(e).HTMLElement;return e instanceof t||e instanceof HTMLElement}o(Yr,"isHTMLElement");function ty(e){if(typeof ShadowRoot=="undefined")return!1;var t=Br(e).ShadowRoot;return e instanceof t||e instanceof ShadowRoot}o(ty,"isShadowRoot");function r3(e){var t=e.state;Object.keys(t.elements).forEach(function(n){var l=t.styles[n]||{},d=t.attributes[n]||{},h=t.elements[n];!Yr(h)||!Pn(h)||(Object.assign(h.style,l),Object.keys(d).forEach(function(c){var v=d[c];v===!1?h.removeAttribute(c):h.setAttribute(c,v===!0?"":v)}))})}o(r3,"applyStyles");function n3(e){var t=e.state,n={popper:{position:t.options.strategy,left:"0",top:"0",margin:"0"},arrow:{position:"absolute"},reference:{}};return Object.assign(t.elements.popper.style,n.popper),t.styles=n,t.elements.arrow&&Object.assign(t.elements.arrow.style,n.arrow),function(){Object.keys(t.elements).forEach(function(l){var d=t.elements[l],h=t.attributes[l]||{},c=Object.keys(t.styles.hasOwnProperty(l)?t.styles[l]:n[l]),v=c.reduce(function(C,k){return C[k]="",C},{});!Yr(d)||!Pn(d)||(Object.assign(d.style,v),Object.keys(h).forEach(function(C){d.removeAttribute(C)}))})}}o(n3,"effect");var KN={name:"applyStyles",enabled:!0,phase:"write",fn:r3,effect:n3,requires:["computeStyles"]};function On(e){return e.split("-")[0]}o(On,"getBasePlacement");var lu=Math.round;function Mo(e,t){t===void 0&&(t=!1);var n=e.getBoundingClientRect(),l=1,d=1;return Yr(e)&&t&&(l=n.width/e.offsetWidth||1,d=n.height/e.offsetHeight||1),{width:lu(n.width/l),height:lu(n.height/d),top:lu(n.top/d),right:lu(n.right/l),bottom:lu(n.bottom/d),left:lu(n.left/l),x:lu(n.left/l),y:lu(n.top/d)}}o(Mo,"getBoundingClientRect");function If(e){var t=Mo(e),n=e.offsetWidth,l=e.offsetHeight;return Math.abs(t.width-n)<=1&&(n=t.width),Math.abs(t.height-l)<=1&&(l=t.height),{x:e.offsetLeft,y:e.offsetTop,width:n,height:l}}o(If,"getLayoutRect");function fm(e,t){var n=t.getRootNode&&t.getRootNode();if(e.contains(t))return!0;if(n&&ty(n)){var l=t;do{if(l&&e.isSameNode(l))return!0;l=l.parentNode||l.host}while(l)}return!1}o(fm,"contains");function wi(e){return Br(e).getComputedStyle(e)}o(wi,"getComputedStyle");function WS(e){return["table","td","th"].indexOf(Pn(e))>=0}o(WS,"isTableElement");function Bn(e){return((Il(e)?e.ownerDocument:e.document)||window.document).documentElement}o(Bn,"getDocumentElement");function Fl(e){return Pn(e)==="html"?e:e.assignedSlot||e.parentNode||(ty(e)?e.host:null)||Bn(e)}o(Fl,"getParentNode");function GN(e){return!Yr(e)||wi(e).position==="fixed"?null:e.offsetParent}o(GN,"getTrueOffsetParent");function i3(e){var t=navigator.userAgent.toLowerCase().indexOf("firefox")!==-1,n=navigator.userAgent.indexOf("Trident")!==-1;if(n&&Yr(e)){var l=wi(e);if(l.position==="fixed")return null}for(var d=Fl(e);Yr(d)&&["html","body"].indexOf(Pn(d))<0;){var h=wi(d);if(h.transform!=="none"||h.perspective!=="none"||h.contain==="paint"||["transform","perspective"].indexOf(h.willChange)!==-1||t&&h.willChange==="filter"||t&&h.filter&&h.filter!=="none")return d;d=d.parentNode}return null}o(i3,"getContainingBlock");function as(e){for(var t=Br(e),n=GN(e);n&&WS(n)&&wi(n).position==="static";)n=GN(n);return n&&(Pn(n)==="html"||Pn(n)==="body"&&wi(n).position==="static")?t:n||i3(e)||t}o(as,"getOffsetParent");function Ff(e){return["top","bottom"].indexOf(e)>=0?"x":"y"}o(Ff,"getMainAxisFromPlacement");var Ao=Math.max,au=Math.min,cm=Math.round;function Bf(e,t,n){return Ao(e,au(t,n))}o(Bf,"within");function pm(){return{top:0,right:0,bottom:0,left:0}}o(pm,"getFreshSideObject");function dm(e){return Object.assign({},pm(),e)}o(dm,"mergePaddingObject");function hm(e,t){return t.reduce(function(n,l){return n[l]=e,n},{})}o(hm,"expandToHashMap");var o3=o(function(t,n){return t=typeof t=="function"?t(Object.assign({},n.rects,{placement:n.placement})):t,dm(typeof t!="number"?t:hm(t,su))},"toPaddingObject");function s3(e){var t,n=e.state,l=e.name,d=e.options,h=n.elements.arrow,c=n.modifiersData.popperOffsets,v=On(n.placement),C=Ff(v),k=[an,ln].indexOf(v)>=0,O=k?"height":"width";if(!(!h||!c)){var j=o3(d.padding,n),B=If(h),X=C==="y"?Gr:an,J=C==="y"?Ln:ln,Z=n.rects.reference[O]+n.rects.reference[C]-c[C]-n.rects.popper[O],R=c[C]-n.rects.reference[C],A=as(h),I=A?C==="y"?A.clientHeight||0:A.clientWidth||0:0,G=Z/2-R/2,K=j[X],se=I-B[O]-j[J],ne=I/2-B[O]/2+G,pe=Bf(K,ne,se),me=C;n.modifiersData[l]=(t={},t[me]=pe,t.centerOffset=pe-ne,t)}}o(s3,"arrow");function l3(e){var t=e.state,n=e.options,l=n.element,d=l===void 0?"[data-popper-arrow]":l;d!=null&&(typeof d=="string"&&(d=t.elements.popper.querySelector(d),!d)||!fm(t.elements.popper,d)||(t.elements.arrow=d))}o(l3,"effect");var YN={name:"arrow",enabled:!0,phase:"main",fn:s3,effect:l3,requires:["popperOffsets"],requiresIfExists:["preventOverflow"]};var a3={top:"auto",right:"auto",bottom:"auto",left:"auto"};function u3(e){var t=e.x,n=e.y,l=window,d=l.devicePixelRatio||1;return{x:cm(cm(t*d)/d)||0,y:cm(cm(n*d)/d)||0}}o(u3,"roundOffsetsByDPR");function XN(e){var t,n=e.popper,l=e.popperRect,d=e.placement,h=e.offsets,c=e.position,v=e.gpuAcceleration,C=e.adaptive,k=e.roundOffsets,O=k===!0?u3(h):typeof k=="function"?k(h):h,j=O.x,B=j===void 0?0:j,X=O.y,J=X===void 0?0:X,Z=h.hasOwnProperty("x"),R=h.hasOwnProperty("y"),A=an,I=Gr,G=window;if(C){var K=as(n),se="clientHeight",ne="clientWidth";K===Br(n)&&(K=Bn(n),wi(K).position!=="static"&&(se="scrollHeight",ne="scrollWidth")),K=K,d===Gr&&(I=Ln,J-=K[se]-l.height,J*=v?1:-1),d===an&&(A=ln,B-=K[ne]-l.width,B*=v?1:-1)}var pe=Object.assign({position:c},C&&a3);if(v){var me;return Object.assign({},pe,(me={},me[I]=R?"0":"",me[A]=Z?"0":"",me.transform=(G.devicePixelRatio||1)<2?"translate("+B+"px, "+J+"px)":"translate3d("+B+"px, "+J+"px, 0)",me))}return Object.assign({},pe,(t={},t[I]=R?J+"px":"",t[A]=Z?B+"px":"",t.transform="",t))}o(XN,"mapToStyles");function f3(e){var t=e.state,n=e.options,l=n.gpuAcceleration,d=l===void 0?!0:l,h=n.adaptive,c=h===void 0?!0:h,v=n.roundOffsets,C=v===void 0?!0:v;if(!1)var k;var O={placement:On(t.placement),popper:t.elements.popper,popperRect:t.rects.popper,gpuAcceleration:d};t.modifiersData.popperOffsets!=null&&(t.styles.popper=Object.assign({},t.styles.popper,XN(Object.assign({},O,{offsets:t.modifiersData.popperOffsets,position:t.options.strategy,adaptive:c,roundOffsets:C})))),t.modifiersData.arrow!=null&&(t.styles.arrow=Object.assign({},t.styles.arrow,XN(Object.assign({},O,{offsets:t.modifiersData.arrow,position:"absolute",adaptive:!1,roundOffsets:C})))),t.attributes.popper=Object.assign({},t.attributes.popper,{"data-popper-placement":t.placement})}o(f3,"computeStyles");var QN={name:"computeStyles",enabled:!0,phase:"beforeWrite",fn:f3,data:{}};var ry={passive:!0};function c3(e){var t=e.state,n=e.instance,l=e.options,d=l.scroll,h=d===void 0?!0:d,c=l.resize,v=c===void 0?!0:c,C=Br(t.elements.popper),k=[].concat(t.scrollParents.reference,t.scrollParents.popper);return h&&k.forEach(function(O){O.addEventListener("scroll",n.update,ry)}),v&&C.addEventListener("resize",n.update,ry),function(){h&&k.forEach(function(O){O.removeEventListener("scroll",n.update,ry)}),v&&C.removeEventListener("resize",n.update,ry)}}o(c3,"effect");var ZN={name:"eventListeners",enabled:!0,phase:"write",fn:o(function(){},"fn"),effect:c3,data:{}};var p3={left:"right",right:"left",bottom:"top",top:"bottom"};function zp(e){return e.replace(/left|right|bottom|top/g,function(t){return p3[t]})}o(zp,"getOppositePlacement");var d3={start:"end",end:"start"};function ny(e){return e.replace(/start|end/g,function(t){return d3[t]})}o(ny,"getOppositeVariationPlacement");function Hf(e){var t=Br(e),n=t.pageXOffset,l=t.pageYOffset;return{scrollLeft:n,scrollTop:l}}o(Hf,"getWindowScroll");function Wf(e){return Mo(Bn(e)).left+Hf(e).scrollLeft}o(Wf,"getWindowScrollBarX");function US(e){var t=Br(e),n=Bn(e),l=t.visualViewport,d=n.clientWidth,h=n.clientHeight,c=0,v=0;return l&&(d=l.width,h=l.height,/^((?!chrome|android).)*safari/i.test(navigator.userAgent)||(c=l.offsetLeft,v=l.offsetTop)),{width:d,height:h,x:c+Wf(e),y:v}}o(US,"getViewportRect");function zS(e){var t,n=Bn(e),l=Hf(e),d=(t=e.ownerDocument)==null?void 0:t.body,h=Ao(n.scrollWidth,n.clientWidth,d?d.scrollWidth:0,d?d.clientWidth:0),c=Ao(n.scrollHeight,n.clientHeight,d?d.scrollHeight:0,d?d.clientHeight:0),v=-l.scrollLeft+Wf(e),C=-l.scrollTop;return wi(d||n).direction==="rtl"&&(v+=Ao(n.clientWidth,d?d.clientWidth:0)-h),{width:h,height:c,x:v,y:C}}o(zS,"getDocumentRect");function Uf(e){var t=wi(e),n=t.overflow,l=t.overflowX,d=t.overflowY;return/auto|scroll|overlay|hidden/.test(n+d+l)}o(Uf,"isScrollParent");function iy(e){return["html","body","#document"].indexOf(Pn(e))>=0?e.ownerDocument.body:Yr(e)&&Uf(e)?e:iy(Fl(e))}o(iy,"getScrollParent");function uu(e,t){var n;t===void 0&&(t=[]);var l=iy(e),d=l===((n=e.ownerDocument)==null?void 0:n.body),h=Br(l),c=d?[h].concat(h.visualViewport||[],Uf(l)?l:[]):l,v=t.concat(c);return d?v:v.concat(uu(Fl(c)))}o(uu,"listScrollParents");function $p(e){return Object.assign({},e,{left:e.x,top:e.y,right:e.x+e.width,bottom:e.y+e.height})}o($p,"rectToClientRect");function h3(e){var t=Mo(e);return t.top=t.top+e.clientTop,t.left=t.left+e.clientLeft,t.bottom=t.top+e.clientHeight,t.right=t.left+e.clientWidth,t.width=e.clientWidth,t.height=e.clientHeight,t.x=t.left,t.y=t.top,t}o(h3,"getInnerBoundingClientRect");function JN(e,t){return t===J0?$p(US(e)):Yr(t)?h3(t):$p(zS(Bn(e)))}o(JN,"getClientRectFromMixedType");function m3(e){var t=uu(Fl(e)),n=["absolute","fixed"].indexOf(wi(e).position)>=0,l=n&&Yr(e)?as(e):e;return Il(l)?t.filter(function(d){return Il(d)&&fm(d,l)&&Pn(d)!=="body"}):[]}o(m3,"getClippingParents");function $S(e,t,n){var l=t==="clippingParents"?m3(e):[].concat(t),d=[].concat(l,[n]),h=d[0],c=d.reduce(function(v,C){var k=JN(e,C);return v.top=Ao(k.top,v.top),v.right=au(k.right,v.right),v.bottom=au(k.bottom,v.bottom),v.left=Ao(k.left,v.left),v},JN(e,h));return c.width=c.right-c.left,c.height=c.bottom-c.top,c.x=c.left,c.y=c.top,c}o($S,"getClippingRect");function Qs(e){return e.split("-")[1]}o(Qs,"getVariation");function mm(e){var t=e.reference,n=e.element,l=e.placement,d=l?On(l):null,h=l?Qs(l):null,c=t.x+t.width/2-n.width/2,v=t.y+t.height/2-n.height/2,C;switch(d){case Gr:C={x:c,y:t.y-n.height};break;case Ln:C={x:c,y:t.y+t.height};break;case ln:C={x:t.x+t.width,y:v};break;case an:C={x:t.x-n.width,y:v};break;default:C={x:t.x,y:t.y}}var k=d?Ff(d):null;if(k!=null){var O=k==="y"?"height":"width";switch(h){case Rl:C[k]=C[k]-(t[O]/2-n[O]/2);break;case Z0:C[k]=C[k]+(t[O]/2-n[O]/2);break;default:}}return C}o(mm,"computeOffsets");function us(e,t){t===void 0&&(t={});var n=t,l=n.placement,d=l===void 0?e.placement:l,h=n.boundary,c=h===void 0?jN:h,v=n.rootBoundary,C=v===void 0?J0:v,k=n.elementContext,O=k===void 0?Up:k,j=n.altBoundary,B=j===void 0?!1:j,X=n.padding,J=X===void 0?0:X,Z=dm(typeof J!="number"?J:hm(J,su)),R=O===Up?qN:Up,A=e.elements.reference,I=e.rects.popper,G=e.elements[B?R:O],K=$S(Il(G)?G:G.contextElement||Bn(e.elements.popper),c,C),se=Mo(A),ne=mm({reference:se,element:I,strategy:"absolute",placement:d}),pe=$p(Object.assign({},I,ne)),me=O===Up?pe:se,xe={top:K.top-me.top+Z.top,bottom:me.bottom-K.bottom+Z.bottom,left:K.left-me.left+Z.left,right:me.right-K.right+Z.right},Ve=e.modifiersData.offset;if(O===Up&&Ve){var tt=Ve[d];Object.keys(xe).forEach(function(_e){var St=[ln,Ln].indexOf(_e)>=0?1:-1,We=[Gr,Ln].indexOf(_e)>=0?"y":"x";xe[_e]+=tt[We]*St})}return xe}o(us,"detectOverflow");function jS(e,t){t===void 0&&(t={});var n=t,l=n.placement,d=n.boundary,h=n.rootBoundary,c=n.padding,v=n.flipVariations,C=n.allowedAutoPlacements,k=C===void 0?ey:C,O=Qs(l),j=O?v?HS:HS.filter(function(J){return Qs(J)===O}):su,B=j.filter(function(J){return k.indexOf(J)>=0});B.length===0&&(B=j);var X=B.reduce(function(J,Z){return J[Z]=us(e,{placement:Z,boundary:d,rootBoundary:h,padding:c})[On(Z)],J},{});return Object.keys(X).sort(function(J,Z){return X[J]-X[Z]})}o(jS,"computeAutoPlacement");function g3(e){if(On(e)===Q0)return[];var t=zp(e);return[ny(e),t,ny(t)]}o(g3,"getExpandedFallbackPlacements");function v3(e){var t=e.state,n=e.options,l=e.name;if(!t.modifiersData[l]._skip){for(var d=n.mainAxis,h=d===void 0?!0:d,c=n.altAxis,v=c===void 0?!0:c,C=n.fallbackPlacements,k=n.padding,O=n.boundary,j=n.rootBoundary,B=n.altBoundary,X=n.flipVariations,J=X===void 0?!0:X,Z=n.allowedAutoPlacements,R=t.options.placement,A=On(R),I=A===R,G=C||(I||!J?[zp(R)]:g3(R)),K=[R].concat(G).reduce(function(ut,Lr){return ut.concat(On(Lr)===Q0?jS(t,{placement:Lr,boundary:O,rootBoundary:j,padding:k,flipVariations:J,allowedAutoPlacements:Z}):Lr)},[]),se=t.rects.reference,ne=t.rects.popper,pe=new Map,me=!0,xe=K[0],Ve=0;Ve=0,Ke=We?"width":"height",Ge=us(t,{placement:tt,boundary:O,rootBoundary:j,altBoundary:B,padding:k}),Xe=We?St?ln:an:St?Ln:Gr;se[Ke]>ne[Ke]&&(Xe=zp(Xe));var nr=zp(Xe),ct=[];if(h&&ct.push(Ge[_e]<=0),v&&ct.push(Ge[Xe]<=0,Ge[nr]<=0),ct.every(function(ut){return ut})){xe=tt,me=!1;break}pe.set(tt,ct)}if(me)for(var Hr=J?3:1,Qt=o(function(Lr){var zt=K.find(function($t){var ie=pe.get($t);if(ie)return ie.slice(0,Lr).every(function(rt){return rt})});if(zt)return xe=zt,"break"},"_loop"),_t=Hr;_t>0;_t--){var Ct=Qt(_t);if(Ct==="break")break}t.placement!==xe&&(t.modifiersData[l]._skip=!0,t.placement=xe,t.reset=!0)}}o(v3,"flip");var eL={name:"flip",enabled:!0,phase:"main",fn:v3,requiresIfExists:["offset"],data:{_skip:!1}};function tL(e,t,n){return n===void 0&&(n={x:0,y:0}),{top:e.top-t.height-n.y,right:e.right-t.width+n.x,bottom:e.bottom-t.height+n.y,left:e.left-t.width-n.x}}o(tL,"getSideOffsets");function rL(e){return[Gr,ln,Ln,an].some(function(t){return e[t]>=0})}o(rL,"isAnySideFullyClipped");function y3(e){var t=e.state,n=e.name,l=t.rects.reference,d=t.rects.popper,h=t.modifiersData.preventOverflow,c=us(t,{elementContext:"reference"}),v=us(t,{altBoundary:!0}),C=tL(c,l),k=tL(v,d,h),O=rL(C),j=rL(k);t.modifiersData[n]={referenceClippingOffsets:C,popperEscapeOffsets:k,isReferenceHidden:O,hasPopperEscaped:j},t.attributes.popper=Object.assign({},t.attributes.popper,{"data-popper-reference-hidden":O,"data-popper-escaped":j})}o(y3,"hide");var nL={name:"hide",enabled:!0,phase:"main",requiresIfExists:["preventOverflow"],fn:y3};function w3(e,t,n){var l=On(e),d=[an,Gr].indexOf(l)>=0?-1:1,h=typeof n=="function"?n(Object.assign({},t,{placement:e})):n,c=h[0],v=h[1];return c=c||0,v=(v||0)*d,[an,ln].indexOf(l)>=0?{x:v,y:c}:{x:c,y:v}}o(w3,"distanceAndSkiddingToXY");function x3(e){var t=e.state,n=e.options,l=e.name,d=n.offset,h=d===void 0?[0,0]:d,c=ey.reduce(function(O,j){return O[j]=w3(j,t.rects,h),O},{}),v=c[t.placement],C=v.x,k=v.y;t.modifiersData.popperOffsets!=null&&(t.modifiersData.popperOffsets.x+=C,t.modifiersData.popperOffsets.y+=k),t.modifiersData[l]=c}o(x3,"offset");var iL={name:"offset",enabled:!0,phase:"main",requires:["popperOffsets"],fn:x3};function S3(e){var t=e.state,n=e.name;t.modifiersData[n]=mm({reference:t.rects.reference,element:t.rects.popper,strategy:"absolute",placement:t.placement})}o(S3,"popperOffsets");var oL={name:"popperOffsets",enabled:!0,phase:"read",fn:S3,data:{}};function qS(e){return e==="x"?"y":"x"}o(qS,"getAltAxis");function C3(e){var t=e.state,n=e.options,l=e.name,d=n.mainAxis,h=d===void 0?!0:d,c=n.altAxis,v=c===void 0?!1:c,C=n.boundary,k=n.rootBoundary,O=n.altBoundary,j=n.padding,B=n.tether,X=B===void 0?!0:B,J=n.tetherOffset,Z=J===void 0?0:J,R=us(t,{boundary:C,rootBoundary:k,padding:j,altBoundary:O}),A=On(t.placement),I=Qs(t.placement),G=!I,K=Ff(A),se=qS(K),ne=t.modifiersData.popperOffsets,pe=t.rects.reference,me=t.rects.popper,xe=typeof Z=="function"?Z(Object.assign({},t.rects,{placement:t.placement})):Z,Ve={x:0,y:0};if(!!ne){if(h||v){var tt=K==="y"?Gr:an,_e=K==="y"?Ln:ln,St=K==="y"?"height":"width",We=ne[K],Ke=ne[K]+R[tt],Ge=ne[K]-R[_e],Xe=X?-me[St]/2:0,nr=I===Rl?pe[St]:me[St],ct=I===Rl?-me[St]:-pe[St],Hr=t.elements.arrow,Qt=X&&Hr?If(Hr):{width:0,height:0},_t=t.modifiersData["arrow#persistent"]?t.modifiersData["arrow#persistent"].padding:pm(),Ct=_t[tt],ut=_t[_e],Lr=Bf(0,pe[St],Qt[St]),zt=G?pe[St]/2-Xe-Lr-Ct-xe:nr-Lr-Ct-xe,$t=G?-pe[St]/2+Xe+Lr+ut+xe:ct+Lr+ut+xe,ie=t.elements.arrow&&as(t.elements.arrow),rt=ie?K==="y"?ie.clientTop||0:ie.clientLeft||0:0,Pr=t.modifiersData.offset?t.modifiersData.offset[t.placement][K]:0,Gt=ne[K]+zt-Pr-rt,Yt=ne[K]+$t-Pr;if(h){var Se=Bf(X?au(Ke,Gt):Ke,We,X?Ao(Ge,Yt):Ge);ne[K]=Se,Ve[K]=Se-We}if(v){var Or=K==="x"?Gr:an,fn=K==="x"?Ln:ln,Un=ne[se],si=Un+R[Or],cn=Un-R[fn],Zt=Bf(X?au(si,Gt):si,Un,X?Ao(cn,Yt):cn);ne[se]=Zt,Ve[se]=Zt-Un}}t.modifiersData[l]=Ve}}o(C3,"preventOverflow");var sL={name:"preventOverflow",enabled:!0,phase:"main",fn:C3,requiresIfExists:["offset"]};function VS(e){return{scrollLeft:e.scrollLeft,scrollTop:e.scrollTop}}o(VS,"getHTMLElementScroll");function KS(e){return e===Br(e)||!Yr(e)?Hf(e):VS(e)}o(KS,"getNodeScroll");function b3(e){var t=e.getBoundingClientRect(),n=t.width/e.offsetWidth||1,l=t.height/e.offsetHeight||1;return n!==1||l!==1}o(b3,"isElementScaled");function GS(e,t,n){n===void 0&&(n=!1);var l=Yr(t),d=Yr(t)&&b3(t),h=Bn(t),c=Mo(e,d),v={scrollLeft:0,scrollTop:0},C={x:0,y:0};return(l||!l&&!n)&&((Pn(t)!=="body"||Uf(h))&&(v=KS(t)),Yr(t)?(C=Mo(t,!0),C.x+=t.clientLeft,C.y+=t.clientTop):h&&(C.x=Wf(h))),{x:c.left+v.scrollLeft-C.x,y:c.top+v.scrollTop-C.y,width:c.width,height:c.height}}o(GS,"getCompositeRect");function E3(e){var t=new Map,n=new Set,l=[];e.forEach(function(h){t.set(h.name,h)});function d(h){n.add(h.name);var c=[].concat(h.requires||[],h.requiresIfExists||[]);c.forEach(function(v){if(!n.has(v)){var C=t.get(v);C&&d(C)}}),l.push(h)}return o(d,"sort"),e.forEach(function(h){n.has(h.name)||d(h)}),l}o(E3,"order");function YS(e){var t=E3(e);return VN.reduce(function(n,l){return n.concat(t.filter(function(d){return d.phase===l}))},[])}o(YS,"orderModifiers");function XS(e){var t;return function(){return t||(t=new Promise(function(n){Promise.resolve().then(function(){t=void 0,n(e())})})),t}}o(XS,"debounce");function QS(e){var t=e.reduce(function(n,l){var d=n[l.name];return n[l.name]=d?Object.assign({},d,l,{options:Object.assign({},d.options,l.options),data:Object.assign({},d.data,l.data)}):l,n},{});return Object.keys(t).map(function(n){return t[n]})}o(QS,"mergeByName");var lL={placement:"bottom",modifiers:[],strategy:"absolute"};function aL(){for(var e=arguments.length,t=new Array(e),n=0;nxi.default.createElement("li",{role:"separator",className:"divider"}),"Divider");function ii(l){var d=l,{onClick:e,children:t}=d,n=Ws(d,["onClick","children"]);return xi.default.createElement("li",null,xi.default.createElement("a",ke({href:"#",onClick:o(c=>{c.preventDefault(),e()},"click")},n),t))}o(ii,"MenuItem");var cu=xi.default.memo(o(function(v){var C=v,{text:t,children:n,options:l,className:d,onOpen:h}=C,c=Ws(C,["text","children","options","className","onOpen"]);let[k,O]=(0,xi.useState)(null),[j,B]=(0,xi.useState)(!1),[X,J]=(0,xi.useState)(null),{styles:Z,attributes:R}=JS(k,X,ke({},l)),A=o(G=>{B(G),h&&h(G)},"setOpen");(0,xi.useEffect)(()=>{!X||document.addEventListener("click",G=>{X.contains(G.target)?document.addEventListener("click",()=>A(!1),{once:!0}):(G.preventDefault(),G.stopPropagation(),A(!1))},{once:!0,capture:!0})},[X]);let I;return j?I=xi.default.createElement("ul",ke({className:"dropdown-menu show",ref:J,style:Z.popper},R.popper),n):I=null,xi.default.createElement(xi.default.Fragment,null,xi.default.createElement("a",ke({href:"#",ref:O,className:(0,dL.default)(d,{open:j}),onClick:G=>{G.preventDefault(),A(!0)}},c),t),I)},"Dropdown"));function gm({value:e,onChange:t}){let n=qe(d=>d.backendState.contentViews||[]),l=zf.default.createElement("span",null,zf.default.createElement("i",{className:"fa fa-fw fa-files-o"}),"\xA0",zf.default.createElement("b",null,"View:")," ",e.toLowerCase()," ",zf.default.createElement("span",{className:"caret"}));return zf.default.createElement(cu,{text:l,className:"btn btn-default btn-xs",options:{placement:"top-start"}},n.map(d=>zf.default.createElement(ii,{key:d,onClick:()=>t(d)},d.toLowerCase().replace("_"," "))))}o(gm,"ViewSelector");function eC({flow:e,message:t}){let n=Xt(),l=e.request===t?"request":"response",d=qe(J=>J.ui.flow.contentViewFor[e.id+l]||"Auto"),h=(0,tr.useRef)(null),[c,v]=(0,tr.useState)(qe(J=>J.options.content_view_lines_cutoff)),C=(0,tr.useCallback)(()=>v(Math.max(1024,c*2)),[c]),[k,O]=(0,tr.useState)(!1),j;k?j=Kr.getContentURL(e,t):j=Kr.getContentURL(e,t,d,c+1);let B=K0(j,t.contentHash),X=(0,tr.useMemo)(()=>{if(B&&!k)try{return JSON.parse(B)}catch(J){return{description:"Network Error",lines:[[["error",`${B}`]]]}}else return},[B]);if(k)return tr.default.createElement("div",{className:"contentview",key:"edit"},tr.default.createElement("div",{className:"controls"},tr.default.createElement("h5",null,"[Editing]"),tr.default.createElement(kr,{onClick:o(()=>Ia(this,null,function*(){var R;let Z=(R=h.current)==null?void 0:R.getContent();yield n(Wi(e,{[l]:{content:Z}})),O(!1)}),"save"),icon:"fa-check text-success",className:"btn-xs"},"Done"),"\xA0",tr.default.createElement(kr,{onClick:()=>O(!1),icon:"fa-times text-danger",className:"btn-xs"},"Cancel")),tr.default.createElement(um,{ref:h,initialContent:B||""}));{let J=X?X.description:"Loading...";return tr.default.createElement("div",{className:"contentview",key:"view"},tr.default.createElement("div",{className:"controls"},tr.default.createElement("h5",null,J),tr.default.createElement(kr,{onClick:()=>O(!0),icon:"fa-edit",className:"btn-xs"},"Edit"),"\xA0",tr.default.createElement(G0,{icon:"fa-upload",text:"Replace",title:"Upload a file to replace the content.",onOpenFile:Z=>n(nN(e,Z,l)),className:"btn btn-default btn-xs"}),"\xA0",tr.default.createElement(gm,{value:d,onChange:Z=>n(w0(e.id+l,Z))})),tC.matches(t)&&tr.default.createElement(tC,{flow:e,message:t}),tr.default.createElement(Y0,{lines:(X==null?void 0:X.lines)||[],maxLines:c,showMore:C}))}}o(eC,"HttpMessage");var O3=/^image\/(png|jpe?g|gif|webp|vnc.microsoft.icon|x-icon)$/i;tC.matches=e=>O3.test(Kr.getContentType(e)||"");function tC({flow:e,message:t}){return tr.default.createElement("div",{className:"flowview-image"},tr.default.createElement("img",{src:Kr.getContentURL(e,t),alt:"preview",className:"img-thumbnail"}))}o(tC,"ViewImage");function M3({flow:e}){let t=Xt();return Ut.createElement("div",{className:"first-line request-line"},Ut.createElement("div",null,Ut.createElement(Df,{content:e.request.method,onEditDone:n=>t(Wi(e,{request:{method:n}})),isValid:n=>n.length>0}),"\xA0",Ut.createElement(Df,{content:Oo.pretty_url(e.request),onEditDone:n=>t(Wi(e,{request:ke({path:""},hS(n))})),isValid:n=>{var l;return!!((l=hS(n))==null?void 0:l.host)}}),"\xA0",Ut.createElement(Df,{content:e.request.http_version,onEditDone:n=>t(Wi(e,{request:{http_version:n}})),isValid:mS})))}o(M3,"RequestLine");function A3({flow:e}){let t=Xt();return Ut.createElement("div",{className:"first-line response-line"},Ut.createElement(Df,{content:e.response.http_version,onEditDone:n=>t(Wi(e,{response:{http_version:n}})),isValid:mS}),"\xA0",Ut.createElement(Df,{content:e.response.status_code+"",onEditDone:n=>t(Wi(e,{response:{code:parseInt(n)}})),isValid:n=>/^\d+$/.test(n)}),e.response.http_version!=="HTTP/2.0"&&Ut.createElement(Ut.Fragment,null,"\xA0",Ut.createElement(Xs,{content:e.response.reason,onEditDone:n=>t(Wi(e,{response:{msg:n}}))})))}o(A3,"ResponseLine");function D3({flow:e,message:t}){let n=Xt(),l=e.request===t?"request":"response";return Ut.createElement(Bp,{className:"headers",data:t.headers,onChange:d=>n(Wi(e,{[l]:{headers:d}}))})}o(D3,"Headers");function R3({flow:e,message:t}){let n=Xt(),l=e.request===t?"request":"response";return!Kr.get_first_header(t,/^trailer$/i)?null:Ut.createElement(Ut.Fragment,null,Ut.createElement("hr",null),Ut.createElement("h5",null,"HTTP Trailers"),Ut.createElement(Bp,{className:"trailers",data:t.trailers,onChange:h=>n(Wi(e,{[l]:{trailers:h}}))}))}o(R3,"Trailers");var mL=Ut.memo(o(function({flow:t,message:n}){let l=t.request===n?"request":"response",d=t.request===n?M3:A3;return Ut.createElement("section",{className:l},Ut.createElement(d,{flow:t}),Ut.createElement(D3,{flow:t,message:n}),Ut.createElement("hr",null),Ut.createElement(eC,{key:t.id+l,flow:t,message:n}),Ut.createElement(R3,{flow:t,message:n}))},"Message"));function rC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ut.createElement(mL,{flow:e,message:e.request})}o(rC,"Request");rC.displayName="Request";function nC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ut.createElement(mL,{flow:e,message:e.response})}o(nC,"Response");nC.displayName="Response";var Ye=fe(Oe());var I3=o(({message:e})=>Ye.createElement("div",null,e.query?e.op_code:e.response_code,"\xA0",e.truncation?"(Truncated)":""),"Summary"),F3=o(({message:e})=>Ye.createElement(Ye.Fragment,null,Ye.createElement("h5",null,e.recursion_desired?"Recursive ":"","Question"),Ye.createElement("table",null,Ye.createElement("thead",null,Ye.createElement("tr",null,Ye.createElement("th",null,"Name"),Ye.createElement("th",null,"Type"),Ye.createElement("th",null,"Class"))),Ye.createElement("tbody",null,e.questions.map((t,n)=>Ye.createElement("tr",{key:n},Ye.createElement("td",null,t.name),Ye.createElement("td",null,t.type),Ye.createElement("td",null,t.class)))))),"Questions"),iC=o(({name:e,values:t})=>Ye.createElement(Ye.Fragment,null,Ye.createElement("h5",null,e),t.length>0?Ye.createElement("table",null,Ye.createElement("thead",null,Ye.createElement("tr",null,Ye.createElement("th",null,"Name"),Ye.createElement("th",null,"Type"),Ye.createElement("th",null,"Class"),Ye.createElement("th",null,"TTL"),Ye.createElement("th",null,"Data"))),Ye.createElement("tbody",null,t.map((n,l)=>Ye.createElement("tr",{key:l},Ye.createElement("td",null,n.name),Ye.createElement("td",null,n.type),Ye.createElement("td",null,n.class),Ye.createElement("td",null,n.ttl),Ye.createElement("td",null,n.data))))):"\u2014"),"ResourceRecords"),gL=o(({type:e,message:t})=>Ye.createElement("section",{className:"dns-"+e},Ye.createElement("div",{className:`first-line ${e}-line`},Ye.createElement(I3,{message:t})),Ye.createElement(F3,{message:t}),Ye.createElement("hr",null),Ye.createElement(iC,{name:`${t.authoritative_answer?"Authoritative ":""}${t.recursion_available?"Recursive ":""}Answer`,values:t.answers}),Ye.createElement("hr",null),Ye.createElement(iC,{name:"Authority",values:t.authorities}),Ye.createElement("hr",null),Ye.createElement(iC,{name:"Additional",values:t.additionals})),"Message");function oC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ye.createElement(gL,{type:"request",message:e.request})}o(oC,"Request");oC.displayName="Request";function sC(){let e=qe(t=>t.flows.byId[t.flows.selected[0]]);return Ye.createElement(gL,{type:"response",message:e.response})}o(sC,"Response");sC.displayName="Response";var Ee=fe(Oe());function vL({conn:e}){var n,l,d;let t=null;return"address"in e?t=Ee.createElement(Ee.Fragment,null,Ee.createElement("tr",null,Ee.createElement("td",null,"Address:"),Ee.createElement("td",null,(n=e.address)==null?void 0:n.join(":"))),e.peername&&Ee.createElement("tr",null,Ee.createElement("td",null,"Resolved address:"),Ee.createElement("td",null,e.peername.join(":"))),e.sockname&&Ee.createElement("tr",null,Ee.createElement("td",null,"Source address:"),Ee.createElement("td",null,e.sockname.join(":")))):((l=e.peername)==null?void 0:l[0])&&(t=Ee.createElement(Ee.Fragment,null,Ee.createElement("tr",null,Ee.createElement("td",null,"Address:"),Ee.createElement("td",null,(d=e.peername)==null?void 0:d.join(":"))))),Ee.createElement("table",{className:"connection-table"},Ee.createElement("tbody",null,t,e.sni?Ee.createElement("tr",null,Ee.createElement("td",null,Ee.createElement("abbr",{title:"TLS Server Name Indication"},"SNI"),":"),Ee.createElement("td",null,e.sni)):null,e.alpn?Ee.createElement("tr",null,Ee.createElement("td",null,Ee.createElement("abbr",{title:"ALPN protocol negotiated"},"ALPN"),":"),Ee.createElement("td",null,e.alpn)):null,e.tls_version?Ee.createElement("tr",null,Ee.createElement("td",null,"TLS Version:"),Ee.createElement("td",null,e.tls_version)):null,e.cipher?Ee.createElement("tr",null,Ee.createElement("td",null,"TLS Cipher:"),Ee.createElement("td",null,e.cipher)):null))}o(vL,"ConnectionInfo");function yL(e){return Ee.createElement("dl",{className:"cert-attributes"},e.map(([t,n])=>Ee.createElement(Ee.Fragment,{key:t},Ee.createElement("dt",null,t),Ee.createElement("dd",null,n))))}o(yL,"attrList");function B3({flow:e}){var n;let t=(n=e.server_conn)==null?void 0:n.cert;return t?Ee.createElement(Ee.Fragment,null,Ee.createElement("h4",{key:"name"},"Server Certificate"),Ee.createElement("table",{className:"certificate-table"},Ee.createElement("tbody",null,Ee.createElement("tr",null,Ee.createElement("td",null,"Type"),Ee.createElement("td",null,t.keyinfo[0],", ",t.keyinfo[1]," bits")),Ee.createElement("tr",null,Ee.createElement("td",null,"SHA256 digest"),Ee.createElement("td",null,t.sha256)),Ee.createElement("tr",null,Ee.createElement("td",null,"Valid from"),Ee.createElement("td",null,no(t.notbefore,{milliseconds:!1}))),Ee.createElement("tr",null,Ee.createElement("td",null,"Valid to"),Ee.createElement("td",null,no(t.notafter,{milliseconds:!1}))),Ee.createElement("tr",null,Ee.createElement("td",null,"Subject Alternative Names"),Ee.createElement("td",null,t.altnames.join(", "))),Ee.createElement("tr",null,Ee.createElement("td",null,"Subject"),Ee.createElement("td",null,yL(t.subject))),Ee.createElement("tr",null,Ee.createElement("td",null,"Issuer"),Ee.createElement("td",null,yL(t.issuer))),Ee.createElement("tr",null,Ee.createElement("td",null,"Serial"),Ee.createElement("td",null,t.serial))))):Ee.createElement(Ee.Fragment,null)}o(B3,"CertificateInfo");function sy({flow:e}){var t;return Ee.createElement("section",{className:"detail"},Ee.createElement("h4",null,"Client Connection"),Ee.createElement(vL,{conn:e.client_conn}),((t=e.server_conn)==null?void 0:t.address)&&Ee.createElement(Ee.Fragment,null,Ee.createElement("h4",null,"Server Connection"),Ee.createElement(vL,{conn:e.server_conn})),Ee.createElement(B3,{flow:e}))}o(sy,"Connection");sy.displayName="Connection";var vm=fe(Oe());function ly({flow:e}){return vm.createElement("section",{className:"error"},vm.createElement("div",{className:"alert alert-warning"},e.error.msg,vm.createElement("div",null,vm.createElement("small",null,no(e.error.timestamp)))))}o(ly,"Error");ly.displayName="Error";var fs=fe(Oe());function H3({t:e,deltaTo:t,title:n}){return e?fs.createElement("tr",null,fs.createElement("td",null,n,":"),fs.createElement("td",null,no(e),t&&fs.createElement("span",{className:"text-muted"},"(",S0(1e3*(e-t)),")"))):fs.createElement("tr",null)}o(H3,"TimeStamp");function ay({flow:e}){var l,d,h,c,v,C;let t;e.type==="http"?t=e.request.timestamp_start:t=e.client_conn.timestamp_start;let n=[{title:"Server conn. initiated",t:(l=e.server_conn)==null?void 0:l.timestamp_start,deltaTo:t},{title:"Server conn. TCP handshake",t:(d=e.server_conn)==null?void 0:d.timestamp_tcp_setup,deltaTo:t},{title:"Server conn. TLS handshake",t:(h=e.server_conn)==null?void 0:h.timestamp_tls_setup,deltaTo:t},{title:"Server conn. closed",t:(c=e.server_conn)==null?void 0:c.timestamp_end,deltaTo:t},{title:"Client conn. established",t:e.client_conn.timestamp_start,deltaTo:e.type==="http"?t:void 0},{title:"Client conn. TLS handshake",t:e.client_conn.timestamp_tls_setup,deltaTo:t},{title:"Client conn. closed",t:e.client_conn.timestamp_end,deltaTo:t}];return e.type==="http"&&n.push({title:"First request byte",t:e.request.timestamp_start},{title:"Request complete",t:e.request.timestamp_end,deltaTo:t},{title:"First response byte",t:(v=e.response)==null?void 0:v.timestamp_start,deltaTo:t},{title:"Response complete",t:(C=e.response)==null?void 0:C.timestamp_end,deltaTo:t}),fs.createElement("section",{className:"timing"},fs.createElement("h4",null,"Timing"),fs.createElement("table",{className:"timing-table"},fs.createElement("tbody",null,n.filter(k=>!!k.t).sort((k,O)=>k.t-O.t).map(k=>fs.createElement(H3,ke({key:k.title},k))))))}o(ay,"Timing");ay.displayName="Timing";var pu=fe(Oe());var Zs=fe(Oe()),jp=fe(Oe());function $f({flow:e,messages_meta:t}){let n=Xt(),l=qe(k=>k.ui.flow.contentViewFor[e.id+"messages"]||"Auto"),[d,h]=(0,jp.useState)(qe(k=>k.options.content_view_lines_cutoff)),c=(0,jp.useCallback)(()=>h(Math.max(1024,d*2)),[d]),v=K0(Kr.getContentURL(e,"messages",l,d+1),e.id+t.count),C=(0,jp.useMemo)(()=>{if(v)try{return JSON.parse(v)}catch(k){return[{description:"Network Error",lines:[[["error",`${v}`]]]}]}},[v])||[];return Zs.createElement("div",{className:"contentview"},Zs.createElement("div",{className:"controls"},Zs.createElement("h5",null,t.count," Messages"),Zs.createElement(gm,{value:l,onChange:k=>n(w0(e.id+"messages",k))})),C.map((k,O)=>{let j=`fa fa-fw fa-arrow-${k.from_client?"right text-primary":"left text-danger"}`,B=Zs.createElement("div",{key:O},Zs.createElement("small",null,Zs.createElement("i",{className:j}),Zs.createElement("span",{className:"pull-right"},k.timestamp&&no(k.timestamp))),Zs.createElement(Y0,{lines:k.lines,maxLines:d,showMore:c}));return d-=k.lines.length,B}))}o($f,"Messages");function uy({flow:e}){return pu.createElement("section",{className:"websocket"},pu.createElement("h4",null,"WebSocket"),pu.createElement($f,{flow:e,messages_meta:e.websocket.messages_meta}),pu.createElement(W3,{websocket:e.websocket}))}o(uy,"WebSocket");uy.displayName="WebSocket";function W3({websocket:e}){if(!e.timestamp_end)return null;let t=e.close_reason?`(${e.close_reason})`:"";return pu.createElement("div",null,pu.createElement("i",{className:"fa fa-fw fa-window-close text-muted"}),"\xA0 Closed by ",e.closed_by_client?"client":"server"," with code ",e.close_code," ",t,".",pu.createElement("small",{className:"pull-right"},no(e.timestamp_end)))}o(W3,"CloseSummary");var wL=fe(ti());var lC=fe(Oe());function fy({flow:e}){return lC.createElement("section",{className:"tcp"},lC.createElement($f,{flow:e,messages_meta:e.messages_meta}))}o(fy,"TcpMessages");fy.displayName="Stream Data";var aC=fe(Oe());function cy({flow:e}){return aC.createElement("section",{className:"udp"},aC.createElement($f,{flow:e,messages_meta:e.messages_meta}))}o(cy,"UdpMessages");cy.displayName="Datagrams";var xL={request:rC,response:nC,error:ly,connection:sy,timing:ay,websocket:uy,tcpmessages:fy,udpmessages:cy,dnsrequest:oC,dnsresponse:sC};function py(e){let t;switch(e.type){case"http":t=["request","response","websocket"].filter(n=>e[n]);break;case"tcp":t=["tcpmessages"];break;case"udp":t=["udpmessages"];break;case"dns":t=["request","response"].filter(n=>e[n]).map(n=>"dns"+n);break}return e.error&&t.push("error"),t.push("connection"),t.push("timing"),t}o(py,"tabsForFlow");function uC(){let e=Xt(),t=qe(h=>h.flows.byId[h.flows.selected[0]]),n=py(t),l=qe(h=>h.ui.flow.tab);n.indexOf(l)<0&&(l==="response"&&t.error?l="error":l==="error"&&"response"in t?l="response":l=n[0]);let d=xL[l];return ym.createElement("div",{className:"flow-detail"},ym.createElement("nav",{className:"nav-tabs nav-tabs-sm"},n.map(h=>ym.createElement("a",{key:h,href:"#",className:(0,wL.default)({active:l===h}),onClick:c=>{c.preventDefault(),e(Lf(h))}},xL[h].displayName))),ym.createElement(d,{flow:t}))}o(uC,"FlowView");function SL(e){if(e.ctrlKey||e.metaKey)return()=>{};let t=e.key;return e.preventDefault(),(n,l)=>{let d=l().flows,h=d.byId[l().flows.selected[0]];switch(t){case"k":case"ArrowUp":n(Mf(d,-1));break;case"j":case"ArrowDown":n(Mf(d,1));break;case" ":case"PageDown":n(Mf(d,10));break;case"PageUp":n(Mf(d,-10));break;case"End":n(Mf(d,1e10));break;case"Home":n(Mf(d,-1e10));break;case"Escape":l().ui.modal.activeModal?n(z0()):n(Af(void 0));break;case"ArrowLeft":{if(!h)break;let c=py(h),v=l().ui.flow.tab,C=c[(Math.max(0,c.indexOf(v))-1+c.length)%c.length];n(Lf(C));break}case"Tab":case"ArrowRight":{if(!h)break;let c=py(h),v=l().ui.flow.tab,C=c[(Math.max(0,c.indexOf(v))+1)%c.length];n(Lf(C));break}case"d":{if(!h)return;n(F0(h));break}case"n":{Pf("view.flows.create","get","https://example.com/");break}case"D":{if(!h)return;n(B0(h));break}case"a":{h&&h.intercepted&&n(Op(h));break}case"A":{n(R0());break}case"r":{h&&n(Mp(h));break}case"v":{h&&h.modified&&n(H0(h));break}case"x":{h&&h.intercepted&&n(I0(h));break}case"X":{n(rN());break}case"z":{n(W0());break}default:return}}}o(SL,"onKeyDown");var Zp=fe(Oe());var wm=fe(Oe()),xm=fe(iu()),CL=fe(ti()),qp=class extends wm.Component{constructor(t,n){super(t,n);this.state={applied:!1,startX:0,startY:0},this.onMouseMove=this.onMouseMove.bind(this),this.onMouseDown=this.onMouseDown.bind(this),this.onMouseUp=this.onMouseUp.bind(this),this.onDragEnd=this.onDragEnd.bind(this)}onMouseDown(t){this.setState({startX:t.pageX,startY:t.pageY}),window.addEventListener("mousemove",this.onMouseMove),window.addEventListener("mouseup",this.onMouseUp),window.addEventListener("dragend",this.onDragEnd)}onDragEnd(){xm.default.findDOMNode(this).style.transform="",window.removeEventListener("dragend",this.onDragEnd),window.removeEventListener("mouseup",this.onMouseUp),window.removeEventListener("mousemove",this.onMouseMove)}onMouseUp(t){this.onDragEnd();let n=xm.default.findDOMNode(this),l=n.previousElementSibling,d=l.offsetHeight+t.pageY-this.state.startY;this.props.axis==="x"&&(d=l.offsetWidth+t.pageX-this.state.startX),l.style.flex=`0 0 ${Math.max(0,d)}px`,n.nextElementSibling.style.flex="1 1 auto",this.setState({applied:!0}),this.onResize()}onMouseMove(t){let n=0,l=0;this.props.axis==="x"?n=t.pageX-this.state.startX:l=t.pageY-this.state.startY,xm.default.findDOMNode(this).style.transform=`translate(${n}px, ${l}px)`}onResize(){window.setTimeout(()=>window.dispatchEvent(new CustomEvent("resize")),1)}reset(t){if(!this.state.applied)return;let n=xm.default.findDOMNode(this);n.previousElementSibling.style.flex="",n.nextElementSibling.style.flex="",t||this.setState({applied:!1}),this.onResize()}componentWillUnmount(){this.reset(!0)}render(){return wm.default.createElement("div",{className:(0,CL.default)("splitter",this.props.axis==="x"?"splitter-x":"splitter-y")},wm.default.createElement("div",{onMouseDown:this.onMouseDown,draggable:"true"}))}};o(qp,"Splitter"),qp.defaultProps={axis:"x"};var cs=fe(Oe()),bm=fe(Sm()),hy=fe(iu());var FL=fe(fC());var cC=fe(iu()),OL=Symbol("shouldStick"),ML=o(e=>e.scrollTop+e.clientHeight===e.scrollHeight,"isAtBottom"),dy=o(e=>{var t;return Object.assign((o(t=class extends e{UNSAFE_componentWillUpdate(){let l=cC.default.findDOMNode(this);this[OL]=l.scrollTop&&ML(l),super.UNSAFE_componentWillUpdate&&super.UNSAFE_componentWillUpdate(),super.componentWillUpdate&&super.componentWillUpdate()}componentDidUpdate(){let l=cC.default.findDOMNode(this);this[OL]&&!ML(l)&&(l.scrollTop=l.scrollHeight),super.componentDidUpdate&&super.componentDidUpdate()}},"AutoScrollWrapper"),t.displayName=e.name,t),e)},"default");function jf(e=void 0){if(!e)return{start:0,end:0,paddingTop:0,paddingBottom:0};let{itemCount:t,rowHeight:n,viewportTop:l,viewportHeight:d,itemHeights:h}=e,c=l+d,v=0,C=0,k=0,O=0;if(h){let j=0;for(let B=0;B0&&jv.flows.sort.desc),l=qe(v=>v.flows.sort.column),d=qe(v=>v.options.web_columns),h=n?"sort-desc":"sort-asc",c=d.map(v=>nm[v]).filter(v=>v).concat(Pp);return Cm.createElement("tr",null,c.map(v=>Cm.createElement("th",{className:(0,AL.default)(`col-${v.name}`,l===v.name&&h),key:v.name,onClick:()=>t(tN(v.name===l&&n?void 0:v.name,v.name!==l?!1:!n))},v.headerName)))},"FlowTableHead"));var Vp=fe(Oe()),RL=fe(ti());var IL=Vp.default.memo(o(function({flow:t,selected:n,highlighted:l}){let d=Xt(),h=qe(k=>k.options.web_columns),c=(0,RL.default)({selected:n,highlighted:l,intercepted:t.intercepted,"has-request":t.type==="http"&&t.request,"has-response":t.type==="http"&&t.response}),v=(0,Vp.useCallback)(k=>{let O=k.target;for(;O.parentNode;){if(O.classList.contains("col-quickactions"))return;O=O.parentNode}d(Af(t.id))},[t]),C=h.map(k=>nm[k]).filter(k=>k).concat(Pp);return Vp.default.createElement("tr",{className:c,onClick:v},C.map(k=>Vp.default.createElement(k,{key:k.name,flow:t})))},"FlowRow"));var Em=class extends cs.Component{constructor(t,n){super(t,n);this.state={vScroll:jf()},this.onViewportUpdate=this.onViewportUpdate.bind(this)}UNSAFE_componentWillMount(){window.addEventListener("resize",this.onViewportUpdate)}componentDidMount(){this.onViewportUpdate()}UNSAFE_componentWillUnmount(){window.removeEventListener("resize",this.onViewportUpdate)}componentDidUpdate(){if(this.onViewportUpdate(),!this.shouldScrollIntoView)return;this.shouldScrollIntoView=!1;let{rowHeight:t,flows:n,selected:l}=this.props,d=hy.default.findDOMNode(this),h=hy.default.findDOMNode(this.refs.head),c=h?h.offsetHeight:0,v=n.indexOf(l)*t+c,C=v+t,k=d.scrollTop,O=d.offsetHeight;v-ck+O&&(d.scrollTop=C-O)}UNSAFE_componentWillReceiveProps(t){t.selected&&t.selected!==this.props.selected&&(this.shouldScrollIntoView=!0)}onViewportUpdate(){let t=hy.default.findDOMNode(this),n=t.scrollTop||0,l=jf({viewportTop:n,viewportHeight:t.offsetHeight||0,itemCount:this.props.flows.length,rowHeight:this.props.rowHeight});if(this.state.viewportTop!==n||!(0,FL.default)(this.state.vScroll,l)){let d=Math.min(n,l.end*this.props.rowHeight);this.setState({vScroll:l,viewportTop:d})}}render(){let{vScroll:t,viewportTop:n}=this.state,{flows:l,selected:d,highlight:h}=this.props,c=h?Of.parse(h):()=>!1;return cs.createElement("div",{className:"flow-table",onScroll:this.onViewportUpdate},cs.createElement("table",null,cs.createElement("thead",{ref:"head",style:{transform:`translateY(${n}px)`}},cs.createElement(DL,null)),cs.createElement("tbody",null,cs.createElement("tr",{style:{height:t.paddingTop}}),l.slice(t.start,t.end).map(v=>cs.createElement(IL,{key:v.id,flow:v,selected:v===d,highlighted:c(v)})),cs.createElement("tr",{style:{height:t.paddingBottom}}))))}};o(Em,"FlowTable"),Vc(Em,"propTypes",{flows:bm.default.array.isRequired,rowHeight:bm.default.number,highlight:bm.default.string,selected:bm.default.object}),Vc(Em,"defaultProps",{rowHeight:32});var $3=dy(Em),BL=Hi(e=>({flows:e.flows.view,highlight:e.flows.highlight,selected:e.flows.byId[e.flows.selected[0]]}))($3);var Nr=fe(Oe()),Ty=fe(Oe());var UP=fe(WP());function RC(){let e=qe(n=>n.backendState.servers),t;return e.length===0?t="":e.length===1?t="Configure your client to use the following proxy server:":t="Configure your client to use one of the following proxy servers:",Nr.createElement("div",{style:{padding:"1em 2em"}},Nr.createElement("h3",null,"mitmproxy is running."),Nr.createElement("p",null,"No flows have been recorded yet.",Nr.createElement("br",null),t),Nr.createElement("ul",{className:"fa-ul"},e.map((n,l)=>Nr.createElement("li",{key:n.full_spec},Nr.createElement(BB,ke({},n))))))}o(RC,"CaptureSetup");function BB({description:e,listen_addrs:t,last_exception:n,is_running:l,full_spec:d,wireguard_conf:h}){let c=(0,Ty.useRef)(null);(0,Ty.useEffect)(()=>{h&&c.current&&UP.default.toCanvas(c.current,h,{margin:0,scale:3})},[h]);let v,C=t.length===1||t.length===2&&t[0][1]===t[1][1],k=t.every(B=>["::","0.0.0.0"].includes(B[0]));C&&k?v=rS(["*",t[0][1]]):v=t.map(rS).join(" and "),e=e[0].toUpperCase()+e.substr(1);let O,j;return n?(j="fa-exclamation text-error",O=Nr.createElement(Nr.Fragment,null,e," (",d,"):",Nr.createElement("br",null),n)):l?(j="fa-check text-success",O=`${e} listening at ${v}.`,h&&(O=Nr.createElement(Nr.Fragment,null,O,Nr.createElement("div",{className:"wireguard-config"},Nr.createElement("pre",null,h),Nr.createElement("canvas",{ref:c}))))):(j="fa-pause text-warning",O=Nr.createElement(Nr.Fragment,null,e," (",d,")")),Nr.createElement(Nr.Fragment,null,Nr.createElement("i",{className:`fa fa-li ${j}`}),O)}o(BB,"ServerDescription");function IC(){let e=qe(n=>!!n.flows.byId[n.flows.selected[0]]),t=qe(n=>n.flows.list.length>0);return Zp.createElement("div",{className:"main-view"},t?Zp.createElement(BL,null):Zp.createElement(RC,null),e&&Zp.createElement(qp,{key:"splitter"}),e&&Zp.createElement(uC,{key:"flowDetails"}))}o(IC,"MainView");var Io=fe(Oe()),GP=fe(ti());var oi=fe(Oe());var ps=fe(Oe()),ky=fe(iu()),zP=fe(ti());var oo=fe(Oe());var so=class extends oo.Component{constructor(t,n){super(t,n);this.state={doc:so.doc}}componentDidMount(){so.xhr||(so.xhr=kt("/filter-help").then(t=>t.json()),so.xhr.catch(()=>{so.xhr=null})),this.state.doc||so.xhr.then(t=>{so.doc=t,this.setState({doc:t})})}render(){let{doc:t}=this.state;return t?oo.default.createElement("table",{className:"table table-condensed"},oo.default.createElement("tbody",null,t.commands.map(n=>oo.default.createElement("tr",{key:n[1],onClick:l=>this.props.selectHandler(n[0].split(" ")[0]+" ")},oo.default.createElement("td",null,n[0].replace(" ","\xA0")),oo.default.createElement("td",null,n[1]))),oo.default.createElement("tr",{key:"docs-link"},oo.default.createElement("td",{colSpan:2},oo.default.createElement("a",{href:"https://mitmproxy.org/docs/latest/concepts-filters/",target:"_blank"},oo.default.createElement("i",{className:"fa fa-external-link"}),"\xA0 mitmproxy docs"))))):oo.default.createElement("i",{className:"fa fa-spinner fa-spin"})}};o(so,"FilterDocs");var Yf=class extends ps.Component{constructor(t,n){super(t,n);this.state={value:this.props.value,focus:!1,mousefocus:!1},this.onChange=this.onChange.bind(this),this.onFocus=this.onFocus.bind(this),this.onBlur=this.onBlur.bind(this),this.onKeyDown=this.onKeyDown.bind(this),this.onMouseEnter=this.onMouseEnter.bind(this),this.onMouseLeave=this.onMouseLeave.bind(this),this.selectFilter=this.selectFilter.bind(this)}UNSAFE_componentWillReceiveProps(t){this.setState({value:t.value})}isValid(t){try{return t&&Of.parse(t),!0}catch(n){return!1}}getDesc(){if(!this.state.value)return ps.default.createElement(so,{selectHandler:this.selectFilter});try{return Of.parse(this.state.value).desc}catch(t){return""+t}}onChange(t){let n=t.target.value;this.setState({value:n}),this.isValid(n)&&this.props.onChange(n)}onFocus(){this.setState({focus:!0})}onBlur(){this.setState({focus:!1})}onMouseEnter(){this.setState({mousefocus:!0})}onMouseLeave(){this.setState({mousefocus:!1})}onKeyDown(t){(t.key==="Escape"||t.key==="Enter")&&(this.blur(),this.setState({mousefocus:!1})),t.stopPropagation()}selectFilter(t){this.setState({value:t}),ky.default.findDOMNode(this.refs.input).focus()}blur(){ky.default.findDOMNode(this.refs.input).blur()}select(){ky.default.findDOMNode(this.refs.input).select()}render(){let{type:t,color:n,placeholder:l}=this.props,{value:d,focus:h,mousefocus:c}=this.state;return ps.default.createElement("div",{className:(0,zP.default)("filter-input input-group",{"has-error":!this.isValid(d)})},ps.default.createElement("span",{className:"input-group-addon"},ps.default.createElement("i",{className:"fa fa-fw fa-"+t,style:{color:n}})),ps.default.createElement("input",{type:"text",ref:"input",placeholder:l,className:"form-control",value:d,onChange:this.onChange,onFocus:this.onFocus,onBlur:this.onBlur,onKeyDown:this.onKeyDown}),(h||c)&&ps.default.createElement("div",{className:"popover bottom",onMouseEnter:this.onMouseEnter,onMouseLeave:this.onMouseLeave},ps.default.createElement("div",{className:"arrow"}),ps.default.createElement("div",{className:"popover-content"},this.getDesc())))}};o(Yf,"FilterInput");Jp.title="Start";function Jp(){return oi.createElement("div",{className:"main-menu"},oi.createElement("div",{className:"menu-group"},oi.createElement("div",{className:"menu-content"},oi.createElement(WB,null),oi.createElement(UB,null)),oi.createElement("div",{className:"menu-legend"},"Find")),oi.createElement("div",{className:"menu-group"},oi.createElement("div",{className:"menu-content"},oi.createElement(HB,null),oi.createElement(zB,null)),oi.createElement("div",{className:"menu-legend"},"Intercept")))}o(Jp,"StartMenu");function HB(){let e=Xt(),t=qe(n=>n.options.intercept);return oi.createElement(Yf,{value:t||"",placeholder:"Intercept",type:"pause",color:"hsl(208, 56%, 53%)",onChange:n=>e(Ip("intercept",n))})}o(HB,"InterceptInput");function WB(){let e=Xt(),t=qe(n=>n.flows.filter);return oi.createElement(Yf,{value:t||"",placeholder:"Search",type:"search",color:"black",onChange:n=>e(A0(n))})}o(WB,"FlowFilterInput");function UB(){let e=Xt(),t=qe(n=>n.flows.highlight);return oi.createElement(Yf,{value:t||"",placeholder:"Highlight",type:"tag",color:"hsl(48, 100%, 50%)",onChange:n=>e(D0(n))})}o(UB,"HighlightInput");function zB(){let e=Xt();return oi.createElement(kr,{className:"btn-sm",title:"[a]ccept all",icon:"fa-forward text-success",onClick:()=>e(R0())},"Resume All")}o(zB,"ResumeAll");var Qr=fe(Oe());var Xf=fe(Oe());function FC({value:e,onChange:t,children:n}){return Xf.createElement("div",{className:"menu-entry"},Xf.createElement("label",null,Xf.createElement("input",{type:"checkbox",checked:e,onChange:t}),n))}o(FC,"MenuToggle");function Ny({name:e,children:t}){let n=Xt(),l=qe(d=>d.options[e]);return Xf.createElement(FC,{value:!!l,onChange:()=>n(Ip(e,!l))},t)}o(Ny,"OptionsToggle");function $P(){let e=Gs(),t=qe(n=>n.eventLog.visible);return Xf.createElement(FC,{value:t,onChange:()=>e(Rp())},"Display Event Log")}o($P,"EventlogToggle");function jP(){let e=Gs(),t=qe(n=>n.commandBar.visible);return Xf.createElement(FC,{value:t,onChange:()=>e(V0())},"Display Command Bar")}o(jP,"CommandBarToggle");var BC=fe(Oe());function HC({children:e,resource:t}){let n=`https://docs.mitmproxy.org/stable/${t}`;return BC.createElement("a",{target:"_blank",href:n},e||BC.createElement("i",{className:"fa fa-question-circle"}))}o(HC,"DocsLink");var Ly=fe(Oe());function Ro({children:e}){return window.MITMWEB_STATIC?null:Ly.createElement(Ly.Fragment,null,e)}o(Ro,"HideInStatic");Py.title="Options";function Py(){let e=Xt(),t=o(()=>sN("OptionModal"),"openOptions");return Qr.createElement("div",null,Qr.createElement(Ro,null,Qr.createElement("div",{className:"menu-group"},Qr.createElement("div",{className:"menu-content"},Qr.createElement(kr,{title:"Open Options",icon:"fa-cogs text-primary",onClick:()=>e(t())},"Edit Options ",Qr.createElement("sup",null,"alpha"))),Qr.createElement("div",{className:"menu-legend"},"Options Editor")),Qr.createElement("div",{className:"menu-group"},Qr.createElement("div",{className:"menu-content"},Qr.createElement(Ny,{name:"anticache"},"Strip cache headers ",Qr.createElement(HC,{resource:"overview-features/#anticache"})),Qr.createElement(Ny,{name:"showhost"},"Use host header for display"),Qr.createElement(Ny,{name:"ssl_insecure"},"Don't verify server certificates")),Qr.createElement("div",{className:"menu-legend"},"Quick Options"))),Qr.createElement("div",{className:"menu-group"},Qr.createElement("div",{className:"menu-content"},Qr.createElement($P,null),Qr.createElement(jP,null)),Qr.createElement("div",{className:"menu-legend"},"View Options")))}o(Py,"OptionMenu");var Hn=fe(Oe());var qP=Hn.memo(o(function(){let t=Gs(),n=qe(l=>l.flows.filter);return Hn.createElement(cu,{className:"pull-left special",text:"File",options:{placement:"bottom-start"}},Hn.createElement("li",null,Hn.createElement(G0,{icon:"fa-folder-open",text:"\xA0Open...",onClick:l=>l.stopPropagation(),onOpenFile:l=>{t(iN(l)),document.body.click()}})),Hn.createElement(ii,{onClick:()=>location.replace("/flows/dump")},Hn.createElement("i",{className:"fa fa-fw fa-floppy-o"}),"\xA0Save"),Hn.createElement(ii,{onClick:()=>location.replace("/flows/dump?filter="+n)},Hn.createElement("i",{className:"fa fa-fw fa-floppy-o"}),"\xA0Save filtered"),Hn.createElement(ii,{onClick:()=>confirm("Delete all flows?")&&t(W0())},Hn.createElement("i",{className:"fa fa-fw fa-trash"}),"\xA0Clear All"),Hn.createElement(Ro,null,Hn.createElement(hL,null),Hn.createElement("li",null,Hn.createElement("a",{href:"http://mitm.it/",target:"_blank"},Hn.createElement("i",{className:"fa fa-fw fa-external-link"}),"\xA0Install Certificates..."))))},"FileMenu"));var at=fe(Oe());function VP(e){if(navigator.clipboard&&window.isSecureContext)return navigator.clipboard.writeText(e);{let t=document.createElement("textarea");t.value=e,t.style.position="absolute",t.style.opacity="0",document.body.appendChild(t);try{return t.focus(),t.select(),document.execCommand("copy"),Promise.resolve()}catch(n){return alert(e),Promise.reject(n)}finally{t.remove()}}}o(VP,"copyToClipboard");var ed=o((e,t)=>Ia(void 0,null,function*(){let n=yield Pf("export",t,`@${e.id}`);n.value?yield VP(n.value):n.error?alert(n.error):console.error(n)}),"copy");td.title="Flow";function td(){let e=Xt(),t=qe(n=>n.flows.byId[n.flows.selected[0]]);return t?at.createElement("div",{className:"flow-menu"},at.createElement(Ro,null,at.createElement("div",{className:"menu-group"},at.createElement("div",{className:"menu-content"},at.createElement(kr,{title:"[r]eplay flow",icon:"fa-repeat text-primary",onClick:()=>e(Mp(t)),disabled:!E0(t)},"Replay"),at.createElement(kr,{title:"[D]uplicate flow",icon:"fa-copy text-info",onClick:()=>e(B0(t))},"Duplicate"),at.createElement(kr,{disabled:!t||!t.modified,title:"revert changes to flow [V]",icon:"fa-history text-warning",onClick:()=>e(H0(t))},"Revert"),at.createElement(kr,{title:"[d]elete flow",icon:"fa-trash text-danger",onClick:()=>e(F0(t))},"Delete"),at.createElement(VB,{flow:t})),at.createElement("div",{className:"menu-legend"},"Flow Modification"))),at.createElement("div",{className:"menu-group"},at.createElement("div",{className:"menu-content"},at.createElement($B,{flow:t}),at.createElement(jB,{flow:t})),at.createElement("div",{className:"menu-legend"},"Export")),at.createElement(Ro,null,at.createElement("div",{className:"menu-group"},at.createElement("div",{className:"menu-content"},at.createElement(kr,{disabled:!t||!t.intercepted,title:"[a]ccept intercepted flow",icon:"fa-play text-success",onClick:()=>e(Op(t))},"Resume"),at.createElement(kr,{disabled:!t||!t.intercepted,title:"kill intercepted flow [x]",icon:"fa-times text-danger",onClick:()=>e(I0(t))},"Abort")),at.createElement("div",{className:"menu-legend"},"Interception")))):at.createElement("div",null)}o(td,"FlowMenu");var Oy=o(e=>{let t=window.open(e,"_blank","noopener,noreferrer");t&&(t.opener=null)},"openInNewTab");function $B({flow:e}){var t;if(e.type!=="http")return at.createElement(kr,{icon:"fa-download",onClick:()=>0,disabled:!0},"Download");if(e.request.contentLength&&!((t=e.response)==null?void 0:t.contentLength))return at.createElement(kr,{icon:"fa-download",onClick:()=>Oy(Kr.getContentURL(e,e.request))},"Download");if(e.response){let n=e.response;if(!e.request.contentLength&&e.response.contentLength)return at.createElement(kr,{icon:"fa-download",onClick:()=>Oy(Kr.getContentURL(e,n))},"Download");if(e.request.contentLength&&e.response.contentLength)return at.createElement(cu,{text:at.createElement(kr,{icon:"fa-download",onClick:()=>1},"Download\u25BE"),options:{placement:"bottom-start"}},at.createElement(ii,{onClick:()=>Oy(Kr.getContentURL(e,e.request))},"Download request"),at.createElement(ii,{onClick:()=>Oy(Kr.getContentURL(e,n))},"Download response"))}return null}o($B,"DownloadButton");function jB({flow:e}){return at.createElement(cu,{className:"",text:at.createElement(kr,{title:"Export flow.",icon:"fa-clone",onClick:()=>1,disabled:e.type!=="http"},"Export\u25BE"),options:{placement:"bottom-start"}},at.createElement(ii,{onClick:()=>ed(e,"raw_request")},"Copy raw request"),at.createElement(ii,{onClick:()=>ed(e,"raw_response")},"Copy raw response"),at.createElement(ii,{onClick:()=>ed(e,"raw")},"Copy raw request and response"),at.createElement(ii,{onClick:()=>ed(e,"curl")},"Copy as cURL"),at.createElement(ii,{onClick:()=>ed(e,"httpie")},"Copy as HTTPie"))}o(jB,"ExportButton");var qB={":red_circle:":"\u{1F534}",":orange_circle:":"\u{1F7E0}",":yellow_circle:":"\u{1F7E1}",":green_circle:":"\u{1F7E2}",":large_blue_circle:":"\u{1F535}",":purple_circle:":"\u{1F7E3}",":brown_circle:":"\u{1F7E4}"};function VB({flow:e}){let t=Xt();return at.createElement(cu,{className:"",text:at.createElement(kr,{title:"mark flow",icon:"fa-paint-brush text-success",onClick:()=>1},"Mark\u25BE"),options:{placement:"bottom-start"}},at.createElement(ii,{onClick:()=>t(Wi(e,{marked:""}))},"\u26AA (no marker)"),Object.entries(qB).map(([n,l])=>at.createElement(ii,{key:n,onClick:()=>t(Wi(e,{marked:n}))},l," ",n.replace(/[:_]/g," "))))}o(VB,"MarkButton");var vu=fe(Oe());var KP=vu.memo(o(function(){let t=qe(l=>l.connection.state),n=qe(l=>l.connection.message);switch(t){case ni.INIT:return vu.createElement("span",{className:"connection-indicator init"},"connecting\u2026");case ni.FETCHING:return vu.createElement("span",{className:"connection-indicator fetching"},"fetching data\u2026");case ni.ESTABLISHED:return vu.createElement("span",{className:"connection-indicator established"},"connected");case ni.ERROR:return vu.createElement("span",{className:"connection-indicator error",title:n},"connection lost");case ni.OFFLINE:return vu.createElement("span",{className:"connection-indicator offline"},"offline");default:let l=t;throw"unknown connection state"}},"ConnectionIndicator"));function WC(){let e=qe(v=>v.flows.selected.filter(C=>C in v.flows.byId)),[t,n]=(0,Io.useState)(()=>Jp),[l,d]=(0,Io.useState)(!1),h=[Jp,Py];e.length>0?(l||(n(()=>td),d(!0)),h.push(td)):(l&&d(!1),t===td&&n(()=>Jp));function c(v,C){C.preventDefault(),n(()=>v)}return o(c,"handleClick"),Io.default.createElement("header",null,Io.default.createElement("nav",{className:"nav-tabs nav-tabs-lg"},Io.default.createElement(qP,null),h.map(v=>Io.default.createElement("a",{key:v.title,href:"#",className:(0,GP.default)({active:v===t}),onClick:C=>c(v,C)},v.title)),Io.default.createElement(Ro,null,Io.default.createElement(KP,null))),Io.default.createElement("div",null,Io.default.createElement(t,null)))}o(WC,"Header");var Je=fe(Oe()),YP=fe(ti());var My=function(){"use strict";function e(l,d){function h(){this.constructor=l}o(h,"ctor"),h.prototype=d.prototype,l.prototype=new h}o(e,"peg$subclass");function t(l,d,h,c){this.message=l,this.expected=d,this.found=h,this.location=c,this.name="SyntaxError",typeof Error.captureStackTrace=="function"&&Error.captureStackTrace(this,t)}o(t,"peg$SyntaxError"),e(t,Error);function n(l){var d=arguments.length>1?arguments[1]:{},h=this,c={},v={Expr:Cr},C=Cr,k=o(function(H,ee){return[H,...ee]},"peg$c0"),O=o(function(H){return[H]},"peg$c1"),j=o(function(){return""},"peg$c2"),B={type:"other",description:"string"},X='"',J={type:"literal",value:'"',description:'"\\""'},Z=o(function(H){return H.join("")},"peg$c6"),R="'",A={type:"literal",value:"'",description:`"'"`},I=/^["\\]/,G={type:"class",value:'["\\\\]',description:'["\\\\]'},K={type:"any",description:"any character"},se=o(function(H){return H},"peg$c12"),ne="\\",pe={type:"literal",value:"\\",description:'"\\\\"'},me=/^['\\]/,xe={type:"class",value:"['\\\\]",description:"['\\\\]"},Ve=/^['"\\]/,tt={type:"class",value:`['"\\\\]`,description:`['"\\\\]`},_e="n",St={type:"literal",value:"n",description:'"n"'},We=o(function(){return` `},"peg$c21"),Ke="r",Ge={type:"literal",value:"r",description:'"r"'},Xe=o(function(){return"\r"},"peg$c24"),nr="t",ct={type:"literal",value:"t",description:'"t"'},Hr=o(function(){return" "},"peg$c27"),Qt={type:"other",description:"whitespace"},_t=/^[ \t\n\r]/,Ct={type:"class",value:"[ \\t\\n\\r]",description:"[ \\t\\n\\r]"},ut={type:"other",description:"control character"},Lr=/^[|&!()~"]/,zt={type:"class",value:'[|&!()~"]',description:'[|&!()~"]'},$t={type:"other",description:"optional whitespace"},ie=0,rt=0,Pr=[{line:1,column:1,seenCR:!1}],Gt=0,Yt=[],Se=0,Or;if("startRule"in d){if(!(d.startRule in v))throw new Error(`Can't start parsing from rule "`+d.startRule+'".');C=v[d.startRule]}function fn(){return l.substring(rt,ie)}o(fn,"text");function Un(){return gr(rt,ie)}o(Un,"location");function si(H){throw Ho(null,[{type:"other",description:H}],l.substring(rt,ie),gr(rt,ie))}o(si,"expected");function cn(H){throw Ho(H,null,l.substring(rt,ie),gr(rt,ie))}o(cn,"error");function Zt(H){var ee=Pr[H],he,Te;if(ee)return ee;for(he=H-1;!Pr[he];)he--;for(ee=Pr[he],ee={line:ee.line,column:ee.column,seenCR:ee.seenCR};heGt&&(Gt=ie,Yt=[]),Yt.push(H))}o(pt,"peg$fail");function Ho(H,ee,he,Te){function ir(Ft){var Wr=1;for(Ft.sort(function(or,li){return or.descriptionli.description?1:0});Wr1?li.slice(0,-1).join(", ")+" or "+li[Ft.length-1]:li[0],lo=Wr?'"'+or(Wr)+'"':"end of input","Expected "+ds+" but "+lo+" found."}return o(Ul,"buildMessage"),ee!==null&&ir(ee),new t(H!==null?H:Ul(ee,he),ee,he,Te)}o(Ho,"peg$buildException");function Cr(){var H,ee,he,Te;if(H=ie,ee=Ui(),ee!==c){if(he=[],Te=$n(),Te!==c)for(;Te!==c;)he.push(Te),Te=$n();else he=c;he!==c?(Te=Cr(),Te!==c?(rt=H,ee=k(ee,Te),H=ee):(ie=H,H=c)):(ie=H,H=c)}else ie=H,H=c;if(H===c&&(H=ie,ee=Ui(),ee!==c&&(rt=H,ee=O(ee)),H=ee,H===c)){for(H=ie,ee=[],he=$n();he!==c;)ee.push(he),he=$n();ee!==c&&(rt=H,ee=j()),H=ee}return H}o(Cr,"peg$parseExpr");function Ui(){var H,ee,he,Te;if(Se++,H=ie,l.charCodeAt(ie)===34?(ee=X,ie++):(ee=c,Se===0&&pt(J)),ee!==c){for(he=[],Te=pn();Te!==c;)he.push(Te),Te=pn();he!==c?(l.charCodeAt(ie)===34?(Te=X,ie++):(Te=c,Se===0&&pt(J)),Te!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)):(ie=H,H=c)}else ie=H,H=c;if(H===c){if(H=ie,l.charCodeAt(ie)===39?(ee=R,ie++):(ee=c,Se===0&&pt(A)),ee!==c){for(he=[],Te=zn();Te!==c;)he.push(Te),Te=zn();he!==c?(l.charCodeAt(ie)===39?(Te=R,ie++):(Te=c,Se===0&&pt(A)),Te!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)):(ie=H,H=c)}else ie=H,H=c;if(H===c){if(H=ie,ee=ie,Se++,he=Mn(),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c){if(he=[],Te=Si(),Te!==c)for(;Te!==c;)he.push(Te),Te=Si();else he=c;he!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)}else ie=H,H=c;if(H===c){if(H=ie,l.charCodeAt(ie)===34?(ee=X,ie++):(ee=c,Se===0&&pt(J)),ee!==c){for(he=[],Te=pn();Te!==c;)he.push(Te),Te=pn();he!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)}else ie=H,H=c;if(H===c)if(H=ie,l.charCodeAt(ie)===39?(ee=R,ie++):(ee=c,Se===0&&pt(A)),ee!==c){for(he=[],Te=zn();Te!==c;)he.push(Te),Te=zn();he!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)}else ie=H,H=c}}}return Se--,H===c&&(ee=c,Se===0&&pt(B)),H}o(Ui,"peg$parseStringLiteral");function pn(){var H,ee,he;return H=ie,ee=ie,Se++,I.test(l.charAt(ie))?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(G)),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c?(l.length>ie?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(K)),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c),H===c&&(H=ie,l.charCodeAt(ie)===92?(ee=ne,ie++):(ee=c,Se===0&&pt(pe)),ee!==c?(he=Ci(),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c)),H}o(pn,"peg$parseDoubleStringChar");function zn(){var H,ee,he;return H=ie,ee=ie,Se++,me.test(l.charAt(ie))?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(xe)),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c?(l.length>ie?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(K)),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c),H===c&&(H=ie,l.charCodeAt(ie)===92?(ee=ne,ie++):(ee=c,Se===0&&pt(pe)),ee!==c?(he=Ci(),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c)),H}o(zn,"peg$parseSingleStringChar");function Si(){var H,ee,he;return H=ie,ee=ie,Se++,he=$n(),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c?(l.length>ie?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(K)),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c),H}o(Si,"peg$parseUnquotedStringChar");function Ci(){var H,ee;return Ve.test(l.charAt(ie))?(H=l.charAt(ie),ie++):(H=c,Se===0&&pt(tt)),H===c&&(H=ie,l.charCodeAt(ie)===110?(ee=_e,ie++):(ee=c,Se===0&&pt(St)),ee!==c&&(rt=H,ee=We()),H=ee,H===c&&(H=ie,l.charCodeAt(ie)===114?(ee=Ke,ie++):(ee=c,Se===0&&pt(Ge)),ee!==c&&(rt=H,ee=Xe()),H=ee,H===c&&(H=ie,l.charCodeAt(ie)===116?(ee=nr,ie++):(ee=c,Se===0&&pt(ct)),ee!==c&&(rt=H,ee=Hr()),H=ee))),H}o(Ci,"peg$parseEscapeSequence");function $n(){var H,ee;return Se++,_t.test(l.charAt(ie))?(H=l.charAt(ie),ie++):(H=c,Se===0&&pt(Ct)),Se--,H===c&&(ee=c,Se===0&&pt(Qt)),H}o($n,"peg$parsews");function Mn(){var H,ee;return Se++,Lr.test(l.charAt(ie))?(H=l.charAt(ie),ie++):(H=c,Se===0&&pt(zt)),Se--,H===c&&(ee=c,Se===0&&pt(ut)),H}o(Mn,"peg$parsecc");function Js(){var H,ee;for(Se++,H=[],ee=$n();ee!==c;)H.push(ee),ee=$n();return Se--,H===c&&(ee=c,Se===0&&pt($t)),H}if(o(Js,"peg$parse__"),Or=C(),Or!==c&&ie===l.length)return Or;throw Or!==c&&ie{t&&t.current.addEventListener("DOMNodeInserted",n=>{let l=n.currentTarget;l.scroll({top:l.scrollHeight,behavior:"auto"})})},[]),Je.default.createElement("div",{className:"command-result",ref:t},e.map((n,l)=>Je.default.createElement("div",{key:l},Je.default.createElement("div",null,Je.default.createElement("strong",null,"$ ",n.command)),n.result)))}o(KB,"Results");function GB({nextArgs:e,currentArg:t,help:n,description:l,availableCommands:d}){let h=[];for(let c=0;c0&&Je.default.createElement("div",null,Je.default.createElement("strong",null,"Argument suggestion:")," ",h),(n==null?void 0:n.includes("->"))&&Je.default.createElement("div",null,Je.default.createElement("strong",null,"Signature help: "),n),l&&Je.default.createElement("div",null,"# ",l),Je.default.createElement("div",null,Je.default.createElement("strong",null,"Available Commands: "),Je.default.createElement("p",{className:"available-commands"},JSON.stringify(d)))))}o(GB,"CommandHelp");function zC(){let[e,t]=(0,Je.useState)(""),[n,l]=(0,Je.useState)(""),[d,h]=(0,Je.useState)(0),[c,v]=(0,Je.useState)([]),[C,k]=(0,Je.useState)([]),[O,j]=(0,Je.useState)({}),[B,X]=(0,Je.useState)([]),[J,Z]=(0,Je.useState)(0),[R,A]=(0,Je.useState)(""),[I,G]=(0,Je.useState)(""),[K,se]=(0,Je.useState)([]),[ne,pe]=(0,Je.useState)([]),[me,xe]=(0,Je.useState)(void 0);(0,Je.useEffect)(()=>{kt("/commands",{method:"GET"}).then(We=>We.json()).then(We=>{j(We),v(UC(We)),k(Object.keys(We))}).catch(We=>console.error(We))},[]),(0,Je.useEffect)(()=>{Pf("commands.history.get").then(We=>{pe(We.value)}).catch(We=>console.error(We))},[]);let Ve=o((We,Ke)=>{var ct,Hr,Qt;let Ge=Dy.parse(Ke),Xe=Dy.parse(We);A((ct=O[Ge[0]])==null?void 0:ct.signature_help),G(((Hr=O[Ge[0]])==null?void 0:Hr.help)||""),v(UC(O,Xe[0])),k(UC(O,Ge[0]));let nr=(Qt=O[Ge[0]])==null?void 0:Qt.parameters.map(_t=>_t.name);nr&&(X([Ge[0],...nr]),Z(Ge.length-1))},"parseCommand"),tt=o(We=>{t(We.target.value),l(We.target.value),h(0)},"onChange"),_e=o(We=>{if(We.key==="Enter"){let[Ke,...Ge]=Dy.parse(e);pe([...ne,e]),Pf("commands.history.add",e).catch(()=>0),kt.post(`/commands/${Ke}`,{arguments:Ge}).then(Xe=>Xe.json()).then(Xe=>{xe(void 0),X([]),se([...K,{command:e,result:JSON.stringify(Xe.value||Xe.error)}])}).catch(Xe=>{xe(void 0),X([]),se([...K,{command:e,result:Xe.toString()}])}),A(""),G(""),t(""),l(""),h(0),v(C)}if(We.key==="ArrowUp"){let Ke;me===void 0?Ke=ne.length-1:Ke=Math.max(0,me-1),t(ne[Ke]),l(ne[Ke]),xe(Ke)}if(We.key==="ArrowDown"){if(me===void 0)return;if(me==ne.length-1)t(""),l(""),xe(void 0);else{let Ke=me+1;t(ne[Ke]),l(ne[Ke]),xe(Ke)}}We.key==="Tab"&&(t(c[d]),h((d+1)%c.length),We.preventDefault()),We.stopPropagation()},"onKeyDown"),St=o(We=>{if(!e){k(Object.keys(O));return}Ve(n,e),We.stopPropagation()},"onKeyUp");return Je.default.createElement("div",{className:"command"},Je.default.createElement("div",{className:"command-title"},"Command Result"),Je.default.createElement(KB,{results:K}),Je.default.createElement(GB,{nextArgs:B,currentArg:J,help:R,description:I,availableCommands:C}),Je.default.createElement("div",{className:(0,YP.default)("command-input input-group")},Je.default.createElement("span",{className:"input-group-addon"},Je.default.createElement("i",{className:"fa fa-fw fa-terminal"})),Je.default.createElement("input",{type:"text",placeholder:"Enter command",className:"form-control",value:e||"",onChange:tt,onKeyDown:_e,onKeyUp:St})))}o(zC,"CommandBar");var Wl=fe(Oe()),rd=fe(Sm());var $C=fe(Oe());function jC({checked:e,onToggle:t,text:n}){return $C.default.createElement("div",{className:"btn btn-toggle "+(e?"btn-primary":"btn-default"),onClick:t},$C.default.createElement("i",{className:"fa fa-fw "+(e?"fa-check-square-o":"fa-square-o")}),"\xA0",n)}o(jC,"ToggleButton");var Hl=fe(Oe()),qC=fe(Sm()),XP=fe(iu()),QP=fe(fC());var Am=class extends Hl.Component{constructor(t){super(t);this.heights={},this.state={vScroll:jf()},this.onViewportUpdate=this.onViewportUpdate.bind(this)}componentDidMount(){window.addEventListener("resize",this.onViewportUpdate),this.onViewportUpdate()}componentWillUnmount(){window.removeEventListener("resize",this.onViewportUpdate)}componentDidUpdate(){this.onViewportUpdate()}onViewportUpdate(){let t=XP.default.findDOMNode(this),n=jf({itemCount:this.props.events.length,rowHeight:this.props.rowHeight,viewportTop:t.scrollTop,viewportHeight:t.offsetHeight,itemHeights:this.props.events.map(l=>this.heights[l.id])});(0,QP.default)(this.state.vScroll,n)||this.setState({vScroll:n})}setHeight(t,n){if(n&&!this.heights[t]){let l=n.offsetHeight;this.heights[t]!==l&&(this.heights[t]=l,this.onViewportUpdate())}}render(){let{vScroll:t}=this.state,{events:n}=this.props;return Hl.default.createElement("pre",{onScroll:this.onViewportUpdate},Hl.default.createElement("div",{style:{height:t.paddingTop}}),n.slice(t.start,t.end).map(l=>Hl.default.createElement("div",{key:l.id,ref:d=>this.setHeight(l.id,d)},Hl.default.createElement(YB,{event:l}),l.message)),Hl.default.createElement("div",{style:{height:t.paddingBottom}}))}};o(Am,"EventLogList"),Am.propTypes={events:qC.default.array.isRequired,rowHeight:qC.default.number},Am.defaultProps={rowHeight:18};function YB({event:e}){let t={web:"html5",debug:"bug",warn:"exclamation-triangle",error:"ban"}[e.level]||"info";return Hl.default.createElement("i",{className:`fa fa-fw fa-${t}`})}o(YB,"LogIcon");var ZP=my(Am);var Dm=class extends Wl.Component{constructor(t,n){super(t,n);this.state={height:this.props.defaultHeight},this.onDragStart=this.onDragStart.bind(this),this.onDragMove=this.onDragMove.bind(this),this.onDragStop=this.onDragStop.bind(this)}onDragStart(t){t.preventDefault(),this.dragStart=this.state.height+t.pageY,window.addEventListener("mousemove",this.onDragMove),window.addEventListener("mouseup",this.onDragStop),window.addEventListener("dragend",this.onDragStop)}onDragMove(t){t.preventDefault(),this.setState({height:this.dragStart-t.pageY})}onDragStop(t){t.preventDefault(),window.removeEventListener("mousemove",this.onDragMove)}render(){let{height:t}=this.state,{filters:n,events:l,toggleFilter:d,close:h}=this.props;return Wl.default.createElement("div",{className:"eventlog",style:{height:t}},Wl.default.createElement("div",{onMouseDown:this.onDragStart},"Eventlog",Wl.default.createElement("div",{className:"pull-right"},["debug","info","web","warn","error"].map(c=>Wl.default.createElement(jC,{key:c,text:c,checked:n[c],onToggle:()=>d(c)})),Wl.default.createElement("i",{onClick:h,className:"fa fa-close"}))),Wl.default.createElement(ZP,{events:l}))}};o(Dm,"PureEventLog"),Vc(Dm,"propTypes",{filters:rd.default.object.isRequired,events:rd.default.array.isRequired,toggleFilter:rd.default.func.isRequired,close:rd.default.func.isRequired,defaultHeight:rd.default.number}),Vc(Dm,"defaultProps",{defaultHeight:200});var JP=Hi(e=>({filters:e.eventLog.filters,events:e.eventLog.view}),{close:Rp,toggleFilter:gN})(Dm);var un=fe(Oe());function VC(){let e=qe(A=>A.backendState.version),{mode:t,intercept:n,showhost:l,upstream_cert:d,rawtcp:h,http2:c,websocket:v,anticache:C,anticomp:k,stickyauth:O,stickycookie:j,stream_large_bodies:B,listen_host:X,listen_port:J,server:Z,ssl_insecure:R}=qe(A=>A.options);return un.createElement("footer",null,t&&(t.length!==1||t[0]!=="regular")&&un.createElement("span",{className:"label label-success"},t.join(",")),n&&un.createElement("span",{className:"label label-success"},"Intercept: ",n),R&&un.createElement("span",{className:"label label-danger"},"ssl_insecure"),l&&un.createElement("span",{className:"label label-success"},"showhost"),!d&&un.createElement("span",{className:"label label-success"},"no-upstream-cert"),!h&&un.createElement("span",{className:"label label-success"},"no-raw-tcp"),!c&&un.createElement("span",{className:"label label-success"},"no-http2"),!v&&un.createElement("span",{className:"label label-success"},"no-websocket"),C&&un.createElement("span",{className:"label label-success"},"anticache"),k&&un.createElement("span",{className:"label label-success"},"anticomp"),O&&un.createElement("span",{className:"label label-success"},"stickyauth: ",O),j&&un.createElement("span",{className:"label label-success"},"stickycookie: ",j),B&&un.createElement("span",{className:"label label-success"},"stream: ",x0(B)),un.createElement("div",{className:"pull-right"},un.createElement(Ro,null,Z&&un.createElement("span",{className:"label label-primary",title:"HTTP Proxy Server Address"},X||"*",":",J||8080)),un.createElement("span",{className:"label label-default",title:"Mitmproxy Version"},"mitmproxy ",e)))}o(VC,"Footer");var JC=fe(Oe());var ZC=fe(Oe());var nd=fe(Oe());function KC({children:e}){return nd.createElement("div",null,nd.createElement("div",{className:"modal-backdrop fade in"}),nd.createElement("div",{className:"modal modal-visible",id:"optionsModal",tabIndex:-1,role:"dialog","aria-labelledby":"options"},nd.createElement("div",{className:"modal-dialog modal-lg",role:"document"},nd.createElement("div",{className:"modal-content"},e))))}o(KC,"ModalLayout");var mr=fe(Oe());var Fo=fe(Oe()),Bo=fe(Sm());var eO=fe(ti()),XB=o(e=>{e.key!=="Escape"&&e.stopPropagation()},"stopPropagation");GC.propTypes={value:Bo.default.bool.isRequired,onChange:Bo.default.func.isRequired};function GC(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);return Fo.default.createElement("div",{className:"checkbox"},Fo.default.createElement("label",null,Fo.default.createElement("input",ke({type:"checkbox",checked:e,onChange:h=>t(h.target.checked)},n)),"Enable"))}o(GC,"BooleanOption");YC.propTypes={value:Bo.default.string,onChange:Bo.default.func.isRequired};function YC(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);return Fo.default.createElement("input",ke({type:"text",value:e||"",onChange:h=>t(h.target.value)},n))}o(YC,"StringOption");function tO(e){return function(l){var d=l,{onChange:t}=d,n=Ws(d,["onChange"]);return Fo.default.createElement(e,ke({onChange:h=>t(h||null)},n))}}o(tO,"Optional");XC.propTypes={value:Bo.default.number.isRequired,onChange:Bo.default.func.isRequired};function XC(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);return Fo.default.createElement("input",ke({type:"number",value:e,onChange:h=>t(parseInt(h.target.value))},n))}o(XC,"NumberOption");rO.propTypes={value:Bo.default.string.isRequired,onChange:Bo.default.func.isRequired};function rO(d){var h=d,{value:e,onChange:t,choices:n}=h,l=Ws(h,["value","onChange","choices"]);return Fo.default.createElement("select",ke({onChange:c=>t(c.target.value),value:e},l),n.map(c=>Fo.default.createElement("option",{key:c,value:c},c)))}o(rO,"ChoicesOption");nO.propTypes={value:Bo.default.arrayOf(Bo.default.string).isRequired,onChange:Bo.default.func.isRequired};function nO(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);let h=Math.max(e.length,1);return Fo.default.createElement("textarea",ke({rows:h,value:e.join(` +`?(ee.seenCR||ee.line++,ee.column=1,ee.seenCR=!1):Te==="\r"||Te==="\u2028"||Te==="\u2029"?(ee.line++,ee.column=1,ee.seenCR=!0):(ee.column++,ee.seenCR=!1),he++;return Pr[H]=ee,ee}o(Zt,"peg$computePosDetails");function gr(H,ee){var he=Zt(H),Te=Zt(ee);return{start:{offset:H,line:he.line,column:he.column},end:{offset:ee,line:Te.line,column:Te.column}}}o(gr,"peg$computeLocation");function pt(H){ieGt&&(Gt=ie,Yt=[]),Yt.push(H))}o(pt,"peg$fail");function Ho(H,ee,he,Te){function ir(Ft){var Wr=1;for(Ft.sort(function(or,li){return or.descriptionli.description?1:0});Wr1?li.slice(0,-1).join(", ")+" or "+li[Ft.length-1]:li[0],lo=Wr?'"'+or(Wr)+'"':"end of input","Expected "+ds+" but "+lo+" found."}return o(Ul,"buildMessage"),ee!==null&&ir(ee),new t(H!==null?H:Ul(ee,he),ee,he,Te)}o(Ho,"peg$buildException");function Cr(){var H,ee,he,Te;if(H=ie,ee=Ui(),ee!==c){if(he=[],Te=$n(),Te!==c)for(;Te!==c;)he.push(Te),Te=$n();else he=c;he!==c?(Te=Cr(),Te!==c?(rt=H,ee=k(ee,Te),H=ee):(ie=H,H=c)):(ie=H,H=c)}else ie=H,H=c;if(H===c&&(H=ie,ee=Ui(),ee!==c&&(rt=H,ee=O(ee)),H=ee,H===c)){for(H=ie,ee=[],he=$n();he!==c;)ee.push(he),he=$n();ee!==c&&(rt=H,ee=j()),H=ee}return H}o(Cr,"peg$parseExpr");function Ui(){var H,ee,he,Te;if(Se++,H=ie,l.charCodeAt(ie)===34?(ee=X,ie++):(ee=c,Se===0&&pt(J)),ee!==c){for(he=[],Te=pn();Te!==c;)he.push(Te),Te=pn();he!==c?(l.charCodeAt(ie)===34?(Te=X,ie++):(Te=c,Se===0&&pt(J)),Te!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)):(ie=H,H=c)}else ie=H,H=c;if(H===c){if(H=ie,l.charCodeAt(ie)===39?(ee=R,ie++):(ee=c,Se===0&&pt(A)),ee!==c){for(he=[],Te=zn();Te!==c;)he.push(Te),Te=zn();he!==c?(l.charCodeAt(ie)===39?(Te=R,ie++):(Te=c,Se===0&&pt(A)),Te!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)):(ie=H,H=c)}else ie=H,H=c;if(H===c){if(H=ie,ee=ie,Se++,he=Mn(),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c){if(he=[],Te=Si(),Te!==c)for(;Te!==c;)he.push(Te),Te=Si();else he=c;he!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)}else ie=H,H=c;if(H===c){if(H=ie,l.charCodeAt(ie)===34?(ee=X,ie++):(ee=c,Se===0&&pt(J)),ee!==c){for(he=[],Te=pn();Te!==c;)he.push(Te),Te=pn();he!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)}else ie=H,H=c;if(H===c)if(H=ie,l.charCodeAt(ie)===39?(ee=R,ie++):(ee=c,Se===0&&pt(A)),ee!==c){for(he=[],Te=zn();Te!==c;)he.push(Te),Te=zn();he!==c?(rt=H,ee=Z(he),H=ee):(ie=H,H=c)}else ie=H,H=c}}}return Se--,H===c&&(ee=c,Se===0&&pt(B)),H}o(Ui,"peg$parseStringLiteral");function pn(){var H,ee,he;return H=ie,ee=ie,Se++,I.test(l.charAt(ie))?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(G)),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c?(l.length>ie?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(K)),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c),H===c&&(H=ie,l.charCodeAt(ie)===92?(ee=ne,ie++):(ee=c,Se===0&&pt(pe)),ee!==c?(he=Ci(),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c)),H}o(pn,"peg$parseDoubleStringChar");function zn(){var H,ee,he;return H=ie,ee=ie,Se++,me.test(l.charAt(ie))?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(xe)),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c?(l.length>ie?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(K)),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c),H===c&&(H=ie,l.charCodeAt(ie)===92?(ee=ne,ie++):(ee=c,Se===0&&pt(pe)),ee!==c?(he=Ci(),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c)),H}o(zn,"peg$parseSingleStringChar");function Si(){var H,ee,he;return H=ie,ee=ie,Se++,he=$n(),Se--,he===c?ee=void 0:(ie=ee,ee=c),ee!==c?(l.length>ie?(he=l.charAt(ie),ie++):(he=c,Se===0&&pt(K)),he!==c?(rt=H,ee=se(he),H=ee):(ie=H,H=c)):(ie=H,H=c),H}o(Si,"peg$parseUnquotedStringChar");function Ci(){var H,ee;return Ve.test(l.charAt(ie))?(H=l.charAt(ie),ie++):(H=c,Se===0&&pt(tt)),H===c&&(H=ie,l.charCodeAt(ie)===110?(ee=_e,ie++):(ee=c,Se===0&&pt(St)),ee!==c&&(rt=H,ee=We()),H=ee,H===c&&(H=ie,l.charCodeAt(ie)===114?(ee=Ke,ie++):(ee=c,Se===0&&pt(Ge)),ee!==c&&(rt=H,ee=Xe()),H=ee,H===c&&(H=ie,l.charCodeAt(ie)===116?(ee=nr,ie++):(ee=c,Se===0&&pt(ct)),ee!==c&&(rt=H,ee=Hr()),H=ee))),H}o(Ci,"peg$parseEscapeSequence");function $n(){var H,ee;return Se++,_t.test(l.charAt(ie))?(H=l.charAt(ie),ie++):(H=c,Se===0&&pt(Ct)),Se--,H===c&&(ee=c,Se===0&&pt(Qt)),H}o($n,"peg$parsews");function Mn(){var H,ee;return Se++,Lr.test(l.charAt(ie))?(H=l.charAt(ie),ie++):(H=c,Se===0&&pt(zt)),Se--,H===c&&(ee=c,Se===0&&pt(ut)),H}o(Mn,"peg$parsecc");function Js(){var H,ee;for(Se++,H=[],ee=$n();ee!==c;)H.push(ee),ee=$n();return Se--,H===c&&(ee=c,Se===0&&pt($t)),H}if(o(Js,"peg$parse__"),Or=C(),Or!==c&&ie===l.length)return Or;throw Or!==c&&ie{t&&t.current.addEventListener("DOMNodeInserted",n=>{let l=n.currentTarget;l.scroll({top:l.scrollHeight,behavior:"auto"})})},[]),Je.default.createElement("div",{className:"command-result",ref:t},e.map((n,l)=>Je.default.createElement("div",{key:l},Je.default.createElement("div",null,Je.default.createElement("strong",null,"$ ",n.command)),n.result)))}o(KB,"Results");function GB({nextArgs:e,currentArg:t,help:n,description:l,availableCommands:d}){let h=[];for(let c=0;c0&&Je.default.createElement("div",null,Je.default.createElement("strong",null,"Argument suggestion:")," ",h),(n==null?void 0:n.includes("->"))&&Je.default.createElement("div",null,Je.default.createElement("strong",null,"Signature help: "),n),l&&Je.default.createElement("div",null,"# ",l),Je.default.createElement("div",null,Je.default.createElement("strong",null,"Available Commands: "),Je.default.createElement("p",{className:"available-commands"},JSON.stringify(d)))))}o(GB,"CommandHelp");function zC(){let[e,t]=(0,Je.useState)(""),[n,l]=(0,Je.useState)(""),[d,h]=(0,Je.useState)(0),[c,v]=(0,Je.useState)([]),[C,k]=(0,Je.useState)([]),[O,j]=(0,Je.useState)({}),[B,X]=(0,Je.useState)([]),[J,Z]=(0,Je.useState)(0),[R,A]=(0,Je.useState)(""),[I,G]=(0,Je.useState)(""),[K,se]=(0,Je.useState)([]),[ne,pe]=(0,Je.useState)([]),[me,xe]=(0,Je.useState)(void 0);(0,Je.useEffect)(()=>{kt("/commands",{method:"GET"}).then(We=>We.json()).then(We=>{j(We),v(UC(We)),k(Object.keys(We))}).catch(We=>console.error(We))},[]),(0,Je.useEffect)(()=>{Pf("commands.history.get").then(We=>{pe(We.value)}).catch(We=>console.error(We))},[]);let Ve=o((We,Ke)=>{var ct,Hr,Qt;let Ge=My.parse(Ke),Xe=My.parse(We);A((ct=O[Ge[0]])==null?void 0:ct.signature_help),G(((Hr=O[Ge[0]])==null?void 0:Hr.help)||""),v(UC(O,Xe[0])),k(UC(O,Ge[0]));let nr=(Qt=O[Ge[0]])==null?void 0:Qt.parameters.map(_t=>_t.name);nr&&(X([Ge[0],...nr]),Z(Ge.length-1))},"parseCommand"),tt=o(We=>{t(We.target.value),l(We.target.value),h(0)},"onChange"),_e=o(We=>{if(We.key==="Enter"){let[Ke,...Ge]=My.parse(e);pe([...ne,e]),Pf("commands.history.add",e).catch(()=>0),kt.post(`/commands/${Ke}`,{arguments:Ge}).then(Xe=>Xe.json()).then(Xe=>{xe(void 0),X([]),se([...K,{command:e,result:JSON.stringify(Xe.value||Xe.error)}])}).catch(Xe=>{xe(void 0),X([]),se([...K,{command:e,result:Xe.toString()}])}),A(""),G(""),t(""),l(""),h(0),v(C)}if(We.key==="ArrowUp"){let Ke;me===void 0?Ke=ne.length-1:Ke=Math.max(0,me-1),t(ne[Ke]),l(ne[Ke]),xe(Ke)}if(We.key==="ArrowDown"){if(me===void 0)return;if(me==ne.length-1)t(""),l(""),xe(void 0);else{let Ke=me+1;t(ne[Ke]),l(ne[Ke]),xe(Ke)}}We.key==="Tab"&&(t(c[d]),h((d+1)%c.length),We.preventDefault()),We.stopPropagation()},"onKeyDown"),St=o(We=>{if(!e){k(Object.keys(O));return}Ve(n,e),We.stopPropagation()},"onKeyUp");return Je.default.createElement("div",{className:"command"},Je.default.createElement("div",{className:"command-title"},"Command Result"),Je.default.createElement(KB,{results:K}),Je.default.createElement(GB,{nextArgs:B,currentArg:J,help:R,description:I,availableCommands:C}),Je.default.createElement("div",{className:(0,YP.default)("command-input input-group")},Je.default.createElement("span",{className:"input-group-addon"},Je.default.createElement("i",{className:"fa fa-fw fa-terminal"})),Je.default.createElement("input",{type:"text",placeholder:"Enter command",className:"form-control",value:e||"",onChange:tt,onKeyDown:_e,onKeyUp:St})))}o(zC,"CommandBar");var Wl=fe(Oe()),rd=fe(Sm());var $C=fe(Oe());function jC({checked:e,onToggle:t,text:n}){return $C.default.createElement("div",{className:"btn btn-toggle "+(e?"btn-primary":"btn-default"),onClick:t},$C.default.createElement("i",{className:"fa fa-fw "+(e?"fa-check-square-o":"fa-square-o")}),"\xA0",n)}o(jC,"ToggleButton");var Hl=fe(Oe()),qC=fe(Sm()),XP=fe(iu()),QP=fe(fC());var Am=class extends Hl.Component{constructor(t){super(t);this.heights={},this.state={vScroll:jf()},this.onViewportUpdate=this.onViewportUpdate.bind(this)}componentDidMount(){window.addEventListener("resize",this.onViewportUpdate),this.onViewportUpdate()}componentWillUnmount(){window.removeEventListener("resize",this.onViewportUpdate)}componentDidUpdate(){this.onViewportUpdate()}onViewportUpdate(){let t=XP.default.findDOMNode(this),n=jf({itemCount:this.props.events.length,rowHeight:this.props.rowHeight,viewportTop:t.scrollTop,viewportHeight:t.offsetHeight,itemHeights:this.props.events.map(l=>this.heights[l.id])});(0,QP.default)(this.state.vScroll,n)||this.setState({vScroll:n})}setHeight(t,n){if(n&&!this.heights[t]){let l=n.offsetHeight;this.heights[t]!==l&&(this.heights[t]=l,this.onViewportUpdate())}}render(){let{vScroll:t}=this.state,{events:n}=this.props;return Hl.default.createElement("pre",{onScroll:this.onViewportUpdate},Hl.default.createElement("div",{style:{height:t.paddingTop}}),n.slice(t.start,t.end).map(l=>Hl.default.createElement("div",{key:l.id,ref:d=>this.setHeight(l.id,d)},Hl.default.createElement(YB,{event:l}),l.message)),Hl.default.createElement("div",{style:{height:t.paddingBottom}}))}};o(Am,"EventLogList"),Am.propTypes={events:qC.default.array.isRequired,rowHeight:qC.default.number},Am.defaultProps={rowHeight:18};function YB({event:e}){let t={web:"html5",debug:"bug",warn:"exclamation-triangle",error:"ban"}[e.level]||"info";return Hl.default.createElement("i",{className:`fa fa-fw fa-${t}`})}o(YB,"LogIcon");var ZP=dy(Am);var Dm=class extends Wl.Component{constructor(t,n){super(t,n);this.state={height:this.props.defaultHeight},this.onDragStart=this.onDragStart.bind(this),this.onDragMove=this.onDragMove.bind(this),this.onDragStop=this.onDragStop.bind(this)}onDragStart(t){t.preventDefault(),this.dragStart=this.state.height+t.pageY,window.addEventListener("mousemove",this.onDragMove),window.addEventListener("mouseup",this.onDragStop),window.addEventListener("dragend",this.onDragStop)}onDragMove(t){t.preventDefault(),this.setState({height:this.dragStart-t.pageY})}onDragStop(t){t.preventDefault(),window.removeEventListener("mousemove",this.onDragMove)}render(){let{height:t}=this.state,{filters:n,events:l,toggleFilter:d,close:h}=this.props;return Wl.default.createElement("div",{className:"eventlog",style:{height:t}},Wl.default.createElement("div",{onMouseDown:this.onDragStart},"Eventlog",Wl.default.createElement("div",{className:"pull-right"},["debug","info","web","warn","error"].map(c=>Wl.default.createElement(jC,{key:c,text:c,checked:n[c],onToggle:()=>d(c)})),Wl.default.createElement("i",{onClick:h,className:"fa fa-close"}))),Wl.default.createElement(ZP,{events:l}))}};o(Dm,"PureEventLog"),Vc(Dm,"propTypes",{filters:rd.default.object.isRequired,events:rd.default.array.isRequired,toggleFilter:rd.default.func.isRequired,close:rd.default.func.isRequired,defaultHeight:rd.default.number}),Vc(Dm,"defaultProps",{defaultHeight:200});var JP=Hi(e=>({filters:e.eventLog.filters,events:e.eventLog.view}),{close:Rp,toggleFilter:gN})(Dm);var un=fe(Oe());function VC(){let e=qe(A=>A.backendState.version),{mode:t,intercept:n,showhost:l,upstream_cert:d,rawtcp:h,http2:c,websocket:v,anticache:C,anticomp:k,stickyauth:O,stickycookie:j,stream_large_bodies:B,listen_host:X,listen_port:J,server:Z,ssl_insecure:R}=qe(A=>A.options);return un.createElement("footer",null,t&&(t.length!==1||t[0]!=="regular")&&un.createElement("span",{className:"label label-success"},t.join(",")),n&&un.createElement("span",{className:"label label-success"},"Intercept: ",n),R&&un.createElement("span",{className:"label label-danger"},"ssl_insecure"),l&&un.createElement("span",{className:"label label-success"},"showhost"),!d&&un.createElement("span",{className:"label label-success"},"no-upstream-cert"),!h&&un.createElement("span",{className:"label label-success"},"no-raw-tcp"),!c&&un.createElement("span",{className:"label label-success"},"no-http2"),!v&&un.createElement("span",{className:"label label-success"},"no-websocket"),C&&un.createElement("span",{className:"label label-success"},"anticache"),k&&un.createElement("span",{className:"label label-success"},"anticomp"),O&&un.createElement("span",{className:"label label-success"},"stickyauth: ",O),j&&un.createElement("span",{className:"label label-success"},"stickycookie: ",j),B&&un.createElement("span",{className:"label label-success"},"stream: ",x0(B)),un.createElement("div",{className:"pull-right"},un.createElement(Ro,null,Z&&un.createElement("span",{className:"label label-primary",title:"HTTP Proxy Server Address"},X||"*",":",J||8080)),un.createElement("span",{className:"label label-default",title:"Mitmproxy Version"},"mitmproxy ",e)))}o(VC,"Footer");var JC=fe(Oe());var ZC=fe(Oe());var nd=fe(Oe());function KC({children:e}){return nd.createElement("div",null,nd.createElement("div",{className:"modal-backdrop fade in"}),nd.createElement("div",{className:"modal modal-visible",id:"optionsModal",tabIndex:-1,role:"dialog","aria-labelledby":"options"},nd.createElement("div",{className:"modal-dialog modal-lg",role:"document"},nd.createElement("div",{className:"modal-content"},e))))}o(KC,"ModalLayout");var mr=fe(Oe());var Fo=fe(Oe()),Bo=fe(Sm());var eO=fe(ti()),XB=o(e=>{e.key!=="Escape"&&e.stopPropagation()},"stopPropagation");GC.propTypes={value:Bo.default.bool.isRequired,onChange:Bo.default.func.isRequired};function GC(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);return Fo.default.createElement("div",{className:"checkbox"},Fo.default.createElement("label",null,Fo.default.createElement("input",ke({type:"checkbox",checked:e,onChange:h=>t(h.target.checked)},n)),"Enable"))}o(GC,"BooleanOption");YC.propTypes={value:Bo.default.string,onChange:Bo.default.func.isRequired};function YC(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);return Fo.default.createElement("input",ke({type:"text",value:e||"",onChange:h=>t(h.target.value)},n))}o(YC,"StringOption");function tO(e){return function(l){var d=l,{onChange:t}=d,n=Ws(d,["onChange"]);return Fo.default.createElement(e,ke({onChange:h=>t(h||null)},n))}}o(tO,"Optional");XC.propTypes={value:Bo.default.number.isRequired,onChange:Bo.default.func.isRequired};function XC(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);return Fo.default.createElement("input",ke({type:"number",value:e,onChange:h=>t(parseInt(h.target.value))},n))}o(XC,"NumberOption");rO.propTypes={value:Bo.default.string.isRequired,onChange:Bo.default.func.isRequired};function rO(d){var h=d,{value:e,onChange:t,choices:n}=h,l=Ws(h,["value","onChange","choices"]);return Fo.default.createElement("select",ke({onChange:c=>t(c.target.value),value:e},l),n.map(c=>Fo.default.createElement("option",{key:c,value:c},c)))}o(rO,"ChoicesOption");nO.propTypes={value:Bo.default.arrayOf(Bo.default.string).isRequired,onChange:Bo.default.func.isRequired};function nO(l){var d=l,{value:e,onChange:t}=d,n=Ws(d,["value","onChange"]);let h=Math.max(e.length,1);return Fo.default.createElement("textarea",ke({rows:h,value:e.join(` `),onChange:c=>t(c.target.value.split(` -`))},n))}o(nO,"StringSequenceOption");var QB={bool:GC,str:YC,int:XC,"optional str":tO(YC),"optional int":tO(XC),"sequence of str":nO};function ZB({choices:e,type:t,value:n,onChange:l,name:d,error:h}){let c,v={};if(e)c=rO,v.choices=e;else if(c=QB[t],!c)throw`unknown option type ${t}`;return c!==GC&&(v.className="form-control"),Fo.default.createElement("div",{className:(0,eO.default)({"has-error":h})},Fo.default.createElement(c,ke({name:d,value:n,onChange:l,onKeyDown:XB},v)))}o(ZB,"PureOption");var iO=Hi((e,{name:t})=>ke(ke({},e.options_meta[t]),e.ui.optionsEditor[t]),(e,{name:t})=>({onChange:n=>e(Ip(t,n))}))(ZB);var Ry=fe(Qh());function JB({help:e}){return mr.default.createElement("div",{className:"help-block small"},e)}o(JB,"PureOptionHelp");var e4=Hi((e,{name:t})=>({help:e.options_meta[t].help}))(JB);function t4({error:e}){return e?mr.default.createElement("div",{className:"small text-danger"},e):null}o(t4,"PureOptionError");var r4=Hi((e,{name:t})=>({error:e.ui.optionsEditor[t]&&e.ui.optionsEditor[t].error}))(t4);function n4({value:e,defaultVal:t}){if(e===t)return null;if(typeof t=="boolean")t=t?"true":"false";else if(Array.isArray(t)){if(Ry.default.isEmpty(Ry.default.compact(e))&&Ry.default.isEmpty(t))return null;t="[ ]"}else t===""?t='""':t===null&&(t="null");return mr.default.createElement("div",{className:"small"},"Default: ",mr.default.createElement("strong",null," ",t," ")," ")}o(n4,"PureOptionDefault");var i4=Hi((e,{name:t})=>({value:e.options[t],defaultVal:e.options_meta[t].default}))(n4),QC=class extends mr.Component{constructor(t,n){super(t,n);this.state={title:"Options"}}componentWillUnmount(){}render(){let{hideModal:t,options:n}=this.props,{title:l}=this.state;return mr.default.createElement("div",null,mr.default.createElement("div",{className:"modal-header"},mr.default.createElement("button",{type:"button",className:"close","data-dismiss":"modal",onClick:()=>{t()}},mr.default.createElement("i",{className:"fa fa-fw fa-times"})),mr.default.createElement("div",{className:"modal-title"},mr.default.createElement("h4",null,l))),mr.default.createElement("div",{className:"modal-body"},mr.default.createElement("div",{className:"form-horizontal"},n.map(d=>mr.default.createElement("div",{key:d,className:"form-group"},mr.default.createElement("div",{className:"col-xs-6"},mr.default.createElement("label",{htmlFor:d},d),mr.default.createElement(e4,{name:d})),mr.default.createElement("div",{className:"col-xs-6"},mr.default.createElement(iO,{name:d}),mr.default.createElement(r4,{name:d}),mr.default.createElement(i4,{name:d})))))),mr.default.createElement("div",{className:"modal-footer"}))}};o(QC,"PureOptionModal");var oO=Hi(e=>({options:Object.keys(e.options_meta).sort()}),{hideModal:z0,save:LN})(QC);function o4(){return ZC.createElement(KC,null,ZC.createElement(oO,null))}o(o4,"OptionModal");var sO=[o4];function eb(){let e=qe(n=>n.ui.modal.activeModal),t=sO.find(n=>n.name===e);return e&&t!==void 0?JC.createElement(t,null):JC.createElement("div",null)}o(eb,"PureModal");var tb=class extends Wn.Component{constructor(){super(...arguments);this.state={};this.render=o(()=>{var l;let{showEventLog:t,showCommandBar:n}=this.props;return this.state.error?(console.log("ERR",this.state),Wn.default.createElement("div",{className:"container"},Wn.default.createElement("h1",null,"mitmproxy has crashed."),Wn.default.createElement("pre",null,this.state.error.stack,Wn.default.createElement("br",null),Wn.default.createElement("br",null),"Component Stack:",(l=this.state.errorInfo)==null?void 0:l.componentStack),Wn.default.createElement("p",null,"Please lodge a bug report at ",Wn.default.createElement("a",{href:"https://github.com/mitmproxy/mitmproxy/issues"},"https://github.com/mitmproxy/mitmproxy/issues"),"."))):Wn.default.createElement("div",{id:"container",tabIndex:0},Wn.default.createElement(WC,null),Wn.default.createElement(IC,null),n&&Wn.default.createElement(zC,{key:"commandbar"}),t&&Wn.default.createElement(JP,{key:"eventlog"}),Wn.default.createElement(VC,null),Wn.default.createElement(eb,null))},"render")}componentDidMount(){window.addEventListener("keydown",this.props.onKeyDown)}componentWillUnmount(){window.removeEventListener("keydown",this.props.onKeyDown)}componentDidCatch(t,n){this.setState({error:t,errorInfo:n})}};o(tb,"ProxyAppMain");var lO=Hi(e=>({showEventLog:e.eventLog.visible,showCommandBar:e.commandBar.visible}),{onKeyDown:SL})(tb);var yu={SEARCH:"s",HIGHLIGHT:"h",SHOW_EVENTLOG:"e",SHOW_COMMANDBAR:"c"};function s4(e){let[t,n]=window.location.hash.substr(1).split("?",2),l=t.substr(1).split("/");if(l[0]==="flows"&&l.length==3){let[d,h]=l.slice(1);e.dispatch(Af(d)),e.dispatch(Lf(h))}n&&n.split("&").forEach(d=>{let[h,c]=d.split("=",2);switch(c=decodeURIComponent(c),h){case yu.SEARCH:e.dispatch(A0(c));break;case yu.HIGHLIGHT:e.dispatch(D0(c));break;case yu.SHOW_EVENTLOG:e.getState().eventLog.visible||e.dispatch(Rp());break;case yu.SHOW_COMMANDBAR:e.getState().commandBar.visible||e.dispatch(V0());break;default:console.error(`unimplemented query arg: ${d}`)}})}o(s4,"updateStoreFromUrl");function l4(e){let t=e.getState(),n={[yu.SEARCH]:t.flows.filter,[yu.HIGHLIGHT]:t.flows.highlight,[yu.SHOW_EVENTLOG]:t.eventLog.visible,[yu.SHOW_COMMANDBAR]:t.commandBar.visible},l=Object.keys(n).filter(c=>n[c]).map(c=>`${c}=${encodeURIComponent(n[c])}`).join("&"),d;t.flows.selected.length>0?d=`/flows/${t.flows.selected[0]}/${t.ui.flow.tab}`:d="/flows",l&&(d+="?"+l);let h=window.location.pathname;h==="blank"&&(h="/"),window.location.hash.substr(1)!==d&&history.replaceState(void 0,"",`${h}#${d}`)}o(l4,"updateUrlFromStore");function rb(e){s4(e),e.subscribe(()=>l4(e))}o(rb,"initialize");var a4="reset",Rm=class{constructor(t){this.activeFetches={},this.store=t,this.connect()}connect(){this.socket=new WebSocket(location.origin.replace("http","ws")+"/updates"),this.socket.addEventListener("open",()=>this.onOpen()),this.socket.addEventListener("close",t=>this.onClose(t)),this.socket.addEventListener("message",t=>this.onMessage(JSON.parse(t.data))),this.socket.addEventListener("error",t=>this.onError(t))}onOpen(){this.fetchData("state"),this.fetchData("flows"),this.fetchData("events"),this.fetchData("options"),this.store.dispatch(_N())}fetchData(t){let n=[];this.activeFetches[t]=n,kt(`./${t}`).then(l=>l.json()).then(l=>{this.activeFetches[t]===n&&this.receive(t,l)})}onMessage(t){if(t.cmd===a4)return this.fetchData(t.resource);if(t.resource in this.activeFetches)this.activeFetches[t.resource].push(t);else{let n=`${t.resource}_${t.cmd}`.toUpperCase();this.store.dispatch(ke({type:n},t))}}receive(t,n){let l=`${t}_RECEIVE`.toUpperCase();this.store.dispatch({type:l,cmd:"receive",resource:t,data:n});let d=this.activeFetches[t];delete this.activeFetches[t],d.forEach(h=>this.onMessage(h)),Object.keys(this.activeFetches).length===0&&this.store.dispatch(TN())}onClose(t){this.store.dispatch(kN(`Connection closed at ${new Date().toUTCString()} with error code ${t.code}.`)),console.error("websocket connection closed",t)}onError(t){console.error("websocket connection errored",arguments)}};o(Rm,"WebsocketBackend");var Im=class{constructor(t){this.store=t,this.onOpen()}onOpen(){this.fetchData("flows"),this.fetchData("options")}fetchData(t){kt(`./${t}`).then(n=>n.json()).then(n=>{this.receive(t,n)})}receive(t,n){let l=`${t}_RECEIVE`.toUpperCase();this.store.dispatch({type:l,cmd:"receive",resource:t,data:n})}};o(Im,"StaticBackend");rb(Fp);window.MITMWEB_STATIC?window.backend=new Im(Fp):window.backend=new Rm(Fp);window.addEventListener("error",e=>{Fp.dispatch(vN(`${e.message} -${e.error.stack}`))});document.addEventListener("DOMContentLoaded",()=>{(0,aO.render)(nb.createElement(Ux,{store:Fp},nb.createElement(lO,null)),document.getElementById("mitmproxy"))});})(); +`))},n))}o(nO,"StringSequenceOption");var QB={bool:GC,str:YC,int:XC,"optional str":tO(YC),"optional int":tO(XC),"sequence of str":nO};function ZB({choices:e,type:t,value:n,onChange:l,name:d,error:h}){let c,v={};if(e)c=rO,v.choices=e;else if(c=QB[t],!c)throw`unknown option type ${t}`;return c!==GC&&(v.className="form-control"),Fo.default.createElement("div",{className:(0,eO.default)({"has-error":h})},Fo.default.createElement(c,ke({name:d,value:n,onChange:l,onKeyDown:XB},v)))}o(ZB,"PureOption");var iO=Hi((e,{name:t})=>ke(ke({},e.options_meta[t]),e.ui.optionsEditor[t]),(e,{name:t})=>({onChange:n=>e(Ip(t,n))}))(ZB);var Ay=fe(Qh());function JB({help:e}){return mr.default.createElement("div",{className:"help-block small"},e)}o(JB,"PureOptionHelp");var eH=Hi((e,{name:t})=>({help:e.options_meta[t].help}))(JB);function tH({error:e}){return e?mr.default.createElement("div",{className:"small text-danger"},e):null}o(tH,"PureOptionError");var rH=Hi((e,{name:t})=>({error:e.ui.optionsEditor[t]&&e.ui.optionsEditor[t].error}))(tH);function nH({value:e,defaultVal:t}){if(e===t)return null;if(typeof t=="boolean")t=t?"true":"false";else if(Array.isArray(t)){if(Ay.default.isEmpty(Ay.default.compact(e))&&Ay.default.isEmpty(t))return null;t="[ ]"}else t===""?t='""':t===null&&(t="null");return mr.default.createElement("div",{className:"small"},"Default: ",mr.default.createElement("strong",null," ",t," ")," ")}o(nH,"PureOptionDefault");var iH=Hi((e,{name:t})=>({value:e.options[t],defaultVal:e.options_meta[t].default}))(nH),QC=class extends mr.Component{constructor(t,n){super(t,n);this.state={title:"Options"}}componentWillUnmount(){}render(){let{hideModal:t,options:n}=this.props,{title:l}=this.state;return mr.default.createElement("div",null,mr.default.createElement("div",{className:"modal-header"},mr.default.createElement("button",{type:"button",className:"close","data-dismiss":"modal",onClick:()=>{t()}},mr.default.createElement("i",{className:"fa fa-fw fa-times"})),mr.default.createElement("div",{className:"modal-title"},mr.default.createElement("h4",null,l))),mr.default.createElement("div",{className:"modal-body"},mr.default.createElement("div",{className:"form-horizontal"},n.map(d=>mr.default.createElement("div",{key:d,className:"form-group"},mr.default.createElement("div",{className:"col-xs-6"},mr.default.createElement("label",{htmlFor:d},d),mr.default.createElement(eH,{name:d})),mr.default.createElement("div",{className:"col-xs-6"},mr.default.createElement(iO,{name:d}),mr.default.createElement(rH,{name:d}),mr.default.createElement(iH,{name:d})))))),mr.default.createElement("div",{className:"modal-footer"}))}};o(QC,"PureOptionModal");var oO=Hi(e=>({options:Object.keys(e.options_meta).sort()}),{hideModal:z0,save:LN})(QC);function oH(){return ZC.createElement(KC,null,ZC.createElement(oO,null))}o(oH,"OptionModal");var sO=[oH];function eb(){let e=qe(n=>n.ui.modal.activeModal),t=sO.find(n=>n.name===e);return e&&t!==void 0?JC.createElement(t,null):JC.createElement("div",null)}o(eb,"PureModal");var tb=class extends Wn.Component{constructor(){super(...arguments);this.state={};this.render=o(()=>{var l;let{showEventLog:t,showCommandBar:n}=this.props;return this.state.error?(console.log("ERR",this.state),Wn.default.createElement("div",{className:"container"},Wn.default.createElement("h1",null,"mitmproxy has crashed."),Wn.default.createElement("pre",null,this.state.error.stack,Wn.default.createElement("br",null),Wn.default.createElement("br",null),"Component Stack:",(l=this.state.errorInfo)==null?void 0:l.componentStack),Wn.default.createElement("p",null,"Please lodge a bug report at ",Wn.default.createElement("a",{href:"https://github.com/mitmproxy/mitmproxy/issues"},"https://github.com/mitmproxy/mitmproxy/issues"),"."))):Wn.default.createElement("div",{id:"container",tabIndex:0},Wn.default.createElement(WC,null),Wn.default.createElement(IC,null),n&&Wn.default.createElement(zC,{key:"commandbar"}),t&&Wn.default.createElement(JP,{key:"eventlog"}),Wn.default.createElement(VC,null),Wn.default.createElement(eb,null))},"render")}componentDidMount(){window.addEventListener("keydown",this.props.onKeyDown)}componentWillUnmount(){window.removeEventListener("keydown",this.props.onKeyDown)}componentDidCatch(t,n){this.setState({error:t,errorInfo:n})}};o(tb,"ProxyAppMain");var lO=Hi(e=>({showEventLog:e.eventLog.visible,showCommandBar:e.commandBar.visible}),{onKeyDown:SL})(tb);var yu={SEARCH:"s",HIGHLIGHT:"h",SHOW_EVENTLOG:"e",SHOW_COMMANDBAR:"c"};function sH(e){let[t,n]=window.location.hash.substr(1).split("?",2),l=t.substr(1).split("/");if(l[0]==="flows"&&l.length==3){let[d,h]=l.slice(1);e.dispatch(Af(d)),e.dispatch(Lf(h))}n&&n.split("&").forEach(d=>{let[h,c]=d.split("=",2);switch(c=decodeURIComponent(c),h){case yu.SEARCH:e.dispatch(A0(c));break;case yu.HIGHLIGHT:e.dispatch(D0(c));break;case yu.SHOW_EVENTLOG:e.getState().eventLog.visible||e.dispatch(Rp());break;case yu.SHOW_COMMANDBAR:e.getState().commandBar.visible||e.dispatch(V0());break;default:console.error(`unimplemented query arg: ${d}`)}})}o(sH,"updateStoreFromUrl");function lH(e){let t=e.getState(),n={[yu.SEARCH]:t.flows.filter,[yu.HIGHLIGHT]:t.flows.highlight,[yu.SHOW_EVENTLOG]:t.eventLog.visible,[yu.SHOW_COMMANDBAR]:t.commandBar.visible},l=Object.keys(n).filter(c=>n[c]).map(c=>`${c}=${encodeURIComponent(n[c])}`).join("&"),d;t.flows.selected.length>0?d=`/flows/${t.flows.selected[0]}/${t.ui.flow.tab}`:d="/flows",l&&(d+="?"+l);let h=window.location.pathname;h==="blank"&&(h="/"),window.location.hash.substr(1)!==d&&history.replaceState(void 0,"",`${h}#${d}`)}o(lH,"updateUrlFromStore");function rb(e){sH(e),e.subscribe(()=>lH(e))}o(rb,"initialize");var aH="reset",Rm=class{constructor(t){this.activeFetches={},this.store=t,this.connect()}connect(){this.socket=new WebSocket(location.origin.replace("http","ws")+"/updates"),this.socket.addEventListener("open",()=>this.onOpen()),this.socket.addEventListener("close",t=>this.onClose(t)),this.socket.addEventListener("message",t=>this.onMessage(JSON.parse(t.data))),this.socket.addEventListener("error",t=>this.onError(t))}onOpen(){this.fetchData("state"),this.fetchData("flows"),this.fetchData("events"),this.fetchData("options"),this.store.dispatch(_N())}fetchData(t){let n=[];this.activeFetches[t]=n,kt(`./${t}`).then(l=>l.json()).then(l=>{this.activeFetches[t]===n&&this.receive(t,l)})}onMessage(t){if(t.cmd===aH)return this.fetchData(t.resource);if(t.resource in this.activeFetches)this.activeFetches[t.resource].push(t);else{let n=`${t.resource}_${t.cmd}`.toUpperCase();this.store.dispatch(ke({type:n},t))}}receive(t,n){let l=`${t}_RECEIVE`.toUpperCase();this.store.dispatch({type:l,cmd:"receive",resource:t,data:n});let d=this.activeFetches[t];delete this.activeFetches[t],d.forEach(h=>this.onMessage(h)),Object.keys(this.activeFetches).length===0&&this.store.dispatch(TN())}onClose(t){this.store.dispatch(kN(`Connection closed at ${new Date().toUTCString()} with error code ${t.code}.`)),console.error("websocket connection closed",t)}onError(t){console.error("websocket connection errored",arguments)}};o(Rm,"WebsocketBackend");var Im=class{constructor(t){this.store=t,this.onOpen()}onOpen(){this.fetchData("flows"),this.fetchData("options")}fetchData(t){kt(`./${t}`).then(n=>n.json()).then(n=>{this.receive(t,n)})}receive(t,n){let l=`${t}_RECEIVE`.toUpperCase();this.store.dispatch({type:l,cmd:"receive",resource:t,data:n})}};o(Im,"StaticBackend");rb(Fp);window.MITMWEB_STATIC?window.backend=new Im(Fp):window.backend=new Rm(Fp);window.addEventListener("error",e=>{Fp.dispatch(vN(`${e.message} +${e.error.stack}`))});document.addEventListener("DOMContentLoaded",()=>{(0,aO.render)(nb.createElement(Hx,{store:Fp},nb.createElement(lO,null)),document.getElementById("mitmproxy"))});})(); /* object-assign (c) Sindre Sorhus From e6cb337994b08007d4bb280619dbf2863a3a5149 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 29 Nov 2022 13:53:22 +0100 Subject: [PATCH 135/695] add `dependabot.yml` --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..c05de27708 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: "monthly" From 8361c81cdfd81f3886caba70006e186967f34442 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 29 Nov 2022 13:53:33 +0100 Subject: [PATCH 136/695] add autofix.ci --- .github/workflows/autofix.yml | 40 +++++++++++++++++++ CHANGELOG.md | 1 + README.md | 1 + docs/scripts/api-events.py | 16 ++++++-- examples/contrib/mitmproxywrapper.py | 3 +- .../data/addonscripts/import_error.py | 2 + 6 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/autofix.yml diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml new file mode 100644 index 0000000000..90aa5dd413 --- /dev/null +++ b/.github/workflows/autofix.yml @@ -0,0 +1,40 @@ +name: autofix.ci +on: [ push, pull_request ] +permissions: + contents: read + +jobs: + autofix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: install-pinned/pyupgrade@847ef2b8aa35a3817372540b887f4130d864d6b7 + - name: Run pyupgrade + run: | + shopt -s globstar + export GLOBIGNORE='mitmproxy/contrib/**' + pyupgrade --exit-zero-even-if-changed --keep-runtime-typing --py39-plus **/*.py + + - uses: install-pinned/reorder_python_imports@97c3e89c53ae5513cc41716e876e26daff8bbdd6 + - name: Run reorder-python-imports + run: | + shopt -s globstar + export GLOBIGNORE='mitmproxy/contrib/**' + reorder-python-imports --exit-zero-even-if-changed --py39-plus **/*.py + - uses: install-pinned/yesqa@b752c9eed899985c6df094e35d7a5a5bd1b94acb + + - name: Run yesqa + run: | + shopt -s globstar + export GLOBIGNORE='mitmproxy/contrib/**' + yesqa **/*.py || true + - uses: install-pinned/autoflake@fa3c1715169ac36d903ee9d492d64beb5cad331f + - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . + + - uses: install-pinned/black@81e6dbf82145462d413a6662dd703fa382edeb11 + - run: black --extend-exclude mitmproxy/contrib . + + - uses: mhils/add-pr-ref-in-changelog@main + + - uses: autofix-ci/action@8bc06253bec489732e5f9c52884c7cace15c0160 diff --git a/CHANGELOG.md b/CHANGELOG.md index c460a474f0..64292246ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) * ASGI/WSGI apps can now listen on all ports for a specific hostname. This makes it simpler to accept both HTTP and HTTPS. + ([#5725](https://github.com/mitmproxy/mitmproxy/pull/5725), @mhils) ### Breaking Changes diff --git a/README.md b/README.md index 4b65402f72..928374c0b3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # mitmproxy [![Continuous Integration Status](https://github.com/mitmproxy/mitmproxy/workflows/CI/badge.svg?branch=main)](https://github.com/mitmproxy/mitmproxy/actions?query=branch%3Amain) +[![autofix.ci: enabled](https://shields.mitmproxy.org/badge/autofix.ci-yes-success?logo=data:image/svg+xml;base64,PHN2ZyBmaWxsPSIjZmZmIiB2aWV3Qm94PSIwIDAgMTI4IDEyOCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48cGF0aCB0cmFuc2Zvcm09InNjYWxlKDAuMDYxLC0wLjA2MSkgdHJhbnNsYXRlKC0yNTAsLTE3NTApIiBkPSJNMTMyNSAtMzQwcS0xMTUgMCAtMTY0LjUgMzIuNXQtNDkuNSAxMTQuNXEwIDMyIDUgNzAuNXQxMC41IDcyLjV0NS41IDU0djIyMHEtMzQgLTkgLTY5LjUgLTE0dC03MS41IC01cS0xMzYgMCAtMjUxLjUgNjJ0LTE5MSAxNjl0LTkyLjUgMjQxcS05MCAxMjAgLTkwIDI2NnEwIDEwOCA0OC41IDIwMC41dDEzMiAxNTUuNXQxODguNSA4MXExNSA5OSAxMDAuNSAxODAuNXQyMTcgMTMwLjV0MjgyLjUgNDlxMTM2IDAgMjU2LjUgLTQ2IHQyMDkgLTEyNy41dDEyOC41IC0xODkuNXExNDkgLTgyIDIyNyAtMjEzLjV0NzggLTI5OS41cTAgLTEzNiAtNTggLTI0NnQtMTY1LjUgLTE4NC41dC0yNTYuNSAtMTAzLjVsLTI0MyAtMzAwdi01MnEwIC0yNyAzLjUgLTU2LjV0Ni41IC01Ny41dDMgLTUycTAgLTg1IC00MS41IC0xMTguNXQtMTU3LjUgLTMzLjV6TTEzMjUgLTI2MHE3NyAwIDk4IDE0LjV0MjEgNTcuNXEwIDI5IC0zIDY4dC02LjUgNzN0LTMuNSA0OHY2NGwyMDcgMjQ5IHEtMzEgMCAtNjAgNS41dC01NCAxMi41bC0xMDQgLTEyM3EtMSAzNCAtMiA2My41dC0xIDU0LjVxMCA2OSA5IDEyM2wzMSAyMDBsLTExNSAtMjhsLTQ2IC0yNzFsLTIwNSAyMjZxLTE5IC0xNSAtNDMgLTI4LjV0LTU1IC0yNi41bDIxOSAtMjQydi0yNzZxMCAtMjAgLTUuNSAtNjB0LTEwLjUgLTc5dC01IC01OHEwIC00MCAzMCAtNTMuNXQxMDQgLTEzLjV6TTEyNjIgNjE2cS0xMTkgMCAtMjI5LjUgMzQuNXQtMTkzLjUgOTYuNWw0OCA2NCBxNzMgLTU1IDE3MC41IC04NXQyMDQuNSAtMzBxMTM3IDAgMjQ5IDQ1LjV0MTc5IDEyMXQ2NyAxNjUuNWg4MHEwIC0xMTQgLTc3LjUgLTIwNy41dC0yMDggLTE0OXQtMjg5LjUgLTU1LjV6TTgwMyA1OTVxODAgMCAxNDkgMjkuNXQxMDggNzIuNWwyMjEgLTY3bDMwOSA4NnE0NyAtMzIgMTA0LjUgLTUwdDExNy41IC0xOHE5MSAwIDE2NSAzOHQxMTguNSAxMDMuNXQ0NC41IDE0Ni41cTAgNzYgLTM0LjUgMTQ5dC05NS41IDEzNHQtMTQzIDk5IHEtMzcgMTA3IC0xMTUuNSAxODMuNXQtMTg2IDExNy41dC0yMzAuNSA0MXEtMTAzIDAgLTE5Ny41IC0yNnQtMTY5IC03Mi41dC0xMTcuNSAtMTA4dC00MyAtMTMxLjVxMCAtMzQgMTQuNSAtNjIuNXQ0MC41IC01MC41bC01NSAtNTlxLTM0IDI5IC01NCA2NS41dC0yNSA4MS41cS04MSAtMTggLTE0NSAtNzB0LTEwMSAtMTI1LjV0LTM3IC0xNTguNXEwIC0xMDIgNDguNSAtMTgwLjV0MTI5LjUgLTEyM3QxNzkgLTQ0LjV6Ii8+PC9zdmc+)](https://autofix.ci) [![Coverage Status](https://shields.mitmproxy.org/codecov/c/github/mitmproxy/mitmproxy/main.svg?label=codecov)](https://codecov.io/gh/mitmproxy/mitmproxy) [![Latest Version](https://shields.mitmproxy.org/pypi/v/mitmproxy.svg)](https://pypi.python.org/pypi/mitmproxy) [![Supported Python versions](https://shields.mitmproxy.org/pypi/pyversions/mitmproxy.svg)](https://pypi.python.org/pypi/mitmproxy) diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index d4d934bf3f..94f65f0de6 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -4,9 +4,19 @@ import textwrap from pathlib import Path -from mitmproxy import hooks, log, addonmanager -from mitmproxy.proxy import server_hooks, layer -from mitmproxy.proxy.layers import dns, http, modes, quic, tcp, tls, udp, websocket +from mitmproxy import addonmanager +from mitmproxy import hooks +from mitmproxy import log +from mitmproxy.proxy import layer +from mitmproxy.proxy import server_hooks +from mitmproxy.proxy.layers import dns +from mitmproxy.proxy.layers.http import _hooks as http +from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tcp +from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import udp +from mitmproxy.proxy.layers import websocket known = set() diff --git a/examples/contrib/mitmproxywrapper.py b/examples/contrib/mitmproxywrapper.py index 361093e7b5..075cc3c04e 100644 --- a/examples/contrib/mitmproxywrapper.py +++ b/examples/contrib/mitmproxywrapper.py @@ -87,8 +87,7 @@ def connected_service_names(self): service_names = [] for service_id in service_ids: - scutil_script = 'show Setup:/Network/Service/{}\n'.format( - service_id) + scutil_script = f"show Setup:/Network/Service/{service_id}\n" stdout = self.run_command_with_input( '/usr/sbin/scutil', scutil_script) diff --git a/test/mitmproxy/data/addonscripts/import_error.py b/test/mitmproxy/data/addonscripts/import_error.py index e868bcc347..1418bc5fd1 100644 --- a/test/mitmproxy/data/addonscripts/import_error.py +++ b/test/mitmproxy/data/addonscripts/import_error.py @@ -1 +1,3 @@ import nonexistent + +nonexistent.foo() From 8c2428c9d355ca5fbc3dd90e9820ceb1cc795837 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 13:28:41 +0000 Subject: [PATCH 137/695] [autofix.ci] apply automated fixes --- docs/scripts/api-events.py | 5 +- docs/scripts/clirecording/clidirector.py | 7 +- docs/scripts/clirecording/record.py | 3 +- docs/scripts/clirecording/screenplays.py | 1 - docs/scripts/examples.py | 1 - docs/scripts/filters.py | 1 - docs/scripts/options.py | 7 +- examples/addons/contentview-custom-grpc.py | 8 +- examples/addons/contentview.py | 3 +- examples/addons/filter-flows.py | 3 +- examples/addons/http-stream-modify.py | 3 +- examples/addons/http-trailers.py | 1 - examples/addons/io-read-saved-flows.py | 6 +- examples/addons/io-write-flow-file.py | 3 +- examples/addons/log-events.py | 5 +- examples/addons/nonblocking.py | 1 - examples/addons/shutdown.py | 3 +- examples/addons/tcp-simple.py | 2 +- examples/addons/websocket-inject-message.py | 3 +- examples/addons/wsgi-flask-app.py | 2 +- examples/contrib/all_markers.py | 13 +- examples/contrib/block_dns_over_https.py | 328 +++++++--- examples/contrib/change_upstream_proxy.py | 1 - examples/contrib/check_ssl_pinning.py | 32 +- examples/contrib/custom_next_layer.py | 3 +- examples/contrib/domain_fronting.py | 6 +- examples/contrib/har_dump.py | 111 ++-- examples/contrib/http_manipulate_cookies.py | 16 +- examples/contrib/httpdump.py | 5 +- examples/contrib/jsondump.py | 149 +++-- examples/contrib/link_expander.py | 18 +- examples/contrib/mitmproxywrapper.py | 99 ++- examples/contrib/modify_body_inject_iframe.py | 15 +- examples/contrib/ntlm_upstream_proxy.py | 114 ++-- examples/contrib/remote-debug.py | 5 +- examples/contrib/save_streamed_data.py | 23 +- examples/contrib/search.py | 37 +- examples/contrib/sslstrip.py | 44 +- examples/contrib/suppress_error_responses.py | 2 +- examples/contrib/test_har_dump.py | 42 +- examples/contrib/test_jsondump.py | 42 +- examples/contrib/test_xss_scanner.py | 591 +++++++++++------- examples/contrib/tls_passthrough.py | 17 +- examples/contrib/webscanner_helper/mapping.py | 44 +- .../webscanner_helper/proxyauth_selenium.py | 35 +- .../contrib/webscanner_helper/test_mapping.py | 23 +- .../test_proxyauth_selenium.py | 44 +- .../contrib/webscanner_helper/test_urldict.py | 6 +- .../webscanner_helper/test_urlindex.py | 48 +- .../webscanner_helper/test_urlinjection.py | 32 +- .../webscanner_helper/test_watchdog.py | 21 +- examples/contrib/webscanner_helper/urldict.py | 9 +- .../contrib/webscanner_helper/urlindex.py | 26 +- .../contrib/webscanner_helper/urlinjection.py | 55 +- .../contrib/webscanner_helper/watchdog.py | 32 +- examples/contrib/xss_scanner.py | 375 +++++++---- mitmproxy/addonmanager.py | 9 +- mitmproxy/addons/__init__.py | 12 +- mitmproxy/addons/asgiapp.py | 3 +- mitmproxy/addons/blocklist.py | 6 +- mitmproxy/addons/browser.py | 7 +- mitmproxy/addons/clientplayback.py | 18 +- mitmproxy/addons/command_history.py | 2 +- mitmproxy/addons/comment.py | 4 +- mitmproxy/addons/core.py | 11 +- mitmproxy/addons/cut.py | 16 +- mitmproxy/addons/dns_resolver.py | 5 +- mitmproxy/addons/dumper.py | 24 +- mitmproxy/addons/errorcheck.py | 1 - mitmproxy/addons/eventstore.py | 3 +- mitmproxy/addons/export.py | 9 +- mitmproxy/addons/intercept.py | 5 +- mitmproxy/addons/keepserving.py | 1 + mitmproxy/addons/maplocal.py | 6 +- mitmproxy/addons/mapremote.py | 5 +- mitmproxy/addons/modifybody.py | 6 +- mitmproxy/addons/modifyheaders.py | 5 +- mitmproxy/addons/next_layer.py | 73 ++- mitmproxy/addons/onboarding.py | 2 +- mitmproxy/addons/onboardingapp/__init__.py | 6 +- mitmproxy/addons/proxyauth.py | 12 +- mitmproxy/addons/proxyserver.py | 81 +-- mitmproxy/addons/readfile.py | 5 +- mitmproxy/addons/save.py | 7 +- mitmproxy/addons/script.py | 17 +- mitmproxy/addons/serverplayback.py | 12 +- mitmproxy/addons/stickyauth.py | 2 +- mitmproxy/addons/stickycookie.py | 9 +- mitmproxy/addons/termlog.py | 17 +- mitmproxy/addons/tlsconfig.py | 36 +- mitmproxy/addons/upstream_auth.py | 9 +- mitmproxy/addons/view.py | 24 +- mitmproxy/certs.py | 21 +- mitmproxy/command.py | 19 +- mitmproxy/connection.py | 17 +- mitmproxy/contentviews/__init__.py | 56 +- mitmproxy/contentviews/auto.py | 2 +- mitmproxy/contentviews/base.py | 11 +- mitmproxy/contentviews/graphql.py | 6 +- mitmproxy/contentviews/grpc.py | 20 +- mitmproxy/contentviews/hex.py | 2 +- mitmproxy/contentviews/http3.py | 68 +- mitmproxy/contentviews/image/view.py | 2 +- mitmproxy/contentviews/javascript.py | 2 +- mitmproxy/contentviews/json.py | 11 +- mitmproxy/contentviews/mqtt.py | 11 +- mitmproxy/contentviews/msgpack.py | 19 +- mitmproxy/contentviews/multipart.py | 2 +- mitmproxy/contentviews/protobuf.py | 5 +- mitmproxy/contentviews/raw.py | 2 +- mitmproxy/contentviews/urlencoded.py | 2 +- mitmproxy/contentviews/wbxml.py | 2 +- mitmproxy/contentviews/xml_html.py | 8 +- mitmproxy/coretypes/multidict.py | 4 +- mitmproxy/coretypes/serializable.py | 33 +- mitmproxy/dns.py | 12 +- mitmproxy/eventsequence.py | 4 +- mitmproxy/flow.py | 12 +- mitmproxy/flowfilter.py | 32 +- mitmproxy/hooks.py | 8 +- mitmproxy/http.py | 18 +- mitmproxy/io/__init__.py | 5 +- mitmproxy/io/compat.py | 7 +- mitmproxy/io/io.py | 6 +- mitmproxy/io/tnetstring.py | 4 +- mitmproxy/log.py | 23 +- mitmproxy/master.py | 19 +- mitmproxy/net/check.py | 4 +- mitmproxy/net/encoding.py | 6 +- mitmproxy/net/http/cookies.py | 2 +- mitmproxy/net/http/headers.py | 2 +- mitmproxy/net/http/http1/__init__.py | 24 +- mitmproxy/net/http/http1/read.py | 9 +- mitmproxy/net/http/multipart.py | 2 +- mitmproxy/net/http/url.py | 9 +- mitmproxy/net/http/user_agents.py | 2 - mitmproxy/net/local_ip.py | 1 + mitmproxy/net/server_spec.py | 14 +- mitmproxy/net/tls.py | 13 +- mitmproxy/net/udp.py | 10 +- mitmproxy/options.py | 18 +- mitmproxy/optmanager.py | 23 +- mitmproxy/platform/__init__.py | 3 +- mitmproxy/platform/windows.py | 19 +- mitmproxy/proxy/commands.py | 10 +- mitmproxy/proxy/context.py | 3 +- mitmproxy/proxy/events.py | 10 +- mitmproxy/proxy/layer.py | 13 +- mitmproxy/proxy/layers/__init__.py | 8 +- mitmproxy/proxy/layers/dns.py | 25 +- mitmproxy/proxy/layers/http/__init__.py | 103 +-- mitmproxy/proxy/layers/http/_base.py | 4 +- mitmproxy/proxy/layers/http/_events.py | 2 +- mitmproxy/proxy/layers/http/_http1.py | 60 +- mitmproxy/proxy/layers/http/_http2.py | 66 +- mitmproxy/proxy/layers/http/_http3.py | 126 ++-- mitmproxy/proxy/layers/http/_http_h2.py | 16 +- mitmproxy/proxy/layers/http/_http_h3.py | 85 +-- .../proxy/layers/http/_upstream_proxy.py | 21 +- mitmproxy/proxy/layers/modes.py | 11 +- mitmproxy/proxy/layers/quic.py | 390 ++++++++---- mitmproxy/proxy/layers/tcp.py | 10 +- mitmproxy/proxy/layers/tls.py | 48 +- mitmproxy/proxy/layers/udp.py | 9 +- mitmproxy/proxy/layers/websocket.py | 16 +- mitmproxy/proxy/mode_servers.py | 86 ++- mitmproxy/proxy/mode_specs.py | 38 +- mitmproxy/proxy/server.py | 45 +- mitmproxy/proxy/server_hooks.py | 2 +- mitmproxy/proxy/tunnel.py | 19 +- mitmproxy/script/concurrent.py | 2 +- mitmproxy/tcp.py | 3 +- mitmproxy/test/taddons.py | 5 +- mitmproxy/test/tflow.py | 12 +- mitmproxy/tls.py | 7 +- .../tools/console/commander/commander.py | 3 +- mitmproxy/tools/console/commandexecutor.py | 1 - mitmproxy/tools/console/commands.py | 3 +- mitmproxy/tools/console/common.py | 17 +- mitmproxy/tools/console/consoleaddons.py | 3 +- mitmproxy/tools/console/eventlog.py | 3 +- mitmproxy/tools/console/flowdetailview.py | 6 +- mitmproxy/tools/console/flowview.py | 1 - .../tools/console/grideditor/__init__.py | 24 +- mitmproxy/tools/console/grideditor/base.py | 17 +- .../tools/console/grideditor/col_bytes.py | 1 + .../tools/console/grideditor/col_subgrid.py | 9 +- .../tools/console/grideditor/col_text.py | 1 - .../tools/console/grideditor/col_viewany.py | 1 + mitmproxy/tools/console/grideditor/editors.py | 3 +- mitmproxy/tools/console/keybindings.py | 3 +- mitmproxy/tools/console/keymap.py | 11 +- mitmproxy/tools/console/master.py | 11 +- mitmproxy/tools/console/options.py | 9 +- mitmproxy/tools/console/overlay.py | 4 +- mitmproxy/tools/console/palettes.py | 4 +- mitmproxy/tools/console/quickhelp.py | 6 +- mitmproxy/tools/console/signals.py | 8 +- mitmproxy/tools/console/statusbar.py | 31 +- mitmproxy/tools/console/window.py | 5 +- mitmproxy/tools/dump.py | 6 +- mitmproxy/tools/main.py | 24 +- mitmproxy/tools/web/app.py | 52 +- mitmproxy/tools/web/master.py | 13 +- mitmproxy/tools/web/static_viewer.py | 6 +- mitmproxy/types.py | 10 +- mitmproxy/udp.py | 3 +- mitmproxy/utils/arg_check.py | 2 +- mitmproxy/utils/data.py | 2 +- mitmproxy/utils/emoji.py | 1 - mitmproxy/utils/human.py | 10 +- mitmproxy/utils/magisk.py | 19 +- mitmproxy/utils/signals.py | 21 +- mitmproxy/utils/sliding_window.py | 5 +- mitmproxy/utils/strutils.py | 6 +- mitmproxy/utils/typecheck.py | 2 +- mitmproxy/utils/vt_codes.py | 1 - mitmproxy/websocket.py | 8 +- release/build-and-deploy-docker.py | 3 +- release/build.py | 16 +- release/release.py | 79 ++- release/selftest.py | 5 +- setup.py | 5 +- test/conftest.py | 1 + test/examples/test_examples.py | 4 +- test/filename_matching.py | 3 +- test/full_coverage_plugin.py | 5 +- test/helper_tools/dumperview.py | 3 +- test/helper_tools/getcert | 2 - test/helper_tools/linkify-changelog.py | 2 +- test/helper_tools/loggrep.py | 2 +- test/helper_tools/memoryleak.py | 4 +- test/individual_coverage.py | 12 +- test/mitmproxy/addons/test_anticache.py | 3 +- test/mitmproxy/addons/test_anticomp.py | 3 +- test/mitmproxy/addons/test_browser.py | 4 +- test/mitmproxy/addons/test_clientplayback.py | 9 +- test/mitmproxy/addons/test_command_history.py | 2 +- test/mitmproxy/addons/test_comment.py | 3 +- test/mitmproxy/addons/test_core.py | 5 +- test/mitmproxy/addons/test_cut.py | 12 +- test/mitmproxy/addons/test_disable_h2c.py | 3 +- test/mitmproxy/addons/test_dns_resolver.py | 7 +- test/mitmproxy/addons/test_export.py | 6 +- test/mitmproxy/addons/test_intercept.py | 2 +- test/mitmproxy/addons/test_keepserving.py | 2 +- test/mitmproxy/addons/test_maplocal.py | 6 +- test/mitmproxy/addons/test_modifyheaders.py | 3 +- test/mitmproxy/addons/test_next_layer.py | 61 +- test/mitmproxy/addons/test_proxyserver.py | 88 ++- test/mitmproxy/addons/test_readfile.py | 2 +- test/mitmproxy/addons/test_save.py | 2 +- test/mitmproxy/addons/test_script.py | 21 +- test/mitmproxy/addons/test_stickyauth.py | 7 +- test/mitmproxy/addons/test_stickycookie.py | 5 +- test/mitmproxy/addons/test_termlog.py | 5 +- test/mitmproxy/addons/test_tlsconfig.py | 41 +- test/mitmproxy/addons/test_upstream_auth.py | 3 +- test/mitmproxy/addons/test_view.py | 10 +- .../mitmproxy/contentviews/image/test_view.py | 2 +- test/mitmproxy/contentviews/test_auto.py | 2 +- test/mitmproxy/contentviews/test_base.py | 1 + test/mitmproxy/contentviews/test_css.py | 2 +- test/mitmproxy/contentviews/test_graphql.py | 2 +- test/mitmproxy/contentviews/test_grpc.py | 18 +- test/mitmproxy/contentviews/test_hex.py | 2 +- test/mitmproxy/contentviews/test_http3.py | 81 +-- .../mitmproxy/contentviews/test_javascript.py | 2 +- test/mitmproxy/contentviews/test_json.py | 79 ++- test/mitmproxy/contentviews/test_mqtt.py | 17 +- test/mitmproxy/contentviews/test_msgpack.py | 95 ++- test/mitmproxy/contentviews/test_multipart.py | 2 +- test/mitmproxy/contentviews/test_protobuf.py | 2 +- test/mitmproxy/contentviews/test_query.py | 2 +- test/mitmproxy/contentviews/test_raw.py | 2 +- .../mitmproxy/contentviews/test_urlencoded.py | 2 +- test/mitmproxy/contentviews/test_wbxml.py | 2 +- test/mitmproxy/contentviews/test_xml_html.py | 2 +- test/mitmproxy/coretypes/test_bidi.py | 1 + test/mitmproxy/coretypes/test_serializable.py | 60 +- .../data/addonscripts/concurrent_decorator.py | 1 + .../concurrent_decorator_class.py | 1 + test/mitmproxy/io/test_compat.py | 2 +- test/mitmproxy/io/test_io.py | 9 +- test/mitmproxy/io/test_tnetstring.py | 6 +- test/mitmproxy/net/dns/test_domain_names.py | 14 +- .../mitmproxy/net/http/http1/test_assemble.py | 21 +- test/mitmproxy/net/http/http1/test_read.py | 23 +- test/mitmproxy/net/http/test_cookies.py | 3 +- test/mitmproxy/net/http/test_headers.py | 3 +- test/mitmproxy/net/test_encoding.py | 1 + test/mitmproxy/net/test_tls.py | 6 +- test/mitmproxy/net/test_udp.py | 19 +- test/mitmproxy/platform/test_pf.py | 2 + test/mitmproxy/proxy/bench.py | 6 +- test/mitmproxy/proxy/conftest.py | 13 +- .../layers/http/hyper_h2_test_helpers.py | 25 +- test/mitmproxy/proxy/layers/http/test_http.py | 63 +- .../mitmproxy/proxy/layers/http/test_http1.py | 23 +- .../mitmproxy/proxy/layers/http/test_http2.py | 53 +- .../mitmproxy/proxy/layers/http/test_http3.py | 235 ++++--- .../proxy/layers/http/test_http_fuzz.py | 78 +-- .../layers/http/test_http_version_interop.py | 22 +- test/mitmproxy/proxy/layers/test_dns.py | 17 +- test/mitmproxy/proxy/layers/test_modes.py | 74 ++- test/mitmproxy/proxy/layers/test_quic.py | 215 ++++--- .../proxy/layers/test_socks5_fuzz.py | 13 +- test/mitmproxy/proxy/layers/test_tcp.py | 15 +- test/mitmproxy/proxy/layers/test_tls.py | 66 +- test/mitmproxy/proxy/layers/test_tls_fuzz.py | 8 +- test/mitmproxy/proxy/layers/test_udp.py | 14 +- test/mitmproxy/proxy/layers/test_websocket.py | 28 +- test/mitmproxy/proxy/test_context.py | 3 +- test/mitmproxy/proxy/test_events.py | 3 +- test/mitmproxy/proxy/test_layer.py | 4 +- test/mitmproxy/proxy/test_mode_servers.py | 45 +- test/mitmproxy/proxy/test_mode_specs.py | 8 +- test/mitmproxy/proxy/test_tunnel.py | 20 +- test/mitmproxy/proxy/test_tutils.py | 4 +- test/mitmproxy/proxy/tutils.py | 19 +- test/mitmproxy/script/test_concurrent.py | 2 +- test/mitmproxy/test_addonmanager.py | 3 +- test/mitmproxy/test_certs.py | 9 +- test/mitmproxy/test_command_lexer.py | 3 +- test/mitmproxy/test_connection.py | 18 +- test/mitmproxy/test_dns.py | 3 +- test/mitmproxy/test_flow.py | 6 +- test/mitmproxy/test_flowfilter.py | 7 +- test/mitmproxy/test_http.py | 10 +- test/mitmproxy/test_optmanager.py | 4 +- test/mitmproxy/test_tcp.py | 2 +- test/mitmproxy/test_tls.py | 13 +- test/mitmproxy/test_types.py | 11 +- test/mitmproxy/test_udp.py | 2 +- test/mitmproxy/test_websocket.py | 2 +- .../tools/console/test_contentview.py | 2 +- test/mitmproxy/tools/console/test_keymap.py | 6 +- .../mitmproxy/tools/console/test_quickhelp.py | 5 +- test/mitmproxy/tools/web/test_app.py | 30 +- test/mitmproxy/tools/web/test_master.py | 5 +- .../mitmproxy/tools/web/test_static_viewer.py | 9 +- test/mitmproxy/utils/test_arg_check.py | 2 +- test/mitmproxy/utils/test_data.py | 1 + test/mitmproxy/utils/test_debug.py | 1 + test/mitmproxy/utils/test_emoji.py | 2 +- test/mitmproxy/utils/test_human.py | 6 +- test/mitmproxy/utils/test_magisk.py | 6 +- test/mitmproxy/utils/test_signals.py | 4 +- test/mitmproxy/utils/test_spec.py | 1 + test/mitmproxy/utils/test_strutils.py | 6 +- test/mitmproxy/utils/test_typecheck.py | 7 +- 351 files changed, 4679 insertions(+), 2801 deletions(-) diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index 94f65f0de6..8f3a72e726 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -10,20 +10,21 @@ from mitmproxy.proxy import layer from mitmproxy.proxy import server_hooks from mitmproxy.proxy.layers import dns -from mitmproxy.proxy.layers.http import _hooks as http from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers import quic from mitmproxy.proxy.layers import tcp from mitmproxy.proxy.layers import tls from mitmproxy.proxy.layers import udp from mitmproxy.proxy.layers import websocket +from mitmproxy.proxy.layers.http import _hooks as http known = set() def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: all_params = [ - list(inspect.signature(hook.__init__, eval_str=True).parameters.values())[1:] for hook in hooks + list(inspect.signature(hook.__init__, eval_str=True).parameters.values())[1:] + for hook in hooks ] # slightly overengineered, but this was fun to write. ¯\_(ツ)_/¯ diff --git a/docs/scripts/clirecording/clidirector.py b/docs/scripts/clirecording/clidirector.py index db286b2b2a..e861e95563 100644 --- a/docs/scripts/clirecording/clidirector.py +++ b/docs/scripts/clirecording/clidirector.py @@ -1,11 +1,12 @@ import json -from typing import NamedTuple, Optional - -import libtmux import random import subprocess import threading import time +from typing import NamedTuple +from typing import Optional + +import libtmux class InstructionSpec(NamedTuple): diff --git a/docs/scripts/clirecording/record.py b/docs/scripts/clirecording/record.py index 54ba1be2a7..6e91674e8c 100644 --- a/docs/scripts/clirecording/record.py +++ b/docs/scripts/clirecording/record.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 - -from clidirector import CliDirector import screenplays +from clidirector import CliDirector if __name__ == "__main__": diff --git a/docs/scripts/clirecording/screenplays.py b/docs/scripts/clirecording/screenplays.py index ea871e7a7f..5f916dac1d 100644 --- a/docs/scripts/clirecording/screenplays.py +++ b/docs/scripts/clirecording/screenplays.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - from clidirector import CliDirector diff --git a/docs/scripts/examples.py b/docs/scripts/examples.py index 4dd742d500..953cd1fccf 100755 --- a/docs/scripts/examples.py +++ b/docs/scripts/examples.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import re from pathlib import Path diff --git a/docs/scripts/filters.py b/docs/scripts/filters.py index 32634196a8..c002a4ed24 100755 --- a/docs/scripts/filters.py +++ b/docs/scripts/filters.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - from mitmproxy import flowfilter diff --git a/docs/scripts/options.py b/docs/scripts/options.py index 3747d3fb77..6ee4c34af4 100755 --- a/docs/scripts/options.py +++ b/docs/scripts/options.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 import asyncio -from mitmproxy import options, optmanager -from mitmproxy.tools import dump, console, web +from mitmproxy import options +from mitmproxy import optmanager +from mitmproxy.tools import console +from mitmproxy.tools import dump +from mitmproxy.tools import web masters = { "mitmproxy": console.master.ConsoleMaster, diff --git a/examples/addons/contentview-custom-grpc.py b/examples/addons/contentview-custom-grpc.py index c84da91c89..37ba99dd75 100644 --- a/examples/addons/contentview-custom-grpc.py +++ b/examples/addons/contentview-custom-grpc.py @@ -4,7 +4,9 @@ """ from mitmproxy import contentviews -from mitmproxy.contentviews.grpc import ViewGrpcProtobuf, ViewConfig, ProtoParser +from mitmproxy.contentviews.grpc import ProtoParser +from mitmproxy.contentviews.grpc import ViewConfig +from mitmproxy.contentviews.grpc import ViewGrpcProtobuf config: ViewConfig = ViewConfig() config.parser_rules = [ @@ -68,13 +70,13 @@ tag_prefixes=["1.5.1", "1.5.3", "1.5.4", "1.5.5", "1.5.6"], name="latitude", intended_decoding=ProtoParser.DecodedTypes.double, - ), # noqa: E501 + ), ProtoParser.ParserFieldDefinition( tag=".2", tag_prefixes=["1.5.1", "1.5.3", "1.5.4", "1.5.5", "1.5.6"], name="longitude", intended_decoding=ProtoParser.DecodedTypes.double, - ), # noqa: E501 + ), ProtoParser.ParserFieldDefinition(tag="7", name="app"), ], ), diff --git a/examples/addons/contentview.py b/examples/addons/contentview.py index a485c25a81..b96a81ca5a 100644 --- a/examples/addons/contentview.py +++ b/examples/addons/contentview.py @@ -7,7 +7,8 @@ """ from typing import Optional -from mitmproxy import contentviews, flow +from mitmproxy import contentviews +from mitmproxy import flow from mitmproxy import http diff --git a/examples/addons/filter-flows.py b/examples/addons/filter-flows.py index b69406c503..fda546fde7 100644 --- a/examples/addons/filter-flows.py +++ b/examples/addons/filter-flows.py @@ -2,10 +2,11 @@ Use mitmproxy's filter pattern in scripts. """ from __future__ import annotations + import logging -from mitmproxy import http from mitmproxy import flowfilter +from mitmproxy import http class Filter: diff --git a/examples/addons/http-stream-modify.py b/examples/addons/http-stream-modify.py index a200fe5631..76cb33e513 100644 --- a/examples/addons/http-stream-modify.py +++ b/examples/addons/http-stream-modify.py @@ -7,7 +7,8 @@ - If you want to replace all occurrences of "foobar", make sure to catch the cases where one chunk ends with [...]foo" and the next starts with "bar[...]. """ -from typing import Iterable, Union +from collections.abc import Iterable +from typing import Union def modify(data: bytes) -> Union[bytes, Iterable[bytes]]: diff --git a/examples/addons/http-trailers.py b/examples/addons/http-trailers.py index 26a51f23bd..4a7f56d615 100644 --- a/examples/addons/http-trailers.py +++ b/examples/addons/http-trailers.py @@ -6,7 +6,6 @@ headers by name, so the receiving endpoint can wait and read them after the body. """ - from mitmproxy import http from mitmproxy.http import Headers diff --git a/examples/addons/io-read-saved-flows.py b/examples/addons/io-read-saved-flows.py index f6a177be4c..32da842d34 100644 --- a/examples/addons/io-read-saved-flows.py +++ b/examples/addons/io-read-saved-flows.py @@ -2,11 +2,13 @@ """ Read a mitmproxy dump file. """ -from mitmproxy import io, http -from mitmproxy.exceptions import FlowReadException import pprint import sys +from mitmproxy import http +from mitmproxy import io +from mitmproxy.exceptions import FlowReadException + with open(sys.argv[1], "rb") as logfile: freader = io.FlowReader(logfile) pp = pprint.PrettyPrinter(indent=4) diff --git a/examples/addons/io-write-flow-file.py b/examples/addons/io-write-flow-file.py index ecc0528e7f..a348749de2 100644 --- a/examples/addons/io-write-flow-file.py +++ b/examples/addons/io-write-flow-file.py @@ -11,7 +11,8 @@ import sys from typing import BinaryIO -from mitmproxy import io, http +from mitmproxy import http +from mitmproxy import io class Writer: diff --git a/examples/addons/log-events.py b/examples/addons/log-events.py index f5a1c91b29..900be1b268 100644 --- a/examples/addons/log-events.py +++ b/examples/addons/log-events.py @@ -8,4 +8,7 @@ def load(l): logging.info("This is some informative text.") logging.warning("This is a warning.") logging.error("This is an error.") - logging.log(ALERT, "This is an alert. It has the same urgency as info, but will also pop up in the status bar.") + logging.log( + ALERT, + "This is an alert. It has the same urgency as info, but will also pop up in the status bar.", + ) diff --git a/examples/addons/nonblocking.py b/examples/addons/nonblocking.py index ae59db80a3..fb3d3fee2c 100644 --- a/examples/addons/nonblocking.py +++ b/examples/addons/nonblocking.py @@ -3,7 +3,6 @@ """ import asyncio import logging - import time from mitmproxy.script import concurrent diff --git a/examples/addons/shutdown.py b/examples/addons/shutdown.py index 6a6d5069ad..13629eeff9 100644 --- a/examples/addons/shutdown.py +++ b/examples/addons/shutdown.py @@ -10,7 +10,8 @@ """ import logging -from mitmproxy import ctx, http +from mitmproxy import ctx +from mitmproxy import http def request(flow: http.HTTPFlow) -> None: diff --git a/examples/addons/tcp-simple.py b/examples/addons/tcp-simple.py index 242e971403..ed90ba48f1 100644 --- a/examples/addons/tcp-simple.py +++ b/examples/addons/tcp-simple.py @@ -12,8 +12,8 @@ """ import logging -from mitmproxy.utils import strutils from mitmproxy import tcp +from mitmproxy.utils import strutils def tcp_message(flow: tcp.TCPFlow): diff --git a/examples/addons/websocket-inject-message.py b/examples/addons/websocket-inject-message.py index a0b73d24c2..5916edc1a5 100644 --- a/examples/addons/websocket-inject-message.py +++ b/examples/addons/websocket-inject-message.py @@ -5,7 +5,8 @@ """ import asyncio -from mitmproxy import ctx, http +from mitmproxy import ctx +from mitmproxy import http # Simple example: Inject a message as a response to an event diff --git a/examples/addons/wsgi-flask-app.py b/examples/addons/wsgi-flask-app.py index 4f117f05ab..7feab9d58b 100644 --- a/examples/addons/wsgi-flask-app.py +++ b/examples/addons/wsgi-flask-app.py @@ -6,6 +6,7 @@ a single simplest-possible page. """ from flask import Flask + from mitmproxy.addons import asgiapp app = Flask("proxapp") @@ -24,5 +25,4 @@ def hello_world() -> str: # mitmproxy will connect to said domain and use its certificate but won't send any data. # By using `--set upstream_cert=false` and `--set connection_strategy_lazy` the local certificate is used instead. # asgiapp.WSGIApp(app, "example.com", 443), - ] diff --git a/examples/contrib/all_markers.py b/examples/contrib/all_markers.py index 4e9043f330..153818d03b 100644 --- a/examples/contrib/all_markers.py +++ b/examples/contrib/all_markers.py @@ -1,10 +1,13 @@ -from mitmproxy import ctx, command +from mitmproxy import command +from mitmproxy import ctx from mitmproxy.utils import emoji -@command.command('all.markers') +@command.command("all.markers") def all_markers(): - 'Create a new flow showing all marker values' + "Create a new flow showing all marker values" for marker in emoji.emoji: - ctx.master.commands.call('view.flows.create', 'get', f'https://example.com/{marker}') - ctx.master.commands.call('flow.mark', [ctx.master.view.focus.flow], marker) + ctx.master.commands.call( + "view.flows.create", "get", f"https://example.com/{marker}" + ) + ctx.master.commands.call("flow.mark", [ctx.master.view.focus.flow], marker) diff --git a/examples/contrib/block_dns_over_https.py b/examples/contrib/block_dns_over_https.py index 4fe71c44c7..2933ce6ce4 100644 --- a/examples/contrib/block_dns_over_https.py +++ b/examples/contrib/block_dns_over_https.py @@ -9,79 +9,250 @@ # known DoH providers' hostnames and IP addresses to block default_blocklist: dict = { "hostnames": [ - "dns.adguard.com", "dns-family.adguard.com", "dns.google", "cloudflare-dns.com", - "mozilla.cloudflare-dns.com", "security.cloudflare-dns.com", "family.cloudflare-dns.com", - "dns.quad9.net", "dns9.quad9.net", "dns10.quad9.net", "dns11.quad9.net", "doh.opendns.com", - "doh.familyshield.opendns.com", "doh.cleanbrowsing.org", "doh.xfinity.com", "dohdot.coxlab.net", - "odvr.nic.cz", "doh.dnslify.com", "dns.nextdns.io", "dns.dnsoverhttps.net", "doh.crypto.sx", - "doh.powerdns.org", "doh-fi.blahdns.com", "doh-jp.blahdns.com", "doh-de.blahdns.com", - "doh.ffmuc.net", "dns.dns-over-https.com", "doh.securedns.eu", "dns.rubyfish.cn", - "dns.containerpi.com", "dns.containerpi.com", "dns.containerpi.com", "doh-2.seby.io", - "doh.seby.io", "commons.host", "doh.dnswarden.com", "doh.dnswarden.com", "doh.dnswarden.com", - "dns-nyc.aaflalo.me", "dns.aaflalo.me", "doh.applied-privacy.net", "doh.captnemo.in", - "doh.tiar.app", "doh.tiarap.org", "doh.dns.sb", "rdns.faelix.net", "doh.li", "doh.armadillodns.net", - "jp.tiar.app", "jp.tiarap.org", "doh.42l.fr", "dns.hostux.net", "dns.hostux.net", "dns.aa.net.uk", - "adblock.mydns.network", "ibksturm.synology.me", "jcdns.fun", "ibuki.cgnat.net", "dns.twnic.tw", - "example.doh.blockerdns.com", "dns.digitale-gesellschaft.ch", "doh.libredns.gr", - "doh.centraleu.pi-dns.com", "doh.northeu.pi-dns.com", "doh.westus.pi-dns.com", - "doh.eastus.pi-dns.com", "dns.flatuslifir.is", "private.canadianshield.cira.ca", - "protected.canadianshield.cira.ca", "family.canadianshield.cira.ca", "dns.google.com", - "dns.google.com" + "dns.adguard.com", + "dns-family.adguard.com", + "dns.google", + "cloudflare-dns.com", + "mozilla.cloudflare-dns.com", + "security.cloudflare-dns.com", + "family.cloudflare-dns.com", + "dns.quad9.net", + "dns9.quad9.net", + "dns10.quad9.net", + "dns11.quad9.net", + "doh.opendns.com", + "doh.familyshield.opendns.com", + "doh.cleanbrowsing.org", + "doh.xfinity.com", + "dohdot.coxlab.net", + "odvr.nic.cz", + "doh.dnslify.com", + "dns.nextdns.io", + "dns.dnsoverhttps.net", + "doh.crypto.sx", + "doh.powerdns.org", + "doh-fi.blahdns.com", + "doh-jp.blahdns.com", + "doh-de.blahdns.com", + "doh.ffmuc.net", + "dns.dns-over-https.com", + "doh.securedns.eu", + "dns.rubyfish.cn", + "dns.containerpi.com", + "dns.containerpi.com", + "dns.containerpi.com", + "doh-2.seby.io", + "doh.seby.io", + "commons.host", + "doh.dnswarden.com", + "doh.dnswarden.com", + "doh.dnswarden.com", + "dns-nyc.aaflalo.me", + "dns.aaflalo.me", + "doh.applied-privacy.net", + "doh.captnemo.in", + "doh.tiar.app", + "doh.tiarap.org", + "doh.dns.sb", + "rdns.faelix.net", + "doh.li", + "doh.armadillodns.net", + "jp.tiar.app", + "jp.tiarap.org", + "doh.42l.fr", + "dns.hostux.net", + "dns.hostux.net", + "dns.aa.net.uk", + "adblock.mydns.network", + "ibksturm.synology.me", + "jcdns.fun", + "ibuki.cgnat.net", + "dns.twnic.tw", + "example.doh.blockerdns.com", + "dns.digitale-gesellschaft.ch", + "doh.libredns.gr", + "doh.centraleu.pi-dns.com", + "doh.northeu.pi-dns.com", + "doh.westus.pi-dns.com", + "doh.eastus.pi-dns.com", + "dns.flatuslifir.is", + "private.canadianshield.cira.ca", + "protected.canadianshield.cira.ca", + "family.canadianshield.cira.ca", + "dns.google.com", + "dns.google.com", ], "ips": [ - "104.16.248.249", "104.16.248.249", "104.16.249.249", "104.16.249.249", "104.18.2.55", - "104.18.26.128", "104.18.27.128", "104.18.3.55", "104.18.44.204", "104.18.44.204", - "104.18.45.204", "104.18.45.204", "104.182.57.196", "104.236.178.232", "104.24.122.53", - "104.24.123.53", "104.28.0.106", "104.28.1.106", "104.31.90.138", "104.31.91.138", - "115.159.131.230", "116.202.176.26", "116.203.115.192", "136.144.215.158", "139.59.48.222", - "139.99.222.72", "146.112.41.2", "146.112.41.3", "146.185.167.43", "149.112.112.10", - "149.112.112.11", "149.112.112.112", "149.112.112.9", "149.112.121.10", "149.112.121.20", - "149.112.121.30", "149.112.122.10", "149.112.122.20", "149.112.122.30", "159.69.198.101", - "168.235.81.167", "172.104.93.80", "172.65.3.223", "174.138.29.175", "174.68.248.77", - "176.103.130.130", "176.103.130.131", "176.103.130.132", "176.103.130.134", "176.56.236.175", - "178.62.214.105", "185.134.196.54", "185.134.197.54", "185.213.26.187", "185.216.27.142", - "185.228.168.10", "185.228.168.168", "185.235.81.1", "185.26.126.37", "185.26.126.37", - "185.43.135.1", "185.95.218.42", "185.95.218.43", "195.30.94.28", "2001:148f:fffe::1", - "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "2001:19f0:7001:5554:5400:2ff:fe57:3077", - "2001:19f0:7001:5554:5400:2ff:fe57:3077", "2001:19f0:7001:5554:5400:2ff:fe57:3077", - "2001:4860:4860::8844", "2001:4860:4860::8888", - "2001:4b98:dc2:43:216:3eff:fe86:1d28", "2001:558:fe21:6b:96:113:151:149", - "2001:608:a01::3", "2001:678:888:69:c45d:2738:c3f2:1878", "2001:8b0::2022", "2001:8b0::2023", - "2001:c50:ffff:1:101:101:101:101", "210.17.9.228", "217.169.20.22", "217.169.20.23", - "2400:6180:0:d0::5f73:4001", "2400:8902::f03c:91ff:feda:c514", "2604:180:f3::42", - "2604:a880:1:20::51:f001", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "2606:4700::6812:1a80", - "2606:4700::6812:1b80", "2606:4700::6812:237", "2606:4700::6812:337", "2606:4700:3033::6812:2ccc", - "2606:4700:3033::6812:2dcc", "2606:4700:3033::6818:7b35", "2606:4700:3034::681c:16a", - "2606:4700:3035::6818:7a35", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681c:6a", - "2606:4700:3036::681f:5b8a", "2606:4700:60:0:a71e:6467:cef8:2a56", "2620:10a:80bb::10", - "2620:10a:80bb::20", "2620:10a:80bb::30" "2620:10a:80bc::10", "2620:10a:80bc::20", - "2620:10a:80bc::30", "2620:119:fc::2", "2620:119:fc::3", "2620:fe::10", "2620:fe::11", - "2620:fe::9", "2620:fe::fe:10", "2620:fe::fe:11", "2620:fe::fe:9", "2620:fe::fe", - "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "2a00:5a60::bad1:ff", "2a00:5a60::bad2:ff", - "2a00:d880:5:bf0::7c93", "2a01:4f8:1c0c:8233::1", "2a01:4f8:1c1c:6b4b::1", "2a01:4f8:c2c:52bf::1", - "2a01:4f9:c010:43ce::1", "2a01:4f9:c01f:4::abcd", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", - "2a01:9e00::54", "2a01:9e00::55", "2a01:9e01::54", "2a01:9e01::55", - "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "2a03:4000:38:53c::2", - "2a03:b0c0:0:1010::e9a:3001", "2a04:bdc7:100:70::abcd", "2a05:fc84::42", "2a05:fc84::43", - "2a07:a8c0::", "2a0d:4d00:81::1", "2a0d:5600:33:3::abcd", "35.198.2.76", "35.231.247.227", - "45.32.55.94", "45.67.219.208", "45.76.113.31", "45.77.180.10", "45.90.28.0", - "46.101.66.244", "46.227.200.54", "46.227.200.55", "46.239.223.80", "8.8.4.4", - "8.8.8.8", "83.77.85.7", "88.198.91.187", "9.9.9.10", "9.9.9.11", "9.9.9.9", - "94.130.106.88", "95.216.181.228", "95.216.212.177", "96.113.151.148", - ] + "104.16.248.249", + "104.16.248.249", + "104.16.249.249", + "104.16.249.249", + "104.18.2.55", + "104.18.26.128", + "104.18.27.128", + "104.18.3.55", + "104.18.44.204", + "104.18.44.204", + "104.18.45.204", + "104.18.45.204", + "104.182.57.196", + "104.236.178.232", + "104.24.122.53", + "104.24.123.53", + "104.28.0.106", + "104.28.1.106", + "104.31.90.138", + "104.31.91.138", + "115.159.131.230", + "116.202.176.26", + "116.203.115.192", + "136.144.215.158", + "139.59.48.222", + "139.99.222.72", + "146.112.41.2", + "146.112.41.3", + "146.185.167.43", + "149.112.112.10", + "149.112.112.11", + "149.112.112.112", + "149.112.112.9", + "149.112.121.10", + "149.112.121.20", + "149.112.121.30", + "149.112.122.10", + "149.112.122.20", + "149.112.122.30", + "159.69.198.101", + "168.235.81.167", + "172.104.93.80", + "172.65.3.223", + "174.138.29.175", + "174.68.248.77", + "176.103.130.130", + "176.103.130.131", + "176.103.130.132", + "176.103.130.134", + "176.56.236.175", + "178.62.214.105", + "185.134.196.54", + "185.134.197.54", + "185.213.26.187", + "185.216.27.142", + "185.228.168.10", + "185.228.168.168", + "185.235.81.1", + "185.26.126.37", + "185.26.126.37", + "185.43.135.1", + "185.95.218.42", + "185.95.218.43", + "195.30.94.28", + "2001:148f:fffe::1", + "2001:19f0:7001:3259:5400:2ff:fe71:bc9", + "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "2001:4860:4860::8844", + "2001:4860:4860::8888", + "2001:4b98:dc2:43:216:3eff:fe86:1d28", + "2001:558:fe21:6b:96:113:151:149", + "2001:608:a01::3", + "2001:678:888:69:c45d:2738:c3f2:1878", + "2001:8b0::2022", + "2001:8b0::2023", + "2001:c50:ffff:1:101:101:101:101", + "210.17.9.228", + "217.169.20.22", + "217.169.20.23", + "2400:6180:0:d0::5f73:4001", + "2400:8902::f03c:91ff:feda:c514", + "2604:180:f3::42", + "2604:a880:1:20::51:f001", + "2606:4700::6810:f8f9", + "2606:4700::6810:f9f9", + "2606:4700::6812:1a80", + "2606:4700::6812:1b80", + "2606:4700::6812:237", + "2606:4700::6812:337", + "2606:4700:3033::6812:2ccc", + "2606:4700:3033::6812:2dcc", + "2606:4700:3033::6818:7b35", + "2606:4700:3034::681c:16a", + "2606:4700:3035::6818:7a35", + "2606:4700:3035::681f:5a8a", + "2606:4700:3036::681c:6a", + "2606:4700:3036::681f:5b8a", + "2606:4700:60:0:a71e:6467:cef8:2a56", + "2620:10a:80bb::10", + "2620:10a:80bb::20", + "2620:10a:80bb::30" "2620:10a:80bc::10", + "2620:10a:80bc::20", + "2620:10a:80bc::30", + "2620:119:fc::2", + "2620:119:fc::3", + "2620:fe::10", + "2620:fe::11", + "2620:fe::9", + "2620:fe::fe:10", + "2620:fe::fe:11", + "2620:fe::fe:9", + "2620:fe::fe", + "2a00:5a60::ad1:ff", + "2a00:5a60::ad2:ff", + "2a00:5a60::bad1:ff", + "2a00:5a60::bad2:ff", + "2a00:d880:5:bf0::7c93", + "2a01:4f8:1c0c:8233::1", + "2a01:4f8:1c1c:6b4b::1", + "2a01:4f8:c2c:52bf::1", + "2a01:4f9:c010:43ce::1", + "2a01:4f9:c01f:4::abcd", + "2a01:7c8:d002:1ef:5054:ff:fe40:3703", + "2a01:9e00::54", + "2a01:9e00::55", + "2a01:9e01::54", + "2a01:9e01::55", + "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", + "2a03:4000:38:53c::2", + "2a03:b0c0:0:1010::e9a:3001", + "2a04:bdc7:100:70::abcd", + "2a05:fc84::42", + "2a05:fc84::43", + "2a07:a8c0::", + "2a0d:4d00:81::1", + "2a0d:5600:33:3::abcd", + "35.198.2.76", + "35.231.247.227", + "45.32.55.94", + "45.67.219.208", + "45.76.113.31", + "45.77.180.10", + "45.90.28.0", + "46.101.66.244", + "46.227.200.54", + "46.227.200.55", + "46.239.223.80", + "8.8.4.4", + "8.8.8.8", + "83.77.85.7", + "88.198.91.187", + "9.9.9.10", + "9.9.9.11", + "9.9.9.9", + "94.130.106.88", + "95.216.181.228", + "95.216.212.177", + "96.113.151.148", + ], } # additional hostnames to block -additional_doh_names: list[str] = [ - 'dns.google.com' -] +additional_doh_names: list[str] = ["dns.google.com"] # additional IPs to block -additional_doh_ips: list[str] = [ - -] +additional_doh_ips: list[str] = [] -doh_hostnames, doh_ips = default_blocklist['hostnames'], default_blocklist['ips'] +doh_hostnames, doh_ips = default_blocklist["hostnames"], default_blocklist["ips"] # convert to sets for faster lookups doh_hostnames = set(doh_hostnames) @@ -95,9 +266,9 @@ def _has_dns_message_content_type(flow): :param flow: mitmproxy flow :return: True if 'Content-Type' header is DNS-looking, False otherwise """ - doh_content_types = ['application/dns-message'] - if 'Content-Type' in flow.request.headers: - if flow.request.headers['Content-Type'] in doh_content_types: + doh_content_types = ["application/dns-message"] + if "Content-Type" in flow.request.headers: + if flow.request.headers["Content-Type"] in doh_content_types: return True return False @@ -109,7 +280,7 @@ def _request_has_dns_query_string(flow): :param flow: mitmproxy flow :return: True is 'dns' is a parameter in the query string, False otherwise """ - return 'dns' in flow.request.query + return "dns" in flow.request.query def _request_is_dns_json(flow): @@ -127,12 +298,12 @@ def _request_is_dns_json(flow): """ # Header 'Accept: application/dns-json' is required in Cloudflare's DoH JSON API # or they return a 400 HTTP response code - if 'Accept' in flow.request.headers: - if flow.request.headers['Accept'] == 'application/dns-json': + if "Accept" in flow.request.headers: + if flow.request.headers["Accept"] == "application/dns-json": return True # Google's DoH JSON API is https://dns.google/resolve - path = flow.request.path.split('?')[0] - if flow.request.host == 'dns.google' and path == '/resolve': + path = flow.request.path.split("?")[0] + if flow.request.host == "dns.google" and path == "/resolve": return True return False @@ -146,9 +317,9 @@ def _request_has_doh_looking_path(flow): :return: True if path looks like it's DoH, otherwise False """ doh_paths = [ - '/dns-query', # used in example in RFC 8484 (see https://tools.ietf.org/html/rfc8484#section-4.1.1) + "/dns-query", # used in example in RFC 8484 (see https://tools.ietf.org/html/rfc8484#section-4.1.1) ] - path = flow.request.path.split('?')[0] + path = flow.request.path.split("?")[0] return path in doh_paths @@ -171,7 +342,7 @@ def _requested_hostname_is_in_doh_blocklist(flow): _request_has_dns_query_string, _request_is_dns_json, _requested_hostname_is_in_doh_blocklist, - _request_has_doh_looking_path + _request_has_doh_looking_path, ] @@ -179,6 +350,9 @@ def request(flow): for check in doh_request_detection_checks: is_doh = check(flow) if is_doh: - logging.warning("[DoH Detection] DNS over HTTPS request detected via method \"%s\"" % check.__name__) + logging.warning( + '[DoH Detection] DNS over HTTPS request detected via method "%s"' + % check.__name__ + ) flow.kill() break diff --git a/examples/contrib/change_upstream_proxy.py b/examples/contrib/change_upstream_proxy.py index 7f6d56bc4a..4a824131c8 100644 --- a/examples/contrib/change_upstream_proxy.py +++ b/examples/contrib/change_upstream_proxy.py @@ -1,4 +1,3 @@ - from mitmproxy import http from mitmproxy.connection import Server from mitmproxy.net.server_spec import ServerSpec diff --git a/examples/contrib/check_ssl_pinning.py b/examples/contrib/check_ssl_pinning.py index 8bc0b24aab..87a96181f8 100644 --- a/examples/contrib/check_ssl_pinning.py +++ b/examples/contrib/check_ssl_pinning.py @@ -1,14 +1,17 @@ +import ipaddress +import time + +import OpenSSL + import mitmproxy from mitmproxy import ctx from mitmproxy.certs import Cert -import ipaddress -import OpenSSL -import time # Certificate for client connection is generated in dummy_cert() in certs.py. Monkeypatching # the function to generate test cases for SSL Pinning. + def monkey_dummy_cert(privkey, cacert, commonname, sans): ss = [] for i in sans: @@ -42,7 +45,7 @@ def monkey_dummy_cert(privkey, cacert, commonname, sans): if ctx.options.certwrongCN: # append an extra char to make certs common name different than original one. # APpending a char in the end of the domain name. - new_cn = commonname + b'm' + new_cn = commonname + b"m" cert.get_subject().CN = new_cn else: @@ -52,7 +55,8 @@ def monkey_dummy_cert(privkey, cacert, commonname, sans): if ss: cert.set_version(2) cert.add_extensions( - [OpenSSL.crypto.X509Extension(b"subjectAltName", False, ss)]) + [OpenSSL.crypto.X509Extension(b"subjectAltName", False, ss)] + ) cert.set_pubkey(cacert.get_pubkey()) cert.sign(privkey, "sha256") return Cert(cert) @@ -61,23 +65,29 @@ def monkey_dummy_cert(privkey, cacert, commonname, sans): class CheckSSLPinning: def load(self, loader): loader.add_option( - "certbeginon", bool, False, + "certbeginon", + bool, + False, """ Sets SSL Certificate's 'Begins On' time in future. - """ + """, ) loader.add_option( - "certexpire", bool, False, + "certexpire", + bool, + False, """ Sets SSL Certificate's 'Expires On' time in the past. - """ + """, ) loader.add_option( - "certwrongCN", bool, False, + "certwrongCN", + bool, + False, """ Sets SSL Certificate's CommonName(CN) different from the domain name. - """ + """, ) def clientconnect(self, layer): diff --git a/examples/contrib/custom_next_layer.py b/examples/contrib/custom_next_layer.py index 917272dcbd..31e0887fc6 100644 --- a/examples/contrib/custom_next_layer.py +++ b/examples/contrib/custom_next_layer.py @@ -11,7 +11,8 @@ import logging from mitmproxy import ctx -from mitmproxy.proxy import layer, layers +from mitmproxy.proxy import layer +from mitmproxy.proxy import layers def running(): diff --git a/examples/contrib/domain_fronting.py b/examples/contrib/domain_fronting.py index fd73d29856..0a477d0b51 100644 --- a/examples/contrib/domain_fronting.py +++ b/examples/contrib/domain_fronting.py @@ -1,6 +1,8 @@ -from typing import Optional, Union import json from dataclasses import dataclass +from typing import Optional +from typing import Union + from mitmproxy import ctx from mitmproxy.addonmanager import Loader from mitmproxy.http import HTTPFlow @@ -79,7 +81,7 @@ def _resolve_addresses(self, host: str) -> Optional[Mapping]: index = host.find(".", index) if index == -1: break - super_domain = host[(index + 1):] + super_domain = host[(index + 1) :] mapping = self.star_mappings.get(super_domain) if mapping is not None: return mapping diff --git a/examples/contrib/har_dump.py b/examples/contrib/har_dump.py index e1337af467..0a7d6faaee 100644 --- a/examples/contrib/har_dump.py +++ b/examples/contrib/har_dump.py @@ -7,16 +7,14 @@ filename endwith '.zhar' will be compressed: mitmdump -s ./har_dump.py --set hardump=./dump.zhar """ - import base64 import json import logging import os +import zlib from datetime import datetime from datetime import timezone -import zlib - import mitmproxy from mitmproxy import connection from mitmproxy import ctx @@ -33,27 +31,28 @@ def load(l): l.add_option( - "hardump", str, "", "HAR dump path.", + "hardump", + str, + "", + "HAR dump path.", ) def configure(updated): - HAR.update({ - "log": { - "version": "1.2", - "creator": { - "name": "mitmproxy har_dump", - "version": "0.1", - "comment": "mitmproxy version %s" % version.MITMPROXY - }, - "pages": [ - { - "pageTimings": {} - } - ], - "entries": [] + HAR.update( + { + "log": { + "version": "1.2", + "creator": { + "name": "mitmproxy har_dump", + "version": "0.1", + "comment": "mitmproxy version %s" % version.MITMPROXY, + }, + "pages": [{"pageTimings": {}}], + "entries": [], + } } - }) + ) # The `pages` attribute is needed for Firefox Dev Tools to load the HAR file. # An empty value works fine. @@ -65,12 +64,15 @@ def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: connect_time = -1 if flow.server_conn and flow.server_conn not in SERVERS_SEEN: - connect_time = (flow.server_conn.timestamp_tcp_setup - - flow.server_conn.timestamp_start) + connect_time = ( + flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start + ) if flow.server_conn.timestamp_tls_setup is not None: - ssl_time = (flow.server_conn.timestamp_tls_setup - - flow.server_conn.timestamp_tcp_setup) + ssl_time = ( + flow.server_conn.timestamp_tls_setup + - flow.server_conn.timestamp_tcp_setup + ) SERVERS_SEEN.add(flow.server_conn) @@ -81,28 +83,31 @@ def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: # spent waiting between request.timestamp_end and response.timestamp_start # thus it correlates to HAR wait instead. timings_raw = { - 'send': flow.request.timestamp_end - flow.request.timestamp_start, - 'receive': flow.response.timestamp_end - flow.response.timestamp_start, - 'wait': flow.response.timestamp_start - flow.request.timestamp_end, - 'connect': connect_time, - 'ssl': ssl_time, + "send": flow.request.timestamp_end - flow.request.timestamp_start, + "receive": flow.response.timestamp_end - flow.response.timestamp_start, + "wait": flow.response.timestamp_start - flow.request.timestamp_end, + "connect": connect_time, + "ssl": ssl_time, } # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = { - k: int(1000 * v) if v != -1 else -1 - for k, v in timings_raw.items() - } + timings = {k: int(1000 * v) if v != -1 else -1 for k, v in timings_raw.items()} # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. full_time = sum(v for v in timings.values() if v > -1) - started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, timezone.utc).isoformat() + started_date_time = datetime.fromtimestamp( + flow.request.timestamp_start, timezone.utc + ).isoformat() # Response body size and encoding - response_body_size = len(flow.response.raw_content) if flow.response.raw_content else 0 - response_body_decoded_size = len(flow.response.content) if flow.response.content else 0 + response_body_size = ( + len(flow.response.raw_content) if flow.response.raw_content else 0 + ) + response_body_decoded_size = ( + len(flow.response.content) if flow.response.content else 0 + ) response_body_compression = response_body_decoded_size - response_body_size entry = { @@ -127,9 +132,9 @@ def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: "content": { "size": response_body_size, "compression": response_body_compression, - "mimeType": flow.response.headers.get('Content-Type', '') + "mimeType": flow.response.headers.get("Content-Type", ""), }, - "redirectURL": flow.response.headers.get('Location', ''), + "redirectURL": flow.response.headers.get("Location", ""), "headersSize": len(str(flow.response.headers)), "bodySize": response_body_size, }, @@ -139,7 +144,9 @@ def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: # Store binary data as base64 if strutils.is_mostly_bin(flow.response.content): - entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode() + entry["response"]["content"]["text"] = base64.b64encode( + flow.response.content + ).decode() entry["response"]["content"]["encoding"] = "base64" else: entry["response"]["content"]["text"] = flow.response.get_text(strict=False) @@ -152,7 +159,7 @@ def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: entry["request"]["postData"] = { "mimeType": flow.request.headers.get("Content-Type", ""), "text": flow.request.get_text(strict=False), - "params": params + "params": params, } if flow.server_conn.connected: @@ -165,7 +172,7 @@ def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: def response(flow: mitmproxy.http.HTTPFlow): """ - Called when a server response has been received. + Called when a server response has been received. """ if flow.websocket is None: flow_entry(flow) @@ -182,29 +189,29 @@ def websocket_end(flow: mitmproxy.http.HTTPFlow): else: data = base64.b64encode(message.content).decode() websocket_message = { - 'type': 'send' if message.from_client else 'receive', - 'time': message.timestamp, - 'opcode': message.type.value, - 'data': data + "type": "send" if message.from_client else "receive", + "time": message.timestamp, + "opcode": message.type.value, + "data": data, } websocket_messages.append(websocket_message) - entry['_resourceType'] = 'websocket' - entry['_webSocketMessages'] = websocket_messages + entry["_resourceType"] = "websocket" + entry["_webSocketMessages"] = websocket_messages def done(): """ - Called once on script shutdown, after any other events. + Called once on script shutdown, after any other events. """ if ctx.options.hardump: json_dump: str = json.dumps(HAR, indent=2) - if ctx.options.hardump == '-': + if ctx.options.hardump == "-": print(json_dump) else: raw: bytes = json_dump.encode() - if ctx.options.hardump.endswith('.zhar'): + if ctx.options.hardump.endswith(".zhar"): raw = zlib.compress(raw, 9) with open(os.path.expanduser(ctx.options.hardump), "wb") as f: @@ -234,7 +241,9 @@ def format_cookies(cookie_list): # Expiration time needs to be formatted expire_ts = cookies.get_expiration_ts(attrs) if expire_ts is not None: - cookie_har["expires"] = datetime.fromtimestamp(expire_ts, timezone.utc).isoformat() + cookie_har["expires"] = datetime.fromtimestamp( + expire_ts, timezone.utc + ).isoformat() rv.append(cookie_har) @@ -251,6 +260,6 @@ def format_response_cookies(fields): def name_value(obj): """ - Convert (key, value) pairs to HAR format. + Convert (key, value) pairs to HAR format. """ return [{"name": k, "value": v} for k, v in obj.items()] diff --git a/examples/contrib/http_manipulate_cookies.py b/examples/contrib/http_manipulate_cookies.py index b91018c6e1..aaad41227c 100644 --- a/examples/contrib/http_manipulate_cookies.py +++ b/examples/contrib/http_manipulate_cookies.py @@ -15,9 +15,10 @@ """ import json -from mitmproxy import http from typing import Union +from mitmproxy import http + PATH_TO_COOKIES = "./cookies.json" # insert your path to the cookie file here FILTER_COOKIES = { @@ -43,7 +44,14 @@ def stringify_cookies(cookies: list[dict[str, Union[str, None]]]) -> str: """ Creates a cookie string from a list of cookie dicts. """ - return "; ".join([f"{c['name']}={c['value']}" if c.get("value", None) is not None else f"{c['name']}" for c in cookies]) + return "; ".join( + [ + f"{c['name']}={c['value']}" + if c.get("value", None) is not None + else f"{c['name']}" + for c in cookies + ] + ) def parse_cookies(cookie_string: str) -> list[dict[str, Union[str, None]]]: @@ -52,7 +60,9 @@ def parse_cookies(cookie_string: str) -> list[dict[str, Union[str, None]]]: """ return [ {"name": g[0], "value": g[1]} if len(g) == 2 else {"name": g[0], "value": None} - for g in [k.split("=", 1) for k in [c.strip() for c in cookie_string.split(";")] if k] + for g in [ + k.split("=", 1) for k in [c.strip() for c in cookie_string.split(";")] if k + ] ] diff --git a/examples/contrib/httpdump.py b/examples/contrib/httpdump.py index e8c1665578..532ed1e5d5 100644 --- a/examples/contrib/httpdump.py +++ b/examples/contrib/httpdump.py @@ -14,8 +14,9 @@ import os from pathlib import Path -from mitmproxy import ctx, http +from mitmproxy import ctx from mitmproxy import flowfilter +from mitmproxy import http class HTTPDump: @@ -32,7 +33,7 @@ def load(self, loader): name="open_browser", typespec=bool, default=True, - help="open integrated browser at start" + help="open integrated browser at start", ) def running(self): diff --git a/examples/contrib/jsondump.py b/examples/contrib/jsondump.py index 7617902f32..cfde9b75c1 100644 --- a/examples/contrib/jsondump.py +++ b/examples/contrib/jsondump.py @@ -34,7 +34,8 @@ import json import logging from queue import Queue -from threading import Lock, Thread +from threading import Lock +from threading import Thread import requests @@ -66,76 +67,77 @@ def done(self): self.outfile.close() fields = { - 'timestamp': ( - ('error', 'timestamp'), - - ('request', 'timestamp_start'), - ('request', 'timestamp_end'), - - ('response', 'timestamp_start'), - ('response', 'timestamp_end'), - - ('client_conn', 'timestamp_start'), - ('client_conn', 'timestamp_end'), - ('client_conn', 'timestamp_tls_setup'), - - ('server_conn', 'timestamp_start'), - ('server_conn', 'timestamp_end'), - ('server_conn', 'timestamp_tls_setup'), - ('server_conn', 'timestamp_tcp_setup'), + "timestamp": ( + ("error", "timestamp"), + ("request", "timestamp_start"), + ("request", "timestamp_end"), + ("response", "timestamp_start"), + ("response", "timestamp_end"), + ("client_conn", "timestamp_start"), + ("client_conn", "timestamp_end"), + ("client_conn", "timestamp_tls_setup"), + ("server_conn", "timestamp_start"), + ("server_conn", "timestamp_end"), + ("server_conn", "timestamp_tls_setup"), + ("server_conn", "timestamp_tcp_setup"), ), - 'ip': ( - ('server_conn', 'source_address'), - ('server_conn', 'ip_address'), - ('server_conn', 'address'), - ('client_conn', 'address'), + "ip": ( + ("server_conn", "source_address"), + ("server_conn", "ip_address"), + ("server_conn", "address"), + ("client_conn", "address"), ), - 'ws_messages': ( - ('messages',), + "ws_messages": (("messages",),), + "headers": ( + ("request", "headers"), + ("response", "headers"), ), - 'headers': ( - ('request', 'headers'), - ('response', 'headers'), - ), - 'content': ( - ('request', 'content'), - ('response', 'content'), + "content": ( + ("request", "content"), + ("response", "content"), ), } def _init_transformations(self): self.transformations = [ { - 'fields': self.fields['headers'], - 'func': dict, + "fields": self.fields["headers"], + "func": dict, }, { - 'fields': self.fields['timestamp'], - 'func': lambda t: int(t * 1000), + "fields": self.fields["timestamp"], + "func": lambda t: int(t * 1000), }, { - 'fields': self.fields['ip'], - 'func': lambda addr: { - 'host': addr[0].replace('::ffff:', ''), - 'port': addr[1], + "fields": self.fields["ip"], + "func": lambda addr: { + "host": addr[0].replace("::ffff:", ""), + "port": addr[1], }, }, { - 'fields': self.fields['ws_messages'], - 'func': lambda ms: [{ - 'type': m[0], - 'from_client': m[1], - 'content': base64.b64encode(bytes(m[2], 'utf-8')) if self.encode else m[2], - 'timestamp': int(m[3] * 1000), - } for m in ms], - } + "fields": self.fields["ws_messages"], + "func": lambda ms: [ + { + "type": m[0], + "from_client": m[1], + "content": base64.b64encode(bytes(m[2], "utf-8")) + if self.encode + else m[2], + "timestamp": int(m[3] * 1000), + } + for m in ms + ], + }, ] if self.encode: - self.transformations.append({ - 'fields': self.fields['content'], - 'func': base64.b64encode, - }) + self.transformations.append( + { + "fields": self.fields["content"], + "func": base64.b64encode, + } + ) @staticmethod def transform_field(obj, path, func): @@ -156,8 +158,10 @@ def convert_to_strings(cls, obj): Recursively convert all list/dict elements of type `bytes` into strings. """ if isinstance(obj, dict): - return {cls.convert_to_strings(key): cls.convert_to_strings(value) - for key, value in obj.items()} + return { + cls.convert_to_strings(key): cls.convert_to_strings(value) + for key, value in obj.items() + } elif isinstance(obj, list) or isinstance(obj, tuple): return [cls.convert_to_strings(element) for element in obj] elif isinstance(obj, bytes): @@ -175,8 +179,8 @@ def dump(self, frame): Transform and dump (write / send) a data frame. """ for tfm in self.transformations: - for field in tfm['fields']: - self.transform_field(frame, field, tfm['func']) + for field in tfm["fields"]: + self.transform_field(frame, field, tfm["func"]) frame = self.convert_to_strings(frame) if self.outfile: @@ -191,14 +195,21 @@ def load(loader): """ Extra options to be specified in `~/.mitmproxy/config.yaml`. """ - loader.add_option('dump_encodecontent', bool, False, - 'Encode content as base64.') - loader.add_option('dump_destination', str, 'jsondump.out', - 'Output destination: path to a file or URL.') - loader.add_option('dump_username', str, '', - 'Basic auth username for URL destinations.') - loader.add_option('dump_password', str, '', - 'Basic auth password for URL destinations.') + loader.add_option( + "dump_encodecontent", bool, False, "Encode content as base64." + ) + loader.add_option( + "dump_destination", + str, + "jsondump.out", + "Output destination: path to a file or URL.", + ) + loader.add_option( + "dump_username", str, "", "Basic auth username for URL destinations." + ) + loader.add_option( + "dump_password", str, "", "Basic auth password for URL destinations." + ) def configure(self, _): """ @@ -207,18 +218,18 @@ def configure(self, _): """ self.encode = ctx.options.dump_encodecontent - if ctx.options.dump_destination.startswith('http'): + if ctx.options.dump_destination.startswith("http"): self.outfile = None self.url = ctx.options.dump_destination - logging.info('Sending all data frames to %s' % self.url) + logging.info("Sending all data frames to %s" % self.url) if ctx.options.dump_username and ctx.options.dump_password: self.auth = (ctx.options.dump_username, ctx.options.dump_password) - logging.info('HTTP Basic auth enabled.') + logging.info("HTTP Basic auth enabled.") else: - self.outfile = open(ctx.options.dump_destination, 'a') + self.outfile = open(ctx.options.dump_destination, "a") self.url = None self.lock = Lock() - logging.info('Writing all data frames to %s' % ctx.options.dump_destination) + logging.info("Writing all data frames to %s" % ctx.options.dump_destination) self._init_transformations() diff --git a/examples/contrib/link_expander.py b/examples/contrib/link_expander.py index 0edf7c9866..7e7e6b5d82 100644 --- a/examples/contrib/link_expander.py +++ b/examples/contrib/link_expander.py @@ -2,27 +2,33 @@ # relative links () and expands them to absolute links # In practice this can be used to front an indexing spider that may not have the capability to expand relative page links. # Usage: mitmdump -s link_expander.py or mitmproxy -s link_expander.py - import re from urllib.parse import urljoin def response(flow): - if "Content-Type" in flow.response.headers and flow.response.headers["Content-Type"].find("text/html") != -1: + if ( + "Content-Type" in flow.response.headers + and flow.response.headers["Content-Type"].find("text/html") != -1 + ): pageUrl = flow.request.url pageText = flow.response.text - pattern = (r"]*?\s+)?href=(?P[\"'])" - r"(?P(?!https?:\/\/|ftps?:\/\/|\/\/|#|javascript:|mailto:).*?)(?P=delimiter)") + pattern = ( + r"]*?\s+)?href=(?P[\"'])" + r"(?P(?!https?:\/\/|ftps?:\/\/|\/\/|#|javascript:|mailto:).*?)(?P=delimiter)" + ) rel_matcher = re.compile(pattern, flags=re.IGNORECASE) rel_matches = rel_matcher.finditer(pageText) map_dict = {} for match_num, match in enumerate(rel_matches): (delimiter, rel_link) = match.group("delimiter", "link") abs_link = urljoin(pageUrl, rel_link) - map_dict["{0}{1}{0}".format(delimiter, rel_link)] = "{0}{1}{0}".format(delimiter, abs_link) + map_dict["{0}{1}{0}".format(delimiter, rel_link)] = "{0}{1}{0}".format( + delimiter, abs_link + ) for map in map_dict.items(): pageText = pageText.replace(*map) # Uncomment the following to print the expansion mapping # print("{0} -> {1}".format(*map)) - flow.response.text = pageText \ No newline at end of file + flow.response.text = pageText diff --git a/examples/contrib/mitmproxywrapper.py b/examples/contrib/mitmproxywrapper.py index 075cc3c04e..a484ff386d 100644 --- a/examples/contrib/mitmproxywrapper.py +++ b/examples/contrib/mitmproxywrapper.py @@ -6,12 +6,11 @@ # # mitmproxywrapper.py -h # - -import subprocess -import re import argparse import contextlib import os +import re +import subprocess import sys @@ -21,59 +20,50 @@ def __init__(self, port, extra_arguments=None): self.extra_arguments = extra_arguments def run_networksetup_command(self, *arguments): - return subprocess.check_output( - ['sudo', 'networksetup'] + list(arguments)) + return subprocess.check_output(["sudo", "networksetup"] + list(arguments)) def proxy_state_for_service(self, service): - state = self.run_networksetup_command( - '-getwebproxy', - service).splitlines() - return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state]) + state = self.run_networksetup_command("-getwebproxy", service).splitlines() + return dict([re.findall(r"([^:]+): (.*)", line)[0] for line in state]) def enable_proxy_for_service(self, service): - print(f'Enabling proxy on {service}...') - for subcommand in ['-setwebproxy', '-setsecurewebproxy']: + print(f"Enabling proxy on {service}...") + for subcommand in ["-setwebproxy", "-setsecurewebproxy"]: self.run_networksetup_command( - subcommand, service, '127.0.0.1', str( - self.port)) + subcommand, service, "127.0.0.1", str(self.port) + ) def disable_proxy_for_service(self, service): - print(f'Disabling proxy on {service}...') - for subcommand in ['-setwebproxystate', '-setsecurewebproxystate']: - self.run_networksetup_command(subcommand, service, 'Off') + print(f"Disabling proxy on {service}...") + for subcommand in ["-setwebproxystate", "-setsecurewebproxystate"]: + self.run_networksetup_command(subcommand, service, "Off") def interface_name_to_service_name_map(self): - order = self.run_networksetup_command('-listnetworkserviceorder') + order = self.run_networksetup_command("-listnetworkserviceorder") mapping = re.findall( - r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', - order, - re.MULTILINE) + r"\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$", order, re.MULTILINE + ) return {b: a for (a, b) in mapping} def run_command_with_input(self, command, input): - popen = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) + popen = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE) (stdout, stderr) = popen.communicate(input) return stdout def primary_interace_name(self): - scutil_script = 'get State:/Network/Global/IPv4\nd.show\n' - stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script) - interface, = re.findall(r'PrimaryInterface\s*:\s*(.+)', stdout) + scutil_script = "get State:/Network/Global/IPv4\nd.show\n" + stdout = self.run_command_with_input("/usr/sbin/scutil", scutil_script) + (interface,) = re.findall(r"PrimaryInterface\s*:\s*(.+)", stdout) return interface def primary_service_name(self): - return self.interface_name_to_service_name_map()[ - self.primary_interace_name()] + return self.interface_name_to_service_name_map()[self.primary_interace_name()] def proxy_enabled_for_service(self, service): - return self.proxy_state_for_service(service)['Enabled'] == 'Yes' + return self.proxy_state_for_service(service)["Enabled"] == "Yes" def toggle_proxy(self): - new_state = not self.proxy_enabled_for_service( - self.primary_service_name()) + new_state = not self.proxy_enabled_for_service(self.primary_service_name()) for service_name in self.connected_service_names(): if self.proxy_enabled_for_service(service_name) and not new_state: self.disable_proxy_for_service(service_name) @@ -81,31 +71,29 @@ def toggle_proxy(self): self.enable_proxy_for_service(service_name) def connected_service_names(self): - scutil_script = 'list\n' - stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script) - service_ids = re.findall(r'State:/Network/Service/(.+)/IPv4', stdout) + scutil_script = "list\n" + stdout = self.run_command_with_input("/usr/sbin/scutil", scutil_script) + service_ids = re.findall(r"State:/Network/Service/(.+)/IPv4", stdout) service_names = [] for service_id in service_ids: scutil_script = f"show Setup:/Network/Service/{service_id}\n" - stdout = self.run_command_with_input( - '/usr/sbin/scutil', - scutil_script) - service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout) + stdout = self.run_command_with_input("/usr/sbin/scutil", scutil_script) + (service_name,) = re.findall(r"UserDefinedName\s*:\s*(.+)", stdout) service_names.append(service_name) return service_names def wrap_mitmproxy(self): with self.wrap_proxy(): - cmd = ['mitmproxy', '-p', str(self.port)] + cmd = ["mitmproxy", "-p", str(self.port)] if self.extra_arguments: cmd.extend(self.extra_arguments) subprocess.check_call(cmd) def wrap_honeyproxy(self): with self.wrap_proxy(): - popen = subprocess.Popen('honeyproxy.sh') + popen = subprocess.Popen("honeyproxy.sh") try: popen.wait() except KeyboardInterrupt: @@ -127,26 +115,29 @@ def wrap_proxy(self): @classmethod def ensure_superuser(cls): if os.getuid() != 0: - print('Relaunching with sudo...') - os.execv('/usr/bin/sudo', ['/usr/bin/sudo'] + sys.argv) + print("Relaunching with sudo...") + os.execv("/usr/bin/sudo", ["/usr/bin/sudo"] + sys.argv) @classmethod def main(cls): parser = argparse.ArgumentParser( - description='Helper tool for OS X proxy configuration and mitmproxy.', - epilog='Any additional arguments will be passed on unchanged to mitmproxy.') + description="Helper tool for OS X proxy configuration and mitmproxy.", + epilog="Any additional arguments will be passed on unchanged to mitmproxy.", + ) parser.add_argument( - '-t', - '--toggle', - action='store_true', - help='just toggle the proxy configuration') + "-t", + "--toggle", + action="store_true", + help="just toggle the proxy configuration", + ) # parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy') parser.add_argument( - '-p', - '--port', + "-p", + "--port", type=int, - help='override the default port of 8080', - default=8080) + help="override the default port of 8080", + default=8080, + ) args, extra_arguments = parser.parse_known_args() wrapper = cls(port=args.port, extra_arguments=extra_arguments) @@ -159,6 +150,6 @@ def main(cls): wrapper.wrap_mitmproxy() -if __name__ == '__main__': +if __name__ == "__main__": Wrapper.ensure_superuser() Wrapper.main() diff --git a/examples/contrib/modify_body_inject_iframe.py b/examples/contrib/modify_body_inject_iframe.py index 595bd9f281..1736efd34e 100644 --- a/examples/contrib/modify_body_inject_iframe.py +++ b/examples/contrib/modify_body_inject_iframe.py @@ -1,24 +1,21 @@ # (this script works best with --anticache) from bs4 import BeautifulSoup -from mitmproxy import ctx, http + +from mitmproxy import ctx +from mitmproxy import http class Injector: def load(self, loader): - loader.add_option( - "iframe", str, "", "IFrame to inject" - ) + loader.add_option("iframe", str, "", "IFrame to inject") def response(self, flow: http.HTTPFlow) -> None: if ctx.options.iframe: html = BeautifulSoup(flow.response.content, "html.parser") if html.body: iframe = html.new_tag( - "iframe", - src=ctx.options.iframe, - frameborder=0, - height=0, - width=0) + "iframe", src=ctx.options.iframe, frameborder=0, height=0, width=0 + ) html.body.insert(0, iframe) flow.response.content = str(html).encode("utf8") diff --git a/examples/contrib/ntlm_upstream_proxy.py b/examples/contrib/ntlm_upstream_proxy.py index 656d48b3ad..f11a0b77a8 100644 --- a/examples/contrib/ntlm_upstream_proxy.py +++ b/examples/contrib/ntlm_upstream_proxy.py @@ -1,28 +1,34 @@ import base64 +import binascii import logging import socket -from typing import Any, Optional +from typing import Any +from typing import Optional -import binascii -from ntlm_auth import gss_channel_bindings, ntlm +from ntlm_auth import gss_channel_bindings +from ntlm_auth import ntlm -from mitmproxy import addonmanager, http +from mitmproxy import addonmanager from mitmproxy import ctx +from mitmproxy import http from mitmproxy.net.http import http1 -from mitmproxy.proxy import commands, layer +from mitmproxy.proxy import commands +from mitmproxy.proxy import layer from mitmproxy.proxy.context import Context -from mitmproxy.proxy.layers.http import HttpConnectUpstreamHook, HttpLayer, HttpStream +from mitmproxy.proxy.layers.http import HttpConnectUpstreamHook +from mitmproxy.proxy.layers.http import HttpLayer +from mitmproxy.proxy.layers.http import HttpStream from mitmproxy.proxy.layers.http._upstream_proxy import HttpUpstreamProxy class NTLMUpstreamAuth: """ - This addon handles authentication to systems upstream from us for the - upstream proxy and reverse proxy mode. There are 3 cases: - - Upstream proxy CONNECT requests should have authentication added, and - subsequent already connected requests should not. - - Upstream proxy regular requests - - Reverse proxy regular requests (CONNECT is invalid in this mode) + This addon handles authentication to systems upstream from us for the + upstream proxy and reverse proxy mode. There are 3 cases: + - Upstream proxy CONNECT requests should have authentication added, and + subsequent already connected requests should not. + - Upstream proxy regular requests + - Reverse proxy regular requests (CONNECT is invalid in this mode) """ def load(self, loader: addonmanager.Loader) -> None: @@ -34,7 +40,7 @@ def load(self, loader: addonmanager.Loader) -> None: help=""" Add HTTP NTLM authentication to upstream proxy requests. Format: username:password. - """ + """, ) loader.add_option( name="upstream_ntlm_domain", @@ -42,7 +48,7 @@ def load(self, loader: addonmanager.Loader) -> None: default=None, help=""" Add HTTP NTLM domain for authentication to upstream proxy requests. - """ + """, ) loader.add_option( name="upstream_proxy_address", @@ -50,7 +56,7 @@ def load(self, loader: addonmanager.Loader) -> None: default=None, help=""" upstream poxy address. - """ + """, ) loader.add_option( name="upstream_ntlm_compatibility", @@ -59,7 +65,7 @@ def load(self, loader: addonmanager.Loader) -> None: help=""" Add HTTP NTLM compatibility for authentication to upstream proxy requests. Valid values are 0-5 (Default: 3) - """ + """, ) logging.debug("AddOn: NTLM Upstream Authentication - Loaded") @@ -69,9 +75,13 @@ def extract_flow_from_context(context: Context) -> http.HTTPFlow: for l in context.layers: if isinstance(l, HttpLayer): for _, stream in l.streams.items(): - return stream.flow if isinstance(stream, HttpStream) else None + return ( + stream.flow if isinstance(stream, HttpStream) else None + ) - def build_connect_flow(context: Context, connect_header: tuple) -> http.HTTPFlow: + def build_connect_flow( + context: Context, connect_header: tuple + ) -> http.HTTPFlow: flow = extract_flow_from_context(context) if not flow: logging.error("failed to build connect flow") @@ -85,23 +95,27 @@ def patched_start_handshake(self) -> layer.CommandGenerator[None]: assert self.conn.address self.ntlm_context = CustomNTLMContext(ctx) proxy_authorization = self.ntlm_context.get_ntlm_start_negotiate_message() - self.flow = build_connect_flow(self.context, ("Proxy-Authorization", proxy_authorization)) + self.flow = build_connect_flow( + self.context, ("Proxy-Authorization", proxy_authorization) + ) yield HttpConnectUpstreamHook(self.flow) raw = http1.assemble_request(self.flow.request) yield commands.SendData(self.tunnel_connection, raw) def extract_proxy_authenticate_msg(response_head: list) -> str: for header in response_head: - if b'Proxy-Authenticate' in header: - challenge_message = str(bytes(header).decode('utf-8')) + if b"Proxy-Authenticate" in header: + challenge_message = str(bytes(header).decode("utf-8")) try: - token = challenge_message.split(': ')[1] + token = challenge_message.split(": ")[1] except IndexError: logging.error("Failed to extract challenge_message") raise return token - def patched_receive_handshake_data(self, data) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: + def patched_receive_handshake_data( + self, data + ) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: self.buf += data response_head = self.buf.maybe_extract_lines() if response_head: @@ -119,8 +133,14 @@ def patched_receive_handshake_data(self, data) -> layer.CommandGenerator[tuple[b else: if not challenge_message: return True, None - proxy_authorization = self.ntlm_context.get_ntlm_challenge_response_message(challenge_message) - self.flow = build_connect_flow(self.context, ("Proxy-Authorization", proxy_authorization)) + proxy_authorization = ( + self.ntlm_context.get_ntlm_challenge_response_message( + challenge_message + ) + ) + self.flow = build_connect_flow( + self.context, ("Proxy-Authorization", proxy_authorization) + ) raw = http1.assemble_request(self.flow.request) yield commands.SendData(self.tunnel_connection, raw) return False, None @@ -131,19 +151,19 @@ def patched_receive_handshake_data(self, data) -> layer.CommandGenerator[tuple[b HttpUpstreamProxy.receive_handshake_data = patched_receive_handshake_data def done(self): - logging.info('close ntlm session') + logging.info("close ntlm session") -addons = [ - NTLMUpstreamAuth() -] +addons = [NTLMUpstreamAuth()] class CustomNTLMContext: - def __init__(self, - ctx, - preferred_type: str = 'NTLM', - cbt_data: gss_channel_bindings.GssChannelBindingsStruct = None): + def __init__( + self, + ctx, + preferred_type: str = "NTLM", + cbt_data: gss_channel_bindings.GssChannelBindingsStruct = None, + ): # TODO:// take care the cbt_data auth: str = ctx.options.upstream_ntlm_auth domain: str = str(ctx.options.upstream_ntlm_domain).upper() @@ -158,29 +178,39 @@ def __init__(self, domain=domain, workstation=workstation, ntlm_compatibility=ntlm_compatibility, - cbt_data=cbt_data) + cbt_data=cbt_data, + ) def get_ntlm_start_negotiate_message(self) -> str: negotiate_message = self.ntlm_context.step() negotiate_message_base_64_in_bytes = base64.b64encode(negotiate_message) - negotiate_message_base_64_ascii = negotiate_message_base_64_in_bytes.decode("ascii") - negotiate_message_base_64_final = f'{self.preferred_type} {negotiate_message_base_64_ascii}' + negotiate_message_base_64_ascii = negotiate_message_base_64_in_bytes.decode( + "ascii" + ) + negotiate_message_base_64_final = ( + f"{self.preferred_type} {negotiate_message_base_64_ascii}" + ) logging.debug( - f'{self.preferred_type} Authentication, negotiate message: {negotiate_message_base_64_final}' + f"{self.preferred_type} Authentication, negotiate message: {negotiate_message_base_64_final}" ) return negotiate_message_base_64_final def get_ntlm_challenge_response_message(self, challenge_message: str) -> Any: challenge_message = challenge_message.replace(self.preferred_type + " ", "", 1) try: - challenge_message_ascii_bytes = base64.b64decode(challenge_message, validate=True) + challenge_message_ascii_bytes = base64.b64decode( + challenge_message, validate=True + ) except binascii.Error as err: - logging.debug(f'{self.preferred_type} Authentication fail with error {err.__str__()}') + logging.debug( + f"{self.preferred_type} Authentication fail with error {err.__str__()}" + ) return False authenticate_message = self.ntlm_context.step(challenge_message_ascii_bytes) - negotiate_message_base_64 = '{} {}'.format(self.preferred_type, - base64.b64encode(authenticate_message).decode('ascii')) + negotiate_message_base_64 = "{} {}".format( + self.preferred_type, base64.b64encode(authenticate_message).decode("ascii") + ) logging.debug( - f'{self.preferred_type} Authentication, response to challenge message: {negotiate_message_base_64}' + f"{self.preferred_type} Authentication, response to challenge message: {negotiate_message_base_64}" ) return negotiate_message_base_64 diff --git a/examples/contrib/remote-debug.py b/examples/contrib/remote-debug.py index 767d828cde..323b88d1b8 100644 --- a/examples/contrib/remote-debug.py +++ b/examples/contrib/remote-debug.py @@ -18,4 +18,7 @@ def load(l): import pydevd_pycharm - pydevd_pycharm.settrace("localhost", port=5678, stdoutToServer=True, stderrToServer=True, suspend=False) + + pydevd_pycharm.settrace( + "localhost", port=5678, stdoutToServer=True, stderrToServer=True, suspend=False + ) diff --git a/examples/contrib/save_streamed_data.py b/examples/contrib/save_streamed_data.py index 283a6b52bd..6407705964 100644 --- a/examples/contrib/save_streamed_data.py +++ b/examples/contrib/save_streamed_data.py @@ -58,12 +58,13 @@ def __call__(self, data): return data if not self.fh: - self.path = datetime.fromtimestamp(self.flow.request.timestamp_start).strftime( - ctx.options.save_streamed_data) - self.path = self.path.replace('%+T', str(self.flow.request.timestamp_start)) - self.path = self.path.replace('%+I', str(self.flow.client_conn.id)) - self.path = self.path.replace('%+D', self.direction) - self.path = self.path.replace('%+C', self.flow.client_conn.address[0]) + self.path = datetime.fromtimestamp( + self.flow.request.timestamp_start + ).strftime(ctx.options.save_streamed_data) + self.path = self.path.replace("%+T", str(self.flow.request.timestamp_start)) + self.path = self.path.replace("%+I", str(self.flow.client_conn.id)) + self.path = self.path.replace("%+D", self.direction) + self.path = self.path.replace("%+C", self.flow.client_conn.address[0]) self.path = os.path.expanduser(self.path) parent = Path(self.path).parent @@ -89,25 +90,27 @@ def __call__(self, data): def load(loader): loader.add_option( - "save_streamed_data", Optional[str], None, + "save_streamed_data", + Optional[str], + None, "Format string for saving streamed data to files. If set each streamed request or response is written " "to a file with a name derived from the string. In addition to formating supported by python " "strftime() (using the request start time) the code '%+T' is replaced with the time stamp of the request, " "'%+D' by 'req' or 'rsp' depending on the direction of the data, '%+C' by the client IP addresses and " - "'%+I' by the client connection ID." + "'%+I' by the client connection ID.", ) def requestheaders(flow): if ctx.options.save_streamed_data and flow.request.stream: - flow.request.stream = StreamSaver(flow, 'req') + flow.request.stream = StreamSaver(flow, "req") def responseheaders(flow): if isinstance(flow.request.stream, StreamSaver): flow.request.stream.done() if ctx.options.save_streamed_data and flow.response.stream: - flow.response.stream = StreamSaver(flow, 'rsp') + flow.response.stream = StreamSaver(flow, "rsp") def response(flow): diff --git a/examples/contrib/search.py b/examples/contrib/search.py index e9c935ac61..73d775d2be 100644 --- a/examples/contrib/search.py +++ b/examples/contrib/search.py @@ -3,20 +3,19 @@ from collections.abc import Sequence from json import dumps -from mitmproxy import command, flow +from mitmproxy import command +from mitmproxy import flow -MARKER = ':mag:' -RESULTS_STR = 'Search Results: ' +MARKER = ":mag:" +RESULTS_STR = "Search Results: " class Search: def __init__(self): self.exp = None - @command.command('search') - def _search(self, - flows: Sequence[flow.Flow], - regex: str) -> None: + @command.command("search") + def _search(self, flows: Sequence[flow.Flow], regex: str) -> None: """ Defines a command named "search" that matches the given regular expression against most parts @@ -49,11 +48,11 @@ def _search(self, for _flow in flows: # Erase previous results while preserving other comments: comments = list() - for c in _flow.comment.split('\n'): + for c in _flow.comment.split("\n"): if c.startswith(RESULTS_STR): break comments.append(c) - _flow.comment = '\n'.join(comments) + _flow.comment = "\n".join(comments) if _flow.marked == MARKER: _flow.marked = False @@ -62,7 +61,7 @@ def _search(self, if results: comments.append(RESULTS_STR) comments.append(dumps(results, indent=2)) - _flow.comment = '\n'.join(comments) + _flow.comment = "\n".join(comments) _flow.marked = MARKER def header_results(self, message): @@ -71,22 +70,16 @@ def header_results(self, message): def flow_results(self, _flow): results = dict() - results.update( - {'flow_comment': self.exp.findall(_flow.comment)}) + results.update({"flow_comment": self.exp.findall(_flow.comment)}) if _flow.request is not None: - results.update( - {'request_path': self.exp.findall(_flow.request.path)}) - results.update( - {'request_headers': self.header_results(_flow.request)}) + results.update({"request_path": self.exp.findall(_flow.request.path)}) + results.update({"request_headers": self.header_results(_flow.request)}) if _flow.request.text: - results.update( - {'request_body': self.exp.findall(_flow.request.text)}) + results.update({"request_body": self.exp.findall(_flow.request.text)}) if _flow.response is not None: - results.update( - {'response_headers': self.header_results(_flow.response)}) + results.update({"response_headers": self.header_results(_flow.response)}) if _flow.response.text: - results.update( - {'response_body': self.exp.findall(_flow.response.text)}) + results.update({"response_body": self.exp.findall(_flow.response.text)}) return results diff --git a/examples/contrib/sslstrip.py b/examples/contrib/sslstrip.py index 05aa5f3e5f..6b88c39565 100644 --- a/examples/contrib/sslstrip.py +++ b/examples/contrib/sslstrip.py @@ -12,15 +12,15 @@ def request(flow: http.HTTPFlow) -> None: - flow.request.headers.pop('If-Modified-Since', None) - flow.request.headers.pop('Cache-Control', None) + flow.request.headers.pop("If-Modified-Since", None) + flow.request.headers.pop("Cache-Control", None) # do not force https redirection - flow.request.headers.pop('Upgrade-Insecure-Requests', None) + flow.request.headers.pop("Upgrade-Insecure-Requests", None) # proxy connections to SSL-enabled hosts if flow.request.pretty_host in secure_hosts: - flow.request.scheme = 'https' + flow.request.scheme = "https" flow.request.port = 443 # We need to update the request destination to whatever is specified in the host header: @@ -31,32 +31,36 @@ def request(flow: http.HTTPFlow) -> None: def response(flow: http.HTTPFlow) -> None: assert flow.response - flow.response.headers.pop('Strict-Transport-Security', None) - flow.response.headers.pop('Public-Key-Pins', None) + flow.response.headers.pop("Strict-Transport-Security", None) + flow.response.headers.pop("Public-Key-Pins", None) # strip links in response body - flow.response.content = flow.response.content.replace(b'https://', b'http://') + flow.response.content = flow.response.content.replace(b"https://", b"http://") # strip meta tag upgrade-insecure-requests in response body - csp_meta_tag_pattern = br'' - flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE) + csp_meta_tag_pattern = rb'' + flow.response.content = re.sub( + csp_meta_tag_pattern, b"", flow.response.content, flags=re.IGNORECASE + ) # strip links in 'Location' header - if flow.response.headers.get('Location', '').startswith('https://'): - location = flow.response.headers['Location'] + if flow.response.headers.get("Location", "").startswith("https://"): + location = flow.response.headers["Location"] hostname = urllib.parse.urlparse(location).hostname if hostname: secure_hosts.add(hostname) - flow.response.headers['Location'] = location.replace('https://', 'http://', 1) + flow.response.headers["Location"] = location.replace("https://", "http://", 1) # strip upgrade-insecure-requests in Content-Security-Policy header - csp_header = flow.response.headers.get('Content-Security-Policy', '') - if re.search('upgrade-insecure-requests', csp_header, flags=re.IGNORECASE): - csp = flow.response.headers['Content-Security-Policy'] - new_header = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) - flow.response.headers['Content-Security-Policy'] = new_header + csp_header = flow.response.headers.get("Content-Security-Policy", "") + if re.search("upgrade-insecure-requests", csp_header, flags=re.IGNORECASE): + csp = flow.response.headers["Content-Security-Policy"] + new_header = re.sub( + r"upgrade-insecure-requests[;\s]*", "", csp, flags=re.IGNORECASE + ) + flow.response.headers["Content-Security-Policy"] = new_header # strip secure flag from 'Set-Cookie' headers - cookies = flow.response.headers.get_all('Set-Cookie') - cookies = [re.sub(r';\s*secure\s*', '', s) for s in cookies] - flow.response.headers.set_all('Set-Cookie', cookies) + cookies = flow.response.headers.get_all("Set-Cookie") + cookies = [re.sub(r";\s*secure\s*", "", s) for s in cookies] + flow.response.headers.set_all("Set-Cookie", cookies) diff --git a/examples/contrib/suppress_error_responses.py b/examples/contrib/suppress_error_responses.py index e087a78da8..5cb319ef65 100644 --- a/examples/contrib/suppress_error_responses.py +++ b/examples/contrib/suppress_error_responses.py @@ -10,7 +10,7 @@ def error(self, flow: http.HTTPFlow): """Kills the flow if it has an error different to HTTPSyntaxException. - Sometimes, web scanners generate malformed HTTP syntax on purpose and we do not want to kill these requests. + Sometimes, web scanners generate malformed HTTP syntax on purpose and we do not want to kill these requests. """ if flow.error is not None and not isinstance(flow.error, HttpSyntaxException): flow.kill() diff --git a/examples/contrib/test_har_dump.py b/examples/contrib/test_har_dump.py index 88c27a9b4c..77b8db6c74 100644 --- a/examples/contrib/test_har_dump.py +++ b/examples/contrib/test_har_dump.py @@ -1,13 +1,13 @@ import json +from mitmproxy.net.http import cookies +from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test import tutils -from mitmproxy.test import taddons -from mitmproxy.net.http import cookies class TestHARDump: - def flow(self, resp_content=b'message'): + def flow(self, resp_content=b"message"): times = dict( timestamp_start=746203272, timestamp_end=746203272, @@ -15,8 +15,8 @@ def flow(self, resp_content=b'message'): # Create a dummy flow for testing return tflow.tflow( - req=tutils.treq(method=b'GET', **times), - resp=tutils.tresp(content=resp_content, **times) + req=tutils.treq(method=b"GET", **times), + resp=tutils.tresp(content=resp_content, **times), ) def test_simple(self, tmpdir, tdata): @@ -26,7 +26,7 @@ def test_simple(self, tmpdir, tdata): a = tctx.script(tdata.path("../examples/contrib/har_dump.py")) # check script is read without errors assert tctx.master.logs == [] - assert a.name_value # last function in har_dump.py + assert a.name_value # last function in har_dump.py path = str(tmpdir.join("somefile")) tctx.configure(a, hardump=path) @@ -46,7 +46,9 @@ def test_base64(self, tmpdir, tdata): a.done() with open(path) as inp: har = json.load(inp) - assert har["log"]["entries"][0]["response"]["content"]["encoding"] == "base64" + assert ( + har["log"]["entries"][0]["response"]["content"]["encoding"] == "base64" + ) def test_format_cookies(self, tdata): with taddons.context() as tctx: @@ -55,17 +57,21 @@ def test_format_cookies(self, tdata): CA = cookies.CookieAttrs f = a.format_cookies([("n", "v", CA([("k", "v")]))])[0] - assert f['name'] == "n" - assert f['value'] == "v" - assert not f['httpOnly'] - assert not f['secure'] - - f = a.format_cookies([("n", "v", CA([("httponly", None), ("secure", None)]))])[0] - assert f['httpOnly'] - assert f['secure'] - - f = a.format_cookies([("n", "v", CA([("expires", "Mon, 24-Aug-2037 00:00:00 GMT")]))])[0] - assert f['expires'] + assert f["name"] == "n" + assert f["value"] == "v" + assert not f["httpOnly"] + assert not f["secure"] + + f = a.format_cookies( + [("n", "v", CA([("httponly", None), ("secure", None)]))] + )[0] + assert f["httpOnly"] + assert f["secure"] + + f = a.format_cookies( + [("n", "v", CA([("expires", "Mon, 24-Aug-2037 00:00:00 GMT")]))] + )[0] + assert f["expires"] def test_binary(self, tmpdir, tdata): with taddons.context() as tctx: diff --git a/examples/contrib/test_jsondump.py b/examples/contrib/test_jsondump.py index 106a0ecbf2..abb85c2903 100644 --- a/examples/contrib/test_jsondump.py +++ b/examples/contrib/test_jsondump.py @@ -1,21 +1,21 @@ -import json import base64 +import json +import requests_mock + +from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test import tutils -from mitmproxy.test import taddons - -import requests_mock example_dir = tutils.test_data.push("../examples") class TestJSONDump: def echo_response(self, request, context): - self.request = {'json': request.json(), 'headers': request.headers} - return '' + self.request = {"json": request.json(), "headers": request.headers} + return "" - def flow(self, resp_content=b'message'): + def flow(self, resp_content=b"message"): times = dict( timestamp_start=746203272, timestamp_end=746203272, @@ -23,8 +23,8 @@ def flow(self, resp_content=b'message'): # Create a dummy flow for testing return tflow.tflow( - req=tutils.treq(method=b'GET', **times), - resp=tutils.tresp(content=resp_content, **times) + req=tutils.treq(method=b"GET", **times), + resp=tutils.tresp(content=resp_content, **times), ) def test_simple(self, tmpdir): @@ -36,7 +36,7 @@ def test_simple(self, tmpdir): tctx.invoke(a, "done") with open(path) as inp: entry = json.loads(inp.readline()) - assert entry['response']['content'] == 'message' + assert entry["response"]["content"] == "message" def test_contentencode(self, tmpdir): with taddons.context() as tctx: @@ -45,24 +45,28 @@ def test_contentencode(self, tmpdir): content = b"foo" + b"\xFF" * 10 tctx.configure(a, dump_destination=path, dump_encodecontent=True) - tctx.invoke( - a, "response", self.flow(resp_content=content) - ) + tctx.invoke(a, "response", self.flow(resp_content=content)) tctx.invoke(a, "done") with open(path) as inp: entry = json.loads(inp.readline()) - assert entry['response']['content'] == base64.b64encode(content).decode('utf-8') + assert entry["response"]["content"] == base64.b64encode(content).decode( + "utf-8" + ) def test_http(self, tmpdir): with requests_mock.Mocker() as mock: - mock.post('http://my-server', text=self.echo_response) + mock.post("http://my-server", text=self.echo_response) with taddons.context() as tctx: a = tctx.script(example_dir.path("complex/jsondump.py")) - tctx.configure(a, dump_destination='http://my-server', - dump_username='user', dump_password='pass') + tctx.configure( + a, + dump_destination="http://my-server", + dump_username="user", + dump_password="pass", + ) tctx.invoke(a, "response", self.flow()) tctx.invoke(a, "done") - assert self.request['json']['response']['content'] == 'message' - assert self.request['headers']['Authorization'] == 'Basic dXNlcjpwYXNz' + assert self.request["json"]["response"]["content"] == "message" + assert self.request["headers"]["Authorization"] == "Basic dXNlcjpwYXNz" diff --git a/examples/contrib/test_xss_scanner.py b/examples/contrib/test_xss_scanner.py index f277528256..2f89bab481 100644 --- a/examples/contrib/test_xss_scanner.py +++ b/examples/contrib/test_xss_scanner.py @@ -1,229 +1,331 @@ import pytest import requests + from examples.complex import xss_scanner as xss -from mitmproxy.test import tflow, tutils +from mitmproxy.test import tflow +from mitmproxy.test import tutils -class TestXSSScanner(): +class TestXSSScanner: def test_get_XSS_info(self): # First type of exploit: # Exploitable: - xss_info = xss.get_XSS_data(b"" % - xss.FULL_PAYLOAD, - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData('https://example.com', - "End of URL", - '" % xss.FULL_PAYLOAD, + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + "" % - xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b'"', b"%22"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - '" + % xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b'"', b"%22"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + "" % - xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b'"', b"%22").replace(b"/", b"%2F"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"" + % xss.FULL_PAYLOAD.replace(b"'", b"%27") + .replace(b'"', b"%22") + .replace(b"/", b"%2F"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Second type of exploit: # Exploitable: - xss_info = xss.get_XSS_data(b"" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").replace(b"\"", b"%22"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - "';alert(0);g='", - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E") - .replace(b"\"", b"%22").decode('utf-8')) + xss_info = xss.get_XSS_data( + b"" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b'"', b"%22"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + "';alert(0);g='", + xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b'"', b"%22") + .decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b"\"", b"%22").replace(b"'", b"%22"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b'"', b"%22") + .replace(b"'", b"%22"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Third type of exploit: # Exploitable: - xss_info = xss.get_XSS_data(b"" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").replace(b"'", b"%27"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - '";alert(0);g="', - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E") - .replace(b"'", b"%27").decode('utf-8')) + xss_info = xss.get_XSS_data( + b'' + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b"'", b"%27"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + '";alert(0);g="', + xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b"'", b"%27") + .decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b"'", b"%27").replace(b"\"", b"%22"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b'' + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b"'", b"%27") + .replace(b'"', b"%22"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Fourth type of exploit: Test # Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD, - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - "'>", - xss.FULL_PAYLOAD.decode('utf-8')) + xss_info = xss.get_XSS_data( + b"Test" % xss.FULL_PAYLOAD, + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + "'>", + xss.FULL_PAYLOAD.decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"'", b"%27"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"'", b"%27"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Fifth type of exploit: Test # Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"'", b"%27"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - "\">", - xss.FULL_PAYLOAD.replace(b"'", b"%27").decode('utf-8')) + xss_info = xss.get_XSS_data( + b'Test' + % xss.FULL_PAYLOAD.replace(b"'", b"%27"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + '">', + xss.FULL_PAYLOAD.replace(b"'", b"%27").decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b"\"", b"%22"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b'Test' + % xss.FULL_PAYLOAD.replace(b"'", b"%27").replace(b'"', b"%22"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Sixth type of exploit: Test # Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD, - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - ">", - xss.FULL_PAYLOAD.decode('utf-8')) + xss_info = xss.get_XSS_data( + b"Test" % xss.FULL_PAYLOAD, + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + ">", + xss.FULL_PAYLOAD.decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E") - .replace(b"=", b"%3D"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b"=", b"%3D"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Seventh type of exploit: PAYLOAD # Exploitable: - xss_info = xss.get_XSS_data(b"%s" % - xss.FULL_PAYLOAD, - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - "", - xss.FULL_PAYLOAD.decode('utf-8')) + xss_info = xss.get_XSS_data( + b"%s" % xss.FULL_PAYLOAD, + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + "", + xss.FULL_PAYLOAD.decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable - xss_info = xss.get_XSS_data(b"%s" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").replace(b"/", b"%2F"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"%s" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b"/", b"%2F"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Eighth type of exploit: Test # Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - "Javascript:alert(0)", - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8')) + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + "Javascript:alert(0)", + xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E") - .replace(b"=", b"%3D"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b"=", b"%3D"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Ninth type of exploit: Test # Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - '" onmouseover="alert(0)" t="', - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8')) + xss_info = xss.get_XSS_data( + b'Test' + % xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + '" onmouseover="alert(0)" t="', + xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E") - .replace(b'"', b"%22"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b'Test' + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b'"', b"%22"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Tenth type of exploit: Test # Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - "' onmouseover='alert(0)' t='", - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8')) + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + "' onmouseover='alert(0)' t='", + xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E") - .replace(b"'", b"%22"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b"'", b"%22"), + "https://example.com", + "End of URL", + ) assert xss_info is None # Eleventh type of exploit: Test # Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), - "https://example.com", - "End of URL") - expected_xss_info = xss.XSSData("https://example.com", - "End of URL", - " onmouseover=alert(0) t=", - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E").decode('utf-8')) + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E"), + "https://example.com", + "End of URL", + ) + expected_xss_info = xss.XSSData( + "https://example.com", + "End of URL", + " onmouseover=alert(0) t=", + xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .decode("utf-8"), + ) assert xss_info == expected_xss_info # Non-Exploitable: - xss_info = xss.get_XSS_data(b"Test" % - xss.FULL_PAYLOAD.replace(b"<", b"%3C").replace(b">", b"%3E") - .replace(b"=", b"%3D"), - "https://example.com", - "End of URL") + xss_info = xss.get_XSS_data( + b"Test" + % xss.FULL_PAYLOAD.replace(b"<", b"%3C") + .replace(b">", b"%3E") + .replace(b"=", b"%3D"), + "https://example.com", + "End of URL", + ) assert xss_info is None def test_get_SQLi_data(self): - sqli_data = xss.get_SQLi_data("SQL syntax MySQL", - "", - "https://example.com", - "End of URL") - expected_sqli_data = xss.SQLiData("https://example.com", - "End of URL", - "SQL syntax.*MySQL", - "MySQL") + sqli_data = xss.get_SQLi_data( + "SQL syntax MySQL", + "", + "https://example.com", + "End of URL", + ) + expected_sqli_data = xss.SQLiData( + "https://example.com", "End of URL", "SQL syntax.*MySQL", "MySQL" + ) assert sqli_data == expected_sqli_data - sqli_data = xss.get_SQLi_data("SQL syntax MySQL", - "SQL syntax MySQL", - "https://example.com", - "End of URL") + sqli_data = xss.get_SQLi_data( + "SQL syntax MySQL", + "SQL syntax MySQL", + "https://example.com", + "End of URL", + ) assert sqli_data is None def test_inside_quote(self): @@ -233,9 +335,12 @@ def test_inside_quote(self): assert not xss.inside_quote("'", b"longStringNotInIt", 1, b"short") def test_paths_to_text(self): - text = xss.paths_to_text("""

STRING

+ text = xss.paths_to_text( + """

STRING

- """, "STRING") + """, + "STRING", + ) expected_text = ["/html/head/h1", "/html/script"] assert text == expected_text assert xss.paths_to_text("""""", "STRING") == [] @@ -244,114 +349,156 @@ def mocked_requests_vuln(*args, headers=None, cookies=None): class MockResponse: def __init__(self, html, headers=None, cookies=None): self.text = html + return MockResponse("%s" % xss.FULL_PAYLOAD) def mocked_requests_invuln(*args, headers=None, cookies=None): class MockResponse: def __init__(self, html, headers=None, cookies=None): self.text = html + return MockResponse("") def test_test_end_of_url_injection(self, get_request_vuln): - xss_info = xss.test_end_of_URL_injection("", "https://example.com/index.html", {})[0] - expected_xss_info = xss.XSSData('https://example.com/index.html/1029zxcs\'d"aoso[sb]po(pc)se;sl/bsl\\eq=3847asd', - 'End of URL', - '', - '1029zxcs\\\'d"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd') - sqli_info = xss.test_end_of_URL_injection("", "https://example.com/", {})[1] + xss_info = xss.test_end_of_URL_injection( + "", "https://example.com/index.html", {} + )[0] + expected_xss_info = xss.XSSData( + "https://example.com/index.html/1029zxcs'd\"aoso[sb]po(pc)se;sl/bsl\\eq=3847asd", + "End of URL", + "", + "1029zxcs\\'d\"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd", + ) + sqli_info = xss.test_end_of_URL_injection( + "", "https://example.com/", {} + )[1] assert xss_info == expected_xss_info assert sqli_info is None def test_test_referer_injection(self, get_request_vuln): - xss_info = xss.test_referer_injection("", "https://example.com/", {})[0] - expected_xss_info = xss.XSSData('https://example.com/', - 'Referer', - '', - '1029zxcs\\\'d"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd') - sqli_info = xss.test_referer_injection("", "https://example.com/", {})[1] + xss_info = xss.test_referer_injection( + "", "https://example.com/", {} + )[0] + expected_xss_info = xss.XSSData( + "https://example.com/", + "Referer", + "", + "1029zxcs\\'d\"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd", + ) + sqli_info = xss.test_referer_injection( + "", "https://example.com/", {} + )[1] assert xss_info == expected_xss_info assert sqli_info is None def test_test_user_agent_injection(self, get_request_vuln): - xss_info = xss.test_user_agent_injection("", "https://example.com/", {})[0] - expected_xss_info = xss.XSSData('https://example.com/', - 'User Agent', - '', - '1029zxcs\\\'d"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd') - sqli_info = xss.test_user_agent_injection("", "https://example.com/", {})[1] + xss_info = xss.test_user_agent_injection( + "", "https://example.com/", {} + )[0] + expected_xss_info = xss.XSSData( + "https://example.com/", + "User Agent", + "", + "1029zxcs\\'d\"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd", + ) + sqli_info = xss.test_user_agent_injection( + "", "https://example.com/", {} + )[1] assert xss_info == expected_xss_info assert sqli_info is None def test_test_query_injection(self, get_request_vuln): - xss_info = xss.test_query_injection("", "https://example.com/vuln.php?cmd=ls", {})[0] - expected_xss_info = xss.XSSData('https://example.com/vuln.php?cmd=1029zxcs\'d"aoso[sb]po(pc)se;sl/bsl\\eq=3847asd', - 'Query', - '', - '1029zxcs\\\'d"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd') - sqli_info = xss.test_query_injection("", "https://example.com/vuln.php?cmd=ls", {})[1] + xss_info = xss.test_query_injection( + "", "https://example.com/vuln.php?cmd=ls", {} + )[0] + expected_xss_info = xss.XSSData( + "https://example.com/vuln.php?cmd=1029zxcs'd\"aoso[sb]po(pc)se;sl/bsl\\eq=3847asd", + "Query", + "", + "1029zxcs\\'d\"aoso[sb]po(pc)se;sl/bsl\\\\eq=3847asd", + ) + sqli_info = xss.test_query_injection( + "", "https://example.com/vuln.php?cmd=ls", {} + )[1] assert xss_info == expected_xss_info assert sqli_info is None - @pytest.fixture(scope='function') + @pytest.fixture(scope="function") def get_request_vuln(self, monkeypatch): - monkeypatch.setattr(requests, 'get', self.mocked_requests_vuln) + monkeypatch.setattr(requests, "get", self.mocked_requests_vuln) - @pytest.fixture(scope='function') + @pytest.fixture(scope="function") def get_request_invuln(self, monkeypatch): - monkeypatch.setattr(requests, 'get', self.mocked_requests_invuln) + monkeypatch.setattr(requests, "get", self.mocked_requests_invuln) - @pytest.fixture(scope='function') + @pytest.fixture(scope="function") def mock_gethostbyname(self, monkeypatch): def gethostbyname(domain): claimed_domains = ["google.com"] if domain not in claimed_domains: from socket import gaierror + raise gaierror("[Errno -2] Name or service not known") else: - return '216.58.221.46' + return "216.58.221.46" monkeypatch.setattr("socket.gethostbyname", gethostbyname) def test_find_unclaimed_URLs(self, logger, mock_gethostbyname): - xss.find_unclaimed_URLs("", - "https://example.com") + xss.find_unclaimed_URLs( + '', + "https://example.com", + ) assert logger.args == [] - xss.find_unclaimed_URLs("", - "https://example.com") - assert logger.args[0] == 'XSS found in https://example.com due to unclaimed URL "http://unclaimedDomainName.com".' - xss.find_unclaimed_URLs("", - "https://example.com") - assert logger.args[1] == 'XSS found in https://example.com due to unclaimed URL "http://unclaimedDomainName.com".' - xss.find_unclaimed_URLs("", - "https://example.com") - assert logger.args[2] == 'XSS found in https://example.com due to unclaimed URL "http://unclaimedDomainName.com".' + xss.find_unclaimed_URLs( + '', + "https://example.com", + ) + assert ( + logger.args[0] + == 'XSS found in https://example.com due to unclaimed URL "http://unclaimedDomainName.com".' + ) + xss.find_unclaimed_URLs( + '', + "https://example.com", + ) + assert ( + logger.args[1] + == 'XSS found in https://example.com due to unclaimed URL "http://unclaimedDomainName.com".' + ) + xss.find_unclaimed_URLs( + '', + "https://example.com", + ) + assert ( + logger.args[2] + == 'XSS found in https://example.com due to unclaimed URL "http://unclaimedDomainName.com".' + ) def test_log_XSS_data(self, logger): xss.log_XSS_data(None) assert logger.args == [] # self, url: str, injection_point: str, exploit: str, line: str - xss.log_XSS_data(xss.XSSData('https://example.com', - 'Location', - 'String', - 'Line of HTML')) - assert logger.args[0] == '===== XSS Found ====' - assert logger.args[1] == 'XSS URL: https://example.com' - assert logger.args[2] == 'Injection Point: Location' - assert logger.args[3] == 'Suggested Exploit: String' - assert logger.args[4] == 'Line: Line of HTML' + xss.log_XSS_data( + xss.XSSData("https://example.com", "Location", "String", "Line of HTML") + ) + assert logger.args[0] == "===== XSS Found ====" + assert logger.args[1] == "XSS URL: https://example.com" + assert logger.args[2] == "Injection Point: Location" + assert logger.args[3] == "Suggested Exploit: String" + assert logger.args[4] == "Line: Line of HTML" def test_log_SQLi_data(self, logger): xss.log_SQLi_data(None) assert logger.args == [] - xss.log_SQLi_data(xss.SQLiData('https://example.com', - 'Location', - 'Oracle.*Driver', - 'Oracle')) - assert logger.args[0] == '===== SQLi Found =====' - assert logger.args[1] == 'SQLi URL: https://example.com' - assert logger.args[2] == 'Injection Point: Location' - assert logger.args[3] == 'Regex used: Oracle.*Driver' + xss.log_SQLi_data( + xss.SQLiData("https://example.com", "Location", "Oracle.*Driver", "Oracle") + ) + assert logger.args[0] == "===== SQLi Found =====" + assert logger.args[1] == "SQLi URL: https://example.com" + assert logger.args[2] == "Injection Point: Location" + assert logger.args[3] == "Regex used: Oracle.*Driver" def test_get_cookies(self): mocked_req = tutils.treq() @@ -363,7 +510,7 @@ def test_get_cookies(self): def test_response(self, get_request_invuln, logger): mocked_flow = tflow.tflow( req=tutils.treq(path=b"index.html?q=1"), - resp=tutils.tresp(content=b'') + resp=tutils.tresp(content=b""), ) xss.response(mocked_flow) assert logger.args == [] diff --git a/examples/contrib/tls_passthrough.py b/examples/contrib/tls_passthrough.py index 16d90ddad2..ab50d41915 100644 --- a/examples/contrib/tls_passthrough.py +++ b/examples/contrib/tls_passthrough.py @@ -17,10 +17,13 @@ import collections import logging import random -from abc import ABC, abstractmethod +from abc import ABC +from abc import abstractmethod from enum import Enum -from mitmproxy import connection, ctx, tls +from mitmproxy import connection +from mitmproxy import ctx +from mitmproxy import tls from mitmproxy.utils import human @@ -54,6 +57,7 @@ class ConservativeStrategy(TlsStrategy): Conservative Interception Strategy - only intercept if there haven't been any failed attempts in the history. """ + def should_intercept(self, server_address: connection.Address) -> bool: return InterceptionResult.FAILURE not in self.history[server_address] @@ -62,6 +66,7 @@ class ProbabilisticStrategy(TlsStrategy): """ Fixed probability that we intercept a given connection. """ + def __init__(self, p: float): self.p = p super().__init__() @@ -75,7 +80,9 @@ class MaybeTls: def load(self, l): l.add_option( - "tls_strategy", int, 0, + "tls_strategy", + int, + 0, "TLS passthrough strategy. If set to 0, connections will be passed through after the first unsuccessful " "handshake. If set to 0 < p <= 100, connections with be passed through with probability p.", ) @@ -97,7 +104,9 @@ def tls_clienthello(self, data: tls.ClientHelloData): def tls_established_client(self, data: tls.TlsData): server_address = data.context.server.peername - logging.info(f"TLS handshake successful: {human.format_address(server_address)}") + logging.info( + f"TLS handshake successful: {human.format_address(server_address)}" + ) self.strategy.record_success(server_address) def tls_failed_client(self, data: tls.TlsData): diff --git a/examples/contrib/webscanner_helper/mapping.py b/examples/contrib/webscanner_helper/mapping.py index 333809f537..52509730d2 100644 --- a/examples/contrib/webscanner_helper/mapping.py +++ b/examples/contrib/webscanner_helper/mapping.py @@ -3,8 +3,8 @@ from bs4 import BeautifulSoup -from mitmproxy.http import HTTPFlow from examples.contrib.webscanner_helper.urldict import URLDict +from mitmproxy.http import HTTPFlow NO_CONTENT = object() @@ -14,7 +14,7 @@ class MappingAddonConfig: class MappingAddon: - """ The mapping add-on can be used in combination with web application scanners to reduce their false positives. + """The mapping add-on can be used in combination with web application scanners to reduce their false positives. Many web application scanners produce false positives caused by dynamically changing content of web applications such as the current time or current measurements. When testing for injection vulnerabilities, web application @@ -45,7 +45,7 @@ class MappingAddon: """Whether to store all new content in the configuration file.""" def __init__(self, filename: str, persistent: bool = False) -> None: - """ Initializes the mapping add-on + """Initializes the mapping add-on Args: filename: str that provides the name of the file in which the urls and css selectors to mapped content is @@ -71,12 +71,16 @@ def __init__(self, filename: str, persistent: bool = False) -> None: def load(self, loader): loader.add_option( - self.OPT_MAPPING_FILE, str, "", - "File where replacement configuration is stored." + self.OPT_MAPPING_FILE, + str, + "", + "File where replacement configuration is stored.", ) loader.add_option( - self.OPT_MAP_PERSISTENT, bool, False, - "Whether to store all new content in the configuration file." + self.OPT_MAP_PERSISTENT, + bool, + False, + "Whether to store all new content in the configuration file.", ) def configure(self, updated): @@ -88,23 +92,33 @@ def configure(self, updated): if self.OPT_MAP_PERSISTENT in updated: self.persistent = updated[self.OPT_MAP_PERSISTENT] - def replace(self, soup: BeautifulSoup, css_sel: str, replace: BeautifulSoup) -> None: + def replace( + self, soup: BeautifulSoup, css_sel: str, replace: BeautifulSoup + ) -> None: """Replaces the content of soup that matches the css selector with the given replace content.""" for content in soup.select(css_sel): - self.logger.debug(f"replace \"{content}\" with \"{replace}\"") + self.logger.debug(f'replace "{content}" with "{replace}"') content.replace_with(copy.copy(replace)) - def apply_template(self, soup: BeautifulSoup, template: dict[str, BeautifulSoup]) -> None: + def apply_template( + self, soup: BeautifulSoup, template: dict[str, BeautifulSoup] + ) -> None: """Applies the given mapping template to the given soup.""" for css_sel, replace in template.items(): mapped = soup.select(css_sel) if not mapped: - self.logger.warning(f"Could not find \"{css_sel}\", can not freeze anything.") + self.logger.warning( + f'Could not find "{css_sel}", can not freeze anything.' + ) else: - self.replace(soup, css_sel, BeautifulSoup(replace, features=MappingAddonConfig.HTML_PARSER)) + self.replace( + soup, + css_sel, + BeautifulSoup(replace, features=MappingAddonConfig.HTML_PARSER), + ) def response(self, flow: HTTPFlow) -> None: - """If a response is received, check if we should replace some content. """ + """If a response is received, check if we should replace some content.""" try: templates = self.mapping_templates[flow] res = flow.response @@ -118,7 +132,9 @@ def response(self, flow: HTTPFlow) -> None: self.apply_template(content, template) res.content = content.encode(encoding) else: - self.logger.warning(f"Unsupported content type '{content_type}' or content encoding '{encoding}'") + self.logger.warning( + f"Unsupported content type '{content_type}' or content encoding '{encoding}'" + ) except KeyError: pass diff --git a/examples/contrib/webscanner_helper/proxyauth_selenium.py b/examples/contrib/webscanner_helper/proxyauth_selenium.py index 6ac1d94de6..579fcc3d20 100644 --- a/examples/contrib/webscanner_helper/proxyauth_selenium.py +++ b/examples/contrib/webscanner_helper/proxyauth_selenium.py @@ -3,13 +3,15 @@ import random import string import time -from typing import Any, cast +from typing import Any +from typing import cast + +from selenium import webdriver import mitmproxy.http from mitmproxy import flowfilter from mitmproxy import master from mitmproxy.script import concurrent -from selenium import webdriver logger = logging.getLogger(__name__) @@ -18,14 +20,14 @@ "expires": "Expires", "domain": "Domain", "is_http_only": "HttpOnly", - "is_secure": "Secure" + "is_secure": "Secure", } def randomString(string_length=10): - """Generate a random string of fixed length """ + """Generate a random string of fixed length""" letters = string.ascii_lowercase - return ''.join(random.choice(letters) for i in range(string_length)) + return "".join(random.choice(letters) for i in range(string_length)) class AuthorizationOracle(abc.ABC): @@ -41,7 +43,7 @@ def is_unauthorized_response(self, flow: mitmproxy.http.HTTPFlow) -> bool: class SeleniumAddon: - """ This Addon can be used in combination with web application scanners in order to help them to authenticate + """This Addon can be used in combination with web application scanners in order to help them to authenticate against a web application. Since the authentication is highly dependant on the web application, this add-on includes the abstract method @@ -50,8 +52,7 @@ class SeleniumAddon: application. In addition, an authentication oracle which inherits from AuthorizationOracle should be created. """ - def __init__(self, fltr: str, domain: str, - auth_oracle: AuthorizationOracle): + def __init__(self, fltr: str, domain: str, auth_oracle: AuthorizationOracle): self.filter = flowfilter.parse(fltr) self.auth_oracle = auth_oracle self.domain = domain @@ -62,9 +63,8 @@ def __init__(self, fltr: str, domain: str, options.headless = True profile = webdriver.FirefoxProfile() - profile.set_preference('network.proxy.type', 0) - self.browser = webdriver.Firefox(firefox_profile=profile, - options=options) + profile.set_preference("network.proxy.type", 0) + self.browser = webdriver.Firefox(firefox_profile=profile, options=options) self.cookies: list[dict[str, str]] = [] def _login(self, flow): @@ -76,7 +76,9 @@ def _login(self, flow): def request(self, flow: mitmproxy.http.HTTPFlow): if flow.request.is_replay: logger.warning("Caught replayed request: " + str(flow)) - if (not self.filter or self.filter(flow)) and self.auth_oracle.is_unauthorized_request(flow): + if ( + not self.filter or self.filter(flow) + ) and self.auth_oracle.is_unauthorized_request(flow): logger.debug("unauthorized request detected, perform login") self._login(flow) @@ -88,7 +90,7 @@ def response(self, flow: mitmproxy.http.HTTPFlow): if self.auth_oracle.is_unauthorized_response(flow): self._login(flow) new_flow = flow.copy() - if master and hasattr(master, 'commands'): + if master and hasattr(master, "commands"): # cast necessary for mypy cast(Any, master).commands.call("replay.client", [new_flow]) count = 0 @@ -99,7 +101,9 @@ def response(self, flow: mitmproxy.http.HTTPFlow): if new_flow.response: flow.response = new_flow.response else: - logger.warning("Could not call 'replay.client' command since master was not initialized yet.") + logger.warning( + "Could not call 'replay.client' command since master was not initialized yet." + ) if self.set_cookies and flow.response: logger.debug("set set-cookie header for response") @@ -124,7 +128,8 @@ def _set_set_cookie_headers(self, flow: mitmproxy.http.HTTPFlow): def _set_request_cookies(self, flow: mitmproxy.http.HTTPFlow): if self.cookies: cookies = "; ".join( - map(lambda c: f"{c['name']}={c['value']}", self.cookies)) + map(lambda c: f"{c['name']}={c['value']}", self.cookies) + ) flow.request.headers["cookie"] = cookies @abc.abstractmethod diff --git a/examples/contrib/webscanner_helper/test_mapping.py b/examples/contrib/webscanner_helper/test_mapping.py index 340522837c..c88b11983a 100644 --- a/examples/contrib/webscanner_helper/test_mapping.py +++ b/examples/contrib/webscanner_helper/test_mapping.py @@ -1,15 +1,15 @@ -from typing import TextIO, Callable +from typing import Callable +from typing import TextIO from unittest import mock from unittest.mock import MagicMock +from examples.contrib.webscanner_helper.mapping import MappingAddon +from examples.contrib.webscanner_helper.mapping import MappingAddonConfig from mitmproxy.test import tflow from mitmproxy.test import tutils -from examples.contrib.webscanner_helper.mapping import MappingAddon, MappingAddonConfig - class TestConfig: - def test_config(self): assert MappingAddonConfig.HTML_PARSER == "html.parser" @@ -20,7 +20,6 @@ def test_config(self): class TestMappingAddon: - def test_init(self, tmpdir): tmpfile = tmpdir.join("tmpfile") with open(tmpfile, "w") as tfile: @@ -36,8 +35,8 @@ def test_load(self, tmpdir): loader = MagicMock() mapping.load(loader) - assert 'mapping_file' in str(loader.add_option.call_args_list) - assert 'map_persistent' in str(loader.add_option.call_args_list) + assert "mapping_file" in str(loader.add_option.call_args_list) + assert "map_persistent" in str(loader.add_option.call_args_list) def test_configure(self, tmpdir): tmpfile = tmpdir.join("tmpfile") @@ -45,7 +44,10 @@ def test_configure(self, tmpdir): tfile.write(mapping_content) mapping = MappingAddon(tmpfile) new_filename = "My new filename" - updated = {str(mapping.OPT_MAPPING_FILE): new_filename, str(mapping.OPT_MAP_PERSISTENT): True} + updated = { + str(mapping.OPT_MAPPING_FILE): new_filename, + str(mapping.OPT_MAP_PERSISTENT): True, + } open_mock = mock.mock_open(read_data="{}") with mock.patch("builtins.open", open_mock): @@ -161,5 +163,8 @@ def test_dump(selfself, tmpdir): with open(tmpfile, "w") as tfile: tfile.write("{}") mapping = MappingAddon(tmpfile, persistent=True) - with mock.patch('examples.complex.webscanner_helper.urldict.URLDict.dump', selfself.mock_dump): + with mock.patch( + "examples.complex.webscanner_helper.urldict.URLDict.dump", + selfself.mock_dump, + ): mapping.done() diff --git a/examples/contrib/webscanner_helper/test_proxyauth_selenium.py b/examples/contrib/webscanner_helper/test_proxyauth_selenium.py index 58e035068f..e755c776c6 100644 --- a/examples/contrib/webscanner_helper/test_proxyauth_selenium.py +++ b/examples/contrib/webscanner_helper/test_proxyauth_selenium.py @@ -3,16 +3,16 @@ import pytest +from examples.contrib.webscanner_helper.proxyauth_selenium import AuthorizationOracle +from examples.contrib.webscanner_helper.proxyauth_selenium import logger +from examples.contrib.webscanner_helper.proxyauth_selenium import randomString +from examples.contrib.webscanner_helper.proxyauth_selenium import SeleniumAddon +from mitmproxy.http import HTTPFlow from mitmproxy.test import tflow from mitmproxy.test import tutils -from mitmproxy.http import HTTPFlow - -from examples.contrib.webscanner_helper.proxyauth_selenium import logger, randomString, AuthorizationOracle, \ - SeleniumAddon class TestRandomString: - def test_random_string(self): res = randomString() assert isinstance(res, str) @@ -36,8 +36,11 @@ def is_unauthorized_response(self, flow: HTTPFlow) -> bool: @pytest.fixture(scope="module", autouse=True) def selenium_addon(request): - addon = SeleniumAddon(fltr=r"~u http://example\.com/login\.php", domain=r"~d http://example\.com", - auth_oracle=oracle) + addon = SeleniumAddon( + fltr=r"~u http://example\.com/login\.php", + domain=r"~d http://example\.com", + auth_oracle=oracle, + ) browser = MagicMock() addon.browser = browser yield addon @@ -49,11 +52,10 @@ def fin(): class TestSeleniumAddon: - def test_request_replay(self, selenium_addon): f = tflow.tflow(resp=tutils.tresp()) f.request.is_replay = True - with mock.patch.object(logger, 'warning') as mock_warning: + with mock.patch.object(logger, "warning") as mock_warning: selenium_addon.request(f) mock_warning.assert_called() @@ -62,7 +64,7 @@ def test_request(self, selenium_addon): f.request.url = "http://example.com/login.php" selenium_addon.set_cookies = False assert not selenium_addon.set_cookies - with mock.patch.object(logger, 'debug') as mock_debug: + with mock.patch.object(logger, "debug") as mock_debug: selenium_addon.request(f) mock_debug.assert_called() assert selenium_addon.set_cookies @@ -79,9 +81,11 @@ def test_request_cookies(self, selenium_addon): f.request.url = "http://example.com/login.php" selenium_addon.set_cookies = False assert not selenium_addon.set_cookies - with mock.patch.object(logger, 'debug') as mock_debug: - with mock.patch('examples.complex.webscanner_helper.proxyauth_selenium.SeleniumAddon.login', - return_value=[{"name": "cookie", "value": "test"}]) as mock_login: + with mock.patch.object(logger, "debug") as mock_debug: + with mock.patch( + "examples.complex.webscanner_helper.proxyauth_selenium.SeleniumAddon.login", + return_value=[{"name": "cookie", "value": "test"}], + ) as mock_login: selenium_addon.request(f) mock_debug.assert_called() assert selenium_addon.set_cookies @@ -95,7 +99,7 @@ def test_request_filter_None(self, selenium_addon): selenium_addon.set_cookies = False assert not selenium_addon.set_cookies - with mock.patch.object(logger, 'debug') as mock_debug: + with mock.patch.object(logger, "debug") as mock_debug: selenium_addon.request(f) mock_debug.assert_called() selenium_addon.filter = fltr @@ -105,8 +109,10 @@ def test_response(self, selenium_addon): f = tflow.tflow(resp=tutils.tresp()) f.request.url = "http://example.com/login.php" selenium_addon.set_cookies = False - with mock.patch('examples.complex.webscanner_helper.proxyauth_selenium.SeleniumAddon.login', - return_value=[]) as mock_login: + with mock.patch( + "examples.complex.webscanner_helper.proxyauth_selenium.SeleniumAddon.login", + return_value=[], + ) as mock_login: selenium_addon.response(f) mock_login.assert_called() @@ -114,7 +120,9 @@ def test_response_cookies(self, selenium_addon): f = tflow.tflow(resp=tutils.tresp()) f.request.url = "http://example.com/login.php" selenium_addon.set_cookies = False - with mock.patch('examples.complex.webscanner_helper.proxyauth_selenium.SeleniumAddon.login', - return_value=[{"name": "cookie", "value": "test"}]) as mock_login: + with mock.patch( + "examples.complex.webscanner_helper.proxyauth_selenium.SeleniumAddon.login", + return_value=[{"name": "cookie", "value": "test"}], + ) as mock_login: selenium_addon.response(f) mock_login.assert_called() diff --git a/examples/contrib/webscanner_helper/test_urldict.py b/examples/contrib/webscanner_helper/test_urldict.py index 102c9ee35f..066566237c 100644 --- a/examples/contrib/webscanner_helper/test_urldict.py +++ b/examples/contrib/webscanner_helper/test_urldict.py @@ -1,5 +1,6 @@ -from mitmproxy.test import tflow, tutils from examples.contrib.webscanner_helper.urldict import URLDict +from mitmproxy.test import tflow +from mitmproxy.test import tutils url = "http://10.10.10.10" new_content_body = "New Body" @@ -11,11 +12,10 @@ class TestUrlDict: - def test_urldict_empty(self): urldict = URLDict() dump = urldict.dumps() - assert dump == '{}' + assert dump == "{}" def test_urldict_loads(self): urldict = URLDict.loads(input_file_content) diff --git a/examples/contrib/webscanner_helper/test_urlindex.py b/examples/contrib/webscanner_helper/test_urlindex.py index 058a36068f..d3dd5f4807 100644 --- a/examples/contrib/webscanner_helper/test_urlindex.py +++ b/examples/contrib/webscanner_helper/test_urlindex.py @@ -4,17 +4,18 @@ from unittest import mock from unittest.mock import patch +from examples.contrib.webscanner_helper.urlindex import filter_404 +from examples.contrib.webscanner_helper.urlindex import JSONUrlIndexWriter +from examples.contrib.webscanner_helper.urlindex import SetEncoder +from examples.contrib.webscanner_helper.urlindex import TextUrlIndexWriter +from examples.contrib.webscanner_helper.urlindex import UrlIndexAddon +from examples.contrib.webscanner_helper.urlindex import UrlIndexWriter +from examples.contrib.webscanner_helper.urlindex import WRITER from mitmproxy.test import tflow from mitmproxy.test import tutils -from examples.contrib.webscanner_helper.urlindex import UrlIndexWriter, SetEncoder, JSONUrlIndexWriter, \ - TextUrlIndexWriter, WRITER, \ - filter_404, \ - UrlIndexAddon - class TestBaseClass: - @patch.multiple(UrlIndexWriter, __abstractmethods__=set()) def test_base_class(self, tmpdir): tmpfile = tmpdir.join("tmpfile") @@ -25,14 +26,13 @@ def test_base_class(self, tmpdir): class TestSetEncoder: - def test_set_encoder_set(self): test_set = {"foo", "bar", "42"} result = SetEncoder.default(SetEncoder(), test_set) assert isinstance(result, list) - assert 'foo' in result - assert 'bar' in result - assert '42' in result + assert "foo" in result + assert "bar" in result + assert "42" in result def test_set_encoder_str(self): test_str = "test" @@ -45,18 +45,18 @@ def test_set_encoder_str(self): class TestJSONUrlIndexWriter: - def test_load(self, tmpdir): tmpfile = tmpdir.join("tmpfile") with open(tmpfile, "w") as tfile: tfile.write( - "{\"http://example.com:80\": {\"/\": {\"GET\": [301]}}, \"http://www.example.com:80\": {\"/\": {\"GET\": [302]}}}") + '{"http://example.com:80": {"/": {"GET": [301]}}, "http://www.example.com:80": {"/": {"GET": [302]}}}' + ) writer = JSONUrlIndexWriter(filename=tmpfile) writer.load() - assert 'http://example.com:80' in writer.host_urls - assert '/' in writer.host_urls['http://example.com:80'] - assert 'GET' in writer.host_urls['http://example.com:80']['/'] - assert 301 in writer.host_urls['http://example.com:80']['/']['GET'] + assert "http://example.com:80" in writer.host_urls + assert "/" in writer.host_urls["http://example.com:80"] + assert "GET" in writer.host_urls["http://example.com:80"]["/"] + assert 301 in writer.host_urls["http://example.com:80"]["/"]["GET"] def test_load_empty(self, tmpdir): tmpfile = tmpdir.join("tmpfile") @@ -102,7 +102,8 @@ def test_load(self, tmpdir): tmpfile = tmpdir.join("tmpfile") with open(tmpfile, "w") as tfile: tfile.write( - "2020-04-22T05:41:08.679231 STATUS: 200 METHOD: GET URL:http://example.com") + "2020-04-22T05:41:08.679231 STATUS: 200 METHOD: GET URL:http://example.com" + ) writer = TextUrlIndexWriter(filename=tmpfile) writer.load() assert True @@ -173,7 +174,6 @@ def test_filter_false(self): class TestUrlIndexAddon: - def test_init(self, tmpdir): tmpfile = tmpdir.join("tmpfile") UrlIndexAddon(tmpfile) @@ -202,7 +202,9 @@ def test_init_append(self, tmpdir): tfile.write("") url_index = UrlIndexAddon(tmpfile, append=False) f = tflow.tflow(resp=tutils.tresp()) - with mock.patch('examples.complex.webscanner_helper.urlindex.JSONUrlIndexWriter.add_url'): + with mock.patch( + "examples.complex.webscanner_helper.urlindex.JSONUrlIndexWriter.add_url" + ): url_index.response(f) assert not Path(tmpfile).exists() @@ -210,7 +212,9 @@ def test_response(self, tmpdir): tmpfile = tmpdir.join("tmpfile") url_index = UrlIndexAddon(tmpfile) f = tflow.tflow(resp=tutils.tresp()) - with mock.patch('examples.complex.webscanner_helper.urlindex.JSONUrlIndexWriter.add_url') as mock_add_url: + with mock.patch( + "examples.complex.webscanner_helper.urlindex.JSONUrlIndexWriter.add_url" + ) as mock_add_url: url_index.response(f) mock_add_url.assert_called() @@ -229,6 +233,8 @@ def test_response_None(self, tmpdir): def test_done(self, tmpdir): tmpfile = tmpdir.join("tmpfile") url_index = UrlIndexAddon(tmpfile) - with mock.patch('examples.complex.webscanner_helper.urlindex.JSONUrlIndexWriter.save') as mock_save: + with mock.patch( + "examples.complex.webscanner_helper.urlindex.JSONUrlIndexWriter.save" + ) as mock_save: url_index.done() mock_save.assert_called() diff --git a/examples/contrib/webscanner_helper/test_urlinjection.py b/examples/contrib/webscanner_helper/test_urlinjection.py index b1c412d21a..b6a841721f 100644 --- a/examples/contrib/webscanner_helper/test_urlinjection.py +++ b/examples/contrib/webscanner_helper/test_urlinjection.py @@ -1,20 +1,22 @@ import json from unittest import mock +from examples.contrib.webscanner_helper.urlinjection import HTMLInjection +from examples.contrib.webscanner_helper.urlinjection import InjectionGenerator +from examples.contrib.webscanner_helper.urlinjection import logger +from examples.contrib.webscanner_helper.urlinjection import RobotsInjection +from examples.contrib.webscanner_helper.urlinjection import SitemapInjection +from examples.contrib.webscanner_helper.urlinjection import UrlInjectionAddon from mitmproxy import flowfilter from mitmproxy.test import tflow from mitmproxy.test import tutils -from examples.contrib.webscanner_helper.urlinjection import InjectionGenerator, HTMLInjection, RobotsInjection, \ - SitemapInjection, \ - UrlInjectionAddon, logger - index = json.loads( - "{\"http://example.com:80\": {\"/\": {\"GET\": [301]}}, \"http://www.example.com:80\": {\"/test\": {\"POST\": [302]}}}") + '{"http://example.com:80": {"/": {"GET": [301]}}, "http://www.example.com:80": {"/test": {"POST": [302]}}}' +) class TestInjectionGenerator: - def test_inject(self): f = tflow.tflow(resp=tutils.tresp()) injection_generator = InjectionGenerator() @@ -23,12 +25,11 @@ def test_inject(self): class TestHTMLInjection: - def test_inject_not404(self): html_injection = HTMLInjection() f = tflow.tflow(resp=tutils.tresp()) - with mock.patch.object(logger, 'warning') as mock_warning: + with mock.patch.object(logger, "warning") as mock_warning: html_injection.inject(index, f) assert mock_warning.called @@ -57,12 +58,11 @@ def test_inject_404(self): class TestRobotsInjection: - def test_inject_not404(self): robots_injection = RobotsInjection() f = tflow.tflow(resp=tutils.tresp()) - with mock.patch.object(logger, 'warning') as mock_warning: + with mock.patch.object(logger, "warning") as mock_warning: robots_injection.inject(index, f) assert mock_warning.called @@ -76,12 +76,11 @@ def test_inject_404(self): class TestSitemapInjection: - def test_inject_not404(self): sitemap_injection = SitemapInjection() f = tflow.tflow(resp=tutils.tresp()) - with mock.patch.object(logger, 'warning') as mock_warning: + with mock.patch.object(logger, "warning") as mock_warning: sitemap_injection.inject(index, f) assert mock_warning.called @@ -89,19 +88,22 @@ def test_inject_404(self): sitemap_injection = SitemapInjection() f = tflow.tflow(resp=tutils.tresp()) f.response.status_code = 404 - assert "http://example.com:80/" not in str(f.response.content) + assert "http://example.com:80/" not in str( + f.response.content + ) sitemap_injection.inject(index, f) assert "http://example.com:80/" in str(f.response.content) class TestUrlInjectionAddon: - def test_init(self, tmpdir): tmpfile = tmpdir.join("tmpfile") with open(tmpfile, "w") as tfile: json.dump(index, tfile) flt = f"~u .*/site.html$" - url_injection = UrlInjectionAddon(f"~u .*/site.html$", tmpfile, HTMLInjection(insert=True)) + url_injection = UrlInjectionAddon( + f"~u .*/site.html$", tmpfile, HTMLInjection(insert=True) + ) assert "http://example.com:80" in url_injection.url_store fltr = flowfilter.parse(flt) f = tflow.tflow(resp=tutils.tresp()) diff --git a/examples/contrib/webscanner_helper/test_watchdog.py b/examples/contrib/webscanner_helper/test_watchdog.py index f6a34a61b1..d5382072bd 100644 --- a/examples/contrib/webscanner_helper/test_watchdog.py +++ b/examples/contrib/webscanner_helper/test_watchdog.py @@ -1,18 +1,17 @@ +import multiprocessing import time from pathlib import Path from unittest import mock +from examples.contrib.webscanner_helper.watchdog import logger +from examples.contrib.webscanner_helper.watchdog import WatchdogAddon from mitmproxy.connections import ServerConnection from mitmproxy.exceptions import HttpSyntaxException from mitmproxy.test import tflow from mitmproxy.test import tutils -import multiprocessing - -from examples.contrib.webscanner_helper.watchdog import WatchdogAddon, logger class TestWatchdog: - def test_init_file(self, tmpdir): tmpfile = tmpdir.join("tmpfile") with open(tmpfile, "w") as tfile: @@ -35,14 +34,18 @@ def test_init_dir(self, tmpdir): def test_serverconnect(self, tmpdir): event = multiprocessing.Event() w = WatchdogAddon(event, Path(tmpdir), timeout=10) - with mock.patch('mitmproxy.connections.ServerConnection.settimeout') as mock_set_timeout: + with mock.patch( + "mitmproxy.connections.ServerConnection.settimeout" + ) as mock_set_timeout: w.serverconnect(ServerConnection("127.0.0.1")) mock_set_timeout.assert_called() def test_serverconnect_None(self, tmpdir): event = multiprocessing.Event() w = WatchdogAddon(event, Path(tmpdir)) - with mock.patch('mitmproxy.connections.ServerConnection.settimeout') as mock_set_timeout: + with mock.patch( + "mitmproxy.connections.ServerConnection.settimeout" + ) as mock_set_timeout: w.serverconnect(ServerConnection("127.0.0.1")) assert not mock_set_timeout.called @@ -52,7 +55,7 @@ def test_trigger(self, tmpdir): f = tflow.tflow(resp=tutils.tresp()) f.error = "Test Error" - with mock.patch.object(logger, 'error') as mock_error: + with mock.patch.object(logger, "error") as mock_error: open_mock = mock.mock_open() with mock.patch("pathlib.Path.open", open_mock, create=True): w.error(f) @@ -66,7 +69,7 @@ def test_trigger_http_synatx(self, tmpdir): f.error = HttpSyntaxException() assert isinstance(f.error, HttpSyntaxException) - with mock.patch.object(logger, 'error') as mock_error: + with mock.patch.object(logger, "error") as mock_error: open_mock = mock.mock_open() with mock.patch("pathlib.Path.open", open_mock, create=True): w.error(f) @@ -79,6 +82,6 @@ def test_timeout(self, tmpdir): assert w.not_in_timeout(None, None) assert w.not_in_timeout(time.time, None) - with mock.patch('time.time', return_value=5): + with mock.patch("time.time", return_value=5): assert not w.not_in_timeout(3, 20) assert w.not_in_timeout(3, 1) diff --git a/examples/contrib/webscanner_helper/urldict.py b/examples/contrib/webscanner_helper/urldict.py index 7e990f1afc..a5b02af21a 100644 --- a/examples/contrib/webscanner_helper/urldict.py +++ b/examples/contrib/webscanner_helper/urldict.py @@ -1,7 +1,12 @@ import itertools import json +from collections.abc import Generator from collections.abc import MutableMapping -from typing import Any, Callable, Generator, TextIO, Union, cast +from typing import Any +from typing import Callable +from typing import cast +from typing import TextIO +from typing import Union from mitmproxy import flowfilter from mitmproxy.http import HTTPFlow @@ -76,7 +81,7 @@ def loads(cls, json_str: str, value_loader: Callable = f_id): def _dump(self, value_dumper: Callable = f_id) -> dict: dumped: dict[Union[flowfilter.TFilter, str], Any] = {} for fltr, value in self.store.items(): - if hasattr(fltr, 'pattern'): + if hasattr(fltr, "pattern"): # cast necessary for mypy dumped[cast(Any, fltr).pattern] = value_dumper(value) else: diff --git a/examples/contrib/webscanner_helper/urlindex.py b/examples/contrib/webscanner_helper/urlindex.py index 650e47c015..09f9ef2e8c 100644 --- a/examples/contrib/webscanner_helper/urlindex.py +++ b/examples/contrib/webscanner_helper/urlindex.py @@ -3,7 +3,8 @@ import json import logging from pathlib import Path -from typing import Optional, Union +from typing import Optional +from typing import Union from mitmproxy import flowfilter from mitmproxy.http import HTTPFlow @@ -67,7 +68,9 @@ def add_url(self, flow: HTTPFlow): res = flow.response if req is not None and res is not None: - urls = self.host_urls.setdefault(f"{req.scheme}://{req.host}:{req.port}", dict()) + urls = self.host_urls.setdefault( + f"{req.scheme}://{req.host}:{req.port}", dict() + ) methods = urls.setdefault(req.path, {}) codes = methods.setdefault(req.method, set()) codes.add(res.status_code) @@ -88,8 +91,10 @@ def add_url(self, flow: HTTPFlow): req = flow.request if res is not None and req is not None: with self.filepath.open("a+") as f: - f.write(f"{datetime.datetime.utcnow().isoformat()} STATUS: {res.status_code} METHOD: " - f"{req.method} URL:{req.url}\n") + f.write( + f"{datetime.datetime.utcnow().isoformat()} STATUS: {res.status_code} METHOD: " + f"{req.method} URL:{req.url}\n" + ) def save(self): pass @@ -120,9 +125,14 @@ class UrlIndexAddon: OPT_APPEND = "URLINDEX_APPEND" OPT_INDEX_FILTER = "URLINDEX_FILTER" - def __init__(self, file_path: Union[str, Path], append: bool = True, - index_filter: Union[str, flowfilter.TFilter] = filter_404, index_format: str = "json"): - """ Initializes the urlindex add-on. + def __init__( + self, + file_path: Union[str, Path], + append: bool = True, + index_filter: Union[str, flowfilter.TFilter] = filter_404, + index_format: str = "json", + ): + """Initializes the urlindex add-on. Args: file_path: Path to file to which the URL index will be written. Can either be given as str or Path. @@ -153,7 +163,7 @@ def __init__(self, file_path: Union[str, Path], append: bool = True, def response(self, flow: HTTPFlow): """Checks if the response should be included in the URL based on the index_filter and adds it to the URL index - if appropriate. + if appropriate. """ if isinstance(self.index_filter, str) or self.index_filter is None: raise ValueError("Invalid filter expression.") diff --git a/examples/contrib/webscanner_helper/urlinjection.py b/examples/contrib/webscanner_helper/urlinjection.py index 6c4f982915..8cd96313db 100644 --- a/examples/contrib/webscanner_helper/urlinjection.py +++ b/examples/contrib/webscanner_helper/urlinjection.py @@ -11,6 +11,7 @@ class InjectionGenerator: """Abstract class for an generator of the injection content in order to inject the URL index.""" + ENCODING = "UTF8" @abc.abstractmethod @@ -32,11 +33,11 @@ def __init__(self, insert: bool = False): @classmethod def _form_html(cls, url): - return f"
" + return f'
' @classmethod def _link_html(cls, url): - return f"link to {url}" + return f'link to {url}' @classmethod def index_html(cls, index): @@ -54,9 +55,9 @@ def index_html(cls, index): @classmethod def landing_page(cls, index): return ( - "" - + cls.index_html(index) - + "" + '' + + cls.index_html(index) + + "" ) def inject(self, index, flow: HTTPFlow): @@ -64,19 +65,21 @@ def inject(self, index, flow: HTTPFlow): if flow.response.status_code != 404 and not self.insert: logger.warning( f"URL '{flow.request.url}' didn't return 404 status, " - f"index page would overwrite valid page.") + f"index page would overwrite valid page." + ) elif self.insert: - content = (flow.response - .content - .decode(self.ENCODING, "backslashreplace")) + content = flow.response.content.decode( + self.ENCODING, "backslashreplace" + ) if "" in content: - content = content.replace("", self.index_html(index) + "") + content = content.replace( + "", self.index_html(index) + "" + ) else: content += self.index_html(index) flow.response.content = content.encode(self.ENCODING) else: - flow.response.content = (self.landing_page(index) - .encode(self.ENCODING)) + flow.response.content = self.landing_page(index).encode(self.ENCODING) class RobotsInjection(InjectionGenerator): @@ -98,11 +101,12 @@ def inject(self, index, flow: HTTPFlow): if flow.response.status_code != 404: logger.warning( f"URL '{flow.request.url}' didn't return 404 status, " - f"index page would overwrite valid page.") + f"index page would overwrite valid page." + ) else: - flow.response.content = self.robots_txt(index, - self.directive).encode( - self.ENCODING) + flow.response.content = self.robots_txt(index, self.directive).encode( + self.ENCODING + ) class SitemapInjection(InjectionGenerator): @@ -111,7 +115,8 @@ class SitemapInjection(InjectionGenerator): @classmethod def sitemap(cls, index): lines = [ - ""] + '' + ] for scheme_netloc, paths in index.items(): for path, methods in paths.items(): url = scheme_netloc + path @@ -124,13 +129,14 @@ def inject(self, index, flow: HTTPFlow): if flow.response.status_code != 404: logger.warning( f"URL '{flow.request.url}' didn't return 404 status, " - f"index page would overwrite valid page.") + f"index page would overwrite valid page." + ) else: flow.response.content = self.sitemap(index).encode(self.ENCODING) class UrlInjectionAddon: - """ The UrlInjection add-on can be used in combination with web application scanners to improve their crawling + """The UrlInjection add-on can be used in combination with web application scanners to improve their crawling performance. The given URls will be injected into the web application. With this, web application scanners can find pages to @@ -143,8 +149,9 @@ class UrlInjectionAddon: The URL index needed for the injection can be generated by the UrlIndex Add-on. """ - def __init__(self, flt: str, url_index_file: str, - injection_gen: InjectionGenerator): + def __init__( + self, flt: str, url_index_file: str, injection_gen: InjectionGenerator + ): """Initializes the UrlIndex add-on. Args: @@ -168,5 +175,7 @@ def response(self, flow: HTTPFlow): self.injection_gen.inject(self.url_store, flow) flow.response.status_code = 200 flow.response.headers["content-type"] = "text/html" - logger.debug(f"Set status code to 200 and set content to logged " - f"urls. Method: {self.injection_gen}") + logger.debug( + f"Set status code to 200 and set content to logged " + f"urls. Method: {self.injection_gen}" + ) diff --git a/examples/contrib/webscanner_helper/watchdog.py b/examples/contrib/webscanner_helper/watchdog.py index 48f58d9c08..361f72a434 100644 --- a/examples/contrib/webscanner_helper/watchdog.py +++ b/examples/contrib/webscanner_helper/watchdog.py @@ -1,19 +1,20 @@ +import logging import pathlib import time -import logging from datetime import datetime from typing import Union import mitmproxy.connections import mitmproxy.http -from mitmproxy.addons.export import curl_command, raw +from mitmproxy.addons.export import curl_command +from mitmproxy.addons.export import raw from mitmproxy.exceptions import HttpSyntaxException logger = logging.getLogger(__name__) -class WatchdogAddon(): - """ The Watchdog Add-on can be used in combination with web application scanners in oder to check if the device +class WatchdogAddon: + """The Watchdog Add-on can be used in combination with web application scanners in oder to check if the device under test responds correctls to the scanner's responses. The Watchdog Add-on checks if the device under test responds correctly to the scanner's responses. @@ -45,10 +46,14 @@ def serverconnect(self, conn: mitmproxy.connections.ServerConnection): @classmethod def not_in_timeout(cls, last_triggered, timeout): """Checks if current error lies not in timeout after last trigger (potential reset of connection).""" - return last_triggered is None or timeout is None or (time.time() - last_triggered > timeout) + return ( + last_triggered is None + or timeout is None + or (time.time() - last_triggered > timeout) + ) def error(self, flow): - """ Checks if the watchdog will be triggered. + """Checks if the watchdog will be triggered. Only triggers watchdog for timeouts after last reset and if flow.error is set (shows that error is a server error). Ignores HttpSyntaxException Errors since this can be triggered on purpose by web application scanner. @@ -56,8 +61,11 @@ def error(self, flow): Args: flow: mitmproxy.http.flow """ - if (self.not_in_timeout(self.last_trigger, self.timeout) - and flow.error is not None and not isinstance(flow.error, HttpSyntaxException)): + if ( + self.not_in_timeout(self.last_trigger, self.timeout) + and flow.error is not None + and not isinstance(flow.error, HttpSyntaxException) + ): self.last_trigger = time.time() logger.error(f"Watchdog triggered! Cause: {flow}") @@ -65,7 +73,11 @@ def error(self, flow): # save the request which might have caused the problem if flow.request: - with (self.flow_dir / f"{datetime.utcnow().isoformat()}.curl").open("w") as f: + with (self.flow_dir / f"{datetime.utcnow().isoformat()}.curl").open( + "w" + ) as f: f.write(curl_command(flow)) - with (self.flow_dir / f"{datetime.utcnow().isoformat()}.raw").open("wb") as f: + with (self.flow_dir / f"{datetime.utcnow().isoformat()}.raw").open( + "wb" + ) as f: f.write(raw(flow)) diff --git a/examples/contrib/xss_scanner.py b/examples/contrib/xss_scanner.py index 287982fb34..c942281c8c 100644 --- a/examples/contrib/xss_scanner.py +++ b/examples/contrib/xss_scanner.py @@ -38,7 +38,9 @@ import re import socket from html.parser import HTMLParser -from typing import NamedTuple, Optional, Union +from typing import NamedTuple +from typing import Optional +from typing import Union from urllib.parse import urlparse import requests @@ -82,14 +84,14 @@ class SQLiData(NamedTuple): def get_cookies(flow: http.HTTPFlow) -> Cookies: - """ Return a dict going from cookie names to cookie values - - Note that it includes both the cookies sent in the original request and - the cookies sent by the server """ + """Return a dict going from cookie names to cookie values + - Note that it includes both the cookies sent in the original request and + the cookies sent by the server""" return {name: value for name, value in flow.request.cookies.fields} def find_unclaimed_URLs(body, requestUrl): - """ Look for unclaimed URLs in script tags and log them if found""" + """Look for unclaimed URLs in script tags and log them if found""" def getValue(attrs: list[tuple[str, str]], attrName: str) -> Optional[str]: for name, value in attrs: @@ -101,9 +103,15 @@ class ScriptURLExtractor(HTMLParser): script_URLs: list[str] = [] def handle_starttag(self, tag, attrs): - if (tag == "script" or tag == "iframe") and "src" in [name for name, value in attrs]: + if (tag == "script" or tag == "iframe") and "src" in [ + name for name, value in attrs + ]: self.script_URLs.append(getValue(attrs, "src")) - if tag == "link" and getValue(attrs, "rel") == "stylesheet" and "href" in [name for name, value in attrs]: + if ( + tag == "link" + and getValue(attrs, "rel") == "stylesheet" + and "href" in [name for name, value in attrs] + ): self.script_URLs.append(getValue(attrs, "href")) parser = ScriptURLExtractor() @@ -114,17 +122,21 @@ def handle_starttag(self, tag, attrs): try: socket.gethostbyname(domain) except socket.gaierror: - logging.error(f"XSS found in {requestUrl} due to unclaimed URL \"{url}\".") + logging.error(f'XSS found in {requestUrl} due to unclaimed URL "{url}".') -def test_end_of_URL_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData: - """ Test the given URL for XSS via injection onto the end of the URL and - log the XSS if found """ +def test_end_of_URL_injection( + original_body: str, request_URL: str, cookies: Cookies +) -> VulnData: + """Test the given URL for XSS via injection onto the end of the URL and + log the XSS if found""" parsed_URL = urlparse(request_URL) path = parsed_URL.path if path != "" and path[-1] != "/": # ensure the path ends in a / path += "/" - path += FULL_PAYLOAD.decode('utf-8') # the path must be a string while the payload is bytes + path += FULL_PAYLOAD.decode( + "utf-8" + ) # the path must be a string while the payload is bytes url = parsed_URL._replace(path=path).geturl() body = requests.get(url, cookies=cookies).text.lower() xss_info = get_XSS_data(body, url, "End of URL") @@ -132,31 +144,42 @@ def test_end_of_URL_injection(original_body: str, request_URL: str, cookies: Coo return xss_info, sqli_info -def test_referer_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData: - """ Test the given URL for XSS via injection into the referer and - log the XSS if found """ - body = requests.get(request_URL, headers={'referer': FULL_PAYLOAD}, cookies=cookies).text.lower() +def test_referer_injection( + original_body: str, request_URL: str, cookies: Cookies +) -> VulnData: + """Test the given URL for XSS via injection into the referer and + log the XSS if found""" + body = requests.get( + request_URL, headers={"referer": FULL_PAYLOAD}, cookies=cookies + ).text.lower() xss_info = get_XSS_data(body, request_URL, "Referer") sqli_info = get_SQLi_data(body, original_body, request_URL, "Referer") return xss_info, sqli_info -def test_user_agent_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData: - """ Test the given URL for XSS via injection into the user agent and - log the XSS if found """ - body = requests.get(request_URL, headers={'User-Agent': FULL_PAYLOAD}, cookies=cookies).text.lower() +def test_user_agent_injection( + original_body: str, request_URL: str, cookies: Cookies +) -> VulnData: + """Test the given URL for XSS via injection into the user agent and + log the XSS if found""" + body = requests.get( + request_URL, headers={"User-Agent": FULL_PAYLOAD}, cookies=cookies + ).text.lower() xss_info = get_XSS_data(body, request_URL, "User Agent") sqli_info = get_SQLi_data(body, original_body, request_URL, "User Agent") return xss_info, sqli_info def test_query_injection(original_body: str, request_URL: str, cookies: Cookies): - """ Test the given URL for XSS via injection into URL queries and - log the XSS if found """ + """Test the given URL for XSS via injection into URL queries and + log the XSS if found""" parsed_URL = urlparse(request_URL) query_string = parsed_URL.query # queries is a list of parameters where each parameter is set to the payload - queries = [query.split("=")[0] + "=" + FULL_PAYLOAD.decode('utf-8') for query in query_string.split("&")] + queries = [ + query.split("=")[0] + "=" + FULL_PAYLOAD.decode("utf-8") + for query in query_string.split("&") + ] new_query_string = "&".join(queries) new_URL = parsed_URL._replace(query=new_query_string).geturl() body = requests.get(new_URL, cookies=cookies).text.lower() @@ -166,7 +189,7 @@ def test_query_injection(original_body: str, request_URL: str, cookies: Cookies) def log_XSS_data(xss_info: Optional[XSSData]) -> None: - """ Log information about the given XSS to mitmproxy """ + """Log information about the given XSS to mitmproxy""" # If it is None, then there is no info to log if not xss_info: return @@ -178,7 +201,7 @@ def log_XSS_data(xss_info: Optional[XSSData]) -> None: def log_SQLi_data(sqli_info: Optional[SQLiData]) -> None: - """ Log information about the given SQLi to mitmproxy """ + """Log information about the given SQLi to mitmproxy""" if not sqli_info: return logging.error("===== SQLi Found =====") @@ -189,51 +212,88 @@ def log_SQLi_data(sqli_info: Optional[SQLiData]) -> None: return -def get_SQLi_data(new_body: str, original_body: str, request_URL: str, injection_point: str) -> Optional[SQLiData]: - """ Return a SQLiDict if there is a SQLi otherwise return None - String String URL String -> (SQLiDict or None) """ +def get_SQLi_data( + new_body: str, original_body: str, request_URL: str, injection_point: str +) -> Optional[SQLiData]: + """Return a SQLiDict if there is a SQLi otherwise return None + String String URL String -> (SQLiDict or None)""" # Regexes taken from Damn Small SQLi Scanner: https://github.com/stamparm/DSSS/blob/master/dsss.py#L17 DBMS_ERRORS = { - "MySQL": (r"SQL syntax.*MySQL", r"Warning.*mysql_.*", r"valid MySQL result", r"MySqlClient\."), - "PostgreSQL": (r"PostgreSQL.*ERROR", r"Warning.*\Wpg_.*", r"valid PostgreSQL result", r"Npgsql\."), - "Microsoft SQL Server": (r"Driver.* SQL[\-\_\ ]*Server", r"OLE DB.* SQL Server", r"(\W|\A)SQL Server.*Driver", - r"Warning.*mssql_.*", r"(\W|\A)SQL Server.*[0-9a-fA-F]{8}", - r"(?s)Exception.*\WSystem\.Data\.SqlClient\.", r"(?s)Exception.*\WRoadhouse\.Cms\."), - "Microsoft Access": (r"Microsoft Access Driver", r"JET Database Engine", r"Access Database Engine"), - "Oracle": (r"\bORA-[0-9][0-9][0-9][0-9]", r"Oracle error", r"Oracle.*Driver", r"Warning.*\Woci_.*", r"Warning.*\Wora_.*"), + "MySQL": ( + r"SQL syntax.*MySQL", + r"Warning.*mysql_.*", + r"valid MySQL result", + r"MySqlClient\.", + ), + "PostgreSQL": ( + r"PostgreSQL.*ERROR", + r"Warning.*\Wpg_.*", + r"valid PostgreSQL result", + r"Npgsql\.", + ), + "Microsoft SQL Server": ( + r"Driver.* SQL[\-\_\ ]*Server", + r"OLE DB.* SQL Server", + r"(\W|\A)SQL Server.*Driver", + r"Warning.*mssql_.*", + r"(\W|\A)SQL Server.*[0-9a-fA-F]{8}", + r"(?s)Exception.*\WSystem\.Data\.SqlClient\.", + r"(?s)Exception.*\WRoadhouse\.Cms\.", + ), + "Microsoft Access": ( + r"Microsoft Access Driver", + r"JET Database Engine", + r"Access Database Engine", + ), + "Oracle": ( + r"\bORA-[0-9][0-9][0-9][0-9]", + r"Oracle error", + r"Oracle.*Driver", + r"Warning.*\Woci_.*", + r"Warning.*\Wora_.*", + ), "IBM DB2": (r"CLI Driver.*DB2", r"DB2 SQL error", r"\bdb2_\w+\("), - "SQLite": (r"SQLite/JDBCDriver", r"SQLite.Exception", r"System.Data.SQLite.SQLiteException", r"Warning.*sqlite_.*", - r"Warning.*SQLite3::", r"\[SQLITE_ERROR\]"), - "Sybase": (r"(?i)Warning.*sybase.*", r"Sybase message", r"Sybase.*Server message.*"), + "SQLite": ( + r"SQLite/JDBCDriver", + r"SQLite.Exception", + r"System.Data.SQLite.SQLiteException", + r"Warning.*sqlite_.*", + r"Warning.*SQLite3::", + r"\[SQLITE_ERROR\]", + ), + "Sybase": ( + r"(?i)Warning.*sybase.*", + r"Sybase message", + r"Sybase.*Server message.*", + ), } for dbms, regexes in DBMS_ERRORS.items(): for regex in regexes: # type: ignore - if re.search(regex, new_body, re.IGNORECASE) and not re.search(regex, original_body, re.IGNORECASE): - return SQLiData(request_URL, - injection_point, - regex, - dbms) + if re.search(regex, new_body, re.IGNORECASE) and not re.search( + regex, original_body, re.IGNORECASE + ): + return SQLiData(request_URL, injection_point, regex, dbms) return None # A qc is either ' or " -def inside_quote(qc: str, substring_bytes: bytes, text_index: int, body_bytes: bytes) -> bool: - """ Whether the Numberth occurrence of the first string in the second - string is inside quotes as defined by the supplied QuoteChar """ - substring = substring_bytes.decode('utf-8') - body = body_bytes.decode('utf-8') +def inside_quote( + qc: str, substring_bytes: bytes, text_index: int, body_bytes: bytes +) -> bool: + """Whether the Numberth occurrence of the first string in the second + string is inside quotes as defined by the supplied QuoteChar""" + substring = substring_bytes.decode("utf-8") + body = body_bytes.decode("utf-8") num_substrings_found = 0 in_quote = False for index, char in enumerate(body): # Whether the next chunk of len(substring) chars is the substring - next_part_is_substring = ( - (not (index + len(substring) > len(body))) and - (body[index:index + len(substring)] == substring) + next_part_is_substring = (not (index + len(substring) > len(body))) and ( + body[index : index + len(substring)] == substring ) # Whether this char is escaped with a \ - is_not_escaped = ( - (index - 1 < 0 or index - 1 > len(body)) or - (body[index - 1] != "\\") + is_not_escaped = (index - 1 < 0 or index - 1 > len(body)) or ( + body[index - 1] != "\\" ) if char == qc and is_not_escaped: in_quote = not in_quote @@ -245,25 +305,27 @@ def inside_quote(qc: str, substring_bytes: bytes, text_index: int, body_bytes: b def paths_to_text(html: str, string: str) -> list[str]: - """ Return list of Paths to a given str in the given HTML tree - - Note that it does a BFS """ + """Return list of Paths to a given str in the given HTML tree + - Note that it does a BFS""" def remove_last_occurence_of_sub_string(string: str, substr: str) -> str: - """ Delete the last occurrence of substr from str + """Delete the last occurrence of substr from str String String -> String """ index = string.rfind(substr) - return string[:index] + string[index + len(substr):] + return string[:index] + string[index + len(substr) :] class PathHTMLParser(HTMLParser): currentPath = "" paths: list[str] = [] def handle_starttag(self, tag, attrs): - self.currentPath += ("/" + tag) + self.currentPath += "/" + tag def handle_endtag(self, tag): - self.currentPath = remove_last_occurence_of_sub_string(self.currentPath, "/" + tag) + self.currentPath = remove_last_occurence_of_sub_string( + self.currentPath, "/" + tag + ) def handle_data(self, data): if string in data: @@ -274,13 +336,15 @@ def handle_data(self, data): return parser.paths -def get_XSS_data(body: Union[str, bytes], request_URL: str, injection_point: str) -> Optional[XSSData]: - """ Return a XSSDict if there is a XSS otherwise return None """ +def get_XSS_data( + body: Union[str, bytes], request_URL: str, injection_point: str +) -> Optional[XSSData]: + """Return a XSSDict if there is a XSS otherwise return None""" def in_script(text, index, body) -> bool: - """ Whether the Numberth occurrence of the first string in the second - string is inside a script tag """ - paths = paths_to_text(body.decode('utf-8'), text.decode("utf-8")) + """Whether the Numberth occurrence of the first string in the second + string is inside a script tag""" + paths = paths_to_text(body.decode("utf-8"), text.decode("utf-8")) try: path = paths[index] return "script" in path @@ -288,12 +352,12 @@ def in_script(text, index, body) -> bool: return False def in_HTML(text: bytes, index: int, body: bytes) -> bool: - """ Whether the Numberth occurrence of the first string in the second - string is inside the HTML but not inside a script tag or part of - a HTML attribute""" + """Whether the Numberth occurrence of the first string in the second + string is inside the HTML but not inside a script tag or part of + a HTML attribute""" # if there is a < then lxml will interpret that as a tag, so only search for the stuff before it text = text.split(b"<")[0] - paths = paths_to_text(body.decode('utf-8'), text.decode("utf-8")) + paths = paths_to_text(body.decode("utf-8"), text.decode("utf-8")) try: path = paths[index] return "script" not in path @@ -301,14 +365,14 @@ def in_HTML(text: bytes, index: int, body: bytes) -> bool: return False def inject_javascript_handler(html: str) -> bool: - """ Whether you can inject a Javascript:alert(0) as a link """ + """Whether you can inject a Javascript:alert(0) as a link""" class injectJSHandlerHTMLParser(HTMLParser): injectJSHandler = False def handle_starttag(self, tag, attrs): for name, value in attrs: - if name == "href" and value.startswith(FRONT_WALL.decode('utf-8')): + if name == "href" and value.startswith(FRONT_WALL.decode("utf-8")): self.injectJSHandler = True parser = injectJSHandlerHTMLParser() @@ -317,7 +381,7 @@ def handle_starttag(self, tag, attrs): # Only convert the body to bytes if needed if isinstance(body, str): - body = bytes(body, 'utf-8') + body = bytes(body, "utf-8") # Regex for between 24 and 72 (aka 24*3) characters encapsulated by the walls regex = re.compile(b"""%s.{24,72}?%s""" % (FRONT_WALL, BACK_WALL)) matches = regex.findall(body) @@ -336,64 +400,121 @@ def handle_starttag(self, tag, attrs): inject_slash = b"sl/bsl" in match # forward slashes inject_semi = b"se;sl" in match # semicolons inject_equals = b"eq=" in match # equals sign - if in_script_val and inject_slash and inject_open_angle and inject_close_angle: # e.g. - return XSSData(request_URL, - injection_point, - ' - return XSSData(request_URL, - injection_point, - "';alert(0);g='", - match.decode('utf-8')) - elif in_script_val and in_double_quotes and inject_double_quotes and inject_semi: # e.g. - return XSSData(request_URL, - injection_point, - '";alert(0);g="', - match.decode('utf-8')) - elif in_tag and in_single_quotes and inject_single_quotes and inject_open_angle and inject_close_angle and inject_slash: + if ( + in_script_val and inject_slash and inject_open_angle and inject_close_angle + ): # e.g. + return XSSData( + request_URL, + injection_point, + " + return XSSData( + request_URL, injection_point, "';alert(0);g='", match.decode("utf-8") + ) + elif ( + in_script_val and in_double_quotes and inject_double_quotes and inject_semi + ): # e.g. + return XSSData( + request_URL, injection_point, '";alert(0);g="', match.decode("utf-8") + ) + elif ( + in_tag + and in_single_quotes + and inject_single_quotes + and inject_open_angle + and inject_close_angle + and inject_slash + ): # e.g. Test - return XSSData(request_URL, - injection_point, - "'>", - match.decode('utf-8')) - elif in_tag and in_double_quotes and inject_double_quotes and inject_open_angle and inject_close_angle and inject_slash: + return XSSData( + request_URL, + injection_point, + "'>", + match.decode("utf-8"), + ) + elif ( + in_tag + and in_double_quotes + and inject_double_quotes + and inject_open_angle + and inject_close_angle + and inject_slash + ): # e.g. Test - return XSSData(request_URL, - injection_point, - '">', - match.decode('utf-8')) - elif in_tag and not in_double_quotes and not in_single_quotes and inject_open_angle and inject_close_angle and inject_slash: + return XSSData( + request_URL, + injection_point, + '">', + match.decode("utf-8"), + ) + elif ( + in_tag + and not in_double_quotes + and not in_single_quotes + and inject_open_angle + and inject_close_angle + and inject_slash + ): # e.g. Test - return XSSData(request_URL, - injection_point, - '>', - match.decode('utf-8')) - elif inject_javascript_handler(body.decode('utf-8')): # e.g. Test - return XSSData(request_URL, - injection_point, - 'Javascript:alert(0)', - match.decode('utf-8')) - elif in_tag and in_double_quotes and inject_double_quotes and inject_equals: # e.g. Test - return XSSData(request_URL, - injection_point, - '" onmouseover="alert(0)" t="', - match.decode('utf-8')) - elif in_tag and in_single_quotes and inject_single_quotes and inject_equals: # e.g. Test - return XSSData(request_URL, - injection_point, - "' onmouseover='alert(0)' t='", - match.decode('utf-8')) - elif in_tag and not in_single_quotes and not in_double_quotes and inject_equals: # e.g. Test - return XSSData(request_URL, - injection_point, - " onmouseover=alert(0) t=", - match.decode('utf-8')) - elif in_HTML_val and not in_script_val and inject_open_angle and inject_close_angle and inject_slash: # e.g. PAYLOAD - return XSSData(request_URL, - injection_point, - '', - match.decode('utf-8')) + return XSSData( + request_URL, + injection_point, + ">", + match.decode("utf-8"), + ) + elif inject_javascript_handler( + body.decode("utf-8") + ): # e.g. Test + return XSSData( + request_URL, + injection_point, + "Javascript:alert(0)", + match.decode("utf-8"), + ) + elif ( + in_tag and in_double_quotes and inject_double_quotes and inject_equals + ): # e.g. Test + return XSSData( + request_URL, + injection_point, + '" onmouseover="alert(0)" t="', + match.decode("utf-8"), + ) + elif ( + in_tag and in_single_quotes and inject_single_quotes and inject_equals + ): # e.g. Test + return XSSData( + request_URL, + injection_point, + "' onmouseover='alert(0)' t='", + match.decode("utf-8"), + ) + elif ( + in_tag and not in_single_quotes and not in_double_quotes and inject_equals + ): # e.g. Test + return XSSData( + request_URL, + injection_point, + " onmouseover=alert(0) t=", + match.decode("utf-8"), + ) + elif ( + in_HTML_val + and not in_script_val + and inject_open_angle + and inject_close_angle + and inject_slash + ): # e.g. PAYLOAD + return XSSData( + request_URL, + injection_point, + "", + match.decode("utf-8"), + ) else: return None return None diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 265c811139..5b52750a5c 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -2,13 +2,14 @@ import inspect import logging import pprint +import sys import traceback import types -from collections.abc import Callable, Sequence +from collections.abc import Callable +from collections.abc import Sequence from dataclasses import dataclass -from typing import Any, Optional - -import sys +from typing import Any +from typing import Optional from mitmproxy import exceptions from mitmproxy import flow diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 36ea9bfe55..0d801f577f 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -11,19 +11,19 @@ from mitmproxy.addons import disable_h2c from mitmproxy.addons import dns_resolver from mitmproxy.addons import export +from mitmproxy.addons import maplocal +from mitmproxy.addons import mapremote +from mitmproxy.addons import modifybody +from mitmproxy.addons import modifyheaders from mitmproxy.addons import next_layer from mitmproxy.addons import onboarding -from mitmproxy.addons import proxyserver from mitmproxy.addons import proxyauth +from mitmproxy.addons import proxyserver +from mitmproxy.addons import save from mitmproxy.addons import script from mitmproxy.addons import serverplayback -from mitmproxy.addons import mapremote -from mitmproxy.addons import maplocal -from mitmproxy.addons import modifybody -from mitmproxy.addons import modifyheaders from mitmproxy.addons import stickyauth from mitmproxy.addons import stickycookie -from mitmproxy.addons import save from mitmproxy.addons import tlsconfig from mitmproxy.addons import upstream_auth diff --git a/mitmproxy/addons/asgiapp.py b/mitmproxy/addons/asgiapp.py index f85a88ebcb..5425275a9d 100644 --- a/mitmproxy/addons/asgiapp.py +++ b/mitmproxy/addons/asgiapp.py @@ -7,7 +7,8 @@ import asgiref.compatibility import asgiref.wsgi -from mitmproxy import ctx, http +from mitmproxy import ctx +from mitmproxy import http logger = logging.getLogger(__name__) diff --git a/mitmproxy/addons/blocklist.py b/mitmproxy/addons/blocklist.py index 4d5b7bedff..6c99bca067 100644 --- a/mitmproxy/addons/blocklist.py +++ b/mitmproxy/addons/blocklist.py @@ -1,7 +1,11 @@ from collections.abc import Sequence from typing import NamedTuple -from mitmproxy import ctx, exceptions, flowfilter, http, version +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import flowfilter +from mitmproxy import http +from mitmproxy import version from mitmproxy.net.http.status_codes import NO_RESPONSE from mitmproxy.net.http.status_codes import RESPONSES diff --git a/mitmproxy/addons/browser.py b/mitmproxy/addons/browser.py index 7354f7c1e9..ab2fcc5601 100644 --- a/mitmproxy/addons/browser.py +++ b/mitmproxy/addons/browser.py @@ -74,7 +74,9 @@ def start(self) -> None: cmd = get_browser_cmd() if not cmd: - logging.log(ALERT, "Your platform is not supported yet - please submit a patch.") + logging.log( + ALERT, "Your platform is not supported yet - please submit a patch." + ) return tdir = tempfile.TemporaryDirectory() @@ -85,7 +87,8 @@ def start(self) -> None: *cmd, "--user-data-dir=%s" % str(tdir.name), "--proxy-server={}:{}".format( - ctx.options.listen_host or "127.0.0.1", ctx.options.listen_port or "8080" + ctx.options.listen_host or "127.0.0.1", + ctx.options.listen_port or "8080", ), "--disable-fre", "--no-default-browser-check", diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index d435324e86..7abb83622b 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -1,10 +1,10 @@ import asyncio import logging +import time import traceback from collections.abc import Sequence -from typing import Optional, cast - -import time +from typing import cast +from typing import Optional import mitmproxy.types from mitmproxy import command @@ -13,11 +13,15 @@ from mitmproxy import flow from mitmproxy import http from mitmproxy import io -from mitmproxy.connection import ConnectionState, Server +from mitmproxy.connection import ConnectionState +from mitmproxy.connection import Server from mitmproxy.hooks import UpdateHook from mitmproxy.log import ALERT from mitmproxy.options import Options -from mitmproxy.proxy import commands, events, layers, server +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layers +from mitmproxy.proxy import server from mitmproxy.proxy.context import Context from mitmproxy.proxy.layer import CommandGenerator from mitmproxy.proxy.layers.http import HTTPMode @@ -161,9 +165,7 @@ async def playback(self): else: await h.replay() except Exception: - logger.error( - f"Client replay has crashed!\n{traceback.format_exc()}" - ) + logger.error(f"Client replay has crashed!\n{traceback.format_exc()}") self.queue.task_done() self.inflight = None diff --git a/mitmproxy/addons/command_history.py b/mitmproxy/addons/command_history.py index d75d3cade5..507b60e500 100644 --- a/mitmproxy/addons/command_history.py +++ b/mitmproxy/addons/command_history.py @@ -41,7 +41,7 @@ def configure(self, updated): def done(self): if ctx.options.command_history and len(self.history) >= self.VACUUM_SIZE: # vacuum history so that it doesn't grow indefinitely. - history_str = "\n".join(self.history[-self.VACUUM_SIZE // 2:]) + "\n" + history_str = "\n".join(self.history[-self.VACUUM_SIZE // 2 :]) + "\n" try: self.history_file.write_text(history_str) except Exception as e: diff --git a/mitmproxy/addons/comment.py b/mitmproxy/addons/comment.py index 3e9b549c72..ecb303b0c0 100644 --- a/mitmproxy/addons/comment.py +++ b/mitmproxy/addons/comment.py @@ -1,6 +1,8 @@ from collections.abc import Sequence -from mitmproxy import command, flow, ctx +from mitmproxy import command +from mitmproxy import ctx +from mitmproxy import flow from mitmproxy.hooks import UpdateHook diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 87eda62cd5..5ab1ba0a69 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -3,15 +3,16 @@ from collections.abc import Sequence from typing import Union -from mitmproxy.log import ALERT -from mitmproxy.utils import emoji -from mitmproxy import ctx, hooks -from mitmproxy import exceptions +import mitmproxy.types from mitmproxy import command +from mitmproxy import ctx +from mitmproxy import exceptions from mitmproxy import flow +from mitmproxy import hooks from mitmproxy import optmanager +from mitmproxy.log import ALERT from mitmproxy.net.http import status_codes -import mitmproxy.types +from mitmproxy.utils import emoji logger = logging.getLogger(__name__) diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index 52a21df527..34a7b8fe0a 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -1,18 +1,18 @@ -import io import csv +import io import logging import os.path from collections.abc import Sequence -from typing import Any, Union +from typing import Any +from typing import Union + +import pyperclip +import mitmproxy.types +from mitmproxy import certs from mitmproxy import command from mitmproxy import exceptions from mitmproxy import flow -from mitmproxy import certs -import mitmproxy.types - -import pyperclip - from mitmproxy.log import ALERT logger = logging.getLogger(__name__) @@ -132,7 +132,7 @@ def save( writer.writerow(vals) logger.log( ALERT, - "Saved %s cuts over %d flows as CSV." % (len(cuts), len(flows)) + "Saved %s cuts over %d flows as CSV." % (len(cuts), len(flows)), ) except OSError as e: logger.error(str(e)) diff --git a/mitmproxy/addons/dns_resolver.py b/mitmproxy/addons/dns_resolver.py index fcfb153f2d..714d37e16e 100644 --- a/mitmproxy/addons/dns_resolver.py +++ b/mitmproxy/addons/dns_resolver.py @@ -1,7 +1,10 @@ import asyncio import ipaddress import socket -from typing import Callable, Iterable, Union +from collections.abc import Iterable +from typing import Callable +from typing import Union + from mitmproxy import dns from mitmproxy.proxy import mode_specs diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 8b290d1717..42ecd26008 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -1,10 +1,12 @@ from __future__ import annotations -import logging import itertools +import logging import shutil import sys -from typing import IO, Optional, Union +from typing import IO +from typing import Optional +from typing import Union from wsproto.frame_protocol import CloseReason @@ -17,12 +19,15 @@ from mitmproxy import http from mitmproxy.contrib import click as miniclick from mitmproxy.net.dns import response_codes -from mitmproxy.tcp import TCPFlow, TCPMessage -from mitmproxy.udp import UDPFlow, UDPMessage +from mitmproxy.tcp import TCPFlow +from mitmproxy.tcp import TCPMessage +from mitmproxy.udp import UDPFlow +from mitmproxy.udp import UDPMessage from mitmproxy.utils import human from mitmproxy.utils import strutils from mitmproxy.utils import vt_codes -from mitmproxy.websocket import WebSocketData, WebSocketMessage +from mitmproxy.websocket import WebSocketData +from mitmproxy.websocket import WebSocketMessage def indent(n: int, text: str) -> str: @@ -387,9 +392,12 @@ def dns_response(self, f: dns.DNSFlow): self.style(str(x), fg="bright_blue") for x in f.response.answers ) else: - answers = self.style(response_codes.to_str( - f.response.response_code, - ), fg="red") + answers = self.style( + response_codes.to_str( + f.response.response_code, + ), + fg="red", + ) self.echo(f"{arrows} {answers}") def dns_error(self, f: dns.DNSFlow): diff --git a/mitmproxy/addons/errorcheck.py b/mitmproxy/addons/errorcheck.py index 1015c9efb5..b634623cff 100644 --- a/mitmproxy/addons/errorcheck.py +++ b/mitmproxy/addons/errorcheck.py @@ -1,6 +1,5 @@ import asyncio import logging - import sys from mitmproxy import log diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py index 6f597b9c3d..e719e69797 100644 --- a/mitmproxy/addons/eventstore.py +++ b/mitmproxy/addons/eventstore.py @@ -4,7 +4,8 @@ from collections.abc import Callable from typing import Optional -from mitmproxy import command, log +from mitmproxy import command +from mitmproxy import log from mitmproxy.log import LogEntry from mitmproxy.utils import signals diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 1e2267bffb..1339111c5b 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -1,15 +1,18 @@ import logging import shlex -from collections.abc import Callable, Sequence -from typing import Any, Union +from collections.abc import Callable +from collections.abc import Sequence +from typing import Any +from typing import Union import pyperclip import mitmproxy.types from mitmproxy import command -from mitmproxy import ctx, http +from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import flow +from mitmproxy import http from mitmproxy.net.http.http1 import assemble from mitmproxy.utils import strutils diff --git a/mitmproxy/addons/intercept.py b/mitmproxy/addons/intercept.py index f918762692..24a9173f95 100644 --- a/mitmproxy/addons/intercept.py +++ b/mitmproxy/addons/intercept.py @@ -1,8 +1,9 @@ from typing import Optional -from mitmproxy import flow, flowfilter -from mitmproxy import exceptions from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import flow +from mitmproxy import flowfilter class Intercept: diff --git a/mitmproxy/addons/keepserving.py b/mitmproxy/addons/keepserving.py index 5a149fdec1..199cf25487 100644 --- a/mitmproxy/addons/keepserving.py +++ b/mitmproxy/addons/keepserving.py @@ -1,4 +1,5 @@ import asyncio + from mitmproxy import ctx diff --git a/mitmproxy/addons/maplocal.py b/mitmproxy/addons/maplocal.py index 54c522d93e..5b1abd0b53 100644 --- a/mitmproxy/addons/maplocal.py +++ b/mitmproxy/addons/maplocal.py @@ -8,7 +8,11 @@ from werkzeug.security import safe_join -from mitmproxy import ctx, exceptions, flowfilter, http, version +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import flowfilter +from mitmproxy import http +from mitmproxy import version from mitmproxy.utils.spec import parse_spec diff --git a/mitmproxy/addons/mapremote.py b/mitmproxy/addons/mapremote.py index 245323a034..31a759ada4 100644 --- a/mitmproxy/addons/mapremote.py +++ b/mitmproxy/addons/mapremote.py @@ -2,7 +2,10 @@ from collections.abc import Sequence from typing import NamedTuple -from mitmproxy import ctx, exceptions, flowfilter, http +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import flowfilter +from mitmproxy import http from mitmproxy.utils.spec import parse_spec diff --git a/mitmproxy/addons/modifybody.py b/mitmproxy/addons/modifybody.py index b82059afe1..4148c47999 100644 --- a/mitmproxy/addons/modifybody.py +++ b/mitmproxy/addons/modifybody.py @@ -2,8 +2,10 @@ import re from collections.abc import Sequence -from mitmproxy import ctx, exceptions -from mitmproxy.addons.modifyheaders import parse_modify_spec, ModifySpec +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy.addons.modifyheaders import ModifySpec +from mitmproxy.addons.modifyheaders import parse_modify_spec class ModifyBody: diff --git a/mitmproxy/addons/modifyheaders.py b/mitmproxy/addons/modifyheaders.py index 995005f305..a7e45b0ddd 100644 --- a/mitmproxy/addons/modifyheaders.py +++ b/mitmproxy/addons/modifyheaders.py @@ -4,7 +4,10 @@ from pathlib import Path from typing import NamedTuple -from mitmproxy import ctx, exceptions, flowfilter, http +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import flowfilter +from mitmproxy import http from mitmproxy.http import Headers from mitmproxy.utils import strutils from mitmproxy.utils.spec import parse_spec diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index bf4be48dff..3495ed9198 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -15,22 +15,39 @@ that sets nextlayer.layer works just as well. """ import re -from collections.abc import Sequence import struct -from typing import Any, Callable, Iterable, Optional, Union, cast - -from mitmproxy import ctx, dns, exceptions, connection +from collections.abc import Iterable +from collections.abc import Sequence +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional +from typing import Union + +from mitmproxy import connection +from mitmproxy import ctx +from mitmproxy import dns +from mitmproxy import exceptions from mitmproxy.net.tls import is_tls_record_magic -from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy import context, layer, layers, mode_specs +from mitmproxy.proxy import context +from mitmproxy.proxy import layer +from mitmproxy.proxy import layers +from mitmproxy.proxy import mode_specs from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy.layers.quic import quic_parse_client_hello -from mitmproxy.proxy.layers.tls import HTTP_ALPNS, dtls_parse_client_hello, parse_client_hello +from mitmproxy.proxy.layers.tls import dtls_parse_client_hello +from mitmproxy.proxy.layers.tls import HTTP_ALPNS +from mitmproxy.proxy.layers.tls import parse_client_hello from mitmproxy.tls import ClientHello LayerCls = type[layer.Layer] -ClientSecurityLayerCls = Union[type[layers.ClientTLSLayer], type[layers.ClientQuicLayer]] -ServerSecurityLayerCls = Union[type[layers.ServerTLSLayer], type[layers.ServerQuicLayer]] +ClientSecurityLayerCls = Union[ + type[layers.ClientTLSLayer], type[layers.ClientQuicLayer] +] +ServerSecurityLayerCls = Union[ + type[layers.ServerTLSLayer], type[layers.ServerQuicLayer] +] def stack_match( @@ -77,7 +94,7 @@ def ignore_connection( data_client: bytes, *, is_tls: Callable[[bytes], bool] = is_tls_record_magic, - client_hello: Callable[[bytes], Optional[ClientHello]] = parse_client_hello + client_hello: Callable[[bytes], Optional[ClientHello]] = parse_client_hello, ) -> Optional[bool]: """ Returns: @@ -148,7 +165,9 @@ def s(*layers): ret.child_layer = client_layer_cls(context) return ret - def is_destination_in_hosts(self, context: context.Context, hosts: Iterable[re.Pattern]) -> bool: + def is_destination_in_hosts( + self, context: context.Context, hosts: Iterable[re.Pattern] + ) -> bool: return any( (context.server.address and rex.search(context.server.address[0])) or (context.client.sni and rex.search(context.client.sni)) @@ -168,15 +187,15 @@ def s(*layers): ): return layers.HttpLayer(context, HTTPMode.regular) # ... or an upstream proxy. - if ( - s(modes.HttpUpstreamProxy) - or - s(modes.HttpUpstreamProxy, (layers.ClientTLSLayer, layers.ClientQuicLayer)) + if s(modes.HttpUpstreamProxy) or s( + modes.HttpUpstreamProxy, (layers.ClientTLSLayer, layers.ClientQuicLayer) ): return layers.HttpLayer(context, HTTPMode.upstream) return None - def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls]]: + def detect_udp_tls( + self, data_client: bytes + ) -> Optional[tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls]]: if len(data_client) == 0: return None @@ -198,23 +217,23 @@ def detect_udp_tls(self, data_client: bytes) -> Optional[tuple[ClientHello, Clie # that's all we currently have to offer return None - def raw_udp_layer(self, context: context.Context, ignore: bool = False) -> layer.Layer: + def raw_udp_layer( + self, context: context.Context, ignore: bool = False + ) -> layer.Layer: def s(*layers): return stack_match(context, layers) # for regular and upstream HTTP3, if we already created a client QUIC layer # we need a server and raw QUIC layer as well - if ( - s(modes.HttpProxy, layers.ClientQuicLayer) - or - s(modes.HttpUpstreamProxy, layers.ClientQuicLayer) + if s(modes.HttpProxy, layers.ClientQuicLayer) or s( + modes.HttpUpstreamProxy, layers.ClientQuicLayer ): server_layer = layers.ServerQuicLayer(context) server_layer.child_layer = layers.RawQuicLayer(context, ignore=ignore) return server_layer # for reverse HTTP3 and QUIC, we need a client and raw QUIC layer - elif (s(modes.ReverseProxy, layers.ServerQuicLayer)): + elif s(modes.ReverseProxy, layers.ServerQuicLayer): client_layer = layers.ClientQuicLayer(context) client_layer.child_layer = layers.RawQuicLayer(context, ignore=ignore) return client_layer @@ -243,11 +262,7 @@ def _next_layer( if context.client.transport_protocol == "tcp": is_quic_stream = isinstance(context.layers[-1], layers.QuicStreamLayer) - if ( - len(data_client) < 3 - and not data_server - and not is_quic_stream - ): + if len(data_client) < 3 and not data_server and not is_quic_stream: return None # not enough data yet to make a decision # 1. check for --ignore/--allow @@ -292,7 +307,7 @@ def _next_layer( context.server.address, data_client, is_tls=lambda _: tls is not None, - client_hello=lambda _: None if tls is None else tls[0] + client_hello=lambda _: None if tls is None else tls[0], ): return self.raw_udp_layer(context, ignore=True) @@ -310,7 +325,7 @@ def _next_layer( return self.raw_udp_layer(context) # 5. Check for reverse modes - if (isinstance(context.layers[0], modes.ReverseProxy)): + if isinstance(context.layers[0], modes.ReverseProxy): scheme = cast(mode_specs.ReverseMode, context.client.proxy_mode).scheme if scheme in ("udp", "dtls"): return layers.UDPLayer(context) diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 78ff110334..02cf4bd204 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -1,6 +1,6 @@ +from mitmproxy import ctx from mitmproxy.addons import asgiapp from mitmproxy.addons.onboardingapp import app -from mitmproxy import ctx APP_HOST = "mitm.it" diff --git a/mitmproxy/addons/onboardingapp/__init__.py b/mitmproxy/addons/onboardingapp/__init__.py index f8fafd20c2..a4aed19858 100644 --- a/mitmproxy/addons/onboardingapp/__init__.py +++ b/mitmproxy/addons/onboardingapp/__init__.py @@ -1,8 +1,10 @@ import os -from flask import Flask, render_template +from flask import Flask +from flask import render_template -from mitmproxy.options import CONF_BASENAME, CONF_DIR +from mitmproxy.options import CONF_BASENAME +from mitmproxy.options import CONF_DIR from mitmproxy.utils.magisk import write_magisk_module app = Flask(__name__) diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 653fa48ae0..5ff57feeee 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -2,14 +2,16 @@ import binascii import weakref -from abc import ABC, abstractmethod -from typing import MutableMapping +from abc import ABC +from abc import abstractmethod +from collections.abc import MutableMapping from typing import Optional import ldap3 import passlib.apache -from mitmproxy import connection, ctx +from mitmproxy import connection +from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import http from mitmproxy.net.http import status_codes @@ -141,7 +143,9 @@ def is_http_proxy(f: http.HTTPFlow) -> bool: - True, if authentication is done as if mitmproxy is a proxy - False, if authentication is done as if mitmproxy is an HTTP server """ - return isinstance(f.client_conn.proxy_mode, (mode_specs.RegularMode, mode_specs.UpstreamMode)) + return isinstance( + f.client_conn.proxy_mode, (mode_specs.RegularMode, mode_specs.UpstreamMode) + ) def mkauth(username: str, password: str, scheme: str = "basic") -> str: diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index c28980bf08..cb17bcfbf0 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -7,29 +7,34 @@ import collections import ipaddress import logging +from collections.abc import Iterable +from collections.abc import Iterator from contextlib import contextmanager -from typing import Iterable, Iterator, Optional +from typing import Optional from wsproto.frame_protocol import Opcode -from mitmproxy import ( - command, - ctx, - exceptions, - http, - platform, - tcp, - udp, - websocket, -) +from mitmproxy import command +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import http +from mitmproxy import platform +from mitmproxy import tcp +from mitmproxy import udp +from mitmproxy import websocket from mitmproxy.connection import Address from mitmproxy.flow import Flow -from mitmproxy.proxy import events, mode_specs, server_hooks +from mitmproxy.proxy import events +from mitmproxy.proxy import mode_specs +from mitmproxy.proxy import server_hooks from mitmproxy.proxy.layers.tcp import TcpMessageInjected from mitmproxy.proxy.layers.udp import UdpMessageInjected from mitmproxy.proxy.layers.websocket import WebSocketMessageInjected -from mitmproxy.proxy.mode_servers import ProxyConnectionHandler, ServerInstance, ServerManager -from mitmproxy.utils import human, signals +from mitmproxy.proxy.mode_servers import ProxyConnectionHandler +from mitmproxy.proxy.mode_servers import ServerInstance +from mitmproxy.proxy.mode_servers import ServerManager +from mitmproxy.utils import human +from mitmproxy.utils import signals logger = logging.getLogger(__name__) @@ -64,7 +69,8 @@ async def update(self, modes: Iterable[mode_specs.ProxyMode]) -> bool: # Shutdown modes that have been removed from the list. stop_tasks = [ - s.stop() for spec, s in self._instances.items() + s.stop() + for spec, s in self._instances.items() if spec not in new_instances ] @@ -101,6 +107,7 @@ class Proxyserver(ServerManager): """ This addon runs the actual proxy server. """ + connections: dict[tuple, ProxyConnectionHandler] servers: Servers @@ -116,7 +123,9 @@ def __repr__(self): return f"Proxyserver({len(self.connections)} active conns)" @contextmanager - def register_connection(self, connection_id: tuple, handler: ProxyConnectionHandler): + def register_connection( + self, connection_id: tuple, handler: ProxyConnectionHandler + ): self.connections[connection_id] = handler try: yield @@ -217,7 +226,10 @@ def configure(self, updated) -> None: if "connect_addr" in updated: try: if ctx.options.connect_addr: - self._connect_addr = str(ipaddress.ip_address(ctx.options.connect_addr)), 0 + self._connect_addr = ( + str(ipaddress.ip_address(ctx.options.connect_addr)), + 0, + ) else: self._connect_addr = None except ValueError: @@ -229,25 +241,27 @@ def configure(self, updated) -> None: modes: list[mode_specs.ProxyMode] = [] for mode in ctx.options.mode: try: - modes.append( - mode_specs.ProxyMode.parse(mode) - ) + modes.append(mode_specs.ProxyMode.parse(mode)) except ValueError as e: - raise exceptions.OptionsError(f"Invalid proxy mode specification: {mode} ({e})") + raise exceptions.OptionsError( + f"Invalid proxy mode specification: {mode} ({e})" + ) # ...and don't listen on the same address. listen_addrs = [ ( m.listen_host(ctx.options.listen_host), m.listen_port(ctx.options.listen_port), - m.transport_protocol + m.transport_protocol, ) for m in modes ] if len(set(listen_addrs)) != len(listen_addrs): (host, port, _) = collections.Counter(listen_addrs).most_common(1)[0][0] dup_addr = human.format_address((host or "0.0.0.0", port)) - raise exceptions.OptionsError(f"Cannot spawn multiple servers on the same address: {dup_addr}") + raise exceptions.OptionsError( + f"Cannot spawn multiple servers on the same address: {dup_addr}" + ) if ctx.options.mode and not ctx.master.addons.get("nextlayer"): logger.warning("Warning: Running proxyserver without nextlayer addon!") @@ -255,20 +269,20 @@ def configure(self, updated) -> None: if platform.original_addr: platform.init_transparent_mode() else: - raise exceptions.OptionsError("Transparent mode not supported on this platform.") + raise exceptions.OptionsError( + "Transparent mode not supported on this platform." + ) if self.is_running: asyncio.create_task(self.servers.update(modes)) async def setup_servers(self) -> bool: - return await self.servers.update([mode_specs.ProxyMode.parse(m) for m in ctx.options.mode]) + return await self.servers.update( + [mode_specs.ProxyMode.parse(m) for m in ctx.options.mode] + ) def listen_addrs(self) -> list[Address]: - return [ - addr - for server in self.servers - for addr in server.listen_addrs - ] + return [addr for server in self.servers for addr in server.listen_addrs] def inject_event(self, event: events.MessageInjected): connection_id = ( @@ -330,12 +344,7 @@ def server_connect(self, data: server_hooks.ServerConnectionHookData): for listen_host, listen_port, *_ in server.listen_addrs: self_connect = ( connect_port == listen_port - and connect_host in ( - "localhost", - "127.0.0.1", - "::1", - listen_host - ) + and connect_host in ("localhost", "127.0.0.1", "::1", listen_host) and server.mode.transport_protocol == data.server.transport_protocol ) if self_connect: diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py index f54708659e..e9f0a5cbf7 100644 --- a/mitmproxy/addons/readfile.py +++ b/mitmproxy/addons/readfile.py @@ -2,13 +2,14 @@ import logging import os.path import sys -from typing import BinaryIO, Optional +from typing import BinaryIO +from typing import Optional +from mitmproxy import command from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import io -from mitmproxy import command class ReadFile: diff --git a/mitmproxy/addons/save.py b/mitmproxy/addons/save.py index 2a3db39b71..f2b0b2133c 100644 --- a/mitmproxy/addons/save.py +++ b/mitmproxy/addons/save.py @@ -5,10 +5,11 @@ from datetime import datetime from functools import lru_cache from pathlib import Path -from typing import Literal, Optional +from typing import Literal +from typing import Optional import mitmproxy.types -from mitmproxy import command, tcp, udp +from mitmproxy import command from mitmproxy import ctx from mitmproxy import dns from mitmproxy import exceptions @@ -16,6 +17,8 @@ from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import io +from mitmproxy import tcp +from mitmproxy import udp from mitmproxy.log import ALERT diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 12586d327c..3e5f0beff7 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -1,21 +1,22 @@ import asyncio +import importlib.machinery +import importlib.util import logging import os -import importlib.util -import importlib.machinery import sys -import types import traceback +import types from collections.abc import Sequence from typing import Optional -from mitmproxy import addonmanager, hooks -from mitmproxy import exceptions -from mitmproxy import flow +import mitmproxy.types as mtypes +from mitmproxy import addonmanager from mitmproxy import command -from mitmproxy import eventsequence from mitmproxy import ctx -import mitmproxy.types as mtypes +from mitmproxy import eventsequence +from mitmproxy import exceptions +from mitmproxy import flow +from mitmproxy import hooks from mitmproxy.utils import asyncio_utils logger = logging.getLogger(__name__) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 1973547c23..7419f8ac5f 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,14 +1,18 @@ import hashlib import logging import urllib -from collections.abc import Hashable, Sequence -from typing import Any, Optional +from collections.abc import Hashable +from collections.abc import Sequence +from typing import Any +from typing import Optional import mitmproxy.types -from mitmproxy import command, hooks -from mitmproxy import ctx, http +from mitmproxy import command +from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import flow +from mitmproxy import hooks +from mitmproxy import http from mitmproxy import io diff --git a/mitmproxy/addons/stickyauth.py b/mitmproxy/addons/stickyauth.py index 15d98c33d2..bd3b4e49d2 100644 --- a/mitmproxy/addons/stickyauth.py +++ b/mitmproxy/addons/stickyauth.py @@ -1,8 +1,8 @@ from typing import Optional +from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import flowfilter -from mitmproxy import ctx class StickyAuth: diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index ef33f5bc41..becaafa442 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -2,7 +2,10 @@ from http import cookiejar from typing import Optional -from mitmproxy import http, flowfilter, ctx, exceptions +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import flowfilter +from mitmproxy import http from mitmproxy.net.http import cookies TOrigin = tuple[str, int, str] @@ -31,7 +34,9 @@ def domain_match(a: str, b: str) -> bool: class StickyCookie: def __init__(self) -> None: - self.jar: collections.defaultdict[TOrigin, dict[str, str]] = collections.defaultdict(dict) + self.jar: collections.defaultdict[ + TOrigin, dict[str, str] + ] = collections.defaultdict(dict) self.flt: Optional[flowfilter.TFilter] = None def load(self, loader): diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index aa15ae39b4..9da47e141e 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -1,19 +1,17 @@ from __future__ import annotations + import asyncio import logging -from typing import IO - import sys +from typing import IO -from mitmproxy import ctx, log +from mitmproxy import ctx +from mitmproxy import log from mitmproxy.utils import vt_codes class TermLog: - def __init__( - self, - out: IO[str] | None = None - ): + def __init__(self, out: IO[str] | None = None): self.logger = TermLogHandler(out) self.logger.install() @@ -41,10 +39,7 @@ async def _teardown(self): class TermLogHandler(log.MitmLogHandler): - def __init__( - self, - out: IO[str] | None = None - ): + def __init__(self, out: IO[str] | None = None): super().__init__() self.file: IO[str] = out or sys.stdout self.has_vt_codes = vt_codes.ensure_supported(self.file) diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 40bbee735b..0d7faf52de 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -1,19 +1,28 @@ import ipaddress import logging import os -from pathlib import Path import ssl -from typing import Any, Optional, TypedDict +from pathlib import Path +from typing import Any +from typing import Optional +from typing import TypedDict from aioquic.h3.connection import H3_ALPN from aioquic.tls import CipherSuite -from OpenSSL import SSL, crypto -from mitmproxy import certs, ctx, exceptions, connection, tls +from OpenSSL import crypto +from OpenSSL import SSL + +from mitmproxy import certs +from mitmproxy import connection +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import tls from mitmproxy.net import tls as net_tls from mitmproxy.options import CONF_BASENAME from mitmproxy.proxy import context from mitmproxy.proxy.layers import modes -from mitmproxy.proxy.layers import tls as proxy_tls, quic +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tls as proxy_tls # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://ssl-config.mozilla.org/#config=old @@ -166,7 +175,9 @@ def tls_start_client(self, tls_start: tls.TlsData) -> None: extra_chain_certs = [] ssl_ctx = net_tls.create_client_proxy_context( - method=net_tls.Method.DTLS_SERVER_METHOD if tls_start.is_dtls else net_tls.Method.TLS_SERVER_METHOD, + method=net_tls.Method.DTLS_SERVER_METHOD + if tls_start.is_dtls + else net_tls.Method.TLS_SERVER_METHOD, min_version=net_tls.Version[ctx.options.tls_version_client_min], max_version=net_tls.Version[ctx.options.tls_version_client_max], cipher_list=tuple(cipher_list), @@ -179,7 +190,9 @@ def tls_start_client(self, tls_start: tls.TlsData) -> None: tls_start.ssl_conn = SSL.Connection(ssl_ctx) tls_start.ssl_conn.use_certificate(entry.cert.to_pyopenssl()) - tls_start.ssl_conn.use_privatekey(crypto.PKey.from_cryptography_key(entry.privatekey)) + tls_start.ssl_conn.use_privatekey( + crypto.PKey.from_cryptography_key(entry.privatekey) + ) # Force HTTP/1 for secure web proxies, we currently don't support CONNECT over HTTP/2. # There is a proof-of-concept branch at https://github.com/mhils/mitmproxy/tree/http2-proxy, @@ -256,7 +269,9 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: client_cert = p ssl_ctx = net_tls.create_proxy_server_context( - method=net_tls.Method.DTLS_CLIENT_METHOD if tls_start.is_dtls else net_tls.Method.TLS_CLIENT_METHOD, + method=net_tls.Method.DTLS_CLIENT_METHOD + if tls_start.is_dtls + else net_tls.Method.TLS_CLIENT_METHOD, min_version=net_tls.Version[ctx.options.tls_version_server_min], max_version=net_tls.Version[ctx.options.tls_version_server_max], cipher_list=tuple(cipher_list), @@ -328,9 +343,8 @@ def quic_start_client(self, tls_start: quic.QuicTlsData) -> None: # if we don't have upstream ALPN, we allow all offered by the client tls_start.settings.alpn_protocols = [ alpn.decode("ascii") - for alpn in [ - alpn for alpn in (client.alpn, server.alpn) if alpn - ] or client.alpn_offers + for alpn in [alpn for alpn in (client.alpn, server.alpn) if alpn] + or client.alpn_offers ] # set the certificates diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py index da9b395348..63b0d32bb0 100644 --- a/mitmproxy/addons/upstream_auth.py +++ b/mitmproxy/addons/upstream_auth.py @@ -1,9 +1,9 @@ -import re import base64 +import re from typing import Optional -from mitmproxy import exceptions from mitmproxy import ctx +from mitmproxy import exceptions from mitmproxy import http from mitmproxy.proxy import mode_specs from mitmproxy.utils import strutils @@ -53,7 +53,10 @@ def http_connect_upstream(self, f: http.HTTPFlow): def requestheaders(self, f: http.HTTPFlow): if self.auth: - if isinstance(f.client_conn.proxy_mode, mode_specs.UpstreamMode) and f.request.scheme == "http": + if ( + isinstance(f.client_conn.proxy_mode, mode_specs.UpstreamMode) + and f.request.scheme == "http" + ): f.request.headers["Proxy-Authorization"] = self.auth elif isinstance(f.client_conn.proxy_mode, mode_specs.ReverseMode): f.request.headers["Authorization"] = self.auth diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 965d9fc483..06049715fe 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -11,25 +11,29 @@ import collections import logging import re -from collections.abc import Iterator, MutableMapping, Sequence -from typing import Any, Optional +from collections.abc import Iterator +from collections.abc import MutableMapping +from collections.abc import Sequence +from typing import Any +from typing import Optional import sortedcontainers import mitmproxy.flow from mitmproxy import command +from mitmproxy import connection from mitmproxy import ctx from mitmproxy import dns from mitmproxy import exceptions -from mitmproxy import hooks -from mitmproxy import connection from mitmproxy import flowfilter +from mitmproxy import hooks from mitmproxy import http from mitmproxy import io from mitmproxy import tcp from mitmproxy import udp from mitmproxy.log import ALERT -from mitmproxy.utils import human, signals +from mitmproxy.utils import human +from mitmproxy.utils import signals # The underlying sorted list implementation expects the sort key to be stable @@ -144,7 +148,9 @@ def _sig_view_remove(flow: mitmproxy.flow.Flow, index: int) -> None: class View(collections.abc.Sequence): def __init__(self) -> None: super().__init__() - self._store: collections.OrderedDict[str, mitmproxy.flow.Flow] = collections.OrderedDict() + self._store: collections.OrderedDict[ + str, mitmproxy.flow.Flow + ] = collections.OrderedDict() self.filter = flowfilter.match_all # Should we show only marked flows? self.show_marked = False @@ -475,7 +481,11 @@ def create(self, method: str, url: str) -> None: except ValueError as e: raise exceptions.CommandError("Invalid URL: %s" % e) - c = connection.Client(peername=("", 0), sockname=("", 0), timestamp_start=req.timestamp_start - 0.0001) + c = connection.Client( + peername=("", 0), + sockname=("", 0), + timestamp_start=req.timestamp_start - 0.0001, + ) s = connection.Server(address=(req.host, req.port)) f = http.HTTPFlow(c, s) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 70d847075f..9b0a824894 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -6,15 +6,21 @@ import sys from dataclasses import dataclass from pathlib import Path -from typing import NewType, Optional, Union +from typing import NewType +from typing import Optional +from typing import Union +import OpenSSL from cryptography import x509 -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa, dsa, ec +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import dsa +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import pkcs12 -from cryptography.x509 import NameOID, ExtendedKeyUsageOID +from cryptography.x509 import ExtendedKeyUsageOID +from cryptography.x509 import NameOID -import OpenSSL from mitmproxy.coretypes import serializable # Default expiry must not be too long: https://github.com/mitmproxy/mitmproxy/issues/815 @@ -315,7 +321,10 @@ def __init__( self.default_chain_certs = ( [ Cert.from_pem(chunk) - for chunk in re.split(rb"(?=-----BEGIN( [A-Z]+)+-----)", self.default_chain_file.read_bytes()) + for chunk in re.split( + rb"(?=-----BEGIN( [A-Z]+)+-----)", + self.default_chain_file.read_bytes(), + ) if chunk.startswith(b"-----BEGIN CERTIFICATE-----") ] if self.default_chain_file diff --git a/mitmproxy/command.py b/mitmproxy/command.py index 950fa44ef3..f62b93c221 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -4,16 +4,21 @@ import functools import inspect import logging - -import pyparsing import sys import textwrap import types -from collections.abc import Sequence, Callable, Iterable -from typing import Any, NamedTuple, Optional +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Sequence +from typing import Any +from typing import NamedTuple +from typing import Optional + +import pyparsing import mitmproxy.types -from mitmproxy import exceptions, command_lexer +from mitmproxy import command_lexer +from mitmproxy import exceptions from mitmproxy.command_lexer import unquote @@ -195,7 +200,9 @@ def parse_partial( Parse a possibly partial command. Return a sequence of ParseResults and a sequence of remainder type help items. """ - parts: pyparsing.ParseResults = command_lexer.expr.parseString(cmdstr, parseAll=True) + parts: pyparsing.ParseResults = command_lexer.expr.parseString( + cmdstr, parseAll=True + ) parsed: list[ParseResult] = [] next_params: list[CommandParameter] = [ diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index 20782057ad..e0e748ff0c 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -1,18 +1,20 @@ import dataclasses import sys import time -from dataclasses import dataclass, field import uuid import warnings from abc import ABCMeta from collections.abc import Sequence +from dataclasses import dataclass +from dataclasses import field from enum import Flag -from typing import Literal, Optional +from typing import Literal +from typing import Optional from mitmproxy import certs from mitmproxy.coretypes import serializable -from mitmproxy.proxy import mode_specs from mitmproxy.net import server_spec +from mitmproxy.proxy import mode_specs from mitmproxy.utils import human @@ -48,12 +50,15 @@ class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): The connection object only exposes metadata about the connection, but not the underlying socket object. This is intentional, all I/O should be handled by `mitmproxy.proxy.server` exclusively. """ + peername: Optional[Address] """The remote's `(ip, port)` tuple for this connection.""" sockname: Optional[Address] """Our local `(ip, port)` tuple for this connection.""" - state: ConnectionState = field(default=ConnectionState.CLOSED, metadata={"serialize": False}) + state: ConnectionState = field( + default=ConnectionState.CLOSED, metadata={"serialize": False} + ) """The current connection state.""" # all connections have a unique id. While @@ -177,7 +182,9 @@ class Client(Connection): The certificate used by mitmproxy to establish TLS with the client. """ - proxy_mode: mode_specs.ProxyMode = field(default=mode_specs.ProxyMode.parse("regular")) + proxy_mode: mode_specs.ProxyMode = field( + default=mode_specs.ProxyMode.parse("regular") + ) """The proxy server type this client has been connecting to.""" timestamp_start: float = field(default_factory=time.time) diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 7c62b1f6f4..2949c8fa3f 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -12,37 +12,41 @@ `base.View`. """ import traceback -from typing import Union from typing import Optional +from typing import Union -from mitmproxy import flow, tcp, udp -from mitmproxy import http -from mitmproxy.utils import signals, strutils -from . import ( - auto, - raw, - hex, - json, - xml_html, - wbxml, - javascript, - css, - urlencoded, - multipart, - image, - query, - protobuf, - msgpack, - graphql, - grpc, - mqtt, - http3, -) - -from .base import View, KEY_MAX, format_text, format_dict, TViewResult +from . import auto +from . import css +from . import graphql +from . import grpc +from . import hex +from . import http3 +from . import image +from . import javascript +from . import json +from . import mqtt +from . import msgpack +from . import multipart +from . import protobuf +from . import query +from . import raw +from . import urlencoded +from . import wbxml +from . import xml_html from ..tcp import TCPMessage from ..udp import UDPMessage from ..websocket import WebSocketMessage +from .base import format_dict +from .base import format_text +from .base import KEY_MAX +from .base import TViewResult +from .base import View +from mitmproxy import flow +from mitmproxy import http +from mitmproxy import tcp +from mitmproxy import udp +from mitmproxy.utils import signals +from mitmproxy.utils import strutils views: list[View] = [] diff --git a/mitmproxy/contentviews/auto.py b/mitmproxy/contentviews/auto.py index d86dcf8108..e8acfd94e3 100644 --- a/mitmproxy/contentviews/auto.py +++ b/mitmproxy/contentviews/auto.py @@ -1,5 +1,5 @@ -from mitmproxy import contentviews from . import base +from mitmproxy import contentviews class ViewAuto(base.View): diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index d8baa9f25a..9788eb6888 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -1,7 +1,12 @@ # Default view cutoff *in lines* -from abc import ABC, abstractmethod -from collections.abc import Iterable, Iterator, Mapping -from typing import ClassVar, Optional, Union +from abc import ABC +from abc import abstractmethod +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from typing import ClassVar +from typing import Optional +from typing import Union from mitmproxy import flow from mitmproxy import http diff --git a/mitmproxy/contentviews/graphql.py b/mitmproxy/contentviews/graphql.py index c179828e87..198082e807 100644 --- a/mitmproxy/contentviews/graphql.py +++ b/mitmproxy/contentviews/graphql.py @@ -1,8 +1,10 @@ import json -from typing import Any, Optional +from typing import Any +from typing import Optional from mitmproxy.contentviews import base -from mitmproxy.contentviews.json import parse_json, PARSE_ERROR +from mitmproxy.contentviews.json import PARSE_ERROR +from mitmproxy.contentviews.json import parse_json def format_graphql(data): diff --git a/mitmproxy/contentviews/grpc.py b/mitmproxy/contentviews/grpc.py index 332310af27..faa60079e2 100644 --- a/mitmproxy/contentviews/grpc.py +++ b/mitmproxy/contentviews/grpc.py @@ -2,11 +2,17 @@ import logging import struct -from dataclasses import dataclass, field +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Iterator +from dataclasses import dataclass +from dataclasses import field from enum import Enum -from typing import Generator, Iterable, Iterator -from mitmproxy import contentviews, flow, flowfilter, http +from mitmproxy import contentviews +from mitmproxy import flow +from mitmproxy import flowfilter +from mitmproxy import http from mitmproxy.contentviews import base from mitmproxy.net.encoding import decode @@ -259,7 +265,9 @@ def read_packed_fields( packed_field: ProtoParser.Field, ) -> list[ProtoParser.Field]: if not isinstance(packed_field.wire_value, bytes): - raise ValueError(f"can not unpack field with data other than bytes: {type(packed_field.wire_value)}") + raise ValueError( + f"can not unpack field with data other than bytes: {type(packed_field.wire_value)}" + ) wire_data: bytes = packed_field.wire_value tag: int = packed_field.tag options: ProtoParser.ParserOptions = packed_field.options @@ -953,7 +961,9 @@ def format_grpc( @dataclass class ViewConfig: - parser_options: ProtoParser.ParserOptions = field(default_factory=ProtoParser.ParserOptions) + parser_options: ProtoParser.ParserOptions = field( + default_factory=ProtoParser.ParserOptions + ) parser_rules: list[ProtoParser.ParserRule] = field(default_factory=list) diff --git a/mitmproxy/contentviews/hex.py b/mitmproxy/contentviews/hex.py index 5b53202c6b..c5079929c5 100644 --- a/mitmproxy/contentviews/hex.py +++ b/mitmproxy/contentviews/hex.py @@ -1,5 +1,5 @@ -from mitmproxy.utils import strutils from . import base +from mitmproxy.utils import strutils class ViewHex(base.View): diff --git a/mitmproxy/contentviews/http3.py b/mitmproxy/contentviews/http3.py index df014b99db..4237720342 100644 --- a/mitmproxy/contentviews/http3.py +++ b/mitmproxy/contentviews/http3.py @@ -1,22 +1,27 @@ from collections import defaultdict from collections.abc import Iterator -from dataclasses import dataclass, field -from typing import Optional, Union +from dataclasses import dataclass +from dataclasses import field +from typing import Optional +from typing import Union -from aioquic.h3.connection import Setting, parse_settings +import pylsqpack +from aioquic.buffer import Buffer +from aioquic.buffer import BufferReadError +from aioquic.h3.connection import parse_settings +from aioquic.h3.connection import Setting -from mitmproxy import flow, tcp from . import base -from .hex import ViewHex from ..proxy.layers.http import is_h3_alpn - -from aioquic.buffer import Buffer, BufferReadError -import pylsqpack +from .hex import ViewHex +from mitmproxy import flow +from mitmproxy import tcp @dataclass(frozen=True) class Frame: """Representation of an HTTP/3 frame.""" + type: int data: bytes @@ -27,10 +32,7 @@ def pretty(self): elif self.type == 1: try: hdrs = pylsqpack.Decoder(4096, 16).feed_header(0, self.data)[1] - return [ - [("header", "HEADERS Frame")], - *base.format_pairs(hdrs) - ] + return [[("header", "HEADERS Frame")], *base.format_pairs(hdrs)] except Exception as e: frame_name = f"HEADERS Frame (error: {e})" elif self.type == 4: @@ -46,10 +48,7 @@ def pretty(self): except ValueError: key = f"0x{k:x}" settings.append((key, f"0x{v:x}")) - return [ - [("header", "SETTINGS Frame")], - *base.format_pairs(settings) - ] + return [[("header", "SETTINGS Frame")], *base.format_pairs(settings)] return [ [("header", frame_name)], *ViewHex._format(self.data), @@ -59,6 +58,7 @@ def pretty(self): @dataclass(frozen=True) class StreamType: """Representation of an HTTP/3 stream types.""" + type: int def pretty(self): @@ -68,9 +68,7 @@ def pretty(self): 0x02: "QPACK Encoder Stream", 0x03: "QPACK Decoder Stream", }.get(self.type, f"0x{self.type:x} Stream") - return [ - [("header", stream_type)] - ] + return [[("header", stream_type)]] @dataclass @@ -85,21 +83,23 @@ class ViewHttp3(base.View): name = "HTTP/3 Frames" def __init__(self) -> None: - self.connections: defaultdict[tcp.TCPFlow, ConnectionState] = defaultdict(ConnectionState) + self.connections: defaultdict[tcp.TCPFlow, ConnectionState] = defaultdict( + ConnectionState + ) def __call__( self, data, flow: Optional[flow.Flow] = None, tcp_message: Optional[tcp.TCPMessage] = None, - **metadata + **metadata, ): assert isinstance(flow, tcp.TCPFlow) assert tcp_message state = self.connections[flow] - for message in flow.messages[state.message_count:]: + for message in flow.messages[state.message_count :]: if message.from_client: buf = state.client_buf else: @@ -111,9 +111,7 @@ def __call__( stream_type = h3_buf.pull_uint_var() consumed = h3_buf.tell() del buf[:consumed] - state.frames[0] = [ - StreamType(stream_type) - ] + state.frames[0] = [StreamType(stream_type)] while True: h3_buf = Buffer(data=bytes(buf[:16])) @@ -128,29 +126,33 @@ def __call__( if len(buf) < consumed + frame_size: break - frame_data = bytes(buf[consumed:consumed + frame_size]) + frame_data = bytes(buf[consumed : consumed + frame_size]) frame = Frame(frame_type, frame_data) state.frames.setdefault(state.message_count, []).append(frame) - del buf[:consumed + frame_size] + del buf[: consumed + frame_size] state.message_count += 1 frames = state.frames.get(flow.messages.index(tcp_message), []) if not frames: - return "HTTP/3", [] # base.format_text(f"(no complete frames here, {state=})") + return ( + "HTTP/3", + [], + ) # base.format_text(f"(no complete frames here, {state=})") else: return "HTTP/3", fmt_frames(frames) def render_priority( - self, - data: bytes, - flow: Optional[flow.Flow] = None, - **metadata + self, data: bytes, flow: Optional[flow.Flow] = None, **metadata ) -> float: - return 2 * float(bool(flow and is_h3_alpn(flow.client_conn.alpn))) * float(isinstance(flow, tcp.TCPFlow)) + return ( + 2 + * float(bool(flow and is_h3_alpn(flow.client_conn.alpn))) + * float(isinstance(flow, tcp.TCPFlow)) + ) def fmt_frames(frames: list[Union[Frame, StreamType]]) -> Iterator[base.TViewLine]: diff --git a/mitmproxy/contentviews/image/view.py b/mitmproxy/contentviews/image/view.py index a414a1a7f0..5d621133fa 100644 --- a/mitmproxy/contentviews/image/view.py +++ b/mitmproxy/contentviews/image/view.py @@ -1,9 +1,9 @@ import imghdr from typing import Optional +from . import image_parser from mitmproxy.contentviews import base from mitmproxy.coretypes import multidict -from . import image_parser def test_ico(h, f): diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/javascript.py index de04668386..33ecac2ae9 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/javascript.py @@ -2,8 +2,8 @@ import re from typing import Optional -from mitmproxy.utils import strutils from mitmproxy.contentviews import base +from mitmproxy.utils import strutils DELIMITERS = "{};\n" SPECIAL_AREAS = ( diff --git a/mitmproxy/contentviews/json.py b/mitmproxy/contentviews/json.py index d8952e80bc..23ec86a0d9 100644 --- a/mitmproxy/contentviews/json.py +++ b/mitmproxy/contentviews/json.py @@ -1,8 +1,9 @@ -import re import json +import re from collections.abc import Iterator from functools import lru_cache -from typing import Any, Optional +from typing import Any +from typing import Optional from mitmproxy.contentviews import base @@ -28,7 +29,11 @@ def format_json(data: Any) -> Iterator[base.TViewLine]: yield current_line current_line = [] if re.match(r'\s*"', chunk): - if len(current_line) == 1 and current_line[0][0] == "text" and current_line[0][1].isspace(): + if ( + len(current_line) == 1 + and current_line[0][0] == "text" + and current_line[0][1].isspace() + ): current_line.append(("Token_Name_Tag", chunk)) else: current_line.append(("Token_Literal_String", chunk)) diff --git a/mitmproxy/contentviews/mqtt.py b/mitmproxy/contentviews/mqtt.py index 1b870341c8..1c3b92a37f 100644 --- a/mitmproxy/contentviews/mqtt.py +++ b/mitmproxy/contentviews/mqtt.py @@ -1,10 +1,9 @@ +import struct from typing import Optional from mitmproxy.contentviews import base from mitmproxy.utils import strutils -import struct - # from https://github.com/nikitastupin/mitmproxy-mqtt-script @@ -211,9 +210,13 @@ def _parse_connect_payload(self): self.payload["WillTopic"] = f.decode("utf-8") elif self.connect_flags["Will"] and "WillMessage" not in self.payload: self.payload["WillMessage"] = f - elif self.connect_flags["UserName"] and "UserName" not in self.payload: # pragma: no cover + elif ( + self.connect_flags["UserName"] and "UserName" not in self.payload + ): # pragma: no cover self.payload["UserName"] = f.decode("utf-8") - elif self.connect_flags["Password"] and "Password" not in self.payload: # pragma: no cover + elif ( + self.connect_flags["Password"] and "Password" not in self.payload + ): # pragma: no cover self.payload["Password"] = f else: raise AssertionError(f"Unknown field in CONNECT payload: {f}") diff --git a/mitmproxy/contentviews/msgpack.py b/mitmproxy/contentviews/msgpack.py index 7e845bd118..92aeb8b39c 100644 --- a/mitmproxy/contentviews/msgpack.py +++ b/mitmproxy/contentviews/msgpack.py @@ -1,8 +1,8 @@ -from typing import Any, Optional +from typing import Any +from typing import Optional import msgpack - from mitmproxy.contentviews import base PARSE_ERROR = object() @@ -15,14 +15,16 @@ def parse_msgpack(s: bytes) -> Any: return PARSE_ERROR -def format_msgpack(data: Any, output = None, indent_count: int = 0) -> list[base.TViewLine]: +def format_msgpack( + data: Any, output=None, indent_count: int = 0 +) -> list[base.TViewLine]: if output is None: output = [[]] indent = ("text", " " * indent_count) if type(data) is str: - token = [("Token_Literal_String", f"\"{data}\"")] + token = [("Token_Literal_String", f'"{data}"')] output[-1] += token # Need to return if single value, but return is discarded in dict/list loop @@ -43,7 +45,14 @@ def format_msgpack(data: Any, output = None, indent_count: int = 0) -> list[base elif type(data) is dict: output[-1] += [("text", "{")] for key in data: - output.append([indent, ("text", " "), ("Token_Name_Tag", f'"{key}"'), ("text", ": ")]) + output.append( + [ + indent, + ("text", " "), + ("Token_Name_Tag", f'"{key}"'), + ("text", ": "), + ] + ) format_msgpack(data[key], output, indent_count + 1) if key != list(data)[-1]: diff --git a/mitmproxy/contentviews/multipart.py b/mitmproxy/contentviews/multipart.py index 9485824ca0..a8cef5f660 100644 --- a/mitmproxy/contentviews/multipart.py +++ b/mitmproxy/contentviews/multipart.py @@ -1,8 +1,8 @@ from typing import Optional +from . import base from mitmproxy.coretypes import multidict from mitmproxy.net.http import multipart -from . import base class ViewMultipart(base.View): diff --git a/mitmproxy/contentviews/protobuf.py b/mitmproxy/contentviews/protobuf.py index 50d349eb59..0d836be102 100644 --- a/mitmproxy/contentviews/protobuf.py +++ b/mitmproxy/contentviews/protobuf.py @@ -2,6 +2,7 @@ from typing import Optional from kaitaistruct import KaitaiStream + from . import base from mitmproxy.contrib.kaitaistruct import google_protobuf @@ -26,7 +27,9 @@ def _parse_proto(raw: bytes) -> list[google_protobuf.GoogleProtobuf.Pair]: """Parse a bytestring into protobuf pairs and make sure that all pairs have a valid wire type.""" buf = google_protobuf.GoogleProtobuf(KaitaiStream(io.BytesIO(raw))) for pair in buf.pairs: - if not isinstance(pair.wire_type, google_protobuf.GoogleProtobuf.Pair.WireTypes): + if not isinstance( + pair.wire_type, google_protobuf.GoogleProtobuf.Pair.WireTypes + ): raise ValueError("Not a protobuf.") return buf.pairs diff --git a/mitmproxy/contentviews/raw.py b/mitmproxy/contentviews/raw.py index a0b0884ecb..c19872534b 100644 --- a/mitmproxy/contentviews/raw.py +++ b/mitmproxy/contentviews/raw.py @@ -1,5 +1,5 @@ -from mitmproxy.utils import strutils from . import base +from mitmproxy.utils import strutils class ViewRaw(base.View): diff --git a/mitmproxy/contentviews/urlencoded.py b/mitmproxy/contentviews/urlencoded.py index 2988d85271..27e4e83d2f 100644 --- a/mitmproxy/contentviews/urlencoded.py +++ b/mitmproxy/contentviews/urlencoded.py @@ -1,7 +1,7 @@ from typing import Optional -from mitmproxy.net.http import url from . import base +from mitmproxy.net.http import url class ViewURLEncoded(base.View): diff --git a/mitmproxy/contentviews/wbxml.py b/mitmproxy/contentviews/wbxml.py index 4cd7fda89b..3faaa86d29 100644 --- a/mitmproxy/contentviews/wbxml.py +++ b/mitmproxy/contentviews/wbxml.py @@ -1,7 +1,7 @@ from typing import Optional -from mitmproxy.contrib.wbxml import ASCommandResponse from . import base +from mitmproxy.contrib.wbxml import ASCommandResponse class ViewWBXML(base.View): diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index b8e8f05f6f..747b5b9c39 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -1,10 +1,12 @@ import io import re import textwrap -from typing import Iterable, Optional +from collections.abc import Iterable +from typing import Optional from mitmproxy.contentviews import base -from mitmproxy.utils import sliding_window, strutils +from mitmproxy.utils import sliding_window +from mitmproxy.utils import strutils """ A custom XML/HTML prettifier. Compared to other prettifiers, its main features are: @@ -46,7 +48,7 @@ def __init__(self, data): self.data = data def __repr__(self): - return "{}({})".format(type(self).__name__, self.data) + return f"{type(self).__name__}({self.data})" class Text(Token): diff --git a/mitmproxy/coretypes/multidict.py b/mitmproxy/coretypes/multidict.py index 6710346e62..8a90c7327b 100644 --- a/mitmproxy/coretypes/multidict.py +++ b/mitmproxy/coretypes/multidict.py @@ -1,6 +1,8 @@ from abc import ABCMeta from abc import abstractmethod -from collections.abc import Iterator, MutableMapping, Sequence +from collections.abc import Iterator +from collections.abc import MutableMapping +from collections.abc import Sequence from typing import TypeVar from mitmproxy.coretypes import serializable diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index 8bfd959d15..f91d202c8b 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -10,8 +10,10 @@ try: from types import UnionType, NoneType except ImportError: # pragma: no cover + class UnionType: # type: ignore pass + NoneType = type(None) # type: ignore T = TypeVar("T", bound="Serializable") @@ -59,7 +61,6 @@ def copy(self: T) -> T: class SerializableDataclass(Serializable): - @classmethod @cache def __fields(cls) -> tuple[dataclasses.Field, ...]: @@ -111,7 +112,9 @@ def set_state(self, state: State) -> None: raise if state: - raise ValueError(f"Unexpected fields in {type(self).__name__}.set_state: {state}") + raise ValueError( + f"Unexpected fields in {type(self).__name__}.set_state: {state}" + ) V = TypeVar("V") @@ -121,11 +124,15 @@ def _process(attr_val: typing.Any, attr_type: type[V], attr_name: str, make: boo origin = typing.get_origin(attr_type) if origin is typing.Literal: if attr_val not in typing.get_args(attr_type): - raise ValueError(f"Invalid value for {attr_name}: {attr_val!r} does not match any literal value.") + raise ValueError( + f"Invalid value for {attr_name}: {attr_val!r} does not match any literal value." + ) return attr_val if origin in (UnionType, typing.Union): attr_type, nt = typing.get_args(attr_type) - assert nt is NoneType, f"{attr_name}: only `x | None` union types are supported`" # noqa + assert ( + nt is NoneType + ), f"{attr_name}: only `x | None` union types are supported`" if attr_val is None: return None # type: ignore else: @@ -146,24 +153,32 @@ def _process(attr_val: typing.Any, attr_type: type[V], attr_name: str, make: boo # We don't have a good way to represent tuple[str,int] | tuple[str,int,int,int], so we do a dirty hack here. if attr_name in ("peername", "sockname"): return tuple( - _process(x, T, attr_name, make) for x, T in zip(attr_val, [str, int, int, int]) + _process(x, T, attr_name, make) + for x, T in zip(attr_val, [str, int, int, int]) ) # type: ignore Ts = typing.get_args(attr_type) if len(Ts) != len(attr_val): - raise ValueError(f"Invalid data for {attr_name}. Expected {Ts}, got {attr_val}.") + raise ValueError( + f"Invalid data for {attr_name}. Expected {Ts}, got {attr_val}." + ) return tuple(_process(x, T, attr_name, make) for T, x in zip(Ts, attr_val)) # type: ignore elif origin is dict: k_cls, v_cls = typing.get_args(attr_type) return { - _process(k, k_cls, attr_name, make): _process(v, v_cls, attr_name, make) for k, v in attr_val.items() + _process(k, k_cls, attr_name, make): _process(v, v_cls, attr_name, make) + for k, v in attr_val.items() } # type: ignore elif attr_type in (int, float): if not isinstance(attr_val, (int, float)): - raise ValueError(f"Invalid value for {attr_name}. Expected {attr_type}, got {attr_val} ({type(attr_val)}).") + raise ValueError( + f"Invalid value for {attr_name}. Expected {attr_type}, got {attr_val} ({type(attr_val)})." + ) return attr_type(attr_val) # type: ignore elif attr_type in (str, bytes, bool): if not isinstance(attr_val, attr_type): - raise ValueError(f"Invalid value for {attr_name}. Expected {attr_type}, got {attr_val} ({type(attr_val)}).") + raise ValueError( + f"Invalid value for {attr_name}. Expected {attr_type}, got {attr_val} ({type(attr_val)})." + ) return attr_type(attr_val) # type: ignore elif isinstance(attr_type, type) and issubclass(attr_type, enum.Enum): if make: diff --git a/mitmproxy/dns.py b/mitmproxy/dns.py index 8ccd48c05e..8d0e2879dc 100644 --- a/mitmproxy/dns.py +++ b/mitmproxy/dns.py @@ -1,15 +1,21 @@ from __future__ import annotations -from dataclasses import dataclass + import itertools import random import struct -from ipaddress import IPv4Address, IPv6Address import time +from dataclasses import dataclass +from ipaddress import IPv4Address +from ipaddress import IPv6Address from typing import ClassVar from mitmproxy import flow from mitmproxy.coretypes import serializable -from mitmproxy.net.dns import classes, domain_names, op_codes, response_codes, types +from mitmproxy.net.dns import classes +from mitmproxy.net.dns import domain_names +from mitmproxy.net.dns import op_codes +from mitmproxy.net.dns import response_codes +from mitmproxy.net.dns import types # DNS parameters taken from https://www.iana.org/assignments/dns-parameters/dns-parameters.xml diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index b00feaa34f..57cf241d05 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -1,4 +1,6 @@ -from typing import Any, Callable, Iterator +from collections.abc import Iterator +from typing import Any +from typing import Callable from mitmproxy import dns from mitmproxy import flow diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 69514f2981..889e3ae445 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -1,10 +1,14 @@ from __future__ import annotations + import asyncio import copy import time import uuid -from dataclasses import dataclass, field -from typing import Any, ClassVar, Optional +from dataclasses import dataclass +from dataclasses import field +from typing import Any +from typing import ClassVar +from typing import Optional from mitmproxy import connection from mitmproxy import exceptions @@ -128,7 +132,9 @@ def __init__( __types: dict[str, type[Flow]] = {} - type: ClassVar[str] # automatically derived from the class name in __init_subclass__ + type: ClassVar[ + str + ] # automatically derived from the class name in __init_subclass__ """The flow type, for example `http`, `tcp`, or `dns`.""" def __init_subclass__(cls, **kwargs): diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index aaec9e1f5f..a596e288ad 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -32,15 +32,21 @@ ~c CODE Response code. rex Equivalent to ~u rex """ - import functools import re import sys from collections.abc import Sequence -from typing import ClassVar, Protocol, Union +from typing import ClassVar +from typing import Protocol +from typing import Union + import pyparsing as pp -from mitmproxy import dns, flow, http, tcp, udp +from mitmproxy import dns +from mitmproxy import flow +from mitmproxy import http +from mitmproxy import tcp +from mitmproxy import udp def only(*types): @@ -288,10 +294,16 @@ class FBod(_Rex): @only(http.HTTPFlow, tcp.TCPFlow, udp.UDPFlow, dns.DNSFlow) def __call__(self, f): if isinstance(f, http.HTTPFlow): - if f.request and (content := f.request.get_content(strict=False)) is not None: + if ( + f.request + and (content := f.request.get_content(strict=False)) is not None + ): if self.re.search(content): return True - if f.response and (content := f.response.get_content(strict=False)) is not None: + if ( + f.response + and (content := f.response.get_content(strict=False)) is not None + ): if self.re.search(content): return True if f.websocket: @@ -318,7 +330,10 @@ class FBodRequest(_Rex): @only(http.HTTPFlow, tcp.TCPFlow, udp.UDPFlow, dns.DNSFlow) def __call__(self, f): if isinstance(f, http.HTTPFlow): - if f.request and (content := f.request.get_content(strict=False)) is not None: + if ( + f.request + and (content := f.request.get_content(strict=False)) is not None + ): if self.re.search(content): return True if f.websocket: @@ -342,7 +357,10 @@ class FBodResponse(_Rex): @only(http.HTTPFlow, tcp.TCPFlow, udp.UDPFlow, dns.DNSFlow) def __call__(self, f): if isinstance(f, http.HTTPFlow): - if f.response and (content := f.response.get_content(strict=False)) is not None: + if ( + f.response + and (content := f.response.get_content(strict=False)) is not None + ): if self.re.search(content): return True if f.websocket: diff --git a/mitmproxy/hooks.py b/mitmproxy/hooks.py index 2c6c8574e8..4acb926691 100644 --- a/mitmproxy/hooks.py +++ b/mitmproxy/hooks.py @@ -1,8 +1,12 @@ import re import warnings from collections.abc import Sequence -from dataclasses import dataclass, is_dataclass, fields -from typing import Any, ClassVar, TYPE_CHECKING +from dataclasses import dataclass +from dataclasses import fields +from dataclasses import is_dataclass +from typing import Any +from typing import ClassVar +from typing import TYPE_CHECKING import mitmproxy.flow diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 6409480446..f9f32beb4c 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -1,26 +1,25 @@ import binascii +import json import os import re import time import urllib.parse -import json import warnings +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping from dataclasses import dataclass from dataclasses import fields from email.utils import formatdate from email.utils import mktime_tz from email.utils import parsedate_tz +from typing import Any from typing import Callable -from typing import Iterable -from typing import Iterator -from typing import Mapping +from typing import cast from typing import Optional from typing import Union -from typing import cast -from typing import Any from mitmproxy import flow -from mitmproxy.websocket import WebSocketData from mitmproxy.coretypes import multidict from mitmproxy.coretypes import serializable from mitmproxy.net import encoding @@ -35,6 +34,7 @@ from mitmproxy.utils import typecheck from mitmproxy.utils.strutils import always_bytes from mitmproxy.utils.strutils import always_str +from mitmproxy.websocket import WebSocketData # While headers _should_ be ASCII, it's not uncommon for certain headers to be utf-8 encoded. @@ -1262,7 +1262,9 @@ def get_state(self) -> serializable.State: def set_state(self, state: serializable.State) -> None: self.request = Request.from_state(state.pop("request")) self.response = Response.from_state(r) if (r := state.pop("response")) else None - self.websocket = WebSocketData.from_state(w) if (w := state.pop("websocket")) else None + self.websocket = ( + WebSocketData.from_state(w) if (w := state.pop("websocket")) else None + ) super().set_state(state) def __repr__(self): diff --git a/mitmproxy/io/__init__.py b/mitmproxy/io/__init__.py index 8d068d5699..541f743ab3 100644 --- a/mitmproxy/io/__init__.py +++ b/mitmproxy/io/__init__.py @@ -1,4 +1,7 @@ -from .io import FlowWriter, FlowReader, FilteredFlowWriter, read_flows_from_paths +from .io import FilteredFlowWriter +from .io import FlowReader +from .io import FlowWriter +from .io import read_flows_from_paths __all__ = ["FlowWriter", "FlowReader", "FilteredFlowWriter", "read_flows_from_paths"] diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 466c14e03d..9a923c67a9 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -7,7 +7,8 @@ """ import copy import uuid -from typing import Any, Union +from typing import Any +from typing import Union from mitmproxy import version from mitmproxy.utils import strutils @@ -406,7 +407,9 @@ def convert_18_19(data): for name in ["peername", "sockname", "address"]: if data[conn].get(name) and isinstance(data[conn][name][0], bytes): - data[conn][name][0] = data[conn][name][0].decode(errors="backslashreplace") + data[conn][name][0] = data[conn][name][0].decode( + errors="backslashreplace" + ) if data["server_conn"]["sni"] is True: data["server_conn"]["sni"] = data["server_conn"]["address"][0] diff --git a/mitmproxy/io/io.py b/mitmproxy/io/io.py index cd2b095bbf..f5957bf724 100644 --- a/mitmproxy/io/io.py +++ b/mitmproxy/io/io.py @@ -1,5 +1,9 @@ import os -from typing import Any, BinaryIO, Iterable, Union, cast +from collections.abc import Iterable +from typing import Any +from typing import BinaryIO +from typing import cast +from typing import Union from mitmproxy import exceptions from mitmproxy import flow diff --git a/mitmproxy/io/tnetstring.py b/mitmproxy/io/tnetstring.py index e08a729eba..b11580e9ec 100644 --- a/mitmproxy/io/tnetstring.py +++ b/mitmproxy/io/tnetstring.py @@ -39,9 +39,9 @@ :License: MIT """ - import collections -from typing import BinaryIO, Union +from typing import BinaryIO +from typing import Union TSerializable = Union[None, str, bool, int, float, bytes, list, tuple, dict] diff --git a/mitmproxy/log.py b/mitmproxy/log.py index 05b609c7d7..d56d5f29d9 100644 --- a/mitmproxy/log.py +++ b/mitmproxy/log.py @@ -1,10 +1,12 @@ from __future__ import annotations + import logging import os import warnings from dataclasses import dataclass -from mitmproxy import hooks, master +from mitmproxy import hooks +from mitmproxy import master from mitmproxy.contrib import click as miniclick from mitmproxy.utils import human @@ -42,7 +44,7 @@ def __init__(self, colorize: bool): self.without_client = f"{time} %s" default_time_format = "%H:%M:%S" - default_msec_format = '%s.%03d' + default_msec_format = "%s.%03d" def format(self, record: logging.LogRecord) -> str: time = self.formatTime(record) @@ -67,19 +69,18 @@ def __init__(self, *args, **kwargs): def filter(self, record: logging.LogRecord) -> bool: # We can't remove stale handlers here because that would modify .handlers during iteration! - return ( - super().filter(record) - and - ( - not self._initiated_in_test - or self._initiated_in_test == os.environ.get("PYTEST_CURRENT_TEST") - ) + return super().filter(record) and ( + not self._initiated_in_test + or self._initiated_in_test == os.environ.get("PYTEST_CURRENT_TEST") ) def install(self) -> None: if self._initiated_in_test: for h in list(logging.getLogger().handlers): - if isinstance(h, MitmLogHandler) and h._initiated_in_test != self._initiated_in_test: + if ( + isinstance(h, MitmLogHandler) + and h._initiated_in_test != self._initiated_in_test + ): h.uninstall() logging.getLogger().addHandler(self) @@ -90,6 +91,7 @@ def uninstall(self) -> None: # everything below is deprecated! + class LogEntry: def __init__(self, msg, level): # it's important that we serialize to string here already so that we don't pick up changes @@ -194,6 +196,7 @@ def __call__(self, text, level="info"): class LegacyLogEvents(MitmLogHandler): """Emit deprecated `add_log` events from stdlib logging.""" + def __init__( self, master: master.Master, diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 0947e6eb01..a8e6bfc345 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -3,14 +3,15 @@ import traceback from typing import Optional -from mitmproxy import addonmanager, hooks +from . import ctx as mitmproxy_ctx +from .proxy.mode_specs import ReverseMode +from mitmproxy import addonmanager from mitmproxy import command from mitmproxy import eventsequence +from mitmproxy import hooks from mitmproxy import http from mitmproxy import log from mitmproxy import options -from . import ctx as mitmproxy_ctx -from .proxy.mode_specs import ReverseMode logger = logging.getLogger(__name__) @@ -22,7 +23,11 @@ class Master: event_loop: asyncio.AbstractEventLoop - def __init__(self, opts: options.Options, event_loop: Optional[asyncio.AbstractEventLoop] = None): + def __init__( + self, + opts: options.Options, + event_loop: Optional[asyncio.AbstractEventLoop] = None, + ): self.options: options.Options = opts or options.Options() self.commands = command.CommandManager(self) self.addons = addonmanager.AddonManager(self) @@ -102,7 +107,11 @@ async def load_flow(self, f): Loads a flow """ - if isinstance(f, http.HTTPFlow) and len(self.options.mode) == 1 and self.options.mode[0].startswith("reverse:"): + if ( + isinstance(f, http.HTTPFlow) + and len(self.options.mode) == 1 + and self.options.mode[0].startswith("reverse:") + ): # When we load flows in reverse proxy mode, we adjust the target host to # the reverse proxy destination for all flows we load. This makes it very # easy to replay saved flows against a different host. diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py index 9a0bdec496..476170032f 100644 --- a/mitmproxy/net/check.py +++ b/mitmproxy/net/check.py @@ -1,11 +1,11 @@ import ipaddress import re +from typing import AnyStr # Allow underscore in host name # Note: This could be a DNS label, a hostname, a FQDN, or an IP -from typing import AnyStr -_label_valid = re.compile(br"[A-Z\d\-_]{1,63}$", re.IGNORECASE) +_label_valid = re.compile(rb"[A-Z\d\-_]{1,63}$", re.IGNORECASE) def is_valid_host(host: AnyStr) -> bool: diff --git a/mitmproxy/net/encoding.py b/mitmproxy/net/encoding.py index 32e61b62c6..29553651fe 100644 --- a/mitmproxy/net/encoding.py +++ b/mitmproxy/net/encoding.py @@ -1,13 +1,13 @@ """ Utility functions for decoding response bodies. """ - import codecs import collections import gzip import zlib from io import BytesIO -from typing import Union, overload +from typing import overload +from typing import Union import brotli import zstandard as zstd @@ -184,7 +184,7 @@ def decode_zstd(content: bytes) -> bytes: except zstd.ZstdError: # If the zstd stream is streamed without a size header, # try decoding with a 10MiB output buffer - return zstd_ctx.decompress(content, max_output_size=10 * 2 ** 20) + return zstd_ctx.decompress(content, max_output_size=10 * 2**20) def encode_zstd(content: bytes) -> bytes: diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 4b2ddd9413..3e961ae830 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -1,7 +1,7 @@ import email.utils import re import time -from typing import Iterable +from collections.abc import Iterable from mitmproxy.coretypes import multidict diff --git a/mitmproxy/net/http/headers.py b/mitmproxy/net/http/headers.py index e3c00994a7..6204040aac 100644 --- a/mitmproxy/net/http/headers.py +++ b/mitmproxy/net/http/headers.py @@ -33,4 +33,4 @@ def assemble_content_type(type, subtype, parameters): if not parameters: return f"{type}/{subtype}" params = "; ".join(f"{k}={v}" for k, v in parameters.items()) - return "{}/{}; {}".format(type, subtype, params) + return f"{type}/{subtype}; {params}" diff --git a/mitmproxy/net/http/http1/__init__.py b/mitmproxy/net/http/http1/__init__.py index 3049e02fb9..b9b6e071e6 100644 --- a/mitmproxy/net/http/http1/__init__.py +++ b/mitmproxy/net/http/http1/__init__.py @@ -1,17 +1,13 @@ -from .read import ( - read_request_head, - read_response_head, - connection_close, - expected_http_body_size, - validate_headers, -) -from .assemble import ( - assemble_request, - assemble_request_head, - assemble_response, - assemble_response_head, - assemble_body, -) +from .assemble import assemble_body +from .assemble import assemble_request +from .assemble import assemble_request_head +from .assemble import assemble_response +from .assemble import assemble_response_head +from .read import connection_close +from .read import expected_http_body_size +from .read import read_request_head +from .read import read_response_head +from .read import validate_headers __all__ = [ diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index 1da4583e1d..2986c489d4 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -1,8 +1,11 @@ import re import time -from typing import Iterable, Optional +from collections.abc import Iterable +from typing import Optional -from mitmproxy.http import Request, Headers, Response +from mitmproxy.http import Headers +from mitmproxy.http import Request +from mitmproxy.http import Response from mitmproxy.net.http import url @@ -214,7 +217,7 @@ def expected_http_body_size( def raise_if_http_version_unknown(http_version: bytes) -> None: - if not re.match(br"^HTTP/\d\.\d$", http_version): + if not re.match(rb"^HTTP/\d\.\d$", http_version): raise ValueError(f"Unknown HTTP version: {http_version!r}") diff --git a/mitmproxy/net/http/multipart.py b/mitmproxy/net/http/multipart.py index 0079995875..4685d80e01 100644 --- a/mitmproxy/net/http/multipart.py +++ b/mitmproxy/net/http/multipart.py @@ -56,7 +56,7 @@ def decode(content_type: Optional[str], content: bytes) -> list[tuple[bytes, byt except (KeyError, UnicodeError): return [] - rx = re.compile(br'\bname="([^"]+)"') + rx = re.compile(rb'\bname="([^"]+)"') r = [] if content is not None: for i in content.split(b"--" + boundary): diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index 274f229fb1..abc038abf4 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -1,16 +1,19 @@ from __future__ import annotations + import re import urllib.parse from collections.abc import Sequence -from typing import AnyStr, Optional +from typing import AnyStr +from typing import Optional from mitmproxy.net import check +from mitmproxy.net.check import is_valid_host +from mitmproxy.net.check import is_valid_port +from mitmproxy.utils.strutils import always_str # This regex extracts & splits the host header into host and port. # Handles the edge case of IPv6 addresses containing colons. # https://bugzilla.mozilla.org/show_bug.cgi?id=45891 -from mitmproxy.net.check import is_valid_host, is_valid_port -from mitmproxy.utils.strutils import always_str _authority_re = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") diff --git a/mitmproxy/net/http/user_agents.py b/mitmproxy/net/http/user_agents.py index 58aa21eab9..6a83f8fb79 100644 --- a/mitmproxy/net/http/user_agents.py +++ b/mitmproxy/net/http/user_agents.py @@ -2,9 +2,7 @@ A small collection of useful user-agent header strings. These should be kept reasonably current to reflect common usage. """ - # pylint: line-too-long - # A collection of (name, shortcut, string) tuples. UASTRINGS = [ diff --git a/mitmproxy/net/local_ip.py b/mitmproxy/net/local_ip.py index 27468c05c5..bc3087263c 100644 --- a/mitmproxy/net/local_ip.py +++ b/mitmproxy/net/local_ip.py @@ -1,4 +1,5 @@ from __future__ import annotations + import socket diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py index 945565f196..f0d5edd609 100644 --- a/mitmproxy/net/server_spec.py +++ b/mitmproxy/net/server_spec.py @@ -9,7 +9,7 @@ ServerSpec = tuple[ Literal["http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns", "quic"], - tuple[str, int] + tuple[str, int], ] server_spec_re = re.compile( @@ -45,7 +45,17 @@ def parse(server_spec: str, default_scheme: str) -> ServerSpec: scheme = m.group("scheme") else: scheme = default_scheme - if scheme not in ("http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns", "quic"): + if scheme not in ( + "http", + "https", + "http3", + "tls", + "dtls", + "tcp", + "udp", + "dns", + "quic", + ): raise ValueError(f"Invalid server scheme: {scheme}") host = m.group("host") diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index d87dc1a67c..59fd229e32 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -1,15 +1,18 @@ import os import threading +from collections.abc import Iterable from enum import Enum from functools import lru_cache from pathlib import Path -from typing import Any, BinaryIO, Callable, Iterable, Optional +from typing import Any +from typing import BinaryIO +from typing import Callable +from typing import Optional import certifi - +from OpenSSL import SSL from OpenSSL.crypto import X509 -from OpenSSL import SSL from mitmproxy import certs @@ -18,8 +21,8 @@ class Method(Enum): TLS_SERVER_METHOD = SSL.TLS_SERVER_METHOD TLS_CLIENT_METHOD = SSL.TLS_CLIENT_METHOD # Type-pyopenssl does not know about these DTLS constants. - DTLS_SERVER_METHOD = SSL.DTLS_SERVER_METHOD # type: ignore - DTLS_CLIENT_METHOD = SSL.DTLS_CLIENT_METHOD # type: ignore + DTLS_SERVER_METHOD = SSL.DTLS_SERVER_METHOD # type: ignore + DTLS_CLIENT_METHOD = SSL.DTLS_CLIENT_METHOD # type: ignore try: diff --git a/mitmproxy/net/udp.py b/mitmproxy/net/udp.py index 00565f6b4e..d51aee1239 100644 --- a/mitmproxy/net/udp.py +++ b/mitmproxy/net/udp.py @@ -3,7 +3,11 @@ import asyncio import logging import socket -from typing import Any, Callable, Optional, Union, cast +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional +from typing import Union from mitmproxy.connection import Address from mitmproxy.net import udp_wireguard @@ -183,7 +187,9 @@ def __init__( self._closed = None @property - def _protocol(self) -> DrainableDatagramProtocol | udp_wireguard.WireGuardDatagramTransport: + def _protocol( + self, + ) -> DrainableDatagramProtocol | udp_wireguard.WireGuardDatagramTransport: return self._transport.get_protocol() # type: ignore def write(self, data: bytes) -> None: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 2ca6ccb58e..f3cf11bedc 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -90,11 +90,19 @@ def __init__(self, **kwargs) -> None: """, ) self.add_option("allow_hosts", Sequence[str], [], "Opposite of --ignore-hosts.") - self.add_option("listen_host", str, "", - "Address to bind proxy server(s) to (may be overridden for individual modes, see `mode`).") - self.add_option("listen_port", Optional[int], None, - "Port to bind proxy server(s) to (may be overridden for individual modes, see `mode`). " - "By default, the port is mode-specific. The default regular HTTP proxy spawns on port 8080.") + self.add_option( + "listen_host", + str, + "", + "Address to bind proxy server(s) to (may be overridden for individual modes, see `mode`).", + ) + self.add_option( + "listen_port", + Optional[int], + None, + "Port to bind proxy server(s) to (may be overridden for individual modes, see `mode`). " + "By default, the port is mode-specific. The default regular HTTP proxy spawns on port 8080.", + ) self.add_option( "mode", Sequence[str], diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index ae3bb54ac6..32d448323c 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -1,18 +1,25 @@ from __future__ import annotations + import contextlib import copy -import weakref -from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass import os import pprint import textwrap -from typing import Any, Optional, TextIO, Union +import weakref +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any +from typing import Optional +from typing import TextIO +from typing import Union import ruamel.yaml from mitmproxy import exceptions -from mitmproxy.utils import signals, typecheck +from mitmproxy.utils import signals +from mitmproxy.utils import typecheck """ The base implementation for Options. @@ -150,9 +157,7 @@ def subscribe(self, func, opts): if i not in self._options: raise exceptions.OptionsError("No such option: %s" % i) - self._subscriptions.append( - (signals.make_weak_ref(func), set(opts)) - ) + self._subscriptions.append((signals.make_weak_ref(func), set(opts))) def _notify_subscribers(self, updated) -> None: cleanup = False @@ -526,7 +531,7 @@ def parse(text): snip = v.problem_mark.get_snippet() raise exceptions.OptionsError( "Config error at line %s:\n%s\n%s" - % (v.problem_mark.line + 1, snip, getattr(v, 'problem', '')) + % (v.problem_mark.line + 1, snip, getattr(v, "problem", "")) ) else: raise exceptions.OptionsError("Could not parse options.") diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py index e6fdcd7c8d..0b0c492ada 100644 --- a/mitmproxy/platform/__init__.py +++ b/mitmproxy/platform/__init__.py @@ -1,7 +1,8 @@ import re import socket import sys -from typing import Callable, Optional +from typing import Callable +from typing import Optional def init_transparent_mode() -> None: diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index 1ff8876618..1e065544b2 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -1,8 +1,7 @@ from __future__ import annotations -import collections + import collections.abc import contextlib -import ctypes import ctypes.wintypes import json import os @@ -12,12 +11,16 @@ import threading import time from collections.abc import Callable -from typing import Any, ClassVar, IO, Optional, cast +from typing import Any +from typing import cast +from typing import ClassVar +from typing import IO +from typing import Optional -import pydivert import pydivert.consts -from mitmproxy.net.local_ip import get_local_ip, get_local_ip6 +from mitmproxy.net.local_ip import get_local_ip +from mitmproxy.net.local_ip import get_local_ip6 REDIRECT_API_HOST = "127.0.0.1" REDIRECT_API_PORT = 8085 @@ -98,7 +101,9 @@ def handle(self) -> None: if c is None: return try: - server = proxifier.client_server_map[cast(tuple[str, int], tuple(c))] + server = proxifier.client_server_map[ + cast(tuple[str, int], tuple(c)) + ] except KeyError: server = None write(server, self.wfile) @@ -397,7 +402,7 @@ class TransparentProxy: local: Optional[RedirectLocal] = None # really weird linting error here. - forward: Optional[Redirect] = None # noqa + forward: Optional[Redirect] = None response: Redirect icmp: Redirect diff --git a/mitmproxy/proxy/commands.py b/mitmproxy/proxy/commands.py index 04b471e02d..e6749dccda 100644 --- a/mitmproxy/proxy/commands.py +++ b/mitmproxy/proxy/commands.py @@ -8,10 +8,12 @@ """ import logging import warnings -from typing import Union, TYPE_CHECKING +from typing import TYPE_CHECKING +from typing import Union import mitmproxy.hooks -from mitmproxy.connection import Connection, Server +from mitmproxy.connection import Connection +from mitmproxy.connection import Server if TYPE_CHECKING: import mitmproxy.proxy.layer @@ -133,6 +135,7 @@ class Log(Command): This could also be implemented with some more playbook magic in the future, but for now we keep the current approach as the fully sans-io one. """ + message: str level: int @@ -144,7 +147,8 @@ def __init__( if isinstance(level, str): # pragma: no cover warnings.warn( "commands.Log() now expects an integer log level, not a string.", - DeprecationWarning, stacklevel=2 + DeprecationWarning, + stacklevel=2, ) level = getattr(logging, level.upper()) self.message = message diff --git a/mitmproxy/proxy/context.py b/mitmproxy/proxy/context.py index 1be73b39b1..29987418fc 100644 --- a/mitmproxy/proxy/context.py +++ b/mitmproxy/proxy/context.py @@ -38,8 +38,7 @@ def __init__( self.client = client self.options = options self.server = connection.Server( - address=None, - transport_protocol=client.transport_protocol + address=None, transport_protocol=client.transport_protocol ) self.layers = [] diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index fad8029ae8..e741fbcfb7 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -5,12 +5,16 @@ """ import typing import warnings -from dataclasses import dataclass, is_dataclass -from typing import Any, Generic, Optional, TypeVar +from dataclasses import dataclass +from dataclasses import is_dataclass +from typing import Any +from typing import Generic +from typing import Optional +from typing import TypeVar from mitmproxy import flow -from mitmproxy.proxy import commands from mitmproxy.connection import Connection +from mitmproxy.proxy import commands class Event: diff --git a/mitmproxy/proxy/layer.py b/mitmproxy/proxy/layer.py index d486e9b8f0..79aed95c9a 100644 --- a/mitmproxy/proxy/layer.py +++ b/mitmproxy/proxy/layer.py @@ -5,13 +5,20 @@ import textwrap from abc import abstractmethod from collections.abc import Callable +from collections.abc import Generator from dataclasses import dataclass from logging import DEBUG -from typing import Any, ClassVar, Generator, NamedTuple, Optional, TypeVar +from typing import Any +from typing import ClassVar +from typing import NamedTuple +from typing import Optional +from typing import TypeVar from mitmproxy.connection import Connection -from mitmproxy.proxy import commands, events -from mitmproxy.proxy.commands import Command, StartHook +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy.commands import Command +from mitmproxy.proxy.commands import StartHook from mitmproxy.proxy.context import Context T = TypeVar("T") diff --git a/mitmproxy/proxy/layers/__init__.py b/mitmproxy/proxy/layers/__init__.py index 349c32cfcd..e21ba60e08 100644 --- a/mitmproxy/proxy/layers/__init__.py +++ b/mitmproxy/proxy/layers/__init__.py @@ -1,10 +1,14 @@ from . import modes from .dns import DNSLayer from .http import HttpLayer -from .quic import QuicStreamLayer, RawQuicLayer, ClientQuicLayer, ServerQuicLayer +from .quic import ClientQuicLayer +from .quic import QuicStreamLayer +from .quic import RawQuicLayer +from .quic import ServerQuicLayer from .tcp import TCPLayer +from .tls import ClientTLSLayer +from .tls import ServerTLSLayer from .udp import UDPLayer -from .tls import ClientTLSLayer, ServerTLSLayer from .websocket import WebsocketLayer __all__ = [ diff --git a/mitmproxy/proxy/layers/dns.py b/mitmproxy/proxy/layers/dns.py index 0b85ad05ae..e2e5c701bb 100644 --- a/mitmproxy/proxy/layers/dns.py +++ b/mitmproxy/proxy/layers/dns.py @@ -1,8 +1,11 @@ -from dataclasses import dataclass import struct +from dataclasses import dataclass -from mitmproxy import dns, flow as mflow -from mitmproxy.proxy import commands, events, layer +from mitmproxy import dns +from mitmproxy import flow as mflow +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.context import Context from mitmproxy.proxy.utils import expect @@ -45,13 +48,17 @@ def __init__(self, context: Context): super().__init__(context) self.flows = {} - def handle_request(self, flow: dns.DNSFlow, msg: dns.Message) -> layer.CommandGenerator[None]: + def handle_request( + self, flow: dns.DNSFlow, msg: dns.Message + ) -> layer.CommandGenerator[None]: flow.request = msg # if already set, continue and query upstream again yield DnsRequestHook(flow) if flow.response: yield from self.handle_response(flow, flow.response) elif not self.context.server.address: - yield from self.handle_error(flow, "No hook has set a response and there is no upstream server.") + yield from self.handle_error( + flow, "No hook has set a response and there is no upstream server." + ) else: if not self.context.server.connected: err = yield commands.OpenConnection(self.context.server) @@ -61,7 +68,9 @@ def handle_request(self, flow: dns.DNSFlow, msg: dns.Message) -> layer.CommandGe return yield commands.SendData(self.context.server, flow.request.packed) - def handle_response(self, flow: dns.DNSFlow, msg: dns.Message) -> layer.CommandGenerator[None]: + def handle_response( + self, flow: dns.DNSFlow, msg: dns.Message + ) -> layer.CommandGenerator[None]: flow.response = msg yield DnsResponseHook(flow) if flow.response: @@ -92,7 +101,9 @@ def state_query(self, event: events.Event) -> layer.CommandGenerator[None]: try: flow = self.flows[msg.id] except KeyError: - flow = dns.DNSFlow(self.context.client, self.context.server, live=True) + flow = dns.DNSFlow( + self.context.client, self.context.server, live=True + ) self.flows[msg.id] = flow if from_client: yield from self.handle_request(flow, msg) diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 09a31024be..9d7cba4ceb 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -1,53 +1,68 @@ import collections import enum -from logging import DEBUG, WARNING - import time from dataclasses import dataclass from functools import cached_property -from typing import Optional, Union +from logging import DEBUG +from logging import WARNING +from typing import Optional +from typing import Union import wsproto.handshake -from mitmproxy import flow, http -from mitmproxy.connection import Connection, Server, TransportProtocol + +from ...context import Context +from ...mode_specs import ReverseMode +from ...mode_specs import UpstreamMode +from ..quic import QuicStreamEvent +from ._base import HttpCommand +from ._base import HttpConnection +from ._base import ReceiveHttp +from ._base import StreamId +from ._events import HttpEvent +from ._events import RequestData +from ._events import RequestEndOfMessage +from ._events import RequestHeaders +from ._events import RequestProtocolError +from ._events import RequestTrailers +from ._events import ResponseData +from ._events import ResponseEndOfMessage +from ._events import ResponseHeaders +from ._events import ResponseProtocolError +from ._events import ResponseTrailers +from ._hooks import HttpConnectHook +from ._hooks import HttpErrorHook +from ._hooks import HttpRequestHeadersHook +from ._hooks import HttpRequestHook +from ._hooks import HttpResponseHeadersHook +from ._hooks import HttpResponseHook +from ._http1 import Http1Client +from ._http1 import Http1Connection +from ._http1 import Http1Server +from ._http2 import Http2Client +from ._http2 import Http2Server +from ._http3 import Http3Client +from ._http3 import Http3Server +from mitmproxy import flow +from mitmproxy import http +from mitmproxy.connection import Connection +from mitmproxy.connection import Server +from mitmproxy.connection import TransportProtocol from mitmproxy.net import server_spec -from mitmproxy.net.http import status_codes, url +from mitmproxy.net.http import status_codes +from mitmproxy.net.http import url from mitmproxy.net.http.http1 import expected_http_body_size -from mitmproxy.proxy import commands, events, layer, tunnel -from mitmproxy.proxy.layers import quic, tcp, tls, websocket +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer +from mitmproxy.proxy import tunnel +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tcp +from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import websocket from mitmproxy.proxy.layers.http import _upstream_proxy from mitmproxy.proxy.utils import expect from mitmproxy.utils import human from mitmproxy.websocket import WebSocketData -from ._base import HttpCommand, HttpConnection, ReceiveHttp, StreamId -from ._events import ( - HttpEvent, - RequestData, - RequestEndOfMessage, - RequestHeaders, - RequestProtocolError, - RequestTrailers, - ResponseData, - ResponseEndOfMessage, - ResponseHeaders, - ResponseProtocolError, - ResponseTrailers, -) -from ._hooks import ( # noqa - HttpConnectHook, - HttpConnectUpstreamHook, - HttpErrorHook, - HttpRequestHeadersHook, - HttpRequestHook, - HttpResponseHeadersHook, - HttpResponseHook, -) -from ._http1 import Http1Client, Http1Connection, Http1Server -from ._http2 import Http2Client, Http2Server -from ._http3 import Http3Client, Http3Server -from ..quic import QuicStreamEvent -from ...context import Context -from ...mode_specs import ReverseMode, UpstreamMode class HTTPMode(enum.Enum): @@ -228,7 +243,9 @@ def state_wait_for_request_headers( "https" if self.context.client.tls else "http" ) - if self.mode is HTTPMode.regular and not (self.flow.request.is_http2 or self.flow.request.is_http3): + if self.mode is HTTPMode.regular and not ( + self.flow.request.is_http2 or self.flow.request.is_http3 + ): # Set the request target to origin-form for HTTP/1, some servers don't support absolute-form requests. # see https://github.com/mitmproxy/mitmproxy/issues/1759 self.flow.request.authority = "" @@ -707,7 +724,7 @@ def handle_connect_regular(self): 502, f"Cannot connect to {human.format_address(self.context.server.address)}: {err} " f"If you plan to redirect requests away from this server, " - f"consider setting `connection_strategy` to `lazy` to suppress early connections." + f"consider setting `connection_strategy` to `lazy` to suppress early connections.", ) self.child_layer = layer.NextLayer(self.context) yield from self.handle_connect_finish() @@ -1012,7 +1029,9 @@ def get_connection( if not can_use_context_connection: - context.server = Server(address=event.address, transport_protocol=event.transport_protocol) + context.server = Server( + address=event.address, transport_protocol=event.transport_protocol + ) if event.via: context.server.via = event.via @@ -1034,7 +1053,9 @@ def get_connection( elif context.server.transport_protocol == "udp": stack /= quic.ServerQuicLayer(context) else: - raise AssertionError(context.server.transport_protocol) # pragma: no cover + raise AssertionError( + context.server.transport_protocol + ) # pragma: no cover stack /= HttpClient(context) diff --git a/mitmproxy/proxy/layers/http/_base.py b/mitmproxy/proxy/layers/http/_base.py index b5f66d46ba..198fa77fb2 100644 --- a/mitmproxy/proxy/layers/http/_base.py +++ b/mitmproxy/proxy/layers/http/_base.py @@ -4,7 +4,9 @@ from mitmproxy import http from mitmproxy.connection import Connection -from mitmproxy.proxy import commands, events, layer +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.context import Context StreamId = int diff --git a/mitmproxy/proxy/layers/http/_events.py b/mitmproxy/proxy/layers/http/_events.py index f67217b03b..ecdbcd7a29 100644 --- a/mitmproxy/proxy/layers/http/_events.py +++ b/mitmproxy/proxy/layers/http/_events.py @@ -1,9 +1,9 @@ from dataclasses import dataclass from typing import Optional +from ._base import HttpEvent from mitmproxy import http from mitmproxy.http import HTTPFlow -from ._base import HttpEvent @dataclass diff --git a/mitmproxy/proxy/layers/http/_http1.py b/mitmproxy/proxy/layers/http/_http1.py index c7c79cac87..0affd6d68c 100644 --- a/mitmproxy/proxy/layers/http/_http1.py +++ b/mitmproxy/proxy/layers/http/_http1.py @@ -1,30 +1,39 @@ import abc -from typing import Callable, Optional, Union +from typing import Callable +from typing import Optional +from typing import Union import h11 -from h11._readers import ChunkedReader, ContentLengthReader, Http10Reader +from h11._readers import ChunkedReader +from h11._readers import ContentLengthReader +from h11._readers import Http10Reader from h11._receivebuffer import ReceiveBuffer -from mitmproxy import http, version -from mitmproxy.connection import Connection, ConnectionState -from mitmproxy.net.http import http1, status_codes -from mitmproxy.proxy import commands, events, layer -from mitmproxy.proxy.layers.http._base import ReceiveHttp, StreamId +from ...context import Context +from ._base import format_error +from ._base import HttpConnection +from ._events import HttpEvent +from ._events import RequestData +from ._events import RequestEndOfMessage +from ._events import RequestHeaders +from ._events import RequestProtocolError +from ._events import ResponseData +from ._events import ResponseEndOfMessage +from ._events import ResponseHeaders +from ._events import ResponseProtocolError +from mitmproxy import http +from mitmproxy import version +from mitmproxy.connection import Connection +from mitmproxy.connection import ConnectionState +from mitmproxy.net.http import http1 +from mitmproxy.net.http import status_codes +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer +from mitmproxy.proxy.layers.http._base import ReceiveHttp +from mitmproxy.proxy.layers.http._base import StreamId from mitmproxy.proxy.utils import expect from mitmproxy.utils import human -from ._base import HttpConnection, format_error -from ._events import ( - HttpEvent, - RequestData, - RequestEndOfMessage, - RequestHeaders, - RequestProtocolError, - ResponseData, - ResponseEndOfMessage, - ResponseHeaders, - ResponseProtocolError, -) -from ...context import Context TBodyReader = Union[ChunkedReader, Http10Reader, ContentLengthReader] @@ -189,7 +198,10 @@ def mark_done( # If we proxy HTTP/2 to HTTP/1, we only use upstream connections for one request. # This simplifies our connection management quite a bit as we can rely on # the proxyserver's max-connection-per-server throttling. - or ((self.request.is_http2 or self.request.is_http3) and isinstance(self, Http1Client)) + or ( + (self.request.is_http2 or self.request.is_http3) + and isinstance(self, Http1Client) + ) ) if connection_done: yield commands.CloseConnection(self.conn) @@ -245,7 +257,11 @@ def send(self, event: HttpEvent) -> layer.CommandGenerator[None]: elif isinstance(event, ResponseEndOfMessage): assert self.request assert self.response - if self.request.method.upper() != "HEAD" and "chunked" in self.response.headers.get("transfer-encoding", "").lower(): + if ( + self.request.method.upper() != "HEAD" + and "chunked" + in self.response.headers.get("transfer-encoding", "").lower() + ): yield commands.SendData(self.conn, b"0\r\n\r\n") yield from self.mark_done(response=True) elif isinstance(event, ResponseProtocolError): diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index f881612cc1..7c7af9dac1 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -1,10 +1,12 @@ import collections -from logging import DEBUG, ERROR - import time from collections.abc import Sequence from enum import Enum -from typing import ClassVar, Optional, Union +from logging import DEBUG +from logging import ERROR +from typing import ClassVar +from typing import Optional +from typing import Union import h2.config import h2.connection @@ -15,29 +17,40 @@ import h2.stream import h2.utilities -from mitmproxy import http, version -from mitmproxy.connection import Connection -from mitmproxy.net.http import status_codes, url -from mitmproxy.utils import human -from . import ( - RequestData, - RequestEndOfMessage, - RequestHeaders, - RequestProtocolError, - ResponseData, - ResponseEndOfMessage, - ResponseHeaders, - RequestTrailers, - ResponseTrailers, - ResponseProtocolError, -) -from ._base import HttpConnection, HttpEvent, ReceiveHttp, format_error -from ._http_h2 import BufferedH2Connection, H2ConnectionLogger -from ...commands import CloseConnection, Log, SendData, RequestWakeup +from . import RequestData +from . import RequestEndOfMessage +from . import RequestHeaders +from . import RequestProtocolError +from . import RequestTrailers +from . import ResponseData +from . import ResponseEndOfMessage +from . import ResponseHeaders +from . import ResponseProtocolError +from . import ResponseTrailers +from ...commands import CloseConnection +from ...commands import Log +from ...commands import RequestWakeup +from ...commands import SendData from ...context import Context -from ...events import ConnectionClosed, DataReceived, Event, Start, Wakeup +from ...events import ConnectionClosed +from ...events import DataReceived +from ...events import Event +from ...events import Start +from ...events import Wakeup from ...layer import CommandGenerator from ...utils import expect +from ._base import format_error +from ._base import HttpConnection +from ._base import HttpEvent +from ._base import ReceiveHttp +from ._http_h2 import BufferedH2Connection +from ._http_h2 import H2ConnectionLogger +from mitmproxy import http +from mitmproxy import version +from mitmproxy.connection import Connection +from mitmproxy.net.http import status_codes +from mitmproxy.net.http import url +from mitmproxy.utils import human class StreamState(Enum): @@ -70,8 +83,7 @@ def __init__(self, context: Context, conn: Connection): super().__init__(context, conn) if self.debug: self.h2_conf.logger = H2ConnectionLogger( - self.context.client.peername, - self.__class__.__name__ + self.context.client.peername, self.__class__.__name__ ) self.h2_conf.validate_inbound_headers = ( self.context.options.validate_inbound_headers @@ -374,7 +386,9 @@ def _handle_event(self, event: Event) -> CommandGenerator[None]: if self.is_open_for_us(event.stream_id): self.h2_conn.send_headers( event.stream_id, - headers=(yield from format_h2_response_headers(self.context, event)), + headers=( + yield from format_h2_response_headers(self.context, event) + ), end_stream=event.end_stream, ) yield SendData(self.conn, self.h2_conn.data_to_send()) diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index 8a696a24fb..ccafe36f4b 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,50 +1,48 @@ -from abc import abstractmethod import time -from typing import Dict, Union +from abc import abstractmethod +from typing import Union -from aioquic.h3.connection import ( - ErrorCode as H3ErrorCode, - FrameUnexpected as H3FrameUnexpected, -) -from aioquic.h3.events import DataReceived, HeadersReceived, PushPromiseReceived +from aioquic.h3.connection import ErrorCode as H3ErrorCode +from aioquic.h3.connection import FrameUnexpected as H3FrameUnexpected +from aioquic.h3.events import DataReceived +from aioquic.h3.events import HeadersReceived +from aioquic.h3.events import PushPromiseReceived -from mitmproxy import connection, http, version +from . import RequestData +from . import RequestEndOfMessage +from . import RequestHeaders +from . import RequestProtocolError +from . import RequestTrailers +from . import ResponseData +from . import ResponseEndOfMessage +from . import ResponseHeaders +from . import ResponseProtocolError +from . import ResponseTrailers +from ._base import format_error +from ._base import HttpConnection +from ._base import HttpEvent +from ._base import ReceiveHttp +from ._http2 import format_h2_request_headers +from ._http2 import format_h2_response_headers +from ._http2 import parse_h2_request_headers +from ._http2 import parse_h2_response_headers +from ._http_h3 import LayeredH3Connection +from ._http_h3 import StreamReset +from ._http_h3 import TrailersReceived +from mitmproxy import connection +from mitmproxy import http +from mitmproxy import version from mitmproxy.net.http import status_codes -from mitmproxy.proxy import commands, context, events, layer -from mitmproxy.proxy.layers.quic import ( - QuicConnectionClosed, - QuicStreamEvent, - StopQuicStream, - error_code_to_str, -) +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layer +from mitmproxy.proxy.layers.quic import error_code_to_str +from mitmproxy.proxy.layers.quic import QuicConnectionClosed +from mitmproxy.proxy.layers.quic import QuicStreamEvent +from mitmproxy.proxy.layers.quic import StopQuicStream from mitmproxy.proxy.utils import expect -from . import ( - RequestData, - RequestEndOfMessage, - RequestHeaders, - RequestProtocolError, - RequestTrailers, - ResponseData, - ResponseEndOfMessage, - ResponseHeaders, - ResponseProtocolError, - ResponseTrailers, -) -from ._base import ( - HttpConnection, - HttpEvent, - ReceiveHttp, - format_error, -) -from ._http2 import ( - format_h2_request_headers, - format_h2_response_headers, - parse_h2_request_headers, - parse_h2_response_headers, -) -from ._http_h3 import LayeredH3Connection, StreamReset, TrailersReceived - class Http3Connection(HttpConnection): h3_conn: LayeredH3Connection @@ -56,7 +54,9 @@ class Http3Connection(HttpConnection): def __init__(self, context: context.Context, conn: connection.Connection): super().__init__(context, conn) - self.h3_conn = LayeredH3Connection(self.conn, is_client=self.conn is self.context.server) + self.h3_conn = LayeredH3Connection( + self.conn, is_client=self.conn is self.context.server + ) self._stream_protocol_errors: dict[int, int] = {} def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: @@ -74,9 +74,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, RequestHeaders) else format_h2_response_headers(self.context, event) ) - self.h3_conn.send_headers(event.stream_id, headers, end_stream=event.end_stream) + self.h3_conn.send_headers( + event.stream_id, headers, end_stream=event.end_stream + ) elif isinstance(event, (RequestTrailers, ResponseTrailers)): - self.h3_conn.send_trailers(event.stream_id, [*event.trailers.fields]) + self.h3_conn.send_trailers( + event.stream_id, [*event.trailers.fields] + ) elif isinstance(event, (RequestEndOfMessage, ResponseEndOfMessage)): self.h3_conn.end_stream(event.stream_id) elif isinstance(event, (RequestProtocolError, ResponseProtocolError)): @@ -145,9 +149,13 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: elif isinstance(h3_event, DataReceived): if h3_event.push_id is None: if h3_event.data: - yield ReceiveHttp(self.ReceiveData(h3_event.stream_id, h3_event.data)) + yield ReceiveHttp( + self.ReceiveData(h3_event.stream_id, h3_event.data) + ) if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + yield ReceiveHttp( + self.ReceiveEndOfMessage(h3_event.stream_id) + ) elif isinstance(h3_event, HeadersReceived): if h3_event.push_id is None: try: @@ -160,12 +168,20 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: yield ReceiveHttp(receive_event) if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + yield ReceiveHttp( + self.ReceiveEndOfMessage(h3_event.stream_id) + ) elif isinstance(h3_event, TrailersReceived): if h3_event.push_id is None: - yield ReceiveHttp(self.ReceiveTrailers(h3_event.stream_id, http.Headers(h3_event.trailers))) + yield ReceiveHttp( + self.ReceiveTrailers( + h3_event.stream_id, http.Headers(h3_event.trailers) + ) + ) if h3_event.stream_ended: - yield ReceiveHttp(self.ReceiveEndOfMessage(h3_event.stream_id)) + yield ReceiveHttp( + self.ReceiveEndOfMessage(h3_event.stream_id) + ) elif isinstance(h3_event, PushPromiseReceived): # pragma: no cover # we don't support push pass @@ -204,7 +220,9 @@ class Http3Server(Http3Connection): def __init__(self, context: context.Context): super().__init__(context, context.client) - def parse_headers(self, event: HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers( + self, event: HeadersReceived + ) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 ( host, @@ -238,8 +256,8 @@ class Http3Client(Http3Connection): ReceiveProtocolError = ResponseProtocolError ReceiveTrailers = ResponseTrailers - our_stream_id: Dict[int, int] - their_stream_id: Dict[int, int] + our_stream_id: dict[int, int] + their_stream_id: dict[int, int] def __init__(self, context: context.Context): super().__init__(context, context.server) @@ -263,7 +281,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id] yield cmd - def parse_headers(self, event: HeadersReceived) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers( + self, event: HeadersReceived + ) -> Union[RequestHeaders, ResponseHeaders]: # same as HTTP/2 status_code, headers = parse_h2_response_headers(event.headers) response = http.Response( diff --git a/mitmproxy/proxy/layers/http/_http_h2.py b/mitmproxy/proxy/layers/http/_http_h2.py index 8533b7afbf..f5b08d64cb 100644 --- a/mitmproxy/proxy/layers/http/_http_h2.py +++ b/mitmproxy/proxy/layers/http/_http_h2.py @@ -1,6 +1,6 @@ import collections import logging -from typing import Dict, List, NamedTuple, Tuple +from typing import NamedTuple import h2.config import h2.connection @@ -21,9 +21,7 @@ def __init__(self, peername: tuple, conn_type: str): def debug(self, fmtstr, *args): logger.debug( - f"{self.conn_type} {fmtstr}", - *args, - extra={"client": self.peername} + f"{self.conn_type} {fmtstr}", *args, extra={"client": self.peername} ) def trace(self, fmtstr, *args): @@ -31,7 +29,7 @@ def trace(self, fmtstr, *args): logging.DEBUG - 1, f"{self.conn_type} {fmtstr}", *args, - extra={"client": self.peername} + extra={"client": self.peername}, ) @@ -48,7 +46,7 @@ class BufferedH2Connection(h2.connection.H2Connection): """ stream_buffers: collections.defaultdict[int, collections.deque[SendH2Data]] - stream_trailers: Dict[int, List[Tuple[bytes, bytes]]] + stream_trailers: dict[int, list[tuple[bytes, bytes]]] def __init__(self, config: h2.config.H2Configuration): super().__init__(config) @@ -93,7 +91,7 @@ def send_data( # We can't send right now, so we buffer. self.stream_buffers[stream_id].append(SendH2Data(data, end_stream)) - def send_trailers(self, stream_id: int, trailers: List[Tuple[bytes, bytes]]): + def send_trailers(self, stream_id: int, trailers: list[tuple[bytes, bytes]]): if self.stream_buffers.get(stream_id, None): # Though trailers are not subject to flow control, we need to queue them and send strictly after data frames self.stream_trailers[stream_id] = trailers @@ -173,7 +171,9 @@ def stream_window_updated(self, stream_id: int) -> bool: if not self.stream_buffers[stream_id]: del self.stream_buffers[stream_id] if stream_id in self.stream_trailers: - self.send_headers(stream_id, self.stream_trailers.pop(stream_id), end_stream=True) + self.send_headers( + stream_id, self.stream_trailers.pop(stream_id), end_stream=True + ) sent_any_data = True return sent_any_data diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 849f3b1594..52b36e57ee 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -1,31 +1,29 @@ +from collections.abc import Iterable from dataclasses import dataclass -from typing import Iterable, Optional - -from aioquic.h3.connection import ( - FrameUnexpected, - H3Connection, - H3Event, - H3Stream, - Headers, - HeadersState, - StreamType, -) +from typing import Optional + +from aioquic.h3.connection import FrameUnexpected +from aioquic.h3.connection import H3Connection +from aioquic.h3.connection import H3Event +from aioquic.h3.connection import H3Stream +from aioquic.h3.connection import Headers +from aioquic.h3.connection import HeadersState +from aioquic.h3.connection import StreamType from aioquic.h3.events import HeadersReceived from aioquic.quic.configuration import QuicConfiguration from aioquic.quic.events import StreamDataReceived from aioquic.quic.packet import QuicErrorCode from mitmproxy import connection -from mitmproxy.proxy import commands, layer -from mitmproxy.proxy.layers.quic import ( - CloseQuicConnection, - QuicConnectionClosed, - QuicStreamDataReceived, - QuicStreamEvent, - QuicStreamReset, - ResetQuicStream, - SendQuicStreamData, -) +from mitmproxy.proxy import commands +from mitmproxy.proxy import layer +from mitmproxy.proxy.layers.quic import CloseQuicConnection +from mitmproxy.proxy.layers.quic import QuicConnectionClosed +from mitmproxy.proxy.layers.quic import QuicStreamDataReceived +from mitmproxy.proxy.layers.quic import QuicStreamEvent +from mitmproxy.proxy.layers.quic import QuicStreamReset +from mitmproxy.proxy.layers.quic import ResetQuicStream +from mitmproxy.proxy.layers.quic import SendQuicStreamData @dataclass @@ -104,8 +102,12 @@ def get_next_available_stream_id(self, is_unidirectional: bool = False) -> int: def reset_stream(self, stream_id: int, error_code: int) -> None: self.pending_commands.append(ResetQuicStream(self.conn, stream_id, error_code)) - def send_stream_data(self, stream_id: int, data: bytes, end_stream: bool = False) -> None: - self.pending_commands.append(SendQuicStreamData(self.conn, stream_id, data, end_stream)) + def send_stream_data( + self, stream_id: int, data: bytes, end_stream: bool = False + ) -> None: + self.pending_commands.append( + SendQuicStreamData(self.conn, stream_id, data, end_stream) + ) class LayeredH3Connection(H3Connection): @@ -114,7 +116,12 @@ class LayeredH3Connection(H3Connection): Also ensures that headers, data and trailers are sent in that order. """ - def __init__(self, conn: connection.Connection, is_client: bool, enable_webtransport: bool = False) -> None: + def __init__( + self, + conn: connection.Connection, + is_client: bool, + enable_webtransport: bool = False, + ) -> None: self._mock = MockQuic(conn, is_client) super().__init__(self._mock, enable_webtransport) # type: ignore @@ -132,13 +139,18 @@ def _handle_request_or_push_frame( stream_ended: bool, ) -> list[H3Event]: # turn HeadersReceived into TrailersReceived for trailers - events = super()._handle_request_or_push_frame(frame_type, frame_data, stream, stream_ended) + events = super()._handle_request_or_push_frame( + frame_type, frame_data, stream, stream_ended + ) for index, event in enumerate(events): if ( isinstance(event, HeadersReceived) - and self._stream[event.stream_id].headers_recv_state == HeadersState.AFTER_TRAILERS + and self._stream[event.stream_id].headers_recv_state + == HeadersState.AFTER_TRAILERS ): - events[index] = TrailersReceived(event.headers, event.stream_id, event.stream_ended, event.push_id) + events[index] = TrailersReceived( + event.headers, event.stream_id, event.stream_ended, event.push_id + ) return events def close_connection( @@ -173,11 +185,7 @@ def get_open_stream_ids(self, push_id: Optional[int]) -> Iterable[int]: for stream in self._stream.values() if ( stream.push_id == push_id - and stream.stream_type == ( - None - if push_id is None else - StreamType.PUSH - ) + and stream.stream_type == (None if push_id is None else StreamType.PUSH) and not ( stream.headers_recv_state == HeadersState.AFTER_TRAILERS and stream.headers_send_state == HeadersState.AFTER_TRAILERS @@ -206,10 +214,15 @@ def handle_stream_event(self, event: QuicStreamEvent) -> list[H3Event]: if self._get_or_create_stream(event.stream_id).ended: # aioquic will not send us any data events once a stream has ended. # Instead, it will close the connection. We simulate this here for H3 tests. - self.close_connection(error_code=QuicErrorCode.PROTOCOL_VIOLATION, reason_phrase="stream already ended") + self.close_connection( + error_code=QuicErrorCode.PROTOCOL_VIOLATION, + reason_phrase="stream already ended", + ) return [] else: - return self.handle_event(StreamDataReceived(event.data, event.end_stream, event.stream_id)) + return self.handle_event( + StreamDataReceived(event.data, event.end_stream, event.stream_id) + ) # should never happen else: # pragma: no cover @@ -241,7 +254,9 @@ def send_datagram(self, flow_id: int, data: bytes) -> None: # supporting datagrams would require additional information from the underlying QUIC connection raise NotImplementedError() # pragma: no cover - def send_headers(self, stream_id: int, headers: Headers, end_stream: bool = False) -> None: + def send_headers( + self, stream_id: int, headers: Headers, end_stream: bool = False + ) -> None: """Sends headers over the given stream.""" # ensure we haven't sent something before diff --git a/mitmproxy/proxy/layers/http/_upstream_proxy.py b/mitmproxy/proxy/layers/http/_upstream_proxy.py index 4029d5347c..9034fe1450 100644 --- a/mitmproxy/proxy/layers/http/_upstream_proxy.py +++ b/mitmproxy/proxy/layers/http/_upstream_proxy.py @@ -1,15 +1,18 @@ -from logging import DEBUG - import time +from logging import DEBUG from typing import Optional from h11._receivebuffer import ReceiveBuffer -from mitmproxy import http, connection +from mitmproxy import connection +from mitmproxy import http from mitmproxy.net.http import http1 -from mitmproxy.proxy import commands, context, layer, tunnel -from mitmproxy.proxy.layers.http._hooks import HttpConnectUpstreamHook +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import layer +from mitmproxy.proxy import tunnel from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers.http._hooks import HttpConnectUpstreamHook from mitmproxy.utils import human @@ -48,7 +51,9 @@ def start_handshake(self) -> layer.CommandGenerator[None]: return (yield from super().start_handshake()) assert self.conn.address flow = http.HTTPFlow(self.context.client, self.tunnel_connection) - authority = self.conn.address[0].encode("idna") + f":{self.conn.address[1]}".encode() + authority = ( + self.conn.address[0].encode("idna") + f":{self.conn.address[1]}".encode() + ) flow.request = http.Request( host=self.conn.address[0], port=self.conn.address[1], @@ -76,9 +81,7 @@ def receive_handshake_data( response_head = self.buf.maybe_extract_lines() if response_head: try: - response = http1.read_response_head([ - bytes(x) for x in response_head - ]) + response = http1.read_response_head([bytes(x) for x in response_head]) except ValueError as e: proxyaddr = human.format_address(self.tunnel_connection.address) yield commands.Log(f"{proxyaddr}: {e}") diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index cb243d754f..bfc2749678 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -1,14 +1,19 @@ from __future__ import annotations + import socket import struct from abc import ABCMeta from dataclasses import dataclass -from typing import Callable, Optional +from typing import Callable +from typing import Optional from mitmproxy import connection -from mitmproxy.proxy import commands, events, layer +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.layers import quic, tls +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tls from mitmproxy.proxy.mode_specs import ReverseMode from mitmproxy.proxy.utils import expect diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 657df37669..808cb584b6 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,44 +1,54 @@ from __future__ import annotations -from dataclasses import dataclass, field -from logging import DEBUG, ERROR, WARNING -from ssl import VerifyMode + import time +from dataclasses import dataclass +from dataclasses import field +from logging import DEBUG +from logging import ERROR +from logging import WARNING +from ssl import VerifyMode from typing import Callable from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import ErrorCode as H3ErrorCode from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration -from aioquic.quic.connection import ( - QuicConnection, - QuicConnectionError, - QuicConnectionState, - QuicErrorCode, - stream_is_client_initiated, - stream_is_unidirectional, -) -from aioquic.tls import CipherSuite, HandshakeType -from aioquic.quic.packet import ( - PACKET_TYPE_INITIAL, - QuicProtocolVersion, - encode_quic_version_negotiation, - pull_quic_header, -) +from aioquic.quic.connection import QuicConnection +from aioquic.quic.connection import QuicConnectionError +from aioquic.quic.connection import QuicConnectionState +from aioquic.quic.connection import QuicErrorCode +from aioquic.quic.connection import stream_is_client_initiated +from aioquic.quic.connection import stream_is_unidirectional +from aioquic.quic.packet import encode_quic_version_negotiation +from aioquic.quic.packet import PACKET_TYPE_INITIAL +from aioquic.quic.packet import pull_quic_header +from aioquic.quic.packet import QuicProtocolVersion +from aioquic.tls import CipherSuite +from aioquic.tls import HandshakeType from cryptography import x509 -from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa -from mitmproxy import certs, connection, ctx +from cryptography.hazmat.primitives.asymmetric import dsa +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.asymmetric import rsa + +from mitmproxy import certs +from mitmproxy import connection +from mitmproxy import ctx from mitmproxy.net import tls -from mitmproxy.proxy import commands, context, events, layer, tunnel +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layer +from mitmproxy.proxy import tunnel from mitmproxy.proxy.layers.tcp import TCPLayer -from mitmproxy.proxy.layers.tls import ( - TlsClienthelloHook, - TlsEstablishedClientHook, - TlsEstablishedServerHook, - TlsFailedClientHook, - TlsFailedServerHook, -) +from mitmproxy.proxy.layers.tls import TlsClienthelloHook +from mitmproxy.proxy.layers.tls import TlsEstablishedClientHook +from mitmproxy.proxy.layers.tls import TlsEstablishedServerHook +from mitmproxy.proxy.layers.tls import TlsFailedClientHook +from mitmproxy.proxy.layers.tls import TlsFailedServerHook from mitmproxy.proxy.layers.udp import UDPLayer -from mitmproxy.tls import ClientHello, ClientHelloData, TlsData +from mitmproxy.tls import ClientHello +from mitmproxy.tls import ClientHelloData +from mitmproxy.tls import TlsData @dataclass @@ -53,7 +63,9 @@ class QuicTlsSettings: """The certificate to use for the connection.""" certificate_chain: list[x509.Certificate] = field(default_factory=list) """A list of additional certificates to send to the peer.""" - certificate_private_key: dsa.DSAPrivateKey | ec.EllipticCurvePrivateKey | rsa.RSAPrivateKey | None = None + certificate_private_key: dsa.DSAPrivateKey | ec.EllipticCurvePrivateKey | rsa.RSAPrivateKey | None = ( + None + ) """The certificate's private key.""" cipher_suites: list[CipherSuite] | None = None """An optional list of allowed/advertised cipher suites.""" @@ -147,7 +159,13 @@ class SendQuicStreamData(QuicStreamCommand): end_stream: bool """Whether the FIN bit should be set in the STREAM frame.""" - def __init__(self, connection: connection.Connection, stream_id: int, data: bytes, end_stream: bool = False) -> None: + def __init__( + self, + connection: connection.Connection, + stream_id: int, + data: bytes, + end_stream: bool = False, + ) -> None: super().__init__(connection, stream_id) self.data = data self.end_stream = end_stream @@ -159,7 +177,9 @@ class ResetQuicStream(QuicStreamCommand): error_code: int """An error code indicating why the stream is being reset.""" - def __init__(self, connection: connection.Connection, stream_id: int, error_code: int) -> None: + def __init__( + self, connection: connection.Connection, stream_id: int, error_code: int + ) -> None: super().__init__(connection, stream_id) self.error_code = error_code @@ -170,7 +190,9 @@ class StopQuicStream(QuicStreamCommand): error_code: int """An error code indicating why the stream is being stopped.""" - def __init__(self, connection: connection.Connection, stream_id: int, error_code: int) -> None: + def __init__( + self, connection: connection.Connection, stream_id: int, error_code: int + ) -> None: super().__init__(connection, stream_id) self.error_code = error_code @@ -203,6 +225,7 @@ def __init__( class QuicConnectionClosed(events.ConnectionClosed): """QUIC connection has been closed.""" + error_code: int "The error code which was specified when closing the connection." @@ -351,7 +374,12 @@ def initialize_replacement(peer_cid: bytes) -> None: class QuicStreamNextLayer(layer.NextLayer): """`NextLayer` variant that callbacks `QuicStreamLayer` after layer decision.""" - def __init__(self, context: context.Context, stream: QuicStreamLayer, ask_on_start: bool = False) -> None: + def __init__( + self, + context: context.Context, + stream: QuicStreamLayer, + ask_on_start: bool = False, + ) -> None: super().__init__(context, ask_on_start) self._stream = stream self._layer: layer.Layer | None = None @@ -390,8 +418,8 @@ def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> No if stream_is_unidirectional(stream_id): self.client.state = ( connection.ConnectionState.CAN_READ - if stream_is_client_initiated(stream_id) else - connection.ConnectionState.CAN_WRITE + if stream_is_client_initiated(stream_id) + else connection.ConnectionState.CAN_WRITE ) self._client_stream_id = stream_id @@ -406,8 +434,8 @@ def __init__(self, context: context.Context, ignore: bool, stream_id: int) -> No super().__init__(context) self.child_layer = ( TCPLayer(context, ignore=True) - if ignore else - QuicStreamNextLayer(context, self) + if ignore + else QuicStreamNextLayer(context, self) ) self.refresh_metadata() @@ -425,11 +453,11 @@ def open_server_stream(self, server_stream_id) -> None: self.server.state = ( ( connection.ConnectionState.CAN_WRITE - if stream_is_client_initiated(server_stream_id) else - connection.ConnectionState.CAN_READ + if stream_is_client_initiated(server_stream_id) + else connection.ConnectionState.CAN_READ ) - if stream_is_unidirectional(server_stream_id) else - connection.ConnectionState.OPEN + if stream_is_unidirectional(server_stream_id) + else connection.ConnectionState.OPEN ) self.refresh_metadata() @@ -444,8 +472,14 @@ def refresh_metadata(self) -> None: else: break # pragma: no cover if isinstance(child_layer, (UDPLayer, TCPLayer)) and child_layer.flow: - child_layer.flow.metadata["quic_is_unidirectional"] = stream_is_unidirectional(self._client_stream_id) - child_layer.flow.metadata["quic_initiator"] = "client" if stream_is_client_initiated(self._client_stream_id) else "server" + child_layer.flow.metadata[ + "quic_is_unidirectional" + ] = stream_is_unidirectional(self._client_stream_id) + child_layer.flow.metadata["quic_initiator"] = ( + "client" + if stream_is_client_initiated(self._client_stream_id) + else "server" + ) child_layer.flow.metadata["quic_stream_id_client"] = self._client_stream_id child_layer.flow.metadata["quic_stream_id_server"] = self._server_stream_id @@ -483,8 +517,8 @@ def __init__(self, context: context.Context, ignore: bool = False) -> None: self.ignore = ignore self.datagram_layer = ( UDPLayer(self.context.fork(), ignore=True) - if ignore else - layer.NextLayer(self.context.fork()) + if ignore + else layer.NextLayer(self.context.fork()) ) self.client_stream_ids = {} self.server_stream_ids = {} @@ -508,29 +542,34 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # properly forward completion events based on their command elif isinstance(event, events.CommandCompleted): - yield from self.event_to_child(self.command_sources.pop(event.command), event) + yield from self.event_to_child( + self.command_sources.pop(event.command), event + ) # route injected messages based on their connections (prefer client, fallback to server) elif isinstance(event, events.MessageInjected): if event.flow.client_conn in self.connections: - yield from self.event_to_child(self.connections[event.flow.client_conn], event) + yield from self.event_to_child( + self.connections[event.flow.client_conn], event + ) elif event.flow.server_conn in self.connections: - yield from self.event_to_child(self.connections[event.flow.server_conn], event) + yield from self.event_to_child( + self.connections[event.flow.server_conn], event + ) else: raise AssertionError(f"Flow not associated: {event.flow!r}") # handle stream events targeting this context - elif ( - isinstance(event, QuicStreamEvent) - and ( - event.connection is self.context.client - or event.connection is self.context.server - ) + elif isinstance(event, QuicStreamEvent) and ( + event.connection is self.context.client + or event.connection is self.context.server ): from_client = event.connection is self.context.client # fetch or create the layer - stream_ids = self.client_stream_ids if from_client else self.server_stream_ids + stream_ids = ( + self.client_stream_ids if from_client else self.server_stream_ids + ) if event.stream_id in stream_ids: stream_layer = stream_ids[event.stream_id] else: @@ -549,7 +588,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: server_stream_id = event.stream_id # create, register and start the layer - stream_layer = QuicStreamLayer(self.context.fork(), self.ignore, client_stream_id) + stream_layer = QuicStreamLayer( + self.context.fork(), self.ignore, client_stream_id + ) self.client_stream_ids[client_stream_id] = stream_layer if server_stream_id is not None: stream_layer.open_server_stream(server_stream_id) @@ -562,7 +603,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: conn = stream_layer.client if from_client else stream_layer.server if isinstance(event, QuicStreamDataReceived): if event.data: - yield from self.event_to_child(stream_layer, events.DataReceived(conn, event.data)) + yield from self.event_to_child( + stream_layer, events.DataReceived(conn, event.data) + ) if event.end_stream: yield from self.close_stream_layer(stream_layer, from_client) elif isinstance(event, QuicStreamReset): @@ -574,26 +617,27 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: and command.end_stream and not command.data ): - yield ResetQuicStream(command.connection, command.stream_id, event.error_code) + yield ResetQuicStream( + command.connection, command.stream_id, event.error_code + ) else: yield command else: raise AssertionError(f"Unexpected stream event: {event!r}") # handle close events that target this context - elif ( - isinstance(event, QuicConnectionClosed) - and ( - event.connection is self.context.client - or event.connection is self.context.server - ) + elif isinstance(event, QuicConnectionClosed) and ( + event.connection is self.context.client + or event.connection is self.context.server ): from_client = event.connection is self.context.client other_conn = self.context.server if from_client else self.context.client # be done if both connections are closed if other_conn.connected: - yield CloseQuicConnection(other_conn, event.error_code, event.frame_type, event.reason_phrase) + yield CloseQuicConnection( + other_conn, event.error_code, event.frame_type, event.reason_phrase + ) else: self._handle_event = self.done # type: ignore @@ -607,16 +651,14 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: # forward to either the client or server connection of stream layers and swallow empty stream end for conn, child_layer in self.connections.items(): - if ( - isinstance(child_layer, QuicStreamLayer) - and ((conn is child_layer.client) if from_client else (conn is child_layer.server)) + if isinstance(child_layer, QuicStreamLayer) and ( + (conn is child_layer.client) + if from_client + else (conn is child_layer.server) ): conn.state &= ~connection.ConnectionState.CAN_WRITE for command in self.close_stream_layer(child_layer, from_client): - if ( - not isinstance(command, SendQuicStreamData) - or command.data - ): + if not isinstance(command, SendQuicStreamData) or command.data: yield command # all other connection events are routed to their corresponding layer @@ -626,7 +668,9 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: raise AssertionError(f"Unexpected event: {event!r}") - def close_stream_layer(self, stream_layer: QuicStreamLayer, client: bool) -> layer.CommandGenerator[None]: + def close_stream_layer( + self, stream_layer: QuicStreamLayer, client: bool + ) -> layer.CommandGenerator[None]: """Closes the incoming part of a connection.""" conn = stream_layer.client if client else stream_layer.server @@ -636,7 +680,9 @@ def close_stream_layer(self, stream_layer: QuicStreamLayer, client: bool) -> lay conn.timestamp_end = time.time() yield from self.event_to_child(stream_layer, events.ConnectionClosed(conn)) - def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer.CommandGenerator[None]: + def event_to_child( + self, child_layer: layer.Layer, event: events.Event + ) -> layer.CommandGenerator[None]: """Forwards events to child layers and translates commands.""" for command in child_layer.handle_event(event): @@ -664,16 +710,24 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer elif isinstance(command, commands.CloseConnection): assert stream_id is not None if command.connection.state & connection.ConnectionState.CAN_WRITE: - command.connection.state &= ~connection.ConnectionState.CAN_WRITE - yield SendQuicStreamData(quic_conn, stream_id, b"", end_stream=True) + command.connection.state &= ( + ~connection.ConnectionState.CAN_WRITE + ) + yield SendQuicStreamData( + quic_conn, stream_id, b"", end_stream=True + ) # XXX: Use `command.connection.state & connection.ConnectionState.CAN_READ` instead? - only_close_our_half = isinstance(command, commands.CloseTcpConnection) and command.half_close + only_close_our_half = ( + isinstance(command, commands.CloseTcpConnection) + and command.half_close + ) if not only_close_our_half: - if ( - stream_is_client_initiated(stream_id) == to_client - or not stream_is_unidirectional(stream_id) - ): - yield StopQuicStream(quic_conn, stream_id, QuicErrorCode.NO_ERROR) + if stream_is_client_initiated( + stream_id + ) == to_client or not stream_is_unidirectional(stream_id): + yield StopQuicStream( + quic_conn, stream_id, QuicErrorCode.NO_ERROR + ) yield from self.close_stream_layer(child_layer, to_client) # open server connections by reserving the next stream ID @@ -684,14 +738,18 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer assert client_stream_id is not None stream_id = self.get_next_available_stream_id( is_client=True, - is_unidirectional=stream_is_unidirectional(client_stream_id) + is_unidirectional=stream_is_unidirectional(client_stream_id), ) child_layer.open_server_stream(stream_id) self.server_stream_ids[stream_id] = child_layer - yield from self.event_to_child(child_layer, events.OpenConnectionCompleted(command, None)) + yield from self.event_to_child( + child_layer, events.OpenConnectionCompleted(command, None) + ) else: - raise AssertionError(f"Unexpected stream connection command: {command!r}") + raise AssertionError( + f"Unexpected stream connection command: {command!r}" + ) # remember blocking and wakeup commands else: @@ -701,7 +759,9 @@ def event_to_child(self, child_layer: layer.Layer, event: events.Event) -> layer self.connections[command.connection] = child_layer yield command - def get_next_available_stream_id(self, is_client: bool, is_unidirectional: bool = False) -> int: + def get_next_available_stream_id( + self, is_client: bool, is_unidirectional: bool = False + ) -> int: index = (int(is_unidirectional) << 1) | int(not is_client) stream_id = self.next_stream_id[index] self.next_stream_id[index] = stream_id + 4 @@ -715,7 +775,12 @@ class QuicLayer(tunnel.TunnelLayer): quic: QuicConnection | None = None tls: QuicTlsSettings | None = None - def __init__(self, context: context.Context, conn: connection.Connection, time: Callable[[], float] | None) -> None: + def __init__( + self, + context: context.Context, + conn: connection.Connection, + time: Callable[[], float] | None, + ) -> None: super().__init__(context, tunnel_connection=conn, conn=conn) self.child_layer = layer.NextLayer(self.context, ask_on_start=True) self._time = time or ctx.master.event_loop.time @@ -723,10 +788,7 @@ def __init__(self, context: context.Context, conn: connection.Connection, time: conn.tls = True def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - if ( - isinstance(event, events.Wakeup) - and event.command in self._wakeup_commands - ): + if isinstance(event, events.Wakeup) and event.command in self._wakeup_commands: # TunnelLayer has no understanding of wakeups, so we turn this into an empty DataReceived event # which TunnelLayer recognizes as belonging to our connection. assert self.quic @@ -746,15 +808,16 @@ def event_to_child(self, event: events.Event) -> layer.CommandGenerator[None]: if self.quic: yield from self.tls_interact() - def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[None]: + def _handle_command( + self, command: commands.Command + ) -> layer.CommandGenerator[None]: """Turns stream commands into aioquic connection invocations.""" - if ( - isinstance(command, QuicStreamCommand) - and command.connection is self.conn - ): + if isinstance(command, QuicStreamCommand) and command.connection is self.conn: assert self.quic if isinstance(command, SendQuicStreamData): - self.quic.send_stream_data(command.stream_id, command.data, command.end_stream) + self.quic.send_stream_data( + command.stream_id, command.data, command.end_stream + ) elif isinstance(command, ResetQuicStream): self.quic.reset_stream(command.stream_id, command.error_code) elif isinstance(command, StopQuicStream): @@ -766,7 +829,9 @@ def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[N else: yield from super()._handle_command(command) - def start_tls(self, original_destination_connection_id: bytes | None) -> layer.CommandGenerator[None]: + def start_tls( + self, original_destination_connection_id: bytes | None + ) -> layer.CommandGenerator[None]: """Initiates the aioquic connection.""" # must only be called if QUIC is uninitialized @@ -780,7 +845,9 @@ def start_tls(self, original_destination_connection_id: bytes | None) -> layer.C else: yield QuicStartServerHook(tls_data) if not tls_data.settings: - yield commands.Log(f"No QUIC context was provided, failing connection.", ERROR) + yield commands.Log( + f"No QUIC context was provided, failing connection.", ERROR + ) yield commands.CloseConnection(self.conn) return @@ -812,15 +879,16 @@ def tls_interact(self) -> layer.CommandGenerator[None]: # request a new wakeup if all pending requests trigger at a later time timer = self.quic.get_timer() - if ( - timer is not None - and not any(existing <= timer for existing in self._wakeup_commands.values()) + if timer is not None and not any( + existing <= timer for existing in self._wakeup_commands.values() ): command = commands.RequestWakeup(timer - self._time()) self._wakeup_commands[command] = timer yield command - def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bool, str | None]]: + def receive_handshake_data( + self, data: bytes + ) -> layer.CommandGenerator[tuple[bool, str | None]]: assert self.quic # forward incoming data to aioquic @@ -854,18 +922,25 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo f"{self.debug}[quic] tls established: {self.conn}", DEBUG ) if self.conn is self.context.client: - yield TlsEstablishedClientHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + yield TlsEstablishedClientHook( + QuicTlsData(self.conn, self.context, settings=self.tls) + ) else: - yield TlsEstablishedServerHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + yield TlsEstablishedServerHook( + QuicTlsData(self.conn, self.context, settings=self.tls) + ) yield from self.tls_interact() return True, None - elif isinstance(event, ( - quic_events.ConnectionIdIssued, - quic_events.ConnectionIdRetired, - quic_events.PingAcknowledged, - quic_events.ProtocolNegotiated, - )): + elif isinstance( + event, + ( + quic_events.ConnectionIdIssued, + quic_events.ConnectionIdRetired, + quic_events.PingAcknowledged, + quic_events.ProtocolNegotiated, + ), + ): pass else: raise AssertionError(f"Unexpected event: {event!r}") @@ -877,9 +952,13 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: self.conn.error = err if self.conn is self.context.client: - yield TlsFailedClientHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + yield TlsFailedClientHook( + QuicTlsData(self.conn, self.context, settings=self.tls) + ) else: - yield TlsFailedServerHook(QuicTlsData(self.conn, self.context, settings=self.tls)) + yield TlsFailedServerHook( + QuicTlsData(self.conn, self.context, settings=self.tls) + ) yield from super().on_handshake_error(err) def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: @@ -895,7 +974,8 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: if self.debug: reason = event.reason_phrase or error_code_to_str(event.error_code) yield commands.Log( - f"{self.debug}[quic] close_notify {self.conn} (reason={reason})", DEBUG + f"{self.debug}[quic] close_notify {self.conn} (reason={reason})", + DEBUG, ) # We don't rely on `ConnectionTerminated` to dispatch `QuicConnectionClosed`, because # after aioquic receives a termination frame, it still waits for the next `handle_timer` @@ -905,17 +985,28 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: yield commands.CloseConnection(self.tunnel_connection) return # we don't handle any further events, nor do/can we transmit data, so exit elif isinstance(event, quic_events.DatagramFrameReceived): - yield from self.event_to_child(events.DataReceived(self.conn, event.data)) + yield from self.event_to_child( + events.DataReceived(self.conn, event.data) + ) elif isinstance(event, quic_events.StreamDataReceived): - yield from self.event_to_child(QuicStreamDataReceived(self.conn, event.stream_id, event.data, event.end_stream)) + yield from self.event_to_child( + QuicStreamDataReceived( + self.conn, event.stream_id, event.data, event.end_stream + ) + ) elif isinstance(event, quic_events.StreamReset): - yield from self.event_to_child(QuicStreamReset(self.conn, event.stream_id, event.error_code)) - elif isinstance(event, ( - quic_events.ConnectionIdIssued, - quic_events.ConnectionIdRetired, - quic_events.PingAcknowledged, - quic_events.ProtocolNegotiated, - )): + yield from self.event_to_child( + QuicStreamReset(self.conn, event.stream_id, event.error_code) + ) + elif isinstance( + event, + ( + quic_events.ConnectionIdIssued, + quic_events.ConnectionIdRetired, + quic_events.PingAcknowledged, + quic_events.ProtocolNegotiated, + ), + ): pass else: raise AssertionError(f"Unexpected event: {event!r}") @@ -931,7 +1022,12 @@ def receive_close(self) -> layer.CommandGenerator[None]: QuicErrorCode.NO_ERROR, None, "Connection closed." ) yield from self.event_to_child( - QuicConnectionClosed(self.conn, close_event.error_code, close_event.frame_type, close_event.reason_phrase) + QuicConnectionClosed( + self.conn, + close_event.error_code, + close_event.frame_type, + close_event.reason_phrase, + ) ) def send_data(self, data: bytes) -> layer.CommandGenerator[None]: @@ -941,11 +1037,15 @@ def send_data(self, data: bytes) -> layer.CommandGenerator[None]: self.quic.send_datagram_frame(data) yield from self.tls_interact() - def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: + def send_close( + self, command: commands.CloseConnection + ) -> layer.CommandGenerator[None]: # properly close the QUIC connection if self.quic: if isinstance(command, CloseQuicConnection): - self.quic.close(command.error_code, command.frame_type, command.reason_phrase) + self.quic.close( + command.error_code, command.frame_type, command.reason_phrase + ) else: self.quic.close() yield from self.tls_interact() @@ -959,13 +1059,17 @@ class ServerQuicLayer(QuicLayer): wait_for_clienthello: bool = False - def __init__(self, context: context.Context, conn: connection.Server | None = None, time: Callable[[], float] | None = None): + def __init__( + self, + context: context.Context, + conn: connection.Server | None = None, + time: Callable[[], float] | None = None, + ): super().__init__(context, conn or context.server, time) def start_handshake(self) -> layer.CommandGenerator[None]: - wait_for_clienthello = ( - not self.command_to_reply_to - and isinstance(self.child_layer, ClientQuicLayer) + wait_for_clienthello = not self.command_to_reply_to and isinstance( + self.child_layer, ClientQuicLayer ) if wait_for_clienthello: self.wait_for_clienthello = True @@ -999,7 +1103,9 @@ class ClientQuicLayer(QuicLayer): server_tls_available: bool """Indicates whether the parent layer is a ServerQuicLayer.""" - def __init__(self, context: context.Context, time: Callable[[], float] | None = None) -> None: + def __init__( + self, context: context.Context, time: Callable[[], float] | None = None + ) -> None: # same as ClientTLSLayer, we might be nested in some other transport if context.client.tls: context.client.alpn = None @@ -1018,7 +1124,9 @@ def __init__(self, context: context.Context, time: Callable[[], float] | None = def start_handshake(self) -> layer.CommandGenerator[None]: yield from () - def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bool, str | None]]: + def receive_handshake_data( + self, data: bytes + ) -> layer.CommandGenerator[tuple[bool, str | None]]: # if we already had a valid client hello, don't process further packets if self.tls: return (yield from super().receive_handshake_data(data)) @@ -1049,7 +1157,10 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # ensure it's (likely) a client handshake packet if len(data) < 1200 or header.packet_type != PACKET_TYPE_INITIAL: - return False, f"Invalid handshake received, roaming not supported. ({data.hex()})" + return ( + False, + f"Invalid handshake received, roaming not supported. ({data.hex()})", + ) # extract the client hello try: @@ -1068,7 +1179,8 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo # replace the QUIC layer with an UDP layer if requested if tls_clienthello.ignore_connection: self.conn = self.tunnel_connection = connection.Client( - peername=("ignore-conn", 0), sockname=("ignore-conn", 0), + peername=("ignore-conn", 0), + sockname=("ignore-conn", 0), transport_protocol="udp", state=connection.ConnectionState.OPEN, ) @@ -1083,7 +1195,9 @@ def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[tuple[bo parent_layer.handle_event = replacement_layer.handle_event # type: ignore parent_layer._handle_event = replacement_layer._handle_event # type: ignore yield from parent_layer.handle_event(events.Start()) - yield from parent_layer.handle_event(events.DataReceived(self.context.client, data)) + yield from parent_layer.handle_event( + events.DataReceived(self.context.client, data) + ) return True, None # start the server QUIC connection if demanded and available @@ -1122,4 +1236,6 @@ def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: def errored(self, event: events.Event) -> layer.CommandGenerator[None]: if self.debug is not None: - yield commands.Log(f"{self.debug}[quic] Swallowing {event} as handshake failed.", DEBUG) + yield commands.Log( + f"{self.debug}[quic] Swallowing {event} as handshake failed.", DEBUG + ) diff --git a/mitmproxy/proxy/layers/tcp.py b/mitmproxy/proxy/layers/tcp.py index 2d1ff7305b..0272d4ed51 100644 --- a/mitmproxy/proxy/layers/tcp.py +++ b/mitmproxy/proxy/layers/tcp.py @@ -1,10 +1,14 @@ from dataclasses import dataclass from typing import Optional -from mitmproxy import flow, tcp -from mitmproxy.proxy import commands, events, layer +from mitmproxy import flow +from mitmproxy import tcp +from mitmproxy.connection import Connection +from mitmproxy.connection import ConnectionState +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.commands import StartHook -from mitmproxy.connection import ConnectionState, Connection from mitmproxy.proxy.context import Context from mitmproxy.proxy.events import MessageInjected from mitmproxy.proxy.utils import expect diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 1e00b8be8f..bb74b8b496 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -1,18 +1,28 @@ import struct -from logging import DEBUG, ERROR, INFO, WARNING - import time +from collections.abc import Iterator from dataclasses import dataclass -from typing import Iterator, Optional +from logging import DEBUG +from logging import ERROR +from logging import INFO +from logging import WARNING +from typing import Optional from OpenSSL import SSL -from mitmproxy import certs, connection -from mitmproxy.proxy import commands, events, layer, tunnel +from mitmproxy import certs +from mitmproxy import connection +from mitmproxy.proxy import commands from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layer +from mitmproxy.proxy import tunnel from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.layers import tcp, udp -from mitmproxy.tls import ClientHello, ClientHelloData, TlsData +from mitmproxy.proxy.layers import tcp +from mitmproxy.proxy.layers import udp +from mitmproxy.tls import ClientHello +from mitmproxy.tls import ClientHelloData +from mitmproxy.tls import TlsData from mitmproxy.utils import human @@ -96,7 +106,7 @@ def is_dtls_handshake_record(d: bytes) -> bool: True, if the passed bytes start with the DTLS record magic bytes False, otherwise. """ - return len(d) >= 3 and d[0] == 0x16 and d[1] == 0xfe and d[2] == 0xfd + return len(d) >= 3 and d[0] == 0x16 and d[1] == 0xFE and d[2] == 0xFD def dtls_handshake_record_contents(data: bytes) -> Iterator[bytes]: @@ -136,7 +146,9 @@ def get_dtls_client_hello(data: bytes) -> Optional[bytes]: client_hello += d if len(client_hello) >= 13: # comment about slicing: we skip the epoch and sequence number - client_hello_size = struct.unpack("!I", b"\x00" + client_hello[9:12])[0] + 12 + client_hello_size = ( + struct.unpack("!I", b"\x00" + client_hello[9:12])[0] + 12 + ) if len(client_hello) >= client_hello_size: return client_hello[:client_hello_size] return None @@ -257,7 +269,9 @@ def __init__(self, context: context.Context, conn: connection.Connection): conn.tls = True def __repr__(self): - return super().__repr__().replace(")", f" {self.conn.sni!r} {self.conn.alpn!r})") + return ( + super().__repr__().replace(")", f" {self.conn.sni!r} {self.conn.alpn!r})") + ) @property def is_dtls(self): @@ -265,7 +279,7 @@ def is_dtls(self): @property def proto_name(self): - return 'DTLS' if self.is_dtls else 'TLS' + return "DTLS" if self.is_dtls else "TLS" def start_tls(self) -> layer.CommandGenerator[None]: assert not self.tls @@ -421,9 +435,7 @@ def receive_data(self, data: bytes) -> layer.CommandGenerator[None]: if close: self.conn.state &= ~connection.ConnectionState.CAN_READ if self.debug: - yield commands.Log( - f"{self.debug}[tls] close_notify {self.conn}", DEBUG - ) + yield commands.Log(f"{self.debug}[tls] close_notify {self.conn}", DEBUG) yield from self.event_to_child(events.ConnectionClosed(self.conn)) def receive_close(self) -> layer.CommandGenerator[None]: @@ -440,7 +452,9 @@ def send_data(self, data: bytes) -> layer.CommandGenerator[None]: pass yield from self.tls_interact() - def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: + def send_close( + self, command: commands.CloseConnection + ) -> layer.CommandGenerator[None]: # We should probably shutdown the TLS connection properly here. yield from super().send_close(command) @@ -659,7 +673,9 @@ def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: def errored(self, event: events.Event) -> layer.CommandGenerator[None]: if self.debug is not None: - yield commands.Log(f"{self.debug}[tls] Swallowing {event} as handshake failed.", DEBUG) + yield commands.Log( + f"{self.debug}[tls] Swallowing {event} as handshake failed.", DEBUG + ) class MockTLSLayer(TLSLayer): diff --git a/mitmproxy/proxy/layers/udp.py b/mitmproxy/proxy/layers/udp.py index 3026a71cea..e80fc7b9db 100644 --- a/mitmproxy/proxy/layers/udp.py +++ b/mitmproxy/proxy/layers/udp.py @@ -1,10 +1,13 @@ from dataclasses import dataclass from typing import Optional -from mitmproxy import flow, udp -from mitmproxy.proxy import commands, events, layer -from mitmproxy.proxy.commands import StartHook +from mitmproxy import flow +from mitmproxy import udp from mitmproxy.connection import Connection +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer +from mitmproxy.proxy.commands import StartHook from mitmproxy.proxy.context import Context from mitmproxy.proxy.events import MessageInjected from mitmproxy.proxy.utils import expect diff --git a/mitmproxy/proxy/layers/websocket.py b/mitmproxy/proxy/layers/websocket.py index a2c57fee91..24c291b765 100644 --- a/mitmproxy/proxy/layers/websocket.py +++ b/mitmproxy/proxy/layers/websocket.py @@ -1,19 +1,23 @@ import time +from collections.abc import Iterator from dataclasses import dataclass -from typing import Iterator -import wsproto import wsproto.extensions import wsproto.frame_protocol import wsproto.utilities -from mitmproxy import connection, http, websocket -from mitmproxy.proxy import commands, events, layer +from wsproto import ConnectionState +from wsproto.frame_protocol import Opcode + +from mitmproxy import connection +from mitmproxy import http +from mitmproxy import websocket +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.commands import StartHook from mitmproxy.proxy.context import Context from mitmproxy.proxy.events import MessageInjected from mitmproxy.proxy.utils import expect -from wsproto import ConnectionState -from wsproto.frame_protocol import Opcode @dataclass diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 3c7587f0aa..b406156703 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -12,25 +12,36 @@ from __future__ import annotations import asyncio +import errno import json import logging import socket import textwrap import typing -from abc import ABCMeta, abstractmethod +from abc import ABCMeta +from abc import abstractmethod from contextlib import contextmanager from pathlib import Path -from typing import ClassVar, Generic, TypeVar, cast, get_args +from typing import cast +from typing import ClassVar +from typing import Generic +from typing import get_args +from typing import TypeVar -import errno import mitmproxy_wireguard as wg -from mitmproxy import ctx, flow, platform +from mitmproxy import ctx +from mitmproxy import flow +from mitmproxy import platform from mitmproxy.connection import Address from mitmproxy.master import Master -from mitmproxy.net import local_ip, udp +from mitmproxy.net import local_ip +from mitmproxy.net import udp from mitmproxy.net.udp_wireguard import WireGuardDatagramTransport -from mitmproxy.proxy import commands, layers, mode_specs, server +from mitmproxy.proxy import commands +from mitmproxy.proxy import layers +from mitmproxy.proxy import mode_specs +from mitmproxy.proxy import server from mitmproxy.proxy.context import Context from mitmproxy.proxy.layer import Layer from mitmproxy.utils import human @@ -55,14 +66,16 @@ async def handle_hook(self, hook: commands.StartHook) -> None: await data.wait_for_resume() # pragma: no cover -M = TypeVar('M', bound=mode_specs.ProxyMode) +M = TypeVar("M", bound=mode_specs.ProxyMode) class ServerManager(typing.Protocol): connections: dict[tuple, ProxyConnectionHandler] @contextmanager - def register_connection(self, connection_id: tuple, handler: ProxyConnectionHandler): + def register_connection( + self, connection_id: tuple, handler: ProxyConnectionHandler + ): ... # pragma: no cover @@ -89,7 +102,7 @@ def __init_subclass__(cls, **kwargs): @classmethod def make( - cls: typing.Type[Self], + cls: type[Self], mode: mode_specs.ProxyMode | str, manager: ServerManager, ) -> Self: @@ -196,7 +209,9 @@ def handle_udp_datagram( reader = cast(udp.DatagramReader, handler.transports[handler.client].reader) reader.feed_data(data, remote_addr) - async def handle_udp_connection(self, connection_id: tuple, handler: ProxyConnectionHandler) -> None: + async def handle_udp_connection( + self, connection_id: tuple, handler: ProxyConnectionHandler + ) -> None: with self.manager.register_connection(connection_id, handler): await handler.handle_client() @@ -220,7 +235,9 @@ async def start(self) -> None: self.last_exception = e message = f"{self.mode.description} failed to listen on {host or '*'}:{port} with {e}" if e.errno == errno.EADDRINUSE and self.mode.custom_listen_port is None: - assert self.mode.custom_listen_host is None # since [@ [listen_addr:]listen_port] + assert ( + self.mode.custom_listen_host is None + ) # since [@ [listen_addr:]listen_port] message += f"\nTry specifying a different port by using `--mode {self.mode.full_spec}@{port + 1}`." raise OSError(e.errno, message, e.filename) from e except Exception as e: @@ -261,9 +278,13 @@ async def listen(self, host: str, port: int) -> asyncio.Server | udp.UdpServer: s.bind(("", 0)) fixed_port = s.getsockname()[1] s.close() - return await asyncio.start_server(self.handle_tcp_connection, host, fixed_port) + return await asyncio.start_server( + self.handle_tcp_connection, host, fixed_port + ) except Exception as e: - logger.debug(f"Failed to listen on a single port ({e!r}), falling back to default behavior.") + logger.debug( + f"Failed to listen on a single port ({e!r}), falling back to default behavior." + ) return await asyncio.start_server(self.handle_tcp_connection, host, port) elif self.mode.transport_protocol == "udp": # create_datagram_endpoint only creates one socket, so the workaround above doesn't apply @@ -309,17 +330,24 @@ async def start(self) -> None: try: if not conf_path.exists(): conf_path.parent.mkdir(parents=True, exist_ok=True) - conf_path.write_text(json.dumps({ - "server_key": wg.genkey(), - "client_key": wg.genkey(), - }, indent=4)) + conf_path.write_text( + json.dumps( + { + "server_key": wg.genkey(), + "client_key": wg.genkey(), + }, + indent=4, + ) + ) try: c = json.loads(conf_path.read_text()) self.server_key = c["server_key"] self.client_key = c["client_key"] except Exception as e: - raise ValueError(f"Invalid configuration file ({conf_path}): {e}") from e + raise ValueError( + f"Invalid configuration file ({conf_path}): {e}" + ) from e # error early on invalid keys p = wg.pubkey(self.client_key) _ = wg.pubkey(self.server_key) @@ -355,7 +383,8 @@ def client_conf(self) -> str | None: return None host = local_ip.get_local_ip() or local_ip.get_local_ip6() port = self.mode.listen_port(ctx.options.listen_port) - return textwrap.dedent(f""" + return textwrap.dedent( + f""" [Interface] PrivateKey = {self.client_key} Address = 10.0.0.1/32 @@ -365,13 +394,11 @@ def client_conf(self) -> str | None: PublicKey = {wg.pubkey(self.server_key)} AllowedIPs = 0.0.0.0/0 Endpoint = {host}:{port} - """).strip() + """ + ).strip() def to_json(self) -> dict: - return { - "wireguard_conf": self.client_conf(), - **super().to_json() - } + return {"wireguard_conf": self.client_conf(), **super().to_json()} async def stop(self) -> None: assert self._server is not None @@ -390,15 +417,12 @@ def listen_addrs(self) -> tuple[Address, ...]: async def wg_handle_tcp_connection(self, stream: wg.TcpStream) -> None: await self.handle_tcp_connection(stream, stream) - def wg_handle_udp_datagram(self, data: bytes, remote_addr: Address, local_addr: Address) -> None: + def wg_handle_udp_datagram( + self, data: bytes, remote_addr: Address, local_addr: Address + ) -> None: assert self._server is not None transport = WireGuardDatagramTransport(self._server, local_addr, remote_addr) - self.handle_udp_datagram( - transport, - data, - remote_addr, - local_addr - ) + self.handle_udp_datagram(transport, data, remote_addr, local_addr) class RegularInstance(AsyncioServerInstance[mode_specs.RegularMode]): diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index 0092c8077a..b13762ced7 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -19,14 +19,16 @@ RegularMode.parse("socks5") # ValueError """ - from __future__ import annotations import dataclasses -from abc import ABCMeta, abstractmethod +from abc import ABCMeta +from abc import abstractmethod from dataclasses import dataclass from functools import cache -from typing import ClassVar, Literal, Type, TypeVar +from typing import ClassVar +from typing import Literal +from typing import TypeVar from mitmproxy.coretypes.serializable import Serializable from mitmproxy.net import server_spec @@ -41,6 +43,7 @@ class ProxyMode(Serializable, metaclass=ABCMeta): Parsed representation of a proxy mode spec. Subclassed for each specific mode, which then does its own data validation. """ + full_spec: str """The full proxy mode spec as entered by the user.""" data: str @@ -50,9 +53,11 @@ class ProxyMode(Serializable, metaclass=ABCMeta): custom_listen_port: int | None """A custom listen port, if specified in the spec.""" - type_name: ClassVar[str] # automatically derived from the class name in __init_subclass__ + type_name: ClassVar[ + str + ] # automatically derived from the class name in __init_subclass__ """The unique name for this proxy mode, e.g. "regular" or "reverse".""" - __types: ClassVar[dict[str, Type[ProxyMode]]] = {} + __types: ClassVar[dict[str, type[ProxyMode]]] = {} def __init_subclass__(cls, **kwargs): cls.type_name = cls.__name__.removesuffix("Mode").lower() @@ -85,7 +90,7 @@ def transport_protocol(self) -> Literal["tcp", "udp"]: @classmethod @cache - def parse(cls: Type[Self], spec: str) -> Self: + def parse(cls: type[Self], spec: str) -> Self: """ Parse a proxy mode specification and return the corresponding `ProxyMode` instance. """ @@ -121,10 +126,7 @@ def parse(cls: Type[Self], spec: str) -> Self: raise ValueError(f"{mode!r} is not a spec for a {cls.type_name} mode") return mode_cls( - full_spec=spec, - data=data, - custom_listen_host=host, - custom_listen_port=port + full_spec=spec, data=data, custom_listen_host=host, custom_listen_port=port ) def listen_host(self, default: str | None = None) -> str: @@ -165,8 +167,8 @@ def set_state(self, state): raise dataclasses.FrozenInstanceError("Proxy modes are immutable.") -TCP: Literal['tcp', 'udp'] = "tcp" -UDP: Literal['tcp', 'udp'] = "udp" +TCP: Literal["tcp", "udp"] = "tcp" +UDP: Literal["tcp", "udp"] = "udp" def _check_empty(data): @@ -176,6 +178,7 @@ def _check_empty(data): class RegularMode(ProxyMode): """A regular HTTP(S) proxy that is interfaced with `HTTP CONNECT` calls (or absolute-form HTTP requests).""" + description = "HTTP(S) proxy" transport_protocol = TCP @@ -185,6 +188,7 @@ def __post_init__(self) -> None: class TransparentMode(ProxyMode): """A transparent proxy, see https://docs.mitmproxy.org/dev/howto-transparent/""" + description = "transparent proxy" transport_protocol = TCP @@ -194,6 +198,7 @@ def __post_init__(self) -> None: class UpstreamMode(ProxyMode): """A regular HTTP(S) proxy, but all connections are forwarded to a second upstream HTTP(S) proxy.""" + description = "HTTP(S) proxy (upstream mode)" transport_protocol = TCP scheme: Literal["http", "https"] @@ -209,9 +214,12 @@ def __post_init__(self) -> None: class ReverseMode(ProxyMode): """A reverse proxy. This acts like a normal server, but redirects all requests to a fixed target.""" + description = "reverse proxy" transport_protocol = TCP - scheme: Literal["http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns", "quic"] + scheme: Literal[ + "http", "https", "http3", "tls", "dtls", "tcp", "udp", "dns", "quic" + ] address: tuple[str, int] # noinspection PyDataclass @@ -230,6 +238,7 @@ def default_port(self) -> int: class Socks5Mode(ProxyMode): """A SOCKSv5 proxy.""" + description = "SOCKS v5 proxy" default_port = 1080 transport_protocol = TCP @@ -240,6 +249,7 @@ def __post_init__(self) -> None: class DnsMode(ProxyMode): """A DNS server.""" + description = "DNS server" default_port = 53 transport_protocol = UDP @@ -253,6 +263,7 @@ class Http3Mode(ProxyMode): A regular HTTP3 proxy that is interfaced with absolute-form HTTP requests. (This class will be merged into `RegularMode` once the UDP implementation is deemed stable enough.) """ + description = "HTTP3 proxy" transport_protocol = UDP @@ -262,6 +273,7 @@ def __post_init__(self) -> None: class WireGuardMode(ProxyMode): """Proxy Server based on WireGuard""" + description = "WireGuard server" default_port = 51820 transport_protocol = UDP diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 24959848c5..ac717dfae9 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -10,23 +10,35 @@ import asyncio import collections import logging - import time import traceback -from collections.abc import Awaitable, Callable, MutableMapping +from collections.abc import Awaitable +from collections.abc import Callable +from collections.abc import MutableMapping from contextlib import contextmanager from dataclasses import dataclass -from typing import Optional, Union +from typing import Optional +from typing import Union import mitmproxy_wireguard as wg from OpenSSL import SSL -from mitmproxy import http, options as moptions, tls +from mitmproxy import http +from mitmproxy import options as moptions +from mitmproxy import tls +from mitmproxy.connection import Address +from mitmproxy.connection import Client +from mitmproxy.connection import Connection +from mitmproxy.connection import ConnectionState +from mitmproxy.net import udp +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer +from mitmproxy.proxy import layers +from mitmproxy.proxy import mode_specs +from mitmproxy.proxy import server_hooks from mitmproxy.proxy.context import Context from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy import commands, events, layer, layers, mode_specs, server_hooks -from mitmproxy.connection import Address, Client, Connection, ConnectionState -from mitmproxy.net import udp from mitmproxy.utils import asyncio_utils from mitmproxy.utils import human from mitmproxy.utils.data import pkg_data @@ -80,8 +92,12 @@ def disarm(self): @dataclass class ConnectionIO: handler: Optional[asyncio.Task] = None - reader: Optional[Union[asyncio.StreamReader, udp.DatagramReader, wg.TcpStream]] = None - writer: Optional[Union[asyncio.StreamWriter, udp.DatagramWriter, wg.TcpStream]] = None + reader: Optional[ + Union[asyncio.StreamReader, udp.DatagramReader, wg.TcpStream] + ] = None + writer: Optional[ + Union[asyncio.StreamWriter, udp.DatagramWriter, wg.TcpStream] + ] = None class ConnectionHandler(metaclass=abc.ABCMeta): @@ -135,7 +151,10 @@ async def handle_client(self) -> None: self.server_event(events.Start()) await asyncio.wait([handler]) if not handler.cancelled() and (e := handler.exception()): - self.log(f"mitmproxy has crashed!\n{traceback.format_exception(e)}", logging.ERROR) + self.log( + f"mitmproxy has crashed!\n{traceback.format_exception(e)}", + logging.ERROR, + ) watch.cancel() while self.wakeup_timer: @@ -331,11 +350,7 @@ async def handle_hook(self, hook: commands.StartHook) -> None: pass def log(self, message: str, level: int = logging.INFO) -> None: - logger.log( - level, - message, - extra={"client": self.client.peername} - ) + logger.log(level, message, extra={"client": self.client.peername}) def server_event(self, event: events.Event) -> None: self.timeout_watchdog.register_activity() diff --git a/mitmproxy/proxy/server_hooks.py b/mitmproxy/proxy/server_hooks.py index 22e1e5418b..a00c6ca195 100644 --- a/mitmproxy/proxy/server_hooks.py +++ b/mitmproxy/proxy/server_hooks.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from mitmproxy import connection from . import commands +from mitmproxy import connection @dataclass diff --git a/mitmproxy/proxy/tunnel.py b/mitmproxy/proxy/tunnel.py index 435693de6a..5aa42cdedb 100644 --- a/mitmproxy/proxy/tunnel.py +++ b/mitmproxy/proxy/tunnel.py @@ -1,9 +1,14 @@ import time -from enum import Enum, auto -from typing import Optional, Union +from enum import auto +from enum import Enum +from typing import Optional +from typing import Union from mitmproxy import connection -from mitmproxy.proxy import commands, context, events, layer +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.layer import Layer @@ -108,7 +113,9 @@ def _handshake_finished(self, err: Optional[str]): yield from self.event_to_child(evt) self._event_queue.clear() - def _handle_command(self, command: commands.Command) -> layer.CommandGenerator[None]: + def _handle_command( + self, command: commands.Command + ) -> layer.CommandGenerator[None]: if ( isinstance(command, commands.ConnectionCommand) and command.connection == self.conn @@ -170,7 +177,9 @@ def receive_close(self) -> layer.CommandGenerator[None]: def send_data(self, data: bytes) -> layer.CommandGenerator[None]: yield commands.SendData(self.tunnel_connection, data) - def send_close(self, command: commands.CloseConnection) -> layer.CommandGenerator[None]: + def send_close( + self, command: commands.CloseConnection + ) -> layer.CommandGenerator[None]: yield command diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py index 9d9546568d..587a65e78e 100644 --- a/mitmproxy/script/concurrent.py +++ b/mitmproxy/script/concurrent.py @@ -2,9 +2,9 @@ This module provides a @concurrent decorator primitive to offload computations from mitmproxy's main master thread. """ - import asyncio import inspect + from mitmproxy import hooks diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py index 2ad13336c4..eec4c26fc9 100644 --- a/mitmproxy/tcp.py +++ b/mitmproxy/tcp.py @@ -1,6 +1,7 @@ import time -from mitmproxy import connection, flow +from mitmproxy import connection +from mitmproxy import flow from mitmproxy.coretypes import serializable diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index 24dbaebdaa..82ee2de304 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -2,10 +2,11 @@ import mitmproxy.master import mitmproxy.options -from mitmproxy import hooks from mitmproxy import command from mitmproxy import eventsequence -from mitmproxy.addons import script, core +from mitmproxy import hooks +from mitmproxy.addons import core +from mitmproxy.addons import script class context: diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index bcb2d211b4..b632abc67e 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -1,5 +1,8 @@ import uuid -from typing import Optional, Union +from typing import Optional +from typing import Union + +from wsproto.frame_protocol import Opcode from mitmproxy import connection from mitmproxy import dns @@ -10,9 +13,10 @@ from mitmproxy import websocket from mitmproxy.connection import ConnectionState from mitmproxy.proxy.mode_specs import ProxyMode -from mitmproxy.test.tutils import tdnsreq, tdnsresp -from mitmproxy.test.tutils import treq, tresp -from wsproto.frame_protocol import Opcode +from mitmproxy.test.tutils import tdnsreq +from mitmproxy.test.tutils import tdnsresp +from mitmproxy.test.tutils import treq +from mitmproxy.test.tutils import tresp def ttcpflow( diff --git a/mitmproxy/tls.py b/mitmproxy/tls.py index 98fed77a44..a8d93ffdab 100644 --- a/mitmproxy/tls.py +++ b/mitmproxy/tls.py @@ -3,10 +3,11 @@ from typing import Optional from kaitaistruct import KaitaiStream - from OpenSSL import SSL + from mitmproxy import connection -from mitmproxy.contrib.kaitaistruct import tls_client_hello, dtls_client_hello +from mitmproxy.contrib.kaitaistruct import dtls_client_hello +from mitmproxy.contrib.kaitaistruct import tls_client_hello from mitmproxy.net import check from mitmproxy.proxy import context @@ -18,7 +19,7 @@ class ClientHello: _raw_bytes: bytes - def __init__(self, raw_client_hello: bytes, dtls: bool=False): + def __init__(self, raw_client_hello: bytes, dtls: bool = False): """Create a TLS ClientHello object from raw bytes.""" self._raw_bytes = raw_client_hello if dtls: diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index 53acbf4bd2..9c2fcb3a81 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -1,6 +1,7 @@ import abc from collections.abc import Sequence -from typing import NamedTuple, Optional +from typing import NamedTuple +from typing import Optional import urwid from urwid.text_layout import calc_coords diff --git a/mitmproxy/tools/console/commandexecutor.py b/mitmproxy/tools/console/commandexecutor.py index d683a1e4b5..fea007fe9a 100644 --- a/mitmproxy/tools/console/commandexecutor.py +++ b/mitmproxy/tools/console/commandexecutor.py @@ -3,7 +3,6 @@ from mitmproxy import exceptions from mitmproxy import flow - from mitmproxy.tools.console import overlay from mitmproxy.tools.console import signals diff --git a/mitmproxy/tools/console/commands.py b/mitmproxy/tools/console/commands.py index 78564ef176..ee10492549 100644 --- a/mitmproxy/tools/console/commands.py +++ b/mitmproxy/tools/console/commands.py @@ -1,6 +1,7 @@ -import urwid import textwrap +import urwid + from mitmproxy import command from mitmproxy.tools.console import layoutwidget from mitmproxy.tools.console import signals diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 65c12db15d..58f2be69a9 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,22 +1,23 @@ import enum -import platform import math +import platform from collections.abc import Iterable from functools import lru_cache -from typing import Optional, Union +from typing import Optional +from typing import Union -from publicsuffix2 import get_sld, get_tld - -import urwid import urwid.util +from publicsuffix2 import get_sld +from publicsuffix2 import get_tld +from mitmproxy import dns from mitmproxy import flow +from mitmproxy.dns import DNSFlow from mitmproxy.http import HTTPFlow -from mitmproxy.utils import human, emoji from mitmproxy.tcp import TCPFlow from mitmproxy.udp import UDPFlow -from mitmproxy import dns -from mitmproxy.dns import DNSFlow +from mitmproxy.utils import emoji +from mitmproxy.utils import human # Detect Windows Subsystem for Linux and Windows IS_WINDOWS_OR_WSL = ( diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index f88603d0a2..6810bc42a5 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -3,7 +3,8 @@ from collections.abc import Sequence import mitmproxy.types -from mitmproxy import command, command_lexer +from mitmproxy import command +from mitmproxy import command_lexer from mitmproxy import contentviews from mitmproxy import ctx from mitmproxy import dns diff --git a/mitmproxy/tools/console/eventlog.py b/mitmproxy/tools/console/eventlog.py index da53539e11..ab6a03c658 100644 --- a/mitmproxy/tools/console/eventlog.py +++ b/mitmproxy/tools/console/eventlog.py @@ -1,8 +1,9 @@ import collections import urwid -from mitmproxy.tools.console import layoutwidget + from mitmproxy import log +from mitmproxy.tools.console import layoutwidget class LogBufferWalker(urwid.SimpleListWalker): diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 56161d50ce..9b55ef2d76 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -4,8 +4,10 @@ import mitmproxy.flow from mitmproxy import http -from mitmproxy.tools.console import common, searchable -from mitmproxy.utils import human, strutils +from mitmproxy.tools.console import common +from mitmproxy.tools.console import searchable +from mitmproxy.utils import human +from mitmproxy.utils import strutils def maybe_timestamp(base, attr): diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 20e969c395..8aae522a8b 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -1,5 +1,4 @@ import logging - import math import sys from functools import lru_cache diff --git a/mitmproxy/tools/console/grideditor/__init__.py b/mitmproxy/tools/console/grideditor/__init__.py index c13ea70e37..6bcae5b94b 100644 --- a/mitmproxy/tools/console/grideditor/__init__.py +++ b/mitmproxy/tools/console/grideditor/__init__.py @@ -1,17 +1,15 @@ from . import base -from .editors import ( - CookieAttributeEditor, - CookieEditor, - DataViewer, - OptionsEditor, - PathEditor, - QueryEditor, - RequestHeaderEditor, - RequestMultipartEditor, - RequestUrlEncodedEditor, - ResponseHeaderEditor, - SetCookieEditor, -) +from .editors import CookieAttributeEditor +from .editors import CookieEditor +from .editors import DataViewer +from .editors import OptionsEditor +from .editors import PathEditor +from .editors import QueryEditor +from .editors import RequestHeaderEditor +from .editors import RequestMultipartEditor +from .editors import RequestUrlEncodedEditor +from .editors import ResponseHeaderEditor +from .editors import SetCookieEditor __all__ = [ "base", diff --git a/mitmproxy/tools/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py index 5759c0cf38..a59717487f 100644 --- a/mitmproxy/tools/console/grideditor/base.py +++ b/mitmproxy/tools/console/grideditor/base.py @@ -1,16 +1,23 @@ import abc import copy import os -from collections.abc import Callable, Container, Iterable, MutableSequence, Sequence -from typing import Any, AnyStr, ClassVar, Optional +from collections.abc import Callable +from collections.abc import Container +from collections.abc import Iterable +from collections.abc import MutableSequence +from collections.abc import Sequence +from typing import Any +from typing import AnyStr +from typing import ClassVar +from typing import Optional import urwid -from mitmproxy.utils import strutils +import mitmproxy.tools.console.master from mitmproxy import exceptions -from mitmproxy.tools.console import signals from mitmproxy.tools.console import layoutwidget -import mitmproxy.tools.console.master +from mitmproxy.tools.console import signals +from mitmproxy.utils import strutils def read_file(filename: str, escaped: bool) -> AnyStr: diff --git a/mitmproxy/tools/console/grideditor/col_bytes.py b/mitmproxy/tools/console/grideditor/col_bytes.py index f291474170..9af1a3544d 100644 --- a/mitmproxy/tools/console/grideditor/col_bytes.py +++ b/mitmproxy/tools/console/grideditor/col_bytes.py @@ -1,4 +1,5 @@ import urwid + from mitmproxy.tools.console import signals from mitmproxy.tools.console.grideditor import base from mitmproxy.utils import strutils diff --git a/mitmproxy/tools/console/grideditor/col_subgrid.py b/mitmproxy/tools/console/grideditor/col_subgrid.py index be4b4271b5..17887b1af2 100644 --- a/mitmproxy/tools/console/grideditor/col_subgrid.py +++ b/mitmproxy/tools/console/grideditor/col_subgrid.py @@ -1,7 +1,8 @@ import urwid -from mitmproxy.tools.console.grideditor import base -from mitmproxy.tools.console import signals + from mitmproxy.net.http import cookies +from mitmproxy.tools.console import signals +from mitmproxy.tools.console.grideditor import base class Column(base.Column): @@ -20,9 +21,7 @@ def blank(self): def keypress(self, key: str, editor): if key in "rRe": - signals.status_message.send( - message="Press enter to edit this field." - ) + signals.status_message.send(message="Press enter to edit this field.") return elif key == "m_select": self.subeditor.grideditor = editor diff --git a/mitmproxy/tools/console/grideditor/col_text.py b/mitmproxy/tools/console/grideditor/col_text.py index d5ad1cba03..04dbb5ab04 100644 --- a/mitmproxy/tools/console/grideditor/col_text.py +++ b/mitmproxy/tools/console/grideditor/col_text.py @@ -4,7 +4,6 @@ In a nutshell, text columns are actually a proxy class for byte columns, which just encode/decodes contents. """ - from mitmproxy.tools.console import signals from mitmproxy.tools.console.grideditor import col_bytes diff --git a/mitmproxy/tools/console/grideditor/col_viewany.py b/mitmproxy/tools/console/grideditor/col_viewany.py index 2801587c04..b6ffe1f442 100644 --- a/mitmproxy/tools/console/grideditor/col_viewany.py +++ b/mitmproxy/tools/console/grideditor/col_viewany.py @@ -4,6 +4,7 @@ from typing import Any import urwid + from mitmproxy.tools.console.grideditor import base from mitmproxy.utils import strutils diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 4e2677b657..bfc9b38622 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -1,4 +1,5 @@ -from typing import Any, Union +from typing import Any +from typing import Union import urwid diff --git a/mitmproxy/tools/console/keybindings.py b/mitmproxy/tools/console/keybindings.py index 903f71e439..5cb3819b58 100644 --- a/mitmproxy/tools/console/keybindings.py +++ b/mitmproxy/tools/console/keybindings.py @@ -1,6 +1,7 @@ -import urwid import textwrap +import urwid + from mitmproxy.tools.console import layoutwidget from mitmproxy.tools.console import signals from mitmproxy.utils import signals as utils_signals diff --git a/mitmproxy/tools/console/keymap.py b/mitmproxy/tools/console/keymap.py index d4fbcaf819..5cadd010d0 100644 --- a/mitmproxy/tools/console/keymap.py +++ b/mitmproxy/tools/console/keymap.py @@ -4,15 +4,14 @@ from functools import cache from typing import Optional -import ruamel.yaml import ruamel.yaml.error +import mitmproxy.types from mitmproxy import command -from mitmproxy.tools.console import commandexecutor -from mitmproxy.tools.console import signals from mitmproxy import ctx from mitmproxy import exceptions -import mitmproxy.types +from mitmproxy.tools.console import commandexecutor +from mitmproxy.tools.console import signals class KeyBindingError(Exception): @@ -62,7 +61,9 @@ def keyspec(self): return self.key.replace("space", " ") def key_short(self) -> str: - return self.key.replace("enter", "⏎").replace("right", "→").replace("space", "␣") + return ( + self.key.replace("enter", "⏎").replace("right", "→").replace("space", "␣") + ) def sortkey(self): return self.key + ",".join(self.contexts) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 415e4a6751..3bad840960 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -1,6 +1,6 @@ import asyncio +import contextlib import mimetypes -import os import os.path import shlex import shutil @@ -8,20 +8,19 @@ import subprocess import sys import tempfile -import contextlib import threading from typing import TypeVar -from tornado.platform.asyncio import AddThreadSelectorEventLoop - import urwid +from tornado.platform.asyncio import AddThreadSelectorEventLoop from mitmproxy import addons +from mitmproxy import log from mitmproxy import master from mitmproxy import options -from mitmproxy import log -from mitmproxy.addons import errorcheck, intercept +from mitmproxy.addons import errorcheck from mitmproxy.addons import eventstore +from mitmproxy.addons import intercept from mitmproxy.addons import readfile from mitmproxy.addons import view from mitmproxy.contrib.tornado import patch_tornado diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 01b055f9e1..8aca078bac 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -1,16 +1,17 @@ from __future__ import annotations -from collections.abc import Sequence -import urwid -import textwrap import pprint +import textwrap +from collections.abc import Sequence from typing import Optional +import urwid + from mitmproxy import exceptions from mitmproxy import optmanager from mitmproxy.tools.console import layoutwidget -from mitmproxy.tools.console import signals from mitmproxy.tools.console import overlay +from mitmproxy.tools.console import signals HELP_HEIGHT = 5 diff --git a/mitmproxy/tools/console/overlay.py b/mitmproxy/tools/console/overlay.py index cd32161918..17b55bc10a 100644 --- a/mitmproxy/tools/console/overlay.py +++ b/mitmproxy/tools/console/overlay.py @@ -2,10 +2,10 @@ import urwid -from mitmproxy.tools.console import signals from mitmproxy.tools.console import grideditor -from mitmproxy.tools.console import layoutwidget from mitmproxy.tools.console import keymap +from mitmproxy.tools.console import layoutwidget +from mitmproxy.tools.console import signals class SimpleOverlay(urwid.Overlay, layoutwidget.LayoutWidget): diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index afbec3a011..415322be33 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -4,7 +4,9 @@ # http://urwid.org/manual/displayattributes.html # from __future__ import annotations -from collections.abc import Mapping, Sequence + +from collections.abc import Mapping +from collections.abc import Sequence from typing import Optional diff --git a/mitmproxy/tools/console/quickhelp.py b/mitmproxy/tools/console/quickhelp.py index 18e6c90d79..a24f810042 100644 --- a/mitmproxy/tools/console/quickhelp.py +++ b/mitmproxy/tools/console/quickhelp.py @@ -2,7 +2,8 @@ This module is reponsible for drawing the quick key help at the bottom of mitmproxy. """ from dataclasses import dataclass -from typing import Optional, Union +from typing import Optional +from typing import Union import urwid @@ -21,6 +22,7 @@ @dataclass class BasicKeyHelp: """Quick help for urwid-builtin keybindings (i.e. those keys that do not appear in the keymap)""" + key: str @@ -181,7 +183,7 @@ def _make_row(label: str, items: HelpItems, keymap: Keymap) -> urwid.Columns: " ", short, ], - wrap="clip" + wrap="clip", ) cols.append((14, txt)) diff --git a/mitmproxy/tools/console/signals.py b/mitmproxy/tools/console/signals.py index 024fad5007..38c3f5cfac 100644 --- a/mitmproxy/tools/console/signals.py +++ b/mitmproxy/tools/console/signals.py @@ -18,7 +18,9 @@ def _status_message(message: StatusMessage, expire: int = 5) -> None: # Prompt for input -def _status_prompt(prompt: str, text: str | None, callback: Callable[[str], None]) -> None: +def _status_prompt( + prompt: str, text: str | None, callback: Callable[[str], None] +) -> None: ... @@ -26,7 +28,9 @@ def _status_prompt(prompt: str, text: str | None, callback: Callable[[str], None # Prompt for a single keystroke -def _status_prompt_onekey(prompt: str, keys: list[tuple[str, str]], callback: Callable[[str], None]) -> None: +def _status_prompt_onekey( + prompt: str, keys: list[tuple[str, str]], callback: Callable[[str], None] +) -> None: ... diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 1ea3fe7fcb..a09950000a 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -1,4 +1,5 @@ from __future__ import annotations + from collections.abc import Callable from functools import lru_cache from typing import Optional @@ -6,15 +7,19 @@ import urwid import mitmproxy.tools.console.master -from mitmproxy.tools.console import commandexecutor, flowlist, quickhelp +from mitmproxy.tools.console import commandexecutor from mitmproxy.tools.console import common +from mitmproxy.tools.console import flowlist +from mitmproxy.tools.console import quickhelp from mitmproxy.tools.console import signals from mitmproxy.tools.console.commander import commander from mitmproxy.utils import human @lru_cache -def shorten_message(msg: tuple[str, str] | str, max_width: int) -> list[tuple[str, str]]: +def shorten_message( + msg: tuple[str, str] | str, max_width: int +) -> list[tuple[str, str]]: """ Shorten message so that it fits into a single line in the statusbar. """ @@ -69,7 +74,9 @@ def sig_update(self, flow=None) -> None: if not self.prompting and flow is None or flow == self.master.view.focus.flow: self.show_quickhelp() - def sig_message(self, message: tuple[str, str] | str, expire: int | None = 1) -> None: + def sig_message( + self, message: tuple[str, str] | str, expire: int | None = 1 + ) -> None: if self.prompting: return cols, _ = self.master.ui.get_cols_rows() @@ -84,7 +91,9 @@ def cb(): signals.call_in.send(seconds=expire, callback=cb) - def sig_prompt(self, prompt: str, text: str | None, callback: Callable[[str], None]) -> None: + def sig_prompt( + self, prompt: str, text: str | None, callback: Callable[[str], None] + ) -> None: signals.focus.send(section="footer") self.top._w = urwid.Edit(f"{prompt.strip()}: ", text or "") self.bottom._w = urwid.Text("") @@ -109,7 +118,9 @@ def execute_command(self, txt: str) -> None: execute = commandexecutor.CommandExecutor(self.master) execute(txt) - def sig_prompt_onekey(self, prompt: str, keys: list[tuple[str, str]], callback: Callable[[str], None]) -> None: + def sig_prompt_onekey( + self, prompt: str, keys: list[tuple[str, str]], callback: Callable[[str], None] + ) -> None: """ Keys are a set of (word, key) tuples. The appropriate key in the word is highlighted. @@ -315,10 +326,12 @@ def redraw(self) -> None: ("heading", f"{arrow} {marked} [{offset}/{fc}]".ljust(11)), ] - listen_addrs: list[str] = list(dict.fromkeys( - human.format_address(a) - for a in self.master.addons.get("proxyserver").listen_addrs() - )) + listen_addrs: list[str] = list( + dict.fromkeys( + human.format_address(a) + for a in self.master.addons.get("proxyserver").listen_addrs() + ) + ) if listen_addrs: boundaddr = f"[{', '.join(listen_addrs)}]" else: diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index 5c2a2392ee..7c9379b57a 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -2,6 +2,7 @@ import re import urwid + from mitmproxy import flow from mitmproxy.tools.console import commands from mitmproxy.tools.console import common @@ -148,7 +149,9 @@ def __init__(self, master): signals.flow_change.connect(self.flow_changed) signals.pop_view_state.connect(self.pop) - self.master.options.subscribe(self.configure, ["console_layout", "console_layout_headers"]) + self.master.options.subscribe( + self.configure, ["console_layout", "console_layout_headers"] + ) self.pane = 0 self.stacks = [WindowStack(master, "flowlist"), WindowStack(master, "eventlog")] diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 527a93e8b6..6bb269ee3c 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -1,7 +1,11 @@ from mitmproxy import addons from mitmproxy import master from mitmproxy import options -from mitmproxy.addons import dumper, errorcheck, keepserving, readfile, termlog +from mitmproxy.addons import dumper +from mitmproxy.addons import errorcheck +from mitmproxy.addons import keepserving +from mitmproxy.addons import readfile +from mitmproxy.addons import termlog class DumpMaster(master.Master): diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 414745109b..dfcf944d9d 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -1,18 +1,24 @@ from __future__ import annotations + import argparse import asyncio import logging import os import signal import sys -from collections.abc import Callable, Sequence -from typing import Any, Optional, TypeVar - -from mitmproxy import exceptions, master +from collections.abc import Callable +from collections.abc import Sequence +from typing import Any +from typing import Optional +from typing import TypeVar + +from mitmproxy import exceptions +from mitmproxy import master from mitmproxy import options from mitmproxy import optmanager from mitmproxy.tools import cmdline -from mitmproxy.utils import debug, arg_check +from mitmproxy.utils import arg_check +from mitmproxy.utils import debug def process_options(parser, opts, args): @@ -29,9 +35,7 @@ def process_options(parser, opts, args): args.flow_detail = 2 adict = { - key: val - for key, val in vars(args).items() - if key in opts and val is not None + key: val for key, val in vars(args).items() if key in opts and val is not None } opts.update(**adict) @@ -55,7 +59,9 @@ async def main() -> T: logging.getLogger("tornado").setLevel(logging.WARNING) logging.getLogger("asyncio").setLevel(logging.WARNING) logging.getLogger("hpack").setLevel(logging.WARNING) - logging.getLogger("quic").setLevel(logging.WARNING) # aioquic uses a different prefix... + logging.getLogger("quic").setLevel( + logging.WARNING + ) # aioquic uses a different prefix... debug.register_info_dumpers() opts = options.Options() diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 4b58dc2710..153b1afe08 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -1,14 +1,18 @@ from __future__ import annotations + import asyncio import hashlib import json import logging import os.path import re -from collections.abc import Callable, Sequence +from collections.abc import Callable +from collections.abc import Sequence from io import BytesIO from itertools import islice -from typing import ClassVar, Optional, Union +from typing import ClassVar +from typing import Optional +from typing import Union import tornado.escape import tornado.web @@ -16,7 +20,9 @@ import mitmproxy.flow import mitmproxy.tools.web.master -from mitmproxy import certs, command, contentviews +from mitmproxy import certs +from mitmproxy import command +from mitmproxy import contentviews from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import io @@ -25,8 +31,10 @@ from mitmproxy import version from mitmproxy.dns import DNSFlow from mitmproxy.http import HTTPFlow -from mitmproxy.tcp import TCPFlow, TCPMessage -from mitmproxy.udp import UDPFlow, UDPMessage +from mitmproxy.tcp import TCPFlow +from mitmproxy.tcp import TCPMessage +from mitmproxy.udp import UDPFlow +from mitmproxy.udp import UDPMessage from mitmproxy.utils.emoji import emoji from mitmproxy.utils.strutils import always_str from mitmproxy.websocket import WebSocketMessage @@ -191,7 +199,7 @@ class APIError(tornado.web.HTTPError): class RequestHandler(tornado.web.RequestHandler): - application: "Application" + application: Application def write(self, chunk: Union[str, bytes, dict, list]): # Writing arrays on the top level is ok nowadays. @@ -237,11 +245,11 @@ def filecontents(self): return self.request.body @property - def view(self) -> "mitmproxy.addons.view.View": + def view(self) -> mitmproxy.addons.view.View: return self.application.master.view @property - def master(self) -> "mitmproxy.tools.web.master.WebMaster": + def master(self) -> mitmproxy.tools.web.master.WebMaster: return self.application.master @property @@ -322,7 +330,11 @@ def get(self) -> None: match = flowfilter.parse(self.request.arguments["filter"][0].decode()) except ValueError: # thrown py flowfilter.parse if filter is invalid raise APIError(400, f"Invalid filter argument / regex") - except (KeyError, IndexError): # Key+Index: ["filter"][0] can fail, if it's not set + except ( + KeyError, + IndexError, + ): # Key+Index: ["filter"][0] can fail, if it's not set + def match(_) -> bool: return True @@ -617,31 +629,35 @@ def get(self): raise tornado.web.HTTPError( 403, reason="To protect against DNS rebinding, mitmweb can only be accessed by IP at the moment. " - "(https://github.com/mitmproxy/mitmproxy/issues/3234)", + "(https://github.com/mitmproxy/mitmproxy/issues/3234)", ) class State(RequestHandler): def get(self): - self.write({ - "version": version.VERSION, - "contentViews": [v.name for v in contentviews.views if v.name != "Query"], - "servers": [s.to_json() for s in self.master.proxyserver.servers] - }) + self.write( + { + "version": version.VERSION, + "contentViews": [ + v.name for v in contentviews.views if v.name != "Query" + ], + "servers": [s.to_json() for s in self.master.proxyserver.servers], + } + ) class GZipContentAndFlowFiles(tornado.web.GZipContentEncoding): CONTENT_TYPES = { "application/octet-stream", - *tornado.web.GZipContentEncoding.CONTENT_TYPES + *tornado.web.GZipContentEncoding.CONTENT_TYPES, } class Application(tornado.web.Application): - master: "mitmproxy.tools.web.master.WebMaster" + master: mitmproxy.tools.web.master.WebMaster def __init__( - self, master: "mitmproxy.tools.web.master.WebMaster", debug: bool + self, master: mitmproxy.tools.web.master.WebMaster, debug: bool ) -> None: self.master = master super().__init__( diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 62119e09c0..3d0b59bc9d 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -1,6 +1,6 @@ +import errno import logging -import errno import tornado.httpserver import tornado.ioloop @@ -8,16 +8,19 @@ from mitmproxy import flow from mitmproxy import log from mitmproxy import master -from mitmproxy import optmanager from mitmproxy import options -from mitmproxy.addons import errorcheck, eventstore +from mitmproxy import optmanager +from mitmproxy.addons import errorcheck +from mitmproxy.addons import eventstore from mitmproxy.addons import intercept from mitmproxy.addons import readfile from mitmproxy.addons import termlog from mitmproxy.addons import view from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.contrib.tornado import patch_tornado -from mitmproxy.tools.web import app, webaddons, static_viewer +from mitmproxy.tools.web import app +from mitmproxy.tools.web import static_viewer +from mitmproxy.tools.web import webaddons logger = logging.getLogger(__name__) @@ -87,7 +90,7 @@ def _sig_servers_changed(self) -> None: app.ClientConnection.broadcast( resource="state", cmd="update", - data={"servers": [s.to_json() for s in self.proxyserver.servers]} + data={"servers": [s.to_json() for s in self.proxyserver.servers]}, ) async def running(self): diff --git a/mitmproxy/tools/web/static_viewer.py b/mitmproxy/tools/web/static_viewer.py index 7decf12de0..3f9b5dbc62 100644 --- a/mitmproxy/tools/web/static_viewer.py +++ b/mitmproxy/tools/web/static_viewer.py @@ -7,10 +7,12 @@ from collections.abc import Iterable from typing import Optional -from mitmproxy import contentviews, http +from mitmproxy import contentviews from mitmproxy import ctx +from mitmproxy import flow from mitmproxy import flowfilter -from mitmproxy import io, flow +from mitmproxy import http +from mitmproxy import io from mitmproxy import version from mitmproxy.tools.web.app import flow_to_json diff --git a/mitmproxy/types.py b/mitmproxy/types.py index e3161aeeae..8645811ace 100644 --- a/mitmproxy/types.py +++ b/mitmproxy/types.py @@ -1,13 +1,17 @@ import codecs -import os import glob +import os import re from collections.abc import Sequence -from typing import Any, Optional, TYPE_CHECKING, Union +from typing import Any +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union from mitmproxy import exceptions from mitmproxy import flow -from mitmproxy.utils import emoji, strutils +from mitmproxy.utils import emoji +from mitmproxy.utils import strutils if TYPE_CHECKING: # pragma: no cover from mitmproxy.command import CommandManager diff --git a/mitmproxy/udp.py b/mitmproxy/udp.py index e719de0ac1..2d716e9108 100644 --- a/mitmproxy/udp.py +++ b/mitmproxy/udp.py @@ -1,6 +1,7 @@ import time -from mitmproxy import connection, flow +from mitmproxy import connection +from mitmproxy import flow from mitmproxy.coretypes import serializable diff --git a/mitmproxy/utils/arg_check.py b/mitmproxy/utils/arg_check.py index ad43eaf9b8..24923d963a 100644 --- a/mitmproxy/utils/arg_check.py +++ b/mitmproxy/utils/arg_check.py @@ -1,5 +1,5 @@ -import sys import re +import sys DEPRECATED = """ --confdir diff --git a/mitmproxy/utils/data.py b/mitmproxy/utils/data.py index 091640ec9f..baa8c6abd3 100644 --- a/mitmproxy/utils/data.py +++ b/mitmproxy/utils/data.py @@ -1,6 +1,6 @@ -import os.path import importlib import inspect +import os.path class Data: diff --git a/mitmproxy/utils/emoji.py b/mitmproxy/utils/emoji.py index 2bb31432ea..2a45eddc9d 100644 --- a/mitmproxy/utils/emoji.py +++ b/mitmproxy/utils/emoji.py @@ -2,7 +2,6 @@ """ All of the emoji and characters that can be used as flow markers. """ - # auto-generated. run this file to refresh. emoji = { diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index ddb336340b..a417c84af4 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -5,11 +5,11 @@ from typing import Optional SIZE_UNITS = { - "b": 1024 ** 0, - "k": 1024 ** 1, - "m": 1024 ** 2, - "g": 1024 ** 3, - "t": 1024 ** 4, + "b": 1024**0, + "k": 1024**1, + "m": 1024**2, + "g": 1024**3, + "t": 1024**4, } diff --git a/mitmproxy/utils/magisk.py b/mitmproxy/utils/magisk.py index 286aee9269..815e513d91 100644 --- a/mitmproxy/utils/magisk.py +++ b/mitmproxy/utils/magisk.py @@ -1,13 +1,14 @@ -from zipfile import ZipFile import hashlib +import os +from zipfile import ZipFile + from cryptography import x509 from cryptography.hazmat.primitives import serialization -from mitmproxy import certs, ctx +from mitmproxy import certs +from mitmproxy import ctx from mitmproxy.options import CONF_BASENAME -import os - # The following 3 variables are for including in the magisk module as text file MODULE_PROP_TEXT = """id=mitmproxycert name=MITMProxy cert @@ -84,14 +85,14 @@ def get_ca_from_files() -> x509.Certificate: return certstore.default_ca._cert -def subject_hash_old(ca : x509.Certificate) -> str: +def subject_hash_old(ca: x509.Certificate) -> str: # Mimics the -subject_hash_old option of openssl used for android certificate names full_hash = hashlib.md5(ca.subject.public_bytes()).digest() - sho = (full_hash[0] | (full_hash[1] << 8) | (full_hash[2] << 16) | full_hash[3] << 24) + sho = full_hash[0] | (full_hash[1] << 8) | (full_hash[2] << 16) | full_hash[3] << 24 return hex(sho)[2:] -def write_magisk_module(path : str): +def write_magisk_module(path: str): # Makes a zip file that can be loaded by Magisk # Android certs are stored as DER files ca = get_ca_from_files() @@ -103,7 +104,9 @@ def write_magisk_module(path : str): zipp.writestr("config.sh", CONFIG_SH_TEXT) zipp.writestr("META-INF/com/google/android/updater-script", "#MAGISK") zipp.writestr("META-INF/com/google/android/update-binary", UPDATE_BINARY_TEXT) - zipp.writestr("common/file_contexts_image", "/magisk(/.*)? u:object_r:system_file:s0") + zipp.writestr( + "common/file_contexts_image", "/magisk(/.*)? u:object_r:system_file:s0" + ) zipp.writestr("common/post-fs-data.sh", "MODDIR=${0%/*}") zipp.writestr("common/service.sh", "MODDIR=${0%/*}") zipp.writestr("common/system.prop", "") diff --git a/mitmproxy/utils/signals.py b/mitmproxy/utils/signals.py index 37900f68b1..cad5e5d1ff 100644 --- a/mitmproxy/utils/signals.py +++ b/mitmproxy/utils/signals.py @@ -8,11 +8,16 @@ - supports async receivers. """ from __future__ import annotations + import asyncio import inspect import weakref -from collections.abc import Callable, Awaitable -from typing import Any, Generic, TypeVar, cast +from collections.abc import Awaitable +from collections.abc import Callable +from typing import Any +from typing import cast +from typing import Generic +from typing import TypeVar try: from typing import ParamSpec @@ -85,11 +90,13 @@ def disconnect(self, receiver: Callable[P, Awaitable[None] | None]) -> None: super().disconnect(receiver) async def send(self, *args: P.args, **kwargs: P.kwargs) -> None: - await asyncio.gather(*[ - aws - for aws in super().notify(*args, **kwargs) - if aws is not None and inspect.isawaitable(aws) - ]) + await asyncio.gather( + *[ + aws + for aws in super().notify(*args, **kwargs) + if aws is not None and inspect.isawaitable(aws) + ] + ) # noinspection PyPep8Naming diff --git a/mitmproxy/utils/sliding_window.py b/mitmproxy/utils/sliding_window.py index dca71cbd6c..a2cbb300db 100644 --- a/mitmproxy/utils/sliding_window.py +++ b/mitmproxy/utils/sliding_window.py @@ -1,5 +1,8 @@ import itertools -from typing import Iterable, Iterator, Optional, TypeVar +from collections.abc import Iterable +from collections.abc import Iterator +from typing import Optional +from typing import TypeVar T = TypeVar("T") diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 6f61ff54de..0e2c1b2072 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,7 +1,9 @@ import codecs import io import re -from typing import Iterable, Union, overload +from collections.abc import Iterable +from typing import overload +from typing import Union # https://mypy.readthedocs.io/en/stable/more_types.html#function-overloading @@ -236,7 +238,7 @@ def escape_special_areas( """ buf = io.StringIO() parts = split_special_areas(data, area_delimiter) - rex = re.compile(fr"[{control_characters}]") + rex = re.compile(rf"[{control_characters}]") for i, x in enumerate(parts): if i % 2: x = rex.sub(_move_to_private_code_plane, x) diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index 6279cfaeb4..a898a5b387 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -17,7 +17,7 @@ def check_option_type(name: str, value: typing.Any, typeinfo: Type) -> None: TypeError otherwise. This function supports only those types required for options. """ - e = TypeError("Expected {} for {}, but got {}.".format(typeinfo, name, type(value))) + e = TypeError(f"Expected {typeinfo} for {name}, but got {type(value)}.") origin = typing.get_origin(typeinfo) diff --git a/mitmproxy/utils/vt_codes.py b/mitmproxy/utils/vt_codes.py index e33a8f2492..7e71d446c2 100644 --- a/mitmproxy/utils/vt_codes.py +++ b/mitmproxy/utils/vt_codes.py @@ -49,7 +49,6 @@ def ensure_supported(f: IO[str]) -> bool: ) return ok - else: def ensure_supported(f: IO[str]) -> bool: diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 6f301922c4..657f2f6cd9 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -7,13 +7,15 @@ """ import time import warnings -from dataclasses import dataclass, field -from typing import Union +from dataclasses import dataclass +from dataclasses import field from typing import Optional +from typing import Union -from mitmproxy.coretypes import serializable from wsproto.frame_protocol import Opcode +from mitmproxy.coretypes import serializable + WebSocketMessageState = tuple[int, bool, bytes, float, bool, bool] diff --git a/release/build-and-deploy-docker.py b/release/build-and-deploy-docker.py index 50a6ce4963..f407834685 100644 --- a/release/build-and-deploy-docker.py +++ b/release/build-and-deploy-docker.py @@ -51,7 +51,8 @@ f"{root / 'release'}:/release", "localtesting", "mitmdump", - "-s", "/release/selftest.py", + "-s", + "/release/selftest.py", ], capture_output=True, ) diff --git a/release/build.py b/release/build.py index 60177e31a2..6f7f10c77e 100644 --- a/release/build.py +++ b/release/build.py @@ -84,7 +84,9 @@ def archive(path: Path) -> tarfile.TarFile | ZipFile2: def version() -> str: - return os.environ.get("GITHUB_REF_NAME", "").replace("/", "-") or os.environ.get("BUILD_VERSION", "dev") + return os.environ.get("GITHUB_REF_NAME", "").replace("/", "-") or os.environ.get( + "BUILD_VERSION", "dev" + ) def operating_system() -> Literal["windows", "linux", "macos", "unknown"]: @@ -170,7 +172,13 @@ def msix_installer(): manifest = TEMP_DIR / "msix/AppxManifest.xml" app_version = version() if not re.match(r"\d+\.\d+\.\d+", app_version): - app_version = datetime.now().strftime("%y%m.%d.%H%M").replace(".0", ".").replace(".0", ".").replace(".0", ".") + app_version = ( + datetime.now() + .strftime("%y%m.%d.%H%M") + .replace(".0", ".") + .replace(".0", ".") + .replace(".0", ".") + ) manifest.write_text(manifest.read_text().replace("1.2.3", app_version)) makeappx_exe = ( @@ -237,7 +245,9 @@ def report(block, blocksize, total): break ib_setup_hash.update(data) if ib_setup_hash.hexdigest() != IB_SETUP_SHA256: # pragma: no cover - raise RuntimeError(f"InstallBuilder hashes don't match: {ib_setup_hash.hexdigest()}") + raise RuntimeError( + f"InstallBuilder hashes don't match: {ib_setup_hash.hexdigest()}" + ) print("Install InstallBuilder...") subprocess.run( diff --git a/release/release.py b/release/release.py index c64602526c..48c866ead1 100755 --- a/release/release.py +++ b/release/release.py @@ -46,7 +46,10 @@ def get_json(url: str) -> dict: branch = subprocess.run( ["git", "branch", "--show-current"], - cwd=root, check=True, capture_output=True, text=True + cwd=root, + check=True, + capture_output=True, + text=True, ).stdout.strip() print("➡️ Working dir clean?") @@ -56,7 +59,12 @@ def get_json(url: str) -> dict: print(f"⚠️ Skipping status check for {branch}.") else: print(f"➡️ CI is passing for {branch}?") - assert get_json(f"https://api.github.com/repos/{repo}/commits/{branch}/status")["state"] == "success" + assert ( + get_json(f"https://api.github.com/repos/{repo}/commits/{branch}/status")[ + "state" + ] + == "success" + ) print("➡️ Updating CHANGELOG.md...") changelog = root / "CHANGELOG.md" @@ -70,7 +78,9 @@ def get_json(url: str) -> dict: print("➡️ Updating web assets...") subprocess.run(["npm", "ci"], cwd=root / "web", check=True, capture_output=True) - subprocess.run(["npm", "start", "prod"], cwd=root / "web", check=True, capture_output=True) + subprocess.run( + ["npm", "start", "prod"], cwd=root / "web", check=True, capture_output=True + ) print("➡️ Updating version...") version_py = root / "mitmproxy" / "version.py" @@ -80,13 +90,22 @@ def get_json(url: str) -> dict: version_py.write_text(ver, "utf8") print("➡️ Do release commit...") - subprocess.run(["git", "config", "user.email", "noreply@mitmproxy.org"], cwd=root, check=True) - subprocess.run(["git", "config", "user.name", "mitmproxy release bot"], cwd=root, check=True) - subprocess.run(["git", "commit", "-a", "-m", f"mitmproxy {version}"], cwd=root, check=True) + subprocess.run( + ["git", "config", "user.email", "noreply@mitmproxy.org"], cwd=root, check=True + ) + subprocess.run( + ["git", "config", "user.name", "mitmproxy release bot"], cwd=root, check=True + ) + subprocess.run( + ["git", "commit", "-a", "-m", f"mitmproxy {version}"], cwd=root, check=True + ) subprocess.run(["git", "tag", version], cwd=root, check=True) release_sha = subprocess.run( ["git", "rev-parse", "HEAD"], - cwd=root, check=True, capture_output=True, text=True + cwd=root, + check=True, + capture_output=True, + text=True, ).stdout.strip() if branch == "main": @@ -97,15 +116,32 @@ def get_json(url: str) -> dict: version_py.write_text(ver, "utf8") print("➡️ Reopen main for development...") - subprocess.run(["git", "commit", "-a", "-m", f"reopen main for development"], cwd=root, check=True) + subprocess.run( + ["git", "commit", "-a", "-m", f"reopen main for development"], + cwd=root, + check=True, + ) print("➡️ Pushing...") - subprocess.run(["git", "push", "--atomic", "origin", branch, version], cwd=root, check=True) + subprocess.run( + ["git", "push", "--atomic", "origin", branch, version], cwd=root, check=True + ) print("➡️ Creating release on GitHub...") - subprocess.run(["gh", "release", "create", version, - "--title", f"mitmproxy {version}", - "--notes-file", "release/github-release-notes.txt"], cwd=root, check=True) + subprocess.run( + [ + "gh", + "release", + "create", + version, + "--title", + f"mitmproxy {version}", + "--notes-file", + "release/github-release-notes.txt", + ], + cwd=root, + check=True, + ) # We currently have to use a personal access token, which auto-triggers CI. # The default GITHUB_TOKEN cannot push to protected branches, @@ -118,7 +154,9 @@ def get_json(url: str) -> dict: while True: print("⌛ Waiting for CI...") - workflows = get_json(f"https://api.github.com/repos/{repo}/actions/runs?head_sha={release_sha}")["workflow_runs"] + workflows = get_json( + f"https://api.github.com/repos/{repo}/actions/runs?head_sha={release_sha}" + )["workflow_runs"] all_done = True if not workflows: @@ -150,16 +188,23 @@ def get_json(url: str) -> dict: assert resp.status == 200 print(f"➡️ Checking Docker ({version} tag)...") - resp = get(f"https://hub.docker.com/v2/repositories/mitmproxy/mitmproxy/tags/{version}") + resp = get( + f"https://hub.docker.com/v2/repositories/mitmproxy/mitmproxy/tags/{version}" + ) assert resp.status == 200 if branch == "main": print("➡️ Checking Docker (latest tag)...") - docker_latest_data = get_json("https://hub.docker.com/v2/repositories/mitmproxy/mitmproxy/tags/latest") + docker_latest_data = get_json( + "https://hub.docker.com/v2/repositories/mitmproxy/mitmproxy/tags/latest" + ) docker_last_updated = datetime.datetime.fromisoformat( - docker_latest_data["last_updated"].replace("Z", "+00:00")) + docker_latest_data["last_updated"].replace("Z", "+00:00") + ) print(f"Last update: {docker_last_updated.isoformat(timespec='minutes')}") - assert docker_last_updated > datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(hours=2) + assert docker_last_updated > datetime.datetime.now( + datetime.timezone.utc + ) - datetime.timedelta(hours=2) print("") print("✅ All done. 🥳") diff --git a/release/selftest.py b/release/selftest.py index cf1130d6a5..01eae5ca03 100644 --- a/release/selftest.py +++ b/release/selftest.py @@ -27,10 +27,7 @@ async def make_request(): cafile = Path(ctx.options.confdir).expanduser() / "mitmproxy-ca.pem" ssl_ctx = ssl.create_default_context(cafile=cafile) port = ctx.master.addons.get("proxyserver").listen_addrs()[0][1] - reader, writer = await asyncio.open_connection( - "127.0.0.1", port, - ssl=ssl_ctx - ) + reader, writer = await asyncio.open_connection("127.0.0.1", port, ssl=ssl_ctx) writer.write(b"GET / HTTP/1.1\r\nHost: mitm.it\r\nConnection: close\r\n\r\n") await writer.drain() resp = await reader.read() diff --git a/setup.py b/setup.py index c1c4617fca..187a8bcdaa 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,8 @@ import re from codecs import open -from setuptools import find_packages, setup +from setuptools import find_packages +from setuptools import setup # Based on https://github.com/pypa/sampleproject/blob/main/setup.py # and https://python-packaging-user-guide.readthedocs.org/ @@ -67,7 +68,7 @@ ], "pyinstaller40": [ "hook-dirs = mitmproxy.utils.pyinstaller:hook_dirs", - ] + ], }, python_requires=">=3.9", # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#install-requires diff --git a/test/conftest.py b/test/conftest.py index 89d78c0e9c..77be0686b4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,5 @@ from __future__ import annotations + import asyncio import os import socket diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index 50b08a1962..1cf0bd304c 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -1,8 +1,8 @@ from mitmproxy import contentviews +from mitmproxy.http import Headers +from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test import tutils -from mitmproxy.test import taddons -from mitmproxy.http import Headers class TestScripts: diff --git a/test/filename_matching.py b/test/filename_matching.py index 3e9878f020..9d64ede25f 100755 --- a/test/filename_matching.py +++ b/test/filename_matching.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 - +import glob import os import re -import glob import sys diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py index 3d1b8b6787..7513e6600f 100644 --- a/test/full_coverage_plugin.py +++ b/test/full_coverage_plugin.py @@ -1,8 +1,9 @@ -import os import configparser -import pytest +import os import sys +import pytest + here = os.path.abspath(os.path.dirname(__file__)) diff --git a/test/helper_tools/dumperview.py b/test/helper_tools/dumperview.py index 450b7f12f4..9c90f784bb 100755 --- a/test/helper_tools/dumperview.py +++ b/test/helper_tools/dumperview.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 import asyncio + import click from mitmproxy.addons import dumper -from mitmproxy.test import tflow from mitmproxy.test import taddons +from mitmproxy.test import tflow def run_async(coro): diff --git a/test/helper_tools/getcert b/test/helper_tools/getcert index 43ebf11dc2..841fac644d 100644 --- a/test/helper_tools/getcert +++ b/test/helper_tools/getcert @@ -2,9 +2,7 @@ import sys sys.path.insert(0, "../..") import socket -import tempfile import ssl -import subprocess addr = socket.gethostbyname(sys.argv[1]) print(ssl.get_server_certificate((addr, 443))) diff --git a/test/helper_tools/linkify-changelog.py b/test/helper_tools/linkify-changelog.py index f0db26175e..77558d87ce 100644 --- a/test/helper_tools/linkify-changelog.py +++ b/test/helper_tools/linkify-changelog.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -from pathlib import Path import re +from pathlib import Path changelog = Path(__file__).parent / "../../CHANGELOG.md" diff --git a/test/helper_tools/loggrep.py b/test/helper_tools/loggrep.py index a986e47c47..c9528f4913 100755 --- a/test/helper_tools/loggrep.py +++ b/test/helper_tools/loggrep.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import fileinput -import sys import re +import sys if __name__ == "__main__": if len(sys.argv) < 3: diff --git a/test/helper_tools/memoryleak.py b/test/helper_tools/memoryleak.py index d02482353b..12e733ae26 100644 --- a/test/helper_tools/memoryleak.py +++ b/test/helper_tools/memoryleak.py @@ -1,7 +1,9 @@ import gc import threading -from pympler import muppy, refbrowser + from OpenSSL import SSL +from pympler import muppy +from pympler import refbrowser # import os # os.environ["TK_LIBRARY"] = r"C:\Python27\tcl\tcl8.5" diff --git a/test/individual_coverage.py b/test/individual_coverage.py index e3c5ef5342..d6d3a40a5b 100755 --- a/test/individual_coverage.py +++ b/test/individual_coverage.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 - -import io +import configparser import contextlib -import os -import sys import glob -import multiprocessing -import configparser +import io import itertools +import multiprocessing +import os +import sys + import pytest diff --git a/test/mitmproxy/addons/test_anticache.py b/test/mitmproxy/addons/test_anticache.py index b3eb00d332..a0746decc3 100644 --- a/test/mitmproxy/addons/test_anticache.py +++ b/test/mitmproxy/addons/test_anticache.py @@ -1,7 +1,6 @@ -from mitmproxy.test import tflow - from mitmproxy.addons import anticache from mitmproxy.test import taddons +from mitmproxy.test import tflow class TestAntiCache: diff --git a/test/mitmproxy/addons/test_anticomp.py b/test/mitmproxy/addons/test_anticomp.py index 92650332c7..70a97dbf56 100644 --- a/test/mitmproxy/addons/test_anticomp.py +++ b/test/mitmproxy/addons/test_anticomp.py @@ -1,7 +1,6 @@ -from mitmproxy.test import tflow - from mitmproxy.addons import anticomp from mitmproxy.test import taddons +from mitmproxy.test import tflow class TestAntiComp: diff --git a/test/mitmproxy/addons/test_browser.py b/test/mitmproxy/addons/test_browser.py index 31cbe292af..e5b9c15570 100644 --- a/test/mitmproxy/addons/test_browser.py +++ b/test/mitmproxy/addons/test_browser.py @@ -6,7 +6,9 @@ def test_browser(caplog): caplog.set_level("INFO") - with mock.patch("subprocess.Popen") as po, mock.patch("shutil.which") as which, taddons.context(): + with mock.patch("subprocess.Popen") as po, mock.patch( + "shutil.which" + ) as which, taddons.context(): which.return_value = "chrome" b = browser.Browser() b.start() diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 432111a2d7..013d6f1b39 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -3,11 +3,14 @@ import pytest -from mitmproxy.addons.clientplayback import ClientPlayback, ReplayHandler +from mitmproxy.addons.clientplayback import ClientPlayback +from mitmproxy.addons.clientplayback import ReplayHandler from mitmproxy.addons.proxyserver import Proxyserver -from mitmproxy.exceptions import CommandError, OptionsError from mitmproxy.connection import Address -from mitmproxy.test import taddons, tflow +from mitmproxy.exceptions import CommandError +from mitmproxy.exceptions import OptionsError +from mitmproxy.test import taddons +from mitmproxy.test import tflow @asynccontextmanager diff --git a/test/mitmproxy/addons/test_command_history.py b/test/mitmproxy/addons/test_command_history.py index 915eddf3a6..7871e4809e 100644 --- a/test/mitmproxy/addons/test_command_history.py +++ b/test/mitmproxy/addons/test_command_history.py @@ -1,6 +1,6 @@ import os -from unittest.mock import patch from pathlib import Path +from unittest.mock import patch from mitmproxy.addons import command_history from mitmproxy.test import taddons diff --git a/test/mitmproxy/addons/test_comment.py b/test/mitmproxy/addons/test_comment.py index ba628cd49a..b3c9833bbf 100644 --- a/test/mitmproxy/addons/test_comment.py +++ b/test/mitmproxy/addons/test_comment.py @@ -1,5 +1,6 @@ -from mitmproxy.test import tflow, taddons from mitmproxy.addons.comment import Comment +from mitmproxy.test import taddons +from mitmproxy.test import tflow def test_comment(): diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index fc219769d3..be2cb1b139 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -1,8 +1,9 @@ +import pytest + +from mitmproxy import exceptions from mitmproxy.addons import core from mitmproxy.test import taddons from mitmproxy.test import tflow -from mitmproxy import exceptions -import pytest def test_set(): diff --git a/test/mitmproxy/addons/test_cut.py b/test/mitmproxy/addons/test_cut.py index ba045bdfd0..d2a30abb45 100644 --- a/test/mitmproxy/addons/test_cut.py +++ b/test/mitmproxy/addons/test_cut.py @@ -1,12 +1,14 @@ +from unittest import mock + +import pyperclip +import pytest + +from mitmproxy import certs +from mitmproxy import exceptions from mitmproxy.addons import cut from mitmproxy.addons import view -from mitmproxy import exceptions -from mitmproxy import certs from mitmproxy.test import taddons from mitmproxy.test import tflow -import pytest -import pyperclip -from unittest import mock def test_extract(tdata): diff --git a/test/mitmproxy/addons/test_disable_h2c.py b/test/mitmproxy/addons/test_disable_h2c.py index 98ec0e3dda..4d55ecfe48 100644 --- a/test/mitmproxy/addons/test_disable_h2c.py +++ b/test/mitmproxy/addons/test_disable_h2c.py @@ -1,7 +1,8 @@ from mitmproxy import flow from mitmproxy.addons import disable_h2c -from mitmproxy.test import taddons, tutils +from mitmproxy.test import taddons from mitmproxy.test import tflow +from mitmproxy.test import tutils class TestDisableH2CleartextUpgrade: diff --git a/test/mitmproxy/addons/test_dns_resolver.py b/test/mitmproxy/addons/test_dns_resolver.py index db91894b44..0937c97a52 100644 --- a/test/mitmproxy/addons/test_dns_resolver.py +++ b/test/mitmproxy/addons/test_dns_resolver.py @@ -6,10 +6,13 @@ import pytest from mitmproxy import dns -from mitmproxy.addons import dns_resolver, proxyserver +from mitmproxy.addons import dns_resolver +from mitmproxy.addons import proxyserver from mitmproxy.connection import Address from mitmproxy.proxy.mode_specs import ProxyMode -from mitmproxy.test import taddons, tflow, tutils +from mitmproxy.test import taddons +from mitmproxy.test import tflow +from mitmproxy.test import tutils async def test_simple(monkeypatch): diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index 17187d85e4..f3dcb8d760 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -1,15 +1,15 @@ import os import shlex +from unittest import mock -import pytest import pyperclip +import pytest from mitmproxy import exceptions from mitmproxy.addons import export # heh +from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test import tutils -from mitmproxy.test import taddons -from unittest import mock @pytest.fixture diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index 3cabfda283..0e45a28920 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy.addons import intercept from mitmproxy import exceptions +from mitmproxy.addons import intercept from mitmproxy.test import taddons from mitmproxy.test import tflow diff --git a/test/mitmproxy/addons/test_keepserving.py b/test/mitmproxy/addons/test_keepserving.py index 99459bf37b..8ee85f8ce8 100644 --- a/test/mitmproxy/addons/test_keepserving.py +++ b/test/mitmproxy/addons/test_keepserving.py @@ -1,8 +1,8 @@ import asyncio +from mitmproxy import command from mitmproxy.addons import keepserving from mitmproxy.test import taddons -from mitmproxy import command class Dummy: diff --git a/test/mitmproxy/addons/test_maplocal.py b/test/mitmproxy/addons/test_maplocal.py index 917664718c..7edf4e8e92 100644 --- a/test/mitmproxy/addons/test_maplocal.py +++ b/test/mitmproxy/addons/test_maplocal.py @@ -3,10 +3,12 @@ import pytest -from mitmproxy.addons.maplocal import MapLocal, MapLocalSpec, file_candidates -from mitmproxy.utils.spec import parse_spec +from mitmproxy.addons.maplocal import file_candidates +from mitmproxy.addons.maplocal import MapLocal +from mitmproxy.addons.maplocal import MapLocalSpec from mitmproxy.test import taddons from mitmproxy.test import tflow +from mitmproxy.utils.spec import parse_spec @pytest.mark.parametrize( diff --git a/test/mitmproxy/addons/test_modifyheaders.py b/test/mitmproxy/addons/test_modifyheaders.py index 430c824b9b..83a46256fc 100644 --- a/test/mitmproxy/addons/test_modifyheaders.py +++ b/test/mitmproxy/addons/test_modifyheaders.py @@ -1,6 +1,7 @@ import pytest -from mitmproxy.addons.modifyheaders import parse_modify_spec, ModifyHeaders +from mitmproxy.addons.modifyheaders import ModifyHeaders +from mitmproxy.addons.modifyheaders import parse_modify_spec from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test.tutils import tresp diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index 4160b88d2b..6004c8770e 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -5,8 +5,10 @@ from mitmproxy import connection from mitmproxy.addons.next_layer import NextLayer +from mitmproxy.proxy import context +from mitmproxy.proxy import layer +from mitmproxy.proxy import layers from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy import context, layer, layers from mitmproxy.test import taddons from mitmproxy.test import tflow @@ -14,7 +16,11 @@ @pytest.fixture def tctx(): context.Context( - connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), + connection.Client( + peername=("client", 1234), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, + ), tctx.options, ) @@ -38,8 +44,8 @@ def tctx(): dtls_client_hello_with_extensions = bytes.fromhex( - "16fefd00000000000000000085" # record layer - "010000790000000000000079" # handshake layer + "16fefd00000000000000000085" # record layer + "010000790000000000000079" # handshake layer "fefd62bf0e0bf809df43e7669197be831919878b1a72c07a584d3c0a8ca6665878010000000cc02bc02fc00ac014c02cc0" "3001000043000d0010000e0403050306030401050106010807ff01000100000a00080006001d00170018000b00020100001" "7000000000010000e00000b6578616d706c652e636f6d" @@ -185,7 +191,7 @@ def test_next_layer2(self): [ ("dtls", layers.ClientTLSLayer, layers.ServerTLSLayer), ("quic", layers.ClientQuicLayer, layers.ServerQuicLayer), - ] + ], ) def test_next_layer_udp( self, @@ -200,10 +206,7 @@ def is_intercepted_udp(layer: Optional[layer.Layer]): return isinstance(layer, layers.UDPLayer) and layer.flow is not None def is_http(layer: Optional[layer.Layer], mode: HTTPMode): - return ( - isinstance(layer, layers.HttpLayer) - and layer.mode is mode - ) + return isinstance(layer, layers.HttpLayer) and layer.mode is mode client_hello = { "dtls": dtls_client_hello_with_extensions, @@ -246,11 +249,15 @@ def is_http(layer: Optional[layer.Layer], mode: HTTPMode): ctx.layers = [layers.modes.TransparentProxy(ctx)] tctx.configure(nl, udp_hosts=["example.com"]) - assert isinstance(nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.UDPLayer) + assert isinstance( + nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.UDPLayer + ) ctx.layers = [layers.modes.TransparentProxy(ctx)] tctx.configure(nl, udp_hosts=[]) - assert isinstance(nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.DNSLayer) + assert isinstance( + nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.DNSLayer + ) def test_next_layer_reverse_raw(self): nl = NextLayer() @@ -273,7 +280,9 @@ def test_next_layer_reverse_raw(self): ctx.layers = [ layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), - layers.ClientQuicLayer(ctx,), + layers.ClientQuicLayer( + ctx, + ), ] assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) @@ -306,12 +315,16 @@ def test_next_layer_reverse_quic_mode(self): layers.ServerQuicLayer(ctx), ] assert nl._next_layer(ctx, b"", b"") is None - assert isinstance(nl._next_layer(ctx, b"notahandshake", b""), layers.UDPLayer) + assert isinstance( + nl._next_layer(ctx, b"notahandshake", b""), layers.UDPLayer + ) ctx.layers = [ layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), ] - assert isinstance(nl._next_layer(ctx, quic_client_hello, b""), layers.ClientQuicLayer) + assert isinstance( + nl._next_layer(ctx, quic_client_hello, b""), layers.ClientQuicLayer + ) def test_next_layer_reverse_http3_mode(self): nl = NextLayer() @@ -325,7 +338,10 @@ def test_next_layer_reverse_http3_mode(self): layers.modes.ReverseProxy(ctx), layers.ServerQuicLayer(ctx), ] - assert isinstance(nl._next_layer(ctx, b"notahandshakebutignore", b""), layers.ClientQuicLayer) + assert isinstance( + nl._next_layer(ctx, b"notahandshakebutignore", b""), + layers.ClientQuicLayer, + ) assert len(ctx.layers) == 3 decision = nl._next_layer(ctx, b"", b"") assert isinstance(decision, layers.HttpLayer) @@ -352,7 +368,10 @@ def test_next_layer_reverse_dtls_mode(self): ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx)] assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx)] - assert isinstance(nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), layers.ClientTLSLayer) + assert isinstance( + nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), + layers.ClientTLSLayer, + ) assert len(ctx.layers) == 3 assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) @@ -366,7 +385,10 @@ def test_next_layer_reverse_udp_mode(self): ctx.layers = [layers.modes.ReverseProxy(ctx)] assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) ctx.layers = [layers.modes.ReverseProxy(ctx)] - assert isinstance(nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), layers.ClientTLSLayer) + assert isinstance( + nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), + layers.ClientTLSLayer, + ) assert len(ctx.layers) == 2 assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) @@ -380,7 +402,10 @@ def test_next_layer_reverse_dns_mode(self): ctx.layers = [layers.modes.ReverseProxy(ctx)] assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) ctx.layers = [layers.modes.ReverseProxy(ctx)] - assert isinstance(nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), layers.ClientTLSLayer) + assert isinstance( + nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), + layers.ClientTLSLayer, + ) assert len(ctx.layers) == 2 assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 5912d8eac9..3027dc9208 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -1,35 +1,46 @@ from __future__ import annotations import asyncio -from contextlib import asynccontextmanager -from dataclasses import dataclass import socket import ssl -from typing import Any, AsyncGenerator, Callable, ClassVar, Optional, TypeVar +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from typing import Any +from typing import Callable +from typing import ClassVar +from typing import Optional +from typing import TypeVar from unittest.mock import Mock +import pytest from aioquic.asyncio.protocol import QuicConnectionProtocol from aioquic.asyncio.server import QuicServer from aioquic.h3 import events as h3_events -from aioquic.h3.connection import H3Connection, FrameUnexpected +from aioquic.h3.connection import FrameUnexpected +from aioquic.h3.connection import H3Connection from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration -from aioquic.quic.connection import QuicConnection, QuicConnectionError -import pytest -from mitmproxy.addons.next_layer import NextLayer -from mitmproxy.addons.tlsconfig import TlsConfig +from aioquic.quic.connection import QuicConnection +from aioquic.quic.connection import QuicConnectionError import mitmproxy.platform -from mitmproxy import dns, exceptions +from mitmproxy import dns +from mitmproxy import exceptions from mitmproxy.addons import dns_resolver +from mitmproxy.addons.next_layer import NextLayer from mitmproxy.addons.proxyserver import Proxyserver +from mitmproxy.addons.tlsconfig import TlsConfig from mitmproxy.connection import Address from mitmproxy.net import udp -from mitmproxy.proxy import layers, server_hooks +from mitmproxy.proxy import layers +from mitmproxy.proxy import server_hooks from mitmproxy.proxy.layers import tls from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.test import taddons, tflow -from mitmproxy.test.tflow import tclient_conn, tserver_conn +from mitmproxy.test import taddons +from mitmproxy.test import tflow +from mitmproxy.test.tflow import tclient_conn +from mitmproxy.test.tflow import tserver_conn from mitmproxy.test.tutils import tdnsreq from mitmproxy.utils import data @@ -298,7 +309,9 @@ async def test_dns(caplog_async) -> None: resp = dns.Message.unpack(await r.read(udp.MAX_DATAGRAM_SIZE)) assert req.id == resp.id and "8.8.8.8" in str(resp) assert len(ps.connections) == 1 - dns_layer = ps.connections[("udp", w.get_extra_info("sockname"), dns_addr)].layer + dns_layer = ps.connections[ + ("udp", w.get_extra_info("sockname"), dns_addr) + ].layer assert isinstance(dns_layer, layers.DNSLayer) assert len(dns_layer.flows) == 2 @@ -339,10 +352,10 @@ async def test_dtls(monkeypatch, caplog_async) -> None: caplog_async.set_level("INFO") def server_handler( - transport: asyncio.DatagramTransport, - data: bytes, - remote_addr: Address, - _: Address, + transport: asyncio.DatagramTransport, + data: bytes, + remote_addr: Address, + _: Address, ): assert data == b"\x16" transport.sendto(b"\x01", remote_addr) @@ -360,7 +373,9 @@ def server_handler( tctx.configure(ps, mode=[mode]) assert await ps.setup_servers() ps.running() - await caplog_async.await_log(f"reverse proxy to dtls://{server_addr[0]}:{server_addr[1]} listening") + await caplog_async.await_log( + f"reverse proxy to dtls://{server_addr[0]}:{server_addr[1]} listening" + ) assert ps.servers addr = ps.servers[mode].listen_addrs[0] r, w = await udp.open_connection(*addr) @@ -392,9 +407,7 @@ def http_headers_received(self, event: h3_events.HeadersReceived) -> None: response.append((b":status", b"200")) response.append((b"x-response", headers[b"x-request"])) self.http.send_headers( - stream_id=event.stream_id, - headers=response, - end_stream=event.stream_ended + stream_id=event.stream_id, headers=response, end_stream=event.stream_ended ) self.transmit() @@ -441,7 +454,9 @@ def quic_event_received(self, event: quic_events.QuicEvent) -> None: @asynccontextmanager -async def quic_server(create_protocol, alpn: list[str]) -> AsyncGenerator[Address, None]: +async def quic_server( + create_protocol, alpn: list[str] +) -> AsyncGenerator[Address, None]: configuration = QuicConfiguration( is_client=False, alpn_protocols=alpn, @@ -475,9 +490,11 @@ def __init__(self, *args, **kwargs) -> None: def quic_event_received(self, event: quic_events.QuicEvent) -> None: if not self._waiter.done(): if isinstance(event, quic_events.ConnectionTerminated): - self._waiter.set_exception(QuicConnectionError( - event.error_code, event.frame_type, event.reason_phrase - )) + self._waiter.set_exception( + QuicConnectionError( + event.error_code, event.frame_type, event.reason_phrase + ) + ) elif isinstance(event, quic_events.HandshakeCompleted): self._waiter.set_result(None) @@ -501,9 +518,11 @@ def quic_event_received(self, event: quic_events.QuicEvent) -> None: if isinstance(event, quic_events.DatagramFrameReceived): self._datagram.set_result(event.data) elif isinstance(event, quic_events.ConnectionTerminated): - self._datagram.set_exception(QuicConnectionError( - event.error_code, event.frame_type, event.reason_phrase - )) + self._datagram.set_exception( + QuicConnectionError( + event.error_code, event.frame_type, event.reason_phrase + ) + ) def send_datagram(self, data: bytes) -> None: self._quic.send_datagram_frame(data) @@ -532,7 +551,6 @@ def __setattr__(self, name: str, value: Any) -> None: class H3Client(QuicClient): - def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._responses: dict[int, H3Response] = dict() @@ -765,7 +783,9 @@ async def test_reverse_http3_and_quic_stream( tctx.configure(ps, mode=[mode]) assert await ps.setup_servers() ps.running() - await caplog_async.await_log(f"reverse proxy to {scheme}://{server_addr[0]}:{server_addr[1]} listening") + await caplog_async.await_log( + f"reverse proxy to {scheme}://{server_addr[0]}:{server_addr[1]} listening" + ) assert ps.servers addr = ps.servers[mode].listen_addrs[0] async with quic_connect(H3Client, alpn=["h3"], address=addr) as client: @@ -797,10 +817,14 @@ async def test_reverse_quic_datagram(caplog_async, connection_strategy: str) -> tctx.configure(ps, mode=[mode]) assert await ps.setup_servers() ps.running() - await caplog_async.await_log(f"reverse proxy to quic://{server_addr[0]}:{server_addr[1]} listening") + await caplog_async.await_log( + f"reverse proxy to quic://{server_addr[0]}:{server_addr[1]} listening" + ) assert ps.servers addr = ps.servers[mode].listen_addrs[0] - async with quic_connect(QuicDatagramClient, alpn=["dgram"], address=addr) as client: + async with quic_connect( + QuicDatagramClient, alpn=["dgram"], address=addr + ) as client: client.send_datagram(b"echo") assert await client.recv_datagram() == b"echo" diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py index 5c01e95e84..78513d586c 100644 --- a/test/mitmproxy/addons/test_readfile.py +++ b/test/mitmproxy/addons/test_readfile.py @@ -1,8 +1,8 @@ import asyncio import io +from unittest import mock import pytest -from unittest import mock import mitmproxy.io from mitmproxy import exceptions diff --git a/test/mitmproxy/addons/test_save.py b/test/mitmproxy/addons/test_save.py index 8361409ac8..b14f5ccace 100644 --- a/test/mitmproxy/addons/test_save.py +++ b/test/mitmproxy/addons/test_save.py @@ -196,4 +196,4 @@ def _raise(*_): with pytest.raises(SystemExit): sa.response(f) - assert "Error while writing" in capsys.readouterr().err \ No newline at end of file + assert "Error while writing" in capsys.readouterr().err diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 0971fa0e9d..678550a991 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -129,7 +129,10 @@ async def test_import_error(self, monkeypatch, tdata, caplog): tdata.path("mitmproxy/data/addonscripts/import_error.py"), False, ) - assert "Note that mitmproxy's binaries include their own Python environment" in caplog.text + assert ( + "Note that mitmproxy's binaries include their own Python environment" + in caplog.text + ) async def test_optionexceptions(self, tdata, caplog_async): with taddons.context() as tctx: @@ -183,7 +186,9 @@ async def test_script_run(self, tdata, caplog_async): with taddons.context(sc): sc.script_run([tflow.tflow(resp=True)], rp) await caplog_async.await_log("recorder response") - debug = [i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG"] + debug = [ + i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG" + ] assert debug == [ "recorder configure", "recorder running", @@ -267,7 +272,9 @@ async def test_order(self, tdata, caplog_async): ], ) await caplog_async.await_log("configure") - debug = [i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG"] + debug = [ + i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG" + ] assert debug == [ "a load", "a configure", @@ -291,7 +298,9 @@ async def test_order(self, tdata, caplog_async): ) await caplog_async.await_log("b configure") - debug = [i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG"] + debug = [ + i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG" + ] assert debug == [ "c configure", "a configure", @@ -307,7 +316,9 @@ async def test_order(self, tdata, caplog_async): ], ) await caplog_async.await_log("e configure") - debug = [i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG"] + debug = [ + i.msg for i in caplog_async.caplog.records if i.levelname == "DEBUG" + ] assert debug == [ "c done", "b done", diff --git a/test/mitmproxy/addons/test_stickyauth.py b/test/mitmproxy/addons/test_stickyauth.py index 7b422fdd17..a684b8162a 100644 --- a/test/mitmproxy/addons/test_stickyauth.py +++ b/test/mitmproxy/addons/test_stickyauth.py @@ -1,10 +1,9 @@ import pytest -from mitmproxy.test import tflow -from mitmproxy.test import taddons - -from mitmproxy.addons import stickyauth from mitmproxy import exceptions +from mitmproxy.addons import stickyauth +from mitmproxy.test import taddons +from mitmproxy.test import tflow def test_configure(): diff --git a/test/mitmproxy/addons/test_stickycookie.py b/test/mitmproxy/addons/test_stickycookie.py index d3edbbdb76..906087c0d2 100644 --- a/test/mitmproxy/addons/test_stickycookie.py +++ b/test/mitmproxy/addons/test_stickycookie.py @@ -1,9 +1,8 @@ import pytest -from mitmproxy.test import tflow -from mitmproxy.test import taddons - from mitmproxy.addons import stickycookie +from mitmproxy.test import taddons +from mitmproxy.test import tflow from mitmproxy.test import tutils as ntutils diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index 62573e218c..0ebf3601a0 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -13,10 +13,7 @@ @pytest.fixture(autouse=True) def ensure_cleanup(): yield - assert not any( - isinstance(x, termlog.TermLogHandler) - for x in logging.root.handlers - ) + assert not any(isinstance(x, termlog.TermLogHandler) for x in logging.root.handlers) async def test_delayed_teardown(): diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 9bd6142b90..1da747d04c 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -4,15 +4,21 @@ from typing import Union import pytest - from cryptography import x509 from OpenSSL import SSL -from mitmproxy import certs, connection, tls, options + +from mitmproxy import certs +from mitmproxy import connection +from mitmproxy import options +from mitmproxy import tls from mitmproxy.addons import tlsconfig from mitmproxy.proxy import context -from mitmproxy.proxy.layers import modes, quic, tls as proxy_tls +from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tls as proxy_tls from mitmproxy.test import taddons -from test.mitmproxy.proxy.layers import test_quic, test_tls +from test.mitmproxy.proxy.layers import test_quic +from test.mitmproxy.proxy.layers import test_tls def test_alpn_select_callback(): @@ -63,7 +69,11 @@ def test_alpn_select_callback(): def _ctx(opts: options.Options) -> context.Context: return context.Context( - connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), + connection.Client( + peername=("client", 1234), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, + ), opts, ) @@ -172,10 +182,7 @@ def quic_do_handshake( tssl_server.write(tssl_client.read()) tssl_client.write(tssl_server.read()) tssl_server.write(tssl_client.read()) - return ( - tssl_client.handshake_completed() - and tssl_server.handshake_completed() - ) + return tssl_client.handshake_completed() and tssl_server.handshake_completed() def test_tls_start_client(self, tdata): ta = tlsconfig.TlsConfig() @@ -229,8 +236,12 @@ def test_quic_start_client(self, tdata): tssl_client = test_quic.SSLTest(alpn=["h3"]) assert self.quic_do_handshake(tssl_client, tssl_server) - san = tssl_client.quic.tls._peer_certificate.extensions.get_extension_for_class(x509.SubjectAlternativeName) - assert san.value.get_values_for_type(x509.DNSName) == ["example.mitmproxy.org"] + san = tssl_client.quic.tls._peer_certificate.extensions.get_extension_for_class( + x509.SubjectAlternativeName + ) + assert san.value.get_values_for_type(x509.DNSName) == [ + "example.mitmproxy.org" + ] def test_tls_start_server_cannot_verify(self): ta = tlsconfig.TlsConfig() @@ -240,7 +251,9 @@ def test_tls_start_server_cannot_verify(self): ctx.server.sni = "" # explicitly opt out of using the address. tls_start = tls.TlsData(ctx.server, context=ctx) - with pytest.raises(ValueError, match="Cannot validate certificate hostname without SNI"): + with pytest.raises( + ValueError, match="Cannot validate certificate hostname without SNI" + ): ta.tls_start_server(tls_start) def test_tls_start_server_verify_failed(self): @@ -305,7 +318,9 @@ def test_quic_start_server_verify_ok(self, hostname, tdata): ta.quic_start_server(tls_start) assert settings_client is tls_start.settings - tssl_server = test_quic.SSLTest(server_side=True, sni=hostname.encode(), alpn=["h3"]) + tssl_server = test_quic.SSLTest( + server_side=True, sni=hostname.encode(), alpn=["h3"] + ) assert self.quic_do_handshake(tssl_client, tssl_server) def test_tls_start_server_insecure(self): diff --git a/test/mitmproxy/addons/test_upstream_auth.py b/test/mitmproxy/addons/test_upstream_auth.py index 1eb8eb0131..883dabc2f3 100644 --- a/test/mitmproxy/addons/test_upstream_auth.py +++ b/test/mitmproxy/addons/test_upstream_auth.py @@ -1,11 +1,12 @@ import base64 + import pytest from mitmproxy import exceptions +from mitmproxy.addons import upstream_auth from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.test import taddons from mitmproxy.test import tflow -from mitmproxy.addons import upstream_auth def test_configure(): diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 7c1b041cf6..199fc00c4d 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -1,14 +1,14 @@ import pytest -from mitmproxy.test import tflow - -from mitmproxy.addons import view -from mitmproxy import flowfilter from mitmproxy import exceptions +from mitmproxy import flowfilter from mitmproxy import io +from mitmproxy.addons import view from mitmproxy.test import taddons +from mitmproxy.test import tflow from mitmproxy.tools.console import consoleaddons -from mitmproxy.tools.console.common import render_marker, SYMBOL_MARK +from mitmproxy.tools.console.common import render_marker +from mitmproxy.tools.console.common import SYMBOL_MARK def tft(*, method="get", start=0): diff --git a/test/mitmproxy/contentviews/image/test_view.py b/test/mitmproxy/contentviews/image/test_view.py index 67c4b81b4e..61c0c6379e 100644 --- a/test/mitmproxy/contentviews/image/test_view.py +++ b/test/mitmproxy/contentviews/image/test_view.py @@ -1,5 +1,5 @@ -from mitmproxy.contentviews import image from .. import full_eval +from mitmproxy.contentviews import image def test_view_image(tdata): diff --git a/test/mitmproxy/contentviews/test_auto.py b/test/mitmproxy/contentviews/test_auto.py index 459d839f0a..5dfbe2aafe 100644 --- a/test/mitmproxy/contentviews/test_auto.py +++ b/test/mitmproxy/contentviews/test_auto.py @@ -1,6 +1,6 @@ +from . import full_eval from mitmproxy.contentviews import auto from mitmproxy.test import tflow -from . import full_eval def test_view_auto(): diff --git a/test/mitmproxy/contentviews/test_base.py b/test/mitmproxy/contentviews/test_base.py index cd879bfdaa..efa971534e 100644 --- a/test/mitmproxy/contentviews/test_base.py +++ b/test/mitmproxy/contentviews/test_base.py @@ -1,4 +1,5 @@ import pytest + from mitmproxy.contentviews import base diff --git a/test/mitmproxy/contentviews/test_css.py b/test/mitmproxy/contentviews/test_css.py index 7474a6b36f..a2192d12f0 100644 --- a/test/mitmproxy/contentviews/test_css.py +++ b/test/mitmproxy/contentviews/test_css.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy.contentviews import css from . import full_eval +from mitmproxy.contentviews import css @pytest.mark.parametrize( diff --git a/test/mitmproxy/contentviews/test_graphql.py b/test/mitmproxy/contentviews/test_graphql.py index a38eedea00..89beda814c 100644 --- a/test/mitmproxy/contentviews/test_graphql.py +++ b/test/mitmproxy/contentviews/test_graphql.py @@ -1,8 +1,8 @@ from hypothesis import given from hypothesis.strategies import binary -from mitmproxy.contentviews import graphql from . import full_eval +from mitmproxy.contentviews import graphql def test_render_priority(): diff --git a/test/mitmproxy/contentviews/test_grpc.py b/test/mitmproxy/contentviews/test_grpc.py index 8d296fe65d..6a88035260 100644 --- a/test/mitmproxy/contentviews/test_grpc.py +++ b/test/mitmproxy/contentviews/test_grpc.py @@ -1,16 +1,16 @@ +import struct + import pytest +from . import full_eval from mitmproxy.contentviews import grpc -from mitmproxy.contentviews.grpc import ( - ViewGrpcProtobuf, - ViewConfig, - ProtoParser, - parse_grpc_messages, -) +from mitmproxy.contentviews.grpc import parse_grpc_messages +from mitmproxy.contentviews.grpc import ProtoParser +from mitmproxy.contentviews.grpc import ViewConfig +from mitmproxy.contentviews.grpc import ViewGrpcProtobuf from mitmproxy.net.encoding import encode -from mitmproxy.test import tflow, tutils -import struct -from . import full_eval +from mitmproxy.test import tflow +from mitmproxy.test import tutils datadir = "mitmproxy/contentviews/test_grpc_data/" diff --git a/test/mitmproxy/contentviews/test_hex.py b/test/mitmproxy/contentviews/test_hex.py index 90db4bd7c1..8eadf8436c 100644 --- a/test/mitmproxy/contentviews/test_hex.py +++ b/test/mitmproxy/contentviews/test_hex.py @@ -1,5 +1,5 @@ -from mitmproxy.contentviews import hex from . import full_eval +from mitmproxy.contentviews import hex def test_view_hex(): diff --git a/test/mitmproxy/contentviews/test_http3.py b/test/mitmproxy/contentviews/test_http3.py index 157ee6914a..d1b9fc6413 100644 --- a/test/mitmproxy/contentviews/test_http3.py +++ b/test/mitmproxy/contentviews/test_http3.py @@ -1,58 +1,59 @@ import pytest +from . import full_eval +from mitmproxy.contentviews import http3 from mitmproxy.tcp import TCPMessage from mitmproxy.test import tflow -from mitmproxy.contentviews import http3 - -from . import full_eval if http3 is None: pytest.skip("HTTP/3 not available.", allow_module_level=True) -@pytest.mark.parametrize("data", [ - # HEADERS - b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xbc\xda\xe0\xdd", - # broken HEADERS - b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xff\xff\xff\xff", - # headers + data - ( - b'\x01@I\x00\x00\xdb_\'\x93I|\xa5\x89\xd3M\x1fj\x12q\xd8\x82\xa6\x0bP\xb0\xd0C\x1b_M\x90\xd0bXt\x1eT\xad\x8f~\xfdp' - b'\xeb\xc8\xc0\x97\x07V\x96\xd0z\xbe\x94\x08\x94\xdcZ\xd4\x10\x04%\x02\xe5\xc6\xde\xb8\x17\x14\xc5\xa3\x7fT\x03315' - b'\x00A;\r\n<' - b'TITLE>Not Found\r\n\r\n

Not Found

\r\n

HTTP Error 404. The requested resource is not found.

\r\n\r\n' - ), - b"", -]) +@pytest.mark.parametrize( + "data", + [ + # HEADERS + b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xbc\xda\xe0\xdd", + # broken HEADERS + b"\x01\x1d\x00\x00\xd1\xc1\xd7P\x8a\x08\x9d\\\x0b\x81p\xdcx\x0f\x03_P\x88%\xb6P\xc3\xab\xff\xff\xff\xff", + # headers + data + ( + b"\x01@I\x00\x00\xdb_'\x93I|\xa5\x89\xd3M\x1fj\x12q\xd8\x82\xa6\x0bP\xb0\xd0C\x1b_M\x90\xd0bXt\x1eT\xad\x8f~\xfdp" + b"\xeb\xc8\xc0\x97\x07V\x96\xd0z\xbe\x94\x08\x94\xdcZ\xd4\x10\x04%\x02\xe5\xc6\xde\xb8\x17\x14\xc5\xa3\x7fT\x03315" + b'\x00A;\r\n<' + b'TITLE>Not Found\r\n\r\n

Not Found

\r\n

HTTP Error 404. The requested resource is not found.

\r\n\r\n" + ), + b"", + ], +) def test_view_http3(data): v = full_eval(http3.ViewHttp3()) - t = tflow.ttcpflow(messages=[ - TCPMessage(from_client=len(data) > 16, content=data) - ]) + t = tflow.ttcpflow(messages=[TCPMessage(from_client=len(data) > 16, content=data)]) t.metadata["quic_is_unidirectional"] = False - assert (v(b"", flow=t, tcp_message=t.messages[0])) - - -@pytest.mark.parametrize("data", [ - # SETTINGS - b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00", - # unknown setting - b"\x00\x04\r\x3f\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00", - # out of bounds - b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x42\x00", - # incomplete - b"\x00\x04\r\x06\xff\xff\xff", - # QPACK encoder stream - b"\x02", -]) + assert v(b"", flow=t, tcp_message=t.messages[0]) + + +@pytest.mark.parametrize( + "data", + [ + # SETTINGS + b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00", + # unknown setting + b"\x00\x04\r\x3f\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x07\x00", + # out of bounds + b"\x00\x04\r\x06\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x42\x00", + # incomplete + b"\x00\x04\r\x06\xff\xff\xff", + # QPACK encoder stream + b"\x02", + ], +) def test_view_http3_unidirectional(data): v = full_eval(http3.ViewHttp3()) - t = tflow.ttcpflow(messages=[ - TCPMessage(from_client=len(data) > 16, content=data) - ]) + t = tflow.ttcpflow(messages=[TCPMessage(from_client=len(data) > 16, content=data)]) t.metadata["quic_is_unidirectional"] = True - assert (v(b"", flow=t, tcp_message=t.messages[0])) + assert v(b"", flow=t, tcp_message=t.messages[0]) def test_render_priority(): diff --git a/test/mitmproxy/contentviews/test_javascript.py b/test/mitmproxy/contentviews/test_javascript.py index c050adee4f..64647446d2 100644 --- a/test/mitmproxy/contentviews/test_javascript.py +++ b/test/mitmproxy/contentviews/test_javascript.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy.contentviews import javascript from . import full_eval +from mitmproxy.contentviews import javascript def test_view_javascript(): diff --git a/test/mitmproxy/contentviews/test_json.py b/test/mitmproxy/contentviews/test_json.py index 5b3883060f..9711a64659 100644 --- a/test/mitmproxy/contentviews/test_json.py +++ b/test/mitmproxy/contentviews/test_json.py @@ -1,8 +1,8 @@ from hypothesis import given from hypothesis.strategies import binary -from mitmproxy.contentviews import json from . import full_eval +from mitmproxy.contentviews import json def test_parse_json(): @@ -18,31 +18,66 @@ def test_parse_json(): def test_format_json(): assert list(json.format_json({"data": ["str", 42, True, False, None, {}, []]})) assert list(json.format_json({"string": "test"})) == [ - [('text', '{'), ('text', '')], - [('text', ' '), ('Token_Name_Tag', '"string"'), ('text', ': '), ('Token_Literal_String', '"test"'), ('text', '')], - [('text', ''), ('text', '}')]] + [("text", "{"), ("text", "")], + [ + ("text", " "), + ("Token_Name_Tag", '"string"'), + ("text", ": "), + ("Token_Literal_String", '"test"'), + ("text", ""), + ], + [("text", ""), ("text", "}")], + ] assert list(json.format_json({"num": 4})) == [ - [('text', '{'), ('text', '')], - [('text', ' '), ('Token_Name_Tag', '"num"'), ('text', ': '), ('Token_Literal_Number', '4'), ('text', '')], - [('text', ''), ('text', '}')]] + [("text", "{"), ("text", "")], + [ + ("text", " "), + ("Token_Name_Tag", '"num"'), + ("text", ": "), + ("Token_Literal_Number", "4"), + ("text", ""), + ], + [("text", ""), ("text", "}")], + ] assert list(json.format_json({"bool": True})) == [ - [('text', '{'), ('text', '')], - [('text', ' '), ('Token_Name_Tag', '"bool"'), ('text', ': '), ('Token_Keyword_Constant', 'true'), ('text', '')], - [('text', ''), ('text', '}')]] + [("text", "{"), ("text", "")], + [ + ("text", " "), + ("Token_Name_Tag", '"bool"'), + ("text", ": "), + ("Token_Keyword_Constant", "true"), + ("text", ""), + ], + [("text", ""), ("text", "}")], + ] assert list(json.format_json({"object": {"int": 1}})) == [ - [('text', '{'), ('text', '')], - [('text', ' '), ('Token_Name_Tag', '"object"'), ('text', ': '), ('text', '{'), ('text', '')], - [('text', ' '), ('Token_Name_Tag', '"int"'), ('text', ': '), ('Token_Literal_Number', '1'), ('text', '')], - [('text', ' '), ('text', '}'), ('text', '')], - [('text', ''), ('text', '}')]] + [("text", "{"), ("text", "")], + [ + ("text", " "), + ("Token_Name_Tag", '"object"'), + ("text", ": "), + ("text", "{"), + ("text", ""), + ], + [ + ("text", " "), + ("Token_Name_Tag", '"int"'), + ("text", ": "), + ("Token_Literal_Number", "1"), + ("text", ""), + ], + [("text", " "), ("text", "}"), ("text", "")], + [("text", ""), ("text", "}")], + ] assert list(json.format_json({"list": ["string", 1, True]})) == [ - [('text', '{'), ('text', '')], - [('text', ' '), ('Token_Name_Tag', '"list"'), ('text', ': '), ('text', '[')], - [('Token_Literal_String', ' "string"'), ('text', ',')], - [('Token_Literal_Number', ' 1'), ('text', ',')], - [('Token_Keyword_Constant', ' true'), ('text', '')], - [('text', ' '), ('text', ']'), ('text', '')], - [('text', ''), ('text', '}')]] + [("text", "{"), ("text", "")], + [("text", " "), ("Token_Name_Tag", '"list"'), ("text", ": "), ("text", "[")], + [("Token_Literal_String", ' "string"'), ("text", ",")], + [("Token_Literal_Number", " 1"), ("text", ",")], + [("Token_Keyword_Constant", " true"), ("text", "")], + [("text", " "), ("text", "]"), ("text", "")], + [("text", ""), ("text", "}")], + ] def test_view_json(): diff --git a/test/mitmproxy/contentviews/test_mqtt.py b/test/mitmproxy/contentviews/test_mqtt.py index 7acc335412..87cc09d40e 100644 --- a/test/mitmproxy/contentviews/test_mqtt.py +++ b/test/mitmproxy/contentviews/test_mqtt.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy.contentviews import mqtt from . import full_eval +from mitmproxy.contentviews import mqtt @pytest.mark.parametrize( @@ -9,8 +9,14 @@ [ pytest.param(b"\xC0\x00", "[PINGREQ]", id="PINGREQ"), pytest.param(b"\xD0\x00", "[PINGRESP]", id="PINGRESP"), - pytest.param(b"\x90\x00", "Packet type SUBACK is not supported yet!", id="SUBACK"), - pytest.param(b"\xA0\x00", "Packet type UNSUBSCRIBE is not supported yet!", id="UNSUBSCRIBE"), + pytest.param( + b"\x90\x00", "Packet type SUBACK is not supported yet!", id="SUBACK" + ), + pytest.param( + b"\xA0\x00", + "Packet type UNSUBSCRIBE is not supported yet!", + id="UNSUBSCRIBE", + ), pytest.param( b"\x82\x31\x00\x03\x00\x2cxxxx/yy/zzzzzz/56:6F:5E:6A:01:05/messages/in\x01", "[SUBSCRIBE] sent topic filters: 'xxxx/yy/zzzzzz/56:6F:5E:6A:01:05/messages/in'", @@ -52,10 +58,7 @@ def test_view_mqtt(data, expected_text): assert output == [[("text", expected_text)]] -@pytest.mark.parametrize( - "data", - [b"\xC0\xFF\xFF\xFF\xFF"] -) +@pytest.mark.parametrize("data", [b"\xC0\xFF\xFF\xFF\xFF"]) def test_mqtt_malformed(data): v = full_eval(mqtt.ViewMQTT()) with pytest.raises(Exception): diff --git a/test/mitmproxy/contentviews/test_msgpack.py b/test/mitmproxy/contentviews/test_msgpack.py index eeba8b2d13..65c8487f0b 100644 --- a/test/mitmproxy/contentviews/test_msgpack.py +++ b/test/mitmproxy/contentviews/test_msgpack.py @@ -1,10 +1,9 @@ from hypothesis import given from hypothesis.strategies import binary - from msgpack import packb -from mitmproxy.contentviews import msgpack from . import full_eval +from mitmproxy.contentviews import msgpack def msgpack_encode(content): @@ -18,32 +17,86 @@ def test_parse_msgpack(): def test_format_msgpack(): - assert list(msgpack.format_msgpack({"string": "test", "int": 1, "float": 1.44, "bool": True})) == [ - [('text', '{')], - [('text', ''), ('text', ' '), ('Token_Name_Tag', '"string"'), ('text', ': '), ('Token_Literal_String', '"test"'), ('text', ',')], - [('text', ''), ('text', ' '), ('Token_Name_Tag', '"int"'), ('text', ': '), ('Token_Literal_Number', '1'), ('text', ',')], - [('text', ''), ('text', ' '), ('Token_Name_Tag', '"float"'), ('text', ': '), ('Token_Literal_Number', '1.44'), ('text', ',')], - [('text', ''), ('text', ' '), ('Token_Name_Tag', '"bool"'), ('text', ': '), ('Token_Keyword_Constant', 'True')], - [('text', ''), ('text', '}')] + assert list( + msgpack.format_msgpack( + {"string": "test", "int": 1, "float": 1.44, "bool": True} + ) + ) == [ + [("text", "{")], + [ + ("text", ""), + ("text", " "), + ("Token_Name_Tag", '"string"'), + ("text", ": "), + ("Token_Literal_String", '"test"'), + ("text", ","), + ], + [ + ("text", ""), + ("text", " "), + ("Token_Name_Tag", '"int"'), + ("text", ": "), + ("Token_Literal_Number", "1"), + ("text", ","), + ], + [ + ("text", ""), + ("text", " "), + ("Token_Name_Tag", '"float"'), + ("text", ": "), + ("Token_Literal_Number", "1.44"), + ("text", ","), + ], + [ + ("text", ""), + ("text", " "), + ("Token_Name_Tag", '"bool"'), + ("text", ": "), + ("Token_Keyword_Constant", "True"), + ], + [("text", ""), ("text", "}")], ] assert list(msgpack.format_msgpack({"object": {"key": "value"}, "list": [1]})) == [ - [('text', '{')], - [('text', ''), ('text', ' '), ('Token_Name_Tag', '"object"'), ('text', ': '), ('text', '{')], - [('text', ' '), ('text', ' '), ('Token_Name_Tag', '"key"'), ('text', ': '), ('Token_Literal_String', '"value"')], - [('text', ' '), ('text', '}'), ('text', ',')], - [('text', ''), ('text', ' '), ('Token_Name_Tag', '"list"'), ('text', ': '), ('text', '[')], - [('text', ' '), ('text', ' '), ('Token_Literal_Number', '1')], - [('text', ' '), ('text', ']')], - [('text', ''), ('text', '}')]] + [("text", "{")], + [ + ("text", ""), + ("text", " "), + ("Token_Name_Tag", '"object"'), + ("text", ": "), + ("text", "{"), + ], + [ + ("text", " "), + ("text", " "), + ("Token_Name_Tag", '"key"'), + ("text", ": "), + ("Token_Literal_String", '"value"'), + ], + [("text", " "), ("text", "}"), ("text", ",")], + [ + ("text", ""), + ("text", " "), + ("Token_Name_Tag", '"list"'), + ("text", ": "), + ("text", "["), + ], + [("text", " "), ("text", " "), ("Token_Literal_Number", "1")], + [("text", " "), ("text", "]")], + [("text", ""), ("text", "}")], + ] - assert list(msgpack.format_msgpack('string')) == [[('Token_Literal_String', '"string"')]] + assert list(msgpack.format_msgpack("string")) == [ + [("Token_Literal_String", '"string"')] + ] - assert list(msgpack.format_msgpack(1.2)) == [[('Token_Literal_Number', '1.2')]] + assert list(msgpack.format_msgpack(1.2)) == [[("Token_Literal_Number", "1.2")]] - assert list(msgpack.format_msgpack(True)) == [[('Token_Keyword_Constant', 'True')]] + assert list(msgpack.format_msgpack(True)) == [[("Token_Keyword_Constant", "True")]] - assert list(msgpack.format_msgpack(b'\x01\x02\x03')) == [[('text', "b'\\x01\\x02\\x03'")]] + assert list(msgpack.format_msgpack(b"\x01\x02\x03")) == [ + [("text", "b'\\x01\\x02\\x03'")] + ] def test_view_msgpack(): diff --git a/test/mitmproxy/contentviews/test_multipart.py b/test/mitmproxy/contentviews/test_multipart.py index da1f723e00..a748231d64 100644 --- a/test/mitmproxy/contentviews/test_multipart.py +++ b/test/mitmproxy/contentviews/test_multipart.py @@ -1,5 +1,5 @@ -from mitmproxy.contentviews import multipart from . import full_eval +from mitmproxy.contentviews import multipart def test_view_multipart(): diff --git a/test/mitmproxy/contentviews/test_protobuf.py b/test/mitmproxy/contentviews/test_protobuf.py index 5f8d84d2e8..99d6768ede 100644 --- a/test/mitmproxy/contentviews/test_protobuf.py +++ b/test/mitmproxy/contentviews/test_protobuf.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy.contentviews import protobuf from . import full_eval +from mitmproxy.contentviews import protobuf datadir = "mitmproxy/contentviews/test_protobuf_data/" diff --git a/test/mitmproxy/contentviews/test_query.py b/test/mitmproxy/contentviews/test_query.py index af47a02f81..b4b1408eff 100644 --- a/test/mitmproxy/contentviews/test_query.py +++ b/test/mitmproxy/contentviews/test_query.py @@ -1,6 +1,6 @@ +from . import full_eval from mitmproxy.contentviews import query from mitmproxy.test import tutils -from . import full_eval def test_view_query(): diff --git a/test/mitmproxy/contentviews/test_raw.py b/test/mitmproxy/contentviews/test_raw.py index d9fa44f898..0cffcf869f 100644 --- a/test/mitmproxy/contentviews/test_raw.py +++ b/test/mitmproxy/contentviews/test_raw.py @@ -1,5 +1,5 @@ -from mitmproxy.contentviews import raw from . import full_eval +from mitmproxy.contentviews import raw def test_view_raw(): diff --git a/test/mitmproxy/contentviews/test_urlencoded.py b/test/mitmproxy/contentviews/test_urlencoded.py index 84c33dfcea..e6005c0c8a 100644 --- a/test/mitmproxy/contentviews/test_urlencoded.py +++ b/test/mitmproxy/contentviews/test_urlencoded.py @@ -1,6 +1,6 @@ +from . import full_eval from mitmproxy.contentviews import urlencoded from mitmproxy.net.http import url -from . import full_eval def test_view_urlencoded(): diff --git a/test/mitmproxy/contentviews/test_wbxml.py b/test/mitmproxy/contentviews/test_wbxml.py index e37f0da21f..11f2886bf8 100644 --- a/test/mitmproxy/contentviews/test_wbxml.py +++ b/test/mitmproxy/contentviews/test_wbxml.py @@ -1,5 +1,5 @@ -from mitmproxy.contentviews import wbxml from . import full_eval +from mitmproxy.contentviews import wbxml datadir = "mitmproxy/contentviews/test_wbxml_data/" diff --git a/test/mitmproxy/contentviews/test_xml_html.py b/test/mitmproxy/contentviews/test_xml_html.py index 4bb007972e..de2b8d59f5 100644 --- a/test/mitmproxy/contentviews/test_xml_html.py +++ b/test/mitmproxy/contentviews/test_xml_html.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy.contentviews import xml_html from . import full_eval +from mitmproxy.contentviews import xml_html datadir = "mitmproxy/contentviews/test_xml_html_data/" diff --git a/test/mitmproxy/coretypes/test_bidi.py b/test/mitmproxy/coretypes/test_bidi.py index 3bdad3c2c8..b4cff33cba 100644 --- a/test/mitmproxy/coretypes/test_bidi.py +++ b/test/mitmproxy/coretypes/test_bidi.py @@ -1,4 +1,5 @@ import pytest + from mitmproxy.coretypes import bidi diff --git a/test/mitmproxy/coretypes/test_serializable.py b/test/mitmproxy/coretypes/test_serializable.py index 70980a625e..06e10fc547 100644 --- a/test/mitmproxy/coretypes/test_serializable.py +++ b/test/mitmproxy/coretypes/test_serializable.py @@ -5,7 +5,8 @@ import enum from collections.abc import Mapping from dataclasses import dataclass -from typing import Literal, Optional +from typing import Literal +from typing import Optional import pytest @@ -108,16 +109,31 @@ class FrozenWrapper(SerializableDataclass): class TestSerializableDataclass: - @pytest.mark.parametrize("cls, state", [ - (Simple, {"x": 42, "y": 'foo'}), - (Simple, {"x": 42, "y": None}), - (SerializableChild, {"foo": {"x": 42, "y": "foo"}, "maybe_foo": None}), - (SerializableChild, {"foo": {"x": 42, "y": "foo"}, "maybe_foo": {"x": 42, "y": "foo"}}), - (Inheritance, {"x": 42, "y": "foo", "z": True}), - (BuiltinChildren, {"a": [1, 2, 3], "b": {"foo": 42}, "c": (1, 2), "d": [{"x": 42, "y": "foo"}], "e": 1}), - (BuiltinChildren, {"a": None, "b": None, "c": None, "d": [], "e": None}), - (TLiteral, {"l": "foo"}), - ]) + @pytest.mark.parametrize( + "cls, state", + [ + (Simple, {"x": 42, "y": "foo"}), + (Simple, {"x": 42, "y": None}), + (SerializableChild, {"foo": {"x": 42, "y": "foo"}, "maybe_foo": None}), + ( + SerializableChild, + {"foo": {"x": 42, "y": "foo"}, "maybe_foo": {"x": 42, "y": "foo"}}, + ), + (Inheritance, {"x": 42, "y": "foo", "z": True}), + ( + BuiltinChildren, + { + "a": [1, 2, 3], + "b": {"foo": 42}, + "c": (1, 2), + "d": [{"x": 42, "y": "foo"}], + "e": 1, + }, + ), + (BuiltinChildren, {"a": None, "b": None, "c": None, "d": [], "e": None}), + (TLiteral, {"l": "foo"}), + ], + ) def test_roundtrip(self, cls, state): a = cls.from_state(copy.deepcopy(state)) assert a.get_state() == state @@ -142,7 +158,9 @@ def test_invalid_type(self): with pytest.raises(ValueError): Simple.from_state({"x": 42, "y": 42}) with pytest.raises(ValueError): - BuiltinChildren.from_state({"a": None, "b": None, "c": ("foo",), "d": [], "e": None}) + BuiltinChildren.from_state( + {"a": None, "b": None, "c": ("foo",), "d": [], "e": None} + ) def test_invalid_key(self): with pytest.raises(ValueError): @@ -150,7 +168,15 @@ def test_invalid_key(self): def test_invalid_type_in_list(self): with pytest.raises(ValueError, match="Invalid value for x"): - BuiltinChildren.from_state({"a": None, "b": None, "c": None, "d": [{"x": "foo", "y": "foo"}], "e": None}) + BuiltinChildren.from_state( + { + "a": None, + "b": None, + "c": None, + "d": [{"x": "foo", "y": "foo"}], + "e": None, + } + ) def test_unsupported_type(self): with pytest.raises(TypeError): @@ -162,8 +188,12 @@ def test_literal(self): TLiteral.from_state({"l": "unknown"}) def test_peername(self): - assert Addr.from_state({"peername": ("addr", 42)}).get_state() == {"peername": ("addr", 42)} - assert Addr.from_state({"peername": ("addr", 42, 0, 0)}).get_state() == {"peername": ("addr", 42, 0, 0)} + assert Addr.from_state({"peername": ("addr", 42)}).get_state() == { + "peername": ("addr", 42) + } + assert Addr.from_state({"peername": ("addr", 42, 0, 0)}).get_state() == { + "peername": ("addr", 42, 0, 0) + } def test_set_immutable(self): w = FrozenWrapper(Frozen(42)) diff --git a/test/mitmproxy/data/addonscripts/concurrent_decorator.py b/test/mitmproxy/data/addonscripts/concurrent_decorator.py index bf2628958a..0af96c486a 100644 --- a/test/mitmproxy/data/addonscripts/concurrent_decorator.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator.py @@ -1,4 +1,5 @@ import time + from mitmproxy.script import concurrent diff --git a/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py b/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py index b4ef75292c..e08ca0cb13 100644 --- a/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py @@ -1,4 +1,5 @@ import time + from mitmproxy.script import concurrent diff --git a/test/mitmproxy/io/test_compat.py b/test/mitmproxy/io/test_compat.py index 85ba5a0eea..35b11d6191 100644 --- a/test/mitmproxy/io/test_compat.py +++ b/test/mitmproxy/io/test_compat.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy import io from mitmproxy import exceptions +from mitmproxy import io @pytest.mark.parametrize( diff --git a/test/mitmproxy/io/test_io.py b/test/mitmproxy/io/test_io.py index 9d7ad80809..73d067d95d 100644 --- a/test/mitmproxy/io/test_io.py +++ b/test/mitmproxy/io/test_io.py @@ -1,11 +1,14 @@ import io import pytest -from hypothesis import example, given +from hypothesis import example +from hypothesis import given from hypothesis.strategies import binary -from mitmproxy import exceptions, version -from mitmproxy.io import FlowReader, tnetstring +from mitmproxy import exceptions +from mitmproxy import version +from mitmproxy.io import FlowReader +from mitmproxy.io import tnetstring class TestFlowReader: diff --git a/test/mitmproxy/io/test_tnetstring.py b/test/mitmproxy/io/test_tnetstring.py index ccda4cfb8f..6ae50bc82c 100644 --- a/test/mitmproxy/io/test_tnetstring.py +++ b/test/mitmproxy/io/test_tnetstring.py @@ -1,8 +1,8 @@ -import unittest -import random -import math import io +import math +import random import struct +import unittest from mitmproxy.io import tnetstring diff --git a/test/mitmproxy/net/dns/test_domain_names.py b/test/mitmproxy/net/dns/test_domain_names.py index 72e6e5391b..5f1847b556 100644 --- a/test/mitmproxy/net/dns/test_domain_names.py +++ b/test/mitmproxy/net/dns/test_domain_names.py @@ -1,5 +1,6 @@ import re import struct + import pytest from mitmproxy.net.dns import domain_names @@ -15,14 +16,11 @@ def test_unpack_from_with_compression(): domain_names.unpack_from_with_compression( b"\x03www\xc0\x00", 0, domain_names.cache() ) - assert ( - domain_names.unpack_from_with_compression( - b"\xFF\xFF\xFF\x07example\x03org\x00\xFF\xFF\xFF\x03www\xc0\x03", - 19, - domain_names.cache(), - ) - == ("www.example.org", 6) - ) + assert domain_names.unpack_from_with_compression( + b"\xFF\xFF\xFF\x07example\x03org\x00\xFF\xFF\xFF\x03www\xc0\x03", + 19, + domain_names.cache(), + ) == ("www.example.org", 6) def test_unpack(): diff --git a/test/mitmproxy/net/http/http1/test_assemble.py b/test/mitmproxy/net/http/http1/test_assemble.py index 5d17e1bfb1..eb246cf19f 100644 --- a/test/mitmproxy/net/http/http1/test_assemble.py +++ b/test/mitmproxy/net/http/http1/test_assemble.py @@ -1,17 +1,16 @@ import pytest from mitmproxy.http import Headers -from mitmproxy.net.http.http1.assemble import ( - assemble_request, - assemble_request_head, - assemble_response, - assemble_response_head, - _assemble_request_line, - _assemble_request_headers, - _assemble_response_headers, - assemble_body, -) -from mitmproxy.test.tutils import treq, tresp +from mitmproxy.net.http.http1.assemble import _assemble_request_headers +from mitmproxy.net.http.http1.assemble import _assemble_request_line +from mitmproxy.net.http.http1.assemble import _assemble_response_headers +from mitmproxy.net.http.http1.assemble import assemble_body +from mitmproxy.net.http.http1.assemble import assemble_request +from mitmproxy.net.http.http1.assemble import assemble_request_head +from mitmproxy.net.http.http1.assemble import assemble_response +from mitmproxy.net.http.http1.assemble import assemble_response_head +from mitmproxy.test.tutils import treq +from mitmproxy.test.tutils import tresp def test_assemble_request(): diff --git a/test/mitmproxy/net/http/http1/test_read.py b/test/mitmproxy/net/http/http1/test_read.py index 3f48a672e3..a9148e7abc 100644 --- a/test/mitmproxy/net/http/http1/test_read.py +++ b/test/mitmproxy/net/http/http1/test_read.py @@ -1,18 +1,17 @@ import pytest from mitmproxy.http import Headers -from mitmproxy.net.http.http1.read import ( - read_request_head, - read_response_head, - connection_close, - expected_http_body_size, - _read_request_line, - _read_response_line, - _read_headers, - get_header_tokens, - validate_headers, -) -from mitmproxy.test.tutils import treq, tresp +from mitmproxy.net.http.http1.read import _read_headers +from mitmproxy.net.http.http1.read import _read_request_line +from mitmproxy.net.http.http1.read import _read_response_line +from mitmproxy.net.http.http1.read import connection_close +from mitmproxy.net.http.http1.read import expected_http_body_size +from mitmproxy.net.http.http1.read import get_header_tokens +from mitmproxy.net.http.http1.read import read_request_head +from mitmproxy.net.http.http1.read import read_response_head +from mitmproxy.net.http.http1.read import validate_headers +from mitmproxy.test.tutils import treq +from mitmproxy.test.tutils import tresp def test_get_header_tokens(): diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 4b7f3dd652..be5a57c76e 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -1,7 +1,8 @@ import time -import pytest from unittest import mock +import pytest + from mitmproxy.net.http import cookies diff --git a/test/mitmproxy/net/http/test_headers.py b/test/mitmproxy/net/http/test_headers.py index b7dff51d98..473b930f84 100644 --- a/test/mitmproxy/net/http/test_headers.py +++ b/test/mitmproxy/net/http/test_headers.py @@ -1,6 +1,7 @@ import collections -from mitmproxy.net.http.headers import parse_content_type, assemble_content_type +from mitmproxy.net.http.headers import assemble_content_type +from mitmproxy.net.http.headers import parse_content_type def test_parse_content_type(): diff --git a/test/mitmproxy/net/test_encoding.py b/test/mitmproxy/net/test_encoding.py index 9d155961b8..640d318aef 100644 --- a/test/mitmproxy/net/test_encoding.py +++ b/test/mitmproxy/net/test_encoding.py @@ -1,4 +1,5 @@ from unittest import mock + import pytest from mitmproxy.net import encoding diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index c4fb160621..9e26003486 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -1,6 +1,8 @@ from pathlib import Path -from OpenSSL import SSL, crypto +from OpenSSL import crypto +from OpenSSL import SSL + from mitmproxy import certs from mitmproxy.net import tls @@ -58,7 +60,7 @@ def test_sslkeylogfile(tdata, monkeypatch): try: read.do_handshake() except SSL.WantReadError: - write.bio_write(read.bio_read(2 ** 16)) + write.bio_write(read.bio_read(2**16)) else: break read, write = write, read diff --git a/test/mitmproxy/net/test_udp.py b/test/mitmproxy/net/test_udp.py index 29a848cd1f..d52636529c 100644 --- a/test/mitmproxy/net/test_udp.py +++ b/test/mitmproxy/net/test_udp.py @@ -1,8 +1,14 @@ import asyncio from typing import Optional + import pytest + from mitmproxy.connection import Address -from mitmproxy.net.udp import MAX_DATAGRAM_SIZE, DatagramReader, DatagramWriter, open_connection, start_server +from mitmproxy.net.udp import DatagramReader +from mitmproxy.net.udp import DatagramWriter +from mitmproxy.net.udp import MAX_DATAGRAM_SIZE +from mitmproxy.net.udp import open_connection +from mitmproxy.net.udp import start_server async def test_client_server(): @@ -13,7 +19,7 @@ def handle_datagram( transport: asyncio.DatagramTransport, data: bytes, remote_addr: Address, - local_addr: Address + local_addr: Address, ): nonlocal server_reader, server_writer if server_writer is None: @@ -23,9 +29,14 @@ def handle_datagram( server = await start_server(handle_datagram, "127.0.0.1", 0) assert repr(server).startswith(" context.Context: opts = options.Options() Proxyserver().load(opts) return context.Context( - connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), - timestamp_start=1605699329, state=connection.ConnectionState.OPEN), - opts + connection.Client( + peername=("client", 1234), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, + state=connection.ConnectionState.OPEN, + ), + opts, ) diff --git a/test/mitmproxy/proxy/layers/http/hyper_h2_test_helpers.py b/test/mitmproxy/proxy/layers/http/hyper_h2_test_helpers.py index d5f8e01821..9b1e2676df 100644 --- a/test/mitmproxy/proxy/layers/http/hyper_h2_test_helpers.py +++ b/test/mitmproxy/proxy/layers/http/hyper_h2_test_helpers.py @@ -1,6 +1,5 @@ # This file has been copied from https://github.com/python-hyper/hyper-h2/blob/master/test/helpers.py, # MIT License - # -*- coding: utf-8 -*- """ helpers @@ -9,19 +8,17 @@ This module contains helpers for the h2 tests. """ from hpack.hpack import Encoder -from hyperframe.frame import ( - HeadersFrame, - DataFrame, - SettingsFrame, - WindowUpdateFrame, - PingFrame, - GoAwayFrame, - RstStreamFrame, - PushPromiseFrame, - PriorityFrame, - ContinuationFrame, - AltSvcFrame, -) +from hyperframe.frame import AltSvcFrame +from hyperframe.frame import ContinuationFrame +from hyperframe.frame import DataFrame +from hyperframe.frame import GoAwayFrame +from hyperframe.frame import HeadersFrame +from hyperframe.frame import PingFrame +from hyperframe.frame import PriorityFrame +from hyperframe.frame import PushPromiseFrame +from hyperframe.frame import RstStreamFrame +from hyperframe.frame import SettingsFrame +from hyperframe.frame import WindowUpdateFrame SAMPLE_SETTINGS = { SettingsFrame.HEADER_TABLE_SIZE: 4096, diff --git a/test/mitmproxy/proxy/layers/http/test_http.py b/test/mitmproxy/proxy/layers/http/test_http.py index 86ff6fc3a1..0d8e638df5 100644 --- a/test/mitmproxy/proxy/layers/http/test_http.py +++ b/test/mitmproxy/proxy/layers/http/test_http.py @@ -1,27 +1,34 @@ -from logging import WARNING - import gc +from logging import WARNING import pytest -from mitmproxy.connection import ConnectionState, Server -from mitmproxy.http import HTTPFlow, Response +from mitmproxy.connection import ConnectionState +from mitmproxy.connection import Server +from mitmproxy.http import HTTPFlow +from mitmproxy.http import Response from mitmproxy.proxy import layer -from mitmproxy.proxy.commands import CloseConnection, Log, OpenConnection, SendData -from mitmproxy.proxy.events import ConnectionClosed, DataReceived -from mitmproxy.proxy.layers import TCPLayer, http, tls +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import Log +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import SendData +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived +from mitmproxy.proxy.layers import http +from mitmproxy.proxy.layers import TCPLayer +from mitmproxy.proxy.layers import tls from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy.layers.tcp import TcpMessageInjected, TcpStartHook +from mitmproxy.proxy.layers.tcp import TcpMessageInjected +from mitmproxy.proxy.layers.tcp import TcpStartHook from mitmproxy.proxy.layers.websocket import WebsocketStartHook from mitmproxy.proxy.mode_specs import ProxyMode -from mitmproxy.tcp import TCPFlow, TCPMessage -from test.mitmproxy.proxy.tutils import ( - BytesMatching, - Placeholder, - Playbook, - reply, - reply_next_layer, -) +from mitmproxy.tcp import TCPFlow +from mitmproxy.tcp import TCPMessage +from test.mitmproxy.proxy.tutils import BytesMatching +from test.mitmproxy.proxy.tutils import Placeholder +from test.mitmproxy.proxy.tutils import Playbook +from test.mitmproxy.proxy.tutils import reply +from test.mitmproxy.proxy.tutils import reply_next_layer def test_http_proxy(tctx): @@ -744,7 +751,8 @@ def test_upstream_proxy(tctx, redirect, domain, scheme): << OpenConnection(server) >> reply(None) << SendData( - server, b"GET http://%s/ HTTP/1.1\r\nHost: %s\r\n\r\n" % (domain, domain), + server, + b"GET http://%s/ HTTP/1.1\r\nHost: %s\r\n\r\n" % (domain, domain), ) ) @@ -799,7 +807,8 @@ def test_upstream_proxy(tctx, redirect, domain, scheme): if redirect == "change-destination": playbook << SendData( server2, - b"GET http://%s.test/two HTTP/1.1\r\nHost: %s\r\n\r\n" % (domain, domain), + b"GET http://%s.test/two HTTP/1.1\r\nHost: %s\r\n\r\n" + % (domain, domain), ) else: playbook << SendData( @@ -808,7 +817,9 @@ def test_upstream_proxy(tctx, redirect, domain, scheme): ) else: if redirect == "change-destination": - playbook << SendData(server2, b"CONNECT %s.test:443 HTTP/1.1\r\n\r\n" % domain) + playbook << SendData( + server2, b"CONNECT %s.test:443 HTTP/1.1\r\n\r\n" % domain + ) playbook >> DataReceived( server2, b"HTTP/1.1 200 Connection established\r\n\r\n" ) @@ -830,9 +841,7 @@ def test_upstream_proxy(tctx, redirect, domain, scheme): assert flow().server_conn.address[0] == domain.decode("idna") if redirect == "change-proxy": - assert ( - server2().address == flow().server_conn.via[1] == ("other-proxy", 1234) - ) + assert server2().address == flow().server_conn.via[1] == ("other-proxy", 1234) else: assert server2().address == flow().server_conn.via[1] == ("proxy", 8080) @@ -958,8 +967,12 @@ def test_http_proxy_without_empty_chunk_in_head_request(tctx): << OpenConnection(server) >> reply(None) << SendData(server, b"HEAD / HTTP/1.1\r\n\r\n") - >> DataReceived(server, b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") - << SendData(tctx.client, b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") + >> DataReceived( + server, b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + ) + << SendData( + tctx.client, b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + ) ) @@ -1658,7 +1671,7 @@ def test_drop_stream_with_paused_events(tctx): << http.HttpRequestHeadersHook(flow) >> reply() << OpenConnection(server) - >> reply('Connection killed: error') + >> reply("Connection killed: error") << http.HttpErrorHook(flow) >> reply() << SendData(tctx.client, BytesMatching(b"502 Bad Gateway.+Connection killed")) diff --git a/test/mitmproxy/proxy/layers/http/test_http1.py b/test/mitmproxy/proxy/layers/http/test_http1.py index 05e84e1fc1..160493ad1b 100644 --- a/test/mitmproxy/proxy/layers/http/test_http1.py +++ b/test/mitmproxy/proxy/layers/http/test_http1.py @@ -3,18 +3,17 @@ from mitmproxy import http from mitmproxy.proxy.commands import SendData from mitmproxy.proxy.events import DataReceived -from mitmproxy.proxy.layers.http import ( - Http1Server, - ReceiveHttp, - RequestHeaders, - RequestEndOfMessage, - ResponseHeaders, - ResponseEndOfMessage, - RequestData, - Http1Client, - ResponseData, -) -from test.mitmproxy.proxy.tutils import Placeholder, Playbook +from mitmproxy.proxy.layers.http import Http1Client +from mitmproxy.proxy.layers.http import Http1Server +from mitmproxy.proxy.layers.http import ReceiveHttp +from mitmproxy.proxy.layers.http import RequestData +from mitmproxy.proxy.layers.http import RequestEndOfMessage +from mitmproxy.proxy.layers.http import RequestHeaders +from mitmproxy.proxy.layers.http import ResponseData +from mitmproxy.proxy.layers.http import ResponseEndOfMessage +from mitmproxy.proxy.layers.http import ResponseHeaders +from test.mitmproxy.proxy.tutils import Placeholder +from test.mitmproxy.proxy.tutils import Playbook class TestServer: diff --git a/test/mitmproxy/proxy/layers/http/test_http2.py b/test/mitmproxy/proxy/layers/http/test_http2.py index e28bf72f35..a6a5412d82 100644 --- a/test/mitmproxy/proxy/layers/http/test_http2.py +++ b/test/mitmproxy/proxy/layers/http/test_http2.py @@ -1,28 +1,34 @@ +import time + import h2.settings import hpack import hyperframe.frame import pytest -import time from h2.errors import ErrorCodes -from mitmproxy.connection import ConnectionState, Server +from mitmproxy.connection import ConnectionState +from mitmproxy.connection import Server from mitmproxy.flow import Error -from mitmproxy.http import HTTPFlow, Headers, Request +from mitmproxy.http import Headers +from mitmproxy.http import HTTPFlow +from mitmproxy.http import Request from mitmproxy.net.http import status_codes -from mitmproxy.proxy.commands import ( - CloseConnection, - Log, - OpenConnection, - SendData, - RequestWakeup, -) +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import Log +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import RequestWakeup +from mitmproxy.proxy.commands import SendData from mitmproxy.proxy.context import Context -from mitmproxy.proxy.events import ConnectionClosed, DataReceived +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived from mitmproxy.proxy.layers import http from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy.layers.http._http2 import Http2Client, split_pseudo_headers +from mitmproxy.proxy.layers.http._http2 import Http2Client +from mitmproxy.proxy.layers.http._http2 import split_pseudo_headers from test.mitmproxy.proxy.layers.http.hyper_h2_test_helpers import FrameFactory -from test.mitmproxy.proxy.tutils import Placeholder, Playbook, reply +from test.mitmproxy.proxy.tutils import Placeholder +from test.mitmproxy.proxy.tutils import Playbook +from test.mitmproxy.proxy.tutils import reply example_request_headers = ( (b":method", b"GET"), @@ -325,8 +331,7 @@ def test_long_response(tctx: Context, trailers): << http.HttpResponseHeadersHook(flow) >> reply() >> DataReceived( - server, - sff.build_data_frame(b"a" * 10000, flags=[]).serialize() + server, sff.build_data_frame(b"a" * 10000, flags=[]).serialize() ) >> DataReceived( server, @@ -373,9 +378,7 @@ def test_long_response(tctx: Context, trailers): playbook >> DataReceived( server, - sff.build_data_frame( - b'', flags=["END_STREAM"] - ).serialize(), + sff.build_data_frame(b"", flags=["END_STREAM"]).serialize(), ) ) ( @@ -412,10 +415,7 @@ def test_long_response(tctx: Context, trailers): tctx.client, cff.build_data_frame(b"a" * 1).serialize(), ) - << SendData( - tctx.client, - cff.build_data_frame(b"a" * 4464).serialize() - ) + << SendData(tctx.client, cff.build_data_frame(b"a" * 4464).serialize()) << SendData( tctx.client, cff.build_headers_frame( @@ -430,15 +430,10 @@ def test_long_response(tctx: Context, trailers): tctx.client, cff.build_data_frame(b"a" * 1).serialize(), ) + << SendData(tctx.client, cff.build_data_frame(b"a" * 4464).serialize()) << SendData( tctx.client, - cff.build_data_frame(b"a" * 4464).serialize() - ) - << SendData( - tctx.client, - cff.build_data_frame( - b"", flags=["END_STREAM"] - ).serialize(), + cff.build_data_frame(b"", flags=["END_STREAM"]).serialize(), ) ) assert flow().request.url == "http://example.com/" diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index eccdd7bf43..59ffb670a2 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -1,26 +1,33 @@ import collections.abc -from typing import Callable, Iterable, Optional -import pytest -import pylsqpack +from collections.abc import Iterable +from typing import Callable +from typing import Optional +import pylsqpack +import pytest from aioquic._buffer import Buffer -from aioquic.h3.connection import ( - ErrorCode, - FrameType, - Headers as H3Headers, - Setting, - StreamType, - encode_frame, - encode_uint_var, - encode_settings, - parse_settings, -) - -from mitmproxy import connection, version +from aioquic.h3.connection import encode_frame +from aioquic.h3.connection import encode_settings +from aioquic.h3.connection import encode_uint_var +from aioquic.h3.connection import ErrorCode +from aioquic.h3.connection import FrameType +from aioquic.h3.connection import Headers as H3Headers +from aioquic.h3.connection import parse_settings +from aioquic.h3.connection import Setting +from aioquic.h3.connection import StreamType + +from mitmproxy import connection +from mitmproxy import version from mitmproxy.flow import Error -from mitmproxy.http import Headers, HTTPFlow, Request -from mitmproxy.proxy import commands, context, events, layers -from mitmproxy.proxy.layers import http, quic +from mitmproxy.http import Headers +from mitmproxy.http import HTTPFlow +from mitmproxy.http import Request +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layers +from mitmproxy.proxy.layers import http +from mitmproxy.proxy.layers import quic from mitmproxy.proxy.layers.http._http3 import Http3Client from test.mitmproxy.proxy import tutils @@ -47,6 +54,7 @@ def decode_frame(frame_type: int, frame_data: bytes) -> bytes: class CallbackPlaceholder(tutils._Placeholder[bytes]): """Data placeholder that invokes a callback once its bytes get set.""" + def __init__(self, cb: Callable[[bytes], None]): super().__init__(bytes) self._cb = cb @@ -59,6 +67,7 @@ def setdefault(self, value: bytes) -> bytes: class DelayedPlaceholder(tutils._Placeholder[bytes]): """Data placeholder that resolves its bytes when needed.""" + def __init__(self, resolve: Callable[[], bytes]): super().__init__(bytes) self._resolve = resolve @@ -71,6 +80,7 @@ def __call__(self) -> bytes: class MultiPlaybook(tutils.Playbook): """Playbook that allows multiple events and commands to be registered at once.""" + def __lshift__(self, c): if isinstance(c, collections.abc.Iterable): for c_i in c: @@ -90,11 +100,8 @@ def __rshift__(self, e): class FrameFactory: """Helper class for generating QUIC stream events and commands.""" - def __init__( - self, - conn: connection.Connection, - is_client: bool - ) -> None: + + def __init__(self, conn: connection.Connection, is_client: bool) -> None: self.conn = conn self.is_client = is_client self.decoder = pylsqpack.Decoder( @@ -108,11 +115,7 @@ def __init__( self.local_stream_id: dict[StreamType, int] = {} self.max_push_id: Optional[int] = None - def get_default_stream_id( - self, - stream_type: StreamType, - for_local: bool - ) -> int: + def get_default_stream_id(self, stream_type: StreamType, for_local: bool) -> int: if stream_type == StreamType.CONTROL: stream_id = 2 elif stream_type == StreamType.QPACK_ENCODER: @@ -132,9 +135,7 @@ def send_stream_type( ) -> quic.SendQuicStreamData: assert stream_type not in self.peer_stream_id if stream_id is None: - stream_id = self.get_default_stream_id( - stream_type, for_local=False - ) + stream_id = self.get_default_stream_id(stream_type, for_local=False) self.peer_stream_id[stream_type] = stream_id return quic.SendQuicStreamData( connection=self.conn, @@ -150,9 +151,7 @@ def receive_stream_type( ) -> quic.QuicStreamDataReceived: assert stream_type not in self.local_stream_id if stream_id is None: - stream_id = self.get_default_stream_id( - stream_type, for_local=True - ) + stream_id = self.get_default_stream_id(stream_type, for_local=True) self.local_stream_id[stream_type] = stream_id return quic.QuicStreamDataReceived( connection=self.conn, @@ -185,10 +184,12 @@ def cb(data: bytes) -> None: buf = Buffer(data=data) assert buf.pull_uint_var() == FrameType.SETTINGS settings = parse_settings(buf.pull_bytes(buf.pull_uint_var())) - placeholder.setdefault(self.encoder.apply_settings( - max_table_capacity=settings[Setting.QPACK_MAX_TABLE_CAPACITY], - blocked_streams=settings[Setting.QPACK_BLOCKED_STREAMS], - )) + placeholder.setdefault( + self.encoder.apply_settings( + max_table_capacity=settings[Setting.QPACK_MAX_TABLE_CAPACITY], + blocked_streams=settings[Setting.QPACK_BLOCKED_STREAMS], + ) + ) return quic.SendQuicStreamData( connection=self.conn, @@ -368,10 +369,7 @@ def receive_init(self) -> Iterable[quic.QuicStreamDataReceived]: @property def is_done(self) -> bool: - return ( - self.encoder_placeholder is None - and not self.decoder_placeholders - ) + return self.encoder_placeholder is None and not self.decoder_placeholders @pytest.fixture @@ -428,11 +426,14 @@ def test_invalid_header(tctx: context.Context): playbook, cff = start_h3_client(tctx) assert ( playbook - >> cff.receive_headers([ - (b":method", b"CONNECT"), - (b":path", b"/"), - (b":authority", b"example.com"), - ], end_stream=True) + >> cff.receive_headers( + [ + (b":method", b"CONNECT"), + (b":path", b"/"), + (b":authority", b"example.com"), + ], + end_stream=True, + ) << cff.send_decoder() # for receive_headers << quic.CloseQuicConnection( tctx.client, @@ -441,11 +442,14 @@ def test_invalid_header(tctx: context.Context): reason_phrase="Invalid HTTP/3 request headers: Required pseudo header is missing: b':scheme'", ) # ensure that once we close, we don't process messages anymore - >> cff.receive_headers([ - (b":method", b"CONNECT"), - (b":path", b"/"), - (b":authority", b"example.com"), - ], end_stream=True) + >> cff.receive_headers( + [ + (b":method", b"CONNECT"), + (b":path", b"/"), + (b":authority", b"example.com"), + ], + end_stream=True, + ) ) @@ -621,10 +625,7 @@ def enable_streaming(flow: HTTPFlow): >> tutils.reply(to=request) << sff.send_headers(example_request_trailers, end_stream=True) ) - assert ( - playbook - >> sff.receive_decoder() # for send_headers - ) + assert playbook >> sff.receive_decoder() # for send_headers assert cff.is_done and sff.is_done @@ -648,11 +649,13 @@ def test_upstream_error(tctx: context.Context): >> tutils.reply("oops server <> error") << http.HttpErrorHook(flow) >> tutils.reply() - << cff.send_headers([ - (b":status", b"502"), - (b'server', version.MITMPROXY.encode()), - (b'content-type', b'text/html'), - ]) + << cff.send_headers( + [ + (b":status", b"502"), + (b"server", version.MITMPROXY.encode()), + (b"content-type", b"text/html"), + ] + ) << quic.SendQuicStreamData( tctx.client, stream_id=0, @@ -670,9 +673,7 @@ def test_upstream_error(tctx: context.Context): @pytest.mark.parametrize("stream", ["stream", ""]) @pytest.mark.parametrize("when", ["request", "response"]) @pytest.mark.parametrize("how", ["RST", "disconnect", "RST+disconnect"]) -def test_http3_client_aborts( - tctx: context.Context, stream: str, when: str, how: str -): +def test_http3_client_aborts(tctx: context.Context, stream: str, when: str, how: str): """ Test handling of the case where a client aborts during request or response transmission. @@ -698,12 +699,12 @@ def enable_response_streaming(flow: HTTPFlow): if stream and when == "request": assert ( playbook - >> tutils.reply( - side_effect=enable_request_streaming, to=request_headers - ) + >> tutils.reply(side_effect=enable_request_streaming, to=request_headers) << commands.OpenConnection(server) >> tutils.reply(None) - << commands.SendData(server, b"GET / HTTP/1.1\r\n" b"Host: example.com\r\n\r\n") + << commands.SendData( + server, b"GET / HTTP/1.1\r\n" b"Host: example.com\r\n\r\n" + ) ) else: assert playbook >> tutils.reply(to=request_headers) @@ -716,7 +717,7 @@ def enable_response_streaming(flow: HTTPFlow): tctx.client, error_code=ErrorCode.H3_REQUEST_CANCELLED, frame_type=None, - reason_phrase="peer closed connection" + reason_phrase="peer closed connection", ) if stream: @@ -729,7 +730,7 @@ def enable_response_streaming(flow: HTTPFlow): tctx.client, error_code=ErrorCode.H3_NO_ERROR, frame_type=None, - reason_phrase="peer closed connection" + reason_phrase="peer closed connection", ) assert playbook assert ( @@ -746,17 +747,21 @@ def enable_response_streaming(flow: HTTPFlow): << commands.OpenConnection(server) >> tutils.reply(None) << commands.SendData(server, b"GET / HTTP/1.1\r\n" b"Host: example.com\r\n\r\n") - >> events.DataReceived(server, b"HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\n123") + >> events.DataReceived( + server, b"HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\n123" + ) << http.HttpResponseHeadersHook(flow) ) if stream: assert ( playbook >> tutils.reply(side_effect=enable_response_streaming) - << cff.send_headers([ - (b":status", b"200"), - (b"content-length", b"6"), - ]) + << cff.send_headers( + [ + (b":status", b"200"), + (b"content-length", b"6"), + ] + ) << cff.send_data(b"123") ) else: @@ -769,7 +774,7 @@ def enable_response_streaming(flow: HTTPFlow): tctx.client, error_code=ErrorCode.H3_REQUEST_CANCELLED, frame_type=None, - reason_phrase="peer closed connection" + reason_phrase="peer closed connection", ) playbook << commands.CloseConnection(server) @@ -782,7 +787,7 @@ def enable_response_streaming(flow: HTTPFlow): tctx.client, error_code=ErrorCode.H3_REQUEST_CANCELLED, frame_type=None, - reason_phrase="peer closed connection" + reason_phrase="peer closed connection", ) assert playbook @@ -920,14 +925,10 @@ def test_stream_concurrency(tctx: context.Context): assert ( playbook # request client - >> cff.receive_headers( - headers1, stream_id=0, end_stream=True - ) + >> cff.receive_headers(headers1, stream_id=0, end_stream=True) << (request_header1 := http.HttpRequestHeadersHook(flow1)) << cff.send_decoder() # for receive_headers - >> cff.receive_headers( - headers2, stream_id=4, end_stream=True - ) + >> cff.receive_headers(headers2, stream_id=4, end_stream=True) << (request_header2 := http.HttpRequestHeadersHook(flow2)) << cff.send_decoder() # for receive_headers >> tutils.reply(to=request_header1) @@ -940,17 +941,13 @@ def test_stream_concurrency(tctx: context.Context): << commands.OpenConnection(server) >> tutils.reply(None, side_effect=make_h3) << sff.send_init() - << sff.send_headers( - headers2, stream_id=0, end_stream=True - ) + << sff.send_headers(headers2, stream_id=0, end_stream=True) >> sff.receive_init() << sff.send_encoder() >> sff.receive_encoder() >> sff.receive_decoder() # for send_headers >> tutils.reply(to=request1) - << sff.send_headers( - headers1, stream_id=4, end_stream=True - ) + << sff.send_headers(headers1, stream_id=4, end_stream=True) >> sff.receive_decoder() # for send_headers ) assert cff.is_done and sff.is_done @@ -964,23 +961,15 @@ def test_stream_concurrent_get_connection(tctx: context.Context): sff = FrameFactory(server, is_client=False) assert ( playbook - >> cff.receive_headers( - example_request_headers, stream_id=0, end_stream=True - ) + >> cff.receive_headers(example_request_headers, stream_id=0, end_stream=True) << cff.send_decoder() # for receive_headers << (o := commands.OpenConnection(server)) - >> cff.receive_headers( - example_request_headers, stream_id=4, end_stream=True - ) + >> cff.receive_headers(example_request_headers, stream_id=4, end_stream=True) << cff.send_decoder() # for receive_headers >> tutils.reply(None, to=o, side_effect=make_h3) << sff.send_init() - << sff.send_headers( - example_request_headers, stream_id=0, end_stream=True - ) - << sff.send_headers( - example_request_headers, stream_id=4, end_stream=True - ) + << sff.send_headers(example_request_headers, stream_id=0, end_stream=True) + << sff.send_headers(example_request_headers, stream_id=4, end_stream=True) >> sff.receive_init() << sff.send_encoder() >> sff.receive_encoder() @@ -1007,14 +996,10 @@ def kill(flow: HTTPFlow): assert ( playbook # request client - >> cff.receive_headers( - headers1, stream_id=0, end_stream=True - ) + >> cff.receive_headers(headers1, stream_id=0, end_stream=True) << (request_header1 := http.HttpRequestHeadersHook(flow1)) << cff.send_decoder() # for receive_headers - >> cff.receive_headers( - headers2, stream_id=4, end_stream=True - ) + >> cff.receive_headers(headers2, stream_id=4, end_stream=True) << (request_header2 := http.HttpRequestHeadersHook(flow2)) << cff.send_decoder() # for receive_headers >> tutils.reply(to=request_header2, side_effect=kill) @@ -1028,9 +1013,7 @@ def kill(flow: HTTPFlow): << commands.OpenConnection(server) >> tutils.reply(None, side_effect=make_h3) << sff.send_init() - << sff.send_headers( - headers1, stream_id=0, end_stream=True - ) + << sff.send_headers(headers1, stream_id=0, end_stream=True) >> sff.receive_init() << sff.send_encoder() >> sff.receive_encoder() @@ -1052,12 +1035,15 @@ def test_no_data_on_closed_stream(self, tctx: context.Context): << frame_factory.send_encoder() >> frame_factory.receive_encoder() >> http.RequestHeaders(1, req, end_stream=True) - << frame_factory.send_headers([ - (b":method", b"GET"), - (b':scheme', b'http'), - (b':path', b'/'), - (b'content-length', b'0'), - ], end_stream=True) + << frame_factory.send_headers( + [ + (b":method", b"GET"), + (b":scheme", b"http"), + (b":path", b"/"), + (b"content-length", b"0"), + ], + end_stream=True, + ) >> frame_factory.receive_decoder() # for send_headers >> http.RequestEndOfMessage(1) >> frame_factory.receive_headers(resp) @@ -1099,12 +1085,15 @@ def test_ignore_wrong_order(self, tctx: context.Context): "DATA frame is not allowed in this state" ) >> http.RequestHeaders(1, req, end_stream=False) - << frame_factory.send_headers([ - (b":method", b"GET"), - (b':scheme', b'http'), - (b':path', b'/'), - (b'content-length', b'0'), - ], end_stream=False) + << frame_factory.send_headers( + [ + (b":method", b"GET"), + (b":scheme", b"http"), + (b":path", b"/"), + (b"content-length", b"0"), + ], + end_stream=False, + ) >> frame_factory.receive_decoder() # for send_headers >> http.RequestHeaders(1, req, end_stream=False) << commands.Log( diff --git a/test/mitmproxy/proxy/layers/http/test_http_fuzz.py b/test/mitmproxy/proxy/layers/http/test_http_fuzz.py index 67478a02fa..8e139e0c28 100644 --- a/test/mitmproxy/proxy/layers/http/test_http_fuzz.py +++ b/test/mitmproxy/proxy/layers/http/test_http_fuzz.py @@ -2,44 +2,44 @@ import pytest from h2.settings import SettingCodes -from hypothesis import example, given -from hypothesis.strategies import ( - binary, - booleans, - composite, - dictionaries, - integers, - lists, - sampled_from, - sets, - text, - data, -) - -from mitmproxy import options, connection +from hypothesis import example +from hypothesis import given +from hypothesis.strategies import binary +from hypothesis.strategies import booleans +from hypothesis.strategies import composite +from hypothesis.strategies import data +from hypothesis.strategies import dictionaries +from hypothesis.strategies import integers +from hypothesis.strategies import lists +from hypothesis.strategies import sampled_from +from hypothesis.strategies import sets +from hypothesis.strategies import text + +from mitmproxy import connection +from mitmproxy import options from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.connection import Server from mitmproxy.http import HTTPFlow -from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy import context, events -from mitmproxy.proxy.commands import OpenConnection, SendData -from mitmproxy.proxy.events import DataReceived, Start, ConnectionClosed +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import SendData +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived +from mitmproxy.proxy.events import Start from mitmproxy.proxy.layers import http -from test.mitmproxy.proxy.layers.http.hyper_h2_test_helpers import FrameFactory -from test.mitmproxy.proxy.layers.http.test_http2 import ( - make_h2, - example_response_headers, - example_request_headers, - start_h2_client, -) -from test.mitmproxy.proxy.tutils import ( - Placeholder, - Playbook, - reply, - _TracebackInPlaybook, - _eq, -) from mitmproxy.proxy.layers.http import _http2 +from mitmproxy.proxy.layers.http import HTTPMode +from test.mitmproxy.proxy.layers.http.hyper_h2_test_helpers import FrameFactory +from test.mitmproxy.proxy.layers.http.test_http2 import example_request_headers +from test.mitmproxy.proxy.layers.http.test_http2 import example_response_headers +from test.mitmproxy.proxy.layers.http.test_http2 import make_h2 +from test.mitmproxy.proxy.layers.http.test_http2 import start_h2_client +from test.mitmproxy.proxy.tutils import _eq +from test.mitmproxy.proxy.tutils import _TracebackInPlaybook +from test.mitmproxy.proxy.tutils import Placeholder +from test.mitmproxy.proxy.tutils import Playbook +from test.mitmproxy.proxy.tutils import reply opts = options.Options() Proxyserver().load(opts) @@ -217,7 +217,7 @@ def h2_frames(draw): settings=draw( dictionaries( keys=sampled_from(SettingCodes), - values=integers(0, 2 ** 32 - 1), + values=integers(0, 2**32 - 1), max_size=5, ) ), @@ -244,7 +244,7 @@ def h2_frames(draw): draw(binary()), draw(h2_flags), stream_id=draw(h2_stream_ids_nonzero) ) window_update = ff.build_window_update_frame( - draw(h2_stream_ids), draw(integers(0, 2 ** 32 - 1)) + draw(h2_stream_ids), draw(integers(0, 2**32 - 1)) ) frames = draw( @@ -318,8 +318,12 @@ def test_fuzz_h2_request_mutations(chunks): def _tctx() -> context.Context: return context.Context( - connection.Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329), - opts + connection.Client( + peername=("client", 1234), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, + ), + opts, ) diff --git a/test/mitmproxy/proxy/layers/http/test_http_version_interop.py b/test/mitmproxy/proxy/layers/http/test_http_version_interop.py index d0ae84248a..599793e32d 100644 --- a/test/mitmproxy/proxy/layers/http/test_http_version_interop.py +++ b/test/mitmproxy/proxy/layers/http/test_http_version_interop.py @@ -2,19 +2,21 @@ import h2.connection import h2.events +from mitmproxy.connection import Server from mitmproxy.http import HTTPFlow +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import SendData from mitmproxy.proxy.context import Context -from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData -from mitmproxy.connection import Server from mitmproxy.proxy.events import DataReceived from mitmproxy.proxy.layers import http +from mitmproxy.proxy.layers.http import HTTPMode from test.mitmproxy.proxy.layers.http.hyper_h2_test_helpers import FrameFactory -from test.mitmproxy.proxy.layers.http.test_http2 import ( - example_response_headers, - make_h2, -) -from test.mitmproxy.proxy.tutils import Placeholder, Playbook, reply +from test.mitmproxy.proxy.layers.http.test_http2 import example_response_headers +from test.mitmproxy.proxy.layers.http.test_http2 import make_h2 +from test.mitmproxy.proxy.tutils import Placeholder +from test.mitmproxy.proxy.tutils import Playbook +from test.mitmproxy.proxy.tutils import reply example_request_headers = ( (b":method", b"GET"), @@ -77,7 +79,9 @@ def test_h2_to_h1(tctx): >> reply() << OpenConnection(server) >> reply(None) - << SendData(server, b"GET / HTTP/1.1\r\nHost: example.com\r\ncookie: a=1; b=2\r\n\r\n") + << SendData( + server, b"GET / HTTP/1.1\r\nHost: example.com\r\ncookie: a=1; b=2\r\n\r\n" + ) >> DataReceived(server, b"HTTP/1.1 200 OK\r\nContent-Length: 12\r\n\r\n") << http.HttpResponseHeadersHook(flow) >> reply() diff --git a/test/mitmproxy/proxy/layers/test_dns.py b/test/mitmproxy/proxy/layers/test_dns.py index ab520bb625..133ae20b3c 100644 --- a/test/mitmproxy/proxy/layers/test_dns.py +++ b/test/mitmproxy/proxy/layers/test_dns.py @@ -1,11 +1,18 @@ import time -from mitmproxy.proxy.commands import CloseConnection, Log, OpenConnection, SendData -from mitmproxy.proxy.events import ConnectionClosed, DataReceived -from mitmproxy.proxy.layers import dns +from ..tutils import Placeholder +from ..tutils import Playbook +from ..tutils import reply from mitmproxy.dns import DNSFlow -from mitmproxy.test.tutils import tdnsreq, tdnsresp -from ..tutils import Placeholder, Playbook, reply +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import Log +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import SendData +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived +from mitmproxy.proxy.layers import dns +from mitmproxy.test.tutils import tdnsreq +from mitmproxy.test.tutils import tdnsresp def test_invalid_and_dummy_end(tctx): diff --git a/test/mitmproxy/proxy/layers/test_modes.py b/test/mitmproxy/proxy/layers/test_modes.py index 96ca863f5a..6d8040e3b0 100644 --- a/test/mitmproxy/proxy/layers/test_modes.py +++ b/test/mitmproxy/proxy/layers/test_modes.py @@ -1,38 +1,46 @@ import copy import pytest -from mitmproxy import dns +from mitmproxy import dns from mitmproxy.addons.proxyauth import ProxyAuth -from mitmproxy.connection import Client, ConnectionState, Server +from mitmproxy.connection import Client +from mitmproxy.connection import ConnectionState +from mitmproxy.connection import Server from mitmproxy.proxy import layers -from mitmproxy.proxy.commands import ( - CloseConnection, - Log, - OpenConnection, - RequestWakeup, - SendData, -) +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import Log +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import RequestWakeup +from mitmproxy.proxy.commands import SendData from mitmproxy.proxy.context import Context -from mitmproxy.proxy.events import ConnectionClosed, DataReceived -from mitmproxy.proxy.layer import NextLayer, NextLayerHook -from mitmproxy.proxy.layers import http, modes, quic, tcp, tls, udp +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived +from mitmproxy.proxy.layer import NextLayer +from mitmproxy.proxy.layer import NextLayerHook +from mitmproxy.proxy.layers import http +from mitmproxy.proxy.layers import modes +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tcp +from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import udp from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy.layers.tcp import TcpMessageHook, TcpStartHook -from mitmproxy.proxy.layers.tls import ( - ClientTLSLayer, - TlsStartClientHook, - TlsStartServerHook, -) +from mitmproxy.proxy.layers.tcp import TcpMessageHook +from mitmproxy.proxy.layers.tcp import TcpStartHook +from mitmproxy.proxy.layers.tls import ClientTLSLayer +from mitmproxy.proxy.layers.tls import TlsStartClientHook +from mitmproxy.proxy.layers.tls import TlsStartServerHook from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.tcp import TCPFlow -from mitmproxy.test import taddons, tflow +from mitmproxy.test import taddons +from mitmproxy.test import tflow from mitmproxy.udp import UDPFlow -from test.mitmproxy.proxy.layers.test_tls import ( - reply_tls_start_client, - reply_tls_start_server, -) -from test.mitmproxy.proxy.tutils import Placeholder, Playbook, reply, reply_next_layer +from test.mitmproxy.proxy.layers.test_tls import reply_tls_start_client +from test.mitmproxy.proxy.layers.test_tls import reply_tls_start_server +from test.mitmproxy.proxy.tutils import Placeholder +from test.mitmproxy.proxy.tutils import Playbook +from test.mitmproxy.proxy.tutils import reply +from test.mitmproxy.proxy.tutils import reply_next_layer def test_upstream_https(tctx): @@ -45,12 +53,24 @@ def test_upstream_https(tctx): curl -x localhost:8080 -k http://example.com """ tctx1 = Context( - Client(peername=("client", 1234), sockname=("127.0.0.1", 8080), timestamp_start=1605699329, state=ConnectionState.OPEN), + Client( + peername=("client", 1234), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, + state=ConnectionState.OPEN, + ), copy.deepcopy(tctx.options), ) - tctx1.client.proxy_mode = ProxyMode.parse("upstream:https://example.mitmproxy.org:8081") + tctx1.client.proxy_mode = ProxyMode.parse( + "upstream:https://example.mitmproxy.org:8081" + ) tctx2 = Context( - Client(peername=("client", 4321), sockname=("127.0.0.1", 8080), timestamp_start=1605699329, state=ConnectionState.OPEN), + Client( + peername=("client", 4321), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, + state=ConnectionState.OPEN, + ), copy.deepcopy(tctx.options), ) assert tctx2.client.proxy_mode == ProxyMode.parse("regular") diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index 06da7725cf..d635a3385d 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -1,28 +1,42 @@ -from logging import DEBUG, ERROR, WARNING import ssl import time +from logging import DEBUG +from logging import ERROR +from logging import WARNING +from typing import Literal +from typing import Optional +from typing import TypeVar +from unittest.mock import MagicMock + +import pytest from aioquic.buffer import Buffer as QuicBuffer from aioquic.quic import events as quic_events from aioquic.quic.configuration import QuicConfiguration -from aioquic.quic.connection import QuicConnection, pull_quic_header -from typing import Literal, Optional, TypeVar -from unittest.mock import MagicMock -import pytest +from aioquic.quic.connection import pull_quic_header +from aioquic.quic.connection import QuicConnection + from mitmproxy import connection -from mitmproxy.proxy import commands, context, events, layer, tunnel +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy import layers -from mitmproxy.proxy.layers import quic, tcp, tls, udp -from mitmproxy.udp import UDPFlow, UDPMessage +from mitmproxy.proxy import tunnel +from mitmproxy.proxy.layers import quic +from mitmproxy.proxy.layers import tcp +from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import udp +from mitmproxy.tcp import TCPFlow +from mitmproxy.udp import UDPFlow +from mitmproxy.udp import UDPMessage from mitmproxy.utils import data from test.mitmproxy.proxy import tutils -from mitmproxy.tcp import TCPFlow - tlsdata = data.Data(__name__) -T = TypeVar('T', bound=layer.Layer) +T = TypeVar("T", bound=layer.Layer) class DummyLayer(layer.Layer): @@ -44,26 +58,44 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: yield commands.SendData( event.connection, f"open-connection failed: {err}".encode() ) - elif isinstance(event, events.DataReceived) and event.data == b"close-connection": + elif ( + isinstance(event, events.DataReceived) and event.data == b"close-connection" + ): yield commands.CloseConnection(event.connection) - elif isinstance(event, events.DataReceived) and event.data == b"close-connection-error": + elif ( + isinstance(event, events.DataReceived) + and event.data == b"close-connection-error" + ): yield quic.CloseQuicConnection(event.connection, 123, None, "error") elif isinstance(event, events.DataReceived) and event.data == b"stop-stream": yield quic.StopQuicStream(event.connection, 24, 123) - elif isinstance(event, events.DataReceived) and event.data == b"invalid-command": + elif ( + isinstance(event, events.DataReceived) and event.data == b"invalid-command" + ): + class InvalidConnectionCommand(commands.ConnectionCommand): pass + yield InvalidConnectionCommand(event.connection) - elif isinstance(event, events.DataReceived) and event.data == b"invalid-stream-command": + elif ( + isinstance(event, events.DataReceived) + and event.data == b"invalid-stream-command" + ): + class InvalidStreamCommand(quic.QuicStreamCommand): pass + yield InvalidStreamCommand(event.connection, 42) elif isinstance(event, quic.QuicConnectionClosed): self.closed = event elif isinstance(event, quic.QuicStreamDataReceived): - yield quic.SendQuicStreamData(event.connection, event.stream_id, event.data, event.end_stream) + yield quic.SendQuicStreamData( + event.connection, event.stream_id, event.data, event.end_stream + ) elif isinstance(event, quic.QuicStreamReset): - yield quic.ResetQuicStream(event.connection, event.stream_id, event.error_code) + yield quic.ResetQuicStream( + event.connection, event.stream_id, event.error_code + ) else: yield from super()._handle_event(event) @@ -104,7 +136,7 @@ class InvalidStreamCommand(quic.QuicStreamCommand): def test_error_code_to_str(): assert quic.error_code_to_str(0x6) == "FINAL_SIZE_ERROR" assert quic.error_code_to_str(0x104) == "H3_CLOSED_CRITICAL_STREAM" - assert quic.error_code_to_str(0xdead) == f"unknown error (0xdead)" + assert quic.error_code_to_str(0xDEAD) == f"unknown error (0xdead)" def test_is_success_error_code(): @@ -112,7 +144,7 @@ def test_is_success_error_code(): assert not quic.is_success_error_code(0x6) assert quic.is_success_error_code(0x100) assert not quic.is_success_error_code(0x104) - assert not quic.is_success_error_code(0xdead) + assert not quic.is_success_error_code(0xDEAD) @pytest.mark.parametrize("value", ["s1 s2\n", "s1 s2"]) @@ -133,7 +165,7 @@ def test_input(self): ) with pytest.raises(ValueError, match="not initial"): quic.quic_parse_client_hello( - b'\\s\xd8\xd8\xa5dT\x8bc\xd3\xae\x1c\xb2\x8a7-\x1d\x19j\x85\xb0~\x8c\x80\xa5\x8cY\xac\x0ecK\x7fC2f\xbcm\x1b\xac~' + b"\\s\xd8\xd8\xa5dT\x8bc\xd3\xae\x1c\xb2\x8a7-\x1d\x19j\x85\xb0~\x8c\x80\xa5\x8cY\xac\x0ecK\x7fC2f\xbcm\x1b\xac~" ) def test_invalid(self, monkeypatch): @@ -156,7 +188,9 @@ def raise_conn_err(self, data, addr, now): def test_no_return(self): with pytest.raises(ValueError, match="No ClientHello"): - quic.quic_parse_client_hello(client_hello[0:1200] + b'\x00' + client_hello[1200:]) + quic.quic_parse_client_hello( + client_hello[0:1200] + b"\x00" + client_hello[1200:] + ) class TestQuicStreamLayer: @@ -247,13 +281,17 @@ def test_msg_inject(self, tctx: context.Context): << udp.UdpMessageHook(udpflow) >> tutils.reply() << commands.SendData(tctx.server, b"msg2") - >> udp.UdpMessageInjected(UDPFlow(("other", 80), tctx.server), UDPMessage(True, b"msg3")) + >> udp.UdpMessageInjected( + UDPFlow(("other", 80), tctx.server), UDPMessage(True, b"msg3") + ) << udp.UdpMessageHook(udpflow) >> tutils.reply() << commands.SendData(tctx.server, b"msg3") ) with pytest.raises(AssertionError, match="not associated"): - playbook >> udp.UdpMessageInjected(UDPFlow(("notfound", 0), ("noexist", 0)), UDPMessage(True, b"msg2")) + playbook >> udp.UdpMessageInjected( + UDPFlow(("notfound", 0), ("noexist", 0)), UDPMessage(True, b"msg2") + ) assert playbook def test_reset_with_end_hook(self, tctx: context.Context): @@ -316,8 +354,10 @@ def test_invalid_stream_event(self, tctx: context.Context): >> tutils.reply(None) ) with pytest.raises(AssertionError, match="Unexpected stream event"): + class InvalidStreamEvent(quic.QuicStreamEvent): pass + playbook >> InvalidStreamEvent(tctx.client, 0) assert playbook @@ -329,8 +369,10 @@ def test_invalid_event(self, tctx: context.Context): >> tutils.reply(None) ) with pytest.raises(AssertionError, match="Unexpected event"): + class InvalidEvent(events.Event): pass + playbook >> InvalidEvent() assert playbook @@ -359,12 +401,16 @@ def echo_new_server(ctx: context.Context): tutils.Playbook(quic.RawQuicLayer(tctx)) << commands.OpenConnection(tctx.server) >> tutils.reply(None) - >> quic.QuicStreamDataReceived(tctx.client, 0, b"open-connection", end_stream=False) + >> quic.QuicStreamDataReceived( + tctx.client, 0, b"open-connection", end_stream=False + ) << layer.NextLayerHook(tutils.Placeholder()) >> tutils.reply_next_layer(echo_new_server) << commands.OpenConnection(server) >> tutils.reply("uhoh") - << quic.SendQuicStreamData(tctx.client, 0, b"open-connection failed: uhoh", end_stream=False) + << quic.SendQuicStreamData( + tctx.client, 0, b"open-connection failed: uhoh", end_stream=False + ) ) def test_invalid_connection_command(self, tctx: context.Context): @@ -378,8 +424,12 @@ def test_invalid_connection_command(self, tctx: context.Context): >> tutils.reply_next_layer(TlsEchoLayer) << quic.SendQuicStreamData(tctx.client, 0, b"msg1", end_stream=False) ) - with pytest.raises(AssertionError, match="Unexpected stream connection command"): - playbook >> quic.QuicStreamDataReceived(tctx.client, 0, b"invalid-command", end_stream=False) + with pytest.raises( + AssertionError, match="Unexpected stream connection command" + ): + playbook >> quic.QuicStreamDataReceived( + tctx.client, 0, b"invalid-command", end_stream=False + ) assert playbook @@ -412,8 +462,8 @@ def make_mock_quic( quic_layer.quic = mock quic_layer.tunnel_state = ( tls.tunnel.TunnelState.OPEN - if established else - tls.tunnel.TunnelState.ESTABLISHING + if established + else tls.tunnel.TunnelState.ESTABLISHING ) return tutils.Playbook(quic_layer), mock @@ -423,21 +473,19 @@ class TestQuicLayer: def test_invalid_event(self, tctx: context.Context, established: bool): class InvalidEvent(quic_events.QuicEvent): pass + playbook, conn = make_mock_quic( tctx, event=InvalidEvent(), established=established ) with pytest.raises(AssertionError, match="Unexpected event"): - assert ( - playbook - >> events.DataReceived(tctx.client, b"") - ) + assert playbook >> events.DataReceived(tctx.client, b"") def test_invalid_stream_command(self, tctx: context.Context): playbook, conn = make_mock_quic( tctx, quic_events.DatagramFrameReceived(b"invalid-stream-command") ) with pytest.raises(AssertionError, match="Unexpected stream command"): - assert (playbook >> events.DataReceived(tctx.client, b"")) + assert playbook >> events.DataReceived(tctx.client, b"") def test_close(self, tctx: context.Context): playbook, conn = make_mock_quic( @@ -470,7 +518,7 @@ def test_datagram(self, tctx: context.Context): tctx, quic_events.DatagramFrameReceived(b"packet") ) assert not conn._datagrams_pending - assert (playbook >> events.DataReceived(tctx.client, b"")) + assert playbook >> events.DataReceived(tctx.client, b"") assert len(conn._datagrams_pending) == 1 assert conn._datagrams_pending[0] == b"packet" @@ -479,15 +527,13 @@ def test_stream_data(self, tctx: context.Context): tctx, quic_events.StreamDataReceived(b"packet", False, 42) ) assert 42 not in conn._streams - assert (playbook >> events.DataReceived(tctx.client, b"")) + assert playbook >> events.DataReceived(tctx.client, b"") assert b"packet" == conn._streams[42].sender._buffer def test_stream_reset(self, tctx: context.Context): - playbook, conn = make_mock_quic( - tctx, quic_events.StreamReset(123, 42) - ) + playbook, conn = make_mock_quic(tctx, quic_events.StreamReset(123, 42)) assert 42 not in conn._streams - assert (playbook >> events.DataReceived(tctx.client, b"")) + assert playbook >> events.DataReceived(tctx.client, b"") assert conn._streams[42].sender.reset_pending assert conn._streams[42].sender._reset_error_code == 123 @@ -497,7 +543,7 @@ def test_stream_stop(self, tctx: context.Context): ) assert 24 not in conn._streams conn._get_or_create_stream_for_send(24) - assert (playbook >> events.DataReceived(tctx.client, b"")) + assert playbook >> events.DataReceived(tctx.client, b"") assert conn._streams[24].receiver.stop_pending assert conn._streams[24].receiver._stop_error_code == 123 @@ -521,7 +567,9 @@ def __init__( self.ctx.verify_mode = ssl.CERT_OPTIONAL self.ctx.load_verify_locations( - cafile=tlsdata.path("../../net/data/verificationcerts/trusted-root.crt"), + cafile=tlsdata.path( + "../../net/data/verificationcerts/trusted-root.crt" + ), ) if alpn: @@ -613,7 +661,7 @@ def finish_handshake( playbook: tutils.Playbook, conn: connection.Connection, tssl: SSLTest, - child_layer: type[T] + child_layer: type[T], ) -> T: result: Optional[T] = None @@ -644,9 +692,7 @@ def set_layer(next_layer: layer.NextLayer) -> None: return result -def reply_tls_start_client( - alpn: Optional[str] = None, *args, **kwargs -) -> tutils.reply: +def reply_tls_start_client(alpn: Optional[str] = None, *args, **kwargs) -> tutils.reply: """ Helper function to simplify the syntax for quic_start_client hooks. """ @@ -658,9 +704,9 @@ def make_client_conn(tls_start: quic.QuicTlsData) -> None: tlsdata.path("../../net/data/verificationcerts/trusted-leaf.key"), ) tls_start.settings = quic.QuicTlsSettings( - certificate = config.certificate, - certificate_chain = config.certificate_chain, - certificate_private_key = config.private_key, + certificate=config.certificate, + certificate_chain=config.certificate_chain, + certificate_private_key=config.private_key, ) if alpn is not None: tls_start.settings.alpn_protocols = [alpn] @@ -668,9 +714,7 @@ def make_client_conn(tls_start: quic.QuicTlsData) -> None: return tutils.reply(*args, side_effect=make_client_conn, **kwargs) -def reply_tls_start_server( - alpn: Optional[str] = None, *args, **kwargs -) -> tutils.reply: +def reply_tls_start_server(alpn: Optional[str] = None, *args, **kwargs) -> tutils.reply: """ Helper function to simplify the syntax for quic_start_server hooks. """ @@ -799,7 +843,7 @@ def test_untrusted_cert(self, tctx: context.Context): >> events.Wakeup(playbook.actual[9]) << commands.Log( "Server QUIC handshake failed. hostname 'wrong.host.mitmproxy.org' doesn't match 'example.mitmproxy.org'", - WARNING + WARNING, ) << tls.TlsFailedServerHook(tls_hook_data) >> tutils.reply() @@ -810,7 +854,8 @@ def test_untrusted_cert(self, tctx: context.Context): ) ) assert ( - tls_hook_data().conn.error == "hostname 'wrong.host.mitmproxy.org' doesn't match 'example.mitmproxy.org'" + tls_hook_data().conn.error + == "hostname 'wrong.host.mitmproxy.org' doesn't match 'example.mitmproxy.org'" ) assert not tctx.server.tls_established @@ -822,7 +867,11 @@ def make_client_tls_layer( # This is a bit contrived as the client layer expects a server layer as parent. # We also set child layers manually to avoid NextLayer noise. - server_layer = DummyLayer(tctx) if no_server else quic.ServerQuicLayer(tctx, time=lambda: tssl_client.now) + server_layer = ( + DummyLayer(tctx) + if no_server + else quic.ServerQuicLayer(tctx, time=lambda: tssl_client.now) + ) client_layer = quic.ClientQuicLayer(tctx, time=lambda: tssl_client.now) server_layer.child_layer = client_layer playbook = tutils.Playbook(server_layer) @@ -881,13 +930,21 @@ def test_client_only(self, tctx: context.Context): assert ( playbook >> events.Wakeup(playbook.actual[16]) - << commands.Log(" >> Wakeup(command=RequestWakeup({'delay': 0.20000000000000004}))", DEBUG) - << commands.Log(" [quic] close_notify Client(client:1234, state=open, tls) (reason=Idle timeout)", DEBUG) + << commands.Log( + " >> Wakeup(command=RequestWakeup({'delay': 0.20000000000000004}))", + DEBUG, + ) + << commands.Log( + " [quic] close_notify Client(client:1234, state=open, tls) (reason=Idle timeout)", + DEBUG, + ) << commands.CloseConnection(tctx.client) ) @pytest.mark.parametrize("server_state", ["open", "closed"]) - def test_server_required(self, tctx: context.Context, server_state: Literal["open", "closed"]): + def test_server_required( + self, tctx: context.Context, server_state: Literal["open", "closed"] + ): """ Test the scenario where a server connection is required (for example, because of an unknown ALPN) to establish TLS with the client. @@ -959,7 +1016,9 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: _test_echo(playbook, tssl_server, tctx.server) @pytest.mark.parametrize("server_state", ["open", "closed"]) - def test_passthrough_from_clienthello(self, tctx: context.Context, server_state: Literal["open", "closed"]): + def test_passthrough_from_clienthello( + self, tctx: context.Context, server_state: Literal["open", "closed"] + ): """ Test the scenario where the connection is moved to passthrough mode in the tls_clienthello hook. """ @@ -1017,7 +1076,9 @@ def test_cannot_parse_clienthello(self, tctx: context.Context): client_layer.debug = "" assert ( playbook - >> events.DataReceived(connection.Server(address=None), b"data on other stream") + >> events.DataReceived( + connection.Server(address=None), b"data on other stream" + ) << commands.Log(">> DataReceived(server, b'data on other stream')", DEBUG) << commands.Log( "[quic] Swallowing DataReceived(server, b'data on other stream') as handshake failed.", @@ -1093,12 +1154,16 @@ def require_server_conn(client_hello: tls.ClientHelloData) -> None: >> tutils.reply() << commands.Log(f"No QUIC context was provided, failing connection.", ERROR) << commands.CloseConnection(tctx.client) - << commands.Log("Client QUIC handshake failed. connection closed early", WARNING) + << commands.Log( + "Client QUIC handshake failed. connection closed early", WARNING + ) << tls.TlsFailedClientHook(tutils.Placeholder()) ) def test_no_server_tls(self, tctx: context.Context): - playbook, client_layer, tssl_client = make_client_tls_layer(tctx, no_server=True) + playbook, client_layer, tssl_client = make_client_tls_layer( + tctx, no_server=True + ) def require_server_conn(client_hello: tls.ClientHelloData) -> None: client_hello.establish_server_tls_first = True @@ -1129,29 +1194,35 @@ def test_version_negotiation(self, tctx: context.Context): def test_non_init_clienthello(self, tctx: context.Context): playbook, client_layer, tssl_client = make_client_tls_layer(tctx) data = ( - b'\xc2\x00\x00\x00\x01\x08q\xda\x98\x03X-\x13o\x08y\xa5RQv\xbe\xe3\xeb\x00@a\x98\x19\xf95t\xad-\x1c\\a\xdd\x8c\xd0\x15F' - b'\xdf\xdc\x87cb\x1eu\xb0\x95*\xac\xa8\xf7a \xb8\nQ\xbd=\xf5x\xca\r\xe6\x8b\x05 w\x9f\xcd\x8d\xcb\xa0\x06\x1e \x8d.\x8f' - b'T\xda\x12et\xe4\x83\x93X\x8aa\xd1\xb2\x18\xb6\xa7\xf50y\x9b\xc5T\xe1\x87\xdd\x9fqv\xb0\x90\xa7s' - b'\xee\x00\x00\x00\x01\x08q\xda\x98\x03X-\x13o\x08y\xa5RQv\xbe\xe3\xeb@a*.\xa8j\x90\x1b\x1a\x7fZ\x04\x0b\\\xc7\x00\x03' - b'\xd7sC\xf8G\x84\x1e\xba\xcf\x08Z\xdd\x98+\xaa\x98J\xca\xe3\xb7u1\x89\x00\xdf\x8e\x16`\xd9^\xc0@i\x1a\x10\x99\r\xd8' - b'\x1dv3\xc6\xb8"\xb9\xa8F\x95K\x9a/\xbc\'\xd8\xd8\x94\x8f\xe7B/\x05\x9d\xfb\x80\xa9\xda@\xe6\xb0J\xfe\xe0\x0f\x02L}' - b'\xd9\xed\xd2L\xa7\xcf' + b"\xc2\x00\x00\x00\x01\x08q\xda\x98\x03X-\x13o\x08y\xa5RQv\xbe\xe3\xeb\x00@a\x98\x19\xf95t\xad-\x1c\\a\xdd\x8c\xd0\x15F" + b"\xdf\xdc\x87cb\x1eu\xb0\x95*\xac\xa8\xf7a \xb8\nQ\xbd=\xf5x\xca\r\xe6\x8b\x05 w\x9f\xcd\x8d\xcb\xa0\x06\x1e \x8d.\x8f" + b"T\xda\x12et\xe4\x83\x93X\x8aa\xd1\xb2\x18\xb6\xa7\xf50y\x9b\xc5T\xe1\x87\xdd\x9fqv\xb0\x90\xa7s" + b"\xee\x00\x00\x00\x01\x08q\xda\x98\x03X-\x13o\x08y\xa5RQv\xbe\xe3\xeb@a*.\xa8j\x90\x1b\x1a\x7fZ\x04\x0b\\\xc7\x00\x03" + b"\xd7sC\xf8G\x84\x1e\xba\xcf\x08Z\xdd\x98+\xaa\x98J\xca\xe3\xb7u1\x89\x00\xdf\x8e\x16`\xd9^\xc0@i\x1a\x10\x99\r\xd8" + b"\x1dv3\xc6\xb8\"\xb9\xa8F\x95K\x9a/\xbc'\xd8\xd8\x94\x8f\xe7B/\x05\x9d\xfb\x80\xa9\xda@\xe6\xb0J\xfe\xe0\x0f\x02L}" + b"\xd9\xed\xd2L\xa7\xcf" ) assert ( playbook >> events.DataReceived(tctx.client, data) - << commands.Log(f"Client QUIC handshake failed. Invalid handshake received, roaming not supported. ({data.hex()})", WARNING) + << commands.Log( + f"Client QUIC handshake failed. Invalid handshake received, roaming not supported. ({data.hex()})", + WARNING, + ) << tls.TlsFailedClientHook(tutils.Placeholder()) ) assert client_layer.tunnel_state == tls.tunnel.TunnelState.ESTABLISHING def test_invalid_clienthello(self, tctx: context.Context): playbook, client_layer, tssl_client = make_client_tls_layer(tctx) - data = client_hello[0:1200] + b'\x00' + client_hello[1200:] + data = client_hello[0:1200] + b"\x00" + client_hello[1200:] assert ( playbook >> events.DataReceived(tctx.client, data) - << commands.Log(f"Client QUIC handshake failed. Cannot parse ClientHello: No ClientHello returned. ({data.hex()})", WARNING) + << commands.Log( + f"Client QUIC handshake failed. Cannot parse ClientHello: No ClientHello returned. ({data.hex()})", + WARNING, + ) << tls.TlsFailedClientHook(tutils.Placeholder()) ) assert client_layer.tunnel_state == tls.tunnel.TunnelState.ESTABLISHING diff --git a/test/mitmproxy/proxy/layers/test_socks5_fuzz.py b/test/mitmproxy/proxy/layers/test_socks5_fuzz.py index bbefa4b01c..e8762fd35a 100644 --- a/test/mitmproxy/proxy/layers/test_socks5_fuzz.py +++ b/test/mitmproxy/proxy/layers/test_socks5_fuzz.py @@ -8,11 +8,14 @@ from mitmproxy.proxy.layers.modes import Socks5Proxy opts = options.Options() -tctx = Context(Client( - peername=("client", 1234), - sockname=("127.0.0.1", 8080), - timestamp_start=1605699329 -), opts) +tctx = Context( + Client( + peername=("client", 1234), + sockname=("127.0.0.1", 8080), + timestamp_start=1605699329, + ), + opts, +) @given(binary()) diff --git a/test/mitmproxy/proxy/layers/test_tcp.py b/test/mitmproxy/proxy/layers/test_tcp.py index df01fa9f98..7864622599 100644 --- a/test/mitmproxy/proxy/layers/test_tcp.py +++ b/test/mitmproxy/proxy/layers/test_tcp.py @@ -1,11 +1,18 @@ import pytest -from mitmproxy.proxy.commands import CloseConnection, CloseTcpConnection, OpenConnection, SendData -from mitmproxy.proxy.events import ConnectionClosed, DataReceived +from ..tutils import Placeholder +from ..tutils import Playbook +from ..tutils import reply +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import CloseTcpConnection +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import SendData +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived from mitmproxy.proxy.layers import tcp from mitmproxy.proxy.layers.tcp import TcpMessageInjected -from mitmproxy.tcp import TCPFlow, TCPMessage -from ..tutils import Placeholder, Playbook, reply +from mitmproxy.tcp import TCPFlow +from mitmproxy.tcp import TCPMessage def test_open_connection(tctx): diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 7422a53221..1fde306fa1 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -1,20 +1,26 @@ import ssl -from logging import DEBUG, WARNING - import time +from logging import DEBUG +from logging import WARNING from typing import Optional import pytest - from OpenSSL import SSL + from mitmproxy import connection -from mitmproxy.connection import ConnectionState, Server -from mitmproxy.proxy import commands, context, events, layer +from mitmproxy.connection import ConnectionState +from mitmproxy.connection import Server +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.layers import tls -from mitmproxy.tls import ClientHelloData, TlsData +from mitmproxy.tls import ClientHelloData +from mitmproxy.tls import TlsData from mitmproxy.utils import data from test.mitmproxy.proxy import tutils -from test.mitmproxy.proxy.tutils import BytesMatching, StrMatching +from test.mitmproxy.proxy.tutils import BytesMatching +from test.mitmproxy.proxy.tutils import StrMatching tlsdata = data.Data(__name__) @@ -136,7 +142,7 @@ def __init__( def bio_write(self, buf: bytes) -> int: return self.inc.write(buf) - def bio_read(self, bufsize: int = 2 ** 16) -> bytes: + def bio_read(self, bufsize: int = 2**16) -> bytes: return self.out.read(bufsize) def do_handshake(self) -> None: @@ -365,7 +371,9 @@ def test_untrusted_cert(self, tctx): >> events.DataReceived(tctx.server, tssl.bio_read()) << commands.Log( # different casing in OpenSSL < 3.0 - StrMatching("Server TLS handshake failed. Certificate verify failed: [Hh]ostname mismatch"), + StrMatching( + "Server TLS handshake failed. Certificate verify failed: [Hh]ostname mismatch" + ), WARNING, ) << tls.TlsFailedServerHook(tls_hook_data) @@ -374,11 +382,14 @@ def test_untrusted_cert(self, tctx): << commands.SendData( tctx.client, # different casing in OpenSSL < 3.0 - BytesMatching(b"open-connection failed: Certificate verify failed: [Hh]ostname mismatch"), + BytesMatching( + b"open-connection failed: Certificate verify failed: [Hh]ostname mismatch" + ), ) ) assert ( - tls_hook_data().conn.error.lower() == "Certificate verify failed: Hostname mismatch".lower() + tls_hook_data().conn.error.lower() + == "Certificate verify failed: Hostname mismatch".lower() ) assert not tctx.server.tls_established @@ -780,7 +791,9 @@ def test_is_dtls_handshake_record(): def test_dtls_record_contents(): - data = bytes.fromhex("16fefd00000000000000000002beef" "16fefd00000000000000000001ff") + data = bytes.fromhex( + "16fefd00000000000000000002beef" "16fefd00000000000000000001ff" + ) assert list(tls.dtls_handshake_record_contents(data)) == [b"\xbe\xef", b"\xff"] for i in range(12): assert list(tls.dtls_handshake_record_contents(data[:i])) == [] @@ -800,8 +813,8 @@ def test__dtls_record_contents_err(): "cc02bc02fc00ac014c02cc03001000000" ) dtls_client_hello_with_extensions = bytes.fromhex( - "16fefd00000000000000000085" # record layer - "010000790000000000000079" # hanshake layer + "16fefd00000000000000000085" # record layer + "010000790000000000000079" # hanshake layer "fefd62bf0e0bf809df43e7669197be831919878b1a72c07a584d3c0a8ca6665878010000000cc02bc02fc00ac014c02cc0" "3001000043000d0010000e0403050306030401050106010807ff01000100000a00080006001d00170018000b00020100001" "7000000000010000e00000b6578616d706c652e636f6d" @@ -809,26 +822,35 @@ def test__dtls_record_contents_err(): def test_dtls_get_client_hello(): - single_record = bytes.fromhex("16fefd00000000000000000042") + dtls_client_hello_no_extensions + single_record = ( + bytes.fromhex("16fefd00000000000000000042") + dtls_client_hello_no_extensions + ) assert tls.get_dtls_client_hello(single_record) == dtls_client_hello_no_extensions split_over_two_records = ( - bytes.fromhex("16fefd00000000000000000020") - + dtls_client_hello_no_extensions[:32] - + bytes.fromhex("16fefd00000000000000000022") - + dtls_client_hello_no_extensions[32:] + bytes.fromhex("16fefd00000000000000000020") + + dtls_client_hello_no_extensions[:32] + + bytes.fromhex("16fefd00000000000000000022") + + dtls_client_hello_no_extensions[32:] + ) + assert ( + tls.get_dtls_client_hello(split_over_two_records) + == dtls_client_hello_no_extensions ) - assert tls.get_dtls_client_hello(split_over_two_records) == dtls_client_hello_no_extensions incomplete = split_over_two_records[:42] assert tls.get_dtls_client_hello(incomplete) is None def test_dtls_parse_client_hello(): - assert tls.dtls_parse_client_hello(dtls_client_hello_with_extensions).sni == "example.com" + assert ( + tls.dtls_parse_client_hello(dtls_client_hello_with_extensions).sni + == "example.com" + ) assert tls.dtls_parse_client_hello(dtls_client_hello_with_extensions[:50]) is None with pytest.raises(ValueError): tls.dtls_parse_client_hello( # Server Name Length longer than actual Server Name - dtls_client_hello_with_extensions[:-16] + b"\x00\x0e\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + dtls_client_hello_with_extensions[:-16] + + b"\x00\x0e\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" ) diff --git a/test/mitmproxy/proxy/layers/test_tls_fuzz.py b/test/mitmproxy/proxy/layers/test_tls_fuzz.py index 402c08a5fc..e157409891 100644 --- a/test/mitmproxy/proxy/layers/test_tls_fuzz.py +++ b/test/mitmproxy/proxy/layers/test_tls_fuzz.py @@ -1,8 +1,10 @@ -from hypothesis import given, example -from hypothesis.strategies import binary, integers +from hypothesis import example +from hypothesis import given +from hypothesis.strategies import binary +from hypothesis.strategies import integers -from mitmproxy.tls import ClientHello from mitmproxy.proxy.layers.tls import parse_client_hello +from mitmproxy.tls import ClientHello client_hello_with_extensions = bytes.fromhex( "16030300bb" # record layer diff --git a/test/mitmproxy/proxy/layers/test_udp.py b/test/mitmproxy/proxy/layers/test_udp.py index 14b344a574..9b8d3b419f 100644 --- a/test/mitmproxy/proxy/layers/test_udp.py +++ b/test/mitmproxy/proxy/layers/test_udp.py @@ -1,11 +1,17 @@ import pytest -from mitmproxy.proxy.commands import CloseConnection, OpenConnection, SendData -from mitmproxy.proxy.events import ConnectionClosed, DataReceived +from ..tutils import Placeholder +from ..tutils import Playbook +from ..tutils import reply +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import SendData +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived from mitmproxy.proxy.layers import udp from mitmproxy.proxy.layers.udp import UdpMessageInjected -from mitmproxy.udp import UDPFlow, UDPMessage -from ..tutils import Placeholder, Playbook, reply +from mitmproxy.udp import UDPFlow +from mitmproxy.udp import UDPMessage def test_open_connection(tctx): diff --git a/test/mitmproxy/proxy/layers/test_websocket.py b/test/mitmproxy/proxy/layers/test_websocket.py index a1d96133f4..eebca259dd 100644 --- a/test/mitmproxy/proxy/layers/test_websocket.py +++ b/test/mitmproxy/proxy/layers/test_websocket.py @@ -2,19 +2,27 @@ from dataclasses import dataclass import pytest - -import wsproto import wsproto.events -from mitmproxy.http import HTTPFlow, Request, Response -from mitmproxy.proxy.layers.http import HTTPMode -from mitmproxy.proxy.commands import SendData, CloseConnection, Log +from wsproto.frame_protocol import Opcode + from mitmproxy.connection import ConnectionState -from mitmproxy.proxy.events import DataReceived, ConnectionClosed -from mitmproxy.proxy.layers import http, websocket +from mitmproxy.http import HTTPFlow +from mitmproxy.http import Request +from mitmproxy.http import Response +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import Log +from mitmproxy.proxy.commands import SendData +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived +from mitmproxy.proxy.layers import http +from mitmproxy.proxy.layers import websocket +from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy.layers.websocket import WebSocketMessageInjected -from mitmproxy.websocket import WebSocketData, WebSocketMessage -from test.mitmproxy.proxy.tutils import Placeholder, Playbook, reply -from wsproto.frame_protocol import Opcode +from mitmproxy.websocket import WebSocketData +from mitmproxy.websocket import WebSocketMessage +from test.mitmproxy.proxy.tutils import Placeholder +from test.mitmproxy.proxy.tutils import Playbook +from test.mitmproxy.proxy.tutils import reply @dataclass diff --git a/test/mitmproxy/proxy/test_context.py b/test/mitmproxy/proxy/test_context.py index 62e9c9b7a7..1ac2bd8332 100644 --- a/test/mitmproxy/proxy/test_context.py +++ b/test/mitmproxy/proxy/test_context.py @@ -1,5 +1,6 @@ from mitmproxy.proxy import context -from mitmproxy.test import tflow, taddons +from mitmproxy.test import taddons +from mitmproxy.test import tflow def test_context(): diff --git a/test/mitmproxy/proxy/test_events.py b/test/mitmproxy/proxy/test_events.py index c415fadad2..5e867369e9 100644 --- a/test/mitmproxy/proxy/test_events.py +++ b/test/mitmproxy/proxy/test_events.py @@ -3,7 +3,8 @@ import pytest from mitmproxy import connection -from mitmproxy.proxy import events, commands +from mitmproxy.proxy import commands +from mitmproxy.proxy import events @pytest.fixture diff --git a/test/mitmproxy/proxy/test_layer.py b/test/mitmproxy/proxy/test_layer.py index 1d4baef7e2..b646c2e027 100644 --- a/test/mitmproxy/proxy/test_layer.py +++ b/test/mitmproxy/proxy/test_layer.py @@ -2,7 +2,9 @@ import pytest -from mitmproxy.proxy import commands, events, layer +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.context import Context from test.mitmproxy.proxy import tutils diff --git a/test/mitmproxy/proxy/test_mode_servers.py b/test/mitmproxy/proxy/test_mode_servers.py index eea437ebbf..917645fc8d 100644 --- a/test/mitmproxy/proxy/test_mode_servers.py +++ b/test/mitmproxy/proxy/test_mode_servers.py @@ -1,14 +1,18 @@ import asyncio import platform from typing import cast -from unittest.mock import AsyncMock, MagicMock, Mock +from unittest.mock import AsyncMock +from unittest.mock import MagicMock +from unittest.mock import Mock import pytest import mitmproxy.platform from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.net import udp -from mitmproxy.proxy.mode_servers import DnsInstance, ServerInstance, WireGuardServerInstance +from mitmproxy.proxy.mode_servers import DnsInstance +from mitmproxy.proxy.mode_servers import ServerInstance +from mitmproxy.proxy.mode_servers import WireGuardServerInstance from mitmproxy.proxy.server import ConnectionHandler from mitmproxy.test import taddons @@ -18,14 +22,23 @@ def test_make(): context = MagicMock() assert ServerInstance.make("regular", manager) - for mode in ["regular", "http3", "upstream:example.com", "transparent", "reverse:example.com", "socks5"]: + for mode in [ + "regular", + "http3", + "upstream:example.com", + "transparent", + "reverse:example.com", + "socks5", + ]: inst = ServerInstance.make(mode, manager) assert inst assert inst.make_top_layer(context) assert inst.mode.description assert inst.to_json() - with pytest.raises(ValueError, match="is not a spec for a WireGuardServerInstance server."): + with pytest.raises( + ValueError, match="is not a spec for a WireGuardServerInstance server." + ): WireGuardServerInstance.make("regular", manager) @@ -86,7 +99,9 @@ async def test_transparent(failure, monkeypatch, caplog_async): if failure: monkeypatch.setattr(mitmproxy.platform, "original_addr", None) else: - monkeypatch.setattr(mitmproxy.platform, "original_addr", lambda s: ("address", 42)) + monkeypatch.setattr( + mitmproxy.platform, "original_addr", lambda s: ("address", 42) + ) with taddons.context(Proxyserver()) as tctx: tctx.options.connection_strategy = "lazy" @@ -199,12 +214,16 @@ async def test_wireguard_invalid_conf(tmp_path): async def test_tcp_start_error(): manager = MagicMock() - server = await asyncio.start_server(MagicMock(), host="127.0.0.1", port=0, reuse_address=False) + server = await asyncio.start_server( + MagicMock(), host="127.0.0.1", port=0, reuse_address=False + ) port = server.sockets[0].getsockname()[1] with taddons.context() as tctx: inst = ServerInstance.make(f"regular@127.0.0.1:{port}", manager) - with pytest.raises(OSError, match=f"proxy failed to listen on 127\\.0\\.0\\.1:{port}"): + with pytest.raises( + OSError, match=f"proxy failed to listen on 127\\.0\\.0\\.1:{port}" + ): await inst.start() tctx.options.listen_host = "127.0.0.1" tctx.options.listen_port = port @@ -253,7 +272,9 @@ async def test_udp_start_error(): await inst.start() port = inst.listen_addrs[0][1] inst2 = ServerInstance.make(f"dns@127.0.0.1:{port}", manager) - with pytest.raises(OSError, match=f"server failed to listen on 127\\.0\\.0\\.1:{port}"): + with pytest.raises( + OSError, match=f"server failed to listen on 127\\.0\\.0\\.1:{port}" + ): await inst2.start() await inst.stop() @@ -267,8 +288,12 @@ async def test_udp_connection_reuse(monkeypatch): with taddons.context(): inst = cast(DnsInstance, ServerInstance.make("dns", manager)) - inst.handle_udp_datagram(MagicMock(), b"\x00\x00\x01", ("remoteaddr", 0), ("localaddr", 0)) - inst.handle_udp_datagram(MagicMock(), b"\x00\x00\x02", ("remoteaddr", 0), ("localaddr", 0)) + inst.handle_udp_datagram( + MagicMock(), b"\x00\x00\x01", ("remoteaddr", 0), ("localaddr", 0) + ) + inst.handle_udp_datagram( + MagicMock(), b"\x00\x00\x02", ("remoteaddr", 0), ("localaddr", 0) + ) await asyncio.sleep(0) assert len(inst.manager.connections) == 1 diff --git a/test/mitmproxy/proxy/test_mode_specs.py b/test/mitmproxy/proxy/test_mode_specs.py index be83e5238a..b52c721968 100644 --- a/test/mitmproxy/proxy/test_mode_specs.py +++ b/test/mitmproxy/proxy/test_mode_specs.py @@ -2,7 +2,8 @@ import pytest -from mitmproxy.proxy.mode_specs import ProxyMode, Socks5Mode +from mitmproxy.proxy.mode_specs import ProxyMode +from mitmproxy.proxy.mode_specs import Socks5Mode def test_parse(): @@ -45,7 +46,10 @@ def test_listen_addr(): assert ProxyMode.parse("regular").listen_host() == "" assert ProxyMode.parse("regular@127.0.0.2:8080").listen_host() == "127.0.0.2" assert ProxyMode.parse("regular").listen_host(default="127.0.0.3") == "127.0.0.3" - assert ProxyMode.parse("regular@127.0.0.2:8080").listen_host(default="127.0.0.3") == "127.0.0.2" + assert ( + ProxyMode.parse("regular@127.0.0.2:8080").listen_host(default="127.0.0.3") + == "127.0.0.2" + ) assert ProxyMode.parse("reverse:https://1.2.3.4").listen_port() == 8080 assert ProxyMode.parse("reverse:dns://8.8.8.8").listen_port() == 53 diff --git a/test/mitmproxy/proxy/test_tunnel.py b/test/mitmproxy/proxy/test_tunnel.py index ad9af41125..0256390cf3 100644 --- a/test/mitmproxy/proxy/test_tunnel.py +++ b/test/mitmproxy/proxy/test_tunnel.py @@ -2,12 +2,22 @@ import pytest -from mitmproxy.proxy import tunnel, layer -from mitmproxy.proxy.commands import CloseTcpConnection, SendData, Log, CloseConnection, OpenConnection -from mitmproxy.connection import Server, ConnectionState +from mitmproxy.connection import ConnectionState +from mitmproxy.connection import Server +from mitmproxy.proxy import layer +from mitmproxy.proxy import tunnel +from mitmproxy.proxy.commands import CloseConnection +from mitmproxy.proxy.commands import CloseTcpConnection +from mitmproxy.proxy.commands import Log +from mitmproxy.proxy.commands import OpenConnection +from mitmproxy.proxy.commands import SendData from mitmproxy.proxy.context import Context -from mitmproxy.proxy.events import Event, DataReceived, Start, ConnectionClosed -from test.mitmproxy.proxy.tutils import Playbook, reply +from mitmproxy.proxy.events import ConnectionClosed +from mitmproxy.proxy.events import DataReceived +from mitmproxy.proxy.events import Event +from mitmproxy.proxy.events import Start +from test.mitmproxy.proxy.tutils import Playbook +from test.mitmproxy.proxy.tutils import reply class TChildLayer(layer.Layer): diff --git a/test/mitmproxy/proxy/test_tutils.py b/test/mitmproxy/proxy/test_tutils.py index 04880990e4..ec676405b3 100644 --- a/test/mitmproxy/proxy/test_tutils.py +++ b/test/mitmproxy/proxy/test_tutils.py @@ -4,8 +4,10 @@ import pytest -from mitmproxy.proxy import commands, events, layer from . import tutils +from mitmproxy.proxy import commands +from mitmproxy.proxy import events +from mitmproxy.proxy import layer class TEvent(events.Event): diff --git a/test/mitmproxy/proxy/tutils.py b/test/mitmproxy/proxy/tutils.py index 087db71316..618f6da0b2 100644 --- a/test/mitmproxy/proxy/tutils.py +++ b/test/mitmproxy/proxy/tutils.py @@ -1,17 +1,24 @@ import collections.abc import difflib -import logging - import itertools +import logging import re import textwrap import traceback -from collections.abc import Callable, Iterable -from typing import Any, AnyStr, Generic, Optional, TypeVar, Union +from collections.abc import Callable +from collections.abc import Iterable +from typing import Any +from typing import AnyStr +from typing import Generic +from typing import Optional +from typing import TypeVar +from typing import Union -from mitmproxy.proxy import commands, context, layer -from mitmproxy.proxy import events from mitmproxy.connection import ConnectionState +from mitmproxy.proxy import commands +from mitmproxy.proxy import context +from mitmproxy.proxy import events +from mitmproxy.proxy import layer from mitmproxy.proxy.events import command_reply_subclasses from mitmproxy.proxy.layer import Layer diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 4bc5f15826..446fbb8d96 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -4,8 +4,8 @@ import pytest -from mitmproxy.test import tflow from mitmproxy.test import taddons +from mitmproxy.test import tflow class TestConcurrent: diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py index 77057d11b5..4122f377ce 100644 --- a/test/mitmproxy/test_addonmanager.py +++ b/test/mitmproxy/test_addonmanager.py @@ -7,7 +7,8 @@ from mitmproxy import hooks from mitmproxy import master from mitmproxy import options -from mitmproxy.proxy.layers.http import HttpRequestHook, HttpResponseHook +from mitmproxy.proxy.layers.http import HttpRequestHook +from mitmproxy.proxy.layers.http import HttpResponseHook from mitmproxy.test import taddons from mitmproxy.test import tflow diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index 448efdfea3..815a84c613 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -1,13 +1,14 @@ import os -from datetime import datetime, timezone +from datetime import datetime +from datetime import timezone from pathlib import Path -from cryptography import x509 -from cryptography.x509 import NameOID import pytest +from cryptography import x509 +from cryptography.x509 import NameOID -from mitmproxy import certs from ..conftest import skip_windows +from mitmproxy import certs # class TestDNTree: diff --git a/test/mitmproxy/test_command_lexer.py b/test/mitmproxy/test_command_lexer.py index dfe9b27198..dd7be31dd5 100644 --- a/test/mitmproxy/test_command_lexer.py +++ b/test/mitmproxy/test_command_lexer.py @@ -1,6 +1,7 @@ import pyparsing import pytest -from hypothesis import given, example +from hypothesis import example +from hypothesis import given from hypothesis.strategies import text from mitmproxy import command_lexer diff --git a/test/mitmproxy/test_connection.py b/test/mitmproxy/test_connection.py index 300015c76e..eee6607850 100644 --- a/test/mitmproxy/test_connection.py +++ b/test/mitmproxy/test_connection.py @@ -1,14 +1,20 @@ import pytest -from mitmproxy.connection import Server, Client, ConnectionState -from mitmproxy.test.tflow import tclient_conn, tserver_conn +from mitmproxy.connection import Client +from mitmproxy.connection import ConnectionState +from mitmproxy.connection import Server +from mitmproxy.test.tflow import tclient_conn +from mitmproxy.test.tflow import tserver_conn class TestConnection: def test_basic(self): - c = Client(peername=("127.0.0.1", 52314), sockname=("127.0.0.1", 8080), - timestamp_start=1607780791, - state=ConnectionState.OPEN) + c = Client( + peername=("127.0.0.1", 52314), + sockname=("127.0.0.1", 8080), + timestamp_start=1607780791, + state=ConnectionState.OPEN, + ) assert not c.tls_established c.timestamp_tls_setup = 1607780792 assert c.tls_established @@ -34,7 +40,7 @@ def test_basic(self): peername=("127.0.0.1", 52314), sockname=("127.0.0.1", 8080), timestamp_start=1607780791, - cipher_list=["foo", "bar"] + cipher_list=["foo", "bar"], ) assert repr(c) assert str(c) diff --git a/test/mitmproxy/test_dns.py b/test/mitmproxy/test_dns.py index 6a5075c91f..cf9af2ac37 100644 --- a/test/mitmproxy/test_dns.py +++ b/test/mitmproxy/test_dns.py @@ -1,5 +1,6 @@ import ipaddress import struct + import pytest from mitmproxy import dns @@ -111,7 +112,7 @@ def test(what: str, min: int, max: int): with pytest.raises(ValueError): req.packed - test("id", 0, 2 ** 16 - 1) + test("id", 0, 2**16 - 1) test("reserved", 0, 7) test("op_code", 0, 0b1111) test("response_code", 0, 0b1111) diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index a44408c259..29f2eeb93f 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -8,8 +8,10 @@ from mitmproxy import options from mitmproxy.exceptions import FlowReadException from mitmproxy.io import tnetstring -from mitmproxy.proxy import server_hooks, layers -from mitmproxy.test import taddons, tflow +from mitmproxy.proxy import layers +from mitmproxy.proxy import server_hooks +from mitmproxy.test import taddons +from mitmproxy.test import tflow class State: diff --git a/test/mitmproxy/test_flowfilter.py b/test/mitmproxy/test_flowfilter.py index cf6388c1ea..b494c93337 100644 --- a/test/mitmproxy/test_flowfilter.py +++ b/test/mitmproxy/test_flowfilter.py @@ -1,8 +1,11 @@ import io -import pytest from unittest.mock import patch + +import pytest + +from mitmproxy import flowfilter +from mitmproxy import http from mitmproxy.test import tflow -from mitmproxy import flowfilter, http class TestParsing: diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index 169aafd71d..acdff67d9a 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -1,17 +1,21 @@ import asyncio import email -import time import json +import time from unittest import mock import pytest from mitmproxy import flow from mitmproxy import flowfilter -from mitmproxy.http import Headers, Request, Response, HTTPFlow +from mitmproxy.http import Headers +from mitmproxy.http import HTTPFlow +from mitmproxy.http import Request +from mitmproxy.http import Response from mitmproxy.net.http.cookies import CookieAttrs from mitmproxy.test.tflow import tflow -from mitmproxy.test.tutils import treq, tresp +from mitmproxy.test.tutils import treq +from mitmproxy.test.tutils import tresp class TestRequest: diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 391a7d0b7e..62df7d0b5e 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -1,14 +1,14 @@ +import argparse import copy import io from collections.abc import Sequence from typing import Optional import pytest -import argparse +from mitmproxy import exceptions from mitmproxy import options from mitmproxy import optmanager -from mitmproxy import exceptions class TO(optmanager.OptManager): diff --git a/test/mitmproxy/test_tcp.py b/test/mitmproxy/test_tcp.py index 13001c8625..3fdde8b23d 100644 --- a/test/mitmproxy/test_tcp.py +++ b/test/mitmproxy/test_tcp.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy import tcp from mitmproxy import flowfilter +from mitmproxy import tcp from mitmproxy.test import tflow diff --git a/test/mitmproxy/test_tls.py b/test/mitmproxy/test_tls.py index af23bc3ab4..b04491b81b 100644 --- a/test/mitmproxy/test_tls.py +++ b/test/mitmproxy/test_tls.py @@ -103,10 +103,11 @@ def test_extensions(self): assert c.cipher_suites == [2, 3, 10, 5, 4, 9] assert c.alpn_protocols == [b"h2", b"http/1.1"] assert c.extensions == [ - (13, b'\x00\x0e\x04\x03\x05\x03\x06\x03\x04\x01\x05\x01\x06\x01\x08\x07'), - (65281, b'\x00'), - (10, b'\x00\x06\x00\x1d\x00\x17\x00\x18'), - (11, b'\x01\x00'), (23, b''), - (0, b'\x00\x0e\x00\x00\x0bexample.com'), - (16, b'\x00\x0c\x02h2\x08http/1.1') + (13, b"\x00\x0e\x04\x03\x05\x03\x06\x03\x04\x01\x05\x01\x06\x01\x08\x07"), + (65281, b"\x00"), + (10, b"\x00\x06\x00\x1d\x00\x17\x00\x18"), + (11, b"\x01\x00"), + (23, b""), + (0, b"\x00\x0e\x00\x00\x0bexample.com"), + (16, b"\x00\x0c\x02h2\x08http/1.1"), ] diff --git a/test/mitmproxy/test_types.py b/test/mitmproxy/test_types.py index 29d2b1f039..31a33b4299 100644 --- a/test/mitmproxy/test_types.py +++ b/test/mitmproxy/test_types.py @@ -1,17 +1,16 @@ +import contextlib +import os from collections.abc import Sequence import pytest -import os -import contextlib import mitmproxy.exceptions import mitmproxy.types -from mitmproxy.test import taddons -from mitmproxy.test import tflow +from . import test_command from mitmproxy import command from mitmproxy import flow - -from . import test_command +from mitmproxy.test import taddons +from mitmproxy.test import tflow @contextlib.contextmanager diff --git a/test/mitmproxy/test_udp.py b/test/mitmproxy/test_udp.py index 2a6a8dd125..ba652f74f1 100644 --- a/test/mitmproxy/test_udp.py +++ b/test/mitmproxy/test_udp.py @@ -1,7 +1,7 @@ import pytest -from mitmproxy import udp from mitmproxy import flowfilter +from mitmproxy import udp from mitmproxy.test import tflow diff --git a/test/mitmproxy/test_websocket.py b/test/mitmproxy/test_websocket.py index f227d0dc50..08117b0fb0 100644 --- a/test/mitmproxy/test_websocket.py +++ b/test/mitmproxy/test_websocket.py @@ -1,9 +1,9 @@ import pytest +from wsproto.frame_protocol import Opcode from mitmproxy import http from mitmproxy import websocket from mitmproxy.test import tflow -from wsproto.frame_protocol import Opcode class TestWebSocketData: diff --git a/test/mitmproxy/tools/console/test_contentview.py b/test/mitmproxy/tools/console/test_contentview.py index 9819ea9a2e..ee0b727579 100644 --- a/test/mitmproxy/tools/console/test_contentview.py +++ b/test/mitmproxy/tools/console/test_contentview.py @@ -1,6 +1,6 @@ -from mitmproxy.test import tflow from mitmproxy import contentviews from mitmproxy.contentviews.base import format_text +from mitmproxy.test import tflow class TContentView(contentviews.View): diff --git a/test/mitmproxy/tools/console/test_keymap.py b/test/mitmproxy/tools/console/test_keymap.py index fb73471e37..624412ab20 100644 --- a/test/mitmproxy/tools/console/test_keymap.py +++ b/test/mitmproxy/tools/console/test_keymap.py @@ -1,8 +1,10 @@ -from mitmproxy.tools.console import keymap -from mitmproxy.test import taddons from unittest import mock + import pytest +from mitmproxy.test import taddons +from mitmproxy.tools.console import keymap + def test_binding(): b = keymap.Binding("space", "cmd", ["options"], "") diff --git a/test/mitmproxy/tools/console/test_quickhelp.py b/test/mitmproxy/tools/console/test_quickhelp.py index 958bf6ae40..722af3dab8 100644 --- a/test/mitmproxy/tools/console/test_quickhelp.py +++ b/test/mitmproxy/tools/console/test_quickhelp.py @@ -1,7 +1,8 @@ import pytest from mitmproxy.test.tflow import tflow -from mitmproxy.tools.console import defaultkeys, quickhelp +from mitmproxy.tools.console import defaultkeys +from mitmproxy.tools.console import quickhelp from mitmproxy.tools.console.eventlog import EventLog from mitmproxy.tools.console.flowlist import FlowListBox from mitmproxy.tools.console.flowview import FlowView @@ -38,7 +39,7 @@ def keymap() -> Keymap: (EventLog, None, True), (PathEditor, None, False), (SimpleOverlay, None, False), - ] + ], ) def test_quickhelp(widget, flow, keymap, is_root_widget): qh = quickhelp.make(widget, flow, is_root_widget) diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 9a8ece62a0..d4b29906b6 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -1,5 +1,5 @@ -import io import gzip +import io import json import logging import textwrap @@ -14,7 +14,10 @@ from tornado import httpclient from tornado import websocket -from mitmproxy import certs, log, options, optmanager +from mitmproxy import certs +from mitmproxy import log +from mitmproxy import options +from mitmproxy import optmanager from mitmproxy.http import Headers from mitmproxy.proxy.mode_servers import ServerInstance from mitmproxy.test import tflow @@ -159,7 +162,9 @@ async def make_master() -> webmaster.WebMaster: o = options.Options(http2=False) return webmaster.WebMaster(o, with_termlog=False) - m: webmaster.WebMaster = self.io_loop.asyncio_loop.run_until_complete(make_master()) + m: webmaster.WebMaster = self.io_loop.asyncio_loop.run_until_complete( + make_master() + ) f = tflow.tflow(resp=True) f.id = "42" f.request.content = b"foo\nbar" @@ -177,11 +182,13 @@ async def make_master() -> webmaster.WebMaster: si2 = ServerInstance.make("reverse:example.com", m.proxyserver) si2.last_exception = RuntimeError("I failed somehow.") si3 = ServerInstance.make("socks5", m.proxyserver) - m.proxyserver.servers._instances.update({ - si1.mode: si1, - si2.mode: si2, - si3.mode: si3, - }) + m.proxyserver.servers._instances.update( + { + si1.mode: si1, + si2.mode: si2, + si3.mode: si3, + } + ) self.master = m self.view = m.view self.events = m.events @@ -497,11 +504,14 @@ def test_generate_state_js(self): "export function TBackendState(): Required {\n" " return %s\n" "}\n" - % textwrap.indent(json.dumps(data, indent=4, sort_keys=True), " ").lstrip() + % textwrap.indent( + json.dumps(data, indent=4, sort_keys=True), " " + ).lstrip() ) ( - Path(__file__).parent / "../../../../web/src/js/__tests__/ducks/_tbackendstate.ts" + Path(__file__).parent + / "../../../../web/src/js/__tests__/ducks/_tbackendstate.ts" ).write_bytes(content.encode()) def test_err(self): diff --git a/test/mitmproxy/tools/web/test_master.py b/test/mitmproxy/tools/web/test_master.py index c193ea87bb..14b1046949 100644 --- a/test/mitmproxy/tools/web/test_master.py +++ b/test/mitmproxy/tools/web/test_master.py @@ -2,12 +2,15 @@ from unittest.mock import MagicMock import pytest + from mitmproxy.options import Options from mitmproxy.tools.web.master import WebMaster async def test_reuse(): - server = await asyncio.start_server(MagicMock(), host="127.0.0.1", port=0, reuse_address=False) + server = await asyncio.start_server( + MagicMock(), host="127.0.0.1", port=0, reuse_address=False + ) port = server.sockets[0].getsockname()[1] master = WebMaster(Options(), with_termlog=False) master.options.web_host = "127.0.0.1" diff --git a/test/mitmproxy/tools/web/test_static_viewer.py b/test/mitmproxy/tools/web/test_static_viewer.py index 4364e2557c..74473a18f1 100644 --- a/test/mitmproxy/tools/web/test_static_viewer.py +++ b/test/mitmproxy/tools/web/test_static_viewer.py @@ -1,14 +1,13 @@ import json from unittest import mock +from mitmproxy import flowfilter +from mitmproxy.addons import readfile +from mitmproxy.addons import save from mitmproxy.test import taddons from mitmproxy.test import tflow - -from mitmproxy import flowfilter -from mitmproxy.tools.web.app import flow_to_json - from mitmproxy.tools.web import static_viewer -from mitmproxy.addons import save, readfile +from mitmproxy.tools.web.app import flow_to_json def test_save_static(tmpdir): diff --git a/test/mitmproxy/utils/test_arg_check.py b/test/mitmproxy/utils/test_arg_check.py index 97102f49b9..d498d1c430 100644 --- a/test/mitmproxy/utils/test_arg_check.py +++ b/test/mitmproxy/utils/test_arg_check.py @@ -1,5 +1,5 @@ -import io import contextlib +import io from unittest import mock import pytest diff --git a/test/mitmproxy/utils/test_data.py b/test/mitmproxy/utils/test_data.py index f40fc86657..4e7c7af2af 100644 --- a/test/mitmproxy/utils/test_data.py +++ b/test/mitmproxy/utils/test_data.py @@ -1,4 +1,5 @@ import pytest + from mitmproxy.utils import data diff --git a/test/mitmproxy/utils/test_debug.py b/test/mitmproxy/utils/test_debug.py index a61bff8682..6384a59818 100644 --- a/test/mitmproxy/utils/test_debug.py +++ b/test/mitmproxy/utils/test_debug.py @@ -1,6 +1,7 @@ import io import sys from unittest import mock + import pytest from mitmproxy.utils import debug diff --git a/test/mitmproxy/utils/test_emoji.py b/test/mitmproxy/utils/test_emoji.py index a147ba885a..2b099926ba 100644 --- a/test/mitmproxy/utils/test_emoji.py +++ b/test/mitmproxy/utils/test_emoji.py @@ -1,5 +1,5 @@ -from mitmproxy.utils import emoji from mitmproxy.tools.console.common import SYMBOL_MARK +from mitmproxy.utils import emoji def test_emoji(): diff --git a/test/mitmproxy/utils/test_human.py b/test/mitmproxy/utils/test_human.py index 944740611f..d4791de3c2 100644 --- a/test/mitmproxy/utils/test_human.py +++ b/test/mitmproxy/utils/test_human.py @@ -1,5 +1,7 @@ import time + import pytest + from mitmproxy.utils import human @@ -16,8 +18,8 @@ def test_parse_size(): assert human.parse_size("0b") == 0 assert human.parse_size("1") == 1 assert human.parse_size("1k") == 1024 - assert human.parse_size("1m") == 1024 ** 2 - assert human.parse_size("1g") == 1024 ** 3 + assert human.parse_size("1m") == 1024**2 + assert human.parse_size("1g") == 1024**3 with pytest.raises(ValueError): human.parse_size("1f") with pytest.raises(ValueError): diff --git a/test/mitmproxy/utils/test_magisk.py b/test/mitmproxy/utils/test_magisk.py index 83116d7f3c..2382e3921a 100644 --- a/test/mitmproxy/utils/test_magisk.py +++ b/test/mitmproxy/utils/test_magisk.py @@ -1,7 +1,9 @@ -from mitmproxy.utils import magisk +import os + from cryptography import x509 + from mitmproxy.test import taddons -import os +from mitmproxy.utils import magisk def test_get_ca(tdata): diff --git a/test/mitmproxy/utils/test_signals.py b/test/mitmproxy/utils/test_signals.py index 1fcc4f26dd..dd856eb587 100644 --- a/test/mitmproxy/utils/test_signals.py +++ b/test/mitmproxy/utils/test_signals.py @@ -1,7 +1,9 @@ from unittest import mock import pytest -from mitmproxy.utils.signals import AsyncSignal, SyncSignal + +from mitmproxy.utils.signals import AsyncSignal +from mitmproxy.utils.signals import SyncSignal def test_sync_signal() -> None: diff --git a/test/mitmproxy/utils/test_spec.py b/test/mitmproxy/utils/test_spec.py index 6cefcacc72..630dd17998 100644 --- a/test/mitmproxy/utils/test_spec.py +++ b/test/mitmproxy/utils/test_spec.py @@ -1,4 +1,5 @@ import pytest + from mitmproxy.utils.spec import parse_spec diff --git a/test/mitmproxy/utils/test_strutils.py b/test/mitmproxy/utils/test_strutils.py index 3459a673f3..f5b2894ac0 100644 --- a/test/mitmproxy/utils/test_strutils.py +++ b/test/mitmproxy/utils/test_strutils.py @@ -44,7 +44,7 @@ def test_escape_control_characters(): def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"foo") == "foo" assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" - assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" + assert strutils.bytes_to_escaped_str(rb"&!?=\)") == r"&!?=\\)" assert strutils.bytes_to_escaped_str(b"\xc3\xbc") == r"\xc3\xbc" assert strutils.bytes_to_escaped_str(b"'") == r"'" assert strutils.bytes_to_escaped_str(b'"') == r'"' @@ -69,9 +69,9 @@ def test_bytes_to_escaped_str(): def test_escaped_str_to_bytes(): assert strutils.escaped_str_to_bytes("foo") == b"foo" assert strutils.escaped_str_to_bytes("\x08") == b"\b" - assert strutils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)" + assert strutils.escaped_str_to_bytes("&!?=\\\\)") == rb"&!?=\)" assert strutils.escaped_str_to_bytes("\\x08") == b"\b" - assert strutils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)" + assert strutils.escaped_str_to_bytes("&!?=\\\\)") == rb"&!?=\)" assert strutils.escaped_str_to_bytes("\u00fc") == b"\xc3\xbc" with pytest.raises(ValueError): diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index 347e39f303..0f480157ca 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -1,7 +1,10 @@ import io import typing from collections.abc import Sequence -from typing import Any, Optional, TextIO, Union +from typing import Any +from typing import Optional +from typing import TextIO +from typing import Union import pytest @@ -84,4 +87,4 @@ def test_typesec_to_str(): def test_typing_aliases(): assert (typecheck.typespec_to_str(typing.Sequence[str])) == "sequence of str" typecheck.check_option_type("foo", [10], typing.Sequence[int]) - typecheck.check_option_type("foo", (42, "42"), typing.Tuple[int, str]) + typecheck.check_option_type("foo", (42, "42"), tuple[int, str]) From 0b89aede3817831394e6473d82eb04ce572ef621 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 14:39:38 +0100 Subject: [PATCH 138/695] Bump actions/download-artifact from 2 to 3 (#5769) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 2 to 3. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dc6144eaa7..10262bafc8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -228,7 +228,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: .github/python-version.txt - - uses: actions/download-artifact@v2 + - uses: actions/download-artifact@v3 with: name: binaries.linux path: release/dist From 2155599c30e2c89ee5bceb4d882c125051ce0462 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 29 Nov 2022 14:59:52 +0100 Subject: [PATCH 139/695] ci: don't run for dependabot branches (#5775) --- .github/workflows/autofix.yml | 8 +++++++- .github/workflows/main.yml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 90aa5dd413..8f2fda7355 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -1,5 +1,11 @@ name: autofix.ci -on: [ push, pull_request ] + +on: + pull_request: + push: + branches: + - main + permissions: contents: read diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 10262bafc8..8e13f4660f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,12 @@ name: CI -on: [ push, pull_request, workflow_dispatch ] +on: + push: + branches: + - '**' + - '!dependabot/**' + pull_request: + workflow_dispatch: permissions: contents: read From f58c5cfa822201f8442734f039fb7326a8b7c42e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 15:01:23 +0100 Subject: [PATCH 140/695] Bump install-pinned/yesqa (#5772) Bumps [install-pinned/yesqa](https://github.com/install-pinned/yesqa) from b752c9eed899985c6df094e35d7a5a5bd1b94acb to b7b1c5e133f5f516905ae35645a082f751ffa216. - [Release notes](https://github.com/install-pinned/yesqa/releases) - [Commits](https://github.com/install-pinned/yesqa/compare/b752c9eed899985c6df094e35d7a5a5bd1b94acb...b7b1c5e133f5f516905ae35645a082f751ffa216) --- updated-dependencies: - dependency-name: install-pinned/yesqa dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 8f2fda7355..447b55c174 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -28,7 +28,7 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' reorder-python-imports --exit-zero-even-if-changed --py39-plus **/*.py - - uses: install-pinned/yesqa@b752c9eed899985c6df094e35d7a5a5bd1b94acb + - uses: install-pinned/yesqa@b7b1c5e133f5f516905ae35645a082f751ffa216 - name: Run yesqa run: | From a7aed48eab051e982bea8c8f93dcdc0cc0a47f9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 15:01:36 +0100 Subject: [PATCH 141/695] Bump install-pinned/black (#5771) Bumps [install-pinned/black](https://github.com/install-pinned/black) from 81e6dbf82145462d413a6662dd703fa382edeb11 to dde5aed720bc458e86d99144d4d0c1f6c8e08844. - [Release notes](https://github.com/install-pinned/black/releases) - [Commits](https://github.com/install-pinned/black/compare/81e6dbf82145462d413a6662dd703fa382edeb11...dde5aed720bc458e86d99144d4d0c1f6c8e08844) --- updated-dependencies: - dependency-name: install-pinned/black dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 447b55c174..1ba8a398a2 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -38,7 +38,7 @@ jobs: - uses: install-pinned/autoflake@fa3c1715169ac36d903ee9d492d64beb5cad331f - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - - uses: install-pinned/black@81e6dbf82145462d413a6662dd703fa382edeb11 + - uses: install-pinned/black@dde5aed720bc458e86d99144d4d0c1f6c8e08844 - run: black --extend-exclude mitmproxy/contrib . - uses: mhils/add-pr-ref-in-changelog@main From d2af9b61635dcd57c7f68a3d1eec380bacef88aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 15:01:51 +0100 Subject: [PATCH 142/695] Bump install-pinned/autoflake (#5770) Bumps [install-pinned/autoflake](https://github.com/install-pinned/autoflake) from fa3c1715169ac36d903ee9d492d64beb5cad331f to 32877f5112ce1c5b8b30cf57d70593a53d5fca87. - [Release notes](https://github.com/install-pinned/autoflake/releases) - [Commits](https://github.com/install-pinned/autoflake/compare/fa3c1715169ac36d903ee9d492d64beb5cad331f...32877f5112ce1c5b8b30cf57d70593a53d5fca87) --- updated-dependencies: - dependency-name: install-pinned/autoflake dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 1ba8a398a2..29bc8d66e5 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -35,7 +35,7 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' yesqa **/*.py || true - - uses: install-pinned/autoflake@fa3c1715169ac36d903ee9d492d64beb5cad331f + - uses: install-pinned/autoflake@32877f5112ce1c5b8b30cf57d70593a53d5fca87 - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - uses: install-pinned/black@dde5aed720bc458e86d99144d4d0c1f6c8e08844 From 55f7193936dfd4546df5d2e43706cdf1aee433c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Nov 2022 14:09:41 +0000 Subject: [PATCH 143/695] Bump docker/setup-qemu-action from 1.2.0 to 2.1.0 (#5768) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 1.2.0 to 2.1.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/27d0a4f181a40b142cce983c5393082c365d1480...e81a89b1732b9c48d79cd809d8d81d79c4647a18) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e13f4660f..76fffb6da5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -238,7 +238,7 @@ jobs: with: name: binaries.linux path: release/dist - - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1.2.0 + - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 - uses: docker/setup-buildx-action@b1f1f719c7cd5364be7c82e366366da322d01f7c # v1.6.0 - run: python release/build-and-deploy-docker.py From ef1a03181b4aa564ee7a29875b3db99457e04c36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Dec 2022 09:42:49 +0100 Subject: [PATCH 144/695] Bump install-pinned/yesqa (#5781) Bumps [install-pinned/yesqa](https://github.com/install-pinned/yesqa) from b7b1c5e133f5f516905ae35645a082f751ffa216 to cf847492077eea907797bac45e68f87801d4de31. - [Release notes](https://github.com/install-pinned/yesqa/releases) - [Commits](https://github.com/install-pinned/yesqa/compare/b7b1c5e133f5f516905ae35645a082f751ffa216...cf847492077eea907797bac45e68f87801d4de31) --- updated-dependencies: - dependency-name: install-pinned/yesqa dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 29bc8d66e5..be567e048f 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -28,7 +28,7 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' reorder-python-imports --exit-zero-even-if-changed --py39-plus **/*.py - - uses: install-pinned/yesqa@b7b1c5e133f5f516905ae35645a082f751ffa216 + - uses: install-pinned/yesqa@cf847492077eea907797bac45e68f87801d4de31 - name: Run yesqa run: | From f3ba7ce33cb19acd5c1d50453fb79bc184f62a87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Dec 2022 09:43:01 +0100 Subject: [PATCH 145/695] Bump install-pinned/reorder_python_imports (#5782) Bumps [install-pinned/reorder_python_imports](https://github.com/install-pinned/reorder_python_imports) from 97c3e89c53ae5513cc41716e876e26daff8bbdd6 to 7365755886d8ef6679e1b77457975a8698842af6. - [Release notes](https://github.com/install-pinned/reorder_python_imports/releases) - [Commits](https://github.com/install-pinned/reorder_python_imports/compare/97c3e89c53ae5513cc41716e876e26daff8bbdd6...7365755886d8ef6679e1b77457975a8698842af6) --- updated-dependencies: - dependency-name: install-pinned/reorder_python_imports dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index be567e048f..eb858fa07f 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -22,7 +22,7 @@ jobs: export GLOBIGNORE='mitmproxy/contrib/**' pyupgrade --exit-zero-even-if-changed --keep-runtime-typing --py39-plus **/*.py - - uses: install-pinned/reorder_python_imports@97c3e89c53ae5513cc41716e876e26daff8bbdd6 + - uses: install-pinned/reorder_python_imports@7365755886d8ef6679e1b77457975a8698842af6 - name: Run reorder-python-imports run: | shopt -s globstar From daa703c7402c18a497d1a89a6e25fbd6ada044d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Dec 2022 09:43:17 +0100 Subject: [PATCH 146/695] Bump TrueBrain/actions-flake8 from 2.1 to 2.2 (#5783) Bumps [TrueBrain/actions-flake8](https://github.com/TrueBrain/actions-flake8) from 2.1 to 2.2. - [Release notes](https://github.com/TrueBrain/actions-flake8/releases) - [Commits](https://github.com/TrueBrain/actions-flake8/compare/c2deca24d388aa5aedd6478332aa9df4600b5eac...c120815866a4bb260e23a2550dccee02d94a0385) --- updated-dependencies: - dependency-name: TrueBrain/actions-flake8 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 76fffb6da5..c519529f96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: .github/python-version.txt - - uses: TrueBrain/actions-flake8@c2deca24d388aa5aedd6478332aa9df4600b5eac # v2.1 + - uses: TrueBrain/actions-flake8@c120815866a4bb260e23a2550dccee02d94a0385 # v2.2 # mirrored at https://github.com/mitmproxy/mitmproxy/settings/actions lint-local: if: github.event_name == 'push' From 4099822e49f70dacee434e3d28881b827446caed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Dec 2022 09:43:31 +0100 Subject: [PATCH 147/695] Bump install-pinned/black (#5784) Bumps [install-pinned/black](https://github.com/install-pinned/black) from dde5aed720bc458e86d99144d4d0c1f6c8e08844 to 70a27391ba5875c09596f067be1e331d0e81947b. - [Release notes](https://github.com/install-pinned/black/releases) - [Commits](https://github.com/install-pinned/black/compare/dde5aed720bc458e86d99144d4d0c1f6c8e08844...70a27391ba5875c09596f067be1e331d0e81947b) --- updated-dependencies: - dependency-name: install-pinned/black dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index eb858fa07f..f0594f2f11 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -38,7 +38,7 @@ jobs: - uses: install-pinned/autoflake@32877f5112ce1c5b8b30cf57d70593a53d5fca87 - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - - uses: install-pinned/black@dde5aed720bc458e86d99144d4d0c1f6c8e08844 + - uses: install-pinned/black@70a27391ba5875c09596f067be1e331d0e81947b - run: black --extend-exclude mitmproxy/contrib . - uses: mhils/add-pr-ref-in-changelog@main From 689ca0c1b5a0680be31149aad87464b88440e63d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Dec 2022 09:43:41 +0100 Subject: [PATCH 148/695] Bump install-pinned/autoflake (#5785) Bumps [install-pinned/autoflake](https://github.com/install-pinned/autoflake) from 32877f5112ce1c5b8b30cf57d70593a53d5fca87 to 95c53f821b204037c1be14d45d810032e8ddfdcb. - [Release notes](https://github.com/install-pinned/autoflake/releases) - [Commits](https://github.com/install-pinned/autoflake/compare/32877f5112ce1c5b8b30cf57d70593a53d5fca87...95c53f821b204037c1be14d45d810032e8ddfdcb) --- updated-dependencies: - dependency-name: install-pinned/autoflake dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index f0594f2f11..503f1b6c3a 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -35,7 +35,7 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' yesqa **/*.py || true - - uses: install-pinned/autoflake@32877f5112ce1c5b8b30cf57d70593a53d5fca87 + - uses: install-pinned/autoflake@95c53f821b204037c1be14d45d810032e8ddfdcb - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - uses: install-pinned/black@70a27391ba5875c09596f067be1e331d0e81947b From bbb5080e982a6054bcd18568acd2ce5ae156a403 Mon Sep 17 00:00:00 2001 From: Sabin Dcoster Date: Sat, 3 Dec 2022 22:18:36 +0545 Subject: [PATCH 149/695] Add loop as a parameter in DumpMaster (#5790) * Add loop as a parameter in DumpMaster * [autofix.ci] apply automated fixes Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- mitmproxy/tools/dump.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 6bb269ee3c..1f6d62f4db 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -12,10 +12,11 @@ class DumpMaster(master.Master): def __init__( self, options: options.Options, + loop=None, with_termlog=True, with_dumper=True, ) -> None: - super().__init__(options) + super().__init__(options, event_loop=loop) if with_termlog: self.addons.add(termlog.TermLog()) self.addons.add(*addons.default_addons()) From 1536e537adf7847062b92fb4235a638a212b55e6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 15 Dec 2022 17:50:43 +0100 Subject: [PATCH 150/695] readme: fix nits --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 928374c0b3..c48cec2c15 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,6 @@ As an open source project, mitmproxy welcomes contributions of all forms. [![Dev Guide](https://shields.mitmproxy.org/badge/dev_docs-CONTRIBUTING.md-blue)](./CONTRIBUTING.md) -Also, please feel free to join our developer Slack! However, please note that the primary purpose of our Slack is direct communication between maintainers and contributors. If you have questions where the answer might be valuable to others, please use [GitHub Discussions](https://github.com/mitmproxy/mitmproxy/discussions). +Also, please feel free to join our developer Slack! However, please note that the primary purpose of our Slack is direct communication between maintainers and contributors. **If you have questions where the answer might be valuable to others, please use [GitHub Discussions](https://github.com/mitmproxy/mitmproxy/discussions) and not Slack.** [![Slack Developer Chat](https://shields.mitmproxy.org/badge/slack-mitmproxy-E01563.svg)](http://slack.mitmproxy.org/) From 12960c0494ba7558c3125cc921d174991cbde743 Mon Sep 17 00:00:00 2001 From: gpiechnik2 <48253270+gpiechnik2@users.noreply.github.com> Date: Sat, 17 Dec 2022 14:32:38 +0100 Subject: [PATCH 151/695] har: don't set `pages` --- examples/contrib/har_dump.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/contrib/har_dump.py b/examples/contrib/har_dump.py index 0a7d6faaee..bdd758ccb2 100644 --- a/examples/contrib/har_dump.py +++ b/examples/contrib/har_dump.py @@ -48,13 +48,11 @@ def configure(updated): "version": "0.1", "comment": "mitmproxy version %s" % version.MITMPROXY, }, - "pages": [{"pageTimings": {}}], + "pages": [], "entries": [], } } ) - # The `pages` attribute is needed for Firefox Dev Tools to load the HAR file. - # An empty value works fine. def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: From 2a3eca2589ffa3620654483eced02c9bf9dfa0bd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 2 Jan 2023 22:13:12 +0100 Subject: [PATCH 152/695] use latest codecov uploader (#5845) --- .github/workflows/main.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c519529f96..71ac16d349 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -105,11 +105,9 @@ jobs: # run tests with loopback only. We need to sudo for unshare, which means we need an absolute path for tox. sudo unshare --net -- sh -c "ip link set lo up; $(which tox) -e py" if: matrix.os == 'ubuntu-latest' - - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 - # mirrored below and at https://github.com/mitmproxy/mitmproxy/settings/actions + - uses: mhils/better-codecov-action@main with: - file: ./coverage.xml - name: ${{ matrix.os }} + arguments: '--file ./coverage.xml --name ${{ matrix.os }}' build: strategy: @@ -174,11 +172,9 @@ jobs: run: npm ci - working-directory: ./web run: npm test - - uses: codecov/codecov-action@a1ed4b322b4b38cb846afb5a0ebfa17086917d27 - # mirrored above and at https://github.com/mitmproxy/mitmproxy/settings/actions + - uses: mhils/better-codecov-action@main with: - file: ./web/coverage/coverage-final.json - name: web + arguments: '--file ./web/coverage/coverage-final.json' docs: runs-on: ubuntu-latest From f0650a6694665b7aacc30f1350d50540833b39ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:20:05 +0100 Subject: [PATCH 153/695] Bump install-pinned/black (#5839) Bumps [install-pinned/black](https://github.com/install-pinned/black) from 70a27391ba5875c09596f067be1e331d0e81947b to bcf144213c4943c1f2078a257fa566cebec36107. - [Release notes](https://github.com/install-pinned/black/releases) - [Commits](https://github.com/install-pinned/black/compare/70a27391ba5875c09596f067be1e331d0e81947b...bcf144213c4943c1f2078a257fa566cebec36107) --- updated-dependencies: - dependency-name: install-pinned/black dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 503f1b6c3a..ec53dd2085 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -38,7 +38,7 @@ jobs: - uses: install-pinned/autoflake@95c53f821b204037c1be14d45d810032e8ddfdcb - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - - uses: install-pinned/black@70a27391ba5875c09596f067be1e331d0e81947b + - uses: install-pinned/black@bcf144213c4943c1f2078a257fa566cebec36107 - run: black --extend-exclude mitmproxy/contrib . - uses: mhils/add-pr-ref-in-changelog@main From 979e789928c3b62719888a6958cea968df509af7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:20:14 +0100 Subject: [PATCH 154/695] Bump install-pinned/yesqa (#5841) Bumps [install-pinned/yesqa](https://github.com/install-pinned/yesqa) from cf847492077eea907797bac45e68f87801d4de31 to a1262fbe567d4c0b3445afade67b90f3bba2c9a2. - [Release notes](https://github.com/install-pinned/yesqa/releases) - [Commits](https://github.com/install-pinned/yesqa/compare/cf847492077eea907797bac45e68f87801d4de31...a1262fbe567d4c0b3445afade67b90f3bba2c9a2) --- updated-dependencies: - dependency-name: install-pinned/yesqa dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index ec53dd2085..9f580113ff 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -28,7 +28,7 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' reorder-python-imports --exit-zero-even-if-changed --py39-plus **/*.py - - uses: install-pinned/yesqa@cf847492077eea907797bac45e68f87801d4de31 + - uses: install-pinned/yesqa@a1262fbe567d4c0b3445afade67b90f3bba2c9a2 - name: Run yesqa run: | From e640fdfa2349b1c76fc418a9e3affce1b9e125d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:20:26 +0100 Subject: [PATCH 155/695] Bump install-pinned/reorder_python_imports (#5842) Bumps [install-pinned/reorder_python_imports](https://github.com/install-pinned/reorder_python_imports) from 7365755886d8ef6679e1b77457975a8698842af6 to 515035fd9eb355713f61dee238b17a04ce01f4d2. - [Release notes](https://github.com/install-pinned/reorder_python_imports/releases) - [Commits](https://github.com/install-pinned/reorder_python_imports/compare/7365755886d8ef6679e1b77457975a8698842af6...515035fd9eb355713f61dee238b17a04ce01f4d2) --- updated-dependencies: - dependency-name: install-pinned/reorder_python_imports dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 9f580113ff..234c859b80 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -22,7 +22,7 @@ jobs: export GLOBIGNORE='mitmproxy/contrib/**' pyupgrade --exit-zero-even-if-changed --keep-runtime-typing --py39-plus **/*.py - - uses: install-pinned/reorder_python_imports@7365755886d8ef6679e1b77457975a8698842af6 + - uses: install-pinned/reorder_python_imports@515035fd9eb355713f61dee238b17a04ce01f4d2 - name: Run reorder-python-imports run: | shopt -s globstar From e3f589740054af6a99ecd49d9113b36f54ceeb37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 22:23:55 +0100 Subject: [PATCH 156/695] Bump install-pinned/pyupgrade from 847ef2b8aa35a3817372540b887f4130d864d6b7 to 423622e7c2088eeba495a591385ec22074284f90 (#5840) * Bump install-pinned/pyupgrade Bumps [install-pinned/pyupgrade](https://github.com/install-pinned/pyupgrade) from 847ef2b8aa35a3817372540b887f4130d864d6b7 to 423622e7c2088eeba495a591385ec22074284f90. - [Release notes](https://github.com/install-pinned/pyupgrade/releases) - [Commits](https://github.com/install-pinned/pyupgrade/compare/847ef2b8aa35a3817372540b887f4130d864d6b7...423622e7c2088eeba495a591385ec22074284f90) --- updated-dependencies: - dependency-name: install-pinned/pyupgrade dependency-type: direct:production ... Signed-off-by: dependabot[bot] * [autofix.ci] apply automated fixes Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- mitmproxy/optmanager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 234c859b80..d8f81150ae 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: install-pinned/pyupgrade@847ef2b8aa35a3817372540b887f4130d864d6b7 + - uses: install-pinned/pyupgrade@423622e7c2088eeba495a591385ec22074284f90 - name: Run pyupgrade run: | shopt -s globstar diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 32d448323c..5fa10a787a 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -611,5 +611,5 @@ def save(opts: OptManager, path: str, defaults: bool = False) -> None: else: data = "" - with open(path, "wt", encoding="utf8") as f: + with open(path, "w", encoding="utf8") as f: serialize(opts, f, data, defaults) From 385633874cdf447f7a6ce67e47617558f97aa8fc Mon Sep 17 00:00:00 2001 From: Igor Talankin Date: Fri, 6 Jan 2023 20:49:54 +0500 Subject: [PATCH 157/695] Added a command for appending flows to server replay list (#5851) * Added a command to append flows to replay server list * Added changelog entry for `replay.server.add` * Update CHANGELOG.md Co-authored-by: Maximilian Hils --- CHANGELOG.md | 2 ++ mitmproxy/addons/serverplayback.py | 7 +++++++ test/mitmproxy/addons/test_serverplayback.py | 21 ++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64292246ea..fc72f5d1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ * ASGI/WSGI apps can now listen on all ports for a specific hostname. This makes it simpler to accept both HTTP and HTTPS. ([#5725](https://github.com/mitmproxy/mitmproxy/pull/5725), @mhils) +* Add `replay.server.add` command for adding flows to server replay buffer + ([#5851](https://github.com/mitmproxy/mitmproxy/pull/5851), @italankin) ### Breaking Changes diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 7419f8ac5f..6602cb5f80 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -114,6 +114,13 @@ def load_flows(self, flows: Sequence[flow.Flow]) -> None: Replay server responses from flows. """ self.flowmap = {} + self.add_flows(flows) + + @command.command("replay.server.add") + def add_flows(self, flows: Sequence[flow.Flow]) -> None: + """ + Add responses from flows to server replay list. + """ for f in flows: if isinstance(f, http.HTTPFlow): lst = self.flowmap.setdefault(self._hash(f), []) diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index 77d4f28367..b538909fc5 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -58,6 +58,27 @@ def test_server_playback(): assert not sp.flowmap +def test_add_flows(): + sp = serverplayback.ServerPlayback() + with taddons.context(sp) as tctx: + tctx.configure(sp) + f1 = tflow.tflow(resp=True) + f2 = tflow.tflow(resp=True) + + sp.load_flows([f1]) + sp.add_flows([f2]) + + assert sp.next_flow(f1) + assert sp.flowmap + assert sp.next_flow(f2) + assert not sp.flowmap + + sp.add_flows([f1]) + assert sp.flowmap + assert sp.next_flow(f1) + assert not sp.flowmap + + def test_ignore_host(): sp = serverplayback.ServerPlayback() with taddons.context(sp) as tctx: From d06afcc1f98a04065c656197768c2b6f3ef40d0d Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Fri, 6 Jan 2023 22:37:55 +0100 Subject: [PATCH 158/695] Fix tests after 2037 (#5852) Background: As part of my work on reproducible builds for openSUSE, I check that software still gives identical build results in the future. The usual offset is +16 years, because that is how long I expect some software will be used in some places. This showed up failing tests in our package build. See https://reproducible-builds.org/ for why this matters. --- test/mitmproxy/net/http/test_cookies.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index be5a57c76e..13b89d940e 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -170,15 +170,15 @@ def set_cookie_equal(obs, exp): ], ], [ - "foo=bar; expires=Mon, 24 Aug 2037", + "foo=bar; expires=Mon, 24 Aug 2133", [ - ("foo", "bar", (("expires", "Mon, 24 Aug 2037"),)), + ("foo", "bar", (("expires", "Mon, 24 Aug 2133"),)), ], ], [ - "foo=bar; expires=Mon, 24 Aug 2037 00:00:00 GMT, doo=dar", + "foo=bar; expires=Mon, 24 Aug 2133 00:00:00 GMT, doo=dar", [ - ("foo", "bar", (("expires", "Mon, 24 Aug 2037 00:00:00 GMT"),)), + ("foo", "bar", (("expires", "Mon, 24 Aug 2133 00:00:00 GMT"),)), ("doo", "dar", ()), ], ], @@ -200,13 +200,13 @@ def set_cookie_equal(obs, exp): def test_refresh_cookie(): # Invalid expires format, sent to us by Reddit. - c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/" + c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2133 23:59:59 GMT; Path=/" assert cookies.refresh_set_cookie_header(c, 60) c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure" assert "00:21:38" in cookies.refresh_set_cookie_header(c, 60) - c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037; Path=/" + c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2133; Path=/" assert "expires" not in cookies.refresh_set_cookie_header(c, 60) c = "foo,bar" @@ -238,7 +238,7 @@ def test_get_expiration_ts(*args): F = cookies.get_expiration_ts assert F(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT")])) == 0 - assert F(CA([("Expires", "Mon, 24-Aug-2037 00:00:00 GMT")])) == 2134684800 + assert F(CA([("Expires", "Mon, 24-Aug-2133 00:00:00 GMT")])) == 5164128000 assert F(CA([("Max-Age", "0")])) == now_ts assert F(CA([("Max-Age", "31")])) == now_ts + 31 @@ -259,10 +259,10 @@ def test_is_expired(): CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT"), ("Max-Age", "0")]) ) - assert not cookies.is_expired(CA([("Expires", "Mon, 24-Aug-2037 00:00:00 GMT")])) + assert not cookies.is_expired(CA([("Expires", "Mon, 24-Aug-2133 00:00:00 GMT")])) assert not cookies.is_expired(CA([("Max-Age", "1")])) assert not cookies.is_expired( - CA([("Expires", "Wed, 15-Jul-2037 00:00:00 GMT"), ("Max-Age", "1")]) + CA([("Expires", "Wed, 15-Jul-2133 00:00:00 GMT"), ("Max-Age", "1")]) ) assert not cookies.is_expired(CA([("Max-Age", "nan")])) From 2bf96678aefa22a738e2799d1ebce44a395a55b8 Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Fri, 13 Jan 2023 19:04:13 +1000 Subject: [PATCH 159/695] Adapt contrib/mitmproxywrapper.py to Python 3 strings (#5866) * Adapt mitmproxywrapper.py to Python 3 strings This fixes errors like this one: `TypeError: memoryview: a bytes-like object is required, not 'str'` * [autofix.ci] apply automated fixes Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- examples/contrib/mitmproxywrapper.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/contrib/mitmproxywrapper.py b/examples/contrib/mitmproxywrapper.py index a484ff386d..8c95c8702c 100644 --- a/examples/contrib/mitmproxywrapper.py +++ b/examples/contrib/mitmproxywrapper.py @@ -20,7 +20,9 @@ def __init__(self, port, extra_arguments=None): self.extra_arguments = extra_arguments def run_networksetup_command(self, *arguments): - return subprocess.check_output(["sudo", "networksetup"] + list(arguments)) + return subprocess.check_output( + ["sudo", "networksetup"] + list(arguments) + ).decode() def proxy_state_for_service(self, service): state = self.run_networksetup_command("-getwebproxy", service).splitlines() @@ -47,8 +49,8 @@ def interface_name_to_service_name_map(self): def run_command_with_input(self, command, input): popen = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - (stdout, stderr) = popen.communicate(input) - return stdout + (stdout, stderr) = popen.communicate(input.encode()) + return stdout.decode() def primary_interace_name(self): scutil_script = "get State:/Network/Global/IPv4\nd.show\n" From 47e06950695cccbef103043ffab80dc8ae129efb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 15 Jan 2023 17:36:32 +0100 Subject: [PATCH 160/695] Remove workaround for tornado 6.1 (#5870) * remove workaround for tornado 6.1 * remove patch references --- mitmproxy/contrib/tornado/__init__.py | 82 --------------------------- mitmproxy/tools/console/master.py | 2 - mitmproxy/tools/web/master.py | 2 - setup.py | 2 +- 4 files changed, 1 insertion(+), 87 deletions(-) delete mode 100644 mitmproxy/contrib/tornado/__init__.py diff --git a/mitmproxy/contrib/tornado/__init__.py b/mitmproxy/contrib/tornado/__init__.py deleted file mode 100644 index c7000a7524..0000000000 --- a/mitmproxy/contrib/tornado/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -SPDX-License-Identifier: Apache-2.0 - -Vendored partial copy of https://github.com/tornadoweb/tornado/blob/master/tornado/platform/asyncio.py @ e18ea03 -to fix https://github.com/tornadoweb/tornado/issues/3092. Can be removed once tornado >6.1 is out. -""" -import errno - -import select -import tornado -import tornado.platform.asyncio - - -def patch_tornado(): - if tornado.version != "6.1": - return - - def _run_select(self) -> None: - while True: - with self._select_cond: - while self._select_args is None and not self._closing_selector: - self._select_cond.wait() - if self._closing_selector: - return - assert self._select_args is not None - to_read, to_write = self._select_args - self._select_args = None - - # We use the simpler interface of the select module instead of - # the more stateful interface in the selectors module because - # this class is only intended for use on windows, where - # select.select is the only option. The selector interface - # does not have well-documented thread-safety semantics that - # we can rely on so ensuring proper synchronization would be - # tricky. - try: - # On windows, selecting on a socket for write will not - # return the socket when there is an error (but selecting - # for reads works). Also select for errors when selecting - # for writes, and merge the results. - # - # This pattern is also used in - # https://github.com/python/cpython/blob/v3.8.0/Lib/selectors.py#L312-L317 - rs, ws, xs = select.select(to_read, to_write, to_write) - ws = ws + xs - except OSError as e: - # After remove_reader or remove_writer is called, the file - # descriptor may subsequently be closed on the event loop - # thread. It's possible that this select thread hasn't - # gotten into the select system call by the time that - # happens in which case (at least on macOS), select may - # raise a "bad file descriptor" error. If we get that - # error, check and see if we're also being woken up by - # polling the waker alone. If we are, just return to the - # event loop and we'll get the updated set of file - # descriptors on the next iteration. Otherwise, raise the - # original error. - if e.errno == getattr(errno, "WSAENOTSOCK", errno.EBADF): - rs, _, _ = select.select([self._waker_r.fileno()], [], [], 0) - if rs: - ws = [] - else: - raise - else: - raise - - try: - self._real_loop.call_soon_threadsafe(self._handle_select, rs, ws) - except RuntimeError: - # "Event loop is closed". Swallow the exception for - # consistency with PollIOLoop (and logical consistency - # with the fact that we can't guarantee that an - # add_callback that completes without error will - # eventually execute). - pass - except AttributeError: - # ProactorEventLoop may raise this instead of RuntimeError - # if call_soon_threadsafe races with a call to close(). - # Swallow it too for consistency. - pass - - tornado.platform.asyncio.AddThreadSelectorEventLoop._run_select = _run_select diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 3bad840960..28cb52049b 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -23,7 +23,6 @@ from mitmproxy.addons import intercept from mitmproxy.addons import readfile from mitmproxy.addons import view -from mitmproxy.contrib.tornado import patch_tornado from mitmproxy.tools.console import consoleaddons from mitmproxy.tools.console import defaultkeys from mitmproxy.tools.console import keymap @@ -217,7 +216,6 @@ async def running(self) -> None: loop = asyncio.get_running_loop() if isinstance(loop, getattr(asyncio, "ProactorEventLoop", tuple())): - patch_tornado() # fix for https://bugs.python.org/issue37373 loop = AddThreadSelectorEventLoop(loop) # type: ignore self.loop = urwid.MainLoop( diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 3d0b59bc9d..7759de577b 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -17,7 +17,6 @@ from mitmproxy.addons import termlog from mitmproxy.addons import view from mitmproxy.addons.proxyserver import Proxyserver -from mitmproxy.contrib.tornado import patch_tornado from mitmproxy.tools.web import app from mitmproxy.tools.web import static_viewer from mitmproxy.tools.web import webaddons @@ -94,7 +93,6 @@ def _sig_servers_changed(self) -> None: ) async def running(self): - patch_tornado() # Register tornado with the current event loop tornado.ioloop.IOLoop.current() diff --git a/setup.py b/setup.py index 187a8bcdaa..9d525e1adf 100644 --- a/setup.py +++ b/setup.py @@ -94,7 +94,7 @@ "pyperclip>=1.6.0,<1.9", "ruamel.yaml>=0.16,<0.18", "sortedcontainers>=2.3,<2.5", - "tornado>=6.1,<7", + "tornado>=6.2,<7", "urwid>=2.1.1,<2.2", "wsproto>=1.0,<1.3", "publicsuffix2>=2.20190812,<3", From 8801bdd28b5f85c6431050040b5e1390f4600b2a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 15 Jan 2023 17:46:55 +0100 Subject: [PATCH 161/695] urwid -> urwid-mitmproxy (#5871) --- mitmproxy/contrib/urwid/__init__.py | 1 - mitmproxy/contrib/urwid/escape_patches.py | 254 ----- mitmproxy/contrib/urwid/raw_display.py | 1183 --------------------- mitmproxy/contrib/urwid/win32.py | 149 --- mitmproxy/tools/console/window.py | 8 +- setup.py | 2 +- 6 files changed, 2 insertions(+), 1595 deletions(-) delete mode 100644 mitmproxy/contrib/urwid/__init__.py delete mode 100644 mitmproxy/contrib/urwid/escape_patches.py delete mode 100644 mitmproxy/contrib/urwid/raw_display.py delete mode 100644 mitmproxy/contrib/urwid/win32.py diff --git a/mitmproxy/contrib/urwid/__init__.py b/mitmproxy/contrib/urwid/__init__.py deleted file mode 100644 index c83ecbdda8..0000000000 --- a/mitmproxy/contrib/urwid/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import escape_patches diff --git a/mitmproxy/contrib/urwid/escape_patches.py b/mitmproxy/contrib/urwid/escape_patches.py deleted file mode 100644 index ac17c3a422..0000000000 --- a/mitmproxy/contrib/urwid/escape_patches.py +++ /dev/null @@ -1,254 +0,0 @@ -# monkeypatch https://github.com/urwid/urwid/commit/e2423b5069f51d318ea1ac0f355a0efe5448f7eb into the urwid sources. -import urwid.escape - -if urwid.__version__ in ("2.1.1", "2.1.2"): - # fmt: off - urwid.escape.input_sequences = [ - ('[A','up'),('[B','down'),('[C','right'),('[D','left'), - ('[E','5'),('[F','end'),('[G','5'),('[H','home'), - - ('[1~','home'),('[2~','insert'),('[3~','delete'),('[4~','end'), - ('[5~','page up'),('[6~','page down'), - ('[7~','home'),('[8~','end'), - - ('[[A','f1'),('[[B','f2'),('[[C','f3'),('[[D','f4'),('[[E','f5'), - - ('[11~','f1'),('[12~','f2'),('[13~','f3'),('[14~','f4'), - ('[15~','f5'),('[17~','f6'),('[18~','f7'),('[19~','f8'), - ('[20~','f9'),('[21~','f10'),('[23~','f11'),('[24~','f12'), - ('[25~','f13'),('[26~','f14'),('[28~','f15'),('[29~','f16'), - ('[31~','f17'),('[32~','f18'),('[33~','f19'),('[34~','f20'), - - ('OA','up'),('OB','down'),('OC','right'),('OD','left'), - ('OH','home'),('OF','end'), - ('OP','f1'),('OQ','f2'),('OR','f3'),('OS','f4'), - ('Oo','/'),('Oj','*'),('Om','-'),('Ok','+'), - - ('[Z','shift tab'), - ('On', '.'), - - ('[200~', 'begin paste'), ('[201~', 'end paste'), - ] + [ - (prefix + letter, modifier + key) - for prefix, modifier in zip('O[', ('meta ', 'shift ')) - for letter, key in zip('abcd', ('up', 'down', 'right', 'left')) - ] + [ - ("[" + digit + symbol, modifier + key) - for modifier, symbol in zip(('shift ', 'meta '), '$^') - for digit, key in zip('235678', - ('insert', 'delete', 'page up', 'page down', 'home', 'end')) - ] + [ - ('O' + chr(ord('p')+n), str(n)) for n in range(10) - ] + [ - # modified cursor keys + home, end, 5 -- [#X and [1;#X forms - (prefix+digit+letter, urwid.escape.escape_modifier(digit) + key) - for prefix in ("[", "[1;") - for digit in "12345678" - for letter,key in zip("ABCDEFGH", - ('up','down','right','left','5','end','5','home')) - ] + [ - # modified F1-F4 keys -- O#X form - ("O"+digit+letter, urwid.escape.escape_modifier(digit) + key) - for digit in "12345678" - for letter,key in zip("PQRS",('f1','f2','f3','f4')) - ] + [ - # modified F1-F13 keys -- [XX;#~ form - ("["+str(num)+";"+digit+"~", urwid.escape.escape_modifier(digit) + key) - for digit in "12345678" - for num,key in zip( - (3,5,6,11,12,13,14,15,17,18,19,20,21,23,24,25,26,28,29,31,32,33,34), - ('delete', 'page up', 'page down', - 'f1','f2','f3','f4','f5','f6','f7','f8','f9','f10','f11', - 'f12','f13','f14','f15','f16','f17','f18','f19','f20')) - ] + [ - # mouse reporting (special handling done in KeyqueueTrie) - ('[M', 'mouse'), - - # mouse reporting for SGR 1006 - ('[<', 'sgrmouse'), - - # report status response - ('[0n', 'status ok') - ] - - - class KeyqueueTrie(object): - def __init__( self, sequences ): - self.data = {} - for s, result in sequences: - assert type(result) != dict - self.add(self.data, s, result) - - def add(self, root, s, result): - assert type(root) == dict, "trie conflict detected" - assert len(s) > 0, "trie conflict detected" - - if ord(s[0]) in root: - return self.add(root[ord(s[0])], s[1:], result) - if len(s)>1: - d = {} - root[ord(s[0])] = d - return self.add(d, s[1:], result) - root[ord(s)] = result - - def get(self, keys, more_available): - result = self.get_recurse(self.data, keys, more_available) - if not result: - result = self.read_cursor_position(keys, more_available) - return result - - def get_recurse(self, root, keys, more_available): - if type(root) != dict: - if root == "mouse": - return self.read_mouse_info(keys, - more_available) - elif root == "sgrmouse": - return self.read_sgrmouse_info (keys, more_available) - return (root, keys) - if not keys: - # get more keys - if more_available: - raise urwid.escape.MoreInputRequired() - return None - if keys[0] not in root: - return None - return self.get_recurse(root[keys[0]], keys[1:], more_available) - - def read_mouse_info(self, keys, more_available): - if len(keys) < 3: - if more_available: - raise urwid.escape.MoreInputRequired() - return None - - b = keys[0] - 32 - x, y = (keys[1] - 33)%256, (keys[2] - 33)%256 # supports 0-255 - - prefix = "" - if b & 4: prefix = prefix + "shift " - if b & 8: prefix = prefix + "meta " - if b & 16: prefix = prefix + "ctrl " - if (b & urwid.escape.MOUSE_MULTIPLE_CLICK_MASK)>>9 == 1: prefix = prefix + "double " - if (b & urwid.escape.MOUSE_MULTIPLE_CLICK_MASK)>>9 == 2: prefix = prefix + "triple " - - # 0->1, 1->2, 2->3, 64->4, 65->5 - button = ((b&64)//64*3) + (b & 3) + 1 - - if b & 3 == 3: - action = "release" - button = 0 - elif b & urwid.escape.MOUSE_RELEASE_FLAG: - action = "release" - elif b & urwid.escape.MOUSE_DRAG_FLAG: - action = "drag" - elif b & urwid.escape.MOUSE_MULTIPLE_CLICK_MASK: - action = "click" - else: - action = "press" - - return ( (prefix + "mouse " + action, button, x, y), keys[3:] ) - - def read_sgrmouse_info(self, keys, more_available): - # Helpful links: - # https://stackoverflow.com/questions/5966903/how-to-get-mousemove-and-mouseclick-in-bash - # http://invisible-island.net/xterm/ctlseqs/ctlseqs.pdf - - if not keys: - if more_available: - raise urwid.escape.MoreInputRequired() - return None - - value = '' - pos_m = 0 - found_m = False - for k in keys: - value = value + chr(k); - if ((k is ord('M')) or (k is ord('m'))): - found_m = True - break; - pos_m += 1 - if not found_m: - if more_available: - raise urwid.escape.MoreInputRequired() - return None - - (b, x, y) = value[:-1].split(';') - - # shift, meta, ctrl etc. is not communicated on my machine, so I - # can't and won't be able to add support for it. - # Double and triple clicks are not supported as well. They can be - # implemented by using a timer. This timer can check if the last - # registered click is below a certain threshold. This threshold - # is normally set in the operating system itself, so setting one - # here will cause an inconsistent behaviour. I do not plan to use - # that feature, so I won't implement it. - - button = ((int(b) & 64) // 64 * 3) + (int(b) & 3) + 1 - x = int(x) - 1 - y = int(y) - 1 - - if (value[-1] == 'M'): - if int(b) & urwid.escape.MOUSE_DRAG_FLAG: - action = "drag" - else: - action = "press" - else: - action = "release" - - return ( ("mouse " + action, button, x, y), keys[pos_m + 1:] ) - - - def read_cursor_position(self, keys, more_available): - """ - Interpret cursor position information being sent by the - user's terminal. Returned as ('cursor position', x, y) - where (x, y) == (0, 0) is the top left of the screen. - """ - if not keys: - if more_available: - raise urwid.escape.MoreInputRequired() - return None - if keys[0] != ord('['): - return None - # read y value - y = 0 - i = 1 - for k in keys[i:]: - i += 1 - if k == ord(';'): - if not y: - return None - break - if k < ord('0') or k > ord('9'): - return None - if not y and k == ord('0'): - return None - y = y * 10 + k - ord('0') - if not keys[i:]: - if more_available: - raise urwid.escape.MoreInputRequired() - return None - # read x value - x = 0 - for k in keys[i:]: - i += 1 - if k == ord('R'): - if not x: - return None - return (("cursor position", x-1, y-1), keys[i:]) - if k < ord('0') or k > ord('9'): - return None - if not x and k == ord('0'): - return None - x = x * 10 + k - ord('0') - if not keys[i:]: - if more_available: - raise urwid.escape.MoreInputRequired() - return None - - urwid.escape.KeyqueueTrie = KeyqueueTrie - urwid.escape.input_trie = KeyqueueTrie(urwid.escape.input_sequences) - - - ESC = urwid.escape.ESC - urwid.escape.MOUSE_TRACKING_ON = ESC+"[?1000h"+ESC+"[?1002h"+ESC+"[?1006h" - urwid.escape.MOUSE_TRACKING_OFF = ESC+"[?1006l"+ESC+"[?1002l"+ESC+"[?1000l" diff --git a/mitmproxy/contrib/urwid/raw_display.py b/mitmproxy/contrib/urwid/raw_display.py deleted file mode 100644 index 8a0aaa451d..0000000000 --- a/mitmproxy/contrib/urwid/raw_display.py +++ /dev/null @@ -1,1183 +0,0 @@ -#!/usr/bin/python -# -# Urwid raw display module -# Copyright (C) 2004-2009 Ian Ward -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# Urwid web site: http://excess.org/urwid/ - -from __future__ import division, print_function - -""" -Direct terminal UI implementation -""" - -import os -import select -import struct -import sys -import signal -import socket -import threading - - -if os.name == "nt": - IS_WINDOWS = True - from . import win32 - from ctypes import byref -else: - IS_WINDOWS = False - import fcntl - import termios - import tty - -from urwid import util -from urwid import escape -from urwid.display_common import BaseScreen, RealTerminal, \ - UPDATE_PALETTE_ENTRY, AttrSpec, UNPRINTABLE_TRANS_TABLE, \ - INPUT_DESCRIPTORS_CHANGED -from urwid import signals -from urwid.compat import PYTHON3, bytes, B - -from subprocess import Popen, PIPE - -STDIN = object() - - -class Screen(BaseScreen, RealTerminal): - def __init__(self, input=STDIN, output=sys.stdout): - """Initialize a screen that directly prints escape codes to an output - terminal. - """ - super(Screen, self).__init__() - self._pal_escape = {} - self._pal_attrspec = {} - signals.connect_signal(self, UPDATE_PALETTE_ENTRY, - self._on_update_palette_entry) - self.colors = 16 # FIXME: detect this - self.has_underline = True # FIXME: detect this - self._keyqueue = [] - self.prev_input_resize = 0 - self.set_input_timeouts() - self.screen_buf = None - self._screen_buf_canvas = None - self._resized = False - self.maxrow = None - self.gpm_mev = None - self.gpm_event_pending = False - self._mouse_tracking_enabled = False - self.last_bstate = 0 - self._setup_G1_done = False - self._rows_used = None - self._cy = 0 - self.term = os.environ.get('TERM', '') - self.fg_bright_is_bold = not self.term.startswith("xterm") - self.bg_bright_is_blink = (self.term == "linux") - self.back_color_erase = not self.term.startswith("screen") - self.register_palette_entry( None, 'default','default') - self._next_timeout = None - self.signal_handler_setter = signal.signal - - # Our connections to the world - self._term_output_file = output - if input is STDIN: - if IS_WINDOWS: - input, self._send_input = socket.socketpair() - else: - input = sys.stdin - self._term_input_file = input - - # pipe for signalling external event loops about resize events - self._resize_pipe_rd, self._resize_pipe_wr = socket.socketpair() - self._resize_pipe_rd.setblocking(False) - - def _input_fileno(self): - """Returns the fileno of the input stream, or None if it doesn't have one. A stream without a fileno can't participate in whatever. - """ - if hasattr(self._term_input_file, 'fileno'): - return self._term_input_file.fileno() - else: - return None - - def _on_update_palette_entry(self, name, *attrspecs): - # copy the attribute to a dictionary containing the escape seqences - a = attrspecs[{16:0,1:1,88:2,256:3,2**24:4}[self.colors]] - self._pal_attrspec[name] = a - self._pal_escape[name] = self._attrspec_to_escape(a) - - def set_input_timeouts(self, max_wait=None, complete_wait=0.125, - resize_wait=0.125): - """ - Set the get_input timeout values. All values are in floating - point numbers of seconds. - - max_wait -- amount of time in seconds to wait for input when - there is no input pending, wait forever if None - complete_wait -- amount of time in seconds to wait when - get_input detects an incomplete escape sequence at the - end of the available input - resize_wait -- amount of time in seconds to wait for more input - after receiving two screen resize requests in a row to - stop Urwid from consuming 100% cpu during a gradual - window resize operation - """ - self.max_wait = max_wait - if max_wait is not None: - if self._next_timeout is None: - self._next_timeout = max_wait - else: - self._next_timeout = min(self._next_timeout, self.max_wait) - self.complete_wait = complete_wait - self.resize_wait = resize_wait - - def _sigwinch_handler(self, signum, frame=None): - """ - frame -- will always be None when the GLib event loop is being used. - """ - if not self._resized: - self._resize_pipe_wr.send(B("R")) - self._resized = True - self.screen_buf = None - - def _sigcont_handler(self, signum, frame=None): - """ - frame -- will always be None when the GLib event loop is being used. - """ - - self.stop() - self.start() - self._sigwinch_handler(None, None) - - def signal_init(self): - """ - Called in the startup of run wrapper to set the SIGWINCH - and SIGCONT signal handlers. - - Override this function to call from main thread in threaded - applications. - """ - self.signal_handler_setter(signal.SIGWINCH, self._sigwinch_handler) - self.signal_handler_setter(signal.SIGCONT, self._sigcont_handler) - - def signal_restore(self): - """ - Called in the finally block of run wrapper to restore the - SIGWINCH and SIGCONT signal handlers. - - Override this function to call from main thread in threaded - applications. - """ - self.signal_handler_setter(signal.SIGCONT, signal.SIG_DFL) - self.signal_handler_setter(signal.SIGWINCH, signal.SIG_DFL) - - def set_mouse_tracking(self, enable=True): - """ - Enable (or disable) mouse tracking. - - After calling this function get_input will include mouse - click events along with keystrokes. - """ - enable = bool(enable) - if enable == self._mouse_tracking_enabled: - return - - self._mouse_tracking(enable) - self._mouse_tracking_enabled = enable - - def _mouse_tracking(self, enable): - if enable: - self.write(escape.MOUSE_TRACKING_ON) - self._start_gpm_tracking() - else: - self.write(escape.MOUSE_TRACKING_OFF) - self._stop_gpm_tracking() - - def _start_gpm_tracking(self): - if not os.path.isfile("/usr/bin/mev"): - return - if not os.environ.get('TERM',"").lower().startswith("linux"): - return - if not Popen: - return - m = Popen(["/usr/bin/mev","-e","158"], stdin=PIPE, stdout=PIPE, - close_fds=True) - fcntl.fcntl(m.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) - self.gpm_mev = m - - def _stop_gpm_tracking(self): - if not self.gpm_mev: - return - os.kill(self.gpm_mev.pid, signal.SIGINT) - os.waitpid(self.gpm_mev.pid, 0) - self.gpm_mev = None - - _dwOriginalOutMode = None - _dwOriginalInMode = None - - def _start(self, alternate_buffer=True): - """ - Initialize the screen and input mode. - - alternate_buffer -- use alternate screen buffer - """ - if alternate_buffer: - self.write(escape.SWITCH_TO_ALTERNATE_BUFFER) - self._rows_used = None - else: - self._rows_used = 0 - - fd = self._input_fileno() - if fd is not None and os.isatty(fd) and not IS_WINDOWS: - self._old_termios_settings = termios.tcgetattr(fd) - tty.setcbreak(fd) - - if IS_WINDOWS: - hOut = win32.GetStdHandle(win32.STD_OUTPUT_HANDLE) - hIn = win32.GetStdHandle(win32.STD_INPUT_HANDLE) - self._dwOriginalOutMode = win32.DWORD() - self._dwOriginalInMode = win32.DWORD() - win32.GetConsoleMode(hOut, byref(self._dwOriginalOutMode)) - win32.GetConsoleMode(hIn, byref(self._dwOriginalInMode)) - # TODO: Restore on exit - - dwOutMode = win32.DWORD( - self._dwOriginalOutMode.value | win32.ENABLE_VIRTUAL_TERMINAL_PROCESSING | win32.DISABLE_NEWLINE_AUTO_RETURN) - dwInMode = win32.DWORD( - self._dwOriginalInMode.value | win32.ENABLE_WINDOW_INPUT | win32.ENABLE_VIRTUAL_TERMINAL_INPUT - ) - - ok = win32.SetConsoleMode(hOut, dwOutMode) - if not ok: - raise RuntimeError("Error enabling virtual terminal processing, " - "mitmproxy's console interface requires Windows 10 Build 10586 or above.") - ok = win32.SetConsoleMode(hIn, dwInMode) - assert ok - else: - self.signal_init() - self._alternate_buffer = alternate_buffer - self._next_timeout = self.max_wait - - if not self._signal_keys_set and not IS_WINDOWS: - self._old_signal_keys = self.tty_signal_keys(fileno=fd) - - signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED) - # restore mouse tracking to previous state - self._mouse_tracking(self._mouse_tracking_enabled) - - return super(Screen, self)._start() - - def _stop(self): - """ - Restore the screen. - """ - self.clear() - - signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED) - - if not IS_WINDOWS: - self.signal_restore() - - fd = self._input_fileno() - if fd is not None and os.isatty(fd) and not IS_WINDOWS: - termios.tcsetattr(fd, termios.TCSADRAIN, self._old_termios_settings) - - self._mouse_tracking(False) - - move_cursor = "" - if self._alternate_buffer: - move_cursor = escape.RESTORE_NORMAL_BUFFER - elif self.maxrow is not None: - move_cursor = escape.set_cursor_position( - 0, self.maxrow) - self.write( - self._attrspec_to_escape(AttrSpec('','')) - + escape.SI - + move_cursor - + escape.SHOW_CURSOR) - self.flush() - - if self._old_signal_keys: - self.tty_signal_keys(*(self._old_signal_keys + (fd,))) - - if IS_WINDOWS: - hOut = win32.GetStdHandle(win32.STD_OUTPUT_HANDLE) - hIn = win32.GetStdHandle(win32.STD_INPUT_HANDLE) - ok = win32.SetConsoleMode(hOut, self._dwOriginalOutMode) - assert ok - ok = win32.SetConsoleMode(hIn, self._dwOriginalInMode) - assert ok - - super(Screen, self)._stop() - - - def write(self, data): - """Write some data to the terminal. - - You may wish to override this if you're using something other than - regular files for input and output. - """ - self._term_output_file.write(data) - - def flush(self): - """Flush the output buffer. - - You may wish to override this if you're using something other than - regular files for input and output. - """ - self._term_output_file.flush() - - def get_input(self, raw_keys=False): - """Return pending input as a list. - - raw_keys -- return raw keycodes as well as translated versions - - This function will immediately return all the input since the - last time it was called. If there is no input pending it will - wait before returning an empty list. The wait time may be - configured with the set_input_timeouts function. - - If raw_keys is False (default) this function will return a list - of keys pressed. If raw_keys is True this function will return - a ( keys pressed, raw keycodes ) tuple instead. - - Examples of keys returned: - - * ASCII printable characters: " ", "a", "0", "A", "-", "/" - * ASCII control characters: "tab", "enter" - * Escape sequences: "up", "page up", "home", "insert", "f1" - * Key combinations: "shift f1", "meta a", "ctrl b" - * Window events: "window resize" - - When a narrow encoding is not enabled: - - * "Extended ASCII" characters: "\\xa1", "\\xb2", "\\xfe" - - When a wide encoding is enabled: - - * Double-byte characters: "\\xa1\\xea", "\\xb2\\xd4" - - When utf8 encoding is enabled: - - * Unicode characters: u"\\u00a5", u'\\u253c" - - Examples of mouse events returned: - - * Mouse button press: ('mouse press', 1, 15, 13), - ('meta mouse press', 2, 17, 23) - * Mouse drag: ('mouse drag', 1, 16, 13), - ('mouse drag', 1, 17, 13), - ('ctrl mouse drag', 1, 18, 13) - * Mouse button release: ('mouse release', 0, 18, 13), - ('ctrl mouse release', 0, 17, 23) - """ - assert self._started - - self._wait_for_input_ready(self._next_timeout) - keys, raw = self.parse_input(None, None, self.get_available_raw_input()) - - # Avoid pegging CPU at 100% when slowly resizing - if keys==['window resize'] and self.prev_input_resize: - while True: - self._wait_for_input_ready(self.resize_wait) - keys, raw2 = self.parse_input(None, None, self.get_available_raw_input()) - raw += raw2 - #if not keys: - # keys, raw2 = self._get_input( - # self.resize_wait) - # raw += raw2 - if keys!=['window resize']: - break - if keys[-1:]!=['window resize']: - keys.append('window resize') - - if keys==['window resize']: - self.prev_input_resize = 2 - elif self.prev_input_resize == 2 and not keys: - self.prev_input_resize = 1 - else: - self.prev_input_resize = 0 - - if raw_keys: - return keys, raw - return keys - - def get_input_descriptors(self): - """ - Return a list of integer file descriptors that should be - polled in external event loops to check for user input. - - Use this method if you are implementing your own event loop. - - This method is only called by `hook_event_loop`, so if you override - that, you can safely ignore this. - """ - if not self._started: - return [] - - fd_list = [self._resize_pipe_rd] - fd = self._input_fileno() - if fd is not None: - fd_list.append(fd) - if self.gpm_mev is not None: - fd_list.append(self.gpm_mev.stdout.fileno()) - return fd_list - - _current_event_loop_handles = () - - def unhook_event_loop(self, event_loop): - """ - Remove any hooks added by hook_event_loop. - """ - if self._input_thread is not None: - self._input_thread.should_exit = True - self._input_thread = None - - for handle in self._current_event_loop_handles: - try: - event_loop.remove_watch_file(handle) - except KeyError: - pass - - if self._input_timeout: - event_loop.remove_alarm(self._input_timeout) - self._input_timeout = None - - def hook_event_loop(self, event_loop, callback): - """ - Register the given callback with the event loop, to be called with new - input whenever it's available. The callback should be passed a list of - processed keys and a list of unprocessed keycodes. - - Subclasses may wish to use parse_input to wrap the callback. - """ - if IS_WINDOWS: - self._input_thread = ReadInputThread(self._send_input, lambda: self._sigwinch_handler(0)) - self._input_thread.start() - if hasattr(self, 'get_input_nonblocking'): - wrapper = self._make_legacy_input_wrapper(event_loop, callback) - else: - wrapper = lambda: self.parse_input( - event_loop, callback, self.get_available_raw_input()) - fds = self.get_input_descriptors() - handles = [event_loop.watch_file(fd, wrapper) for fd in fds] - self._current_event_loop_handles = handles - - _input_thread = None - _input_timeout = None - _partial_codes = None - - def _make_legacy_input_wrapper(self, event_loop, callback): - """ - Support old Screen classes that still have a get_input_nonblocking and - expect it to work. - """ - def wrapper(): - if self._input_timeout: - event_loop.remove_alarm(self._input_timeout) - self._input_timeout = None - timeout, keys, raw = self.get_input_nonblocking() - if timeout is not None: - self._input_timeout = event_loop.alarm(timeout, wrapper) - - callback(keys, raw) - - return wrapper - - def get_available_raw_input(self): - """ - Return any currently-available input. Does not block. - - This method is only used by the default `hook_event_loop` - implementation; you can safely ignore it if you implement your own. - """ - codes = self._get_gpm_codes() + self._get_keyboard_codes() - - if self._partial_codes: - codes = self._partial_codes + codes - self._partial_codes = None - - # clean out the pipe used to signal external event loops - # that a resize has occurred - try: - while True: self._resize_pipe_rd.recv(1) - except OSError: - pass - - return codes - - def parse_input(self, event_loop, callback, codes, wait_for_more=True): - """ - Read any available input from get_available_raw_input, parses it into - keys, and calls the given callback. - - The current implementation tries to avoid any assumptions about what - the screen or event loop look like; it only deals with parsing keycodes - and setting a timeout when an incomplete one is detected. - - `codes` should be a sequence of keycodes, i.e. bytes. A bytearray is - appropriate, but beware of using bytes, which only iterates as integers - on Python 3. - """ - # Note: event_loop may be None for 100% synchronous support, only used - # by get_input. Not documented because you shouldn't be doing it. - if self._input_timeout and event_loop: - event_loop.remove_alarm(self._input_timeout) - self._input_timeout = None - - original_codes = codes - processed = [] - try: - while codes: - run, codes = escape.process_keyqueue( - codes, wait_for_more) - processed.extend(run) - except escape.MoreInputRequired: - # Set a timer to wait for the rest of the input; if it goes off - # without any new input having come in, use the partial input - k = len(original_codes) - len(codes) - processed_codes = original_codes[:k] - self._partial_codes = codes - - def _parse_incomplete_input(): - self._input_timeout = None - self._partial_codes = None - self.parse_input( - event_loop, callback, codes, wait_for_more=False) - if event_loop: - self._input_timeout = event_loop.alarm( - self.complete_wait, _parse_incomplete_input) - - else: - processed_codes = original_codes - self._partial_codes = None - - if self._resized: - processed.append('window resize') - self._resized = False - - if callback: - callback(processed, processed_codes) - else: - # For get_input - return processed, processed_codes - - def _get_keyboard_codes(self): - codes = [] - while True: - code = self._getch_nodelay() - if code < 0: - break - codes.append(code) - return codes - - def _get_gpm_codes(self): - codes = [] - try: - while self.gpm_mev is not None and self.gpm_event_pending: - codes.extend(self._encode_gpm_event()) - except IOError as e: - if e.args[0] != 11: - raise - return codes - - def _wait_for_input_ready(self, timeout): - ready = None - fd_list = [] - fd = self._input_fileno() - if fd is not None: - fd_list.append(fd) - if self.gpm_mev is not None: - fd_list.append(self.gpm_mev.stdout.fileno()) - while True: - try: - if timeout is None: - ready,w,err = select.select( - fd_list, [], fd_list) - else: - ready,w,err = select.select( - fd_list,[],fd_list, timeout) - break - except select.error as e: - if e.args[0] != 4: - raise - if self._resized: - ready = [] - break - return ready - - def _getch(self, timeout): - ready = self._wait_for_input_ready(timeout) - if self.gpm_mev is not None: - if self.gpm_mev.stdout.fileno() in ready: - self.gpm_event_pending = True - fd = self._input_fileno() - if fd is not None and fd in ready: - if IS_WINDOWS: - return ord(self._term_input_file.recv(1)) - else: - return ord(os.read(fd, 1)) - return -1 - - def _encode_gpm_event( self ): - self.gpm_event_pending = False - s = self.gpm_mev.stdout.readline().decode('ascii') - l = s.split(",") - if len(l) != 6: - # unexpected output, stop tracking - self._stop_gpm_tracking() - signals.emit_signal(self, INPUT_DESCRIPTORS_CHANGED) - return [] - ev, x, y, ign, b, m = s.split(",") - ev = int( ev.split("x")[-1], 16) - x = int( x.split(" ")[-1] ) - y = int( y.lstrip().split(" ")[0] ) - b = int( b.split(" ")[-1] ) - m = int( m.split("x")[-1].rstrip(), 16 ) - - # convert to xterm-like escape sequence - - last = next = self.last_bstate - l = [] - - mod = 0 - if m & 1: mod |= 4 # shift - if m & 10: mod |= 8 # alt - if m & 4: mod |= 16 # ctrl - - def append_button( b ): - b |= mod - l.extend([ 27, ord('['), ord('M'), b+32, x+32, y+32 ]) - - def determine_button_release( flag ): - if b & 4 and last & 1: - append_button( 0 + flag ) - next |= 1 - if b & 2 and last & 2: - append_button( 1 + flag ) - next |= 2 - if b & 1 and last & 4: - append_button( 2 + flag ) - next |= 4 - - if ev == 20 or ev == 36 or ev == 52: # press - if b & 4 and last & 1 == 0: - append_button( 0 ) - next |= 1 - if b & 2 and last & 2 == 0: - append_button( 1 ) - next |= 2 - if b & 1 and last & 4 == 0: - append_button( 2 ) - next |= 4 - elif ev == 146: # drag - if b & 4: - append_button( 0 + escape.MOUSE_DRAG_FLAG ) - elif b & 2: - append_button( 1 + escape.MOUSE_DRAG_FLAG ) - elif b & 1: - append_button( 2 + escape.MOUSE_DRAG_FLAG ) - else: # release - if b & 4 and last & 1: - append_button( 0 + escape.MOUSE_RELEASE_FLAG ) - next &= ~ 1 - if b & 2 and last & 2: - append_button( 1 + escape.MOUSE_RELEASE_FLAG ) - next &= ~ 2 - if b & 1 and last & 4: - append_button( 2 + escape.MOUSE_RELEASE_FLAG ) - next &= ~ 4 - if ev == 40: # double click (release) - if b & 4 and last & 1: - append_button( 0 + escape.MOUSE_MULTIPLE_CLICK_FLAG ) - if b & 2 and last & 2: - append_button( 1 + escape.MOUSE_MULTIPLE_CLICK_FLAG ) - if b & 1 and last & 4: - append_button( 2 + escape.MOUSE_MULTIPLE_CLICK_FLAG ) - elif ev == 52: - if b & 4 and last & 1: - append_button( 0 + escape.MOUSE_MULTIPLE_CLICK_FLAG*2 ) - if b & 2 and last & 2: - append_button( 1 + escape.MOUSE_MULTIPLE_CLICK_FLAG*2 ) - if b & 1 and last & 4: - append_button( 2 + escape.MOUSE_MULTIPLE_CLICK_FLAG*2 ) - - self.last_bstate = next - return l - - def _getch_nodelay(self): - return self._getch(0) - - - def get_cols_rows(self): - """Return the terminal dimensions (num columns, num rows).""" - y, x = 24, 80 - try: - if hasattr(self._term_output_file, 'fileno'): - if IS_WINDOWS: - assert self._term_output_file == sys.stdout - handle = win32.GetStdHandle(win32.STD_OUTPUT_HANDLE) - info = win32.CONSOLE_SCREEN_BUFFER_INFO() - ok = win32.GetConsoleScreenBufferInfo(handle, byref(info)) - if ok == 0: - raise IOError() - y, x = info.dwSize.Y, info.dwSize.X - else: - buf = fcntl.ioctl(self._term_output_file.fileno(), - termios.TIOCGWINSZ, ' '*4) - y, x = struct.unpack('hh', buf) - except IOError: - # Term size could not be determined - pass - self.maxrow = y - return x, y - - def _setup_G1(self): - """ - Initialize the G1 character set to graphics mode if required. - """ - if self._setup_G1_done: - return - - while True: - try: - self.write(escape.DESIGNATE_G1_SPECIAL) - self.flush() - break - except IOError: - pass - self._setup_G1_done = True - - - def draw_screen(self, maxres, r ): - """Paint screen with rendered canvas.""" - - (maxcol, maxrow) = maxres - - assert self._started - - assert maxrow == r.rows() - - # quick return if nothing has changed - if self.screen_buf and r is self._screen_buf_canvas: - return - - self._setup_G1() - - if self._resized: - # handle resize before trying to draw screen - return - - o = [escape.HIDE_CURSOR, self._attrspec_to_escape(AttrSpec('',''))] - - def partial_display(): - # returns True if the screen is in partial display mode - # ie. only some rows belong to the display - return self._rows_used is not None - - if not partial_display(): - o.append(escape.CURSOR_HOME) - - if self.screen_buf: - osb = self.screen_buf - else: - osb = [] - sb = [] - cy = self._cy - y = -1 - - def set_cursor_home(): - if not partial_display(): - return escape.set_cursor_position(0, 0) - return (escape.CURSOR_HOME_COL + - escape.move_cursor_up(cy)) - - def set_cursor_row(y): - if not partial_display(): - return escape.set_cursor_position(0, y) - return escape.move_cursor_down(y - cy) - - def set_cursor_position(x, y): - if not partial_display(): - return escape.set_cursor_position(x, y) - if cy > y: - return ('\b' + escape.CURSOR_HOME_COL + - escape.move_cursor_up(cy - y) + - escape.move_cursor_right(x)) - return ('\b' + escape.CURSOR_HOME_COL + - escape.move_cursor_down(y - cy) + - escape.move_cursor_right(x)) - - def is_blank_row(row): - if len(row) > 1: - return False - if row[0][2].strip(): - return False - return True - - def attr_to_escape(a): - if a in self._pal_escape: - return self._pal_escape[a] - elif isinstance(a, AttrSpec): - return self._attrspec_to_escape(a) - # undefined attributes use default/default - # TODO: track and report these - return self._attrspec_to_escape( - AttrSpec('default','default')) - - def using_standout_or_underline(a): - a = self._pal_attrspec.get(a, a) - return isinstance(a, AttrSpec) and (a.standout or a.underline) - - ins = None - o.append(set_cursor_home()) - cy = 0 - for row in r.content(): - y += 1 - if osb and y < len(osb) and osb[y] == row: - # this row of the screen buffer matches what is - # currently displayed, so we can skip this line - sb.append( osb[y] ) - continue - - sb.append(row) - - # leave blank lines off display when we are using - # the default screen buffer (allows partial screen) - if partial_display() and y > self._rows_used: - if is_blank_row(row): - continue - self._rows_used = y - - if y or partial_display(): - o.append(set_cursor_position(0, y)) - # after updating the line we will be just over the - # edge, but terminals still treat this as being - # on the same line - cy = y - - whitespace_at_end = False - if row: - a, cs, run = row[-1] - if (run[-1:] == B(' ') and self.back_color_erase - and not using_standout_or_underline(a)): - whitespace_at_end = True - row = row[:-1] + [(a, cs, run.rstrip(B(' ')))] - elif y == maxrow-1 and maxcol > 1: - row, back, ins = self._last_row(row) - - first = True - lasta = lastcs = None - for (a,cs, run) in row: - assert isinstance(run, bytes) # canvases should render with bytes - if cs != 'U': - run = run.translate(UNPRINTABLE_TRANS_TABLE) - if first or lasta != a: - o.append(attr_to_escape(a)) - lasta = a - if first or lastcs != cs: - assert cs in [None, "0", "U"], repr(cs) - if lastcs == "U": - o.append( escape.IBMPC_OFF ) - - if cs is None: - o.append( escape.SI ) - elif cs == "U": - o.append( escape.IBMPC_ON ) - else: - o.append( escape.SO ) - lastcs = cs - o.append( run ) - first = False - if ins: - (inserta, insertcs, inserttext) = ins - ias = attr_to_escape(inserta) - assert insertcs in [None, "0", "U"], repr(insertcs) - if cs is None: - icss = escape.SI - elif cs == "U": - icss = escape.IBMPC_ON - else: - icss = escape.SO - o += [ "\x08"*back, - ias, icss, - escape.INSERT_ON, inserttext, - escape.INSERT_OFF ] - - if cs == "U": - o.append(escape.IBMPC_OFF) - if whitespace_at_end: - o.append(escape.ERASE_IN_LINE_RIGHT) - - if r.cursor is not None: - x,y = r.cursor - o += [set_cursor_position(x, y), - escape.SHOW_CURSOR ] - self._cy = y - - if self._resized: - # handle resize before trying to draw screen - return - try: - for l in o: - if isinstance(l, bytes) and PYTHON3: - l = l.decode('utf-8', 'replace') - self.write(l) - self.flush() - except IOError as e: - # ignore interrupted syscall - if e.args[0] != 4: - raise - - self.screen_buf = sb - self._screen_buf_canvas = r - - - def _last_row(self, row): - """On the last row we need to slide the bottom right character - into place. Calculate the new line, attr and an insert sequence - to do that. - - eg. last row: - XXXXXXXXXXXXXXXXXXXXYZ - - Y will be drawn after Z, shifting Z into position. - """ - - new_row = row[:-1] - z_attr, z_cs, last_text = row[-1] - last_cols = util.calc_width(last_text, 0, len(last_text)) - last_offs, z_col = util.calc_text_pos(last_text, 0, - len(last_text), last_cols-1) - if last_offs == 0: - z_text = last_text - del new_row[-1] - # we need another segment - y_attr, y_cs, nlast_text = row[-2] - nlast_cols = util.calc_width(nlast_text, 0, - len(nlast_text)) - z_col += nlast_cols - nlast_offs, y_col = util.calc_text_pos(nlast_text, 0, - len(nlast_text), nlast_cols-1) - y_text = nlast_text[nlast_offs:] - if nlast_offs: - new_row.append((y_attr, y_cs, - nlast_text[:nlast_offs])) - else: - z_text = last_text[last_offs:] - y_attr, y_cs = z_attr, z_cs - nlast_cols = util.calc_width(last_text, 0, - last_offs) - nlast_offs, y_col = util.calc_text_pos(last_text, 0, - last_offs, nlast_cols-1) - y_text = last_text[nlast_offs:last_offs] - if nlast_offs: - new_row.append((y_attr, y_cs, - last_text[:nlast_offs])) - - new_row.append((z_attr, z_cs, z_text)) - return new_row, z_col-y_col, (y_attr, y_cs, y_text) - - - - def clear(self): - """ - Force the screen to be completely repainted on the next - call to draw_screen(). - """ - self.screen_buf = None - self.setup_G1 = True - - - def _attrspec_to_escape(self, a): - """ - Convert AttrSpec instance a to an escape sequence for the terminal - - >>> s = Screen() - >>> s.set_terminal_properties(colors=256) - >>> a2e = s._attrspec_to_escape - >>> a2e(s.AttrSpec('brown', 'dark green')) - '\\x1b[0;33;42m' - >>> a2e(s.AttrSpec('#fea,underline', '#d0d')) - '\\x1b[0;38;5;229;4;48;5;164m' - """ - if self.term == 'fbterm': - fg = escape.ESC + '[1;%d}' % (a.foreground_number,) - bg = escape.ESC + '[2;%d}' % (a.background_number,) - return fg + bg - - if a.foreground_true: - fg = "38;2;%d;%d;%d" %(a.get_rgb_values()[0:3]) - elif a.foreground_high: - fg = "38;5;%d" % a.foreground_number - elif a.foreground_basic: - if a.foreground_number > 7: - if self.fg_bright_is_bold: - fg = "1;%d" % (a.foreground_number - 8 + 30) - else: - fg = "%d" % (a.foreground_number - 8 + 90) - else: - fg = "%d" % (a.foreground_number + 30) - else: - fg = "39" - st = ("1;" * a.bold + "3;" * a.italics + - "4;" * a.underline + "5;" * a.blink + - "7;" * a.standout + "9;" * a.strikethrough) - if a.background_true: - bg = "48;2;%d;%d;%d" %(a.get_rgb_values()[3:6]) - elif a.background_high: - bg = "48;5;%d" % a.background_number - elif a.background_basic: - if a.background_number > 7: - if self.bg_bright_is_blink: - bg = "5;%d" % (a.background_number - 8 + 40) - else: - # this doesn't work on most terminals - bg = "%d" % (a.background_number - 8 + 100) - else: - bg = "%d" % (a.background_number + 40) - else: - bg = "49" - return escape.ESC + "[0;%s;%s%sm" % (fg, st, bg) - - - def set_terminal_properties(self, colors=None, bright_is_bold=None, - has_underline=None): - """ - colors -- number of colors terminal supports (1, 16, 88, 256, or 2**24) - or None to leave unchanged - bright_is_bold -- set to True if this terminal uses the bold - setting to create bright colors (numbers 8-15), set to False - if this Terminal can create bright colors without bold or - None to leave unchanged - has_underline -- set to True if this terminal can use the - underline setting, False if it cannot or None to leave - unchanged - """ - if colors is None: - colors = self.colors - if bright_is_bold is None: - bright_is_bold = self.fg_bright_is_bold - if has_underline is None: - has_underline = self.has_underline - - if colors == self.colors and bright_is_bold == self.fg_bright_is_bold \ - and has_underline == self.has_underline: - return - - self.colors = colors - self.fg_bright_is_bold = bright_is_bold - self.has_underline = has_underline - - self.clear() - self._pal_escape = {} - for p,v in self._palette.items(): - self._on_update_palette_entry(p, *v) - - - - def reset_default_terminal_palette(self): - """ - Attempt to set the terminal palette to default values as taken - from xterm. Uses number of colors from current - set_terminal_properties() screen setting. - """ - if self.colors == 1: - return - elif self.colors == 2**24: - colors = 256 - else: - colors = self.colors - - def rgb_values(n): - if colors == 16: - aspec = AttrSpec("h%d"%n, "", 256) - else: - aspec = AttrSpec("h%d"%n, "", colors) - return aspec.get_rgb_values()[:3] - - entries = [(n,) + rgb_values(n) for n in range(min(colors, 256))] - self.modify_terminal_palette(entries) - - - def modify_terminal_palette(self, entries): - """ - entries - list of (index, red, green, blue) tuples. - - Attempt to set part of the terminal palette (this does not work - on all terminals.) The changes are sent as a single escape - sequence so they should all take effect at the same time. - - 0 <= index < 256 (some terminals will only have 16 or 88 colors) - 0 <= red, green, blue < 256 - """ - - if self.term == 'fbterm': - modify = ["%d;%d;%d;%d" % (index, red, green, blue) - for index, red, green, blue in entries] - self.write("\x1b[3;"+";".join(modify)+"}") - else: - modify = ["%d;rgb:%02x/%02x/%02x" % (index, red, green, blue) - for index, red, green, blue in entries] - self.write("\x1b]4;"+";".join(modify)+"\x1b\\") - self.flush() - - - # shortcut for creating an AttrSpec with this screen object's - # number of colors - AttrSpec = lambda self, fg, bg: AttrSpec(fg, bg, self.colors) - - -class ReadInputThread(threading.Thread): - name = "urwid Windows input reader" - daemon = True - should_exit: bool = False - _input: socket.socket - - def __init__(self, input, resize): - self._input = input - self._resize = resize - super().__init__() - - def run(self) -> None: - hIn = win32.GetStdHandle(win32.STD_INPUT_HANDLE) - MAX = 2048 - - read = win32.DWORD(0) - arrtype = win32.INPUT_RECORD * MAX - input_records = arrtype() - - while True: - win32.ReadConsoleInputW(hIn, byref(input_records), MAX, byref(read)) - if self.should_exit: - return - for i in range(read.value): - inp = input_records[i] - if inp.EventType == win32.EventType.KEY_EVENT: - if not inp.Event.KeyEvent.bKeyDown: - continue - self._input.send(inp.Event.KeyEvent.uChar.UnicodeChar.encode("utf8")) - elif inp.EventType == win32.EventType.WINDOW_BUFFER_SIZE_EVENT: - self._resize() - else: - pass # TODO: handle mouse events - - -def _test(): - import doctest - doctest.testmod() - -if __name__=='__main__': - _test() diff --git a/mitmproxy/contrib/urwid/win32.py b/mitmproxy/contrib/urwid/win32.py deleted file mode 100644 index 581fcdfd2d..0000000000 --- a/mitmproxy/contrib/urwid/win32.py +++ /dev/null @@ -1,149 +0,0 @@ -from ctypes import Structure, Union, windll, POINTER -from ctypes.wintypes import BOOL, DWORD, WCHAR, WORD, SHORT, UINT, HANDLE, LPDWORD, CHAR - -# https://docs.microsoft.com/de-de/windows/console/getstdhandle -STD_INPUT_HANDLE = -10 -STD_OUTPUT_HANDLE = -11 -STD_ERROR_HANDLE = -12 - -# https://docs.microsoft.com/de-de/windows/console/setconsolemode -ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 -DISABLE_NEWLINE_AUTO_RETURN = 0x0008 -ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200 -ENABLE_WINDOW_INPUT = 0x0008 - - -class COORD(Structure): - """https://docs.microsoft.com/en-us/windows/console/coord-str""" - - _fields_ = [ - ("X", SHORT), - ("Y", SHORT), - ] - - -class SMALL_RECT(Structure): - """https://docs.microsoft.com/en-us/windows/console/small-rect-str""" - - _fields_ = [ - ("Left", SHORT), - ("Top", SHORT), - ("Right", SHORT), - ("Bottom", SHORT), - ] - - -class CONSOLE_SCREEN_BUFFER_INFO(Structure): - """https://docs.microsoft.com/en-us/windows/console/console-screen-buffer-info-str""" - - _fields_ = [ - ("dwSize", COORD), - ("dwCursorPosition", COORD), - ("wAttributes", WORD), - ("srWindow", SMALL_RECT), - ("dwMaximumWindowSize", COORD), - ] - - -class uChar(Union): - """https://docs.microsoft.com/en-us/windows/console/key-event-record-str""" - _fields_ = [ - ("AsciiChar", CHAR), - ("UnicodeChar", WCHAR), - ] - - -class KEY_EVENT_RECORD(Structure): - """https://docs.microsoft.com/en-us/windows/console/key-event-record-str""" - - _fields_ = [ - ("bKeyDown", BOOL), - ("wRepeatCount", WORD), - ("wVirtualKeyCode", WORD), - ("wVirtualScanCode", WORD), - ("uChar", uChar), - ("dwControlKeyState", DWORD), - ] - - -class MOUSE_EVENT_RECORD(Structure): - """https://docs.microsoft.com/en-us/windows/console/mouse-event-record-str""" - - _fields_ = [ - ("dwMousePosition", COORD), - ("dwButtonState", DWORD), - ("dwControlKeyState", DWORD), - ("dwEventFlags", DWORD), - ] - - -class WINDOW_BUFFER_SIZE_RECORD(Structure): - """https://docs.microsoft.com/en-us/windows/console/window-buffer-size-record-str""" - - _fields_ = [("dwSize", COORD)] - - -class MENU_EVENT_RECORD(Structure): - """https://docs.microsoft.com/en-us/windows/console/menu-event-record-str""" - - _fields_ = [("dwCommandId", UINT)] - - -class FOCUS_EVENT_RECORD(Structure): - """https://docs.microsoft.com/en-us/windows/console/focus-event-record-str""" - - _fields_ = [("bSetFocus", BOOL)] - - -class Event(Union): - """https://docs.microsoft.com/en-us/windows/console/input-record-str""" - _fields_ = [ - ("KeyEvent", KEY_EVENT_RECORD), - ("MouseEvent", MOUSE_EVENT_RECORD), - ("WindowBufferSizeEvent", WINDOW_BUFFER_SIZE_RECORD), - ("MenuEvent", MENU_EVENT_RECORD), - ("FocusEvent", FOCUS_EVENT_RECORD), - ] - - -class INPUT_RECORD(Structure): - """https://docs.microsoft.com/en-us/windows/console/input-record-str""" - - _fields_ = [ - ("EventType", WORD), - ("Event", Event) - ] - - -class EventType: - FOCUS_EVENT = 0x0010 - KEY_EVENT = 0x0001 - MENU_EVENT = 0x0008 - MOUSE_EVENT = 0x0002 - WINDOW_BUFFER_SIZE_EVENT = 0x0004 - - -# https://docs.microsoft.com/de-de/windows/console/getstdhandle -GetStdHandle = windll.kernel32.GetStdHandle -GetStdHandle.argtypes = [DWORD] -GetStdHandle.restype = HANDLE - -# https://docs.microsoft.com/de-de/windows/console/getconsolemode -GetConsoleMode = windll.kernel32.GetConsoleMode -GetConsoleMode.argtypes = [HANDLE, LPDWORD] -GetConsoleMode.restype = BOOL - -# https://docs.microsoft.com/de-de/windows/console/setconsolemode -SetConsoleMode = windll.kernel32.SetConsoleMode -SetConsoleMode.argtypes = [HANDLE, DWORD] -SetConsoleMode.restype = BOOL - -# https://docs.microsoft.com/de-de/windows/console/readconsoleinput -ReadConsoleInputW = windll.kernel32.ReadConsoleInputW -# ReadConsoleInputW.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, LPDWORD] -ReadConsoleInputW.restype = BOOL - -# https://docs.microsoft.com/en-us/windows/console/getconsolescreenbufferinfo -GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo -GetConsoleScreenBufferInfo.argtypes = [HANDLE, POINTER(CONSOLE_SCREEN_BUFFER_INFO)] -GetConsoleScreenBufferInfo.restype = BOOL diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index 7c9379b57a..6c2e24926b 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -1,4 +1,3 @@ -import os import re import urwid @@ -17,11 +16,6 @@ from mitmproxy.tools.console import signals from mitmproxy.tools.console import statusbar -if os.name == "nt": - from mitmproxy.contrib.urwid import raw_display -else: - from urwid import raw_display # type: ignore - class StackWidget(urwid.Frame): def __init__(self, window, widget, title, focus): @@ -309,7 +303,7 @@ def keypress(self, size, k): return self.master.keymap.handle(self.focus_stack().top_widget().keyctx, k) -class Screen(raw_display.Screen): +class Screen(urwid.raw_display.Screen): def write(self, data): if common.IS_WINDOWS_OR_WSL: # replace urwid's SI/SO, which produce artifacts under WSL. diff --git a/setup.py b/setup.py index 9d525e1adf..4a86dd2008 100644 --- a/setup.py +++ b/setup.py @@ -95,7 +95,7 @@ "ruamel.yaml>=0.16,<0.18", "sortedcontainers>=2.3,<2.5", "tornado>=6.2,<7", - "urwid>=2.1.1,<2.2", + "urwid-mitmproxy>=2.1.1,<2.2", "wsproto>=1.0,<1.3", "publicsuffix2>=2.20190812,<3", "zstandard>=0.11,<0.20", From 3b35b5d675c7548fd5d843626a09ef98a24f56e6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 15 Jan 2023 18:13:48 +0100 Subject: [PATCH 162/695] clarify license for vendored code (#5872) --- mitmproxy/contrib/README | 8 ------ mitmproxy/contrib/README.md | 4 +++ mitmproxy/contrib/click/LICENSE.BSD-3 | 28 +++++++++++++++++++ mitmproxy/contrib/kaitaistruct/LICENSE | 1 + mitmproxy/contrib/kaitaistruct/README.md | 3 ++ .../kaitaistruct/dtls_client_hello.ksy | 1 + 6 files changed, 37 insertions(+), 8 deletions(-) delete mode 100644 mitmproxy/contrib/README create mode 100644 mitmproxy/contrib/README.md create mode 100644 mitmproxy/contrib/click/LICENSE.BSD-3 create mode 100644 mitmproxy/contrib/kaitaistruct/LICENSE create mode 100644 mitmproxy/contrib/kaitaistruct/README.md diff --git a/mitmproxy/contrib/README b/mitmproxy/contrib/README deleted file mode 100644 index 9924f9368a..0000000000 --- a/mitmproxy/contrib/README +++ /dev/null @@ -1,8 +0,0 @@ - -Contribs: - -wbxml - - https://github.com/davidpshaw/PyWBXMLDecoder - -urwid - - Patches vendored from https://github.com/urwid/urwid/pull/448. \ No newline at end of file diff --git a/mitmproxy/contrib/README.md b/mitmproxy/contrib/README.md new file mode 100644 index 0000000000..d327a1f9ae --- /dev/null +++ b/mitmproxy/contrib/README.md @@ -0,0 +1,4 @@ +# mitmproxy/contrib + +This directory includes vendored code from other sources. +See the respective README and LICENSE files for details. diff --git a/mitmproxy/contrib/click/LICENSE.BSD-3 b/mitmproxy/contrib/click/LICENSE.BSD-3 new file mode 100644 index 0000000000..d12a849186 --- /dev/null +++ b/mitmproxy/contrib/click/LICENSE.BSD-3 @@ -0,0 +1,28 @@ +Copyright 2014 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mitmproxy/contrib/kaitaistruct/LICENSE b/mitmproxy/contrib/kaitaistruct/LICENSE new file mode 100644 index 0000000000..45da807914 --- /dev/null +++ b/mitmproxy/contrib/kaitaistruct/LICENSE @@ -0,0 +1 @@ +Either MIT or CC-0 - see the individual .ksy files for the respective license. diff --git a/mitmproxy/contrib/kaitaistruct/README.md b/mitmproxy/contrib/kaitaistruct/README.md new file mode 100644 index 0000000000..ed7104cd94 --- /dev/null +++ b/mitmproxy/contrib/kaitaistruct/README.md @@ -0,0 +1,3 @@ +# Kaitai Struct Formats + +Most of the formats here are vendored from https://github.com/kaitai-io/kaitai_struct_formats/. diff --git a/mitmproxy/contrib/kaitaistruct/dtls_client_hello.ksy b/mitmproxy/contrib/kaitaistruct/dtls_client_hello.ksy index b0a271eca6..43a77cf972 100644 --- a/mitmproxy/contrib/kaitaistruct/dtls_client_hello.ksy +++ b/mitmproxy/contrib/kaitaistruct/dtls_client_hello.ksy @@ -1,6 +1,7 @@ meta: id: dtls_client_hello endian: be + license: MIT seq: - id: version From f0fbaf3bd8d289f985fedc6e0a755500c416fe58 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 15 Jan 2023 22:54:16 +0100 Subject: [PATCH 163/695] use reusable tox workflow --- .github/workflows/main.yml | 77 +++++++++++--------------------------- mitmproxy/version.py | 2 +- 2 files changed, 22 insertions(+), 57 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71ac16d349..1a726aa9af 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,65 +11,31 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - lint-pr: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - uses: TrueBrain/actions-flake8@c120815866a4bb260e23a2550dccee02d94a0385 # v2.2 - # mirrored at https://github.com/mitmproxy/mitmproxy/settings/actions - lint-local: - if: github.event_name == 'push' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox - - run: tox -e flake8 + lint: + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e flake8 + filename-matching: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox - - run: tox -e filename_matching + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e filename_matching + mypy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox - - run: tox -e mypy + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e mypy + individual-coverage: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - persist-credentials: false - fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version-file: .github/python-version.txt - - run: pip install tox - - run: tox -e individual_coverage + uses: mhils/workflows/.github/workflows/python-tox.yml@main + with: + cmd: tox -e individual_coverage + test: strategy: fail-fast: false @@ -155,7 +121,6 @@ jobs: - uses: actions/checkout@v3 with: persist-credentials: false - - run: git rev-parse --abbrev-ref HEAD - uses: actions/setup-node@v3 with: node-version-file: .github/node-version.txt diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 6022c261d9..700d3a4c84 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -18,7 +18,7 @@ def get_dev_version() -> str: mitmproxy_version = VERSION here = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - try: + try: # pragma: no cover # Check that we're in the mitmproxy repository: https://github.com/mitmproxy/mitmproxy/issues/3987 # cb0e3287090786fad566feb67ac07b8ef361b2c3 is the first mitmproxy commit. subprocess.run( From 3cdb8efc5e25a60334c27dc52695c6b9306d0e91 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 15 Jan 2023 22:55:46 +0100 Subject: [PATCH 164/695] add all-greens --- .github/workflows/main.yml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a726aa9af..cc853710c2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -171,6 +171,21 @@ jobs: name: docs-archive path: docs/public + check: + if: always() + needs: + - lint + - filename-matching + - mypy + - individual-coverage + - test + - build + - test-web-ui + - docs + uses: mhils/workflows/.github/workflows/alls-green.yml@main + with: + jobs: ${{ toJSON(needs) }} + # Separate from everything else because slow. build-and-deploy-docker: if: github.repository == 'mitmproxy/mitmproxy' && ( @@ -179,11 +194,7 @@ jobs: || startsWith(github.ref, 'refs/tags/') ) environment: deploy-docker - needs: - - test - - test-web-ui - - build - - docs + needs: check runs-on: ubuntu-latest env: DOCKER_USERNAME: mitmbot @@ -208,11 +219,7 @@ jobs: # In particular, we don't blindly `pip install` anything to minimize the risk of supply chain attacks. if: github.repository == 'mitmproxy/mitmproxy' && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/')) environment: ${{ (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/')) && 'deploy-release' || 'deploy-snapshot' }} - needs: - - test - - test-web-ui - - build - - docs + needs: check runs-on: ubuntu-latest env: # PyPI and MSFT keys are only available for the deploy-release environment From 121368654f27f80708fbe7108310c9284913fc6f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 15 Jan 2023 23:31:10 +0100 Subject: [PATCH 165/695] Adopt Dependabot (#5879) --- .github/dependabot.yml | 5 +++++ .github/workflows/autofix.yml | 4 ++-- .github/workflows/main.yml | 2 +- setup.py | 12 ++++++------ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c05de27708..f2c7fa9c50 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,8 @@ updates: directory: "/" schedule: interval: "monthly" + - package-ecosystem: pip + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 10 diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index d8f81150ae..c951b9167f 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -35,10 +35,10 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' yesqa **/*.py || true - - uses: install-pinned/autoflake@95c53f821b204037c1be14d45d810032e8ddfdcb + - uses: install-pinned/autoflake@1a248450153f02b75d051acf6c2a05df8c797666 - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - - uses: install-pinned/black@bcf144213c4943c1f2078a257fa566cebec36107 + - uses: install-pinned/black@9101a4d68e870eaaaae21c412d1d879b93c9afcb - run: black --extend-exclude mitmproxy/contrib . - uses: mhils/add-pr-ref-in-changelog@main diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cc853710c2..f44f11894d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,7 +211,7 @@ jobs: name: binaries.linux path: release/dist - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 - - uses: docker/setup-buildx-action@b1f1f719c7cd5364be7c82e366366da322d01f7c # v1.6.0 + - uses: docker/setup-buildx-action@165fe681b849eec43aaa64d786b9ec53e690475f # v1.6.0 - run: python release/build-and-deploy-docker.py deploy: diff --git a/setup.py b/setup.py index 4a86dd2008..4f63e3feb4 100644 --- a/setup.py +++ b/setup.py @@ -75,10 +75,10 @@ # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ "aioquic_mitmproxy>=0.9.20,<0.10", - "asgiref>=3.2.10,<3.6", + "asgiref>=3.2.10,<3.7", "Brotli>=1.0,<1.1", "certifi>=2019.9.11", # no semver here - this should always be on the last release! - "cryptography>=38.0,<38.1", + "cryptography>=38.0,<39.1", "flask>=1.1.1,<2.3", "h11>=0.11,<0.15", "h2>=4.1,<5", @@ -89,7 +89,7 @@ "msgpack>=1.0.0, <1.1.0", "passlib>=1.6.5, <1.8", "protobuf>=3.14,<5", - "pyOpenSSL>=22.1,<22.2", + "pyOpenSSL>=22.1,<23.1", "pyparsing>=2.4.2,<3.1", "pyperclip>=1.6.0,<1.9", "ruamel.yaml>=0.16,<0.18", @@ -110,14 +110,14 @@ "hypothesis>=5.8,<7", "parver>=0.1,<2.0", "pdoc>=4.0.0", - "pyinstaller==5.6.2", + "pyinstaller==5.7.0", "pytest-asyncio>=0.17,<0.21", "pytest-cov>=2.7.1,<4.1", "pytest-timeout>=1.3.3,<2.2", - "pytest-xdist>=2.1.0,<3.1", + "pytest-xdist>=2.1.0,<3.2", "pytest>=6.1.0,<8", "requests>=2.9.1,<3", - "tox>=3.5,<4", + "tox>=3.5,<5", "wheel>=0.36.2,<0.39", ], }, From 5df439b7e8e90e7e889163cab4a7d15fafd91ce2 Mon Sep 17 00:00:00 2001 From: Sean Flanigan Date: Wed, 18 Jan 2023 18:02:04 +1000 Subject: [PATCH 166/695] Add enhancements for contrib/mitmproxywrapper.py (#5883) * Add options for mitmweb and random port * Clean up proxy settings on ^C * [autofix.ci] apply automated fixes Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- examples/contrib/mitmproxywrapper.py | 37 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/examples/contrib/mitmproxywrapper.py b/examples/contrib/mitmproxywrapper.py index 8c95c8702c..5ae9db610f 100644 --- a/examples/contrib/mitmproxywrapper.py +++ b/examples/contrib/mitmproxywrapper.py @@ -10,13 +10,16 @@ import contextlib import os import re +import signal +import socketserver import subprocess import sys class Wrapper: - def __init__(self, port, extra_arguments=None): + def __init__(self, port, use_mitmweb, extra_arguments=None): self.port = port + self.use_mitmweb = use_mitmweb self.extra_arguments = extra_arguments def run_networksetup_command(self, *arguments): @@ -88,7 +91,7 @@ def connected_service_names(self): def wrap_mitmproxy(self): with self.wrap_proxy(): - cmd = ["mitmproxy", "-p", str(self.port)] + cmd = ["mitmweb" if self.use_mitmweb else "mitmproxy", "-p", str(self.port)] if self.extra_arguments: cmd.extend(self.extra_arguments) subprocess.check_call(cmd) @@ -124,7 +127,7 @@ def ensure_superuser(cls): def main(cls): parser = argparse.ArgumentParser( description="Helper tool for OS X proxy configuration and mitmproxy.", - epilog="Any additional arguments will be passed on unchanged to mitmproxy.", + epilog="Any additional arguments will be passed on unchanged to mitmproxy/mitmweb.", ) parser.add_argument( "-t", @@ -140,9 +143,35 @@ def main(cls): help="override the default port of 8080", default=8080, ) + parser.add_argument( + "-P", + "--port-random", + action="store_true", + help="choose a random unused port", + ) + parser.add_argument( + "-w", + "--web", + action="store_true", + help="web interface: run mitmweb instead of mitmproxy", + ) args, extra_arguments = parser.parse_known_args() + port = args.port + + # Allocate a random unused port, and hope no other process steals it before mitmproxy/mitmweb uses it. + # Passing the allocated socket to mitmproxy/mitmweb would be nicer of course. + if args.port_random: + with socketserver.TCPServer(("localhost", 0), None) as s: + port = s.server_address[1] + print(f"Using random port {port}...") + + wrapper = cls(port=port, use_mitmweb=args.web, extra_arguments=extra_arguments) + + def handler(signum, frame): + print("Cleaning up proxy settings...") + wrapper.toggle_proxy() - wrapper = cls(port=args.port, extra_arguments=extra_arguments) + signal.signal(signal.SIGINT, handler) if args.toggle: wrapper.toggle_proxy() From 4ebccfa2365a15b88d9342641f35d66b53583548 Mon Sep 17 00:00:00 2001 From: Sujal Singh Date: Thu, 19 Jan 2023 15:13:39 +0530 Subject: [PATCH 167/695] add fedora certificate installation instructions (#5885) --- docs/src/content/concepts-certificates.md | 1 + mitmproxy/addons/onboardingapp/templates/index.html | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 78f5e77f74..f4ba8c343f 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -62,6 +62,7 @@ documentation for some common platforms. The mitmproxy CA cert is located in - [macOS (automated)](https://www.dssw.co.uk/reference/security.html): `sudo security add-trusted-cert -d -p ssl -p basic -k /Library/Keychains/System.keychain ~/.mitmproxy/mitmproxy-ca-cert.pem` - [Ubuntu/Debian]( https://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861) +- [Fedora](https://docs.fedoraproject.org/en-US/quick-docs/using-shared-system-certificates/#proc_adding-new-certificates) - [Mozilla Firefox](https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox) - [Chrome on Linux](https://stackoverflow.com/a/15076602/198996) - [iOS](http://jasdev.me/intercepting-ios-traffic) diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index 8f4a11e5ab..f96870fd06 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -50,6 +50,11 @@
Ubuntu/Debian
  • mv mitmproxy-ca-cert.pem /usr/local/share/ca-certificates/mitmproxy.crt
  • sudo update-ca-certificates
    • +
    Fedora
    +
      +
    1. mv mitmproxy-ca-cert.pem /etc/pki/ca-trust/source/anchors/
      2. +
    2. sudo update-ca-trust
      3. +
    {% endcall %} {% call entry('macOS', 'apple') %}
    Manual Installation
    From 83c57dc53e55e62fa2f6b4003e73bd664e12ff3d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 25 Jan 2023 01:40:48 +0100 Subject: [PATCH 168/695] fix `timestamp_start` docstring, refs #5884 --- mitmproxy/connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index e0e748ff0c..7152244c20 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -271,7 +271,11 @@ class Server(Connection): sockname: Optional[Address] = None timestamp_start: Optional[float] = None - """*Timestamp:* TCP SYN sent.""" + """ + *Timestamp:* Connection establishment started. + + For IP addresses, this corresponds to sending a TCP SYN; for domains, this corresponds to starting a DNS lookup. + """ timestamp_tcp_setup: Optional[float] = None """*Timestamp:* TCP ACK received.""" From 0c4549f4cce06c217f62ee79a3d32fa8b5bbaaed Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 25 Jan 2023 02:15:48 +0100 Subject: [PATCH 169/695] fix lint errors --- mitmproxy/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index 7152244c20..5fc5066cfa 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -272,8 +272,8 @@ class Server(Connection): timestamp_start: Optional[float] = None """ - *Timestamp:* Connection establishment started. - + *Timestamp:* Connection establishment started. + For IP addresses, this corresponds to sending a TCP SYN; for domains, this corresponds to starting a DNS lookup. """ timestamp_tcp_setup: Optional[float] = None From 849a3c33cb10670eeab7f77a797c544f38e09d6b Mon Sep 17 00:00:00 2001 From: stephenspol Date: Sun, 29 Jan 2023 05:56:39 -0500 Subject: [PATCH 170/695] Removed escaping string to get real raw view (#5894) * Removed escaping string to get real raw view * [autofix.ci] apply automated fixes * Updated changelog * extend tests --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Maximilian Hils --- CHANGELOG.md | 2 ++ mitmproxy/contentviews/raw.py | 3 +-- test/mitmproxy/contentviews/test_raw.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc72f5d1ba..3c0175badf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ ([#5725](https://github.com/mitmproxy/mitmproxy/pull/5725), @mhils) * Add `replay.server.add` command for adding flows to server replay buffer ([#5851](https://github.com/mitmproxy/mitmproxy/pull/5851), @italankin) +* Removed string escaping in raw view. + ([#5470](https://github.com/mitmproxy/mitmproxy/issues/5470), @stephenspol) ### Breaking Changes diff --git a/mitmproxy/contentviews/raw.py b/mitmproxy/contentviews/raw.py index c19872534b..0c7b77d923 100644 --- a/mitmproxy/contentviews/raw.py +++ b/mitmproxy/contentviews/raw.py @@ -1,12 +1,11 @@ from . import base -from mitmproxy.utils import strutils class ViewRaw(base.View): name = "Raw" def __call__(self, data, **metadata): - return "Raw", base.format_text(strutils.bytes_to_escaped_str(data, True)) + return "Raw", base.format_text(data) def render_priority(self, data: bytes, **metadata) -> float: return 0.1 * float(bool(data)) diff --git a/test/mitmproxy/contentviews/test_raw.py b/test/mitmproxy/contentviews/test_raw.py index 0cffcf869f..f5020c2793 100644 --- a/test/mitmproxy/contentviews/test_raw.py +++ b/test/mitmproxy/contentviews/test_raw.py @@ -5,6 +5,16 @@ def test_view_raw(): v = full_eval(raw.ViewRaw()) assert v(b"foo") + # unicode + assert v("🫠".encode()) == ( + "Raw", + [[("text", "🫠".encode())]], + ) + # invalid utf8 + assert v(b"\xFF") == ( + "Raw", + [[("text", b"\xFF")]], + ) def test_render_priority(): From e3a79419130118c63be8feb49605a598de1db4d5 Mon Sep 17 00:00:00 2001 From: Sujal Singh Date: Mon, 30 Jan 2023 18:29:21 +0530 Subject: [PATCH 171/695] add delete shortcut to delete flows in mitmweb (#5896) --- web/src/js/ducks/ui/keyboard.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/web/src/js/ducks/ui/keyboard.tsx b/web/src/js/ducks/ui/keyboard.tsx index e318e808fd..d54625db8b 100644 --- a/web/src/js/ducks/ui/keyboard.tsx +++ b/web/src/js/ducks/ui/keyboard.tsx @@ -73,6 +73,7 @@ export function onKeyDown(e: KeyboardEvent) { break } + case "Delete": case "d": { if (!flow) { return From 89d688e9fcda69150dab70f57f607584d519bfd7 Mon Sep 17 00:00:00 2001 From: Sujal Singh Date: Wed, 1 Feb 2023 22:35:09 +0530 Subject: [PATCH 172/695] temporary fix for unhandled AlternativeServiceAvailable event (#5898) * temporary fix for unhandled AlternativeServiceAvailable event * [autofix.ci] apply automated fixes * better log message for AlternativeServiceAvailable service * [autofix.ci] apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- mitmproxy/proxy/layers/http/_http2.py | 4 +++ .../mitmproxy/proxy/layers/http/test_http2.py | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index 7c7af9dac1..f8151bf98e 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -275,6 +275,10 @@ def handle_h2_event(self, event: h2.events.Event) -> CommandGenerator[bool]: # https://http2.github.io/http2-spec/#rfc.section.4.1 # Implementations MUST ignore and discard any frame that has a type that is unknown. yield Log(f"Ignoring unknown HTTP/2 frame type: {event.frame.type}") + elif isinstance(event, h2.events.AlternativeServiceAvailable): + yield Log( + "Received HTTP/2 Alt-Svc frame, which will not be forwarded.", DEBUG + ) else: raise AssertionError(f"Unexpected event: {event!r}") return False diff --git a/test/mitmproxy/proxy/layers/http/test_http2.py b/test/mitmproxy/proxy/layers/http/test_http2.py index a6a5412d82..92f7ed4bab 100644 --- a/test/mitmproxy/proxy/layers/http/test_http2.py +++ b/test/mitmproxy/proxy/layers/http/test_http2.py @@ -1,4 +1,5 @@ import time +from logging import DEBUG import h2.settings import hpack @@ -1191,3 +1192,31 @@ def advance_time(_): >> reply(to=wakeup_command, side_effect=advance_time) << None ) + + +def test_alt_svc(tctx): + playbook, cff = start_h2_client(tctx) + flow = Placeholder(HTTPFlow) + server = Placeholder(Server) + initial = Placeholder(bytes) + + assert ( + playbook + >> DataReceived( + tctx.client, + cff.build_headers_frame( + example_request_headers, flags=["END_STREAM"] + ).serialize(), + ) + << http.HttpRequestHeadersHook(flow) + >> reply() + << http.HttpRequestHook(flow) + >> reply() + << OpenConnection(server) + >> reply(None, side_effect=make_h2) + << SendData(server, initial) + >> DataReceived( + server, cff.build_alt_svc_frame(0, b"example.com", b'h3=":443"').serialize() + ) + << Log("Received HTTP/2 Alt-Svc frame, which will not be forwarded.", DEBUG) + ) From 8683f8392cf8c775fc0ccbb2b46eda3fda00c325 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 22:14:05 +0000 Subject: [PATCH 173/695] Bump install-pinned/autoflake (#5900) Bumps [install-pinned/autoflake](https://github.com/install-pinned/autoflake) from 1a248450153f02b75d051acf6c2a05df8c797666 to 19ecc14a8688d57cca9dc6cfd705f16f200ff097. - [Release notes](https://github.com/install-pinned/autoflake/releases) - [Commits](https://github.com/install-pinned/autoflake/compare/1a248450153f02b75d051acf6c2a05df8c797666...19ecc14a8688d57cca9dc6cfd705f16f200ff097) --- updated-dependencies: - dependency-name: install-pinned/autoflake dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index c951b9167f..744aef18e4 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -35,7 +35,7 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' yesqa **/*.py || true - - uses: install-pinned/autoflake@1a248450153f02b75d051acf6c2a05df8c797666 + - uses: install-pinned/autoflake@19ecc14a8688d57cca9dc6cfd705f16f200ff097 - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - uses: install-pinned/black@9101a4d68e870eaaaae21c412d1d879b93c9afcb From 5859d998f37b406eeb054c7fe869414cf7820d67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 22:18:02 +0000 Subject: [PATCH 174/695] Bump docker/setup-buildx-action (#5902) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 165fe681b849eec43aaa64d786b9ec53e690475f to 11e8a2e2910826a92412015c515187a2d6750279. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/165fe681b849eec43aaa64d786b9ec53e690475f...11e8a2e2910826a92412015c515187a2d6750279) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f44f11894d..2ebca48931 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,7 +211,7 @@ jobs: name: binaries.linux path: release/dist - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 - - uses: docker/setup-buildx-action@165fe681b849eec43aaa64d786b9ec53e690475f # v1.6.0 + - uses: docker/setup-buildx-action@11e8a2e2910826a92412015c515187a2d6750279 # v1.6.0 - run: python release/build-and-deploy-docker.py deploy: From 985a77e77da398c818dc547318fe5ab3a988c3ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 22:18:07 +0000 Subject: [PATCH 175/695] Bump install-pinned/reorder_python_imports (#5901) Bumps [install-pinned/reorder_python_imports](https://github.com/install-pinned/reorder_python_imports) from 515035fd9eb355713f61dee238b17a04ce01f4d2 to 946c8bbd8fe048a3bee76063c90c938d5a59a9aa. - [Release notes](https://github.com/install-pinned/reorder_python_imports/releases) - [Commits](https://github.com/install-pinned/reorder_python_imports/compare/515035fd9eb355713f61dee238b17a04ce01f4d2...946c8bbd8fe048a3bee76063c90c938d5a59a9aa) --- updated-dependencies: - dependency-name: install-pinned/reorder_python_imports dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 744aef18e4..41edcdbe63 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -22,7 +22,7 @@ jobs: export GLOBIGNORE='mitmproxy/contrib/**' pyupgrade --exit-zero-even-if-changed --keep-runtime-typing --py39-plus **/*.py - - uses: install-pinned/reorder_python_imports@515035fd9eb355713f61dee238b17a04ce01f4d2 + - uses: install-pinned/reorder_python_imports@946c8bbd8fe048a3bee76063c90c938d5a59a9aa - name: Run reorder-python-imports run: | shopt -s globstar From 317b7a2be2b29ba63a11a32a3ef35119299bfdf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 22:20:13 +0000 Subject: [PATCH 176/695] Bump install-pinned/yesqa (#5903) Bumps [install-pinned/yesqa](https://github.com/install-pinned/yesqa) from a1262fbe567d4c0b3445afade67b90f3bba2c9a2 to 4af1e53e86a56db346a03ece9e89c19bfd0e5d0e. - [Release notes](https://github.com/install-pinned/yesqa/releases) - [Commits](https://github.com/install-pinned/yesqa/compare/a1262fbe567d4c0b3445afade67b90f3bba2c9a2...4af1e53e86a56db346a03ece9e89c19bfd0e5d0e) --- updated-dependencies: - dependency-name: install-pinned/yesqa dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 41edcdbe63..e258bbea1e 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -28,7 +28,7 @@ jobs: shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' reorder-python-imports --exit-zero-even-if-changed --py39-plus **/*.py - - uses: install-pinned/yesqa@a1262fbe567d4c0b3445afade67b90f3bba2c9a2 + - uses: install-pinned/yesqa@4af1e53e86a56db346a03ece9e89c19bfd0e5d0e - name: Run yesqa run: | From dfeddcf4add66a6ebab1d508c47256185ef6ddb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Feb 2023 23:08:09 +0000 Subject: [PATCH 177/695] Bump install-pinned/black from 9101a4d68e870eaaaae21c412d1d879b93c9afcb to 13c8a20eb904ba800c87f0b34ccfd932ac2ff81d (#5899) * Bump install-pinned/black Bumps [install-pinned/black](https://github.com/install-pinned/black) from 9101a4d68e870eaaaae21c412d1d879b93c9afcb to 13c8a20eb904ba800c87f0b34ccfd932ac2ff81d. - [Release notes](https://github.com/install-pinned/black/releases) - [Commits](https://github.com/install-pinned/black/compare/9101a4d68e870eaaaae21c412d1d879b93c9afcb...13c8a20eb904ba800c87f0b34ccfd932ac2ff81d) --- updated-dependencies: - dependency-name: install-pinned/black dependency-type: direct:production ... Signed-off-by: dependabot[bot] * [autofix.ci] apply automated fixes --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- examples/contrib/domain_fronting.py | 1 - examples/contrib/har_dump.py | 1 - examples/contrib/link_expander.py | 1 - examples/contrib/test_xss_scanner.py | 1 - examples/contrib/webscanner_helper/mapping.py | 1 - examples/contrib/webscanner_helper/urldict.py | 1 - examples/contrib/webscanner_helper/watchdog.py | 1 - mitmproxy/addons/dumper.py | 2 +- mitmproxy/contentviews/__init__.py | 2 +- mitmproxy/contentviews/base.py | 1 - mitmproxy/contentviews/grpc.py | 1 - mitmproxy/io/tnetstring.py | 2 +- mitmproxy/net/http/cookies.py | 1 - mitmproxy/net/http/http1/read.py | 2 +- mitmproxy/optmanager.py | 2 +- mitmproxy/platform/windows.py | 2 ++ mitmproxy/proxy/layers/http/__init__.py | 1 - mitmproxy/proxy/layers/http/_http2.py | 2 +- mitmproxy/proxy/layers/tcp.py | 1 - mitmproxy/proxy/layers/udp.py | 1 - mitmproxy/proxy/layers/websocket.py | 1 - mitmproxy/proxy/server.py | 1 - mitmproxy/tools/console/common.py | 1 - mitmproxy/tools/console/flowview.py | 2 +- mitmproxy/tools/console/grideditor/editors.py | 2 -- mitmproxy/tools/console/quickhelp.py | 2 +- mitmproxy/tools/web/app.py | 2 +- test/mitmproxy/addons/test_clientplayback.py | 1 - test/mitmproxy/net/http/test_cookies.py | 1 - test/mitmproxy/tools/console/test_quickhelp.py | 2 +- test/mitmproxy/tools/web/test_app.py | 1 - 32 files changed, 13 insertions(+), 32 deletions(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index e258bbea1e..00418de7e3 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -38,7 +38,7 @@ jobs: - uses: install-pinned/autoflake@19ecc14a8688d57cca9dc6cfd705f16f200ff097 - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - - uses: install-pinned/black@9101a4d68e870eaaaae21c412d1d879b93c9afcb + - uses: install-pinned/black@13c8a20eb904ba800c87f0b34ccfd932ac2ff81d - run: black --extend-exclude mitmproxy/contrib . - uses: mhils/add-pr-ref-in-changelog@main diff --git a/examples/contrib/domain_fronting.py b/examples/contrib/domain_fronting.py index 0a477d0b51..804ceadf32 100644 --- a/examples/contrib/domain_fronting.py +++ b/examples/contrib/domain_fronting.py @@ -60,7 +60,6 @@ class Mapping: class HttpsDomainFronting: - # configurations for regular ("foo.example.com") mappings: star_mappings: dict[str, Mapping] diff --git a/examples/contrib/har_dump.py b/examples/contrib/har_dump.py index bdd758ccb2..47930dd32d 100644 --- a/examples/contrib/har_dump.py +++ b/examples/contrib/har_dump.py @@ -56,7 +56,6 @@ def configure(updated): def flow_entry(flow: mitmproxy.http.HTTPFlow) -> dict: - # -1 indicates that these values do not apply to current request ssl_time = -1 connect_time = -1 diff --git a/examples/contrib/link_expander.py b/examples/contrib/link_expander.py index 7e7e6b5d82..e62aab5e9d 100644 --- a/examples/contrib/link_expander.py +++ b/examples/contrib/link_expander.py @@ -7,7 +7,6 @@ def response(flow): - if ( "Content-Type" in flow.response.headers and flow.response.headers["Content-Type"].find("text/html") != -1 diff --git a/examples/contrib/test_xss_scanner.py b/examples/contrib/test_xss_scanner.py index 2f89bab481..8aeb524047 100644 --- a/examples/contrib/test_xss_scanner.py +++ b/examples/contrib/test_xss_scanner.py @@ -408,7 +408,6 @@ def test_test_user_agent_injection(self, get_request_vuln): assert sqli_info is None def test_test_query_injection(self, get_request_vuln): - xss_info = xss.test_query_injection( "", "https://example.com/vuln.php?cmd=ls", {} )[0] diff --git a/examples/contrib/webscanner_helper/mapping.py b/examples/contrib/webscanner_helper/mapping.py index 52509730d2..c8069c91f6 100644 --- a/examples/contrib/webscanner_helper/mapping.py +++ b/examples/contrib/webscanner_helper/mapping.py @@ -141,7 +141,6 @@ def response(self, flow: HTTPFlow) -> None: def done(self) -> None: """Dumps all new content into the configuration file if self.persistent is set.""" if self.persistent: - # make sure that all items are strings and not soups. def value_dumper(value): store = {} diff --git a/examples/contrib/webscanner_helper/urldict.py b/examples/contrib/webscanner_helper/urldict.py index a5b02af21a..a4ce1e0fa5 100644 --- a/examples/contrib/webscanner_helper/urldict.py +++ b/examples/contrib/webscanner_helper/urldict.py @@ -50,7 +50,6 @@ def __len__(self): return self.store.__len__() def get_generator(self, flow: HTTPFlow) -> Generator[Any, None, None]: - for fltr, value in self.store.items(): if flowfilter.match(fltr, flow): yield value diff --git a/examples/contrib/webscanner_helper/watchdog.py b/examples/contrib/webscanner_helper/watchdog.py index 361f72a434..7f63ffec5f 100644 --- a/examples/contrib/webscanner_helper/watchdog.py +++ b/examples/contrib/webscanner_helper/watchdog.py @@ -66,7 +66,6 @@ def error(self, flow): and flow.error is not None and not isinstance(flow.error, HttpSyntaxException) ): - self.last_trigger = time.time() logger.error(f"Watchdog triggered! Cause: {flow}") self.error_event.set() diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 42ecd26008..897fc3a151 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -111,7 +111,7 @@ def _echo_trailers(self, trailers: Optional[http.Headers]): def _colorful(self, line): yield " " # we can already indent here - for (style, text) in line: + for style, text in line: yield self.style(text, **CONTENTVIEW_STYLES.get(style, {})) def _echo_message( diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 2949c8fa3f..688869b6c3 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -89,7 +89,7 @@ def safe_to_print(lines, encoding="utf8"): """ for line in lines: clean_line = [] - for (style, text) in line: + for style, text in line: if isinstance(text, bytes): text = text.decode(encoding, "replace") text = strutils.escape_control_characters(text) diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 9788eb6888..7eca342a0b 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -86,7 +86,6 @@ def format_pairs(items: Iterable[tuple[TTextType, TTextType]]) -> Iterator[TView for key, value in items: if isinstance(key, bytes): - key += b":" else: key += ":" diff --git a/mitmproxy/contentviews/grpc.py b/mitmproxy/contentviews/grpc.py index faa60079e2..899d8a601e 100644 --- a/mitmproxy/contentviews/grpc.py +++ b/mitmproxy/contentviews/grpc.py @@ -1113,7 +1113,6 @@ def render_priority( http_message: http.Message | None = None, **unknown_metadata, ) -> float: - if bool(data) and content_type in self.__content_types_grpc: return 1 if bool(data) and content_type in self.__content_types_pb: diff --git a/mitmproxy/io/tnetstring.py b/mitmproxy/io/tnetstring.py index b11580e9ec..73b7aeb370 100644 --- a/mitmproxy/io/tnetstring.py +++ b/mitmproxy/io/tnetstring.py @@ -138,7 +138,7 @@ def _rdumpq(q: collections.deque, size: int, value: TSerializable) -> int: elif isinstance(value, dict): write(b"}") init_size = size = size + 1 - for (k, v) in value.items(): + for k, v in value.items(): size = _rdumpq(q, size, v) size = _rdumpq(q, size, k) span = str(size - init_size).encode() diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 3e961ae830..be8c94f8e7 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -274,7 +274,6 @@ def format_set_cookie_header(set_cookies: list[TSetCookie]) -> str: rv = [] for name, value, attrs in set_cookies: - pairs = [(name, value)] pairs.extend(attrs.fields if hasattr(attrs, "fields") else attrs) diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index 2986c489d4..578fcd7d41 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -56,7 +56,7 @@ def validate_headers(headers: Headers) -> None: te_found = False cl_found = False - for (name, value) in headers.fields: + for name, value in headers.fields: if not _valid_header_name.match(name): raise ValueError( f"Received an invalid header name: {name!r}. Invalid header names may introduce " diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 5fa10a787a..17cec4d983 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -161,7 +161,7 @@ def subscribe(self, func, opts): def _notify_subscribers(self, updated) -> None: cleanup = False - for (ref, opts) in self._subscriptions: + for ref, opts in self._subscriptions: callback = ref() if callback is not None: if opts & updated: diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index 1e065544b2..005bb148a5 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -132,6 +132,7 @@ def __init__(self, proxifier, *args, **kwargs): # IPv6 # + # https://msdn.microsoft.com/en-us/library/windows/desktop/aa366896(v=vs.85).aspx class MIB_TCP6ROW_OWNER_PID(ctypes.Structure): _fields_ = [ @@ -161,6 +162,7 @@ class _MIB_TCP6TABLE_OWNER_PID(ctypes.Structure): # IPv4 # + # https://msdn.microsoft.com/en-us/library/windows/desktop/aa366913(v=vs.85).aspx class MIB_TCPROW_OWNER_PID(ctypes.Structure): _fields_ = [ diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 9d7cba4ceb..56d7189124 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -1028,7 +1028,6 @@ def get_connection( stack = tunnel.LayerStack() if not can_use_context_connection: - context.server = Server( address=event.address, transport_protocol=event.transport_protocol ) diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index f8151bf98e..df32853a0a 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -610,7 +610,7 @@ def split_pseudo_headers( ) -> tuple[dict[bytes, bytes], http.Headers]: pseudo_headers: dict[bytes, bytes] = {} i = 0 - for (header, value) in h2_headers: + for header, value in h2_headers: if header.startswith(b":"): if header in pseudo_headers: raise ValueError(f"Duplicate HTTP/2 pseudo header: {header!r}") diff --git a/mitmproxy/proxy/layers/tcp.py b/mitmproxy/proxy/layers/tcp.py index 0272d4ed51..417009e415 100644 --- a/mitmproxy/proxy/layers/tcp.py +++ b/mitmproxy/proxy/layers/tcp.py @@ -93,7 +93,6 @@ def start(self, _) -> layer.CommandGenerator[None]: @expect(events.DataReceived, events.ConnectionClosed, TcpMessageInjected) def relay_messages(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, TcpMessageInjected): # we just spoof that we received data here and then process that regularly. event = events.DataReceived( diff --git a/mitmproxy/proxy/layers/udp.py b/mitmproxy/proxy/layers/udp.py index e80fc7b9db..ac6643b9a5 100644 --- a/mitmproxy/proxy/layers/udp.py +++ b/mitmproxy/proxy/layers/udp.py @@ -92,7 +92,6 @@ def start(self, _) -> layer.CommandGenerator[None]: @expect(events.DataReceived, events.ConnectionClosed, UdpMessageInjected) def relay_messages(self, event: events.Event) -> layer.CommandGenerator[None]: - if isinstance(event, UdpMessageInjected): # we just spoof that we received data here and then process that regularly. event = events.DataReceived( diff --git a/mitmproxy/proxy/layers/websocket.py b/mitmproxy/proxy/layers/websocket.py index 24c291b765..85b63b4bdb 100644 --- a/mitmproxy/proxy/layers/websocket.py +++ b/mitmproxy/proxy/layers/websocket.py @@ -97,7 +97,6 @@ def __init__(self, context: Context, flow: http.HTTPFlow): @expect(events.Start) def start(self, _) -> layer.CommandGenerator[None]: - client_extensions = [] server_extensions = [] diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index ac717dfae9..69442b0bd9 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -357,7 +357,6 @@ def server_event(self, event: events.Event) -> None: try: layer_commands = self.layer.handle_event(event) for command in layer_commands: - if isinstance(command, commands.OpenConnection): assert command.connection not in self.transports handler = asyncio_utils.create_task( diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 58f2be69a9..9e2aaf2f81 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -550,7 +550,6 @@ def format_http_flow_table( response_style = "" if response_code: - status = str(response_code) status_style = response_style or HTTP_RESPONSE_CODE_STYLE.get( response_code // 100, "code_other" diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 8aae522a8b..f00d2fefe8 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -329,7 +329,7 @@ def _get_content_view(self, viewmode, max_lines, _): text_objects = [] for line in lines: txt = [] - for (style, text) in line: + for style, text in line: if total_chars + len(text) > max_chars: text = text[: max_chars - total_chars] txt.append((style, text)) diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index bfc9b38622..f1b80b8171 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -54,7 +54,6 @@ class RequestMultipartEditor(base.FocusEditor): columns = [col_text.Column("Key"), col_text.Column("Value")] def get_data(self, flow): - return flow.request.multipart_form.items(multi=True) def set_data(self, vals, flow): @@ -66,7 +65,6 @@ class RequestUrlEncodedEditor(base.FocusEditor): columns = [col_text.Column("Key"), col_text.Column("Value")] def get_data(self, flow): - return flow.request.urlencoded_form.items(multi=True) def set_data(self, vals, flow): diff --git a/mitmproxy/tools/console/quickhelp.py b/mitmproxy/tools/console/quickhelp.py index a24f810042..562587ac84 100644 --- a/mitmproxy/tools/console/quickhelp.py +++ b/mitmproxy/tools/console/quickhelp.py @@ -169,7 +169,7 @@ def _make_row(label: str, items: HelpItems, keymap: Keymap) -> urwid.Columns: cols = [ (len(label), urwid.Text(label)), ] - for (short, long) in items.items(): + for short, long in items.items(): if isinstance(long, BasicKeyHelp): key_short = long.key else: diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 153b1afe08..334661cc1e 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -559,7 +559,7 @@ def get(self, flow_id, message, content_view) -> None: class Commands(RequestHandler): def get(self) -> None: commands = {} - for (name, cmd) in self.master.commands.commands.items(): + for name, cmd in self.master.commands.commands.items(): commands[name] = { "help": cmd.help, "parameters": [ diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 013d6f1b39..62e4e2e3be 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -50,7 +50,6 @@ async def handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): with taddons.context(cp, ps) as tctx: tctx.configure(cp, client_replay_concurrency=concurrency) async with tcp_server(handler) as addr: - cp.running() flow = tflow.tflow(live=False) flow.request.content = b"data" diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 13b89d940e..936a6f2c16 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -198,7 +198,6 @@ def set_cookie_equal(obs, exp): def test_refresh_cookie(): - # Invalid expires format, sent to us by Reddit. c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2133 23:59:59 GMT; Path=/" assert cookies.refresh_set_cookie_header(c, 60) diff --git a/test/mitmproxy/tools/console/test_quickhelp.py b/test/mitmproxy/tools/console/test_quickhelp.py index 722af3dab8..e129850d57 100644 --- a/test/mitmproxy/tools/console/test_quickhelp.py +++ b/test/mitmproxy/tools/console/test_quickhelp.py @@ -44,7 +44,7 @@ def keymap() -> Keymap: def test_quickhelp(widget, flow, keymap, is_root_widget): qh = quickhelp.make(widget, flow, is_root_widget) for row in [qh.top_items, qh.bottom_items]: - for (title, v) in row.items(): + for title, v in row.items(): if isinstance(v, quickhelp.BasicKeyHelp): key_short = v.key else: diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index d4b29906b6..66816da6b5 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -130,7 +130,6 @@ def ts_type(t): raise RuntimeError(t) with redirect_stdout(io.StringIO()) as s: - print("/** Auto-generated by test_app.py:test_generate_options_js */") print("export interface OptionsState {") From 977385ceab5d7ae45e5c5d45298a85057f732f9e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 4 Feb 2023 01:01:01 +0100 Subject: [PATCH 178/695] docs: `Server.peername` may be None, refs #5904 --- mitmproxy/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index 5fc5066cfa..f18880c92d 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -267,7 +267,10 @@ class Server(Connection): address: Optional[Address] = None peername: Optional[Address] = None - """The server's resolved `(ip, port)` tuple. Will be set during connection establishment.""" + """ + The server's resolved `(ip, port)` tuple. Will be set during connection establishment. + May be `None` in upstream proxy mode when the address is resolved by the upstream proxy only. + """ sockname: Optional[Address] = None timestamp_start: Optional[float] = None From a7e50c793e9c6840d9f0fc162893e293477a358e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 4 Feb 2023 22:28:15 +0100 Subject: [PATCH 179/695] mitmproxy-wireguard -> mitmproxy_rs (#5909) mitmproxy-rs includes all the fantastic WireGuard work, but will add more non-WireGuard stuff. :) --- mitmproxy/net/udp.py | 9 +++++---- mitmproxy/net/udp_wireguard.py | 35 --------------------------------- mitmproxy/proxy/layers/modes.py | 2 +- mitmproxy/proxy/mode_servers.py | 34 ++++++++++++-------------------- mitmproxy/proxy/server.py | 10 +++++----- setup.py | 2 +- 6 files changed, 25 insertions(+), 67 deletions(-) delete mode 100644 mitmproxy/net/udp_wireguard.py diff --git a/mitmproxy/net/udp.py b/mitmproxy/net/udp.py index d51aee1239..c1bf595e61 100644 --- a/mitmproxy/net/udp.py +++ b/mitmproxy/net/udp.py @@ -9,8 +9,9 @@ from typing import Optional from typing import Union +import mitmproxy_rs + from mitmproxy.connection import Address -from mitmproxy.net import udp_wireguard from mitmproxy.utils import human logger = logging.getLogger(__name__) @@ -162,14 +163,14 @@ async def read(self, n: int) -> bytes: class DatagramWriter: - _transport: asyncio.DatagramTransport + _transport: asyncio.DatagramTransport | mitmproxy_rs.DatagramTransport _remote_addr: Address _reader: DatagramReader | None _closed: asyncio.Event | None def __init__( self, - transport: asyncio.DatagramTransport, + transport: asyncio.DatagramTransport | mitmproxy_rs.DatagramTransport, remote_addr: Address, reader: DatagramReader | None = None, ) -> None: @@ -189,7 +190,7 @@ def __init__( @property def _protocol( self, - ) -> DrainableDatagramProtocol | udp_wireguard.WireGuardDatagramTransport: + ) -> DrainableDatagramProtocol | mitmproxy_rs.DatagramTransport: return self._transport.get_protocol() # type: ignore def write(self, data: bytes) -> None: diff --git a/mitmproxy/net/udp_wireguard.py b/mitmproxy/net/udp_wireguard.py deleted file mode 100644 index b857ec76ca..0000000000 --- a/mitmproxy/net/udp_wireguard.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -This module contains a mock DatagramTransport for use with mitmproxy-wireguard. -""" -import asyncio -from typing import Any - -import mitmproxy_wireguard as wg - -from mitmproxy.connection import Address - - -class WireGuardDatagramTransport(asyncio.DatagramTransport): - def __init__(self, server: wg.Server, local_addr: Address, remote_addr: Address): - self._server: wg.Server = server - self._local_addr: Address = local_addr - self._remote_addr: Address = remote_addr - super().__init__() - - def sendto(self, data, addr=None): - self._server.send_datagram(data, self._local_addr, addr or self._remote_addr) - - def get_extra_info(self, name: str, default: Any = None) -> Any: - if name == "sockname": - return self._server.getsockname() - else: - raise NotImplementedError - - def get_protocol(self): - return self - - async def drain(self) -> None: - pass - - async def wait_closed(self) -> None: - pass diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index bfc2749678..2dbda7a161 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -87,7 +87,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: class TransparentProxy(DestinationKnown): @expect(events.Start) def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - assert self.context.server.address + assert self.context.server.address, "No server address set." self.child_layer = layer.NextLayer(self.context) err = yield from self.finish_start() if err: diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index b406156703..105c996896 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -28,7 +28,7 @@ from typing import get_args from typing import TypeVar -import mitmproxy_wireguard as wg +import mitmproxy_rs from mitmproxy import ctx from mitmproxy import flow @@ -37,7 +37,6 @@ from mitmproxy.master import Master from mitmproxy.net import local_ip from mitmproxy.net import udp -from mitmproxy.net.udp_wireguard import WireGuardDatagramTransport from mitmproxy.proxy import commands from mitmproxy.proxy import layers from mitmproxy.proxy import mode_specs @@ -149,8 +148,8 @@ def to_json(self) -> dict: async def handle_tcp_connection( self, - reader: asyncio.StreamReader | wg.TcpStream, - writer: asyncio.StreamWriter | wg.TcpStream, + reader: asyncio.StreamReader | mitmproxy_rs.TcpStream, + writer: asyncio.StreamWriter | mitmproxy_rs.TcpStream, ) -> None: handler = ProxyConnectionHandler( ctx.master, reader, writer, ctx.options, self.mode @@ -182,7 +181,7 @@ async def handle_tcp_connection( def handle_udp_datagram( self, - transport: asyncio.DatagramTransport, + transport: asyncio.DatagramTransport | mitmproxy_rs.DatagramTransport, data: bytes, remote_addr: Address, local_addr: Address, @@ -304,7 +303,7 @@ def listen_addrs(self) -> tuple[Address, ...]: class WireGuardServerInstance(ServerInstance[mode_specs.WireGuardMode]): - _server: wg.Server | None = None + _server: mitmproxy_rs.WireGuardServer | None = None _listen_addrs: tuple[Address, ...] = tuple() server_key: str @@ -333,8 +332,8 @@ async def start(self) -> None: conf_path.write_text( json.dumps( { - "server_key": wg.genkey(), - "client_key": wg.genkey(), + "server_key": mitmproxy_rs.genkey(), + "client_key": mitmproxy_rs.genkey(), }, indent=4, ) @@ -349,16 +348,16 @@ async def start(self) -> None: f"Invalid configuration file ({conf_path}): {e}" ) from e # error early on invalid keys - p = wg.pubkey(self.client_key) - _ = wg.pubkey(self.server_key) + p = mitmproxy_rs.pubkey(self.client_key) + _ = mitmproxy_rs.pubkey(self.server_key) - self._server = await wg.start_server( + self._server = await mitmproxy_rs.start_wireguard_server( host, port, self.server_key, [p], self.wg_handle_tcp_connection, - self.wg_handle_udp_datagram, + self.handle_udp_datagram, ) self._listen_addrs = (self._server.getsockname(),) except Exception as e: @@ -391,7 +390,7 @@ def client_conf(self) -> str | None: DNS = 10.0.0.53 [Peer] - PublicKey = {wg.pubkey(self.server_key)} + PublicKey = {mitmproxy_rs.pubkey(self.server_key)} AllowedIPs = 0.0.0.0/0 Endpoint = {host}:{port} """ @@ -414,16 +413,9 @@ async def stop(self) -> None: def listen_addrs(self) -> tuple[Address, ...]: return self._listen_addrs - async def wg_handle_tcp_connection(self, stream: wg.TcpStream) -> None: + async def wg_handle_tcp_connection(self, stream: mitmproxy_rs.TcpStream) -> None: await self.handle_tcp_connection(stream, stream) - def wg_handle_udp_datagram( - self, data: bytes, remote_addr: Address, local_addr: Address - ) -> None: - assert self._server is not None - transport = WireGuardDatagramTransport(self._server, local_addr, remote_addr) - self.handle_udp_datagram(transport, data, remote_addr, local_addr) - class RegularInstance(AsyncioServerInstance[mode_specs.RegularMode]): def make_top_layer(self, context: Context) -> Layer: diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 69442b0bd9..eeec51d5f2 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -20,7 +20,7 @@ from typing import Optional from typing import Union -import mitmproxy_wireguard as wg +import mitmproxy_rs from OpenSSL import SSL from mitmproxy import http @@ -93,10 +93,10 @@ def disarm(self): class ConnectionIO: handler: Optional[asyncio.Task] = None reader: Optional[ - Union[asyncio.StreamReader, udp.DatagramReader, wg.TcpStream] + Union[asyncio.StreamReader, udp.DatagramReader, mitmproxy_rs.TcpStream] ] = None writer: Optional[ - Union[asyncio.StreamWriter, udp.DatagramWriter, wg.TcpStream] + Union[asyncio.StreamWriter, udp.DatagramWriter, mitmproxy_rs.TcpStream] ] = None @@ -429,8 +429,8 @@ def close_connection( class LiveConnectionHandler(ConnectionHandler, metaclass=abc.ABCMeta): def __init__( self, - reader: Union[asyncio.StreamReader, wg.TcpStream], - writer: Union[asyncio.StreamWriter, wg.TcpStream], + reader: Union[asyncio.StreamReader, mitmproxy_rs.TcpStream], + writer: Union[asyncio.StreamWriter, mitmproxy_rs.TcpStream], options: moptions.Options, mode: mode_specs.ProxyMode, ) -> None: diff --git a/setup.py b/setup.py index 4f63e3feb4..4578e7752b 100644 --- a/setup.py +++ b/setup.py @@ -85,7 +85,7 @@ "hyperframe>=6.0,<7", "kaitaistruct>=0.10,<0.11", "ldap3>=2.8,<2.10", - "mitmproxy_wireguard>=0.1.6,<0.2", + "mitmproxy_rs>=0.2.0b1,<0.3", "msgpack>=1.0.0, <1.1.0", "passlib>=1.6.5, <1.8", "protobuf>=3.14,<5", From 54185c2c8dbcc3436f8e9103f3390617fd352556 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 6 Feb 2023 17:34:48 +0100 Subject: [PATCH 180/695] Add experimental Windows OS proxy mode (#5912) * add experimental Windows OS proxy mode this is merely a proof-of-concept now, but works under the most ideal circumstances. --- mitmproxy/proxy/mode_servers.py | 256 +++++++++++------- mitmproxy/proxy/mode_specs.py | 17 +- test/mitmproxy/addons/test_proxyserver.py | 12 +- .../mitmproxy/proxy/layers/http/test_http2.py | 3 + .../proxy/layers/http/test_http_fuzz.py | 5 +- test/mitmproxy/proxy/test_mode_servers.py | 90 +++++- test/mitmproxy/proxy/test_mode_specs.py | 5 + test/mitmproxy/tools/web/test_app.py | 10 +- 8 files changed, 286 insertions(+), 112 deletions(-) diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 105c996896..0d64157ae2 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -15,6 +15,7 @@ import errno import json import logging +import os import socket import textwrap import typing @@ -85,10 +86,11 @@ def register_connection( class ServerInstance(Generic[M], metaclass=ABCMeta): __modes: ClassVar[dict[str, type[ServerInstance]]] = {} + last_exception: Exception | None = None + def __init__(self, mode: M, manager: ServerManager): self.mode: M = mode self.manager: ServerManager = manager - self.last_exception: Exception | None = None def __init_subclass__(cls, **kwargs): """Register all subclasses so that make() finds them.""" @@ -119,12 +121,41 @@ def make( def is_running(self) -> bool: pass - @abstractmethod async def start(self) -> None: + try: + await self._start() + except Exception as e: + self.last_exception = e + raise + else: + self.last_exception = None + if self.listen_addrs: + addrs = " and ".join({human.format_address(a) for a in self.listen_addrs}) + logger.info(f"{self.mode.description} listening at {addrs}.") + else: + logger.info(f"{self.mode.description} started.") + + async def stop(self) -> None: + listen_addrs = self.listen_addrs + try: + await self._stop() + except Exception as e: + self.last_exception = e + raise + else: + self.last_exception = None + if listen_addrs: + addrs = " and ".join({human.format_address(a) for a in listen_addrs}) + logger.info(f"{self.mode.description} at {addrs} stopped.") + else: + logger.info(f"{self.mode.description} stopped.") + + @abstractmethod + async def _start(self) -> None: pass @abstractmethod - async def stop(self) -> None: + async def _stop(self) -> None: pass @property @@ -166,10 +197,8 @@ async def handle_tcp_connection( else: handler.layer.context.client.sockname = original_dst handler.layer.context.server.address = original_dst - elif isinstance(self.mode, mode_specs.WireGuardMode): - original_dst = writer.get_extra_info("original_dst") - handler.layer.context.client.sockname = original_dst - handler.layer.context.server.address = original_dst + elif isinstance(self.mode, (mode_specs.WireGuardMode, mode_specs.OsProxyMode)): + handler.layer.context.server.address = handler.layer.context.client.sockname connection_id = ( handler.layer.context.client.transport_protocol, @@ -197,7 +226,9 @@ def handle_udp_datagram( handler.layer = self.make_top_layer(handler.layer.context) handler.layer.context.client.transport_protocol = "udp" handler.layer.context.server.transport_protocol = "udp" - if isinstance(self.mode, mode_specs.WireGuardMode): + if isinstance( + self.mode, (mode_specs.WireGuardMode, mode_specs.OsProxyMode) + ): handler.layer.context.server.address = local_addr # pre-register here - we may get datagrams before the task is executed. @@ -217,13 +248,19 @@ async def handle_udp_connection( class AsyncioServerInstance(ServerInstance[M], metaclass=ABCMeta): _server: asyncio.Server | udp.UdpServer | None = None - _listen_addrs: tuple[Address, ...] = tuple() @property def is_running(self) -> bool: return self._server is not None - async def start(self) -> None: + @property + def listen_addrs(self) -> tuple[Address, ...]: + if self._server is not None: + return tuple(s.getsockname() for s in self._server.sockets) + else: + return tuple() + + async def _start(self) -> None: assert self._server is None host = self.mode.listen_host(ctx.options.listen_host) port = self.mode.listen_port(ctx.options.listen_port) @@ -231,7 +268,6 @@ async def start(self) -> None: self._server = await self.listen(host, port) self._listen_addrs = tuple(s.getsockname() for s in self._server.sockets) except OSError as e: - self.last_exception = e message = f"{self.mode.description} failed to listen on {host or '*'}:{port} with {e}" if e.errno == errno.EADDRINUSE and self.mode.custom_listen_port is None: assert ( @@ -239,31 +275,15 @@ async def start(self) -> None: ) # since [@ [listen_addr:]listen_port] message += f"\nTry specifying a different port by using `--mode {self.mode.full_spec}@{port + 1}`." raise OSError(e.errno, message, e.filename) from e - except Exception as e: - self.last_exception = e - raise - else: - self.last_exception = None - addrs = " and ".join({human.format_address(a) for a in self._listen_addrs}) - logger.info(f"{self.mode.description} listening at {addrs}.") - async def stop(self) -> None: + async def _stop(self) -> None: assert self._server is not None - # we always reset _server and _listen_addrs and ignore failures - server = self._server - listen_addrs = self._listen_addrs - self._server = None - self._listen_addrs = tuple() try: - server.close() - await server.wait_closed() - except Exception as e: - self.last_exception = e - raise - else: - self.last_exception = None - addrs = " and ".join({human.format_address(a) for a in listen_addrs}) - logger.info(f"Stopped {self.mode.description} at {addrs}.") + self._server.close() + await self._server.wait_closed() + finally: + # we always reset _server and ignore failures + self._server = None async def listen(self, host: str, port: int) -> asyncio.Server | udp.UdpServer: if self.mode.transport_protocol == "tcp": @@ -297,14 +317,9 @@ async def listen(self, host: str, port: int) -> asyncio.Server | udp.UdpServer: else: raise AssertionError(self.mode.transport_protocol) - @property - def listen_addrs(self) -> tuple[Address, ...]: - return self._listen_addrs - class WireGuardServerInstance(ServerInstance[mode_specs.WireGuardMode]): _server: mitmproxy_rs.WireGuardServer | None = None - _listen_addrs: tuple[Address, ...] = tuple() server_key: str client_key: str @@ -316,7 +331,14 @@ def make_top_layer(self, context: Context) -> Layer: def is_running(self) -> bool: return self._server is not None - async def start(self) -> None: + @property + def listen_addrs(self) -> tuple[Address, ...]: + if self._server: + return (self._server.getsockname(),) + else: + return tuple() + + async def _start(self) -> None: assert self._server is None host = self.mode.listen_host(ctx.options.listen_host) port = self.mode.listen_port(ctx.options.listen_port) @@ -326,56 +348,40 @@ async def start(self) -> None: else: conf_path = Path(ctx.options.confdir).expanduser() / "wireguard.conf" - try: - if not conf_path.exists(): - conf_path.parent.mkdir(parents=True, exist_ok=True) - conf_path.write_text( - json.dumps( - { - "server_key": mitmproxy_rs.genkey(), - "client_key": mitmproxy_rs.genkey(), - }, - indent=4, - ) + if not conf_path.exists(): + conf_path.parent.mkdir(parents=True, exist_ok=True) + conf_path.write_text( + json.dumps( + { + "server_key": mitmproxy_rs.genkey(), + "client_key": mitmproxy_rs.genkey(), + }, + indent=4, ) - - try: - c = json.loads(conf_path.read_text()) - self.server_key = c["server_key"] - self.client_key = c["client_key"] - except Exception as e: - raise ValueError( - f"Invalid configuration file ({conf_path}): {e}" - ) from e - # error early on invalid keys - p = mitmproxy_rs.pubkey(self.client_key) - _ = mitmproxy_rs.pubkey(self.server_key) - - self._server = await mitmproxy_rs.start_wireguard_server( - host, - port, - self.server_key, - [p], - self.wg_handle_tcp_connection, - self.handle_udp_datagram, ) - self._listen_addrs = (self._server.getsockname(),) + + try: + c = json.loads(conf_path.read_text()) + self.server_key = c["server_key"] + self.client_key = c["client_key"] except Exception as e: - self.last_exception = e - message = f"{self.mode.description} failed to listen on {host or '*'}:{port} with {e}" - raise OSError(message) from e - else: - self.last_exception = None + raise ValueError(f"Invalid configuration file ({conf_path}): {e}") from e + # error early on invalid keys + p = mitmproxy_rs.pubkey(self.client_key) + _ = mitmproxy_rs.pubkey(self.server_key) + + self._server = await mitmproxy_rs.start_wireguard_server( + host, + port, + self.server_key, + [p], + self.wg_handle_tcp_connection, + self.handle_udp_datagram, + ) - addrs = " and ".join({human.format_address(a) for a in self.listen_addrs}) conf = self.client_conf() assert conf - logger.info( - f"{self.mode.description} listening at {addrs}.\n" - + "------------------------------------------------------------\n" - + conf - + "\n------------------------------------------------------------" - ) + logger.info("-" * 60 + "\n" + conf + "\n" + "-" * 60) def client_conf(self) -> str | None: if not self._server: @@ -399,22 +405,84 @@ def client_conf(self) -> str | None: def to_json(self) -> dict: return {"wireguard_conf": self.client_conf(), **super().to_json()} - async def stop(self) -> None: + async def _stop(self) -> None: assert self._server is not None - self._server.close() - await self._server.wait_closed() - self._server = None - self.last_exception = None + try: + self._server.close() + await self._server.wait_closed() + finally: + self._server = None + + async def wg_handle_tcp_connection(self, stream: mitmproxy_rs.TcpStream) -> None: + await self.handle_tcp_connection(stream, stream) - addrs = " and ".join({human.format_address(a) for a in self.listen_addrs}) - logger.info(f"Stopped {self.mode.description} at {addrs}.") + +class OsProxyInstance(ServerInstance[mode_specs.OsProxyMode]): + _server: ClassVar[mitmproxy_rs.OsProxy | None] = None + """The OsProxy server. Will be started once and then reused for all future instances.""" + _instance: ClassVar[OsProxyInstance | None] = None + """The current OsProxy Instance. Will be unset again if an instance is stopped.""" + listen_addrs = () @property - def listen_addrs(self) -> tuple[Address, ...]: - return self._listen_addrs + def is_running(self) -> bool: + return self._instance is not None - async def wg_handle_tcp_connection(self, stream: mitmproxy_rs.TcpStream) -> None: - await self.handle_tcp_connection(stream, stream) + def make_top_layer(self, context: Context) -> Layer: + return layers.modes.TransparentProxy(context) + + @classmethod + async def os_handle_tcp_connection(cls, stream: mitmproxy_rs.TcpStream) -> None: + if cls._instance is not None: + await cls._instance.handle_tcp_connection(stream, stream) + + @classmethod + def os_handle_datagram( + cls, + transport: mitmproxy_rs.DatagramTransport, + data: bytes, + remote_addr: Address, + local_addr: Address, + ) -> None: + if cls._instance is not None: + cls._instance.handle_udp_datagram( + transport=transport, + data=data, + remote_addr=remote_addr, + local_addr=local_addr, + ) + + async def _start(self) -> None: + if self._instance: + raise RuntimeError("Cannot spawn more than one OS proxy instance.") + + if self.mode.data.startswith("!"): + spec = f"{self.mode.data},{os.getpid()}" + elif self.mode.data: + spec = self.mode.data + else: + spec = f"!{os.getpid()}" + + cls = self.__class__ + cls._instance = self # assign before awaiting to avoid races + if cls._server is None: + try: + cls._server = await mitmproxy_rs.start_os_proxy( + cls.os_handle_tcp_connection, + cls.os_handle_datagram, + ) + except Exception: + cls._instance = None + raise + + cls._server.set_intercept(spec) + + async def _stop(self) -> None: + assert self._instance + assert self._server + self.__class__._instance = None + # We're not shutting down the server because we want to avoid additional UAC prompts. + self._server.set_intercept("") class RegularInstance(AsyncioServerInstance[mode_specs.RegularMode]): diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index b13762ced7..8bffb3b532 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -30,6 +30,8 @@ from typing import Literal from typing import TypeVar +import mitmproxy_rs + from mitmproxy.coretypes.serializable import Serializable from mitmproxy.net import server_spec @@ -85,7 +87,7 @@ def default_port(self) -> int: @property @abstractmethod - def transport_protocol(self) -> Literal["tcp", "udp"]: + def transport_protocol(self) -> Literal["tcp", "udp"] | None: """The transport protocol used by this mode's server.""" @classmethod @@ -189,7 +191,7 @@ def __post_init__(self) -> None: class TransparentMode(ProxyMode): """A transparent proxy, see https://docs.mitmproxy.org/dev/howto-transparent/""" - description = "transparent proxy" + description = "Transparent Proxy" transport_protocol = TCP def __post_init__(self) -> None: @@ -280,3 +282,14 @@ class WireGuardMode(ProxyMode): def __post_init__(self) -> None: pass + + +class OsProxyMode(ProxyMode): + """OS-level transparent proxy.""" + + description = "OS proxy" + transport_protocol = None + + def __post_init__(self) -> None: + # should not raise + mitmproxy_rs.OsProxy.describe_spec(self.data) diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 3027dc9208..40f01c501c 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -111,7 +111,7 @@ async def server_handler( await ps.setup_servers() # assert this can always be called without side effects tctx.configure(ps, server=False) - await caplog_async.await_log("Stopped HTTP(S) proxy at") + await caplog_async.await_log("stopped") if ps.servers.is_updating: async with ps.servers._lock: pass # wait until start/stop is finished. @@ -318,7 +318,7 @@ async def test_dns(caplog_async) -> None: w.write(b"\x00") await caplog_async.await_log("sent an invalid message") tctx.configure(ps, server=False) - await caplog_async.await_log("Stopped DNS server at") + await caplog_async.await_log("stopped") def test_validation_no_transparent(monkeypatch): @@ -384,7 +384,7 @@ def server_handler( assert repr(ps) == "Proxyserver(1 active conns)" assert len(ps.connections) == 1 tctx.configure(ps, server=False) - await caplog_async.await_log("Stopped reverse proxy to dtls") + await caplog_async.await_log("stopped") class H3EchoServer(QuicConnectionProtocol): @@ -793,7 +793,7 @@ async def test_reverse_http3_and_quic_stream( assert len(ps.connections) == 1 tctx.configure(ps, server=False) - await caplog_async.await_log(f"Stopped reverse proxy to {scheme}") + await caplog_async.await_log(f"stopped") @pytest.mark.parametrize("connection_strategy", ["lazy", "eager"]) @@ -829,7 +829,7 @@ async def test_reverse_quic_datagram(caplog_async, connection_strategy: str) -> assert await client.recv_datagram() == b"echo" tctx.configure(ps, server=False) - await caplog_async.await_log("Stopped reverse proxy to quic") + await caplog_async.await_log("stopped") async def test_regular_http3(caplog_async, monkeypatch) -> None: @@ -869,4 +869,4 @@ def open_connection_path( assert len(ps.connections) == 1 tctx.configure(ps, server=False) - await caplog_async.await_log("Stopped HTTP3 proxy") + await caplog_async.await_log("stopped") diff --git a/test/mitmproxy/proxy/layers/http/test_http2.py b/test/mitmproxy/proxy/layers/http/test_http2.py index 92f7ed4bab..40364a7085 100644 --- a/test/mitmproxy/proxy/layers/http/test_http2.py +++ b/test/mitmproxy/proxy/layers/http/test_http2.py @@ -86,6 +86,9 @@ def start_h2_client(tctx: Context, keepalive: int = 0) -> tuple[Playbook, FrameF def make_h2(open_connection: OpenConnection) -> None: + assert isinstance( + open_connection, OpenConnection + ), f"Expected OpenConnection event, not {open_connection}" open_connection.connection.alpn = b"h2" diff --git a/test/mitmproxy/proxy/layers/http/test_http_fuzz.py b/test/mitmproxy/proxy/layers/http/test_http_fuzz.py index 8e139e0c28..04c891f59e 100644 --- a/test/mitmproxy/proxy/layers/http/test_http_fuzz.py +++ b/test/mitmproxy/proxy/layers/http/test_http_fuzz.py @@ -273,7 +273,6 @@ def h2_frames(draw): def h2_layer(opts): tctx = _tctx() - tctx.options.http2_ping_keepalive = 0 tctx.client.alpn = b"h2" layer = http.HttpLayer(tctx, HTTPMode.regular) @@ -317,7 +316,7 @@ def test_fuzz_h2_request_mutations(chunks): def _tctx() -> context.Context: - return context.Context( + tctx = context.Context( connection.Client( peername=("client", 1234), sockname=("127.0.0.1", 8080), @@ -325,6 +324,8 @@ def _tctx() -> context.Context: ), opts, ) + tctx.options.http2_ping_keepalive = 0 + return tctx def _h2_response(chunks): diff --git a/test/mitmproxy/proxy/test_mode_servers.py b/test/mitmproxy/proxy/test_mode_servers.py index 917645fc8d..218764382a 100644 --- a/test/mitmproxy/proxy/test_mode_servers.py +++ b/test/mitmproxy/proxy/test_mode_servers.py @@ -5,12 +5,14 @@ from unittest.mock import MagicMock from unittest.mock import Mock +import mitmproxy_rs import pytest import mitmproxy.platform from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.net import udp from mitmproxy.proxy.mode_servers import DnsInstance +from mitmproxy.proxy.mode_servers import OsProxyInstance from mitmproxy.proxy.mode_servers import ServerInstance from mitmproxy.proxy.mode_servers import WireGuardServerInstance from mitmproxy.proxy.server import ConnectionHandler @@ -88,7 +90,7 @@ async def test_tcp_start_stop(caplog_async): assert await caplog_async.await_log("client disconnect") await inst.stop() - assert await caplog_async.await_log("Stopped HTTP(S) proxy") + assert await caplog_async.await_log("stopped") @pytest.mark.parametrize("failure", [True, False]) @@ -107,7 +109,7 @@ async def test_transparent(failure, monkeypatch, caplog_async): tctx.options.connection_strategy = "lazy" inst = ServerInstance.make("transparent@127.0.0.1:0", manager) await inst.start() - await caplog_async.await_log("proxy listening") + await caplog_async.await_log("listening") host, port, *_ = inst.listen_addrs[0] reader, writer = await asyncio.open_connection(host, port) @@ -123,7 +125,7 @@ async def test_transparent(failure, monkeypatch, caplog_async): assert await caplog_async.await_log("client disconnect") await inst.stop() - assert await caplog_async.await_log("Stopped transparent proxy") + assert await caplog_async.await_log("stopped") async def test_wireguard(tdata, monkeypatch, caplog): @@ -176,7 +178,7 @@ async def handle_client(self: ConnectionHandler): raise await inst.stop() - assert "Stopped WireGuard server" in caplog.text + assert "stopped" in caplog.text async def test_wireguard_generate_conf(tmp_path): @@ -205,7 +207,7 @@ async def test_wireguard_invalid_conf(tmp_path): # directory instead of filename inst = WireGuardServerInstance.make(f"wireguard:{tmp_path}", MagicMock()) - with pytest.raises(OSError): + with pytest.raises(ValueError, match="Invalid configuration file"): await inst.start() assert "Invalid configuration file" in repr(inst.last_exception) @@ -261,7 +263,7 @@ async def test_udp_start_stop(caplog_async): writer.close() await inst.stop() - assert await caplog_async.await_log("Stopped") + assert await caplog_async.await_log("stopped") async def test_udp_start_error(): @@ -297,3 +299,79 @@ async def test_udp_connection_reuse(monkeypatch): await asyncio.sleep(0) assert len(inst.manager.connections) == 1 + + +@pytest.fixture() +def patched_os_proxy(monkeypatch): + start_os_proxy = AsyncMock() + monkeypatch.setattr(mitmproxy_rs, "start_os_proxy", start_os_proxy) + # make sure _server and _instance are restored after this test + monkeypatch.setattr(OsProxyInstance, "_server", None) + monkeypatch.setattr(OsProxyInstance, "_instance", None) + return start_os_proxy + + +async def test_os_proxy(patched_os_proxy, caplog_async): + caplog_async.set_level("INFO") + + with taddons.context(): + inst = ServerInstance.make(f"osproxy", MagicMock()) + assert not inst.is_running + + await inst.start() + assert patched_os_proxy.called + assert await caplog_async.await_log("OS proxy started.") + assert inst.is_running + + await inst.stop() + assert await caplog_async.await_log("OS proxy stopped") + assert not inst.is_running + + # just called for coverage + inst.make_top_layer(MagicMock()) + + +async def test_os_proxy_startup_err(patched_os_proxy): + patched_os_proxy.side_effect = RuntimeError("OS proxy startup error") + + with taddons.context(): + inst = ServerInstance.make(f"osproxy:!curl", MagicMock()) + with pytest.raises(RuntimeError): + await inst.start() + assert not inst.is_running + + +async def test_multiple_os_proxies(patched_os_proxy): + manager = MagicMock() + + with taddons.context(): + inst1 = ServerInstance.make(f"osproxy:curl", manager) + await inst1.start() + + inst2 = ServerInstance.make(f"osproxy:wget", manager) + with pytest.raises( + RuntimeError, match="Cannot spawn more than one OS proxy instance" + ): + await inst2.start() + + +async def test_always_uses_current_instance(patched_os_proxy, monkeypatch): + manager = MagicMock() + + with taddons.context(): + inst1 = ServerInstance.make(f"osproxy:curl", manager) + await inst1.start() + await inst1.stop() + + handle_tcp, handle_udp = patched_os_proxy.await_args[0] + + inst2 = ServerInstance.make(f"osproxy:wget", manager) + await inst2.start() + + monkeypatch.setattr(inst2, "handle_tcp_connection", h_tcp := AsyncMock()) + await handle_tcp(Mock()) + assert h_tcp.await_count + + monkeypatch.setattr(inst2, "handle_udp_datagram", h_udp := Mock()) + handle_udp(Mock(), b"", ("", 0), ("", 0)) + assert h_udp.called diff --git a/test/mitmproxy/proxy/test_mode_specs.py b/test/mitmproxy/proxy/test_mode_specs.py index b52c721968..be95990e33 100644 --- a/test/mitmproxy/proxy/test_mode_specs.py +++ b/test/mitmproxy/proxy/test_mode_specs.py @@ -70,6 +70,8 @@ def test_parse_specific_modes(): assert ProxyMode.parse("wireguard:foo.conf").data == "foo.conf" assert ProxyMode.parse("wireguard@51821").listen_port() == 51821 + assert ProxyMode.parse("osproxy") + with pytest.raises(ValueError, match="invalid port"): ProxyMode.parse("regular@invalid-port") @@ -87,3 +89,6 @@ def test_parse_specific_modes(): with pytest.raises(ValueError, match="Port specification missing."): ProxyMode.parse("reverse:dtls://127.0.0.1") + + with pytest.raises(ValueError, match="invalid intercept spec"): + ProxyMode.parse("osproxy:,,,") diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 66816da6b5..249ed00d64 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Optional from unittest import mock +from unittest.mock import Mock import pytest import tornado.testing @@ -176,8 +177,13 @@ async def make_master() -> webmaster.WebMaster: m.events._add_log(log.LogEntry("test log", "info")) m.events.done() si1 = ServerInstance.make("regular", m.proxyserver) - si1._listen_addrs = [("127.0.0.1", 8080), ("::1", 8080)] - si1._server = True # spoof is_running + sock1 = Mock() + sock1.getsockname.return_value = ("127.0.0.1", 8080) + sock2 = Mock() + sock2.getsockname.return_value = ("::1", 8080) + server = Mock() + server.sockets = [sock1, sock2] + si1._server = server si2 = ServerInstance.make("reverse:example.com", m.proxyserver) si2.last_exception = RuntimeError("I failed somehow.") si3 = ServerInstance.make("socks5", m.proxyserver) From b9f357472800e35b179fbdb4b8ba5cdda23f3dfc Mon Sep 17 00:00:00 2001 From: Sujal Singh Date: Mon, 6 Feb 2023 22:05:16 +0530 Subject: [PATCH 181/695] fix `Host` and `:authority` header not being updated on changes to host or port (#5908) * fix map remote addon not setting `Host` header correctly * fix failing tests * fix coverage * fix host and authority headers * raise error when port is not of type int * fix nits * [autofix.ci] apply automated fixes * coverage++ --------- Co-authored-by: Maximilian Hils Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 3 +++ mitmproxy/http.py | 22 +++++++++++++++------- test/mitmproxy/addons/test_mapremote.py | 10 ++++++++++ test/mitmproxy/test_http.py | 15 +++++++++------ 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0175badf..42e2ae9d79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ ([#5851](https://github.com/mitmproxy/mitmproxy/pull/5851), @italankin) * Removed string escaping in raw view. ([#5470](https://github.com/mitmproxy/mitmproxy/issues/5470), @stephenspol) +* Updating `Request.port` now also updates the Host header if present. + This aligns with `Request.host`, which already does this. + ([#5908](https://github.com/mitmproxy/mitmproxy/pull/5908), @sujaldev) ### Breaking Changes diff --git a/mitmproxy/http.py b/mitmproxy/http.py index f9f32beb4c..05729ef90d 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -749,13 +749,7 @@ def host(self) -> str: @host.setter def host(self, val: Union[str, bytes]) -> None: self.data.host = always_str(val, "idna", "strict") - - # Update host header - if "Host" in self.data.headers: - self.data.headers["Host"] = val - # Update authority - if self.data.authority: - self.authority = url.hostport(self.scheme, self.host, self.port) + self._update_host_and_authority() @property def host_header(self) -> Optional[str]: @@ -794,7 +788,21 @@ def port(self) -> int: @port.setter def port(self, port: int) -> None: + if not isinstance(port, int): + raise ValueError(f"Port must be an integer, not {port!r}.") + self.data.port = port + self._update_host_and_authority() + + def _update_host_and_authority(self) -> None: + val = url.hostport(self.scheme, self.host, self.port) + + # Update host header + if "Host" in self.data.headers: + self.data.headers["Host"] = val + # Update authority + if self.data.authority: + self.authority = val @property def path(self) -> str: diff --git a/test/mitmproxy/addons/test_mapremote.py b/test/mitmproxy/addons/test_mapremote.py index 2aff3468d9..f20ca20efd 100644 --- a/test/mitmproxy/addons/test_mapremote.py +++ b/test/mitmproxy/addons/test_mapremote.py @@ -27,6 +27,16 @@ def test_simple(self): mr.request(f) assert f.request.url == "https://mitmproxy.org/img/test.jpg" + def test_host_header(self): + mr = mapremote.MapRemote() + with taddons.context(mr) as tctx: + tctx.configure(mr, map_remote=["|http://[^/]+|http://example.com:4444"]) + f = tflow.tflow() + f.request.url = b"http://example.org/example" + f.request.headers["Host"] = "example.org" + mr.request(f) + assert f.request.headers.get("Host", "") == "example.com:4444" + def test_is_killed(self): mr = mapremote.MapRemote() with taddons.context(mr) as tctx: diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index acdff67d9a..142a49d841 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -2,6 +2,7 @@ import email import json import time +from typing import Any from unittest import mock import pytest @@ -10,6 +11,7 @@ from mitmproxy import flowfilter from mitmproxy.http import Headers from mitmproxy.http import HTTPFlow +from mitmproxy.http import Message from mitmproxy.http import Request from mitmproxy.http import Response from mitmproxy.net.http.cookies import CookieAttrs @@ -158,7 +160,9 @@ def test_scheme(self): _test_decoded_attr(treq(), "scheme") def test_port(self): - _test_passthrough_attr(treq(), "port") + _test_passthrough_attr(treq(), "port", 1234) + with pytest.raises(ValueError): + treq().port = "foo" def test_path(self): _test_decoded_attr(treq(), "path") @@ -199,8 +203,7 @@ def test_host_update_also_updates_header(self): request.headers["Host"] = "foo" request.authority = "foo" request.host = "example.org" - assert request.headers["Host"] == "example.org" - assert request.authority == "example.org:22" + assert request.headers["Host"] == request.authority == "example.org:22" def test_get_host_header(self): no_hdr = treq() @@ -864,10 +867,10 @@ def test_items(self): ] -def _test_passthrough_attr(message, attr): +def _test_passthrough_attr(message: Message, attr: str, value: Any = b"foo") -> None: assert getattr(message, attr) == getattr(message.data, attr) - setattr(message, attr, b"foo") - assert getattr(message.data, attr) == b"foo" + setattr(message, attr, value) + assert getattr(message.data, attr) == value def _test_decoded_attr(message, attr): From 430833e3d9651bb5fe2a32c882643e61c815a66c Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 7 Feb 2023 16:57:24 +0800 Subject: [PATCH 182/695] Fix server addr issue in tls_passthrough example. (#5904) * fix server peername is None issue peername would be None, we should use other not None property as key. * Update tls_passthrough.py --------- Co-authored-by: Maximilian Hils --- CHANGELOG.md | 3 +++ examples/contrib/tls_passthrough.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42e2ae9d79..3f18e7743b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased: mitmproxy next +* Fix a bug where peername would be None in tls_passthrough script, which would make it not working. + ([#5904](https://github.com/mitmproxy/mitmproxy/pull/5904), @truebit) + * Add experimental QUIC support. ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) * ASGI/WSGI apps can now listen on all ports for a specific hostname. diff --git a/examples/contrib/tls_passthrough.py b/examples/contrib/tls_passthrough.py index ab50d41915..54b7058220 100644 --- a/examples/contrib/tls_passthrough.py +++ b/examples/contrib/tls_passthrough.py @@ -95,22 +95,27 @@ def configure(self, updated): else: self.strategy = ConservativeStrategy() + @staticmethod + def get_addr(server: connection.Server): + # .peername may be unset in upstream proxy mode, so we need a fallback. + return server.peername or server.address + def tls_clienthello(self, data: tls.ClientHelloData): - server_address = data.context.server.peername + server_address = self.get_addr(data.context.server) if not self.strategy.should_intercept(server_address): logging.info(f"TLS passthrough: {human.format_address(server_address)}.") data.ignore_connection = True self.strategy.record_skipped(server_address) def tls_established_client(self, data: tls.TlsData): - server_address = data.context.server.peername + server_address = self.get_addr(data.context.server) logging.info( f"TLS handshake successful: {human.format_address(server_address)}" ) self.strategy.record_success(server_address) def tls_failed_client(self, data: tls.TlsData): - server_address = data.context.server.peername + server_address = self.get_addr(data.context.server) logging.info(f"TLS handshake failed: {human.format_address(server_address)}") self.strategy.record_failure(server_address) From 7da3a8e871cfc02ff2a82a797bf3da2388aee032 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 7 Feb 2023 11:29:08 +0100 Subject: [PATCH 183/695] treat multipart as bytes, not str. fixes #5148 (#5917) --- CHANGELOG.md | 2 + mitmproxy/contentviews/multipart.py | 2 +- mitmproxy/http.py | 28 ++++++------ mitmproxy/net/http/multipart.py | 43 +++++++++++++++---- mitmproxy/tools/console/grideditor/editors.py | 2 +- test/mitmproxy/net/http/test_multipart.py | 28 +++++++----- test/mitmproxy/test_http.py | 2 +- 7 files changed, 72 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f18e7743b..6670055d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ * Updating `Request.port` now also updates the Host header if present. This aligns with `Request.host`, which already does this. ([#5908](https://github.com/mitmproxy/mitmproxy/pull/5908), @sujaldev) +* Fix editing of multipart HTTP requests from the CLI. + ([#5148](https://github.com/mitmproxy/mitmproxy/issues/5148), @mhils) ### Breaking Changes diff --git a/mitmproxy/contentviews/multipart.py b/mitmproxy/contentviews/multipart.py index a8cef5f660..df27e53b33 100644 --- a/mitmproxy/contentviews/multipart.py +++ b/mitmproxy/contentviews/multipart.py @@ -16,7 +16,7 @@ def _format(v): def __call__(self, data: bytes, content_type: Optional[str] = None, **metadata): if content_type is None: return - v = multipart.decode(content_type, data) + v = multipart.decode_multipart(content_type, data) if v: return "Multipart form", self._format(v) diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 05729ef90d..4556678cf2 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -8,6 +8,7 @@ from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Mapping +from collections.abc import Sequence from dataclasses import dataclass from dataclasses import fields from email.utils import formatdate @@ -963,7 +964,7 @@ def _get_urlencoded_form(self): return tuple(url.decode(self.get_text(strict=False))) return () - def _set_urlencoded_form(self, form_data): + def _set_urlencoded_form(self, form_data: Sequence[tuple[str, str]]) -> None: """ Sets the body to the URL-encoded form data, and adds the appropriate content-type header. This will overwrite the existing content if there is one. @@ -989,23 +990,22 @@ def urlencoded_form(self) -> multidict.MultiDictView[str, str]: def urlencoded_form(self, value): self._set_urlencoded_form(value) - def _get_multipart_form(self): + def _get_multipart_form(self) -> list[tuple[bytes, bytes]]: is_valid_content_type = ( "multipart/form-data" in self.headers.get("content-type", "").lower() ) if is_valid_content_type and self.content is not None: try: - return multipart.decode(self.headers.get("content-type"), self.content) + return multipart.decode_multipart( + self.headers.get("content-type"), self.content + ) except ValueError: pass - return () + return [] - def _set_multipart_form(self, value): - is_valid_content_type = ( - self.headers.get("content-type", "") - .lower() - .startswith("multipart/form-data") - ) + def _set_multipart_form(self, value: list[tuple[bytes, bytes]]) -> None: + ct = self.headers.get("content-type", "") + is_valid_content_type = ct.lower().startswith("multipart/form-data") if not is_valid_content_type: """ Generate a random boundary here. @@ -1014,8 +1014,10 @@ def _set_multipart_form(self, value): on generating the boundary. """ boundary = "-" * 20 + binascii.hexlify(os.urandom(16)).decode() - self.headers["content-type"] = f"multipart/form-data; boundary={boundary}" - self.content = multipart.encode(self.headers, value) + self.headers[ + "content-type" + ] = ct = f"multipart/form-data; boundary={boundary}" + self.content = multipart.encode_multipart(ct, value) @property def multipart_form(self) -> multidict.MultiDictView[bytes, bytes]: @@ -1032,7 +1034,7 @@ def multipart_form(self) -> multidict.MultiDictView[bytes, bytes]: ) @multipart_form.setter - def multipart_form(self, value): + def multipart_form(self, value: list[tuple[bytes, bytes]]) -> None: self._set_multipart_form(value) diff --git a/mitmproxy/net/http/multipart.py b/mitmproxy/net/http/multipart.py index 4685d80e01..bc53af66a0 100644 --- a/mitmproxy/net/http/multipart.py +++ b/mitmproxy/net/http/multipart.py @@ -1,23 +1,25 @@ +from __future__ import annotations + import mimetypes import re +import warnings from typing import Optional from urllib.parse import quote from mitmproxy.net.http import headers -def encode(head, l): - k = head.get("content-type") - if k: - k = headers.parse_content_type(k) - if k is not None: +def encode_multipart(content_type: str, parts: list[tuple[bytes, bytes]]) -> bytes: + if content_type: + ct = headers.parse_content_type(content_type) + if ct is not None: try: - boundary = k[2]["boundary"].encode("ascii") - boundary = quote(boundary) + raw_boundary = ct[2]["boundary"].encode("ascii") + boundary = quote(raw_boundary) except (KeyError, UnicodeError): return b"" hdrs = [] - for key, value in l: + for key, value in parts: file_type = ( mimetypes.guess_type(str(key))[0] or "text/plain; charset=utf-8" ) @@ -41,9 +43,12 @@ def encode(head, l): hdrs.append(b"--%b--\r\n" % boundary.encode("utf-8")) temp = b"\r\n".join(hdrs) return temp + return b"" -def decode(content_type: Optional[str], content: bytes) -> list[tuple[bytes, bytes]]: +def decode_multipart( + content_type: Optional[str], content: bytes +) -> list[tuple[bytes, bytes]]: """ Takes a multipart boundary encoded string and returns list of (key, value) tuples. """ @@ -69,3 +74,23 @@ def decode(content_type: Optional[str], content: bytes) -> list[tuple[bytes, byt r.append((key, value)) return r return [] + + +def encode(ct, parts): # pragma: no cover + # 2023-02 + warnings.warn( + "multipart.encode is deprecated, use multipart.encode_multipart instead.", + DeprecationWarning, + stacklevel=2, + ) + return encode_multipart(ct, parts) + + +def decode(ct, content): # pragma: no cover + # 2023-02 + warnings.warn( + "multipart.decode is deprecated, use multipart.decode_multipart instead.", + DeprecationWarning, + stacklevel=2, + ) + return encode_multipart(ct, content) diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index f1b80b8171..6f4682e2a4 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -51,7 +51,7 @@ def set_data(self, vals, flow): class RequestMultipartEditor(base.FocusEditor): title = "Edit Multipart Form" - columns = [col_text.Column("Key"), col_text.Column("Value")] + columns = [col_bytes.Column("Key"), col_bytes.Column("Value")] def get_data(self, flow): return flow.request.multipart_form.items(multi=True) diff --git a/test/mitmproxy/net/http/test_multipart.py b/test/mitmproxy/net/http/test_multipart.py index 1045d70c60..b1d83b654e 100644 --- a/test/mitmproxy/net/http/test_multipart.py +++ b/test/mitmproxy/net/http/test_multipart.py @@ -1,6 +1,5 @@ import pytest -from mitmproxy.http import Headers from mitmproxy.net.http import multipart @@ -15,23 +14,28 @@ def test_decode(): "value2\n" "--{0}--".format(boundary).encode() ) - form = multipart.decode(f"multipart/form-data; boundary={boundary}", content) + form = multipart.decode_multipart( + f"multipart/form-data; boundary={boundary}", content + ) assert len(form) == 2 assert form[0] == (b"field1", b"value1") assert form[1] == (b"field2", b"value2") boundary = "boundary茅莽" - result = multipart.decode(f"multipart/form-data; boundary={boundary}", content) + result = multipart.decode_multipart( + f"multipart/form-data; boundary={boundary}", content + ) assert result == [] - assert multipart.decode("", content) == [] + assert multipart.decode_multipart("", content) == [] def test_encode(): data = [(b"file", b"shell.jpg"), (b"file_size", b"1000")] - headers = Headers(content_type="multipart/form-data; boundary=127824672498") - content = multipart.encode(headers, data) + content = multipart.encode_multipart( + "multipart/form-data; boundary=127824672498", data + ) assert b'Content-Disposition: form-data; name="file"' in content assert ( @@ -42,9 +46,13 @@ def test_encode(): assert len(content) == 252 with pytest.raises(ValueError, match=r"boundary found in encoded string"): - multipart.encode(headers, [(b"key", b"--127824672498")]) + multipart.encode_multipart( + "multipart/form-data; boundary=127824672498", [(b"key", b"--127824672498")] + ) - boundary = "boundary茅莽" - headers = Headers(content_type="multipart/form-data; boundary=" + boundary) - result = multipart.encode(headers, data) + result = multipart.encode_multipart( + "multipart/form-data; boundary=boundary茅莽", data + ) assert result == b"" + + assert multipart.encode_multipart("", data) == b"" diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index 142a49d841..c20eb3977b 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -434,7 +434,7 @@ def test_get_multipart_form(self): request.headers["Content-Type"] = "multipart/form-data" assert list(request.multipart_form.items()) == [] - with mock.patch("mitmproxy.net.http.multipart.decode") as m: + with mock.patch("mitmproxy.net.http.multipart.decode_multipart") as m: m.side_effect = ValueError assert list(request.multipart_form.items()) == [] From 8f780524939e5b9262b4b9c91c35707e11b33495 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 7 Feb 2023 11:35:11 +0100 Subject: [PATCH 184/695] update to mypy 1.0 (#5918) --- mitmproxy/connection.py | 8 ++++++-- tox.ini | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index f18880c92d..b7f4ad2f5c 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -197,7 +197,9 @@ def __str__(self): tls_state = ", tls" else: tls_state = "" - return f"Client({human.format_address(self.peername)}, state={self.state.name.lower()}{tls_state})" + state = self.state.name + assert state + return f"Client({human.format_address(self.peername)}, state={state.lower()}{tls_state})" @property def address(self): # pragma: no cover @@ -296,7 +298,9 @@ def __str__(self): local_port = f", src_port={self.sockname[1]}" else: local_port = "" - return f"Server({human.format_address(self.address)}, state={self.state.name.lower()}{tls_state}{local_port})" + state = self.state.name + assert state + return f"Server({human.format_address(self.address)}, state={state.lower()}{tls_state}{local_port})" def __setattr__(self, name, value): if name in ("address", "via"): diff --git a/tox.ini b/tox.ini index b41164d32a..5f3c0df805 100644 --- a/tox.ini +++ b/tox.ini @@ -29,13 +29,13 @@ commands = [testenv:mypy] deps = - mypy==0.991 + mypy==1.0.0 types-certifi==2021.10.8.3 types-Flask==1.1.6 types-Werkzeug==1.0.9 - types-requests==2.28.11.5 + types-requests==2.28.11.11 types-cryptography==3.3.23.2 - types-pyOpenSSL==22.1.0.2 + types-pyOpenSSL==23.0.0.2 -e .[dev] commands = From 41555edab98034fcc669ef90f25753533ab93327 Mon Sep 17 00:00:00 2001 From: Pradyot Ranjan <99216956+pradyotRanjan@users.noreply.github.com> Date: Thu, 9 Feb 2023 16:08:27 +0530 Subject: [PATCH 185/695] Reformatted list (#5919) * Reformatted list * [autofix.ci] apply automated fixes * Reformatted list * [autofix.ci] apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- mitmproxy/contentviews/msgpack.py | 6 ++--- test/mitmproxy/contentviews/test_msgpack.py | 30 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/mitmproxy/contentviews/msgpack.py b/mitmproxy/contentviews/msgpack.py index 92aeb8b39c..8de6c9abbf 100644 --- a/mitmproxy/contentviews/msgpack.py +++ b/mitmproxy/contentviews/msgpack.py @@ -64,11 +64,11 @@ def format_msgpack( elif type(data) is list: output[-1] += [("text", "[")] - for item in data: + + for count, item in enumerate(data): output.append([indent, ("text", " ")]) format_msgpack(item, output, indent_count + 1) - - if item != data[-1]: + if count != len(data) - 1: output[-1] += [("text", ",")] output.append([indent, ("text", "]")]) diff --git a/test/mitmproxy/contentviews/test_msgpack.py b/test/mitmproxy/contentviews/test_msgpack.py index 65c8487f0b..d3c53d3a97 100644 --- a/test/mitmproxy/contentviews/test_msgpack.py +++ b/test/mitmproxy/contentviews/test_msgpack.py @@ -57,7 +57,9 @@ def test_format_msgpack(): [("text", ""), ("text", "}")], ] - assert list(msgpack.format_msgpack({"object": {"key": "value"}, "list": [1]})) == [ + assert list( + msgpack.format_msgpack({"object": {"key": "value"}, "list": [0, 0, 1, 0, 0]}) + ) == [ [("text", "{")], [ ("text", ""), @@ -81,7 +83,31 @@ def test_format_msgpack(): ("text", ": "), ("text", "["), ], - [("text", " "), ("text", " "), ("Token_Literal_Number", "1")], + [ + ("text", " "), + ("text", " "), + ("Token_Literal_Number", "0"), + ("text", ","), + ], + [ + ("text", " "), + ("text", " "), + ("Token_Literal_Number", "0"), + ("text", ","), + ], + [ + ("text", " "), + ("text", " "), + ("Token_Literal_Number", "1"), + ("text", ","), + ], + [ + ("text", " "), + ("text", " "), + ("Token_Literal_Number", "0"), + ("text", ","), + ], + [("text", " "), ("text", " "), ("Token_Literal_Number", "0")], [("text", " "), ("text", "]")], [("text", ""), ("text", "}")], ] From 5969f25db4cbbc3edc91947205b89edca58f0818 Mon Sep 17 00:00:00 2001 From: Maksym Medvied <5236517+medvied@users.noreply.github.com> Date: Thu, 9 Feb 2023 14:43:47 +0400 Subject: [PATCH 186/695] pass signals to mitmproxy in docker-entrypoint.sh (#5920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Current docker-entrypoint.sh [0][1] runs gosu mitmproxy "$@" for mitmproxy, mitmdump and mitmweb. There is a problem with this approach: bash becomes a parent process for mitmproxy [2][3], but when signals are sent by docker-compose to mitmproxy container they are sent to bash, but they are not delivered to mitmproxy [4]. This leads to a slow shutdown of the container, because by default docker sends SIGTERM, waits for 10 seconds and then sends SIGKILL if the container is still alive [5]. This patch solves the issue by replacing bash process with mitmproxy entirely using "exec" - this way the signals are delivered to mitmproxy directly. To test the patch a Dockerfile [6] that applies the patch to the release image from the dockerhub could be used along with slighly modified compose.yml [7]. With the patch bash is no longer running inside the container [8] and the `docker compose down` time on my machine drops from 10.3s to 0.5s [9]. 0. https://github.com/mitmproxy/mitmproxy/blob/main/release/docker/docker-entrypoint.sh 1. To confirm that this is what's actually in the image: ``` > docker run mitmproxy/mitmproxy grep gosu /usr/local/bin/docker-entrypoint.sh gosu mitmproxy "$@" ``` 2. compose.yaml ``` services: mitmproxy-test: image: mitmproxy/mitmproxy command: ["mitmweb"] # https://github.com/mitmproxy/mitmproxy/issues/5727 stdin_open: true tty: true ``` 3. We can see that the parent PID for mitmweb is the pid of bash. ``` > docker compose up -d [+] Running 2/2 ⠿ Network mitmproxy_default Created 0.1s ⠿ Container mitmproxy-mitmproxy-test-1 Started 0.5s > docker compose top mitmproxy-mitmproxy-test-1 UID PID PPID C STIME TTY TIME CMD root 31227 31202 0 16:12 pts/0 00:00:00 /bin/bash /usr/local/bin/docker-entrypoint.sh mitmweb root 31314 31227 1 16:12 pts/0 00:00:01 /usr/local/bin/python /usr/local/bin/mitmweb ``` 4. https://unix.stackexchange.com/a/196053 5. https://docs.docker.com/compose/faq/#why-do-my-services-take-10-seconds-to-recreate-or-stop 6. Dockerfile: ``` FROM mitmproxy/mitmproxy RUN sed -i 's/^ gosu mitmproxy/ exec gosu mitmproxy/' /usr/local/bin/docker-entrypoint.sh ``` 7. compose.yaml to build an image from Dockerfile and use it: ``` services: mitmproxy-test: build: dockerfile: Dockerfile context: . command: ["mitmweb"] # https://github.com/mitmproxy/mitmproxy/issues/5727 stdin_open: true tty: true ``` 8. With the patch: ``` > docker compose top mitmproxy-mitmproxy-test-1 UID PID PPID C STIME TTY TIME CMD root 4994 4970 50 17:00 pts/0 00:00:02 /usr/local/bin/python /usr/local/bin/mitmweb ``` 9. Without the patch: ``` > docker compose down [+] Running 2/2 ⠿ Container mitmproxy-mitmproxy-test-1 Removed 10.2s ⠿ Network mitmproxy_default Removed 0.1s ``` With the patch: ``` > docker compose down [+] Running 2/2 ⠿ Container mitmproxy-mitmproxy-test-1 Removed 0.4s ⠿ Network mitmproxy_default Removed 0.1s ``` --- release/docker/docker-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/docker/docker-entrypoint.sh b/release/docker/docker-entrypoint.sh index 0cf853da6e..8a38c93ec1 100755 --- a/release/docker/docker-entrypoint.sh +++ b/release/docker/docker-entrypoint.sh @@ -18,7 +18,7 @@ usermod -o \ mitmproxy if [[ "$1" = "mitmdump" || "$1" = "mitmproxy" || "$1" = "mitmweb" ]]; then - gosu mitmproxy "$@" + exec gosu mitmproxy "$@" else exec "$@" fi From 09a83a89b299f05a33fd9d790a99ca1594b1379b Mon Sep 17 00:00:00 2001 From: konradh <49533343+konradh@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:22:05 +0100 Subject: [PATCH 187/695] console: fix bug that caused wrong direction indicators in message stream flow view (#5923) --- CHANGELOG.md | 2 ++ mitmproxy/tools/console/flowview.py | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6670055d33..39d55eab7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased: mitmproxy next +* Fix a bug where the direction indicator in the message stream view would be in the wrong direction. + ([#5921](https://github.com/mitmproxy/mitmproxy/issues/5921), @konradh) * Fix a bug where peername would be None in tls_passthrough script, which would make it not working. ([#5904](https://github.com/mitmproxy/mitmproxy/pull/5904), @truebit) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index f00d2fefe8..9f2298728d 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -255,21 +255,17 @@ def view_message_stream(self) -> urwid.Widget: viewmode = self.master.commands.call("console.flowview.mode") widget_lines = [] - - from_client = flow.messages[0].from_client for m in flow.messages: _, lines, _ = contentviews.get_message_content_view(viewmode, m, flow) for line in lines: - if from_client: + if m.from_client: line.insert(0, self.FROM_CLIENT_MARKER) else: line.insert(0, self.TO_CLIENT_MARKER) widget_lines.append(urwid.Text(line)) - from_client = not from_client - if flow.intercepted: markup = widget_lines[-1].get_text()[0] widget_lines[-1].set_text(("intercept", markup)) From 7e3380e6283b97d53fe852fe55c0474648f2320b Mon Sep 17 00:00:00 2001 From: Jurrie Overgoor <1213142+Jurrie@users.noreply.github.com> Date: Mon, 13 Feb 2023 17:07:24 +0100 Subject: [PATCH 188/695] Add a section on using Magisk and Magisk modules in the Android howto (#5924) * Add a section on using Magisk and Magisk modules in the Android howto This allows you to use Google Play builds with mitmproxy. * [autofix.ci] apply automated fixes * Explain how to get the Magisk module from mitmweb --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 2 + ...howto-install-system-trusted-ca-android.md | 78 +++++++++++++++++-- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39d55eab7c..9f330a4d1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ ([#5908](https://github.com/mitmproxy/mitmproxy/pull/5908), @sujaldev) * Fix editing of multipart HTTP requests from the CLI. ([#5148](https://github.com/mitmproxy/mitmproxy/issues/5148), @mhils) +* Added documentation on using Magisk module for intercepting traffic in Android production builds. + ([#5924](https://github.com/mitmproxy/mitmproxy/pull/5924), @Jurrie) ### Breaking Changes diff --git a/docs/src/content/howto-install-system-trusted-ca-android.md b/docs/src/content/howto-install-system-trusted-ca-android.md index b3bf5576ef..f18bc62021 100644 --- a/docs/src/content/howto-install-system-trusted-ca-android.md +++ b/docs/src/content/howto-install-system-trusted-ca-android.md @@ -16,7 +16,7 @@ Please note, that apps can decide to ignore the system certificate store and mai - [Android Studio/Android Sdk](https://developer.android.com/studio) is installed (tested with Version 4.1.3 for Linux 64-bit) - An Android Virtual Device (AVD) was created. Setup documentation available [here](https://developer.android.com/studio/run/managing-avds) - - The AVD must not run a production build (these will prevent you from using `adb root`) + - AVD production builds (those labeled with "Google Play") will prevent you from using `adb root`. You need to use [the Magisk method]({{< ref "#instructions-when-using-magisk" >}}) if you need Google Play installed. - The proxy settings of the AVD are configured to use mitmproxy. Documentation [here](https://developer.android.com/studio/run/emulator-networking#proxy) - Emulator and adb executables from Android Sdk have been added to $PATH variable @@ -45,10 +45,70 @@ By default, the mitmproxy CA certificate is located in this file: `~/.mitmproxy/ ## 3. Insert certificate into system certificate store -Now we have to place our CA certificate inside the system certificate store located at `/system/etc/security/cacerts/` in the Android filesystem. By default, the `/system` partition is mounted as read-only. The following steps describe how to gain write permissions on the `/system` partition and how to copy the certificate created in the previous step. +Now we have to place our CA certificate inside the system certificate store located at `/system/etc/security/cacerts/` in the Android filesystem. By default, the `/system` partition is mounted as read-only. The following steps describe how to gain write permissions on the `/system` partition and how to copy the certificate created in [the previous step]({{< ref "#2-rename-certificate" >}}). -### Instructions for API LEVEL > 28 - Starting from API LEVEL 29 (Android 10), it seems to be impossible to mount the "/" partition as read-write. Google provided a [workaround for this issue](https://android.googlesource.com/platform/system/core/+/master/fs_mgr/README.overlayfs.md) using OverlayFS. Unfortunately, at the time of writing this (11. April 2021), the instructions in this workaround will result in your emulator getting stuck in a [boot loop](https://issuetracker.google.com/issues/144891973). Some smart guy on Stackoverflow [found a way](https://stackoverflow.com/questions/60867956/android-emulator-sdk-10-api-29-wont-start-after-remount-and-reboot) to get the `/system` directory writable anyway. +### Instructions when using Magisk +If you want to use a production build (labeled "Google Play"; it's those builds that have Google Play installed) you can use Magisk to obtain root in your AVD. +[Magisk](https://github.com/topjohnwu/Magisk) allows root on your Android device or emulator. + +See the [instructions here](https://github.com/shakalaca/MagiskOnEmulator) for installing Magisk on your AVD. +The instructions have been tested with API level 30, but are reportedly working with API levels 22 up to and including 30 and 'S' (except API level 28). +Note: the instructions say to start your AVD. Do not supply an `-http-proxy` directive to mitmproxy at this point. + +When you are done with that, your emulator will allow root. You can check this by running a terminal emulator and typing `su`. +Magisk should ask you if you want to grant root to the program. After granting this, typing `whoami` would display `root`. + +However, after you have installed Magisk, you can no longer start your emulator with `-writable-system`. It will cause a boot loop. (Start your AVD with `-show-kernel` to see the error.) +But you can install your mitmproxy certificate by putting it in a Magisk module, and installing that module. +Magisk will take care of copying your certificate to `/system/etc/security/cacerts/` during boot. + +#### Downloading the Magisk module from mitmweb +If you run mitmweb, you can get simply download the Magisk module instead of handcrafting it. +Stop your AVD, and start it again with `-http-proxy 127.0.0.1:8080` (or whatever IP and port combination you are running mitmweb's proxy on). + +Then, *inside* the AVD, start a browser and navigate to `http://mitm.it/cert/magisk`. +You will be prompted to download `mitmproxy-magisk-module.zip`, which is the Magisk module you need. Store that file somewhere (like in 'Downloads'). + +Then open up Magisk, click on `Modules` and install your module. + +Reboot your AVD. + +#### Creating the Magisk module containing your certificate +If you do not run mitmweb, you'll need to create a Magisk module yourself. +See [here](https://topjohnwu.github.io/Magisk/guides.html#magisk-modules) for in-depth information on Magisk modules, but basically it boils down to this: + +Create the following directories: +- `mitmproxycert` (this will be the root of your module) +- `mitmproxycert/com/google/android` +- `mitmproxycert/system/etc/security/cacerts` + +Place your renamed certificate from [step 2]({{< ref "#2-rename-certificate" >}}) inside `mitmproxycert/system/etc/security/cacerts` and `chmod 664` it. + +Save the content of [https://github.com/topjohnwu/Magisk/blob/master/scripts/module_installer.sh](https://github.com/topjohnwu/Magisk/blob/master/scripts/module_installer.sh) as a local file `update-binary` and place it inside `mitmproxycert/com/google/android`. + +Create a file named `updater-script` containing only the string `#MAGISK` and place it inside `mitmproxycert/com/google/android`. + +Create a file named `module.prop` and place it inside `mitmproxycert`. The file should contain something like: + +``` +id=mitmproxycert +name=MITM proxy certificate +version=1 +versionCode=1 +author=mitmproxycert +description=My shiny MITM proxy certificate to reveal all secrets and obtain world domination! +``` + +Zip the module using something like `cd ./mitmproxycert ; zip -r ./../mitmproxycert.zip ./` and push it to your running AVD using `adb push ./../mitmproxycert.zip /storage/emulated/0/Download/`. + +The go to your AVD, open up Magisk, click on `Modules` and install your module (you'll find it in the Downloads folder). + +Reboot your AVD. + +### Instructions for API LEVEL > 28 using `-writable-system` +By default, the `/system` partition is mounted as read-only. The following steps describe how to gain write permissions on the `/system` partition and how to copy the certificate created in chapter 2. + +Starting from API LEVEL 29 (Android 10), it seems to be impossible to mount the "/" partition as read-write. Google provided a [workaround for this issue](https://android.googlesource.com/platform/system/core/+/master/fs_mgr/README.overlayfs.md) using OverlayFS. Unfortunately, at the time of writing this (11. April 2021), the instructions in this workaround will result in your emulator getting stuck in a [boot loop](https://issuetracker.google.com/issues/144891973). Some smart guy on Stackoverflow [found a way](https://stackoverflow.com/questions/60867956/android-emulator-sdk-10-api-29-wont-start-after-remount-and-reboot) to get the `/system` directory writable anyway. **Keep in mind:** You always have to start the emulator using the `-writable-system` option if you want to use your certificate. Otherwise Android will load a "clean" system image. @@ -62,11 +122,11 @@ Tested on emulators running API LEVEL 29 and 30 - reboot device: `adb reboot` - restart adb as root: `adb root` - perform remount of partitions as read-write: `adb remount`. (If adb tells you that you need to reboot, reboot again `adb reboot` and run `adb remount` again.) - - push your renamed certificate from step 2: `adb push /system/etc/security/cacerts` + - push your renamed certificate from [step 2]({{< ref "#2-rename-certificate" >}}): `adb push /system/etc/security/cacerts` - set certificate permissions: `adb shell chmod 664 /system/etc/security/cacerts/` - reboot device: `adb reboot` -### Instructions for API LEVEL <= 28 +### Instructions for API LEVEL <= 28 using `-writable-system` Tested on emulators running API LEVEL 26, 27 and 28 @@ -76,6 +136,10 @@ Tested on emulators running API LEVEL 26, 27 and 28 - Start the desired AVD: `emulator -avd -writable-system` (add `-show-kernel` flag for kernel logs) - restart adb as root: `adb root` - perform remount of partitions as read-write: `adb remount`. (If adb tells you that you need to reboot, reboot again `adb reboot` and run `adb remount` again.) - - push your renamed certificate from step 2: `adb push /system/etc/security/cacerts` + - push your renamed certificate from [step 2]({{< ref "#2-rename-certificate" >}}): `adb push /system/etc/security/cacerts` - set certificate permissions: `adb shell chmod 664 /system/etc/security/cacerts/` - reboot device: `adb reboot` + +### Testing that your certificate is loaded from the system certificate store + +In your AVD, go to Settings → Security → Advanced → Encryption & credentials → Trusted credentials. Find your certificate (default name is `mitmproxy`) in the list. From 244ff35e606e8ae7ca8aa15d7529566a97ec69f5 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 13 Feb 2023 22:45:02 +0100 Subject: [PATCH 189/695] fix usage of `asyncio.create_task` (#5928) this fixes #5926 --- examples/addons/websocket-inject-message.py | 9 ++++++++- mitmproxy/addons/clientplayback.py | 9 ++++++++- mitmproxy/addons/keepserving.py | 6 +++++- mitmproxy/addons/proxyserver.py | 3 ++- mitmproxy/addons/readfile.py | 13 ++++++------- mitmproxy/addons/termlog.py | 4 +++- mitmproxy/proxy/mode_servers.py | 4 +++- mitmproxy/proxy/server.py | 7 ++++++- mitmproxy/tools/web/app.py | 11 +++++++---- test/bench/benchmark.py | 2 +- test/conftest.py | 2 +- test/mitmproxy/test_master.py | 14 ++++++++++---- 12 files changed, 60 insertions(+), 24 deletions(-) diff --git a/examples/addons/websocket-inject-message.py b/examples/addons/websocket-inject-message.py index 5916edc1a5..722cab6914 100644 --- a/examples/addons/websocket-inject-message.py +++ b/examples/addons/websocket-inject-message.py @@ -34,5 +34,12 @@ async def inject_async(flow: http.HTTPFlow): msg = msg[1:] + msg[:1] +# Python 3.11: replace with TaskGroup +tasks = set() + + def websocket_start(flow: http.HTTPFlow): - asyncio.create_task(inject_async(flow)) + # we need to hold a reference to the task, otherwise it will be garbage collected. + t = asyncio.create_task(inject_async(flow)) + tasks.add(t) + t.add_done_callback(tasks.remove) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 7abb83622b..e6d7a85243 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import logging import time @@ -136,11 +138,13 @@ class ClientPlayback: inflight: Optional[http.HTTPFlow] queue: asyncio.Queue options: Options + replay_tasks: set[asyncio.Task] def __init__(self): self.queue = asyncio.Queue() self.inflight = None self.task = None + self.replay_tasks = set() def running(self): self.playback_task = asyncio_utils.create_task( @@ -159,9 +163,12 @@ async def playback(self): assert self.inflight h = ReplayHandler(self.inflight, self.options) if ctx.options.client_replay_concurrency == -1: - asyncio_utils.create_task( + t = asyncio_utils.create_task( h.replay(), name="client playback awaiting response" ) + # keep a reference so this is not garbage collected + self.replay_tasks.add(t) + t.add_done_callback(self.replay_tasks.remove) else: await h.replay() except Exception: diff --git a/mitmproxy/addons/keepserving.py b/mitmproxy/addons/keepserving.py index 199cf25487..efe296ab36 100644 --- a/mitmproxy/addons/keepserving.py +++ b/mitmproxy/addons/keepserving.py @@ -1,9 +1,13 @@ +from __future__ import annotations + import asyncio from mitmproxy import ctx class KeepServing: + _watch_task: asyncio.Task | None = None + def load(self, loader): loader.add_option( "keepserving", @@ -40,4 +44,4 @@ def running(self): ctx.options.rfile, ] if any(opts) and not ctx.options.keepserving: - asyncio.get_running_loop().create_task(self.watch()) + self._watch_task = asyncio.get_running_loop().create_task(self.watch()) diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index cb17bcfbf0..10b6c7cc29 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -113,6 +113,7 @@ class Proxyserver(ServerManager): is_running: bool _connect_addr: Optional[Address] = None + _update_task: Optional[asyncio.Task] = None def __init__(self): self.connections = {} @@ -274,7 +275,7 @@ def configure(self, updated) -> None: ) if self.is_running: - asyncio.create_task(self.servers.update(modes)) + self._update_task = asyncio.create_task(self.servers.update(modes)) async def setup_servers(self) -> bool: return await self.servers.update( diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py index e9f0a5cbf7..63f7b1dbde 100644 --- a/mitmproxy/addons/readfile.py +++ b/mitmproxy/addons/readfile.py @@ -19,7 +19,7 @@ class ReadFile: def __init__(self): self.filter = None - self.is_reading = False + self._read_task: asyncio.Task | None = None def load(self, loader): loader.add_option("rfile", Optional[str], None, "Read flows from file.") @@ -64,22 +64,21 @@ async def load_flows_from_path(self, path: str) -> int: logging.error(f"Cannot load flows: {e}") raise exceptions.FlowReadException(str(e)) from e - async def doread(self, rfile): - self.is_reading = True + async def doread(self, rfile: str) -> None: try: - await self.load_flows_from_path(ctx.options.rfile) + await self.load_flows_from_path(rfile) except exceptions.FlowReadException as e: raise exceptions.OptionsError(e) from e finally: - self.is_reading = False + self._read_task = None def running(self): if ctx.options.rfile: - asyncio.get_running_loop().create_task(self.doread(ctx.options.rfile)) + self._read_task = asyncio.create_task(self.doread(ctx.options.rfile)) @command.command("readfile.reading") def reading(self) -> bool: - return self.is_reading + return bool(self._read_task) class ReadFileStdin(ReadFile): diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 9da47e141e..f6d0038643 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -11,6 +11,8 @@ class TermLog: + _teardown_task: asyncio.Task | None = None + def __init__(self, out: IO[str] | None = None): self.logger = TermLogHandler(out) self.logger.install() @@ -29,7 +31,7 @@ def done(self): t = self._teardown() try: # try to delay teardown a bit. - asyncio.create_task(t) + self._teardown_task = asyncio.create_task(t) except RuntimeError: # no event loop, we're in a test. asyncio.run(t) diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 0d64157ae2..405348e6b8 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -233,7 +233,9 @@ def handle_udp_datagram( # pre-register here - we may get datagrams before the task is executed. self.manager.connections[connection_id] = handler - asyncio.create_task(self.handle_udp_connection(connection_id, handler)) + t = asyncio.create_task(self.handle_udp_connection(connection_id, handler)) + # assign it somewhere so that it does not get garbage-collected. + handler._handle_udp_task = t # type: ignore else: handler = self.manager.connections[connection_id] reader = cast(udp.DatagramReader, handler.transports[handler.client].reader) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index eeec51d5f2..7094ce31a9 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -107,12 +107,14 @@ class ConnectionHandler(metaclass=abc.ABCMeta): max_conns: collections.defaultdict[Address, asyncio.Semaphore] layer: "layer.Layer" wakeup_timer: set[asyncio.Task] + hook_tasks: set[asyncio.Task] def __init__(self, context: Context) -> None: self.client = context.client self.transports = {} self.max_conns = collections.defaultdict(lambda: asyncio.Semaphore(5)) self.wakeup_timer = set() + self.hook_tasks = set() # Ask for the first layer right away. # In a reverse proxy scenario, this is necessary as we would otherwise hang @@ -388,11 +390,14 @@ def server_event(self, event: events.Event) -> None: elif isinstance(command, commands.CloseConnection): self.close_connection(command.connection, False) elif isinstance(command, commands.StartHook): - asyncio_utils.create_task( + t = asyncio_utils.create_task( self.hook_task(command), name=f"handle_hook({command.name})", client=self.client.peername, ) + # Python 3.11 Use TaskGroup instead. + self.hook_tasks.add(t) + t.add_done_callback(self.hook_tasks.remove) elif isinstance(command, commands.Log): self.log(command.message, command.level) else: diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 334661cc1e..7eb71f72ff 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -284,6 +284,7 @@ def get(self): class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): # raise an error if inherited class doesn't specify its own instance. connections: ClassVar[set[WebSocketEventBroadcaster]] + _send_tasks: ClassVar[set[asyncio.Task]] = set() def open(self): self.connections.add(self) @@ -299,7 +300,9 @@ async def wrapper(): except tornado.websocket.WebSocketClosedError: cls.connections.discard(conn) - asyncio.ensure_future(wrapper()) + t = asyncio.create_task(wrapper()) + cls._send_tasks.add(t) + t.add_done_callback(cls._send_tasks.remove) @classmethod def broadcast(cls, **kwargs): @@ -345,11 +348,11 @@ def match(_) -> bool: fw.add(f) self.write(bio.getvalue()) - def post(self): + async def post(self): self.view.clear() bio = BytesIO(self.filecontents) - for i in io.FlowReader(bio).stream(): - asyncio.ensure_future(self.master.load_flow(i)) + for f in io.FlowReader(bio).stream(): + await self.master.load_flow(f) bio.close() diff --git a/test/bench/benchmark.py b/test/bench/benchmark.py index 289df5c164..b0d6902744 100644 --- a/test/bench/benchmark.py +++ b/test/bench/benchmark.py @@ -56,7 +56,7 @@ def load(self, loader): def running(self): if not self.started: self.started = True - asyncio.get_running_loop().create_task(self.procs()) + self._task = asyncio.create_task(self.procs()) def done(self): self.pr.dump_stats(ctx.options.benchmark_save_path + ".prof") diff --git a/test/conftest.py b/test/conftest.py index 77be0686b4..63d2164110 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -47,7 +47,7 @@ async def await_log(self, text, timeout=2): return True else: await asyncio.sleep(0.01) - raise AssertionError(f"Did not find {text!r} in log:\n{self.caplog.text}.") + raise AssertionError(f"Did not find {text!r} in log:\n{self.caplog.text}") def clear(self) -> None: self.caplog.clear() diff --git a/test/mitmproxy/test_master.py b/test/mitmproxy/test_master.py index df0d2c85ad..df7cea8360 100644 --- a/test/mitmproxy/test_master.py +++ b/test/mitmproxy/test_master.py @@ -7,11 +7,17 @@ async def err(): raise RuntimeError -async def test_exception_handler(caplog): +class TaskError: + def running(self): + # not assigned to anything + asyncio.create_task(err()) + + +async def test_exception_handler(caplog_async): + caplog_async.set_level("ERROR") m = Master(None) + m.addons.add(TaskError()) running = asyncio.create_task(m.run()) - asyncio.create_task(err()) - await asyncio.sleep(0) - assert "Traceback" in caplog.text + await caplog_async.await_log("Traceback") m.shutdown() await running From 01c1090ec1a6e7684f0a9ac49ccbbfee7d9878f3 Mon Sep 17 00:00:00 2001 From: James O'Claire Date: Sat, 25 Feb 2023 17:29:26 +0800 Subject: [PATCH 190/695] Was pointing to issue on archived mitmweb issue, instead point to active mitmproxy issue (#5951) --- web/src/js/filt/filt.peg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg index 8d428c7450..e966192da3 100644 --- a/web/src/js/filt/filt.peg +++ b/web/src/js/filt/filt.peg @@ -69,7 +69,7 @@ function body(regex){ function bodyFilter(flow){ return true; } - bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; + bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmproxy/issues/3609"; return bodyFilter; } @@ -79,7 +79,7 @@ function requestBody(regex){ function requestBodyFilter(flow){ return true; } - requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; + requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmproxy/issues/3609"; return requestBodyFilter; } @@ -89,7 +89,7 @@ function responseBody(regex){ function responseBodyFilter(flow){ return true; } - responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; + responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmproxy/issues/3609"; return responseBodyFilter; } From 5259d1e31aefe2d188f99cc521ec2ae0be040ecb Mon Sep 17 00:00:00 2001 From: WenzelTian Date: Sun, 26 Feb 2023 22:25:10 +0800 Subject: [PATCH 191/695] add the architecture determination when running the wireguard test (#5953) --- test/mitmproxy/proxy/test_mode_servers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/mitmproxy/proxy/test_mode_servers.py b/test/mitmproxy/proxy/test_mode_servers.py index 218764382a..f239a3e2e6 100644 --- a/test/mitmproxy/proxy/test_mode_servers.py +++ b/test/mitmproxy/proxy/test_mode_servers.py @@ -150,6 +150,10 @@ async def handle_client(self: ConnectionHandler): else: return pytest.skip("Unsupported platform for wg-test-client.") + arch = platform.machine() + if arch != "AMD64" and arch != "x86_64": + return pytest.skip("Unsupported architecture for wg-test-client.") + test_client_path = tdata.path(f"wg-test-client/{test_client_name}") test_conf = tdata.path(f"wg-test-client/test.conf") From 46bfb354881c74c57fab4f9d7c21f31212c08cda Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 27 Feb 2023 08:21:41 +0100 Subject: [PATCH 192/695] drop support for Python 3.9 --- .github/workflows/autofix.yml | 17 +++++++++-------- .github/workflows/main.yml | 2 -- CHANGELOG.md | 1 + CONTRIBUTING.md | 3 +-- docs/scripts/api-events.py | 13 ++++--------- docs/src/content/overview-installation.md | 2 +- mitmproxy/master.py | 5 +---- mitmproxy/utils/signals.py | 6 +----- setup.py | 3 +-- 9 files changed, 19 insertions(+), 33 deletions(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 00418de7e3..2e5f5316a0 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -15,30 +15,31 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: install-pinned/pyupgrade@423622e7c2088eeba495a591385ec22074284f90 + - uses: install-pinned/pyupgrade@28e8d2633f6f1a03d5b4709682ce155a66324e6a - name: Run pyupgrade run: | shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' - pyupgrade --exit-zero-even-if-changed --keep-runtime-typing --py39-plus **/*.py + pyupgrade --exit-zero-even-if-changed --keep-runtime-typing --py310-plus **/*.py - - uses: install-pinned/reorder_python_imports@946c8bbd8fe048a3bee76063c90c938d5a59a9aa + - uses: install-pinned/reorder_python_imports@2cc264e0f6bc33907796602661e5b26d8199314d - name: Run reorder-python-imports run: | shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' - reorder-python-imports --exit-zero-even-if-changed --py39-plus **/*.py - - uses: install-pinned/yesqa@4af1e53e86a56db346a03ece9e89c19bfd0e5d0e + reorder-python-imports --exit-zero-even-if-changed --py310-plus **/*.py + - uses: install-pinned/yesqa@4896f663e9c294fddfbf5f4e4fc4f9b1a4556658 - name: Run yesqa run: | shopt -s globstar export GLOBIGNORE='mitmproxy/contrib/**' yesqa **/*.py || true - - uses: install-pinned/autoflake@19ecc14a8688d57cca9dc6cfd705f16f200ff097 - - run: autoflake --in-place --remove-all-unused-imports --exclude contrib -r . - - uses: install-pinned/black@13c8a20eb904ba800c87f0b34ccfd932ac2ff81d + - uses: install-pinned/autoflake@dfa39c5f136f5b885c175734a719dc6ad1f11fc7 + - run: autoflake --in-place --remove-all-unused-imports --exclude mitmproxy/contrib -r . + + - uses: install-pinned/black@3375665f712256be11c3212db472c3dafc217fa1 - run: black --extend-exclude mitmproxy/contrib . - uses: mhils/add-pr-ref-in-changelog@main diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ebca48931..514d62ee8f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,8 +49,6 @@ jobs: py: "3.11" - os: ubuntu-latest py: "3.10" - - os: ubuntu-latest - py: "3.9" runs-on: ${{ matrix.os }} steps: - run: printenv diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f330a4d1f..5543af0e1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased: mitmproxy next +* mitmproxy now requires Python 3.10 or above. * Fix a bug where the direction indicator in the message stream view would be in the wrong direction. ([#5921](https://github.com/mitmproxy/mitmproxy/issues/5921), @konradh) * Fix a bug where peername would be None in tls_passthrough script, which would make it not working. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f0496090a..8541833f08 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,8 +14,7 @@ forward, please consider contributing in the following areas: ## Development Setup -To get started hacking on mitmproxy, please install a recent version of Python (we require at least Python 3.9). -Then, do the following: +To get started hacking on mitmproxy, please install the latest version of Python and do the following: ##### Linux / macOS diff --git a/docs/scripts/api-events.py b/docs/scripts/api-events.py index 8f3a72e726..8e70076390 100644 --- a/docs/scripts/api-events.py +++ b/docs/scripts/api-events.py @@ -2,6 +2,7 @@ import contextlib import inspect import textwrap +import typing from pathlib import Path from mitmproxy import addonmanager @@ -33,15 +34,9 @@ def category(name: str, desc: str, hooks: list[type[hooks.Hook]]) -> None: for params in all_params: for param in params: try: - mod = inspect.getmodule(param.annotation).__name__ - if mod == "typing": - # this is ugly, but can be removed once we are on Python 3.9+ only - imports.add( - inspect.getmodule(param.annotation.__args__[0]).__name__ - ) - types.add(param.annotation._name) - else: - imports.add(mod) + imports.add(inspect.getmodule(param.annotation).__name__) + for t in typing.get_args(param.annotation): + imports.add(inspect.getmodule(t).__name__) except AttributeError: raise ValueError(f"Missing type annotation: {params}") imports.discard("builtins") diff --git a/docs/src/content/overview-installation.md b/docs/src/content/overview-installation.md index a65ae0fa49..8f7e86d5e8 100644 --- a/docs/src/content/overview-installation.md +++ b/docs/src/content/overview-installation.md @@ -66,7 +66,7 @@ While there are plenty of options around[^1], we recommend the installation usin packages. Most of them (pip, virtualenv, pipenv, etc.) should just work, but we don't have the capacity to provide support for it. -1. Install a recent version of Python (we require at least 3.9). +1. Install a recent version of Python (we require at least 3.10). 2. Install [pipx](https://pipxproject.github.io/pipx/). 3. `pipx install mitmproxy` diff --git a/mitmproxy/master.py b/mitmproxy/master.py index a8e6bfc345..3484600a8f 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -40,10 +40,7 @@ def __init__( # may want to spawn tasks during the initial configuration phase, # which happens before run(). self.event_loop = event_loop or asyncio.get_running_loop() - try: - self.should_exit = asyncio.Event() - except RuntimeError: # python 3.9 and below - self.should_exit = asyncio.Event(loop=self.event_loop) # type: ignore + self.should_exit = asyncio.Event() mitmproxy_ctx.master = self mitmproxy_ctx.log = self.log # deprecated, do not use. mitmproxy_ctx.options = self.options diff --git a/mitmproxy/utils/signals.py b/mitmproxy/utils/signals.py index cad5e5d1ff..4bdc7282fb 100644 --- a/mitmproxy/utils/signals.py +++ b/mitmproxy/utils/signals.py @@ -17,13 +17,9 @@ from typing import Any from typing import cast from typing import Generic +from typing import ParamSpec from typing import TypeVar -try: - from typing import ParamSpec -except ImportError: # pragma: no cover - # Python 3.9 - from typing_extensions import ParamSpec # type: ignore P = ParamSpec("P") R = TypeVar("R") diff --git a/setup.py b/setup.py index 4578e7752b..1c0f7c4330 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,6 @@ "Operating System :: POSIX", "Operating System :: Microsoft :: Windows", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", @@ -70,7 +69,7 @@ "hook-dirs = mitmproxy.utils.pyinstaller:hook_dirs", ], }, - python_requires=">=3.9", + python_requires=">=3.10", # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#install-requires # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ From 51670861e6f8c11e8309804cfe15d2599a05aee7 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 27 Feb 2023 07:28:30 +0000 Subject: [PATCH 193/695] [autofix.ci] apply automated fixes --- CHANGELOG.md | 1 + docs/scripts/clirecording/clidirector.py | 9 +- examples/addons/contentview.py | 14 +-- examples/addons/http-stream-modify.py | 3 +- examples/contrib/domain_fronting.py | 8 +- examples/contrib/http_manipulate_cookies.py | 7 +- examples/contrib/ntlm_upstream_proxy.py | 2 +- .../contrib/webscanner_helper/test_mapping.py | 2 +- examples/contrib/webscanner_helper/urldict.py | 5 +- .../contrib/webscanner_helper/urlindex.py | 8 +- .../contrib/webscanner_helper/watchdog.py | 5 +- examples/contrib/xss_scanner.py | 13 +- mitmproxy/addonmanager.py | 3 +- mitmproxy/addons/asgiapp.py | 5 +- mitmproxy/addons/browser.py | 7 +- mitmproxy/addons/clientplayback.py | 7 +- mitmproxy/addons/core.py | 3 +- mitmproxy/addons/cut.py | 7 +- mitmproxy/addons/dns_resolver.py | 7 +- mitmproxy/addons/dumper.py | 13 +- mitmproxy/addons/eventstore.py | 3 +- mitmproxy/addons/export.py | 3 +- mitmproxy/addons/intercept.py | 2 +- mitmproxy/addons/next_layer.py | 17 ++- mitmproxy/addons/proxyserver.py | 4 +- mitmproxy/addons/save.py | 6 +- mitmproxy/addons/script.py | 3 +- mitmproxy/addons/serverplayback.py | 3 +- mitmproxy/addons/stickycookie.py | 2 +- mitmproxy/addons/tlsconfig.py | 11 +- mitmproxy/addons/upstream_auth.py | 2 +- mitmproxy/addons/view.py | 14 +-- mitmproxy/certs.py | 28 ++--- mitmproxy/command.py | 9 +- mitmproxy/connection.py | 52 ++++---- mitmproxy/contentviews/__init__.py | 18 ++- mitmproxy/contentviews/base.py | 13 +- mitmproxy/contentviews/css.py | 3 +- mitmproxy/contentviews/graphql.py | 3 +- mitmproxy/contentviews/http3.py | 12 +- mitmproxy/contentviews/image/view.py | 3 +- mitmproxy/contentviews/javascript.py | 3 +- mitmproxy/contentviews/json.py | 3 +- mitmproxy/contentviews/mqtt.py | 3 +- mitmproxy/contentviews/msgpack.py | 3 +- mitmproxy/contentviews/multipart.py | 6 +- mitmproxy/contentviews/protobuf.py | 3 +- mitmproxy/contentviews/query.py | 6 +- mitmproxy/contentviews/urlencoded.py | 4 +- mitmproxy/contentviews/wbxml.py | 4 +- mitmproxy/contentviews/xml_html.py | 15 ++- mitmproxy/contrib/kaitaistruct/exif.py | 2 +- .../contrib/kaitaistruct/google_protobuf.py | 2 +- mitmproxy/contrib/kaitaistruct/ico.py | 2 +- .../contrib/kaitaistruct/vlq_base128_le.py | 2 +- mitmproxy/eventsequence.py | 2 +- mitmproxy/flow.py | 11 +- mitmproxy/flowfilter.py | 3 +- mitmproxy/http.py | 116 +++++++++--------- mitmproxy/io/compat.py | 5 +- mitmproxy/master.py | 3 +- mitmproxy/net/encoding.py | 13 +- mitmproxy/net/http/headers.py | 3 +- mitmproxy/net/http/http1/read.py | 7 +- mitmproxy/net/http/multipart.py | 3 +- mitmproxy/net/http/url.py | 5 +- mitmproxy/net/tls.py | 23 ++-- mitmproxy/net/udp.py | 5 +- mitmproxy/optmanager.py | 11 +- mitmproxy/platform/__init__.py | 5 +- mitmproxy/platform/windows.py | 9 +- mitmproxy/proxy/events.py | 3 +- mitmproxy/proxy/layer.py | 7 +- mitmproxy/proxy/layers/http/__init__.py | 26 ++-- mitmproxy/proxy/layers/http/_events.py | 3 +- mitmproxy/proxy/layers/http/_http1.py | 19 ++- mitmproxy/proxy/layers/http/_http2.py | 14 +-- mitmproxy/proxy/layers/http/_http3.py | 21 ++-- mitmproxy/proxy/layers/http/_http_h3.py | 13 +- .../proxy/layers/http/_upstream_proxy.py | 3 +- mitmproxy/proxy/layers/modes.py | 7 +- mitmproxy/proxy/layers/quic.py | 2 +- mitmproxy/proxy/layers/tcp.py | 3 +- mitmproxy/proxy/layers/tls.py | 19 ++- mitmproxy/proxy/layers/udp.py | 3 +- mitmproxy/proxy/server.py | 24 ++-- mitmproxy/proxy/tunnel.py | 7 +- mitmproxy/test/tflow.py | 24 ++-- mitmproxy/tls.py | 5 +- .../tools/console/commander/commander.py | 3 +- mitmproxy/tools/console/common.py | 58 +++++---- mitmproxy/tools/console/flowdetailview.py | 6 +- mitmproxy/tools/console/flowlist.py | 3 +- mitmproxy/tools/console/flowview.py | 3 +- mitmproxy/tools/console/grideditor/base.py | 11 +- mitmproxy/tools/console/grideditor/editors.py | 7 +- mitmproxy/tools/console/keymap.py | 9 +- mitmproxy/tools/console/palettes.py | 3 +- mitmproxy/tools/console/quickhelp.py | 3 +- mitmproxy/tools/console/statusbar.py | 5 +- mitmproxy/tools/main.py | 7 +- mitmproxy/tools/web/app.py | 16 ++- mitmproxy/types.py | 3 +- mitmproxy/utils/asyncio_utils.py | 7 +- mitmproxy/utils/human.py | 7 +- mitmproxy/utils/sliding_window.py | 7 +- mitmproxy/utils/strutils.py | 11 +- mitmproxy/websocket.py | 14 +-- release/build-and-deploy-docker.py | 5 +- release/deploy.py | 5 +- test/mitmproxy/addons/test_dns_resolver.py | 2 +- test/mitmproxy/addons/test_next_layer.py | 7 +- test/mitmproxy/addons/test_proxyserver.py | 19 ++- test/mitmproxy/addons/test_tlsconfig.py | 5 +- test/mitmproxy/coretypes/test_serializable.py | 15 ++- test/mitmproxy/net/test_udp.py | 3 +- .../mitmproxy/proxy/layers/http/test_http3.py | 11 +- test/mitmproxy/proxy/layers/test_quic.py | 23 ++-- test/mitmproxy/proxy/layers/test_tls.py | 17 +-- test/mitmproxy/proxy/test_tunnel.py | 6 +- test/mitmproxy/proxy/tutils.py | 19 ++- 121 files changed, 496 insertions(+), 649 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5543af0e1f..823cc356bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased: mitmproxy next * mitmproxy now requires Python 3.10 or above. + ([#5954](https://github.com/mitmproxy/mitmproxy/pull/5954), @mhils) * Fix a bug where the direction indicator in the message stream view would be in the wrong direction. ([#5921](https://github.com/mitmproxy/mitmproxy/issues/5921), @konradh) * Fix a bug where peername would be None in tls_passthrough script, which would make it not working. diff --git a/docs/scripts/clirecording/clidirector.py b/docs/scripts/clirecording/clidirector.py index e861e95563..973ca610ff 100644 --- a/docs/scripts/clirecording/clidirector.py +++ b/docs/scripts/clirecording/clidirector.py @@ -4,7 +4,6 @@ import threading import time from typing import NamedTuple -from typing import Optional import libtmux @@ -74,7 +73,7 @@ def end_session(self) -> None: self.tmux_session.kill_session() def press_key( - self, keys: str, count=1, pause: Optional[float] = None, target=None + self, keys: str, count=1, pause: float | None = None, target=None ) -> None: if pause is None: pause = self.pause_between_keys @@ -97,7 +96,7 @@ def press_key( real_pause += 2 * pause self.pause(real_pause) - def type(self, keys: str, pause: Optional[float] = None, target=None) -> None: + def type(self, keys: str, pause: float | None = None, target=None) -> None: if pause is None: pause = self.pause_between_keys if target is None: @@ -128,7 +127,7 @@ def run_external(self, command: str) -> None: def message( self, msg: str, - duration: Optional[int] = None, + duration: int | None = None, add_instruction: bool = True, instruction_html: str = "", ) -> None: @@ -161,7 +160,7 @@ def close_popup(self, duration: float = 0) -> None: self.tmux_pane.cmd("display-popup", "-C") def instruction( - self, instruction: str, duration: float = 3, time_from: Optional[float] = None + self, instruction: str, duration: float = 3, time_from: float | None = None ) -> None: if time_from is None: time_from = self.current_time diff --git a/examples/addons/contentview.py b/examples/addons/contentview.py index b96a81ca5a..6223e96e83 100644 --- a/examples/addons/contentview.py +++ b/examples/addons/contentview.py @@ -5,8 +5,6 @@ which is used to pretty-print HTTP bodies for example. The content view API is explained in the mitmproxy.contentviews module. """ -from typing import Optional - from mitmproxy import contentviews from mitmproxy import flow from mitmproxy import http @@ -19,9 +17,9 @@ def __call__( self, data: bytes, *, - content_type: Optional[str] = None, - flow: Optional[flow.Flow] = None, - http_message: Optional[http.Message] = None, + content_type: str | None = None, + flow: flow.Flow | None = None, + http_message: http.Message | None = None, **unknown_metadata, ) -> contentviews.TViewResult: return "case-swapped text", contentviews.format_text(data.swapcase()) @@ -30,9 +28,9 @@ def render_priority( self, data: bytes, *, - content_type: Optional[str] = None, - flow: Optional[flow.Flow] = None, - http_message: Optional[http.Message] = None, + content_type: str | None = None, + flow: flow.Flow | None = None, + http_message: http.Message | None = None, **unknown_metadata, ) -> float: if content_type == "text/plain": diff --git a/examples/addons/http-stream-modify.py b/examples/addons/http-stream-modify.py index 76cb33e513..9c59a338ed 100644 --- a/examples/addons/http-stream-modify.py +++ b/examples/addons/http-stream-modify.py @@ -8,10 +8,9 @@ where one chunk ends with [...]foo" and the next starts with "bar[...]. """ from collections.abc import Iterable -from typing import Union -def modify(data: bytes) -> Union[bytes, Iterable[bytes]]: +def modify(data: bytes) -> bytes | Iterable[bytes]: """ This function will be called for each chunk of request/response body data that arrives at the proxy, and once at the end of the message with an empty bytes argument (b""). diff --git a/examples/contrib/domain_fronting.py b/examples/contrib/domain_fronting.py index 804ceadf32..7b65a02b00 100644 --- a/examples/contrib/domain_fronting.py +++ b/examples/contrib/domain_fronting.py @@ -1,7 +1,5 @@ import json from dataclasses import dataclass -from typing import Optional -from typing import Union from mitmproxy import ctx from mitmproxy.addonmanager import Loader @@ -55,8 +53,8 @@ @dataclass class Mapping: - server: Union[str, None] - host: Union[str, None] + server: str | None + host: str | None class HttpsDomainFronting: @@ -70,7 +68,7 @@ def __init__(self) -> None: self.strict_mappings = {} self.star_mappings = {} - def _resolve_addresses(self, host: str) -> Optional[Mapping]: + def _resolve_addresses(self, host: str) -> Mapping | None: mapping = self.strict_mappings.get(host) if mapping is not None: return mapping diff --git a/examples/contrib/http_manipulate_cookies.py b/examples/contrib/http_manipulate_cookies.py index aaad41227c..2f86ab5346 100644 --- a/examples/contrib/http_manipulate_cookies.py +++ b/examples/contrib/http_manipulate_cookies.py @@ -15,7 +15,6 @@ """ import json -from typing import Union from mitmproxy import http @@ -29,7 +28,7 @@ # -- Helper functions -- -def load_json_cookies() -> list[dict[str, Union[str, None]]]: +def load_json_cookies() -> list[dict[str, str | None]]: """ Load a particular json file containing a list of cookies. """ @@ -40,7 +39,7 @@ def load_json_cookies() -> list[dict[str, Union[str, None]]]: # NOTE: or just hardcode the cookies as [{"name": "", "value": ""}] -def stringify_cookies(cookies: list[dict[str, Union[str, None]]]) -> str: +def stringify_cookies(cookies: list[dict[str, str | None]]) -> str: """ Creates a cookie string from a list of cookie dicts. """ @@ -54,7 +53,7 @@ def stringify_cookies(cookies: list[dict[str, Union[str, None]]]) -> str: ) -def parse_cookies(cookie_string: str) -> list[dict[str, Union[str, None]]]: +def parse_cookies(cookie_string: str) -> list[dict[str, str | None]]: """ Parses a cookie string into a list of cookie dicts. """ diff --git a/examples/contrib/ntlm_upstream_proxy.py b/examples/contrib/ntlm_upstream_proxy.py index f11a0b77a8..bca568fd4d 100644 --- a/examples/contrib/ntlm_upstream_proxy.py +++ b/examples/contrib/ntlm_upstream_proxy.py @@ -115,7 +115,7 @@ def extract_proxy_authenticate_msg(response_head: list) -> str: def patched_receive_handshake_data( self, data - ) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: + ) -> layer.CommandGenerator[tuple[bool, str | None]]: self.buf += data response_head = self.buf.maybe_extract_lines() if response_head: diff --git a/examples/contrib/webscanner_helper/test_mapping.py b/examples/contrib/webscanner_helper/test_mapping.py index c88b11983a..89ed27c219 100644 --- a/examples/contrib/webscanner_helper/test_mapping.py +++ b/examples/contrib/webscanner_helper/test_mapping.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable from typing import TextIO from unittest import mock from unittest.mock import MagicMock diff --git a/examples/contrib/webscanner_helper/urldict.py b/examples/contrib/webscanner_helper/urldict.py index a4ce1e0fa5..e527667631 100644 --- a/examples/contrib/webscanner_helper/urldict.py +++ b/examples/contrib/webscanner_helper/urldict.py @@ -1,12 +1,11 @@ import itertools import json +from collections.abc import Callable from collections.abc import Generator from collections.abc import MutableMapping from typing import Any -from typing import Callable from typing import cast from typing import TextIO -from typing import Union from mitmproxy import flowfilter from mitmproxy.http import HTTPFlow @@ -78,7 +77,7 @@ def loads(cls, json_str: str, value_loader: Callable = f_id): return cls._load(json_obj, value_loader) def _dump(self, value_dumper: Callable = f_id) -> dict: - dumped: dict[Union[flowfilter.TFilter, str], Any] = {} + dumped: dict[flowfilter.TFilter | str, Any] = {} for fltr, value in self.store.items(): if hasattr(fltr, "pattern"): # cast necessary for mypy diff --git a/examples/contrib/webscanner_helper/urlindex.py b/examples/contrib/webscanner_helper/urlindex.py index 09f9ef2e8c..db43bbe326 100644 --- a/examples/contrib/webscanner_helper/urlindex.py +++ b/examples/contrib/webscanner_helper/urlindex.py @@ -3,8 +3,6 @@ import json import logging from pathlib import Path -from typing import Optional -from typing import Union from mitmproxy import flowfilter from mitmproxy.http import HTTPFlow @@ -118,7 +116,7 @@ class UrlIndexAddon: The injection can be done using the URLInjection Add-on. """ - index_filter: Optional[Union[str, flowfilter.TFilter]] + index_filter: str | flowfilter.TFilter | None writer: UrlIndexWriter OPT_FILEPATH = "URLINDEX_FILEPATH" @@ -127,9 +125,9 @@ class UrlIndexAddon: def __init__( self, - file_path: Union[str, Path], + file_path: str | Path, append: bool = True, - index_filter: Union[str, flowfilter.TFilter] = filter_404, + index_filter: str | flowfilter.TFilter = filter_404, index_format: str = "json", ): """Initializes the urlindex add-on. diff --git a/examples/contrib/webscanner_helper/watchdog.py b/examples/contrib/webscanner_helper/watchdog.py index 7f63ffec5f..ee61b5c6ec 100644 --- a/examples/contrib/webscanner_helper/watchdog.py +++ b/examples/contrib/webscanner_helper/watchdog.py @@ -2,7 +2,6 @@ import pathlib import time from datetime import datetime -from typing import Union import mitmproxy.connections import mitmproxy.http @@ -36,8 +35,8 @@ def __init__(self, event, outdir: pathlib.Path, timeout=None): raise RuntimeError("Watchtdog output path must be a directory.") elif not self.flow_dir.exists(): self.flow_dir.mkdir(parents=True) - self.last_trigger: Union[None, float] = None - self.timeout: Union[None, float] = timeout + self.last_trigger: None | float = None + self.timeout: None | float = timeout def serverconnect(self, conn: mitmproxy.connections.ServerConnection): if self.timeout is not None: diff --git a/examples/contrib/xss_scanner.py b/examples/contrib/xss_scanner.py index c942281c8c..eab9510a07 100644 --- a/examples/contrib/xss_scanner.py +++ b/examples/contrib/xss_scanner.py @@ -40,7 +40,6 @@ from html.parser import HTMLParser from typing import NamedTuple from typing import Optional -from typing import Union from urllib.parse import urlparse import requests @@ -93,7 +92,7 @@ def get_cookies(flow: http.HTTPFlow) -> Cookies: def find_unclaimed_URLs(body, requestUrl): """Look for unclaimed URLs in script tags and log them if found""" - def getValue(attrs: list[tuple[str, str]], attrName: str) -> Optional[str]: + def getValue(attrs: list[tuple[str, str]], attrName: str) -> str | None: for name, value in attrs: if attrName == name: return value @@ -188,7 +187,7 @@ def test_query_injection(original_body: str, request_URL: str, cookies: Cookies) return xss_info, sqli_info -def log_XSS_data(xss_info: Optional[XSSData]) -> None: +def log_XSS_data(xss_info: XSSData | None) -> None: """Log information about the given XSS to mitmproxy""" # If it is None, then there is no info to log if not xss_info: @@ -200,7 +199,7 @@ def log_XSS_data(xss_info: Optional[XSSData]) -> None: logging.error("Line: %s" % xss_info.line) -def log_SQLi_data(sqli_info: Optional[SQLiData]) -> None: +def log_SQLi_data(sqli_info: SQLiData | None) -> None: """Log information about the given SQLi to mitmproxy""" if not sqli_info: return @@ -214,7 +213,7 @@ def log_SQLi_data(sqli_info: Optional[SQLiData]) -> None: def get_SQLi_data( new_body: str, original_body: str, request_URL: str, injection_point: str -) -> Optional[SQLiData]: +) -> SQLiData | None: """Return a SQLiDict if there is a SQLi otherwise return None String String URL String -> (SQLiDict or None)""" # Regexes taken from Damn Small SQLi Scanner: https://github.com/stamparm/DSSS/blob/master/dsss.py#L17 @@ -337,8 +336,8 @@ def handle_data(self, data): def get_XSS_data( - body: Union[str, bytes], request_URL: str, injection_point: str -) -> Optional[XSSData]: + body: str | bytes, request_URL: str, injection_point: str +) -> XSSData | None: """Return a XSSDict if there is a XSS otherwise return None""" def in_script(text, index, body) -> bool: diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 5b52750a5c..45856f22a4 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -9,7 +9,6 @@ from collections.abc import Sequence from dataclasses import dataclass from typing import Any -from typing import Optional from mitmproxy import exceptions from mitmproxy import flow @@ -71,7 +70,7 @@ def add_option( typespec: type, default: Any, help: str, - choices: Optional[Sequence[str]] = None, + choices: Sequence[str] | None = None, ) -> None: """ Add an option to mitmproxy. diff --git a/mitmproxy/addons/asgiapp.py b/mitmproxy/addons/asgiapp.py index 5425275a9d..494671d0a5 100644 --- a/mitmproxy/addons/asgiapp.py +++ b/mitmproxy/addons/asgiapp.py @@ -2,7 +2,6 @@ import logging import traceback import urllib.parse -from typing import Optional import asgiref.compatibility import asgiref.wsgi @@ -22,7 +21,7 @@ class ASGIApp: - It currently only implements the HTTP protocol (Lifespan and WebSocket are unimplemented). """ - def __init__(self, asgi_app, host: str, port: Optional[int]): + def __init__(self, asgi_app, host: str, port: int | None): asgi_app = asgiref.compatibility.guarantee_single_callable(asgi_app) self.asgi_app, self.host, self.port = asgi_app, host, port @@ -45,7 +44,7 @@ async def request(self, flow: http.HTTPFlow) -> None: class WSGIApp(ASGIApp): - def __init__(self, wsgi_app, host: str, port: Optional[int]): + def __init__(self, wsgi_app, host: str, port: int | None): asgi_app = asgiref.wsgi.WsgiToAsgi(wsgi_app) super().__init__(asgi_app, host, port) diff --git a/mitmproxy/addons/browser.py b/mitmproxy/addons/browser.py index ab2fcc5601..4c3a6bc274 100644 --- a/mitmproxy/addons/browser.py +++ b/mitmproxy/addons/browser.py @@ -2,14 +2,13 @@ import shutil import subprocess import tempfile -from typing import Optional from mitmproxy import command from mitmproxy import ctx from mitmproxy.log import ALERT -def get_chrome_executable() -> Optional[str]: +def get_chrome_executable() -> str | None: for browser in ( "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", # https://stackoverflow.com/questions/40674914/google-chrome-path-in-windows-10 @@ -29,7 +28,7 @@ def get_chrome_executable() -> Optional[str]: return None -def get_chrome_flatpak() -> Optional[str]: +def get_chrome_flatpak() -> str | None: if shutil.which("flatpak"): for browser in ( "com.google.Chrome", @@ -50,7 +49,7 @@ def get_chrome_flatpak() -> Optional[str]: return None -def get_browser_cmd() -> Optional[list[str]]: +def get_browser_cmd() -> list[str] | None: if browser := get_chrome_executable(): return [browser] elif browser := get_chrome_flatpak(): diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index e6d7a85243..8c5c671101 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -6,7 +6,6 @@ import traceback from collections.abc import Sequence from typing import cast -from typing import Optional import mitmproxy.types from mitmproxy import command @@ -134,8 +133,8 @@ async def handle_hook(self, hook: commands.StartHook) -> None: class ClientPlayback: - playback_task: Optional[asyncio.Task] = None - inflight: Optional[http.HTTPFlow] + playback_task: asyncio.Task | None = None + inflight: http.HTTPFlow | None queue: asyncio.Queue options: Options replay_tasks: set[asyncio.Task] @@ -176,7 +175,7 @@ async def playback(self): self.queue.task_done() self.inflight = None - def check(self, f: flow.Flow) -> Optional[str]: + def check(self, f: flow.Flow) -> str | None: if f.live or f == self.inflight: return "Can't replay live flow." if f.intercepted: diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 5ab1ba0a69..4cb3dca691 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -1,7 +1,6 @@ import logging import os from collections.abc import Sequence -from typing import Union import mitmproxy.types from mitmproxy import command @@ -135,7 +134,7 @@ def flow_set(self, flows: Sequence[flow.Flow], attr: str, value: str) -> None: """ Quickly set a number of common values on flows. """ - val: Union[int, str] = value + val: int | str = value if attr == "status_code": try: val = int(val) # type: ignore diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index 34a7b8fe0a..c15ac3f539 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -4,7 +4,6 @@ import os.path from collections.abc import Sequence from typing import Any -from typing import Union import pyperclip @@ -28,7 +27,7 @@ def is_addr(v): return isinstance(v, tuple) and len(v) > 1 -def extract(cut: str, f: flow.Flow) -> Union[str, bytes]: +def extract(cut: str, f: flow.Flow) -> str | bytes: path = cut.split(".") current: Any = f for i, spec in enumerate(path): @@ -86,7 +85,7 @@ def cut( or "false", "bytes" are preserved, and all other values are converted to strings. """ - ret: list[list[Union[str, bytes]]] = [] + ret: list[list[str | bytes]] = [] for f in flows: ret.append([extract(c, f) for c in cuts]) return ret # type: ignore @@ -148,7 +147,7 @@ def clip( format is UTF-8 encoded CSV. If there is exactly one row and one column, the data is written to file as-is, with raw bytes preserved. """ - v: Union[str, bytes] + v: str | bytes fp = io.StringIO(newline="") if len(cuts) == 1 and len(flows) == 1: v = extract_str(cuts[0], flows[0]) diff --git a/mitmproxy/addons/dns_resolver.py b/mitmproxy/addons/dns_resolver.py index 714d37e16e..63718050eb 100644 --- a/mitmproxy/addons/dns_resolver.py +++ b/mitmproxy/addons/dns_resolver.py @@ -1,9 +1,8 @@ import asyncio import ipaddress import socket +from collections.abc import Callable from collections.abc import Iterable -from typing import Callable -from typing import Union from mitmproxy import dns from mitmproxy.proxy import mode_specs @@ -24,7 +23,7 @@ async def resolve_question_by_name( question: dns.Question, loop: asyncio.AbstractEventLoop, family: socket.AddressFamily, - ip: Callable[[str], Union[ipaddress.IPv4Address, ipaddress.IPv6Address]], + ip: Callable[[str], ipaddress.IPv4Address | ipaddress.IPv6Address], ) -> Iterable[dns.ResourceRecord]: try: addrinfos = await loop.getaddrinfo(host=question.name, port=0, family=family) @@ -51,7 +50,7 @@ async def resolve_question_by_addr( question: dns.Question, loop: asyncio.AbstractEventLoop, suffix: str, - sockaddr: Callable[[list[str]], Union[tuple[str, int], tuple[str, int, int, int]]], + sockaddr: Callable[[list[str]], tuple[str, int] | tuple[str, int, int, int]], ) -> Iterable[dns.ResourceRecord]: try: addr = sockaddr(question.name[: -len(suffix)].split(".")[::-1]) diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 897fc3a151..c2930c5ad3 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -6,7 +6,6 @@ import sys from typing import IO from typing import Optional -from typing import Union from wsproto.frame_protocol import CloseReason @@ -45,8 +44,8 @@ def indent(n: int, text: str) -> str: class Dumper: - def __init__(self, outfile: Optional[IO[str]] = None): - self.filter: Optional[flowfilter.TFilter] = None + def __init__(self, outfile: IO[str] | None = None): + self.filter: flowfilter.TFilter | None = None self.outfp: IO[str] = outfile or sys.stdout self.out_has_vt_codes = vt_codes.ensure_supported(self.outfp) @@ -103,7 +102,7 @@ def _echo_headers(self, headers: http.Headers): vs = strutils.bytes_to_escaped_str(v) self.echo(f"{ks}: {vs}", ident=4) - def _echo_trailers(self, trailers: Optional[http.Headers]): + def _echo_trailers(self, trailers: http.Headers | None): if not trailers: return self.echo("--- HTTP Trailers", fg="magenta", ident=4) @@ -116,8 +115,8 @@ def _colorful(self, line): def _echo_message( self, - message: Union[http.Message, TCPMessage, UDPMessage, WebSocketMessage], - flow: Union[http.HTTPFlow, TCPFlow, UDPFlow], + message: http.Message | TCPMessage | UDPMessage | WebSocketMessage, + flow: http.HTTPFlow | TCPFlow | UDPFlow, ): _, lines, error = contentviews.get_message_content_view( ctx.options.dumper_default_contentview, message, flow @@ -341,7 +340,7 @@ def tcp_error(self, f): def udp_error(self, f): self._proto_error(f) - def _proto_message(self, f: Union[TCPFlow, UDPFlow]) -> None: + def _proto_message(self, f: TCPFlow | UDPFlow) -> None: if self.match(f): message = f.messages[-1] direction = "->" if message.from_client else "<-" diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py index e719e69797..b474a5f7c0 100644 --- a/mitmproxy/addons/eventstore.py +++ b/mitmproxy/addons/eventstore.py @@ -2,7 +2,6 @@ import collections import logging from collections.abc import Callable -from typing import Optional from mitmproxy import command from mitmproxy import log @@ -27,7 +26,7 @@ def _add_log(self, entry: LogEntry) -> None: self.sig_add.send(entry) @property - def size(self) -> Optional[int]: + def size(self) -> int | None: return self.data.maxlen @command.command("eventstore.clear") diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 1339111c5b..fb5ce2d5db 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -3,7 +3,6 @@ from collections.abc import Callable from collections.abc import Sequence from typing import Any -from typing import Union import pyperclip @@ -142,7 +141,7 @@ def raw(f: flow.Flow, separator=b"\r\n\r\n") -> bytes: raise exceptions.CommandError("Can't export flow with no request or response.") -formats: dict[str, Callable[[flow.Flow], Union[str, bytes]]] = dict( +formats: dict[str, Callable[[flow.Flow], str | bytes]] = dict( curl=curl_command, httpie=httpie_command, raw=raw, diff --git a/mitmproxy/addons/intercept.py b/mitmproxy/addons/intercept.py index 24a9173f95..dff16e459d 100644 --- a/mitmproxy/addons/intercept.py +++ b/mitmproxy/addons/intercept.py @@ -7,7 +7,7 @@ class Intercept: - filt: Optional[flowfilter.TFilter] = None + filt: flowfilter.TFilter | None = None def load(self, loader): loader.add_option("intercept_active", bool, False, "Intercept toggle") diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 3495ed9198..25f6af9e7f 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -16,12 +16,11 @@ """ import re import struct +from collections.abc import Callable from collections.abc import Iterable from collections.abc import Sequence from typing import Any -from typing import Callable from typing import cast -from typing import Optional from typing import Union from mitmproxy import connection @@ -51,7 +50,7 @@ def stack_match( - context: context.Context, layers: Sequence[Union[LayerCls, tuple[LayerCls, ...]]] + context: context.Context, layers: Sequence[LayerCls | tuple[LayerCls, ...]] ) -> bool: if len(context.layers) != len(layers): return False @@ -90,12 +89,12 @@ def configure(self, updated): def ignore_connection( self, - server_address: Optional[connection.Address], + server_address: connection.Address | None, data_client: bytes, *, is_tls: Callable[[bytes], bool] = is_tls_record_magic, - client_hello: Callable[[bytes], Optional[ClientHello]] = parse_client_hello, - ) -> Optional[bool]: + client_hello: Callable[[bytes], ClientHello | None] = parse_client_hello, + ) -> bool | None: """ Returns: True, if the connection should be ignored. @@ -174,7 +173,7 @@ def is_destination_in_hosts( for rex in hosts ) - def get_http_layer(self, context: context.Context) -> Optional[layers.HttpLayer]: + def get_http_layer(self, context: context.Context) -> layers.HttpLayer | None: def s(*layers): return stack_match(context, layers) @@ -195,7 +194,7 @@ def s(*layers): def detect_udp_tls( self, data_client: bytes - ) -> Optional[tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls]]: + ) -> tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls] | None: if len(data_client) == 0: return None @@ -257,7 +256,7 @@ def next_layer(self, nextlayer: layer.NextLayer): def _next_layer( self, context: context.Context, data_client: bytes, data_server: bytes - ) -> Optional[layer.Layer]: + ) -> layer.Layer | None: assert context.layers if context.client.transport_protocol == "tcp": diff --git a/mitmproxy/addons/proxyserver.py b/mitmproxy/addons/proxyserver.py index 10b6c7cc29..58e44acde3 100644 --- a/mitmproxy/addons/proxyserver.py +++ b/mitmproxy/addons/proxyserver.py @@ -112,8 +112,8 @@ class Proxyserver(ServerManager): servers: Servers is_running: bool - _connect_addr: Optional[Address] = None - _update_task: Optional[asyncio.Task] = None + _connect_addr: Address | None = None + _update_task: asyncio.Task | None = None def __init__(self): self.connections = {} diff --git a/mitmproxy/addons/save.py b/mitmproxy/addons/save.py index f2b0b2133c..bca9621d1b 100644 --- a/mitmproxy/addons/save.py +++ b/mitmproxy/addons/save.py @@ -41,10 +41,10 @@ def _mode(path: str) -> Literal["ab", "wb"]: class Save: def __init__(self) -> None: - self.stream: Optional[io.FilteredFlowWriter] = None - self.filt: Optional[flowfilter.TFilter] = None + self.stream: io.FilteredFlowWriter | None = None + self.filt: flowfilter.TFilter | None = None self.active_flows: set[flow.Flow] = set() - self.current_path: Optional[str] = None + self.current_path: str | None = None def load(self, loader): loader.add_option( diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 3e5f0beff7..b320b39717 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -7,7 +7,6 @@ import traceback import types from collections.abc import Sequence -from typing import Optional import mitmproxy.types as mtypes from mitmproxy import addonmanager @@ -22,7 +21,7 @@ logger = logging.getLogger(__name__) -def load_script(path: str) -> Optional[types.ModuleType]: +def load_script(path: str) -> types.ModuleType | None: fullname = "__mitmproxy_script__.{}".format( os.path.splitext(os.path.basename(path))[0] ) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 6602cb5f80..b43d74f596 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -4,7 +4,6 @@ from collections.abc import Hashable from collections.abc import Sequence from typing import Any -from typing import Optional import mitmproxy.types from mitmproxy import command @@ -195,7 +194,7 @@ def _hash(self, flow: http.HTTPFlow) -> Hashable: key.append(headers) return hashlib.sha256(repr(key).encode("utf8", "surrogateescape")).digest() - def next_flow(self, flow: http.HTTPFlow) -> Optional[http.HTTPFlow]: + def next_flow(self, flow: http.HTTPFlow) -> http.HTTPFlow | None: """ Returns the next flow object, or None if no matching flow was found. diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index becaafa442..2164ac55f0 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -37,7 +37,7 @@ def __init__(self) -> None: self.jar: collections.defaultdict[ TOrigin, dict[str, str] ] = collections.defaultdict(dict) - self.flt: Optional[flowfilter.TFilter] = None + self.flt: flowfilter.TFilter | None = None def load(self, loader): loader.add_option( diff --git a/mitmproxy/addons/tlsconfig.py b/mitmproxy/addons/tlsconfig.py index 0d7faf52de..dfa851afb8 100644 --- a/mitmproxy/addons/tlsconfig.py +++ b/mitmproxy/addons/tlsconfig.py @@ -4,7 +4,6 @@ import ssl from pathlib import Path from typing import Any -from typing import Optional from typing import TypedDict from aioquic.h3.connection import H3_ALPN @@ -64,8 +63,8 @@ class AppData(TypedDict): - client_alpn: Optional[bytes] - server_alpn: Optional[bytes] + client_alpn: bytes | None + server_alpn: bytes | None http2: bool @@ -200,7 +199,7 @@ def tls_start_client(self, tls_start: tls.TlsData) -> None: if len(tls_start.context.layers) == 2 and isinstance( tls_start.context.layers[0], modes.HttpProxy ): - client_alpn: Optional[bytes] = b"http/1.1" + client_alpn: bytes | None = b"http/1.1" else: client_alpn = client.alpn @@ -257,7 +256,7 @@ def tls_start_server(self, tls_start: tls.TlsData) -> None: # don't assign to client.cipher_list, doesn't need to be stored. cipher_list = server.cipher_list or DEFAULT_CIPHERS - client_cert: Optional[str] = None + client_cert: str | None = None if ctx.options.client_certs: client_certs = os.path.expanduser(ctx.options.client_certs) if os.path.isfile(client_certs): @@ -455,7 +454,7 @@ def get_cert(self, conn_context: context.Context) -> certs.CertStoreEntry: our certificate should have and then fetches a matching cert from the certstore. """ altnames: list[str] = [] - organization: Optional[str] = None + organization: str | None = None # Use upstream certificate if available. if ctx.options.upstream_cert and conn_context.server.certificate_list: diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py index 63b0d32bb0..655ca7d752 100644 --- a/mitmproxy/addons/upstream_auth.py +++ b/mitmproxy/addons/upstream_auth.py @@ -27,7 +27,7 @@ class UpstreamAuth: - Reverse proxy regular requests (CONNECT is invalid in this mode) """ - auth: Optional[bytes] = None + auth: bytes | None = None def load(self, loader): loader.add_option( diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 06049715fe..fe55ee58fc 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -235,7 +235,7 @@ def _bisect(self, f: mitmproxy.flow.Flow) -> int: return self._rev(v - 1) + 1 def index( - self, f: mitmproxy.flow.Flow, start: int = 0, stop: Optional[int] = None + self, f: mitmproxy.flow.Flow, start: int = 0, stop: int | None = None ) -> int: return self._rev(self._view.index(f, start, stop)) @@ -353,7 +353,7 @@ def set_filter_cmd(self, filter_expr: str) -> None: raise exceptions.CommandError(str(e)) from e self.set_filter(filt) - def set_filter(self, flt: Optional[flowfilter.TFilter]): + def set_filter(self, flt: flowfilter.TFilter | None): self.filter = flt or flowfilter.match_all self._refilter() @@ -524,7 +524,7 @@ def add(self, flows: Sequence[mitmproxy.flow.Flow]) -> None: self.focus.flow = f self.sig_view_add.send(flow=f) - def get_by_id(self, flow_id: str) -> Optional[mitmproxy.flow.Flow]: + def get_by_id(self, flow_id: str) -> mitmproxy.flow.Flow | None: """ Get flow with the given id from the store. Returns None if the flow is not found. @@ -669,7 +669,7 @@ class Focus: def __init__(self, v: View) -> None: self.view = v - self._flow: Optional[mitmproxy.flow.Flow] = None + self._flow: mitmproxy.flow.Flow | None = None self.sig_change = signals.SyncSignal(lambda: None) if len(self.view): self.flow = self.view[0] @@ -678,18 +678,18 @@ def __init__(self, v: View) -> None: v.sig_view_refresh.connect(self._sig_view_refresh) @property - def flow(self) -> Optional[mitmproxy.flow.Flow]: + def flow(self) -> mitmproxy.flow.Flow | None: return self._flow @flow.setter - def flow(self, f: Optional[mitmproxy.flow.Flow]): + def flow(self, f: mitmproxy.flow.Flow | None): if f is not None and f not in self.view: raise ValueError("Attempt to set focus to flow not in view") self._flow = f self.sig_change.send() @property - def index(self) -> Optional[int]: + def index(self) -> int | None: if self.flow: return self.view.index(self.flow) return None diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 9b0a824894..ce6d9443c4 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -131,14 +131,14 @@ def keyinfo(self) -> tuple[str, int]: ) # pragma: no cover @property - def cn(self) -> Optional[str]: + def cn(self) -> str | None: attrs = self._cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME) if attrs: return attrs[0].value return None @property - def organization(self) -> Optional[str]: + def organization(self) -> str | None: attrs = self._cert.subject.get_attributes_for_oid( x509.NameOID.ORGANIZATION_NAME ) @@ -231,9 +231,9 @@ def create_ca( def dummy_cert( privkey: rsa.RSAPrivateKey, cacert: x509.Certificate, - commonname: Optional[str], + commonname: str | None, sans: list[str], - organization: Optional[str] = None, + organization: str | None = None, ) -> Cert: """ Generates a dummy certificate. @@ -288,7 +288,7 @@ def dummy_cert( class CertStoreEntry: cert: Cert privatekey: rsa.RSAPrivateKey - chain_file: Optional[Path] + chain_file: Path | None chain_certs: list[Cert] @@ -312,7 +312,7 @@ def __init__( self, default_privatekey: rsa.RSAPrivateKey, default_ca: Cert, - default_chain_file: Optional[Path], + default_chain_file: Path | None, dhparams: DHParams, ): self.default_privatekey = default_privatekey @@ -365,10 +365,10 @@ def load_dhparam(path: Path) -> DHParams: @classmethod def from_store( cls, - path: Union[Path, str], + path: Path | str, basename: str, key_size: int, - passphrase: Optional[bytes] = None, + passphrase: bytes | None = None, ) -> "CertStore": path = Path(path) ca_file = path / f"{basename}-ca.pem" @@ -379,7 +379,7 @@ def from_store( @classmethod def from_files( - cls, ca_file: Path, dhparam_file: Path, passphrase: Optional[bytes] = None + cls, ca_file: Path, dhparam_file: Path, passphrase: bytes | None = None ) -> "CertStore": raw = ca_file.read_bytes() key = load_pem_private_key(raw, passphrase) @@ -387,7 +387,7 @@ def from_files( certs = re.split(rb"(?=-----BEGIN CERTIFICATE-----)", raw) ca = Cert.from_pem(certs[1]) if len(certs) > 2: - chain_file: Optional[Path] = ca_file + chain_file: Path | None = ca_file else: chain_file = None return cls(key, ca, chain_file, dh) @@ -463,7 +463,7 @@ def create_store( (path / f"{basename}-dhparam.pem").write_bytes(DEFAULT_DHPARAM) def add_cert_file( - self, spec: str, path: Path, passphrase: Optional[bytes] = None + self, spec: str, path: Path, passphrase: bytes | None = None ) -> None: raw = path.read_bytes() cert = Cert.from_pem(raw) @@ -500,9 +500,9 @@ def asterisk_forms(dn: str) -> list[str]: def get_cert( self, - commonname: Optional[str], + commonname: str | None, sans: list[str], - organization: Optional[str] = None, + organization: str | None = None, ) -> CertStoreEntry: """ commonname: Common name for the generated certificate. Must be a @@ -543,7 +543,7 @@ def get_cert( return entry -def load_pem_private_key(data: bytes, password: Optional[bytes]) -> rsa.RSAPrivateKey: +def load_pem_private_key(data: bytes, password: bytes | None) -> rsa.RSAPrivateKey: """ like cryptography's load_pem_private_key, but silently falls back to not using a password if the private key is unencrypted. diff --git a/mitmproxy/command.py b/mitmproxy/command.py index f62b93c221..d9b56b5848 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -12,7 +12,6 @@ from collections.abc import Sequence from typing import Any from typing import NamedTuple -from typing import Optional import pyparsing @@ -66,7 +65,7 @@ class Command: name: str manager: "CommandManager" signature: inspect.Signature - help: Optional[str] + help: str | None def __init__(self, manager: "CommandManager", name: str, func: Callable) -> None: self.name = name @@ -95,7 +94,7 @@ def __init__(self, manager: "CommandManager", name: str, func: Callable) -> None ) @property - def return_type(self) -> Optional[type]: + def return_type(self) -> type | None: return _empty_as_none(self.signature.return_annotation) @property @@ -209,7 +208,7 @@ def parse_partial( CommandParameter("", mitmproxy.types.Cmd), CommandParameter("", mitmproxy.types.CmdArgs), ] - expected: Optional[CommandParameter] = None + expected: CommandParameter | None = None for part in parts: if part.isspace(): parsed.append( @@ -314,7 +313,7 @@ def parsearg(manager: CommandManager, spec: str, argtype: type) -> Any: raise exceptions.CommandError(str(e)) from e -def command(name: Optional[str] = None): +def command(name: str | None = None): def decorator(function): @functools.wraps(function) def wrapper(*args, **kwargs): diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index b7f4ad2f5c..46bb63b327 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -9,7 +9,6 @@ from dataclasses import field from enum import Flag from typing import Literal -from typing import Optional from mitmproxy import certs from mitmproxy.coretypes import serializable @@ -35,10 +34,7 @@ class ConnectionState(Flag): # this version at least provides useful type checking messages. Address = tuple[str, int] -if sys.version_info < (3, 10): # pragma: no cover - kw_only = {} -else: - kw_only = {"kw_only": True} +kw_only = {"kw_only": True} # noinspection PyDataclass @@ -51,9 +47,9 @@ class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): This is intentional, all I/O should be handled by `mitmproxy.proxy.server` exclusively. """ - peername: Optional[Address] + peername: Address | None """The remote's `(ip, port)` tuple for this connection.""" - sockname: Optional[Address] + sockname: Address | None """Our local `(ip, port)` tuple for this connection.""" state: ConnectionState = field( @@ -68,7 +64,7 @@ class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): """A unique UUID to identify the connection.""" transport_protocol: TransportProtocol = field(default="tcp") """The connection protocol in use.""" - error: Optional[str] = None + error: str | None = None """ A string describing a general error with connections to this address. @@ -99,27 +95,27 @@ class Connection(serializable.SerializableDataclass, metaclass=ABCMeta): > TLS version, with the exception of the end-entity certificate which > MUST be first. """ - alpn: Optional[bytes] = None + alpn: bytes | None = None """The application-layer protocol as negotiated using [ALPN](https://en.wikipedia.org/wiki/Application-Layer_Protocol_Negotiation).""" alpn_offers: Sequence[bytes] = () """The ALPN offers as sent in the ClientHello.""" # we may want to add SSL_CIPHER_description here, but that's currently not exposed by cryptography - cipher: Optional[str] = None + cipher: str | None = None """The active cipher name as returned by OpenSSL's `SSL_CIPHER_get_name`.""" cipher_list: Sequence[str] = () """Ciphers accepted by the proxy server on this connection.""" - tls_version: Optional[str] = None + tls_version: str | None = None """The active TLS version.""" - sni: Optional[str] = None + sni: str | None = None """ The [Server Name Indication (SNI)](https://en.wikipedia.org/wiki/Server_Name_Indication) sent in the ClientHello. """ - timestamp_start: Optional[float] = None - timestamp_end: Optional[float] = None + timestamp_start: float | None = None + timestamp_end: float | None = None """*Timestamp:* Connection has been closed.""" - timestamp_tls_setup: Optional[float] = None + timestamp_tls_setup: float | None = None """*Timestamp:* TLS handshake has been completed successfully.""" @property @@ -157,7 +153,7 @@ def __repr__(self): return f"{type(self).__name__}({attrs!r})" @property - def alpn_proto_negotiated(self) -> Optional[bytes]: # pragma: no cover + def alpn_proto_negotiated(self) -> bytes | None: # pragma: no cover """*Deprecated:* An outdated alias for Connection.alpn.""" warnings.warn( "Connection.alpn_proto_negotiated is deprecated, use Connection.alpn instead.", @@ -177,7 +173,7 @@ class Client(Connection): sockname: Address """The local address we received this connection on.""" - mitmcert: Optional[certs.Cert] = None + mitmcert: certs.Cert | None = None """ The certificate used by mitmproxy to establish TLS with the client. """ @@ -221,7 +217,7 @@ def address(self, x): # pragma: no cover self.peername = x @property - def cipher_name(self) -> Optional[str]: # pragma: no cover + def cipher_name(self) -> str | None: # pragma: no cover """*Deprecated:* An outdated alias for Connection.cipher.""" warnings.warn( "Client.cipher_name is deprecated, use Client.cipher instead.", @@ -231,7 +227,7 @@ def cipher_name(self) -> Optional[str]: # pragma: no cover return self.cipher @property - def clientcert(self) -> Optional[certs.Cert]: # pragma: no cover + def clientcert(self) -> certs.Cert | None: # pragma: no cover """*Deprecated:* An outdated alias for Connection.certificate_list[0].""" warnings.warn( "Client.clientcert is deprecated, use Client.certificate_list instead.", @@ -261,30 +257,30 @@ def clientcert(self, val): # pragma: no cover class Server(Connection): """A connection between mitmproxy and an upstream server.""" - address: Optional[Address] # type: ignore + address: Address | None # type: ignore """The server's `(host, port)` address tuple. The host can either be a domain or a plain IP address.""" if sys.version_info < (3, 10): # pragma: no cover # no keyword-only arguments here. - address: Optional[Address] = None + address: Address | None = None - peername: Optional[Address] = None + peername: Address | None = None """ The server's resolved `(ip, port)` tuple. Will be set during connection establishment. May be `None` in upstream proxy mode when the address is resolved by the upstream proxy only. """ - sockname: Optional[Address] = None + sockname: Address | None = None - timestamp_start: Optional[float] = None + timestamp_start: float | None = None """ *Timestamp:* Connection establishment started. For IP addresses, this corresponds to sending a TCP SYN; for domains, this corresponds to starting a DNS lookup. """ - timestamp_tcp_setup: Optional[float] = None + timestamp_tcp_setup: float | None = None """*Timestamp:* TCP ACK received.""" - via: Optional[server_spec.ServerSpec] = None + via: server_spec.ServerSpec | None = None """An optional proxy server specification via which the connection should be established.""" def __str__(self): @@ -315,7 +311,7 @@ def __setattr__(self, name, value): return super().__setattr__(name, value) @property - def ip_address(self) -> Optional[Address]: # pragma: no cover + def ip_address(self) -> Address | None: # pragma: no cover """*Deprecated:* An outdated alias for `Server.peername`.""" warnings.warn( "Server.ip_address is deprecated, use Server.peername instead.", @@ -325,7 +321,7 @@ def ip_address(self) -> Optional[Address]: # pragma: no cover return self.peername @property - def cert(self) -> Optional[certs.Cert]: # pragma: no cover + def cert(self) -> certs.Cert | None: # pragma: no cover """*Deprecated:* An outdated alias for `Connection.certificate_list[0]`.""" warnings.warn( "Server.cert is deprecated, use Server.certificate_list instead.", diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 688869b6c3..cc824b8960 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -12,8 +12,6 @@ `base.View`. """ import traceback -from typing import Optional -from typing import Union from . import auto from . import css @@ -61,7 +59,7 @@ def _update(view: View) -> None: """A contentview has been removed.""" -def get(name: str) -> Optional[View]: +def get(name: str) -> View | None: for i in views: if i.name.lower() == name.lower(): return i @@ -99,7 +97,7 @@ def safe_to_print(lines, encoding="utf8"): def get_message_content_view( viewname: str, - message: Union[http.Message, TCPMessage, UDPMessage, WebSocketMessage], + message: http.Message | TCPMessage | UDPMessage | WebSocketMessage, flow: flow.Flow, ): """ @@ -110,7 +108,7 @@ def get_message_content_view( viewmode = get("auto") assert viewmode - content: Optional[bytes] + content: bytes | None try: content = message.content except ValueError: @@ -162,11 +160,11 @@ def get_content_view( viewmode: View, data: bytes, *, - content_type: Optional[str] = None, - flow: Optional[flow.Flow] = None, - http_message: Optional[http.Message] = None, - tcp_message: Optional[tcp.TCPMessage] = None, - udp_message: Optional[udp.UDPMessage] = None, + content_type: str | None = None, + flow: flow.Flow | None = None, + http_message: http.Message | None = None, + tcp_message: tcp.TCPMessage | None = None, + udp_message: udp.UDPMessage | None = None, ): """ Args: diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 7eca342a0b..9ee0451574 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -5,7 +5,6 @@ from collections.abc import Iterator from collections.abc import Mapping from typing import ClassVar -from typing import Optional from typing import Union from mitmproxy import flow @@ -26,9 +25,9 @@ def __call__( self, data: bytes, *, - content_type: Optional[str] = None, - flow: Optional[flow.Flow] = None, - http_message: Optional[http.Message] = None, + content_type: str | None = None, + flow: flow.Flow | None = None, + http_message: http.Message | None = None, **unknown_metadata, ) -> TViewResult: """ @@ -52,9 +51,9 @@ def render_priority( self, data: bytes, *, - content_type: Optional[str] = None, - flow: Optional[flow.Flow] = None, - http_message: Optional[http.Message] = None, + content_type: str | None = None, + flow: flow.Flow | None = None, + http_message: http.Message | None = None, **unknown_metadata, ) -> float: """ diff --git a/mitmproxy/contentviews/css.py b/mitmproxy/contentviews/css.py index fd878c0afd..0adde59d5e 100644 --- a/mitmproxy/contentviews/css.py +++ b/mitmproxy/contentviews/css.py @@ -1,6 +1,5 @@ import re import time -from typing import Optional from mitmproxy.contentviews import base from mitmproxy.utils import strutils @@ -58,7 +57,7 @@ def __call__(self, data, **metadata): return "CSS", base.format_text(beautified) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float(bool(data) and content_type == "text/css") diff --git a/mitmproxy/contentviews/graphql.py b/mitmproxy/contentviews/graphql.py index 198082e807..1dd41a676c 100644 --- a/mitmproxy/contentviews/graphql.py +++ b/mitmproxy/contentviews/graphql.py @@ -1,6 +1,5 @@ import json from typing import Any -from typing import Optional from mitmproxy.contentviews import base from mitmproxy.contentviews.json import PARSE_ERROR @@ -48,7 +47,7 @@ def __call__(self, data, **metadata): return "GraphQL", base.format_text(format_query_list(data)) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: if content_type != "application/json" or not data: return 0 diff --git a/mitmproxy/contentviews/http3.py b/mitmproxy/contentviews/http3.py index 4237720342..e84ad6ef35 100644 --- a/mitmproxy/contentviews/http3.py +++ b/mitmproxy/contentviews/http3.py @@ -2,8 +2,6 @@ from collections.abc import Iterator from dataclasses import dataclass from dataclasses import field -from typing import Optional -from typing import Union import pylsqpack from aioquic.buffer import Buffer @@ -74,7 +72,7 @@ def pretty(self): @dataclass class ConnectionState: message_count: int = 0 - frames: dict[int, list[Union[Frame, StreamType]]] = field(default_factory=dict) + frames: dict[int, list[Frame | StreamType]] = field(default_factory=dict) client_buf: bytearray = field(default_factory=bytearray) server_buf: bytearray = field(default_factory=bytearray) @@ -90,8 +88,8 @@ def __init__(self) -> None: def __call__( self, data, - flow: Optional[flow.Flow] = None, - tcp_message: Optional[tcp.TCPMessage] = None, + flow: flow.Flow | None = None, + tcp_message: tcp.TCPMessage | None = None, **metadata, ): assert isinstance(flow, tcp.TCPFlow) @@ -146,7 +144,7 @@ def __call__( return "HTTP/3", fmt_frames(frames) def render_priority( - self, data: bytes, flow: Optional[flow.Flow] = None, **metadata + self, data: bytes, flow: flow.Flow | None = None, **metadata ) -> float: return ( 2 @@ -155,7 +153,7 @@ def render_priority( ) -def fmt_frames(frames: list[Union[Frame, StreamType]]) -> Iterator[base.TViewLine]: +def fmt_frames(frames: list[Frame | StreamType]) -> Iterator[base.TViewLine]: for i, frame in enumerate(frames): if i > 0: yield [("text", "")] diff --git a/mitmproxy/contentviews/image/view.py b/mitmproxy/contentviews/image/view.py index 5d621133fa..021cac9920 100644 --- a/mitmproxy/contentviews/image/view.py +++ b/mitmproxy/contentviews/image/view.py @@ -1,5 +1,4 @@ import imghdr -from typing import Optional from . import image_parser from mitmproxy.contentviews import base @@ -36,7 +35,7 @@ def __call__(self, data, **metadata): return view_name, base.format_dict(multidict.MultiDict(image_metadata)) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float( bool( diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/javascript.py index 33ecac2ae9..02b47d53e8 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/javascript.py @@ -1,6 +1,5 @@ import io import re -from typing import Optional from mitmproxy.contentviews import base from mitmproxy.utils import strutils @@ -55,6 +54,6 @@ def __call__(self, data, **metadata): return "JavaScript", base.format_text(res) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float(bool(data) and content_type in self.__content_types) diff --git a/mitmproxy/contentviews/json.py b/mitmproxy/contentviews/json.py index 23ec86a0d9..1b0e22a1e6 100644 --- a/mitmproxy/contentviews/json.py +++ b/mitmproxy/contentviews/json.py @@ -3,7 +3,6 @@ from collections.abc import Iterator from functools import lru_cache from typing import Any -from typing import Optional from mitmproxy.contentviews import base @@ -55,7 +54,7 @@ def __call__(self, data, **metadata): return "JSON", format_json(data) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: if not data: return 0 diff --git a/mitmproxy/contentviews/mqtt.py b/mitmproxy/contentviews/mqtt.py index 1c3b92a37f..ad40482601 100644 --- a/mitmproxy/contentviews/mqtt.py +++ b/mitmproxy/contentviews/mqtt.py @@ -1,5 +1,4 @@ import struct -from typing import Optional from mitmproxy.contentviews import base from mitmproxy.utils import strutils @@ -273,6 +272,6 @@ def __call__(self, data, **metadata): return "MQTT", base.format_text(text) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return 0 diff --git a/mitmproxy/contentviews/msgpack.py b/mitmproxy/contentviews/msgpack.py index 8de6c9abbf..21527b23de 100644 --- a/mitmproxy/contentviews/msgpack.py +++ b/mitmproxy/contentviews/msgpack.py @@ -1,5 +1,4 @@ from typing import Any -from typing import Optional import msgpack @@ -95,6 +94,6 @@ def __call__(self, data, **metadata): return "MsgPack", format_msgpack(data) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float(bool(data) and content_type in self.__content_types) diff --git a/mitmproxy/contentviews/multipart.py b/mitmproxy/contentviews/multipart.py index df27e53b33..6114fff653 100644 --- a/mitmproxy/contentviews/multipart.py +++ b/mitmproxy/contentviews/multipart.py @@ -1,5 +1,3 @@ -from typing import Optional - from . import base from mitmproxy.coretypes import multidict from mitmproxy.net.http import multipart @@ -13,7 +11,7 @@ def _format(v): yield [("highlight", "Form data:\n")] yield from base.format_dict(multidict.MultiDict(v)) - def __call__(self, data: bytes, content_type: Optional[str] = None, **metadata): + def __call__(self, data: bytes, content_type: str | None = None, **metadata): if content_type is None: return v = multipart.decode_multipart(content_type, data) @@ -21,6 +19,6 @@ def __call__(self, data: bytes, content_type: Optional[str] = None, **metadata): return "Multipart form", self._format(v) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float(bool(data) and content_type == "multipart/form-data") diff --git a/mitmproxy/contentviews/protobuf.py b/mitmproxy/contentviews/protobuf.py index 0d836be102..82c888d8a7 100644 --- a/mitmproxy/contentviews/protobuf.py +++ b/mitmproxy/contentviews/protobuf.py @@ -1,5 +1,4 @@ import io -from typing import Optional from kaitaistruct import KaitaiStream @@ -98,6 +97,6 @@ def __call__(self, data, **metadata): return "Protobuf", base.format_text(decoded) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float(bool(data) and content_type in self.__content_types) diff --git a/mitmproxy/contentviews/query.py b/mitmproxy/contentviews/query.py index 49c9107028..d816921e2b 100644 --- a/mitmproxy/contentviews/query.py +++ b/mitmproxy/contentviews/query.py @@ -1,5 +1,3 @@ -from typing import Optional - from . import base from .. import http @@ -8,7 +6,7 @@ class ViewQuery(base.View): name = "Query" def __call__( - self, data: bytes, http_message: Optional[http.Message] = None, **metadata + self, data: bytes, http_message: http.Message | None = None, **metadata ): query = getattr(http_message, "query", None) if query: @@ -17,6 +15,6 @@ def __call__( return "Query", base.format_text("") def render_priority( - self, data: bytes, *, http_message: Optional[http.Message] = None, **metadata + self, data: bytes, *, http_message: http.Message | None = None, **metadata ) -> float: return 0.3 * float(bool(getattr(http_message, "query", False) and not data)) diff --git a/mitmproxy/contentviews/urlencoded.py b/mitmproxy/contentviews/urlencoded.py index 27e4e83d2f..5065dca338 100644 --- a/mitmproxy/contentviews/urlencoded.py +++ b/mitmproxy/contentviews/urlencoded.py @@ -1,5 +1,3 @@ -from typing import Optional - from . import base from mitmproxy.net.http import url @@ -16,6 +14,6 @@ def __call__(self, data, **metadata): return "URLEncoded form", base.format_pairs(d) def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float(bool(data) and content_type == "application/x-www-form-urlencoded") diff --git a/mitmproxy/contentviews/wbxml.py b/mitmproxy/contentviews/wbxml.py index 3faaa86d29..1aa19fec7d 100644 --- a/mitmproxy/contentviews/wbxml.py +++ b/mitmproxy/contentviews/wbxml.py @@ -1,5 +1,3 @@ -from typing import Optional - from . import base from mitmproxy.contrib.wbxml import ASCommandResponse @@ -18,6 +16,6 @@ def __call__(self, data, **metadata): return None def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: return float(bool(data) and content_type in self.__content_types) diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index 747b5b9c39..f402bbe568 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -2,7 +2,6 @@ import re import textwrap from collections.abc import Iterable -from typing import Optional from mitmproxy.contentviews import base from mitmproxy.utils import sliding_window @@ -140,7 +139,7 @@ def indent_text(data: str, prefix: str) -> str: return textwrap.indent(dedented, prefix[:32]) -def is_inline_text(a: Optional[Token], b: Optional[Token], c: Optional[Token]) -> bool: +def is_inline_text(a: Token | None, b: Token | None, c: Token | None) -> bool: if isinstance(a, Tag) and isinstance(b, Text) and isinstance(c, Tag): if a.is_opening and "\n" not in b.data and c.is_closing and a.tag == c.tag: return True @@ -148,11 +147,11 @@ def is_inline_text(a: Optional[Token], b: Optional[Token], c: Optional[Token]) - def is_inline( - prev2: Optional[Token], - prev1: Optional[Token], - t: Optional[Token], - next1: Optional[Token], - next2: Optional[Token], + prev2: Token | None, + prev1: Token | None, + t: Token | None, + next1: Token | None, + next2: Token | None, ) -> bool: if isinstance(t, Text): return is_inline_text(prev1, t, next1) @@ -267,7 +266,7 @@ def __call__(self, data, **metadata): return t, pretty def render_priority( - self, data: bytes, *, content_type: Optional[str] = None, **metadata + self, data: bytes, *, content_type: str | None = None, **metadata ) -> float: if not data: return 0 diff --git a/mitmproxy/contrib/kaitaistruct/exif.py b/mitmproxy/contrib/kaitaistruct/exif.py index f0cadd4ae6..464849e420 100644 --- a/mitmproxy/contrib/kaitaistruct/exif.py +++ b/mitmproxy/contrib/kaitaistruct/exif.py @@ -1,7 +1,7 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild import kaitaistruct -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import KaitaiStream, KaitaiStruct from enum import Enum diff --git a/mitmproxy/contrib/kaitaistruct/google_protobuf.py b/mitmproxy/contrib/kaitaistruct/google_protobuf.py index de06e8d267..48f5e0ec9b 100644 --- a/mitmproxy/contrib/kaitaistruct/google_protobuf.py +++ b/mitmproxy/contrib/kaitaistruct/google_protobuf.py @@ -1,7 +1,7 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild import kaitaistruct -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import KaitaiStream, KaitaiStruct from enum import Enum diff --git a/mitmproxy/contrib/kaitaistruct/ico.py b/mitmproxy/contrib/kaitaistruct/ico.py index 966bfb9481..39b34ae49a 100644 --- a/mitmproxy/contrib/kaitaistruct/ico.py +++ b/mitmproxy/contrib/kaitaistruct/ico.py @@ -1,7 +1,7 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild import kaitaistruct -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import KaitaiStruct if getattr(kaitaistruct, 'API_VERSION', (0, 9)) < (0, 9): diff --git a/mitmproxy/contrib/kaitaistruct/vlq_base128_le.py b/mitmproxy/contrib/kaitaistruct/vlq_base128_le.py index 6f7a37f800..78f28c28d3 100644 --- a/mitmproxy/contrib/kaitaistruct/vlq_base128_le.py +++ b/mitmproxy/contrib/kaitaistruct/vlq_base128_le.py @@ -1,7 +1,7 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild import kaitaistruct -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import KaitaiStruct if getattr(kaitaistruct, 'API_VERSION', (0, 9)) < (0, 9): diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index 57cf241d05..40b4767b9e 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -1,6 +1,6 @@ +from collections.abc import Callable from collections.abc import Iterator from typing import Any -from typing import Callable from mitmproxy import dns from mitmproxy import flow diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 889e3ae445..0415ee1f3f 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -8,7 +8,6 @@ from dataclasses import field from typing import Any from typing import ClassVar -from typing import Optional from mitmproxy import connection from mitmproxy import exceptions @@ -66,7 +65,7 @@ class Flow(serializable.Serializable): with a `timestamp_start` set to `None`. """ - error: Optional[Error] = None + error: Error | None = None """A connection or protocol error affecting this flow.""" intercepted: bool @@ -89,7 +88,7 @@ class Flow(serializable.Serializable): The default marker for the view will be used if the Unicode emoji name can not be interpreted. """ - is_replay: Optional[str] + is_replay: str | None """ This attribute indicates if this flow has been replayed in either direction. @@ -123,10 +122,10 @@ def __init__( self.timestamp_created = time.time() self.intercepted: bool = False - self._resume_event: Optional[asyncio.Event] = None - self._backup: Optional[Flow] = None + self._resume_event: asyncio.Event | None = None + self._backup: Flow | None = None self.marked: str = "" - self.is_replay: Optional[str] = None + self.is_replay: str | None = None self.metadata: dict[str, Any] = dict() self.comment: str = "" diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index a596e288ad..52db22be03 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -38,7 +38,6 @@ from collections.abc import Sequence from typing import ClassVar from typing import Protocol -from typing import Union import pyparsing as pp @@ -662,7 +661,7 @@ def parse(s: str) -> TFilter: raise ValueError(f"Invalid filter expression: {s!r}") from e -def match(flt: Union[str, TFilter], flow: flow.Flow) -> bool: +def match(flt: str | TFilter, flow: flow.Flow) -> bool: """ Matches a flow against a compiled filter expression. Returns True if matched, False if not. diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 4556678cf2..bd0f9d2776 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -5,6 +5,7 @@ import time import urllib.parse import warnings +from collections.abc import Callable from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Mapping @@ -15,10 +16,7 @@ from email.utils import mktime_tz from email.utils import parsedate_tz from typing import Any -from typing import Callable from typing import cast -from typing import Optional -from typing import Union from mitmproxy import flow from mitmproxy.coretypes import multidict @@ -43,7 +41,7 @@ def _native(x: bytes) -> str: return x.decode("utf-8", "surrogateescape") -def _always_bytes(x: Union[str, bytes]) -> bytes: +def _always_bytes(x: str | bytes) -> bytes: return strutils.always_bytes(x, "utf-8", "surrogateescape") @@ -136,7 +134,7 @@ def __bytes__(self) -> bytes: else: return b"" - def __delitem__(self, key: Union[str, bytes]) -> None: + def __delitem__(self, key: str | bytes) -> None: key = _always_bytes(key) super().__delitem__(key) @@ -144,7 +142,7 @@ def __iter__(self) -> Iterator[str]: for x in super().__iter__(): yield _native(x) - def get_all(self, name: Union[str, bytes]) -> list[str]: + def get_all(self, name: str | bytes) -> list[str]: """ Like `Headers.get`, but does not fold multiple headers into a single one. This is useful for Set-Cookie and Cookie headers, which do not support folding. @@ -157,7 +155,7 @@ def get_all(self, name: Union[str, bytes]) -> list[str]: name = _always_bytes(name) return [_native(x) for x in super().get_all(name)] - def set_all(self, name: Union[str, bytes], values: Iterable[Union[str, bytes]]): + def set_all(self, name: str | bytes, values: Iterable[str | bytes]): """ Explicitly set multiple headers for the given key. See `Headers.get_all`. @@ -166,7 +164,7 @@ def set_all(self, name: Union[str, bytes], values: Iterable[Union[str, bytes]]): values = [_always_bytes(x) for x in values] return super().set_all(name, values) - def insert(self, index: int, key: Union[str, bytes], value: Union[str, bytes]): + def insert(self, index: int, key: str | bytes, value: str | bytes): key = _always_bytes(key) value = _always_bytes(value) super().insert(index, key, value) @@ -182,10 +180,10 @@ def items(self, multi=False): class MessageData(serializable.Serializable): http_version: bytes headers: Headers - content: Optional[bytes] - trailers: Optional[Headers] + content: bytes | None + trailers: Headers | None timestamp_start: float - timestamp_end: Optional[float] + timestamp_end: float | None # noinspection PyUnreachableCode if __debug__: @@ -246,7 +244,7 @@ def set_state(self, state): self.data.set_state(state) data: MessageData - stream: Union[Callable[[bytes], Union[Iterable[bytes], bytes]], bool] = False + stream: Callable[[bytes], Iterable[bytes] | bytes] | bool = False """ This attribute controls if the message body should be streamed. @@ -269,7 +267,7 @@ def http_version(self) -> str: return self.data.http_version.decode("utf-8", "surrogateescape") @http_version.setter - def http_version(self, http_version: Union[str, bytes]) -> None: + def http_version(self, http_version: str | bytes) -> None: self.data.http_version = strutils.always_bytes( http_version, "utf-8", "surrogateescape" ) @@ -302,18 +300,18 @@ def headers(self, h: Headers) -> None: self.data.headers = h @property - def trailers(self) -> Optional[Headers]: + def trailers(self) -> Headers | None: """ The [HTTP trailers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Trailer). """ return self.data.trailers @trailers.setter - def trailers(self, h: Optional[Headers]) -> None: + def trailers(self, h: Headers | None) -> None: self.data.trailers = h @property - def raw_content(self) -> Optional[bytes]: + def raw_content(self) -> bytes | None: """ The raw (potentially compressed) HTTP message body. @@ -324,11 +322,11 @@ def raw_content(self) -> Optional[bytes]: return self.data.content @raw_content.setter - def raw_content(self, content: Optional[bytes]) -> None: + def raw_content(self, content: bytes | None) -> None: self.data.content = content @property - def content(self) -> Optional[bytes]: + def content(self) -> bytes | None: """ The uncompressed HTTP message body as bytes. @@ -339,11 +337,11 @@ def content(self) -> Optional[bytes]: return self.get_content() @content.setter - def content(self, value: Optional[bytes]) -> None: + def content(self, value: bytes | None) -> None: self.set_content(value) @property - def text(self) -> Optional[str]: + def text(self) -> str | None: """ The uncompressed and decoded HTTP message body as text. @@ -354,10 +352,10 @@ def text(self) -> Optional[str]: return self.get_text() @text.setter - def text(self, value: Optional[str]) -> None: + def text(self, value: str | None) -> None: self.set_text(value) - def set_content(self, value: Optional[bytes]) -> None: + def set_content(self, value: bytes | None) -> None: if value is None: self.raw_content = None return @@ -382,7 +380,7 @@ def set_content(self, value: Optional[bytes]) -> None: else: self.headers["content-length"] = str(len(self.raw_content)) - def get_content(self, strict: bool = True) -> Optional[bytes]: + def get_content(self, strict: bool = True) -> bytes | None: """ Similar to `Message.content`, but does not raise if `strict` is `False`. Instead, the compressed message body is returned as-is. @@ -404,7 +402,7 @@ def get_content(self, strict: bool = True) -> Optional[bytes]: else: return self.raw_content - def _get_content_type_charset(self) -> Optional[str]: + def _get_content_type_charset(self) -> str | None: ct = parse_content_type(self.headers.get("content-type", "")) if ct: return ct[2].get("charset") @@ -438,7 +436,7 @@ def _guess_encoding(self, content: bytes = b"") -> str: return enc - def set_text(self, text: Optional[str]) -> None: + def set_text(self, text: str | None) -> None: if text is None: self.content = None return @@ -458,7 +456,7 @@ def set_text(self, text: Optional[str]) -> None: enc = "utf8" self.content = text.encode(enc, "surrogateescape") - def get_text(self, strict: bool = True) -> Optional[str]: + def get_text(self, strict: bool = True) -> str | None: """ Similar to `Message.text`, but does not raise if `strict` is `False`. Instead, the message body is returned as surrogate-escaped UTF-8. @@ -486,14 +484,14 @@ def timestamp_start(self, timestamp_start: float) -> None: self.data.timestamp_start = timestamp_start @property - def timestamp_end(self) -> Optional[float]: + def timestamp_end(self) -> float | None: """ *Timestamp:* Last byte received. """ return self.data.timestamp_end @timestamp_end.setter - def timestamp_end(self, timestamp_end: Optional[float]): + def timestamp_end(self, timestamp_end: float | None): self.data.timestamp_end = timestamp_end def decode(self, strict: bool = True) -> None: @@ -558,11 +556,11 @@ def __init__( authority: bytes, path: bytes, http_version: bytes, - headers: Union[Headers, tuple[tuple[bytes, bytes], ...]], - content: Optional[bytes], - trailers: Union[Headers, tuple[tuple[bytes, bytes], ...], None], + headers: Headers | tuple[tuple[bytes, bytes], ...], + content: bytes | None, + trailers: Headers | tuple[tuple[bytes, bytes], ...] | None, timestamp_start: float, - timestamp_end: Optional[float], + timestamp_end: float | None, ): # auto-convert invalid types to retain compatibility with older code. if isinstance(host, bytes): @@ -613,12 +611,10 @@ def make( cls, method: str, url: str, - content: Union[bytes, str] = "", - headers: Union[ - Headers, - dict[Union[str, bytes], Union[str, bytes]], - Iterable[tuple[bytes, bytes]], - ] = (), + content: bytes | str = "", + headers: ( + Headers | dict[str | bytes, str | bytes] | Iterable[tuple[bytes, bytes]] + ) = (), ) -> "Request": """ Simplified API for creating request objects. @@ -693,7 +689,7 @@ def method(self) -> str: return self.data.method.decode("utf-8", "surrogateescape").upper() @method.setter - def method(self, val: Union[str, bytes]) -> None: + def method(self, val: str | bytes) -> None: self.data.method = always_bytes(val, "utf-8", "surrogateescape") @property @@ -704,7 +700,7 @@ def scheme(self) -> str: return self.data.scheme.decode("utf-8", "surrogateescape") @scheme.setter - def scheme(self, val: Union[str, bytes]) -> None: + def scheme(self, val: str | bytes) -> None: self.data.scheme = always_bytes(val, "utf-8", "surrogateescape") @property @@ -726,7 +722,7 @@ def authority(self) -> str: return self.data.authority.decode("utf8", "surrogateescape") @authority.setter - def authority(self, val: Union[str, bytes]) -> None: + def authority(self, val: str | bytes) -> None: if isinstance(val, str): try: val = val.encode("idna", "strict") @@ -748,12 +744,12 @@ def host(self) -> str: return self.data.host @host.setter - def host(self, val: Union[str, bytes]) -> None: + def host(self, val: str | bytes) -> None: self.data.host = always_str(val, "idna", "strict") self._update_host_and_authority() @property - def host_header(self) -> Optional[str]: + def host_header(self) -> str | None: """ The request's host/authority header. @@ -768,7 +764,7 @@ def host_header(self) -> Optional[str]: return self.data.headers.get("Host", None) @host_header.setter - def host_header(self, val: Union[None, str, bytes]) -> None: + def host_header(self, val: None | str | bytes) -> None: if val is None: if self.is_http2 or self.is_http3: self.data.authority = b"" @@ -814,7 +810,7 @@ def path(self) -> str: return self.data.path.decode("utf-8", "surrogateescape") @path.setter - def path(self, val: Union[str, bytes]) -> None: + def path(self, val: str | bytes) -> None: self.data.path = always_bytes(val, "utf-8", "surrogateescape") @property @@ -829,7 +825,7 @@ def url(self) -> str: return url.unparse(self.scheme, self.host, self.port, self.path) @url.setter - def url(self, val: Union[str, bytes]) -> None: + def url(self, val: str | bytes) -> None: val = always_str(val, "utf-8", "surrogateescape") self.scheme, self.host, self.port, self.path = url.parse(val) @@ -1050,11 +1046,11 @@ def __init__( http_version: bytes, status_code: int, reason: bytes, - headers: Union[Headers, tuple[tuple[bytes, bytes], ...]], - content: Optional[bytes], - trailers: Union[None, Headers, tuple[tuple[bytes, bytes], ...]], + headers: Headers | tuple[tuple[bytes, bytes], ...], + content: bytes | None, + trailers: None | Headers | tuple[tuple[bytes, bytes], ...], timestamp_start: float, - timestamp_end: Optional[float], + timestamp_end: float | None, ): # auto-convert invalid types to retain compatibility with older code. if isinstance(http_version, str): @@ -1093,10 +1089,10 @@ def __repr__(self) -> str: def make( cls, status_code: int = 200, - content: Union[bytes, str] = b"", - headers: Union[ - Headers, Mapping[str, Union[str, bytes]], Iterable[tuple[bytes, bytes]] - ] = (), + content: bytes | str = b"", + headers: ( + Headers | Mapping[str, str | bytes] | Iterable[tuple[bytes, bytes]] + ) = (), ) -> "Response": """ Simplified API for creating response objects. @@ -1165,7 +1161,7 @@ def reason(self) -> str: return self.data.reason.decode("ISO-8859-1") @reason.setter - def reason(self, reason: Union[str, bytes]) -> None: + def reason(self, reason: str | bytes) -> None: self.data.reason = strutils.always_bytes(reason, "ISO-8859-1") def _get_cookies(self): @@ -1183,9 +1179,7 @@ def _set_cookies(self, value): @property def cookies( self, - ) -> multidict.MultiDictView[ - str, tuple[str, multidict.MultiDict[str, Optional[str]]] - ]: + ) -> multidict.MultiDictView[str, tuple[str, multidict.MultiDict[str, str | None]]]: """ The response cookies. A possibly empty `MultiDictView`, where the keys are cookie name strings, and values are `(cookie value, attributes)` tuples. Within @@ -1245,9 +1239,9 @@ class HTTPFlow(flow.Flow): request: Request """The client's HTTP request.""" - response: Optional[Response] = None + response: Response | None = None """The server's HTTP response.""" - error: Optional[flow.Error] = None + error: flow.Error | None = None """ A connection or protocol error affecting this flow. @@ -1256,7 +1250,7 @@ class HTTPFlow(flow.Flow): from the server, but there was an error sending it back to the client. """ - websocket: Optional[WebSocketData] = None + websocket: WebSocketData | None = None """ If this HTTP flow initiated a WebSocket connection, this attribute contains all associated WebSocket data. """ diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 9a923c67a9..fe9c4a5972 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -8,7 +8,6 @@ import copy import uuid from typing import Any -from typing import Union from mitmproxy import version from mitmproxy.utils import strutils @@ -491,9 +490,7 @@ def convert_unicode(data: dict) -> dict: } -def migrate_flow( - flow_data: dict[Union[bytes, str], Any] -) -> dict[Union[bytes, str], Any]: +def migrate_flow(flow_data: dict[bytes | str, Any]) -> dict[bytes | str, Any]: while True: flow_version = flow_data.get(b"version", flow_data.get("version")) diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 3484600a8f..ec0d9354f9 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -1,7 +1,6 @@ import asyncio import logging import traceback -from typing import Optional from . import ctx as mitmproxy_ctx from .proxy.mode_specs import ReverseMode @@ -26,7 +25,7 @@ class Master: def __init__( self, opts: options.Options, - event_loop: Optional[asyncio.AbstractEventLoop] = None, + event_loop: asyncio.AbstractEventLoop | None = None, ): self.options: options.Options = opts or options.Options() self.commands = command.CommandManager(self) diff --git a/mitmproxy/net/encoding.py b/mitmproxy/net/encoding.py index 29553651fe..30453120b5 100644 --- a/mitmproxy/net/encoding.py +++ b/mitmproxy/net/encoding.py @@ -7,7 +7,6 @@ import zlib from io import BytesIO from typing import overload -from typing import Union import brotli import zstandard as zstd @@ -31,13 +30,13 @@ def decode(encoded: str, encoding: str, errors: str = "strict") -> str: @overload -def decode(encoded: bytes, encoding: str, errors: str = "strict") -> Union[str, bytes]: +def decode(encoded: bytes, encoding: str, errors: str = "strict") -> str | bytes: ... def decode( - encoded: Union[None, str, bytes], encoding: str, errors: str = "strict" -) -> Union[None, str, bytes]: + encoded: None | str | bytes, encoding: str, errors: str = "strict" +) -> None | str | bytes: """ Decode the given input object @@ -87,7 +86,7 @@ def encode(decoded: None, encoding: str, errors: str = "strict") -> None: @overload -def encode(decoded: str, encoding: str, errors: str = "strict") -> Union[str, bytes]: +def encode(decoded: str, encoding: str, errors: str = "strict") -> str | bytes: ... @@ -97,8 +96,8 @@ def encode(decoded: bytes, encoding: str, errors: str = "strict") -> bytes: def encode( - decoded: Union[None, str, bytes], encoding, errors="strict" -) -> Union[None, str, bytes]: + decoded: None | str | bytes, encoding, errors="strict" +) -> None | str | bytes: """ Encode the given input object diff --git a/mitmproxy/net/http/headers.py b/mitmproxy/net/http/headers.py index 6204040aac..e87efc5032 100644 --- a/mitmproxy/net/http/headers.py +++ b/mitmproxy/net/http/headers.py @@ -1,8 +1,7 @@ import collections -from typing import Optional -def parse_content_type(c: str) -> Optional[tuple[str, str, dict[str, str]]]: +def parse_content_type(c: str) -> tuple[str, str, dict[str, str]] | None: """ A simple parser for content-type values. Returns a (type, subtype, parameters) tuple, where type and subtype are strings, and parameters diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index 578fcd7d41..774dc83f1f 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -1,7 +1,6 @@ import re import time from collections.abc import Iterable -from typing import Optional from mitmproxy.http import Headers from mitmproxy.http import Request @@ -78,8 +77,8 @@ def validate_headers(headers: Headers) -> None: def expected_http_body_size( - request: Request, response: Optional[Response] = None -) -> Optional[int]: + request: Request, response: Response | None = None +) -> int | None: """ Returns: The expected body length: @@ -226,7 +225,7 @@ def _read_request_line( ) -> tuple[str, int, bytes, bytes, bytes, bytes, bytes]: try: method, target, http_version = line.split() - port: Optional[int] + port: int | None if target == b"*" or target.startswith(b"/"): scheme, authority, path = b"", b"", target diff --git a/mitmproxy/net/http/multipart.py b/mitmproxy/net/http/multipart.py index bc53af66a0..c2c0a8bbfa 100644 --- a/mitmproxy/net/http/multipart.py +++ b/mitmproxy/net/http/multipart.py @@ -3,7 +3,6 @@ import mimetypes import re import warnings -from typing import Optional from urllib.parse import quote from mitmproxy.net.http import headers @@ -47,7 +46,7 @@ def encode_multipart(content_type: str, parts: list[tuple[bytes, bytes]]) -> byt def decode_multipart( - content_type: Optional[str], content: bytes + content_type: str | None, content: bytes ) -> list[tuple[bytes, bytes]]: """ Takes a multipart boundary encoded string and returns list of (key, value) tuples. diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index abc038abf4..4407f02acc 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -4,7 +4,6 @@ import urllib.parse from collections.abc import Sequence from typing import AnyStr -from typing import Optional from mitmproxy.net import check from mitmproxy.net.check import is_valid_host @@ -147,7 +146,7 @@ def hostport(scheme: AnyStr, host: AnyStr, port: int) -> AnyStr: return "%s:%d" % (host, port) -def default_port(scheme: AnyStr) -> Optional[int]: +def default_port(scheme: AnyStr) -> int | None: return { "http": 80, b"http": 80, @@ -156,7 +155,7 @@ def default_port(scheme: AnyStr) -> Optional[int]: }.get(scheme, None) -def parse_authority(authority: AnyStr, check: bool) -> tuple[str, Optional[int]]: +def parse_authority(authority: AnyStr, check: bool) -> tuple[str, int | None]: """Extract the host and port from host header/authority information Raises: diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 59fd229e32..ab2adab397 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -1,13 +1,12 @@ import os import threading +from collections.abc import Callable from collections.abc import Iterable from enum import Enum from functools import lru_cache from pathlib import Path from typing import Any from typing import BinaryIO -from typing import Callable -from typing import Optional import certifi from OpenSSL import SSL @@ -55,7 +54,7 @@ class Verify(Enum): class MasterSecretLogger: def __init__(self, filename: Path): self.filename = filename.expanduser() - self.f: Optional[BinaryIO] = None + self.f: BinaryIO | None = None self.lock = threading.Lock() # required for functools.wraps, which pyOpenSSL uses. @@ -76,7 +75,7 @@ def close(self): self.f.close() -def make_master_secret_logger(filename: Optional[str]) -> Optional[MasterSecretLogger]: +def make_master_secret_logger(filename: str | None) -> MasterSecretLogger | None: if filename: return MasterSecretLogger(Path(filename)) return None @@ -92,7 +91,7 @@ def _create_ssl_context( method: Method, min_version: Version, max_version: Version, - cipher_list: Optional[Iterable[str]], + cipher_list: Iterable[str] | None, ) -> SSL.Context: context = SSL.Context(method.value) @@ -127,11 +126,11 @@ def create_proxy_server_context( method: Method, min_version: Version, max_version: Version, - cipher_list: Optional[tuple[str, ...]], + cipher_list: tuple[str, ...] | None, verify: Verify, - ca_path: Optional[str], - ca_pemfile: Optional[str], - client_cert: Optional[str], + ca_path: str | None, + ca_pemfile: str | None, + client_cert: str | None, ) -> SSL.Context: context: SSL.Context = _create_ssl_context( method=method, @@ -167,9 +166,9 @@ def create_client_proxy_context( method: Method, min_version: Version, max_version: Version, - cipher_list: Optional[tuple[str, ...]], - chain_file: Optional[Path], - alpn_select_callback: Optional[Callable[[SSL.Connection, list[bytes]], Any]], + cipher_list: tuple[str, ...] | None, + chain_file: Path | None, + alpn_select_callback: Callable[[SSL.Connection, list[bytes]], Any] | None, request_client_cert: bool, extra_chain_certs: tuple[certs.Cert, ...], dhparams: certs.DHParams, diff --git a/mitmproxy/net/udp.py b/mitmproxy/net/udp.py index c1bf595e61..ac43b82beb 100644 --- a/mitmproxy/net/udp.py +++ b/mitmproxy/net/udp.py @@ -3,10 +3,9 @@ import asyncio import logging import socket +from collections.abc import Callable from typing import Any -from typing import Callable from typing import cast -from typing import Optional from typing import Union import mitmproxy_rs @@ -267,7 +266,7 @@ async def start_server( async def open_connection( - host: str, port: int, *, local_addr: Optional[Address] = None + host: str, port: int, *, local_addr: Address | None = None ) -> tuple[DatagramReader, DatagramWriter]: """UDP variant of asyncio.open_connection.""" diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 17cec4d983..2efd2ddf30 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -13,7 +13,6 @@ from typing import Any from typing import Optional from typing import TextIO -from typing import Union import ruamel.yaml @@ -34,10 +33,10 @@ class _Option: def __init__( self, name: str, - typespec: Union[type, object], # object for Optional[x], which is not a type. + typespec: type | object, # object for Optional[x], which is not a type. default: Any, help: str, - choices: Optional[Sequence[str]], + choices: Sequence[str] | None, ) -> None: typecheck.check_option_type(name, default, typespec) self.name = name @@ -123,10 +122,10 @@ def __init__(self) -> None: def add_option( self, name: str, - typespec: Union[type, object], + typespec: type | object, default: Any, help: str, - choices: Optional[Sequence[str]] = None, + choices: Sequence[str] | None = None, ) -> None: self._options[name] = _Option(name, typespec, default, help, choices) self.changed.send(updated={name}) @@ -373,7 +372,7 @@ def _parse_setval(self, o: _Option, values: list[str]) -> Any: f"Received multiple values for {o.name}: {values}" ) - optstr: Optional[str] + optstr: str | None if values: optstr = values[0] else: diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py index 0b0c492ada..8f3660b18c 100644 --- a/mitmproxy/platform/__init__.py +++ b/mitmproxy/platform/__init__.py @@ -1,8 +1,7 @@ import re import socket import sys -from typing import Callable -from typing import Optional +from collections.abc import Callable def init_transparent_mode() -> None: @@ -11,7 +10,7 @@ def init_transparent_mode() -> None: """ -original_addr: Optional[Callable[[socket.socket], tuple[str, int]]] +original_addr: Callable[[socket.socket], tuple[str, int]] | None """ Get the original destination for the given socket. This function will be None if transparent mode is not supported. diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index 005bb148a5..53539d4b77 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -15,7 +15,6 @@ from typing import cast from typing import ClassVar from typing import IO -from typing import Optional import pydivert.consts @@ -294,7 +293,7 @@ def run(self): def shutdown(self): self.windivert.close() - def recv(self) -> Optional[pydivert.Packet]: + def recv(self) -> pydivert.Packet | None: """ Convenience function that receives a packet from the passed handler and handles error codes. If the process has been shut down, None is returned. @@ -402,9 +401,9 @@ class TransparentProxy: which mitmproxy sees, but this would remove the correct client info from mitmproxy. """ - local: Optional[RedirectLocal] = None + local: RedirectLocal | None = None # really weird linting error here. - forward: Optional[Redirect] = None + forward: Redirect | None = None response: Redirect icmp: Redirect @@ -418,7 +417,7 @@ def __init__( local: bool = True, forward: bool = True, proxy_port: int = 8080, - filter: Optional[str] = "tcp.DstPort == 80 or tcp.DstPort == 443", + filter: str | None = "tcp.DstPort == 80 or tcp.DstPort == 443", ) -> None: self.proxy_port = proxy_port self.filter = ( diff --git a/mitmproxy/proxy/events.py b/mitmproxy/proxy/events.py index e741fbcfb7..c0518631c7 100644 --- a/mitmproxy/proxy/events.py +++ b/mitmproxy/proxy/events.py @@ -9,7 +9,6 @@ from dataclasses import is_dataclass from typing import Any from typing import Generic -from typing import Optional from typing import TypeVar from mitmproxy import flow @@ -106,7 +105,7 @@ def __repr__(self): @dataclass(repr=False) class OpenConnectionCompleted(CommandCompleted): command: commands.OpenConnection - reply: Optional[str] + reply: str | None """error message""" diff --git a/mitmproxy/proxy/layer.py b/mitmproxy/proxy/layer.py index 79aed95c9a..275125a3f8 100644 --- a/mitmproxy/proxy/layer.py +++ b/mitmproxy/proxy/layer.py @@ -11,7 +11,6 @@ from typing import Any from typing import ClassVar from typing import NamedTuple -from typing import Optional from typing import TypeVar from mitmproxy.connection import Connection @@ -60,7 +59,7 @@ def _handle_event(self, event): __last_debug_message: ClassVar[str] = "" context: Context - _paused: Optional[Paused] + _paused: Paused | None """ If execution is currently paused, this attribute stores the paused coroutine and the command for which we are expecting a reply. @@ -70,7 +69,7 @@ def _handle_event(self, event): All events that have occurred since execution was paused. These will be replayed to ._child_layer once we resume. """ - debug: Optional[str] = None + debug: str | None = None """ Enable debug logging by assigning a prefix string for log messages. Different amounts of whitespace for different layers work well. @@ -242,7 +241,7 @@ def __continue(self, event: events.CommandCompleted): class NextLayer(Layer): - layer: Optional[Layer] + layer: Layer | None """The next layer. To be set by an addon.""" events: list[mevents.Event] diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 56d7189124..8fcbf37081 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -5,8 +5,6 @@ from functools import cached_property from logging import DEBUG from logging import WARNING -from typing import Optional -from typing import Union import wsproto.handshake @@ -71,7 +69,7 @@ class HTTPMode(enum.Enum): upstream = 3 -def validate_request(mode: HTTPMode, request: http.Request) -> Optional[str]: +def validate_request(mode: HTTPMode, request: http.Request) -> str | None: if request.scheme not in ("http", "https", ""): return f"Invalid request scheme: {request.scheme}" if mode is HTTPMode.transparent and request.method == "CONNECT": @@ -82,7 +80,7 @@ def validate_request(mode: HTTPMode, request: http.Request) -> Optional[str]: return None -def is_h3_alpn(alpn: Optional[bytes]) -> bool: +def is_h3_alpn(alpn: bytes | None) -> bool: return alpn == b"h3" or (alpn is not None and alpn.startswith(b"h3-")) @@ -95,7 +93,7 @@ class GetHttpConnection(HttpCommand): blocking = True address: tuple[str, int] tls: bool - via: Optional[server_spec.ServerSpec] + via: server_spec.ServerSpec | None transport_protocol: TransportProtocol = "tcp" def __hash__(self): @@ -114,7 +112,7 @@ def connection_spec_matches(self, connection: Connection) -> bool: @dataclass class GetHttpConnectionCompleted(events.CommandCompleted): command: GetHttpConnection - reply: Union[tuple[None, str], tuple[Connection, None]] + reply: tuple[None, str] | tuple[Connection, None] """connection object, error message""" @@ -125,7 +123,7 @@ class RegisterHttpConnection(HttpCommand): """ connection: Connection - err: Optional[str] + err: str | None @dataclass @@ -149,7 +147,7 @@ class HttpStream(layer.Layer): response_body_buf: bytes flow: http.HTTPFlow stream_id: StreamId - child_layer: Optional[layer.Layer] = None + child_layer: layer.Layer | None = None @cached_property def mode(self) -> HTTPMode: @@ -301,7 +299,7 @@ def start_request_stream(self) -> layer.CommandGenerator[None]: @expect(RequestData, RequestTrailers, RequestEndOfMessage) def state_stream_request_body( - self, event: Union[RequestData, RequestEndOfMessage] + self, event: RequestData | RequestEndOfMessage ) -> layer.CommandGenerator[None]: if isinstance(event, RequestData): if callable(self.flow.request.stream): @@ -554,7 +552,7 @@ def check_body_size(self, request: bool) -> layer.CommandGenerator[bool]: # Step 1: Determine the expected body size. This can either come from a known content-length header, # or from the amount of currently buffered bytes (e.g. for chunked encoding). response = not request - expected_size: Optional[int] + expected_size: int | None # the 'late' case: we already started consuming the body if request and self.request_body_buf: expected_size = len(self.request_body_buf) @@ -652,7 +650,7 @@ def check_killed(self, emit_error_hook: bool) -> layer.CommandGenerator[bool]: return False def handle_protocol_error( - self, event: Union[RequestProtocolError, ResponseProtocolError] + self, event: RequestProtocolError | ResponseProtocolError ) -> layer.CommandGenerator[None]: is_client_error_but_we_already_talk_upstream = ( isinstance(event, RequestProtocolError) @@ -930,7 +928,7 @@ def _handle_event(self, event: events.Event): def event_to_child( self, - child: Union[layer.Layer, HttpStream], + child: layer.Layer | HttpStream, event: events.Event, ) -> layer.CommandGenerator[None]: for command in child.handle_event(event): @@ -1068,7 +1066,7 @@ def register_connection( ) -> layer.CommandGenerator[None]: waiting = self.waiting_for_establishment.pop(command.connection) - reply: Union[tuple[None, str], tuple[Connection, None]] + reply: tuple[None, str] | tuple[Connection, None] if command.err: reply = (None, command.err) else: @@ -1098,7 +1096,7 @@ class HttpClient(layer.Layer): @expect(events.Start) def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: - err: Optional[str] + err: str | None if self.context.server.connected: err = None else: diff --git a/mitmproxy/proxy/layers/http/_events.py b/mitmproxy/proxy/layers/http/_events.py index ecdbcd7a29..d977135ff2 100644 --- a/mitmproxy/proxy/layers/http/_events.py +++ b/mitmproxy/proxy/layers/http/_events.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Optional from ._base import HttpEvent from mitmproxy import http @@ -15,7 +14,7 @@ class RequestHeaders(HttpEvent): us to set END_STREAM on headers already (and some servers - Akamai - implicitly expect that). In either case, this event will nonetheless be followed by RequestEndOfMessage. """ - replay_flow: Optional[HTTPFlow] = None + replay_flow: HTTPFlow | None = None """If set, the current request headers belong to a replayed flow, which should be reused.""" diff --git a/mitmproxy/proxy/layers/http/_http1.py b/mitmproxy/proxy/layers/http/_http1.py index 0affd6d68c..a4932b8cd8 100644 --- a/mitmproxy/proxy/layers/http/_http1.py +++ b/mitmproxy/proxy/layers/http/_http1.py @@ -1,6 +1,5 @@ import abc -from typing import Callable -from typing import Optional +from collections.abc import Callable from typing import Union import h11 @@ -39,19 +38,19 @@ class Http1Connection(HttpConnection, metaclass=abc.ABCMeta): - stream_id: Optional[StreamId] = None - request: Optional[http.Request] = None - response: Optional[http.Response] = None + stream_id: StreamId | None = None + request: http.Request | None = None + response: http.Response | None = None request_done: bool = False response_done: bool = False # this is a bit of a hack to make both mypy and PyCharm happy. - state: Union[Callable[[events.Event], layer.CommandGenerator[None]], Callable] + state: Callable[[events.Event], layer.CommandGenerator[None]] | Callable body_reader: TBodyReader buf: ReceiveBuffer - ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] - ReceiveData: type[Union[RequestData, ResponseData]] - ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] + ReceiveProtocolError: type[RequestProtocolError | ResponseProtocolError] + ReceiveData: type[RequestData | ResponseData] + ReceiveEndOfMessage: type[RequestEndOfMessage | ResponseEndOfMessage] def __init__(self, context: Context, conn: Connection): super().__init__(context, conn) @@ -464,7 +463,7 @@ def should_make_pipe(request: http.Request, response: http.Response) -> bool: return False -def make_body_reader(expected_size: Optional[int]) -> TBodyReader: +def make_body_reader(expected_size: int | None) -> TBodyReader: if expected_size is None: return ChunkedReader() elif expected_size == -1: diff --git a/mitmproxy/proxy/layers/http/_http2.py b/mitmproxy/proxy/layers/http/_http2.py index df32853a0a..a41dbf72f3 100644 --- a/mitmproxy/proxy/layers/http/_http2.py +++ b/mitmproxy/proxy/layers/http/_http2.py @@ -5,8 +5,6 @@ from logging import DEBUG from logging import ERROR from typing import ClassVar -from typing import Optional -from typing import Union import h2.config import h2.connection @@ -74,10 +72,10 @@ class Http2Connection(HttpConnection): streams: dict[int, StreamState] """keep track of all active stream ids to send protocol errors on teardown""" - ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] - ReceiveData: type[Union[RequestData, ResponseData]] - ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] - ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] + ReceiveProtocolError: type[RequestProtocolError | ResponseProtocolError] + ReceiveData: type[RequestData | ResponseData] + ReceiveTrailers: type[RequestTrailers | ResponseTrailers] + ReceiveEndOfMessage: type[RequestEndOfMessage | ResponseEndOfMessage] def __init__(self, context: Context, conn: Connection): super().__init__(context, conn) @@ -454,7 +452,7 @@ class Http2Client(Http2Connection): their_stream_id: dict[int, int] stream_queue: collections.defaultdict[int, list[Event]] """Queue of streams that we haven't sent yet because we have reached MAX_CONCURRENT_STREAMS""" - provisional_max_concurrency: Optional[int] = 10 + provisional_max_concurrency: int | None = 10 """A provisional currency limit before we get the server's first settings frame.""" last_activity: float """Timestamp of when we've last seen network activity on this connection.""" @@ -583,7 +581,7 @@ def handle_h2_event(self, event: h2.events.Event) -> CommandGenerator[bool]: # - 102 Processing is WebDAV only and also ignorable. # - 103 Early Hints is not mission-critical. headers = http.Headers(event.headers) - status: Union[str, int] = "" + status: str | int = "" try: status = int(headers[":status"]) reason = status_codes.RESPONSES.get(status, "") diff --git a/mitmproxy/proxy/layers/http/_http3.py b/mitmproxy/proxy/layers/http/_http3.py index ccafe36f4b..51b04f5bc6 100644 --- a/mitmproxy/proxy/layers/http/_http3.py +++ b/mitmproxy/proxy/layers/http/_http3.py @@ -1,6 +1,5 @@ import time from abc import abstractmethod -from typing import Union from aioquic.h3.connection import ErrorCode as H3ErrorCode from aioquic.h3.connection import FrameUnexpected as H3FrameUnexpected @@ -47,10 +46,10 @@ class Http3Connection(HttpConnection): h3_conn: LayeredH3Connection - ReceiveData: type[Union[RequestData, ResponseData]] - ReceiveEndOfMessage: type[Union[RequestEndOfMessage, ResponseEndOfMessage]] - ReceiveProtocolError: type[Union[RequestProtocolError, ResponseProtocolError]] - ReceiveTrailers: type[Union[RequestTrailers, ResponseTrailers]] + ReceiveData: type[RequestData | ResponseData] + ReceiveEndOfMessage: type[RequestEndOfMessage | ResponseEndOfMessage] + ReceiveProtocolError: type[RequestProtocolError | ResponseProtocolError] + ReceiveTrailers: type[RequestTrailers | ResponseTrailers] def __init__(self, context: context.Context, conn: connection.Connection): super().__init__(context, conn) @@ -205,9 +204,7 @@ def done(self, _) -> layer.CommandGenerator[None]: yield from () @abstractmethod - def parse_headers( - self, event: HeadersReceived - ) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers(self, event: HeadersReceived) -> RequestHeaders | ResponseHeaders: pass # pragma: no cover @@ -220,9 +217,7 @@ class Http3Server(Http3Connection): def __init__(self, context: context.Context): super().__init__(context, context.client) - def parse_headers( - self, event: HeadersReceived - ) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers(self, event: HeadersReceived) -> RequestHeaders | ResponseHeaders: # same as HTTP/2 ( host, @@ -281,9 +276,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: cmd.event.stream_id = self.their_stream_id[cmd.event.stream_id] yield cmd - def parse_headers( - self, event: HeadersReceived - ) -> Union[RequestHeaders, ResponseHeaders]: + def parse_headers(self, event: HeadersReceived) -> RequestHeaders | ResponseHeaders: # same as HTTP/2 status_code, headers = parse_h2_response_headers(event.headers) response = http.Response( diff --git a/mitmproxy/proxy/layers/http/_http_h3.py b/mitmproxy/proxy/layers/http/_http_h3.py index 52b36e57ee..1d1ea907d6 100644 --- a/mitmproxy/proxy/layers/http/_http_h3.py +++ b/mitmproxy/proxy/layers/http/_http_h3.py @@ -1,6 +1,5 @@ from collections.abc import Iterable from dataclasses import dataclass -from typing import Optional from aioquic.h3.connection import FrameUnexpected from aioquic.h3.connection import H3Connection @@ -41,7 +40,7 @@ class TrailersReceived(H3Event): stream_ended: bool "Whether the STREAM frame had the FIN bit set." - push_id: Optional[int] = None + push_id: int | None = None "The Push ID or `None` if this is not a push." @@ -57,7 +56,7 @@ class StreamReset(H3Event): error_code: int """The error code indicating why the stream was reset.""" - push_id: Optional[int] = None + push_id: int | None = None "The Push ID or `None` if this is not a push." @@ -81,7 +80,7 @@ def __init__(self, conn: connection.Connection, is_client: bool) -> None: def close( self, error_code: int = QuicErrorCode.NO_ERROR, - frame_type: Optional[int] = None, + frame_type: int | None = None, reason_phrase: str = "", ) -> None: # we'll get closed if a protocol error occurs in `H3Connection.handle_event` @@ -134,7 +133,7 @@ def _after_send(self, stream_id: int, end_stream: bool) -> None: def _handle_request_or_push_frame( self, frame_type: int, - frame_data: Optional[bytes], + frame_data: bytes | None, stream: H3Stream, stream_ended: bool, ) -> list[H3Event]: @@ -156,7 +155,7 @@ def _handle_request_or_push_frame( def close_connection( self, error_code: int = QuicErrorCode.NO_ERROR, - frame_type: Optional[int] = None, + frame_type: int | None = None, reason_phrase: str = "", ) -> None: """Closes the underlying QUIC connection and ignores any incoming events.""" @@ -177,7 +176,7 @@ def get_next_available_stream_id(self, is_unidirectional: bool = False): return self._quic.get_next_available_stream_id(is_unidirectional) - def get_open_stream_ids(self, push_id: Optional[int]) -> Iterable[int]: + def get_open_stream_ids(self, push_id: int | None) -> Iterable[int]: """Iterates over all non-special open streams, optionally for a given push id.""" return ( diff --git a/mitmproxy/proxy/layers/http/_upstream_proxy.py b/mitmproxy/proxy/layers/http/_upstream_proxy.py index 9034fe1450..3220cf9307 100644 --- a/mitmproxy/proxy/layers/http/_upstream_proxy.py +++ b/mitmproxy/proxy/layers/http/_upstream_proxy.py @@ -1,6 +1,5 @@ import time from logging import DEBUG -from typing import Optional from h11._receivebuffer import ReceiveBuffer @@ -74,7 +73,7 @@ def start_handshake(self) -> layer.CommandGenerator[None]: def receive_handshake_data( self, data: bytes - ) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: + ) -> layer.CommandGenerator[tuple[bool, str | None]]: if not self.send_connect: return (yield from super().receive_handshake_data(data)) self.buf += data diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 2dbda7a161..ba79c21356 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -3,9 +3,8 @@ import socket import struct from abc import ABCMeta +from collections.abc import Callable from dataclasses import dataclass -from typing import Callable -from typing import Optional from mitmproxy import connection from mitmproxy.proxy import commands @@ -39,7 +38,7 @@ class DestinationKnown(layer.Layer, metaclass=ABCMeta): child_layer: layer.Layer - def finish_start(self) -> layer.CommandGenerator[Optional[str]]: + def finish_start(self) -> layer.CommandGenerator[str | None]: if ( self.context.options.connection_strategy == "eager" and self.context.server.address @@ -134,7 +133,7 @@ class Socks5Proxy(DestinationKnown): def socks_err( self, message: str, - reply_code: Optional[int] = None, + reply_code: int | None = None, ) -> layer.CommandGenerator[None]: if reply_code is not None: yield commands.SendData( diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index 808cb584b6..d20924c769 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1,13 +1,13 @@ from __future__ import annotations import time +from collections.abc import Callable from dataclasses import dataclass from dataclasses import field from logging import DEBUG from logging import ERROR from logging import WARNING from ssl import VerifyMode -from typing import Callable from aioquic.buffer import Buffer as QuicBuffer from aioquic.h3.connection import ErrorCode as H3ErrorCode diff --git a/mitmproxy/proxy/layers/tcp.py b/mitmproxy/proxy/layers/tcp.py index 417009e415..f0dfc47a19 100644 --- a/mitmproxy/proxy/layers/tcp.py +++ b/mitmproxy/proxy/layers/tcp.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Optional from mitmproxy import flow from mitmproxy import tcp @@ -64,7 +63,7 @@ class TCPLayer(layer.Layer): Simple TCP layer that just relays messages right now. """ - flow: Optional[tcp.TCPFlow] + flow: tcp.TCPFlow | None def __init__(self, context: Context, ignore: bool = False): super().__init__(context) diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index bb74b8b496..5d58c0d0b4 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -6,7 +6,6 @@ from logging import ERROR from logging import INFO from logging import WARNING -from typing import Optional from OpenSSL import SSL @@ -63,7 +62,7 @@ def handshake_record_contents(data: bytes) -> Iterator[bytes]: offset += record_size -def get_client_hello(data: bytes) -> Optional[bytes]: +def get_client_hello(data: bytes) -> bytes | None: """ Read all TLS records that contain the initial ClientHello. Returns the raw handshake packet bytes, without TLS record headers. @@ -78,7 +77,7 @@ def get_client_hello(data: bytes) -> Optional[bytes]: return None -def parse_client_hello(data: bytes) -> Optional[ClientHello]: +def parse_client_hello(data: bytes) -> ClientHello | None: """ Check if the supplied bytes contain a full ClientHello message, and if so, parse it. @@ -136,7 +135,7 @@ def dtls_handshake_record_contents(data: bytes) -> Iterator[bytes]: offset += record_size -def get_dtls_client_hello(data: bytes) -> Optional[bytes]: +def get_dtls_client_hello(data: bytes) -> bytes | None: """ Read all DTLS records that contain the initial ClientHello. Returns the raw handshake packet bytes, without TLS record headers. @@ -154,7 +153,7 @@ def get_dtls_client_hello(data: bytes) -> Optional[bytes]: return None -def dtls_parse_client_hello(data: bytes) -> Optional[ClientHello]: +def dtls_parse_client_hello(data: bytes) -> ClientHello | None: """ Check if the supplied bytes contain a full ClientHello message, and if so, parse it. @@ -309,7 +308,7 @@ def tls_interact(self) -> layer.CommandGenerator[None]: def receive_handshake_data( self, data: bytes - ) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: + ) -> layer.CommandGenerator[tuple[bool, str | None]]: # bio_write errors for b"", so we need to check first if we actually received something. if data: self.tls.bio_write(data) @@ -466,9 +465,7 @@ class ServerTLSLayer(TLSLayer): wait_for_clienthello: bool = False - def __init__( - self, context: context.Context, conn: Optional[connection.Server] = None - ): + def __init__(self, context: context.Context, conn: connection.Server | None = None): super().__init__(context, conn or context.server) def start_handshake(self) -> layer.CommandGenerator[None]: @@ -558,7 +555,7 @@ def start_handshake(self) -> layer.CommandGenerator[None]: def receive_handshake_data( self, data: bytes - ) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: + ) -> layer.CommandGenerator[tuple[bool, str | None]]: if self.client_hello_parsed: return (yield from super().receive_handshake_data(data)) self.recv_buffer.extend(data) @@ -621,7 +618,7 @@ def receive_handshake_data( self.recv_buffer.clear() return ret - def start_server_tls(self) -> layer.CommandGenerator[Optional[str]]: + def start_server_tls(self) -> layer.CommandGenerator[str | None]: """ We often need information from the upstream connection to establish TLS with the client. For example, we need to check if the client does ALPN or not. diff --git a/mitmproxy/proxy/layers/udp.py b/mitmproxy/proxy/layers/udp.py index ac6643b9a5..fd7e65227f 100644 --- a/mitmproxy/proxy/layers/udp.py +++ b/mitmproxy/proxy/layers/udp.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from typing import Optional from mitmproxy import flow from mitmproxy import udp @@ -63,7 +62,7 @@ class UDPLayer(layer.Layer): Simple UDP layer that just relays messages right now. """ - flow: Optional[udp.UDPFlow] + flow: udp.UDPFlow | None def __init__(self, context: Context, ignore: bool = False): super().__init__(context) diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 7094ce31a9..feaab5f276 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -17,8 +17,6 @@ from collections.abc import MutableMapping from contextlib import contextmanager from dataclasses import dataclass -from typing import Optional -from typing import Union import mitmproxy_rs from OpenSSL import SSL @@ -91,13 +89,13 @@ def disarm(self): @dataclass class ConnectionIO: - handler: Optional[asyncio.Task] = None - reader: Optional[ - Union[asyncio.StreamReader, udp.DatagramReader, mitmproxy_rs.TcpStream] - ] = None - writer: Optional[ - Union[asyncio.StreamWriter, udp.DatagramWriter, mitmproxy_rs.TcpStream] - ] = None + handler: asyncio.Task | None = None + reader: None | ( + asyncio.StreamReader | udp.DatagramReader | mitmproxy_rs.TcpStream + ) = None + writer: None | ( + asyncio.StreamWriter | udp.DatagramWriter | mitmproxy_rs.TcpStream + ) = None class ConnectionHandler(metaclass=abc.ABCMeta): @@ -201,8 +199,8 @@ async def open_connection(self, command: commands.OpenConnection) -> None: return async with self.max_conns[command.connection.address]: - reader: Union[asyncio.StreamReader, udp.DatagramReader] - writer: Union[asyncio.StreamWriter, udp.DatagramWriter] + reader: asyncio.StreamReader | udp.DatagramReader + writer: asyncio.StreamWriter | udp.DatagramWriter try: command.connection.timestamp_start = time.time() if command.connection.transport_protocol == "tcp": @@ -434,8 +432,8 @@ def close_connection( class LiveConnectionHandler(ConnectionHandler, metaclass=abc.ABCMeta): def __init__( self, - reader: Union[asyncio.StreamReader, mitmproxy_rs.TcpStream], - writer: Union[asyncio.StreamWriter, mitmproxy_rs.TcpStream], + reader: asyncio.StreamReader | mitmproxy_rs.TcpStream, + writer: asyncio.StreamWriter | mitmproxy_rs.TcpStream, options: moptions.Options, mode: mode_specs.ProxyMode, ) -> None: diff --git a/mitmproxy/proxy/tunnel.py b/mitmproxy/proxy/tunnel.py index 5aa42cdedb..1ed1e73be5 100644 --- a/mitmproxy/proxy/tunnel.py +++ b/mitmproxy/proxy/tunnel.py @@ -1,7 +1,6 @@ import time from enum import auto from enum import Enum -from typing import Optional from typing import Union from mitmproxy import connection @@ -31,7 +30,7 @@ class TunnelLayer(layer.Layer): conn: connection.Connection """The 'inner' connection which provides data I/O""" tunnel_state: TunnelState = TunnelState.INACTIVE - command_to_reply_to: Optional[commands.OpenConnection] = None + command_to_reply_to: commands.OpenConnection | None = None _event_queue: list[events.Event] """ If the connection already exists when we receive the start event, @@ -98,7 +97,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: yield from self.event_to_child(event) - def _handshake_finished(self, err: Optional[str]): + def _handshake_finished(self, err: str | None): if err: self.tunnel_state = TunnelState.CLOSED else: @@ -159,7 +158,7 @@ def start_handshake(self) -> layer.CommandGenerator[None]: def receive_handshake_data( self, data: bytes - ) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: + ) -> layer.CommandGenerator[tuple[bool, str | None]]: """returns a (done, err) tuple""" yield from () return True, None diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index b632abc67e..84654deb7d 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -1,6 +1,4 @@ import uuid -from typing import Optional -from typing import Union from wsproto.frame_protocol import Opcode @@ -123,11 +121,11 @@ def twebsocketflow( def tdnsflow( *, - client_conn: Optional[connection.Client] = None, - server_conn: Optional[connection.Server] = None, - req: Optional[dns.Message] = None, - resp: Union[bool, dns.Message] = False, - err: Union[bool, flow.Error] = False, + client_conn: connection.Client | None = None, + server_conn: connection.Server | None = None, + req: dns.Message | None = None, + resp: bool | dns.Message = False, + err: bool | flow.Error = False, live: bool = True, ) -> dns.DNSFlow: """Create a DNS flow for testing.""" @@ -160,12 +158,12 @@ def tdnsflow( def tflow( *, - client_conn: Optional[connection.Client] = None, - server_conn: Optional[connection.Server] = None, - req: Optional[http.Request] = None, - resp: Union[bool, http.Response] = False, - err: Union[bool, flow.Error] = False, - ws: Union[bool, websocket.WebSocketData] = False, + client_conn: connection.Client | None = None, + server_conn: connection.Server | None = None, + req: http.Request | None = None, + resp: bool | http.Response = False, + err: bool | flow.Error = False, + ws: bool | websocket.WebSocketData = False, live: bool = True, ) -> http.HTTPFlow: """Create a flow for testing.""" diff --git a/mitmproxy/tls.py b/mitmproxy/tls.py index a8d93ffdab..de0aa340e6 100644 --- a/mitmproxy/tls.py +++ b/mitmproxy/tls.py @@ -1,6 +1,5 @@ import io from dataclasses import dataclass -from typing import Optional from kaitaistruct import KaitaiStream from OpenSSL import SSL @@ -68,7 +67,7 @@ def cipher_suites(self) -> list[int]: return self._client_hello.cipher_suites.cipher_suites @property - def sni(self) -> Optional[str]: + def sni(self) -> str | None: """ The [Server Name Indication](https://en.wikipedia.org/wiki/Server_Name_Indication), which indicates which hostname the client wants to connect to. @@ -142,7 +141,7 @@ class TlsData: """The affected connection.""" context: context.Context """The context object for this connection.""" - ssl_conn: Optional[SSL.Connection] = None + ssl_conn: SSL.Connection | None = None """ The associated pyOpenSSL `SSL.Connection` object. This will be set by an addon in the `tls_start_*` event hooks. diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index 9c2fcb3a81..11f2b67926 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -1,7 +1,6 @@ import abc from collections.abc import Sequence from typing import NamedTuple -from typing import Optional import urwid from urwid.text_layout import calc_coords @@ -54,7 +53,7 @@ def __init__(self, master: mitmproxy.master.Master, start: str = "") -> None: self.text = start # Cursor is always within the range [0:len(buffer)]. self._cursor = len(self.text) - self.completion: Optional[CompletionState] = None + self.completion: CompletionState | None = None @property def cursor(self) -> int: diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 9e2aaf2f81..d58cded802 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -3,8 +3,6 @@ import platform from collections.abc import Iterable from functools import lru_cache -from typing import Optional -from typing import Union import urwid.util from publicsuffix2 import get_sld @@ -48,7 +46,7 @@ def highlight_key(str, key, textattr="text", keyattr="key"): def format_keyvals( - entries: Iterable[tuple[str, Union[None, str, urwid.Widget]]], + entries: Iterable[tuple[str, None | str | urwid.Widget]], key_format: str = "key", value_format: str = "text", indent: int = 0, @@ -360,7 +358,7 @@ def format_size(num_bytes: int) -> tuple[str, str]: def format_left_indicators(*, focused: bool, intercepted: bool, timestamp: float): - indicators: list[Union[str, tuple[str, str]]] = [] + indicators: list[str | tuple[str, str]] = [] if focused: indicators.append(("focus", ">>")) else: @@ -378,7 +376,7 @@ def format_right_indicators( replay: bool, marked: str, ): - indicators: list[Union[str, tuple[str, str]]] = [] + indicators: list[str | tuple[str, str]] = [] if replay: indicators.append(("replay", SYMBOL_REPLAY)) else: @@ -406,12 +404,12 @@ def format_http_flow_list( request_timestamp: float, request_is_push_promise: bool, intercepted: bool, - response_code: Optional[int], - response_reason: Optional[str], - response_content_length: Optional[int], - response_content_type: Optional[str], - duration: Optional[float], - error_message: Optional[str], + response_code: int | None, + response_reason: str | None, + response_content_length: int | None, + response_content_type: str | None, + duration: float | None, + error_message: str | None, ) -> urwid.Widget: req = [] @@ -494,7 +492,7 @@ def format_http_flow_table( render_mode: RenderMode, focused: bool, marked: str, - is_replay: Optional[str], + is_replay: str | None, request_method: str, request_scheme: str, request_host: str, @@ -504,12 +502,12 @@ def format_http_flow_table( request_timestamp: float, request_is_push_promise: bool, intercepted: bool, - response_code: Optional[int], - response_reason: Optional[str], - response_content_length: Optional[int], - response_content_type: Optional[str], - duration: Optional[float], - error_message: Optional[str], + response_code: int | None, + response_reason: str | None, + response_content_length: int | None, + response_content_type: str | None, + duration: float | None, + error_message: str | None, ) -> urwid.Widget: items = [ format_left_indicators( @@ -617,8 +615,8 @@ def format_message_flow( client_address, server_address, total_size: int, - duration: Optional[float], - error_message: Optional[str], + duration: float | None, + error_message: str | None, ): conn = f"{human.format_address(client_address)} <-> {human.format_address(server_address)}" @@ -669,16 +667,16 @@ def format_dns_flow( focused: bool, intercepted: bool, marked: str, - is_replay: Optional[str], + is_replay: str | None, op_code: str, request_timestamp: float, domain: str, type: str, - response_code: Optional[str], + response_code: str | None, response_code_http_equiv: int, - answer: Optional[str], + answer: str | None, error_message: str, - duration: Optional[float], + duration: float | None, ): items = [] @@ -749,8 +747,8 @@ def format_flow( relevant for display and call the render with only that. This assures that rows are updated if the flow is changed. """ - duration: Optional[float] - error_message: Optional[str] + duration: float | None + error_message: str | None if f.error: error_message = f.error.msg else: @@ -783,7 +781,7 @@ def format_flow( elif isinstance(f, DNSFlow): if f.response: duration = f.response.timestamp - f.request.timestamp - response_code_str: Optional[str] = dns.response_codes.to_str( + response_code_str: str | None = dns.response_codes.to_str( f.response.response_code ) response_code_http_equiv = dns.response_codes.http_equiv_status_code( @@ -815,14 +813,14 @@ def format_flow( ) elif isinstance(f, HTTPFlow): intercepted = f.intercepted - response_content_length: Optional[int] + response_content_length: int | None if f.response: if f.response.raw_content is not None: response_content_length = len(f.response.raw_content) else: response_content_length = None - response_code: Optional[int] = f.response.status_code - response_reason: Optional[str] = f.response.reason + response_code: int | None = f.response.status_code + response_reason: str | None = f.response.reason response_content_type = f.response.headers.get("content-type") if f.response.timestamp_end: duration = max( diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 9b55ef2d76..b8b2745549 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -1,5 +1,3 @@ -from typing import Optional - import urwid import mitmproxy.flow @@ -27,8 +25,8 @@ def flowdetails(state, flow: mitmproxy.flow.Flow): sc = flow.server_conn cc = flow.client_conn - req: Optional[http.Request] - resp: Optional[http.Response] + req: http.Request | None + resp: http.Response | None if isinstance(flow, http.HTTPFlow): req = flow.request resp = flow.response diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index f0016577f1..e9723e24f4 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -1,5 +1,4 @@ from functools import lru_cache -from typing import Optional import urwid @@ -70,7 +69,7 @@ def set_focus(self, index): self.master.view.focus.index = index @lru_cache(maxsize=None) - def _get(self, pos: int) -> tuple[Optional[FlowItem], Optional[int]]: + def _get(self, pos: int) -> tuple[FlowItem | None, int | None]: if not self.master.view.inbounds(pos): return None, None return FlowItem(self.master, self.master.view[pos]), pos diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 9f2298728d..191edd8c88 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -2,7 +2,6 @@ import math import sys from functools import lru_cache -from typing import Optional import urwid @@ -417,7 +416,7 @@ def conn_text(self, conn): return searchable.Searchable(txt) def dns_message_text( - self, type: str, message: Optional[dns.Message] + self, type: str, message: dns.Message | None ) -> searchable.Searchable: # Keep in sync with web/src/js/components/FlowView/DnsMessages.tsx if message: diff --git a/mitmproxy/tools/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py index a59717487f..830f4bf7b5 100644 --- a/mitmproxy/tools/console/grideditor/base.py +++ b/mitmproxy/tools/console/grideditor/base.py @@ -9,7 +9,6 @@ from typing import Any from typing import AnyStr from typing import ClassVar -from typing import Optional import urwid @@ -65,21 +64,21 @@ def Edit(self, data) -> Cell: def blank(self) -> Any: pass - def keypress(self, key: str, editor: "GridEditor") -> Optional[str]: + def keypress(self, key: str, editor: "GridEditor") -> str | None: return key class GridRow(urwid.WidgetWrap): def __init__( self, - focused: Optional[int], + focused: int | None, editing: bool, editor: "GridEditor", values: tuple[Iterable[bytes], Container[int]], ) -> None: self.focused = focused self.editor = editor - self.edit_col: Optional[Cell] = None + self.edit_col: Cell | None = None errors = values[1] self.fields: Sequence[Any] = [] @@ -128,7 +127,7 @@ def __init__(self, lst: Iterable[list], editor: "GridEditor") -> None: self.editor = editor self.focus = 0 self.focus_col = 0 - self.edit_row: Optional[GridRow] = None + self.edit_row: GridRow | None = None def _modified(self): self.editor.show_empty_msg() @@ -360,7 +359,7 @@ def data_in(self, data: Any) -> Iterable[list]: """ return data - def is_error(self, col: int, val: Any) -> Optional[str]: + def is_error(self, col: int, val: Any) -> str | None: """ Return None, or a string error message. """ diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 6f4682e2a4..4e5eb401df 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -1,5 +1,4 @@ from typing import Any -from typing import Union import urwid @@ -191,11 +190,7 @@ class DataViewer(base.GridEditor, layoutwidget.LayoutWidget): def __init__( self, master, - vals: Union[ - list[list[Any]], - list[Any], - Any, - ], + vals: (list[list[Any]] | list[Any] | Any), ) -> None: if vals is not None: # Whatever vals is, make it a list of rows containing lists of column values. diff --git a/mitmproxy/tools/console/keymap.py b/mitmproxy/tools/console/keymap.py index 5cadd010d0..31640b6f80 100644 --- a/mitmproxy/tools/console/keymap.py +++ b/mitmproxy/tools/console/keymap.py @@ -2,7 +2,6 @@ import os from collections.abc import Sequence from functools import cache -from typing import Optional import ruamel.yaml.error @@ -136,13 +135,13 @@ def unbind(self, binding: Binding) -> None: self.bindings = [b for b in self.bindings if b != binding] self._on_change() - def get(self, context: str, key: str) -> Optional[Binding]: + def get(self, context: str, key: str) -> Binding | None: if context in self.keys: return self.keys[context].get(key, None) return None @cache - def binding_for_help(self, help: str) -> Optional[Binding]: + def binding_for_help(self, help: str) -> Binding | None: for b in self.bindings: if b.help == help: return b @@ -156,7 +155,7 @@ def list(self, context: str) -> Sequence[Binding]: multi.sort(key=lambda x: x.sortkey()) return single + multi - def handle(self, context: str, key: str) -> Optional[str]: + def handle(self, context: str, key: str) -> str | None: """ Returns the key if it has not been handled, or None. """ @@ -166,7 +165,7 @@ def handle(self, context: str, key: str) -> Optional[str]: return None return key - def handle_only(self, context: str, key: str) -> Optional[str]: + def handle_only(self, context: str, key: str) -> str | None: """ Like handle, but ignores global bindings. Returns the key if it has not been handled, or None. diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 415322be33..57266b11fb 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -7,7 +7,6 @@ from collections.abc import Mapping from collections.abc import Sequence -from typing import Optional class Palette: @@ -92,7 +91,7 @@ class Palette: "commander_hint", ] _fields.extend(["gradient_%02d" % i for i in range(100)]) - high: Optional[Mapping[str, Sequence[str]]] = None + high: Mapping[str, Sequence[str]] | None = None low: Mapping[str, Sequence[str]] def palette(self, transparent: bool): diff --git a/mitmproxy/tools/console/quickhelp.py b/mitmproxy/tools/console/quickhelp.py index 562587ac84..d6c959ad9a 100644 --- a/mitmproxy/tools/console/quickhelp.py +++ b/mitmproxy/tools/console/quickhelp.py @@ -2,7 +2,6 @@ This module is reponsible for drawing the quick key help at the bottom of mitmproxy. """ from dataclasses import dataclass -from typing import Optional from typing import Union import urwid @@ -51,7 +50,7 @@ def make_rows(self, keymap: Keymap) -> tuple[urwid.Columns, urwid.Columns]: def make( widget: type[urwid.Widget], - focused_flow: Optional[flow.Flow], + focused_flow: flow.Flow | None, is_root_widget: bool, ) -> QuickHelp: top_label = "" diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index a09950000a..4390fe0237 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -2,7 +2,6 @@ from collections.abc import Callable from functools import lru_cache -from typing import Optional import urwid @@ -99,9 +98,7 @@ def sig_prompt( self.bottom._w = urwid.Text("") self.prompting = callback - def sig_prompt_command( - self, partial: str = "", cursor: Optional[int] = None - ) -> None: + def sig_prompt_command(self, partial: str = "", cursor: int | None = None) -> None: signals.focus.send(section="footer") self.top._w = commander.CommandEdit( self.master, diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index dfcf944d9d..cc3c9ddeeb 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -9,7 +9,6 @@ from collections.abc import Callable from collections.abc import Sequence from typing import Any -from typing import Optional from typing import TypeVar from mitmproxy import exceptions @@ -128,14 +127,14 @@ def _sigterm(*_): return asyncio.run(main()) -def mitmproxy(args=None) -> Optional[int]: # pragma: no cover +def mitmproxy(args=None) -> int | None: # pragma: no cover from mitmproxy.tools import console run(console.master.ConsoleMaster, cmdline.mitmproxy, args) return None -def mitmdump(args=None) -> Optional[int]: # pragma: no cover +def mitmdump(args=None) -> int | None: # pragma: no cover from mitmproxy.tools import dump def extra(args): @@ -152,7 +151,7 @@ def extra(args): return None -def mitmweb(args=None) -> Optional[int]: # pragma: no cover +def mitmweb(args=None) -> int | None: # pragma: no cover from mitmproxy.tools import web run(web.master.WebMaster, cmdline.mitmweb, args) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 7eb71f72ff..f031d6787f 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -11,8 +11,6 @@ from io import BytesIO from itertools import islice from typing import ClassVar -from typing import Optional -from typing import Union import tornado.escape import tornado.web @@ -40,7 +38,7 @@ from mitmproxy.websocket import WebSocketMessage -def cert_to_json(certs: Sequence[certs.Cert]) -> Optional[dict]: +def cert_to_json(certs: Sequence[certs.Cert]) -> dict | None: if not certs: return None cert = certs[0] @@ -111,8 +109,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: f["error"] = flow.error.get_state() if isinstance(flow, HTTPFlow): - content_length: Optional[int] - content_hash: Optional[str] + content_length: int | None + content_hash: str | None if flow.request.raw_content is not None: content_length = len(flow.request.raw_content) @@ -201,7 +199,7 @@ class APIError(tornado.web.HTTPError): class RequestHandler(tornado.web.RequestHandler): application: Application - def write(self, chunk: Union[str, bytes, dict, list]): + def write(self, chunk: str | bytes | dict | list): # Writing arrays on the top level is ok nowadays. # http://flask.pocoo.org/docs/0.11/security/#json-security if isinstance(chunk, list): @@ -509,9 +507,9 @@ class FlowContentView(RequestHandler): def message_to_json( self, viewname: str, - message: Union[http.Message, TCPMessage, UDPMessage, WebSocketMessage], - flow: Union[HTTPFlow, TCPFlow, UDPFlow], - max_lines: Optional[int] = None, + message: http.Message | TCPMessage | UDPMessage | WebSocketMessage, + flow: HTTPFlow | TCPFlow | UDPFlow, + max_lines: int | None = None, ): description, lines, error = contentviews.get_message_content_view( viewname, message, flow diff --git a/mitmproxy/types.py b/mitmproxy/types.py index 8645811ace..27c51d569a 100644 --- a/mitmproxy/types.py +++ b/mitmproxy/types.py @@ -4,7 +4,6 @@ import re from collections.abc import Sequence from typing import Any -from typing import Optional from typing import TYPE_CHECKING from typing import Union @@ -474,7 +473,7 @@ def __init__(self, *types): for t in types: self.typemap[t.typ] = t() - def get(self, t: Optional[type], default=None) -> Optional[_BaseType]: + def get(self, t: type | None, default=None) -> _BaseType | None: if type(t) in self.typemap: return self.typemap[type(t)] return self.typemap.get(t, default) diff --git a/mitmproxy/utils/asyncio_utils.py b/mitmproxy/utils/asyncio_utils.py index 44ec46077f..95c01edd00 100644 --- a/mitmproxy/utils/asyncio_utils.py +++ b/mitmproxy/utils/asyncio_utils.py @@ -1,7 +1,6 @@ import asyncio import time from collections.abc import Coroutine -from typing import Optional from mitmproxy.utils import human @@ -10,7 +9,7 @@ def create_task( coro: Coroutine, *, name: str, - client: Optional[tuple] = None, + client: tuple | None = None, ) -> asyncio.Task: """ Like asyncio.create_task, but also store some debug info on the task object. @@ -24,7 +23,7 @@ def set_task_debug_info( task: asyncio.Task, *, name: str, - client: Optional[tuple] = None, + client: tuple | None = None, ) -> None: """Set debug info for an externally-spawned task.""" task.created = time.time() # type: ignore @@ -36,7 +35,7 @@ def set_task_debug_info( def set_current_task_debug_info( *, name: str, - client: Optional[tuple] = None, + client: tuple | None = None, ) -> None: """Set debug info for the current task.""" task = asyncio.current_task() diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index a417c84af4..8d2d72f647 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -2,7 +2,6 @@ import functools import ipaddress import time -from typing import Optional SIZE_UNITS = { "b": 1024**0, @@ -31,7 +30,7 @@ def pretty_size(size: int) -> str: @functools.lru_cache -def parse_size(s: Optional[str]) -> Optional[int]: +def parse_size(s: str | None) -> int | None: """ Parse a size with an optional k/m/... suffix. Invalid values raise a ValueError. For added convenience, passing `None` returns `None`. @@ -51,7 +50,7 @@ def parse_size(s: Optional[str]) -> Optional[int]: raise ValueError("Invalid size specification.") -def pretty_duration(secs: Optional[float]) -> str: +def pretty_duration(secs: float | None) -> str: formatters = [ (100, "{:.0f}s"), (10, "{:2.1f}s"), @@ -79,7 +78,7 @@ def format_timestamp_with_milli(s): @functools.lru_cache -def format_address(address: Optional[tuple]) -> str: +def format_address(address: tuple | None) -> str: """ This function accepts IPv4/IPv6 tuples and returns the formatted address string with port number diff --git a/mitmproxy/utils/sliding_window.py b/mitmproxy/utils/sliding_window.py index a2cbb300db..6c3853db88 100644 --- a/mitmproxy/utils/sliding_window.py +++ b/mitmproxy/utils/sliding_window.py @@ -1,7 +1,6 @@ import itertools from collections.abc import Iterable from collections.abc import Iterator -from typing import Optional from typing import TypeVar T = TypeVar("T") @@ -9,7 +8,7 @@ def window( iterator: Iterable[T], behind: int = 0, ahead: int = 0 -) -> Iterator[tuple[Optional[T], ...]]: +) -> Iterator[tuple[T | None, ...]]: """ Sliding window for an iterator. @@ -23,9 +22,7 @@ def window( 2 3 None """ # TODO: move into utils - iters: list[Iterator[Optional[T]]] = list( - itertools.tee(iterator, behind + 1 + ahead) - ) + iters: list[Iterator[T | None]] = list(itertools.tee(iterator, behind + 1 + ahead)) for i in range(behind): iters[i] = itertools.chain((behind - i) * [None], iters[i]) for i in range(ahead): diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 0e2c1b2072..d3bdc9852e 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -3,7 +3,6 @@ import re from collections.abc import Iterable from typing import overload -from typing import Union # https://mypy.readthedocs.io/en/stable/more_types.html#function-overloading @@ -15,13 +14,11 @@ def always_bytes(str_or_bytes: None, *encode_args) -> None: @overload -def always_bytes(str_or_bytes: Union[str, bytes], *encode_args) -> bytes: +def always_bytes(str_or_bytes: str | bytes, *encode_args) -> bytes: ... -def always_bytes( - str_or_bytes: Union[None, str, bytes], *encode_args -) -> Union[None, bytes]: +def always_bytes(str_or_bytes: None | str | bytes, *encode_args) -> None | bytes: if str_or_bytes is None or isinstance(str_or_bytes, bytes): return str_or_bytes elif isinstance(str_or_bytes, str): @@ -38,11 +35,11 @@ def always_str(str_or_bytes: None, *encode_args) -> None: @overload -def always_str(str_or_bytes: Union[str, bytes], *encode_args) -> str: +def always_str(str_or_bytes: str | bytes, *encode_args) -> str: ... -def always_str(str_or_bytes: Union[None, str, bytes], *decode_args) -> Union[None, str]: +def always_str(str_or_bytes: None | str | bytes, *decode_args) -> None | str: """ Returns, str_or_bytes unmodified, if diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 657f2f6cd9..7424e12696 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -9,8 +9,6 @@ import warnings from dataclasses import dataclass from dataclasses import field -from typing import Optional -from typing import Union from wsproto.frame_protocol import Opcode @@ -54,10 +52,10 @@ class WebSocketMessage(serializable.Serializable): def __init__( self, - type: Union[int, Opcode], + type: int | Opcode, from_client: bool, content: bytes, - timestamp: Optional[float] = None, + timestamp: float | None = None, dropped: bool = False, injected: bool = False, ) -> None: @@ -156,18 +154,18 @@ class WebSocketData(serializable.SerializableDataclass): messages: list[WebSocketMessage] = field(default_factory=list) """All `WebSocketMessage`s transferred over this connection.""" - closed_by_client: Optional[bool] = None + closed_by_client: bool | None = None """ `True` if the client closed the connection, `False` if the server closed the connection, `None` if the connection is active. """ - close_code: Optional[int] = None + close_code: int | None = None """[Close Code](https://tools.ietf.org/html/rfc6455#section-7.1.5)""" - close_reason: Optional[str] = None + close_reason: str | None = None """[Close Reason](https://tools.ietf.org/html/rfc6455#section-7.1.6)""" - timestamp_end: Optional[float] = None + timestamp_end: float | None = None """*Timestamp:* WebSocket connection closed.""" def __repr__(self): diff --git a/release/build-and-deploy-docker.py b/release/build-and-deploy-docker.py index f407834685..3cf92881dd 100644 --- a/release/build-and-deploy-docker.py +++ b/release/build-and-deploy-docker.py @@ -7,15 +7,14 @@ import shutil import subprocess from pathlib import Path -from typing import Optional # Security: No third-party dependencies here! root = Path(__file__).absolute().parent.parent ref = os.environ["GITHUB_REF"] -branch: Optional[str] = None -tag: Optional[str] = None +branch: str | None = None +tag: str | None = None if ref.startswith("refs/heads/"): branch = ref.replace("refs/heads/", "") elif ref.startswith("refs/tags/"): diff --git a/release/deploy.py b/release/deploy.py index f0774edecd..cd7d41bf41 100755 --- a/release/deploy.py +++ b/release/deploy.py @@ -2,7 +2,6 @@ import os import subprocess from pathlib import Path -from typing import Optional # Security: No third-party dependencies here! @@ -10,8 +9,8 @@ if __name__ == "__main__": ref = os.environ["GITHUB_REF"] - branch: Optional[str] = None - tag: Optional[str] = None + branch: str | None = None + tag: str | None = None if ref.startswith("refs/heads/"): branch = ref.replace("refs/heads/", "") elif ref.startswith("refs/tags/"): diff --git a/test/mitmproxy/addons/test_dns_resolver.py b/test/mitmproxy/addons/test_dns_resolver.py index 0937c97a52..a72d5ed257 100644 --- a/test/mitmproxy/addons/test_dns_resolver.py +++ b/test/mitmproxy/addons/test_dns_resolver.py @@ -1,7 +1,7 @@ import asyncio import ipaddress import socket -from typing import Callable +from collections.abc import Callable import pytest diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index 6004c8770e..8f6f382f28 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -1,4 +1,3 @@ -from typing import Optional from unittest.mock import MagicMock import pytest @@ -199,13 +198,13 @@ def test_next_layer_udp( client_layer: layer.Layer, server_layer: layer.Layer, ): - def is_ignored_udp(layer: Optional[layer.Layer]): + def is_ignored_udp(layer: layer.Layer | None): return isinstance(layer, layers.UDPLayer) and layer.flow is None - def is_intercepted_udp(layer: Optional[layer.Layer]): + def is_intercepted_udp(layer: layer.Layer | None): return isinstance(layer, layers.UDPLayer) and layer.flow is not None - def is_http(layer: Optional[layer.Layer], mode: HTTPMode): + def is_http(layer: layer.Layer | None, mode: HTTPMode): return isinstance(layer, layers.HttpLayer) and layer.mode is mode client_hello = { diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 40f01c501c..beca215e4c 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -4,12 +4,11 @@ import socket import ssl from collections.abc import AsyncGenerator +from collections.abc import Callable from contextlib import asynccontextmanager from dataclasses import dataclass from typing import Any -from typing import Callable from typing import ClassVar -from typing import Optional from typing import TypeVar from unittest.mock import Mock @@ -391,7 +390,7 @@ class H3EchoServer(QuicConnectionProtocol): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._seen_headers: set[int] = set() - self.http: Optional[H3Connection] = None + self.http: H3Connection | None = None def http_headers_received(self, event: h3_events.HeadersReceived) -> None: assert event.push_id is None @@ -498,7 +497,7 @@ def quic_event_received(self, event: quic_events.QuicEvent) -> None: elif isinstance(event, quic_events.HandshakeCompleted): self._waiter.set_result(None) - def connection_lost(self, exc: Optional[Exception]) -> None: + def connection_lost(self, exc: Exception | None) -> None: if not self._waiter.done(): self._waiter.set_exception(exc) return super().connection_lost(exc) @@ -536,10 +535,10 @@ async def recv_datagram(self) -> bytes: class H3Response: waiter: asyncio.Future[H3Response] stream_id: int - headers: Optional[h3_events.H3Event] = None - data: Optional[bytes] = None - trailers: Optional[h3_events.H3Event] = None - callback: Optional[Callable[[str], None]] = None + headers: h3_events.H3Event | None = None + data: bytes | None = None + trailers: h3_events.H3Event | None = None + callback: Callable[[str], None] | None = None async def wait_result(self) -> H3Response: return await asyncio.wait_for(self.waiter, timeout=QuicClient.TIMEOUT) @@ -607,8 +606,8 @@ def quic_event_received(self, event: quic_events.QuicEvent) -> None: def request( self, headers: h3_events.H3Event, - data: Optional[bytes] = None, - trailers: Optional[h3_events.H3Event] = None, + data: bytes | None = None, + trailers: h3_events.H3Event | None = None, end_stream: bool = True, ) -> H3Response: stream_id = self._quic.get_next_available_stream_id() diff --git a/test/mitmproxy/addons/test_tlsconfig.py b/test/mitmproxy/addons/test_tlsconfig.py index 1da747d04c..7f9a5b6223 100644 --- a/test/mitmproxy/addons/test_tlsconfig.py +++ b/test/mitmproxy/addons/test_tlsconfig.py @@ -1,7 +1,6 @@ import ssl import time from pathlib import Path -from typing import Union import pytest from cryptography import x509 @@ -154,8 +153,8 @@ def test_tls_clienthello(self): def do_handshake( self, - tssl_client: Union[test_tls.SSLTest, SSL.Connection], - tssl_server: Union[test_tls.SSLTest, SSL.Connection], + tssl_client: test_tls.SSLTest | SSL.Connection, + tssl_server: test_tls.SSLTest | SSL.Connection, ) -> bool: # ClientHello with pytest.raises((ssl.SSLWantReadError, SSL.WantReadError)): diff --git a/test/mitmproxy/coretypes/test_serializable.py b/test/mitmproxy/coretypes/test_serializable.py index 06e10fc547..e46bb153eb 100644 --- a/test/mitmproxy/coretypes/test_serializable.py +++ b/test/mitmproxy/coretypes/test_serializable.py @@ -6,7 +6,6 @@ from collections.abc import Mapping from dataclasses import dataclass from typing import Literal -from typing import Optional import pytest @@ -50,13 +49,13 @@ def test_copy_id(self): @dataclass class Simple(SerializableDataclass): x: int - y: Optional[str] + y: str | None @dataclass class SerializableChild(SerializableDataclass): foo: Simple - maybe_foo: Optional[Simple] + maybe_foo: Simple | None @dataclass @@ -76,16 +75,16 @@ class TLiteral(SerializableDataclass): @dataclass class BuiltinChildren(SerializableDataclass): - a: Optional[list[int]] - b: Optional[dict[str, int]] - c: Optional[tuple[int, int]] + a: list[int] | None + b: dict[str, int] | None + c: tuple[int, int] | None d: list[Simple] - e: Optional[TEnum] + e: TEnum | None @dataclass class Defaults(SerializableDataclass): - z: Optional[int] = 42 + z: int | None = 42 @dataclass diff --git a/test/mitmproxy/net/test_udp.py b/test/mitmproxy/net/test_udp.py index d52636529c..70a6cd8876 100644 --- a/test/mitmproxy/net/test_udp.py +++ b/test/mitmproxy/net/test_udp.py @@ -1,5 +1,4 @@ import asyncio -from typing import Optional import pytest @@ -13,7 +12,7 @@ async def test_client_server(): server_reader = DatagramReader() - server_writer: Optional[DatagramWriter] = None + server_writer: DatagramWriter | None = None def handle_datagram( transport: asyncio.DatagramTransport, diff --git a/test/mitmproxy/proxy/layers/http/test_http3.py b/test/mitmproxy/proxy/layers/http/test_http3.py index 59ffb670a2..34456455e7 100644 --- a/test/mitmproxy/proxy/layers/http/test_http3.py +++ b/test/mitmproxy/proxy/layers/http/test_http3.py @@ -1,7 +1,6 @@ import collections.abc +from collections.abc import Callable from collections.abc import Iterable -from typing import Callable -from typing import Optional import pylsqpack import pytest @@ -110,10 +109,10 @@ def __init__(self, conn: connection.Connection, is_client: bool) -> None: ) self.decoder_placeholders: list[tutils.Placeholder[bytes]] = [] self.encoder = pylsqpack.Encoder() - self.encoder_placeholder: Optional[tutils.Placeholder[bytes]] = None + self.encoder_placeholder: tutils.Placeholder[bytes] | None = None self.peer_stream_id: dict[StreamType, int] = {} self.local_stream_id: dict[StreamType, int] = {} - self.max_push_id: Optional[int] = None + self.max_push_id: int | None = None def get_default_stream_id(self, stream_type: StreamType, for_local: bool) -> int: if stream_type == StreamType.CONTROL: @@ -131,7 +130,7 @@ def get_default_stream_id(self, stream_type: StreamType, for_local: bool) -> int def send_stream_type( self, stream_type: StreamType, - stream_id: Optional[int] = None, + stream_id: int | None = None, ) -> quic.SendQuicStreamData: assert stream_type not in self.peer_stream_id if stream_id is None: @@ -147,7 +146,7 @@ def send_stream_type( def receive_stream_type( self, stream_type: StreamType, - stream_id: Optional[int] = None, + stream_id: int | None = None, ) -> quic.QuicStreamDataReceived: assert stream_type not in self.local_stream_id if stream_id is None: diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index d635a3385d..e68222ff8d 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -4,7 +4,6 @@ from logging import ERROR from logging import WARNING from typing import Literal -from typing import Optional from typing import TypeVar from unittest.mock import MagicMock @@ -40,7 +39,7 @@ class DummyLayer(layer.Layer): - child_layer: Optional[layer.Layer] + child_layer: layer.Layer | None def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert self.child_layer @@ -48,8 +47,8 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: class TlsEchoLayer(tutils.EchoLayer): - err: Optional[str] = None - closed: Optional[quic.QuicConnectionClosed] = None + err: str | None = None + closed: quic.QuicConnectionClosed | None = None def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.DataReceived) and event.data == b"open-connection": @@ -452,7 +451,7 @@ def get_timer(self): def make_mock_quic( tctx: context.Context, - event: Optional[quic_events.QuicEvent] = None, + event: quic_events.QuicEvent | None = None, established: bool = True, ) -> tuple[tutils.Playbook, MockQuic]: tctx.client.state = connection.ConnectionState.CLOSED @@ -554,10 +553,10 @@ class SSLTest: def __init__( self, server_side: bool = False, - alpn: Optional[list[str]] = None, - sni: Optional[str] = "example.mitmproxy.org", - version: Optional[int] = None, - settings: Optional[quic.QuicTlsSettings] = None, + alpn: list[str] | None = None, + sni: str | None = "example.mitmproxy.org", + version: int | None = None, + settings: quic.QuicTlsSettings | None = None, ): if settings is None: self.ctx = QuicConfiguration( @@ -663,7 +662,7 @@ def finish_handshake( tssl: SSLTest, child_layer: type[T], ) -> T: - result: Optional[T] = None + result: T | None = None def set_layer(next_layer: layer.NextLayer) -> None: nonlocal result @@ -692,7 +691,7 @@ def set_layer(next_layer: layer.NextLayer) -> None: return result -def reply_tls_start_client(alpn: Optional[str] = None, *args, **kwargs) -> tutils.reply: +def reply_tls_start_client(alpn: str | None = None, *args, **kwargs) -> tutils.reply: """ Helper function to simplify the syntax for quic_start_client hooks. """ @@ -714,7 +713,7 @@ def make_client_conn(tls_start: quic.QuicTlsData) -> None: return tutils.reply(*args, side_effect=make_client_conn, **kwargs) -def reply_tls_start_server(alpn: Optional[str] = None, *args, **kwargs) -> tutils.reply: +def reply_tls_start_server(alpn: str | None = None, *args, **kwargs) -> tutils.reply: """ Helper function to simplify the syntax for quic_start_server hooks. """ diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 1fde306fa1..d934604dec 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -2,7 +2,6 @@ import time from logging import DEBUG from logging import WARNING -from typing import Optional import pytest from OpenSSL import SSL @@ -99,9 +98,9 @@ class SSLTest: def __init__( self, server_side: bool = False, - alpn: Optional[list[str]] = None, - sni: Optional[bytes] = b"example.mitmproxy.org", - max_ver: Optional[ssl.TLSVersion] = None, + alpn: list[str] | None = None, + sni: bytes | None = b"example.mitmproxy.org", + max_ver: ssl.TLSVersion | None = None, ): self.inc = ssl.MemoryBIO() self.out = ssl.MemoryBIO() @@ -164,7 +163,7 @@ def _test_echo( class TlsEchoLayer(tutils.EchoLayer): - err: Optional[str] = None + err: str | None = None def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: if isinstance(event, events.DataReceived) and event.data == b"open-connection": @@ -197,9 +196,7 @@ def finish_handshake( tssl.bio_write(data()) -def reply_tls_start_client( - alpn: Optional[bytes] = None, *args, **kwargs -) -> tutils.reply: +def reply_tls_start_client(alpn: bytes | None = None, *args, **kwargs) -> tutils.reply: """ Helper function to simplify the syntax for tls_start_client hooks. """ @@ -226,9 +223,7 @@ def make_client_conn(tls_start: TlsData) -> None: return tutils.reply(*args, side_effect=make_client_conn, **kwargs) -def reply_tls_start_server( - alpn: Optional[bytes] = None, *args, **kwargs -) -> tutils.reply: +def reply_tls_start_server(alpn: bytes | None = None, *args, **kwargs) -> tutils.reply: """ Helper function to simplify the syntax for tls_start_server hooks. """ diff --git a/test/mitmproxy/proxy/test_tunnel.py b/test/mitmproxy/proxy/test_tunnel.py index 0256390cf3..8f5ddf4f6e 100644 --- a/test/mitmproxy/proxy/test_tunnel.py +++ b/test/mitmproxy/proxy/test_tunnel.py @@ -1,5 +1,3 @@ -from typing import Optional - import pytest from mitmproxy.connection import ConnectionState @@ -21,7 +19,7 @@ class TChildLayer(layer.Layer): - child_layer: Optional[layer.Layer] = None + child_layer: layer.Layer | None = None def _handle_event(self, event: Event) -> layer.CommandGenerator[None]: if isinstance(event, Start): @@ -48,7 +46,7 @@ def start_handshake(self) -> layer.CommandGenerator[None]: def receive_handshake_data( self, data: bytes - ) -> layer.CommandGenerator[tuple[bool, Optional[str]]]: + ) -> layer.CommandGenerator[tuple[bool, str | None]]: yield SendData(self.tunnel_connection, data) if data == b"handshake-success": return True, None diff --git a/test/mitmproxy/proxy/tutils.py b/test/mitmproxy/proxy/tutils.py index 618f6da0b2..22543e397d 100644 --- a/test/mitmproxy/proxy/tutils.py +++ b/test/mitmproxy/proxy/tutils.py @@ -10,7 +10,6 @@ from typing import Any from typing import AnyStr from typing import Generic -from typing import Optional from typing import TypeVar from typing import Union @@ -57,8 +56,8 @@ def _eq(a: PlaybookEntry, b: PlaybookEntry) -> bool: def eq( - a: Union[PlaybookEntry, Iterable[PlaybookEntry]], - b: Union[PlaybookEntry, Iterable[PlaybookEntry]], + a: PlaybookEntry | Iterable[PlaybookEntry], + b: PlaybookEntry | Iterable[PlaybookEntry], ): """ Compare an indiviual event/command or a list of events/commands. @@ -146,7 +145,7 @@ def __init__( layer: Layer, hooks: bool = True, logs: bool = False, - expected: Optional[PlaybookEntryList] = None, + expected: PlaybookEntryList | None = None, ): if expected is None: expected = [events.Start()] @@ -299,13 +298,13 @@ def __del__(self): class reply(events.Event): args: tuple[Any, ...] - to: Union[commands.Command, int] + to: commands.Command | int side_effect: Callable[[Any], Any] def __init__( self, *args, - to: Union[commands.Command, int] = -1, + to: commands.Command | int = -1, side_effect: Callable[[Any], None] = lambda x: None, ): """Utility method to reply to the latest hook in playbooks.""" @@ -387,7 +386,7 @@ def __str__(self): # noinspection PyPep8Naming -def Placeholder(cls: type[T] = Any) -> Union[T, _Placeholder[T]]: +def Placeholder(cls: type[T] = Any) -> T | _Placeholder[T]: return _Placeholder(cls) @@ -405,12 +404,12 @@ def setdefault(self, value: AnyStr) -> AnyStr: # noinspection PyPep8Naming -def BytesMatching(match: bytes) -> Union[bytes, _AnyStrPlaceholder[bytes]]: +def BytesMatching(match: bytes) -> bytes | _AnyStrPlaceholder[bytes]: return _AnyStrPlaceholder(match) # noinspection PyPep8Naming -def StrMatching(match: str) -> Union[str, _AnyStrPlaceholder[str]]: +def StrMatching(match: str) -> str | _AnyStrPlaceholder[str]: return _AnyStrPlaceholder(match) @@ -439,7 +438,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: def reply_next_layer( - child_layer: Union[type[Layer], Callable[[context.Context], Layer]], *args, **kwargs + child_layer: type[Layer] | Callable[[context.Context], Layer], *args, **kwargs ) -> reply: """Helper function to simplify the syntax for next_layer events to this: << NextLayerHook(nl) From d9bdaa060aff1c501bf1d94c429bc21d477f8ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20D=C4=85bek?= <373530+szemek@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:42:47 +0100 Subject: [PATCH 194/695] Update links to mitmweb issue (#5952) Replace: https://github.com/mitmproxy/mitmweb/issues/10 with: https://github.com/mitmproxy/mitmproxy/issues/3609 --- web/src/js/filt/filt.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/src/js/filt/filt.js b/web/src/js/filt/filt.js index 6eca0be51a..fc426241dc 100644 --- a/web/src/js/filt/filt.js +++ b/web/src/js/filt/filt.js @@ -2084,7 +2084,7 @@ export default (function() { function bodyFilter(flow){ return true; } - bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; + bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmproxy/issues/3609"; return bodyFilter; } @@ -2094,7 +2094,7 @@ export default (function() { function requestBodyFilter(flow){ return true; } - requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; + requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmproxy/issues/3609"; return requestBodyFilter; } @@ -2104,7 +2104,7 @@ export default (function() { function responseBodyFilter(flow){ return true; } - responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; + responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmproxy/issues/3609"; return responseBodyFilter; } From 7b3e0a693921454d1c613e0c7e9ea8f3d6cf6bdb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:49:02 +0100 Subject: [PATCH 195/695] Update zstandard requirement from <0.20,>=0.11 to >=0.11,<0.21 (#5964) Updates the requirements on [zstandard](https://github.com/indygreg/python-zstandard) to permit the latest version. - [Release notes](https://github.com/indygreg/python-zstandard/releases) - [Changelog](https://github.com/indygreg/python-zstandard/blob/main/docs/news.rst) - [Commits](https://github.com/indygreg/python-zstandard/compare/0.11.0...0.20.0) --- updated-dependencies: - dependency-name: zstandard dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1c0f7c4330..aacb7b3560 100644 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ "urwid-mitmproxy>=2.1.1,<2.2", "wsproto>=1.0,<1.3", "publicsuffix2>=2.20190812,<3", - "zstandard>=0.11,<0.20", + "zstandard>=0.11,<0.21", "typing-extensions>=4.3,<4.5; python_version<'3.10'", ], extras_require={ From 37ac37598bc05aa870a34a25bcc050f432cf9607 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:49:15 +0100 Subject: [PATCH 196/695] Update pytest-xdist requirement from <3.2,>=2.1.0 to >=2.1.0,<3.3 (#5963) Updates the requirements on [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v2.1.0...v3.2.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aacb7b3560..5d20f45e69 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ "pytest-asyncio>=0.17,<0.21", "pytest-cov>=2.7.1,<4.1", "pytest-timeout>=1.3.3,<2.2", - "pytest-xdist>=2.1.0,<3.2", + "pytest-xdist>=2.1.0,<3.3", "pytest>=6.1.0,<8", "requests>=2.9.1,<3", "tox>=3.5,<5", From 80fb8f99d2a0b9791ea6a94e8480eef4dca5f578 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:49:24 +0100 Subject: [PATCH 197/695] Bump pyinstaller from 5.7.0 to 5.8.0 (#5962) Bumps [pyinstaller](https://github.com/pyinstaller/pyinstaller) from 5.7.0 to 5.8.0. - [Release notes](https://github.com/pyinstaller/pyinstaller/releases) - [Changelog](https://github.com/pyinstaller/pyinstaller/blob/develop/doc/CHANGES.rst) - [Commits](https://github.com/pyinstaller/pyinstaller/compare/v5.7.0...v5.8.0) --- updated-dependencies: - dependency-name: pyinstaller dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5d20f45e69..0859da179e 100644 --- a/setup.py +++ b/setup.py @@ -109,7 +109,7 @@ "hypothesis>=5.8,<7", "parver>=0.1,<2.0", "pdoc>=4.0.0", - "pyinstaller==5.7.0", + "pyinstaller==5.8.0", "pytest-asyncio>=0.17,<0.21", "pytest-cov>=2.7.1,<4.1", "pytest-timeout>=1.3.3,<2.2", From 098cf89187cf15b372894152b29708fc6278c556 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:49:33 +0100 Subject: [PATCH 198/695] Bump install-pinned/reorder_python_imports (#5961) Bumps [install-pinned/reorder_python_imports](https://github.com/install-pinned/reorder_python_imports) from 2cc264e0f6bc33907796602661e5b26d8199314d to 9766e7ba4f33497b107014571afe3b5f4c2d946b. - [Release notes](https://github.com/install-pinned/reorder_python_imports/releases) - [Commits](https://github.com/install-pinned/reorder_python_imports/compare/2cc264e0f6bc33907796602661e5b26d8199314d...9766e7ba4f33497b107014571afe3b5f4c2d946b) --- updated-dependencies: - dependency-name: install-pinned/reorder_python_imports dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 2e5f5316a0..8ed0051662 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -22,7 +22,7 @@ jobs: export GLOBIGNORE='mitmproxy/contrib/**' pyupgrade --exit-zero-even-if-changed --keep-runtime-typing --py310-plus **/*.py - - uses: install-pinned/reorder_python_imports@2cc264e0f6bc33907796602661e5b26d8199314d + - uses: install-pinned/reorder_python_imports@9766e7ba4f33497b107014571afe3b5f4c2d946b - name: Run reorder-python-imports run: | shopt -s globstar From 88e0d61093dd54dea133394075d782e54d413a01 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:49:41 +0100 Subject: [PATCH 199/695] Bump docker/setup-buildx-action (#5960) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 11e8a2e2910826a92412015c515187a2d6750279 to a19c1710881d7ce3cf668865cc8459bba5b912aa. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/11e8a2e2910826a92412015c515187a2d6750279...a19c1710881d7ce3cf668865cc8459bba5b912aa) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 514d62ee8f..19b21ab5f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -209,7 +209,7 @@ jobs: name: binaries.linux path: release/dist - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 - - uses: docker/setup-buildx-action@11e8a2e2910826a92412015c515187a2d6750279 # v1.6.0 + - uses: docker/setup-buildx-action@a19c1710881d7ce3cf668865cc8459bba5b912aa # v1.6.0 - run: python release/build-and-deploy-docker.py deploy: From 3e04379eafcaf86df00fe0b6aedfcc69b76bfd6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:49:53 +0100 Subject: [PATCH 200/695] Bump install-pinned/black (#5959) Bumps [install-pinned/black](https://github.com/install-pinned/black) from 3375665f712256be11c3212db472c3dafc217fa1 to eb57934f28e8d533bbcb4caa88d00b613b6ddd00. - [Release notes](https://github.com/install-pinned/black/releases) - [Commits](https://github.com/install-pinned/black/compare/3375665f712256be11c3212db472c3dafc217fa1...eb57934f28e8d533bbcb4caa88d00b613b6ddd00) --- updated-dependencies: - dependency-name: install-pinned/black dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 8ed0051662..8dca853d16 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -39,7 +39,7 @@ jobs: - uses: install-pinned/autoflake@dfa39c5f136f5b885c175734a719dc6ad1f11fc7 - run: autoflake --in-place --remove-all-unused-imports --exclude mitmproxy/contrib -r . - - uses: install-pinned/black@3375665f712256be11c3212db472c3dafc217fa1 + - uses: install-pinned/black@eb57934f28e8d533bbcb4caa88d00b613b6ddd00 - run: black --extend-exclude mitmproxy/contrib . - uses: mhils/add-pr-ref-in-changelog@main From 71d35e18fee8bc7ea3038e0a7ebddd0517079028 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:50:01 +0100 Subject: [PATCH 201/695] Bump install-pinned/yesqa (#5958) Bumps [install-pinned/yesqa](https://github.com/install-pinned/yesqa) from 4896f663e9c294fddfbf5f4e4fc4f9b1a4556658 to e45b0928dd14d5c2a22695e32de8936530ba7a49. - [Release notes](https://github.com/install-pinned/yesqa/releases) - [Commits](https://github.com/install-pinned/yesqa/compare/4896f663e9c294fddfbf5f4e4fc4f9b1a4556658...e45b0928dd14d5c2a22695e32de8936530ba7a49) --- updated-dependencies: - dependency-name: install-pinned/yesqa dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 8dca853d16..9d447cf59b 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -29,7 +29,7 @@ jobs: export GLOBIGNORE='mitmproxy/contrib/**' reorder-python-imports --exit-zero-even-if-changed --py310-plus **/*.py - - uses: install-pinned/yesqa@4896f663e9c294fddfbf5f4e4fc4f9b1a4556658 + - uses: install-pinned/yesqa@e45b0928dd14d5c2a22695e32de8936530ba7a49 - name: Run yesqa run: | shopt -s globstar From 6054d31665b675b9a2d3f05905a85302d028ecc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Mar 2023 00:50:06 +0100 Subject: [PATCH 202/695] Bump install-pinned/pyupgrade (#5957) Bumps [install-pinned/pyupgrade](https://github.com/install-pinned/pyupgrade) from 28e8d2633f6f1a03d5b4709682ce155a66324e6a to af7d65f31bddb01097a24da6c8fb694441f51cba. - [Release notes](https://github.com/install-pinned/pyupgrade/releases) - [Commits](https://github.com/install-pinned/pyupgrade/compare/28e8d2633f6f1a03d5b4709682ce155a66324e6a...af7d65f31bddb01097a24da6c8fb694441f51cba) --- updated-dependencies: - dependency-name: install-pinned/pyupgrade dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 9d447cf59b..a7fcf0b505 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: install-pinned/pyupgrade@28e8d2633f6f1a03d5b4709682ce155a66324e6a + - uses: install-pinned/pyupgrade@af7d65f31bddb01097a24da6c8fb694441f51cba - name: Run pyupgrade run: | shopt -s globstar From 262cadab75d377fbe3a79561e27e477757ee142c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Mar 2023 09:10:01 +0100 Subject: [PATCH 203/695] is_[d]tls_record -> starts_like_[d]tls_record --- mitmproxy/net/tls.py | 37 +++++++++++++++++++++---- mitmproxy/proxy/layers/tls.py | 25 ++--------------- test/mitmproxy/net/test_tls.py | 27 +++++++++++++----- test/mitmproxy/proxy/layers/test_tls.py | 11 -------- 4 files changed, 54 insertions(+), 46 deletions(-) diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index ab2adab397..c2eede2eac 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -224,15 +224,42 @@ def accept_all( return True -def is_tls_record_magic(d): +def starts_like_tls_record(d: bytes) -> bool: """ Returns: - True, if the passed bytes start with the TLS record magic bytes. + True, if the passed bytes could be the start of a TLS record False, otherwise. """ - d = d[:3] - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2, and TLSv1.3 # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello # https://tls13.ulfheim.net/ - return len(d) == 3 and d[0] == 0x16 and d[1] == 0x03 and 0x0 <= d[2] <= 0x03 + match len(d): + case 0: + return True + case 1: + return d[0] == 0x16 + case 2: + return d[0] == 0x16 and d[1] == 0x03 + case _: + return d[0] == 0x16 and d[1] == 0x03 and 0x00 <= d[2] <= 0x03 + + +def starts_like_dtls_record(d: bytes) -> bool: + """ + Returns: + True, if the passed bytes could be the start of a DTLS record + False, otherwise. + """ + # TLS ClientHello magic, works for DTLS 1.1, DTLS 1.2, and DTLS 1.3. + # https://www.rfc-editor.org/rfc/rfc4347#section-4.1 + # https://www.rfc-editor.org/rfc/rfc6347#section-4.1 + # https://www.rfc-editor.org/rfc/rfc9147#section-4-6.2 + match len(d): + case 0: + return True + case 1: + return d[0] == 0x16 + case 2: + return d[0] == 0x16 and d[1] == 0xfe + case _: + return d[0] == 0x16 and d[1] == 0xfe and 0xfd <= d[2] <= 0xfe diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index 5d58c0d0b4..a19be3e70d 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -25,18 +25,6 @@ from mitmproxy.utils import human -def is_tls_handshake_record(d: bytes) -> bool: - """ - Returns: - True, if the passed bytes start with the TLS record magic bytes - False, otherwise. - """ - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2. - # TLS 1.3 mandates legacy_record_version to be 0x0301. - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - return len(d) >= 3 and d[0] == 0x16 and d[1] == 0x03 and 0x0 <= d[2] <= 0x03 - - def handshake_record_contents(data: bytes) -> Iterator[bytes]: """ Returns a generator that yields the bytes contained in each handshake record. @@ -48,7 +36,7 @@ def handshake_record_contents(data: bytes) -> Iterator[bytes]: if len(data) < offset + 5: return record_header = data[offset : offset + 5] - if not is_tls_handshake_record(record_header): + if not starts_like_tls_record(record_header): raise ValueError(f"Expected TLS record, got {record_header!r} instead.") record_size = struct.unpack("!H", record_header[3:])[0] if record_size == 0: @@ -99,15 +87,6 @@ def parse_client_hello(data: bytes) -> ClientHello | None: return None -def is_dtls_handshake_record(d: bytes) -> bool: - """ - Returns: - True, if the passed bytes start with the DTLS record magic bytes - False, otherwise. - """ - return len(d) >= 3 and d[0] == 0x16 and d[1] == 0xFE and d[2] == 0xFD - - def dtls_handshake_record_contents(data: bytes) -> Iterator[bytes]: """ Returns a generator that yields the bytes contained in each handshake record. @@ -120,7 +99,7 @@ def dtls_handshake_record_contents(data: bytes) -> Iterator[bytes]: if len(data) < offset + 13: return record_header = data[offset : offset + 13] - if not is_dtls_handshake_record(record_header): + if not starts_like_dtls_record(record_header): raise ValueError(f"Expected DTLS record, got {record_header!r} instead.") # Length fields starts at 11 record_size = struct.unpack("!H", record_header[11:])[0] diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index 9e26003486..60f1287b16 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -70,10 +70,23 @@ def test_sslkeylogfile(tdata, monkeypatch): def test_is_record_magic(): - assert not tls.is_tls_record_magic(b"POST /") - assert not tls.is_tls_record_magic(b"\x16\x03") - assert not tls.is_tls_record_magic(b"\x16\x03\x04") - assert tls.is_tls_record_magic(b"\x16\x03\x00") - assert tls.is_tls_record_magic(b"\x16\x03\x01") - assert tls.is_tls_record_magic(b"\x16\x03\x02") - assert tls.is_tls_record_magic(b"\x16\x03\x03") + assert not tls.starts_like_tls_record(b"POST /") + assert not tls.starts_like_tls_record(b"\x16\x03\x04") + assert tls.starts_like_tls_record(b"") + assert tls.starts_like_tls_record(b"\x16") + assert tls.starts_like_tls_record(b"\x16\x03") + assert tls.starts_like_tls_record(b"\x16\x03\x00") + assert tls.starts_like_tls_record(b"\x16\x03\x01") + assert tls.starts_like_tls_record(b"\x16\x03\x02") + assert tls.starts_like_tls_record(b"\x16\x03\x03") + + +def test_is_dtls_record_magic(): + assert tls.starts_like_dtls_record(bytes.fromhex("")) + assert tls.starts_like_dtls_record(bytes.fromhex("16")) + assert tls.starts_like_dtls_record(bytes.fromhex("16fe")) + assert tls.starts_like_dtls_record(bytes.fromhex("16fefd")) + assert tls.starts_like_dtls_record(bytes.fromhex("16fefe")) + assert not tls.starts_like_dtls_record(bytes.fromhex("160300")) + assert not tls.starts_like_dtls_record(bytes.fromhex("160304")) + assert not tls.starts_like_dtls_record(bytes.fromhex("150301")) diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index d934604dec..2cc36a8484 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -24,17 +24,6 @@ tlsdata = data.Data(__name__) -def test_is_tls_handshake_record(): - assert tls.is_tls_handshake_record(bytes.fromhex("160300")) - assert tls.is_tls_handshake_record(bytes.fromhex("160301")) - assert tls.is_tls_handshake_record(bytes.fromhex("160302")) - assert tls.is_tls_handshake_record(bytes.fromhex("160303")) - assert not tls.is_tls_handshake_record(bytes.fromhex("ffffff")) - assert not tls.is_tls_handshake_record(bytes.fromhex("")) - assert not tls.is_tls_handshake_record(bytes.fromhex("160304")) - assert not tls.is_tls_handshake_record(bytes.fromhex("150301")) - - def test_record_contents(): data = bytes.fromhex("1603010002beef" "1603010001ff") assert list(tls.handshake_record_contents(data)) == [b"\xbe\xef", b"\xff"] From 0583e8e46c02ccd35ca22630d93ee6bab100b013 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Mar 2023 09:11:21 +0100 Subject: [PATCH 204/695] add MaybeClientTLSLayer --- mitmproxy/proxy/layers/tls.py | 63 +++++++++++++++++++++- test/mitmproxy/proxy/layers/test_tls.py | 70 ++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 8 deletions(-) diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index a19be3e70d..eb8edf4b9e 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -11,12 +11,15 @@ from mitmproxy import certs from mitmproxy import connection +from mitmproxy.net.tls import starts_like_dtls_record +from mitmproxy.net.tls import starts_like_tls_record from mitmproxy.proxy import commands from mitmproxy.proxy import context from mitmproxy.proxy import events from mitmproxy.proxy import layer from mitmproxy.proxy import tunnel from mitmproxy.proxy.commands import StartHook +from mitmproxy.proxy.layer import CommandGenerator from mitmproxy.proxy.layers import tcp from mitmproxy.proxy.layers import udp from mitmproxy.tls import ClientHello @@ -526,7 +529,7 @@ def __init__(self, context: context.Context): context.client.cipher_list = [] super().__init__(context, context.client) - self.server_tls_available = isinstance(self.context.layers[-2], ServerTLSLayer) + self.server_tls_available = len(self.context.layers) > 1 and isinstance(self.context.layers[-2], ServerTLSLayer) self.recv_buffer = bytearray() def start_handshake(self) -> layer.CommandGenerator[None]: @@ -654,6 +657,64 @@ def errored(self, event: events.Event) -> layer.CommandGenerator[None]: ) +class MaybeClientTLSLayer(ClientTLSLayer): + """ + This layer is used in places where the client connection may optionally be wrapped in TLS. + For example, when mitmproxy is running as a reverse proxy to https://example.com, a client may + issue a plaintext request to mitmproxy, or may already use TLS for the client <-> proxy connection as well. + Of course, in both cases the proxy <-> server connection would be HTTPS. + """ + def receive_handshake_data( + self, data: bytes + ) -> layer.CommandGenerator[tuple[bool, str | None]]: + received = bytes(self.recv_buffer + data) + if len(received) < 3: + self.recv_buffer.extend(data) + return False, None + + is_tls = ( + starts_like_dtls_record(received) + if self.is_dtls else + starts_like_tls_record(received) + ) + if is_tls: + self.receive_handshake_data = super().receive_handshake_data + self.on_handshake_error = super().on_handshake_error + self._handshake_finished = super()._handshake_finished + self.event_to_child = super().event_to_child + return (yield from super().receive_handshake_data(data)) + else: + self._event_queue.append(events.DataReceived(self.context.client, received)) + return True, None + + def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: + yield commands.CloseConnection(self.tunnel_connection) + + def _handshake_finished(self, err: str | None) -> layer.CommandGenerator[None]: + # only invoked for non-TLS connections. + # We're not doing TLS, so we assign fake connection objects for the tunnel. + self.conn = self.tunnel_connection = connection.Client( + peername=("ignore-conn", 0), sockname=("ignore-conn", 0) + ) + self.tunnel_state = tunnel.TunnelState.INACTIVE + + assert not self.command_to_reply_to + for evt in self._event_queue: + yield from self.event_to_child(evt) + self._event_queue.clear() + + def event_to_child(self, event: events.Event) -> CommandGenerator[None]: + received_server_data_while_waiting_for_client_hello = ( + self.tunnel_state is tunnel.TunnelState.ESTABLISHING + and isinstance(event, events.DataReceived) + and event.connection != self.conn + ) + if received_server_data_while_waiting_for_client_hello: + yield from self._handshake_finished(None) + + yield from super().event_to_child(event) + + class MockTLSLayer(TLSLayer): """Mock layer to disable actual TLS and use cleartext in tests. diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 2cc36a8484..3b946f589d 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -765,13 +765,69 @@ def test_unsupported_protocol(self, tctx: context.Context): assert tls_hook_data().conn.error -def test_is_dtls_handshake_record(): - assert tls.is_dtls_handshake_record(bytes.fromhex("16fefd")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("160300")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("16fefe")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("160304")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("150301")) +class TestMaybeClientTLS: + @pytest.mark.parametrize("trickle", ["trickle", "full"]) + def test_no_tls(self, tctx: context.Context, trickle: bool): + layer = tls.MaybeClientTLSLayer(tctx) + layer.child_layer = tutils.EchoLayer(tctx) + playbook = tutils.Playbook(layer) + + if trickle == "trickle": + playbook >> events.DataReceived(tctx.client, b"ABC") + else: + playbook >> events.DataReceived(tctx.client, b"A") + playbook >> events.DataReceived(tctx.client, b"B") + playbook >> events.DataReceived(tctx.client, b"C") + + assert ( + playbook + << commands.SendData(tctx.client, b'abc') + >> events.ConnectionClosed(tctx.client) + << commands.CloseConnection(tctx.client) + ) + + @pytest.mark.parametrize("trickle", ["trickle", "full"]) + def test_tls(self, tctx: context.Context, trickle): + layer = tls.MaybeClientTLSLayer(tctx) + layer.child_layer = tutils.EchoLayer(tctx) + data = tutils.Placeholder(bytes) + playbook = tutils.Playbook(layer) + + if trickle == "trickle": + for d in client_hello_tls_1_3: + playbook >> events.DataReceived(tctx.client, bytes([d])) + else: + playbook >> events.DataReceived(tctx.client, client_hello_tls_1_3) + assert ( + playbook + << tls.TlsClienthelloHook(tutils.Placeholder()) + >> tutils.reply() + << tls.TlsStartClientHook(tutils.Placeholder()) + >> reply_tls_start_client() + << commands.SendData(tctx.client, data) + ) + assert data().startswith(b"\x16\x03\x03") + + def test_close_early(self, tctx: context.Context): + layer = tls.MaybeClientTLSLayer(tctx) + layer.child_layer = tutils.EchoLayer(tctx) + + assert ( + tutils.Playbook(layer) + >> events.DataReceived(tctx.client, b"A") + >> events.ConnectionClosed(tctx.client) + << commands.CloseConnection(tctx.client) + ) + + def test_no_side_effects(self, tctx: context.Context): + layer = tls.MaybeClientTLSLayer(tctx) + layer.child_layer = tutils.EchoLayer(tctx) + + assert ( + tutils.Playbook(layer) + >> events.DataReceived(tctx.server, b"A") + << commands.SendData(tctx.server, b"a") + ) def test_dtls_record_contents(): From 524adbf950384f1a7cce77abfa0ed0a4666ce5d7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Mar 2023 09:11:46 +0100 Subject: [PATCH 205/695] Revert "add MaybeClientTLSLayer" This reverts commit e7b0e3f351686c2ef5f1daa13365216b775eefb3. --- mitmproxy/proxy/layers/tls.py | 63 +--------------------- test/mitmproxy/proxy/layers/test_tls.py | 70 +++---------------------- 2 files changed, 8 insertions(+), 125 deletions(-) diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index eb8edf4b9e..a19be3e70d 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -11,15 +11,12 @@ from mitmproxy import certs from mitmproxy import connection -from mitmproxy.net.tls import starts_like_dtls_record -from mitmproxy.net.tls import starts_like_tls_record from mitmproxy.proxy import commands from mitmproxy.proxy import context from mitmproxy.proxy import events from mitmproxy.proxy import layer from mitmproxy.proxy import tunnel from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.layer import CommandGenerator from mitmproxy.proxy.layers import tcp from mitmproxy.proxy.layers import udp from mitmproxy.tls import ClientHello @@ -529,7 +526,7 @@ def __init__(self, context: context.Context): context.client.cipher_list = [] super().__init__(context, context.client) - self.server_tls_available = len(self.context.layers) > 1 and isinstance(self.context.layers[-2], ServerTLSLayer) + self.server_tls_available = isinstance(self.context.layers[-2], ServerTLSLayer) self.recv_buffer = bytearray() def start_handshake(self) -> layer.CommandGenerator[None]: @@ -657,64 +654,6 @@ def errored(self, event: events.Event) -> layer.CommandGenerator[None]: ) -class MaybeClientTLSLayer(ClientTLSLayer): - """ - This layer is used in places where the client connection may optionally be wrapped in TLS. - For example, when mitmproxy is running as a reverse proxy to https://example.com, a client may - issue a plaintext request to mitmproxy, or may already use TLS for the client <-> proxy connection as well. - Of course, in both cases the proxy <-> server connection would be HTTPS. - """ - def receive_handshake_data( - self, data: bytes - ) -> layer.CommandGenerator[tuple[bool, str | None]]: - received = bytes(self.recv_buffer + data) - if len(received) < 3: - self.recv_buffer.extend(data) - return False, None - - is_tls = ( - starts_like_dtls_record(received) - if self.is_dtls else - starts_like_tls_record(received) - ) - if is_tls: - self.receive_handshake_data = super().receive_handshake_data - self.on_handshake_error = super().on_handshake_error - self._handshake_finished = super()._handshake_finished - self.event_to_child = super().event_to_child - return (yield from super().receive_handshake_data(data)) - else: - self._event_queue.append(events.DataReceived(self.context.client, received)) - return True, None - - def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]: - yield commands.CloseConnection(self.tunnel_connection) - - def _handshake_finished(self, err: str | None) -> layer.CommandGenerator[None]: - # only invoked for non-TLS connections. - # We're not doing TLS, so we assign fake connection objects for the tunnel. - self.conn = self.tunnel_connection = connection.Client( - peername=("ignore-conn", 0), sockname=("ignore-conn", 0) - ) - self.tunnel_state = tunnel.TunnelState.INACTIVE - - assert not self.command_to_reply_to - for evt in self._event_queue: - yield from self.event_to_child(evt) - self._event_queue.clear() - - def event_to_child(self, event: events.Event) -> CommandGenerator[None]: - received_server_data_while_waiting_for_client_hello = ( - self.tunnel_state is tunnel.TunnelState.ESTABLISHING - and isinstance(event, events.DataReceived) - and event.connection != self.conn - ) - if received_server_data_while_waiting_for_client_hello: - yield from self._handshake_finished(None) - - yield from super().event_to_child(event) - - class MockTLSLayer(TLSLayer): """Mock layer to disable actual TLS and use cleartext in tests. diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 3b946f589d..2cc36a8484 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -765,69 +765,13 @@ def test_unsupported_protocol(self, tctx: context.Context): assert tls_hook_data().conn.error -class TestMaybeClientTLS: - @pytest.mark.parametrize("trickle", ["trickle", "full"]) - def test_no_tls(self, tctx: context.Context, trickle: bool): - layer = tls.MaybeClientTLSLayer(tctx) - layer.child_layer = tutils.EchoLayer(tctx) - playbook = tutils.Playbook(layer) - - if trickle == "trickle": - playbook >> events.DataReceived(tctx.client, b"ABC") - else: - playbook >> events.DataReceived(tctx.client, b"A") - playbook >> events.DataReceived(tctx.client, b"B") - playbook >> events.DataReceived(tctx.client, b"C") - - assert ( - playbook - << commands.SendData(tctx.client, b'abc') - >> events.ConnectionClosed(tctx.client) - << commands.CloseConnection(tctx.client) - ) - - @pytest.mark.parametrize("trickle", ["trickle", "full"]) - def test_tls(self, tctx: context.Context, trickle): - layer = tls.MaybeClientTLSLayer(tctx) - layer.child_layer = tutils.EchoLayer(tctx) - data = tutils.Placeholder(bytes) - playbook = tutils.Playbook(layer) - - if trickle == "trickle": - for d in client_hello_tls_1_3: - playbook >> events.DataReceived(tctx.client, bytes([d])) - else: - playbook >> events.DataReceived(tctx.client, client_hello_tls_1_3) - assert ( - playbook - << tls.TlsClienthelloHook(tutils.Placeholder()) - >> tutils.reply() - << tls.TlsStartClientHook(tutils.Placeholder()) - >> reply_tls_start_client() - << commands.SendData(tctx.client, data) - ) - assert data().startswith(b"\x16\x03\x03") - - def test_close_early(self, tctx: context.Context): - layer = tls.MaybeClientTLSLayer(tctx) - layer.child_layer = tutils.EchoLayer(tctx) - - assert ( - tutils.Playbook(layer) - >> events.DataReceived(tctx.client, b"A") - >> events.ConnectionClosed(tctx.client) - << commands.CloseConnection(tctx.client) - ) - - def test_no_side_effects(self, tctx: context.Context): - layer = tls.MaybeClientTLSLayer(tctx) - layer.child_layer = tutils.EchoLayer(tctx) - - assert ( - tutils.Playbook(layer) - >> events.DataReceived(tctx.server, b"A") - << commands.SendData(tctx.server, b"a") - ) +def test_is_dtls_handshake_record(): + assert tls.is_dtls_handshake_record(bytes.fromhex("16fefd")) + assert not tls.is_dtls_handshake_record(bytes.fromhex("160300")) + assert not tls.is_dtls_handshake_record(bytes.fromhex("16fefe")) + assert not tls.is_dtls_handshake_record(bytes.fromhex("")) + assert not tls.is_dtls_handshake_record(bytes.fromhex("160304")) + assert not tls.is_dtls_handshake_record(bytes.fromhex("150301")) def test_dtls_record_contents(): From edb85d4af1c4ac71caf578d6ce016644434c062d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Mar 2023 14:49:17 +0100 Subject: [PATCH 206/695] refactor NextLayer addon --- mitmproxy/addons/next_layer.py | 513 ++++++------- mitmproxy/net/tls.py | 22 +- mitmproxy/options.py | 2 +- mitmproxy/proxy/layers/http/__init__.py | 21 +- mitmproxy/proxy/layers/modes.py | 27 +- mitmproxy/proxy/layers/tls.py | 2 + mitmproxy/proxy/tunnel.py | 2 +- setup.cfg | 1 + test/mitmproxy/addons/test_next_layer.py | 832 +++++++++++++--------- test/mitmproxy/addons/test_proxyserver.py | 40 +- test/mitmproxy/contentviews/test_http3.py | 3 - test/mitmproxy/net/test_tls.py | 13 +- test/mitmproxy/proxy/layers/test_modes.py | 11 +- test/mitmproxy/proxy/layers/test_tls.py | 9 - 14 files changed, 837 insertions(+), 661 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 25f6af9e7f..c26e3f6bdc 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -8,49 +8,56 @@ For a typical HTTPS request, this addon is called a couple of times: First to determine that we start with an HTTP layer which processes the `CONNECT` request, a second time to determine that the client then starts negotiating TLS, and a -third time where we check if the protocol within that TLS stream is actually HTTP or something else. +third time when we check if the protocol within that TLS stream is actually HTTP or something else. Sometimes it's useful to hardcode specific logic in next_layer when one wants to do fancy things. In that case it's not necessary to modify mitmproxy's source, adding a custom addon with a next_layer event hook that sets nextlayer.layer works just as well. """ +from __future__ import annotations + +import logging import re import struct -from collections.abc import Callable from collections.abc import Iterable from collections.abc import Sequence from typing import Any +from typing import assert_never from typing import cast -from typing import Union -from mitmproxy import connection from mitmproxy import ctx from mitmproxy import dns from mitmproxy import exceptions -from mitmproxy.net.tls import is_tls_record_magic -from mitmproxy.proxy import context +from mitmproxy.net.tls import starts_like_dtls_record +from mitmproxy.net.tls import starts_like_tls_record from mitmproxy.proxy import layer from mitmproxy.proxy import layers from mitmproxy.proxy import mode_specs +from mitmproxy.proxy import tunnel +from mitmproxy.proxy.context import Context +from mitmproxy.proxy.layer import Layer +from mitmproxy.proxy.layers import ClientQuicLayer +from mitmproxy.proxy.layers import ClientTLSLayer +from mitmproxy.proxy.layers import DNSLayer +from mitmproxy.proxy.layers import HttpLayer +from mitmproxy.proxy.layers import RawQuicLayer +from mitmproxy.proxy.layers import ServerQuicLayer +from mitmproxy.proxy.layers import ServerTLSLayer +from mitmproxy.proxy.layers import TCPLayer +from mitmproxy.proxy.layers import UDPLayer from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy.layers.quic import quic_parse_client_hello -from mitmproxy.proxy.layers.tls import dtls_parse_client_hello from mitmproxy.proxy.layers.tls import HTTP_ALPNS +from mitmproxy.proxy.layers.tls import dtls_parse_client_hello from mitmproxy.proxy.layers.tls import parse_client_hello from mitmproxy.tls import ClientHello -LayerCls = type[layer.Layer] -ClientSecurityLayerCls = Union[ - type[layers.ClientTLSLayer], type[layers.ClientQuicLayer] -] -ServerSecurityLayerCls = Union[ - type[layers.ServerTLSLayer], type[layers.ServerQuicLayer] -] +logger = logging.getLogger(__name__) def stack_match( - context: context.Context, layers: Sequence[LayerCls | tuple[LayerCls, ...]] + context: Context, layers: Sequence[type[Layer] | tuple[type[Layer], ...]] ) -> bool: if len(context.layers) != len(layers): return False @@ -60,11 +67,15 @@ def stack_match( ) +class NeedsMoreData(Exception): + """Signal that the decision on which layer to put next needs to be deferred within the NextLayer addon.""" + + class NextLayer: - ignore_hosts: Iterable[re.Pattern] = () - allow_hosts: Iterable[re.Pattern] = () - tcp_hosts: Iterable[re.Pattern] = () - udp_hosts: Iterable[re.Pattern] = () + ignore_hosts: Sequence[re.Pattern] = () + allow_hosts: Sequence[re.Pattern] = () + tcp_hosts: Sequence[re.Pattern] = () + udp_hosts: Sequence[re.Pattern] = () def configure(self, updated): if "tcp_hosts" in updated: @@ -87,37 +98,123 @@ def configure(self, updated): re.compile(x, re.IGNORECASE) for x in ctx.options.allow_hosts ] - def ignore_connection( + def next_layer(self, nextlayer: layer.NextLayer): + if nextlayer.layer: + return # do not override something another addon has set. + try: + nextlayer.layer = self._next_layer( + nextlayer.context, + nextlayer.data_client(), + nextlayer.data_server(), + ) + except NeedsMoreData: + logger.info( + f"Deferring layer decision, not enough data: {nextlayer.data_client().hex()}" + ) + + def _next_layer( + self, context: Context, data_client: bytes, data_server: bytes + ) -> Layer | None: + assert context.layers + + def s(*layers): + return stack_match(context, layers) + + tcp_based = context.client.transport_protocol == "tcp" + udp_based = context.client.transport_protocol == "udp" + + # 1) check for --ignore/--allow + if self._ignore_connection(context, data_client): + return ( + layers.TCPLayer(context, ignore=True) + if tcp_based + else layers.UDPLayer(context, ignore=True) + ) + + # 2) Handle proxy modes with well-defined next protocol + # 2a) Reverse proxy: derive from spec + if s(modes.ReverseProxy): + return self._setup_reverse_proxy(context, data_client) + # 2b) Explicit HTTP proxies + if s((modes.HttpProxy, modes.HttpUpstreamProxy)): + return self._setup_explicit_http_proxy(context, data_client) + + # 3) Handle security protocols + # 3a) TLS/DTLS + is_tls_or_dtls = ( + tcp_based + and starts_like_tls_record(data_client) + or udp_based + and starts_like_dtls_record(data_client) + ) + if is_tls_or_dtls: + server_tls = ServerTLSLayer(context) + server_tls.child_layer = ClientTLSLayer(context) + return server_tls + # 3b) QUIC + if udp_based and _starts_like_quic(data_client): + server_quic = ServerQuicLayer(context) + server_quic.child_layer = ClientQuicLayer(context) + return server_quic + + # 4) Check for --tcp/--udp + if tcp_based and self._is_destination_in_hosts(context, self.tcp_hosts): + return layers.TCPLayer(context) + if udp_based and self._is_destination_in_hosts(context, self.udp_hosts): + return layers.UDPLayer(context) + + # 5) Handle application protocol + # 5a) Is it DNS? + if udp_based: + try: + # TODO: DNS over TCP + dns.Message.unpack(data_client) # TODO: perf + except struct.error: + pass + else: + return layers.DNSLayer(context) + # 5b) We have no other specialized layers for UDP, so we fall back to raw forwarding. + if udp_based: + return layers.UDPLayer(context) + # 5b) Check for raw tcp mode. + very_likely_http = context.client.alpn and context.client.alpn in HTTP_ALPNS + probably_no_http = not very_likely_http and ( + # the first three bytes should be the HTTP verb, so A-Za-z is expected. + len(data_client) < 3 + or not data_client[:3].isalpha() + # a server greeting would be uncharacteristic. + or data_server + ) + if ctx.options.rawtcp and probably_no_http: + return layers.TCPLayer(context) + # 5c) Assume HTTP by default. + return layers.HttpLayer(context, HTTPMode.transparent) + + def _ignore_connection( self, - server_address: connection.Address | None, + context: Context, data_client: bytes, - *, - is_tls: Callable[[bytes], bool] = is_tls_record_magic, - client_hello: Callable[[bytes], ClientHello | None] = parse_client_hello, ) -> bool | None: """ Returns: True, if the connection should be ignored. False, if it should not be ignored. - None, if we need to wait for more input data. + + Raises: + NeedsMoreData, if we need to wait for more input data. """ if not ctx.options.ignore_hosts and not ctx.options.allow_hosts: return False hostnames: list[str] = [] - if server_address is not None: - hostnames.append(server_address[0]) - if is_tls(data_client): - try: - ch = client_hello(data_client) - if ch is None: # not complete yet - return None - sni = ch.sni - except ValueError: - pass - else: - if sni: - hostnames.append(sni) + if context.server.peername and (peername := context.server.peername[0]): + hostnames.append(peername) + if context.server.address and (server_address := context.server.address[0]): + hostnames.append(server_address) + if ( + client_hello := self._get_client_hello(context, data_client) + ) and client_hello.sni: + hostnames.append(client_hello.sni) if not hostnames: return False @@ -137,35 +234,122 @@ def ignore_connection( else: # pragma: no cover raise AssertionError() - def setup_tls_layer( - self, - context: context.Context, - client_layer_cls: ClientSecurityLayerCls = layers.ClientTLSLayer, - server_layer_cls: ServerSecurityLayerCls = layers.ServerTLSLayer, - ) -> layer.Layer: - def s(*layers): - return stack_match(context, layers) + def _get_client_hello( + self, context: Context, data_client: bytes + ) -> ClientHello | None: + """ + Try to read a TLS/DTLS/QUIC ClientHello from data_client. - # client tls usually requires a server tls layer as parent layer, except: - # - a secure web proxy doesn't have a server part. - # - an upstream proxy uses the mode spec - # - reverse proxy mode manages this itself. - if ( - s(modes.HttpProxy) - or s(modes.HttpUpstreamProxy) - or s(modes.ReverseProxy) - or s(modes.ReverseProxy, server_layer_cls) - ): - return client_layer_cls(context) - else: - # We already assign the next layer here so that the server layer - # knows that it can safely wait for a ClientHello. - ret = server_layer_cls(context) - ret.child_layer = client_layer_cls(context) - return ret - - def is_destination_in_hosts( - self, context: context.Context, hosts: Iterable[re.Pattern] + Returns: + A complete ClientHello, or None, if no ClientHello was found. + + Raises: + NeedsMoreData, if the ClientHello is incomplete. + """ + match context.client.transport_protocol: + case "tcp": + if starts_like_tls_record(data_client): + try: + ch = parse_client_hello(data_client) + except ValueError: + pass + else: + if ch is None: + raise NeedsMoreData + return ch + return None + case "udp": + try: + return quic_parse_client_hello(data_client) + except ValueError: + pass + + try: + ch = dtls_parse_client_hello(data_client) + except ValueError: + pass + else: + if ch is None: + raise NeedsMoreData + return ch + return None + case _: + assert_never(context.client.transport_protocol) + + def _setup_reverse_proxy(self, context: Context, data_client: bytes) -> Layer: + spec = cast(mode_specs.ReverseMode, context.client.proxy_mode) + stack = tunnel.LayerStack() + + match spec.scheme: + case "http": + if starts_like_tls_record(data_client): + stack /= ClientTLSLayer(context) + stack /= HttpLayer(context, HTTPMode.transparent) + case "https": + stack /= ServerTLSLayer(context) + if starts_like_tls_record(data_client): + stack /= ClientTLSLayer(context) + stack /= HttpLayer(context, HTTPMode.transparent) + + case "tcp": + if starts_like_tls_record(data_client): + stack /= ClientTLSLayer(context) + stack /= TCPLayer(context) + case "tls": + stack /= ServerTLSLayer(context) + if starts_like_tls_record(data_client): + stack /= ClientTLSLayer(context) + stack /= TCPLayer(context) + + case "udp": + if starts_like_dtls_record(data_client): + stack /= ClientTLSLayer(context) + stack /= UDPLayer(context) + case "dtls": + stack /= ServerTLSLayer(context) + if starts_like_dtls_record(data_client): + stack /= ClientTLSLayer(context) + stack /= UDPLayer(context) + + case "dns": + # TODO: DNS-over-TLS / DNS-over-DTLS + # is_tls_or_dtls = ( + # context.client.transport_protocol == "tcp" and starts_like_tls_record(data_client) + # or + # context.client.transport_protocol == "udp" and starts_like_dtls_record(data_client) + # ) + # if is_tls_or_dtls: + # stack /= ClientTLSLayer(context) + stack /= DNSLayer(context) + + case "http3": + stack /= ServerQuicLayer(context) + stack /= ClientQuicLayer(context) + stack /= HttpLayer(context, HTTPMode.transparent) + case "quic": + stack /= ServerQuicLayer(context) + stack /= ClientQuicLayer(context) + stack /= RawQuicLayer(context) + + case _: + assert_never(spec.scheme) + + return stack[0] + + def _setup_explicit_http_proxy(self, context: Context, data_client: bytes) -> Layer: + stack = tunnel.LayerStack() + + if context.client.transport_protocol == "udp": + stack /= layers.ClientQuicLayer(context) + elif starts_like_tls_record(data_client): + stack /= layers.ClientTLSLayer(context) + + stack /= layers.HttpLayer(context, HTTPMode.regular) + + return stack[0] + + def _is_destination_in_hosts( + self, context: Context, hosts: Iterable[re.Pattern] ) -> bool: return any( (context.server.address and rex.search(context.server.address[0])) @@ -173,192 +357,13 @@ def is_destination_in_hosts( for rex in hosts ) - def get_http_layer(self, context: context.Context) -> layers.HttpLayer | None: - def s(*layers): - return stack_match(context, layers) - - # Setup the HTTP layer for a regular HTTP proxy ... - if ( - s(modes.HttpProxy) - or - # or a "Secure Web Proxy", see https://www.chromium.org/developers/design-documents/secure-web-proxy - s(modes.HttpProxy, (layers.ClientTLSLayer, layers.ClientQuicLayer)) - ): - return layers.HttpLayer(context, HTTPMode.regular) - # ... or an upstream proxy. - if s(modes.HttpUpstreamProxy) or s( - modes.HttpUpstreamProxy, (layers.ClientTLSLayer, layers.ClientQuicLayer) - ): - return layers.HttpLayer(context, HTTPMode.upstream) - return None - - def detect_udp_tls( - self, data_client: bytes - ) -> tuple[ClientHello, ClientSecurityLayerCls, ServerSecurityLayerCls] | None: - if len(data_client) == 0: - return None - - # first try DTLS (the parser may return None) - try: - client_hello = dtls_parse_client_hello(data_client) - if client_hello is not None: - return (client_hello, layers.ClientTLSLayer, layers.ServerTLSLayer) - except ValueError: - pass - - # next try QUIC - try: - client_hello = quic_parse_client_hello(data_client) - return (client_hello, layers.ClientQuicLayer, layers.ServerQuicLayer) - except (ValueError, TypeError): - pass - - # that's all we currently have to offer - return None - - def raw_udp_layer( - self, context: context.Context, ignore: bool = False - ) -> layer.Layer: - def s(*layers): - return stack_match(context, layers) - - # for regular and upstream HTTP3, if we already created a client QUIC layer - # we need a server and raw QUIC layer as well - if s(modes.HttpProxy, layers.ClientQuicLayer) or s( - modes.HttpUpstreamProxy, layers.ClientQuicLayer - ): - server_layer = layers.ServerQuicLayer(context) - server_layer.child_layer = layers.RawQuicLayer(context, ignore=ignore) - return server_layer - - # for reverse HTTP3 and QUIC, we need a client and raw QUIC layer - elif s(modes.ReverseProxy, layers.ServerQuicLayer): - client_layer = layers.ClientQuicLayer(context) - client_layer.child_layer = layers.RawQuicLayer(context, ignore=ignore) - return client_layer - - # in other cases we assume `setup_tls_layer` happened, so if the - # top layer is `ClientQuicLayer` we return a raw QUIC layer... - elif isinstance(context.layers[-1], layers.ClientQuicLayer): - return layers.RawQuicLayer(context, ignore=ignore) - - # ... otherwise an UDP layer - else: - return layers.UDPLayer(context, ignore=ignore) - def next_layer(self, nextlayer: layer.NextLayer): - if nextlayer.layer is None: - nextlayer.layer = self._next_layer( - nextlayer.context, - nextlayer.data_client(), - nextlayer.data_server(), - ) - - def _next_layer( - self, context: context.Context, data_client: bytes, data_server: bytes - ) -> layer.Layer | None: - assert context.layers - - if context.client.transport_protocol == "tcp": - is_quic_stream = isinstance(context.layers[-1], layers.QuicStreamLayer) - if len(data_client) < 3 and not data_server and not is_quic_stream: - return None # not enough data yet to make a decision - - # 1. check for --ignore/--allow - ignore = self.ignore_connection(context.server.address, data_client) - if ignore is True: - return layers.TCPLayer(context, ignore=True) - if ignore is None and not is_quic_stream: - return None - - # 2. Check for TLS - if is_tls_record_magic(data_client): - return self.setup_tls_layer(context) - - # 3. Check for HTTP - if http_layer := self.get_http_layer(context): - return http_layer - - # 4. Check for --tcp - if self.is_destination_in_hosts(context, self.tcp_hosts): - return layers.TCPLayer(context) - - # 5. Check for raw tcp mode. - very_likely_http = context.client.alpn and context.client.alpn in HTTP_ALPNS - probably_no_http = not very_likely_http and ( - not data_client[ - :3 - ].isalpha() # the first three bytes should be the HTTP verb, so A-Za-z is expected. - or data_server # a server greeting would be uncharacteristic. - ) - if ctx.options.rawtcp and probably_no_http: - return layers.TCPLayer(context) - - # 6. Assume HTTP by default. - return layers.HttpLayer(context, HTTPMode.transparent) - - elif context.client.transport_protocol == "udp": - # unlike TCP, we make a decision immediately - tls = self.detect_udp_tls(data_client) - - # 1. check for --ignore/--allow - if self.ignore_connection( - context.server.address, - data_client, - is_tls=lambda _: tls is not None, - client_hello=lambda _: None if tls is None else tls[0], - ): - return self.raw_udp_layer(context, ignore=True) - - # 2. Check for DTLS/QUIC - if tls is not None: - _, client_layer_cls, server_layer_cls = tls - return self.setup_tls_layer(context, client_layer_cls, server_layer_cls) - - # 3. Check for HTTP - if http_layer := self.get_http_layer(context): - return http_layer - - # 4. Check for --udp - if self.is_destination_in_hosts(context, self.udp_hosts): - return self.raw_udp_layer(context) - - # 5. Check for reverse modes - if isinstance(context.layers[0], modes.ReverseProxy): - scheme = cast(mode_specs.ReverseMode, context.client.proxy_mode).scheme - if scheme in ("udp", "dtls"): - return layers.UDPLayer(context) - elif scheme == "http3": - if isinstance(context.layers[-1], layers.ClientQuicLayer): - return layers.HttpLayer(context, HTTPMode.transparent) - else: - return layers.ClientQuicLayer(context) - elif scheme == "quic": - if isinstance(context.layers[-1], layers.ClientQuicLayer): - # the client supports QUIC, use raw layer - return layers.RawQuicLayer(context) - elif data_client: - # we have received data, which was not a handshake, use UDP - # on the client, and send datagrams over QUIC to the server - return layers.UDPLayer(context) - else: - # wait for client data to make a decision - return None - elif scheme == "dns": - return layers.DNSLayer(context) - else: - raise AssertionError(scheme) - - # 6. Check for DNS - try: - dns.Message.unpack(data_client) - except struct.error: - pass - else: - return layers.DNSLayer(context) - - # 7. Use raw mode. - return self.raw_udp_layer(context) - - else: - raise AssertionError(context.client.transport_protocol) +def _starts_like_quic(data_client: bytes) -> bool: + # FIXME: handle clienthellos distributed over multiple packets? + # FIXME: perf + try: + quic_parse_client_hello(data_client) + except ValueError: + return False + else: + return True diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index c2eede2eac..0f7ad2a2f9 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -233,15 +233,8 @@ def starts_like_tls_record(d: bytes) -> bool: # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2, and TLSv1.3 # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello # https://tls13.ulfheim.net/ - match len(d): - case 0: - return True - case 1: - return d[0] == 0x16 - case 2: - return d[0] == 0x16 and d[1] == 0x03 - case _: - return d[0] == 0x16 and d[1] == 0x03 and 0x00 <= d[2] <= 0x03 + # We assume that a client sending less than 3 bytes initially is not a TLS client. + return len(d) > 2 and d[0] == 0x16 and d[1] == 0x03 and 0x00 <= d[2] <= 0x03 def starts_like_dtls_record(d: bytes) -> bool: @@ -254,12 +247,5 @@ def starts_like_dtls_record(d: bytes) -> bool: # https://www.rfc-editor.org/rfc/rfc4347#section-4.1 # https://www.rfc-editor.org/rfc/rfc6347#section-4.1 # https://www.rfc-editor.org/rfc/rfc9147#section-4-6.2 - match len(d): - case 0: - return True - case 1: - return d[0] == 0x16 - case 2: - return d[0] == 0x16 and d[1] == 0xfe - case _: - return d[0] == 0x16 and d[1] == 0xfe and 0xfd <= d[2] <= 0xfe + # We assume that a client sending less than 3 bytes initially is not a DTLS client. + return len(d) > 2 and d[0] == 0x16 and d[1] == 0xFE and 0xFD <= d[2] <= 0xFE diff --git a/mitmproxy/options.py b/mitmproxy/options.py index f3cf11bedc..1e9f2404be 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -131,7 +131,7 @@ def __init__(self, **kwargs) -> None: "http2", bool, True, - "Enable/disable HTTP/2 support. " "HTTP/2 support is enabled by default.", + "Enable/disable HTTP/2 support. HTTP/2 support is enabled by default.", ) self.add_option( "http2_ping_keepalive", diff --git a/mitmproxy/proxy/layers/http/__init__.py b/mitmproxy/proxy/layers/http/__init__.py index 8fcbf37081..f5696f25a0 100644 --- a/mitmproxy/proxy/layers/http/__init__.py +++ b/mitmproxy/proxy/layers/http/__init__.py @@ -852,22 +852,23 @@ def __init__(self, context: Context, mode: HTTPMode): self.waiting_for_establishment = collections.defaultdict(list) self.streams = {} self.command_sources = {} - - http_conn: HttpConnection - if is_h3_alpn(self.context.client.alpn): - http_conn = Http3Server(context.fork()) - elif self.context.client.alpn == b"h2": - http_conn = Http2Server(context.fork()) - else: - http_conn = Http1Server(context.fork()) - - self.connections = {context.client: http_conn} + self.connections = {} def __repr__(self): return f"HttpLayer({self.mode.name}, conns: {len(self.connections)})" def _handle_event(self, event: events.Event): if isinstance(event, events.Start): + http_conn: HttpConnection + if is_h3_alpn(self.context.client.alpn): + http_conn = Http3Server(self.context.fork()) + elif self.context.client.alpn == b"h2": + http_conn = Http2Server(self.context.fork()) + else: + http_conn = Http1Server(self.context.fork()) + + # may have been set by client playback. + self.connections.setdefault(self.context.client, http_conn) yield from self.event_to_child(self.connections[self.context.client], event) if self.mode is HTTPMode.upstream: proxy_mode = self.context.client.proxy_mode diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index ba79c21356..4499296a5f 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -5,14 +5,18 @@ from abc import ABCMeta from collections.abc import Callable from dataclasses import dataclass +from typing import assert_never from mitmproxy import connection from mitmproxy.proxy import commands from mitmproxy.proxy import events from mitmproxy.proxy import layer +from mitmproxy.proxy import tunnel from mitmproxy.proxy.commands import StartHook +from mitmproxy.proxy.layers import dns from mitmproxy.proxy.layers import quic from mitmproxy.proxy.layers import tls +from mitmproxy.proxy.layers import http from mitmproxy.proxy.mode_specs import ReverseMode from mitmproxy.proxy.utils import expect @@ -65,18 +69,17 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: assert isinstance(spec, ReverseMode) self.context.server.address = spec.address - if spec.scheme in ("http3", "quic"): - if not self.context.options.keep_host_header: - self.context.server.sni = spec.address[0] - self.child_layer = quic.ServerQuicLayer(self.context) - elif spec.scheme in ("https", "tls", "dtls"): - if not self.context.options.keep_host_header: - self.context.server.sni = spec.address[0] - self.child_layer = tls.ServerTLSLayer(self.context) - elif spec.scheme in ("tcp", "http", "udp", "dns"): - self.child_layer = layer.NextLayer(self.context) - else: - raise AssertionError(spec.scheme) # pragma: no cover + self.child_layer = layer.NextLayer(self.context) + + # For secure protocols, set SNI if keep_host_header is false + match spec.scheme: + case "http3" | "quic" | "https" | "tls" | "dtls": + if not self.context.options.keep_host_header: + self.context.server.sni = spec.address[0] + case "tcp" | "http" | "udp" | "dns": + pass + case _: + assert_never(spec.scheme) err = yield from self.finish_start() if err: diff --git a/mitmproxy/proxy/layers/tls.py b/mitmproxy/proxy/layers/tls.py index a19be3e70d..9ed9449989 100644 --- a/mitmproxy/proxy/layers/tls.py +++ b/mitmproxy/proxy/layers/tls.py @@ -11,6 +11,8 @@ from mitmproxy import certs from mitmproxy import connection +from mitmproxy.net.tls import starts_like_dtls_record +from mitmproxy.net.tls import starts_like_tls_record from mitmproxy.proxy import commands from mitmproxy.proxy import context from mitmproxy.proxy import events diff --git a/mitmproxy/proxy/tunnel.py b/mitmproxy/proxy/tunnel.py index 1ed1e73be5..bdda421543 100644 --- a/mitmproxy/proxy/tunnel.py +++ b/mitmproxy/proxy/tunnel.py @@ -97,7 +97,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: else: yield from self.event_to_child(event) - def _handshake_finished(self, err: str | None): + def _handshake_finished(self, err: str | None) -> layer.CommandGenerator[None]: if err: self.tunnel_state = TunnelState.CLOSED else: diff --git a/setup.cfg b/setup.cfg index 346f04b5f2..46f9890247 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ exclude_lines = if TYPE_CHECKING: @overload @abstractmethod + assert_never \.\.\. [mypy] diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index 8f6f382f28..1698dcd892 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -1,28 +1,35 @@ +from __future__ import annotations + +import dataclasses +import logging +from dataclasses import dataclass +from functools import partial +from typing import Sequence from unittest.mock import MagicMock import pytest -from mitmproxy import connection +from mitmproxy.addons.next_layer import NeedsMoreData from mitmproxy.addons.next_layer import NextLayer -from mitmproxy.proxy import context -from mitmproxy.proxy import layer -from mitmproxy.proxy import layers +from mitmproxy.addons.next_layer import stack_match +from mitmproxy.connection import Client +from mitmproxy.connection import TransportProtocol +from mitmproxy.proxy.context import Context +from mitmproxy.proxy.layer import Layer +from mitmproxy.proxy.layers import ClientQuicLayer +from mitmproxy.proxy.layers import ClientTLSLayer +from mitmproxy.proxy.layers import DNSLayer +from mitmproxy.proxy.layers import HttpLayer +from mitmproxy.proxy.layers import RawQuicLayer +from mitmproxy.proxy.layers import ServerQuicLayer +from mitmproxy.proxy.layers import ServerTLSLayer +from mitmproxy.proxy.layers import TCPLayer +from mitmproxy.proxy.layers import UDPLayer +from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers.http import HTTPMode +from mitmproxy.proxy.layers.http import HttpStream +from mitmproxy.proxy.mode_specs import ProxyMode from mitmproxy.test import taddons -from mitmproxy.test import tflow - - -@pytest.fixture -def tctx(): - context.Context( - connection.Client( - peername=("client", 1234), - sockname=("127.0.0.1", 8080), - timestamp_start=1605699329, - ), - tctx.options, - ) - client_hello_no_extensions = bytes.fromhex( "1603030065" # record header @@ -41,7 +48,6 @@ def tctx(): "170018" ) - dtls_client_hello_with_extensions = bytes.fromhex( "16fefd00000000000000000085" # record layer "010000790000000000000079" # handshake layer @@ -50,7 +56,6 @@ def tctx(): "7000000000010000e00000b6578616d706c652e636f6d" ) - quic_client_hello = bytes.fromhex( "ca0000000108c0618c84b54541320823fcce946c38d8210044e6a93bbb283593f75ffb6f2696b16cfdcb5b1255" "577b2af5fc5894188c9568bc65eef253faf7f0520e41341cfa81d6aae573586665ce4e1e41676364820402feec" @@ -83,6 +88,8 @@ def tctx(): "297c0013924e88248684fe8f2098326ce51aa6e5" ) +dns_query = bytes.fromhex("002a01000001000000000000076578616d706c6503636f6d0000010001") + class TestNextLayer: def test_configure(self): @@ -93,325 +100,510 @@ def test_configure(self): nl, allow_hosts=["example.org"], ignore_hosts=["example.com"] ) - def test_ignore_connection(self): - nl = NextLayer() - with taddons.context(nl) as tctx: - assert not nl.ignore_connection(("example.com", 443), b"") - - tctx.configure(nl, ignore_hosts=["example.com"]) - assert nl.ignore_connection(("example.com", 443), b"") - assert nl.ignore_connection(("example.com", 1234), b"") - assert nl.ignore_connection(("com", 443), b"") is False - assert nl.ignore_connection(None, b"") is False - assert nl.ignore_connection(None, client_hello_no_extensions) is False - assert nl.ignore_connection(None, client_hello_with_extensions) - assert nl.ignore_connection(None, client_hello_with_extensions[:-5]) is None - # invalid clienthello - assert ( - nl.ignore_connection( - None, client_hello_no_extensions[:9] + b"\x00" * 200 - ) - is False - ) - # different server name and SNI - assert nl.ignore_connection(("decoy", 1234), client_hello_with_extensions) - - tctx.configure(nl, ignore_hosts=[], allow_hosts=["example.com"]) - assert nl.ignore_connection(("example.com", 443), b"") is False - assert nl.ignore_connection(("example.org", 443), b"") - # different server name and SNI - assert ( - nl.ignore_connection(("decoy", 1234), client_hello_with_extensions) - is False - ) - - def test_next_layer(self, monkeypatch): - ctx = MagicMock() - ctx.client.transport_protocol = "tcp" - nl_layer = layer.NextLayer(ctx) - monkeypatch.setattr(nl_layer, "data_client", lambda: b"\x16\x03\x03") - nl = NextLayer() - - with taddons.context(nl): - nl.next_layer(nl_layer) - assert nl_layer.layer - - def test_next_layer2(self): - nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "tcp" - with taddons.context(nl) as tctx: - ctx.layers = [layers.modes.HttpProxy(ctx)] - - assert nl._next_layer(ctx, b"", b"") is None - - tctx.configure(nl, ignore_hosts=["example.com"]) - assert isinstance(nl._next_layer(ctx, b"123", b""), layers.TCPLayer) - assert nl._next_layer(ctx, client_hello_no_extensions[:10], b"") is None - - tctx.configure(nl, ignore_hosts=[]) - assert isinstance( - nl._next_layer(ctx, client_hello_no_extensions, b""), - layers.ServerTLSLayer, - ) - assert isinstance(ctx.layers[-1], layers.ClientTLSLayer) - - ctx.layers = [layers.modes.HttpProxy(ctx)] - assert isinstance( - nl._next_layer(ctx, client_hello_no_extensions, b""), - layers.ClientTLSLayer, - ) - - ctx.layers = [layers.modes.HttpProxy(ctx)] - assert isinstance( - nl._next_layer(ctx, b"GET http://example.com/ HTTP/1.1\r\n", b""), - layers.HttpLayer, - ) - assert ctx.layers[-1].mode == HTTPMode.regular - - ctx.layers = [layers.modes.HttpUpstreamProxy(ctx)] - assert isinstance( - nl._next_layer(ctx, b"GET http://example.com/ HTTP/1.1\r\n", b""), - layers.HttpLayer, - ) - assert ctx.layers[-1].mode == HTTPMode.upstream - - tctx.configure(nl, tcp_hosts=["example.com"]) - assert isinstance(nl._next_layer(ctx, b"123", b""), layers.TCPLayer) - - tctx.configure(nl, tcp_hosts=[]) - assert isinstance(nl._next_layer(ctx, b"GET /foo", b""), layers.HttpLayer) - assert isinstance(nl._next_layer(ctx, b"", b"hello"), layers.TCPLayer) - @pytest.mark.parametrize( - ("protocol", "client_layer", "server_layer"), + "ignore, allow, transport_protocol, server_address, data_client, result", [ - ("dtls", layers.ClientTLSLayer, layers.ServerTLSLayer), - ("quic", layers.ClientQuicLayer, layers.ServerQuicLayer), + # ignore + pytest.param( + [], [], "example.com", "tcp", b"", False, id="nothing ignored" + ), + pytest.param( + ["example.com"], [], "tcp", "example.com", b"", True, id="address" + ), + pytest.param( + ["1.2.3.4"], [], "tcp", "example.com", b"", True, id="ip address" + ), + pytest.param( + ["example.com"], + [], + "tcp", + "com", + b"", + False, + id="partial address match", + ), + pytest.param( + ["example.com"], [], "tcp", None, b"", False, id="no destination info" + ), + pytest.param( + ["example.com"], + [], + "tcp", + None, + client_hello_no_extensions, + False, + id="no sni", + ), + pytest.param( + ["example.com"], + [], + "tcp", + None, + client_hello_with_extensions, + True, + id="sni", + ), + pytest.param( + ["example.com"], + [], + "tcp", + None, + client_hello_with_extensions[:-5], + NeedsMoreData, + id="incomplete client hello", + ), + pytest.param( + ["example.com"], + [], + "tcp", + None, + client_hello_no_extensions[:9] + b"\x00" * 200, + False, + id="invalid client hello", + ), + pytest.param( + ["example.com"], + [], + "tcp", + "decoy", + client_hello_with_extensions, + True, + id="sni mismatch", + ), + pytest.param( + ["example.com"], + [], + "udp", + None, + dtls_client_hello_with_extensions, + True, + id="dtls sni", + ), + pytest.param( + ["example.com"], + [], + "udp", + None, + dtls_client_hello_with_extensions[:-5], + NeedsMoreData, + id="incomplete dtls client hello", + ), + pytest.param( + ["example.com"], + [], + "udp", + None, + dtls_client_hello_with_extensions[:9] + b"\x00" * 200, + False, + id="invalid dtls client hello", + ), + pytest.param( + ["example.com"], [], "udp", None, quic_client_hello, True, id="quic sni" + ), + # allow + pytest.param( + [], ["example.com"], "tcp", "example.com", b"", False, id="allow: allow" + ), + pytest.param( + [], ["example.com"], "tcp", "example.org", b"", True, id="allow: ignore" + ), + pytest.param( + [], + ["example.com"], + "tcp", + "decoy", + client_hello_with_extensions, + False, + id="allow: sni mismatch", + ), ], ) - def test_next_layer_udp( + def test_ignore_connection( self, - protocol: str, - client_layer: layer.Layer, - server_layer: layer.Layer, + ignore: list[str], + allow: list[str], + transport_protocol: TransportProtocol, + server_address: str, + data_client: bytes, + result: bool | type[NeedsMoreData], ): - def is_ignored_udp(layer: layer.Layer | None): - return isinstance(layer, layers.UDPLayer) and layer.flow is None - - def is_intercepted_udp(layer: layer.Layer | None): - return isinstance(layer, layers.UDPLayer) and layer.flow is not None - - def is_http(layer: layer.Layer | None, mode: HTTPMode): - return isinstance(layer, layers.HttpLayer) and layer.mode is mode - - client_hello = { - "dtls": dtls_client_hello_with_extensions, - "quic": quic_client_hello, - }[protocol] nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" with taddons.context(nl) as tctx: - ctx.layers = [layers.modes.HttpProxy(ctx), client_layer(ctx)] - assert is_http(nl._next_layer(ctx, b"", b""), HTTPMode.regular) - - ctx.layers = [layers.modes.HttpUpstreamProxy(ctx), client_layer(ctx)] - assert is_http(nl._next_layer(ctx, b"", b""), HTTPMode.upstream) - - ctx.layers = [layers.modes.TransparentProxy(ctx)] - is_intercepted_udp(nl._next_layer(ctx, b"", b"")) - - ctx.layers = [layers.modes.TransparentProxy(ctx)] - ctx.server.address = ("nomatch.com", 443) - tctx.configure(nl, ignore_hosts=["example.com"]) - assert is_intercepted_udp(nl._next_layer(ctx, client_hello[:50], b"")) - assert is_ignored_udp(nl._next_layer(ctx, client_hello, b"")) - - ctx.layers = [layers.modes.TransparentProxy(ctx)] - ctx.server.address = ("example.com", 443) - assert is_ignored_udp(nl._next_layer(ctx, client_hello[:50], b"")) - - ctx.layers = [layers.modes.TransparentProxy(ctx)] - tctx.configure(nl, ignore_hosts=[]) - decision = nl._next_layer(ctx, client_hello, b"") - assert isinstance(decision, server_layer) - assert isinstance(decision.child_layer, client_layer) - - ctx.layers = [layers.modes.ReverseProxy(ctx), server_layer(ctx)] - tctx.configure(nl, ignore_hosts=[]) - assert isinstance(nl._next_layer(ctx, client_hello, b""), client_layer) - - ctx.layers = [layers.modes.TransparentProxy(ctx)] - tctx.configure(nl, udp_hosts=["example.com"]) - assert isinstance( - nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.UDPLayer - ) - - ctx.layers = [layers.modes.TransparentProxy(ctx)] - tctx.configure(nl, udp_hosts=[]) - assert isinstance( - nl._next_layer(ctx, tflow.tdnsreq().packed, b""), layers.DNSLayer + if ignore: + tctx.configure(nl, ignore_hosts=ignore) + if allow: + tctx.configure(nl, allow_hosts=allow) + + ctx = Context( + Client(peername=("192.168.0.42", 51234), sockname=("0.0.0.0", 8080)), + tctx.options, ) - - def test_next_layer_reverse_raw(self): + ctx.client.transport_protocol = transport_protocol + if server_address: + ctx.server.address = (server_address, 443) + ctx.server.peername = ("1.2.3.4", 443) + + if result is NeedsMoreData: + with pytest.raises(NeedsMoreData): + nl._ignore_connection(ctx, data_client) + else: + assert nl._ignore_connection(ctx, data_client) is result + + def test_next_layer(self, monkeypatch, caplog): + caplog.set_level(logging.INFO) nl = NextLayer() - with taddons.context(nl): - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - with taddons.context(nl) as tctx: - tctx.configure(nl, ignore_hosts=["example.com"]) - - ctx.layers = [ - layers.modes.HttpProxy(ctx), - layers.ClientQuicLayer(ctx), - ] - decision = nl._next_layer(ctx, b"", b"") - assert isinstance(decision, layers.ServerQuicLayer) - assert isinstance(decision.child_layer, layers.RawQuicLayer) - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - layers.ClientQuicLayer( - ctx, - ), - ] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) + with taddons.context(nl) as tctx: + m = MagicMock() + m.context = Context( + Client(peername=("192.168.0.42", 51234), sockname=("0.0.0.0", 8080)), + tctx.options, + ) + m.context.layers = [modes.TransparentProxy(m.context)] + m.context.server.address = ("example.com", 42) + tctx.configure(nl, ignore_hosts=["example.com"]) - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - ] - decision = nl._next_layer(ctx, b"", b"") - assert isinstance(decision, layers.ClientQuicLayer) - assert isinstance(decision.child_layer, layers.RawQuicLayer) + m.layer = preexisting = object() + nl.next_layer(m) + assert m.layer is preexisting - tctx.configure(nl, ignore_hosts=[]) + m.layer = None + nl.next_layer(m) + assert m.layer - def test_next_layer_reverse_quic_mode(self): - nl = NextLayer() - with taddons.context(nl): - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "quic" - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - layers.ClientQuicLayer(ctx), - ] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.RawQuicLayer) - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - ] - assert nl._next_layer(ctx, b"", b"") is None - assert isinstance( - nl._next_layer(ctx, b"notahandshake", b""), layers.UDPLayer - ) - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - ] - assert isinstance( - nl._next_layer(ctx, quic_client_hello, b""), layers.ClientQuicLayer + m.layer = None + monkeypatch.setattr( + m, "data_client", lambda: client_hello_with_extensions[:-5] ) + nl.next_layer(m) + assert not m.layer + assert "Deferring layer decision" in caplog.text + + +@dataclass +class TestConf: + before: list[type[Layer]] + after: list[type[Layer]] + proxy_mode: str = "regular" + transport_protocol: TransportProtocol = "tcp" + data_client: bytes = b"" + data_server: bytes = b"" + ignore_hosts: Sequence[str] = () + tcp_hosts: Sequence[str] = () + udp_hosts: Sequence[str] = () + ignore_conn: bool = False + + +explicit_proxy_configs = [ + pytest.param( + TestConf( + before=[modes.HttpProxy], + after=[modes.HttpProxy, HttpLayer], + ), + id=f"explicit proxy: regular http", + ), + pytest.param( + TestConf( + before=[modes.HttpProxy], + after=[modes.HttpProxy, ClientTLSLayer, HttpLayer], + data_client=client_hello_no_extensions, + ), + id=f"explicit proxy: secure web proxy", + ), + pytest.param( + TestConf( + before=[modes.HttpUpstreamProxy], + after=[modes.HttpUpstreamProxy, HttpLayer], + ), + id=f"explicit proxy: upstream proxy", + ), + pytest.param( + TestConf( + before=[modes.HttpUpstreamProxy], + after=[modes.HttpUpstreamProxy, ClientQuicLayer, HttpLayer], + transport_protocol="udp", + ), + id=f"explicit proxy: experimental http3", + ), + pytest.param( + TestConf( + before=[ + modes.HttpProxy, + partial(HttpLayer, mode=HTTPMode.regular), + partial(HttpStream, stream_id=1), + ], + after=[modes.HttpProxy, HttpLayer, HttpStream, HttpLayer], + data_client=b"GET / HTTP/1.1\r\n", + ), + id=f"explicit proxy: HTTP over regular proxy", + ), + pytest.param( + TestConf( + before=[ + modes.HttpProxy, + partial(HttpLayer, mode=HTTPMode.regular), + partial(HttpStream, stream_id=1), + ], + after=[ + modes.HttpProxy, + HttpLayer, + HttpStream, + ServerTLSLayer, + ClientTLSLayer, + ], + data_client=client_hello_with_extensions, + ), + id=f"explicit proxy: TLS over regular proxy", + ), + pytest.param( + TestConf( + before=[ + modes.HttpProxy, + partial(HttpLayer, mode=HTTPMode.regular), + partial(HttpStream, stream_id=1), + ServerTLSLayer, + ClientTLSLayer, + ], + after=[ + modes.HttpProxy, + HttpLayer, + HttpStream, + ServerTLSLayer, + ClientTLSLayer, + HttpLayer, + ], + data_client=b"GET / HTTP/1.1\r\n", + ), + id=f"explicit proxy: HTTPS over regular proxy", + ), + pytest.param( + TestConf( + before=[ + modes.HttpProxy, + partial(HttpLayer, mode=HTTPMode.regular), + partial(HttpStream, stream_id=1), + ], + after=[modes.HttpProxy, HttpLayer, HttpStream, TCPLayer], + data_client=b"\xFF", + ), + id=f"explicit proxy: TCP over regular proxy", + ), +] + +reverse_proxy_configs = [] +for proto_plain, proto_enc, app_layer in [ + ("udp", "dtls", UDPLayer), + ("tcp", "tls", TCPLayer), + ("http", "https", HttpLayer), +]: + if proto_plain == "udp": + data_client = dtls_client_hello_with_extensions + else: + data_client = client_hello_with_extensions + + reverse_proxy_configs.extend( + [ + pytest.param( + TestConf( + before=[modes.ReverseProxy], + after=[modes.ReverseProxy, app_layer], + proxy_mode=f"reverse:{proto_plain}://example.com:42", + ), + id=f"reverse proxy: {proto_plain} -> {proto_plain}", + ), + pytest.param( + TestConf( + before=[modes.ReverseProxy], + after=[ + modes.ReverseProxy, + ServerTLSLayer, + ClientTLSLayer, + app_layer, + ], + proxy_mode=f"reverse:{proto_enc}://example.com:42", + data_client=data_client, + ), + id=f"reverse proxy: {proto_enc} -> {proto_enc}", + ), + pytest.param( + TestConf( + before=[modes.ReverseProxy], + after=[modes.ReverseProxy, ClientTLSLayer, app_layer], + proxy_mode=f"reverse:{proto_plain}://example.com:42", + data_client=data_client, + ), + id=f"reverse proxy: {proto_enc} -> {proto_plain}", + ), + pytest.param( + TestConf( + before=[modes.ReverseProxy], + after=[modes.ReverseProxy, ServerTLSLayer, app_layer], + proxy_mode=f"reverse:{proto_enc}://example.com:42", + ), + id=f"reverse proxy: {proto_plain} -> {proto_enc}", + ), + ] + ) - def test_next_layer_reverse_http3_mode(self): - nl = NextLayer() - with taddons.context(nl): - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "http3" - ctx.layers = [ - layers.modes.ReverseProxy(ctx), - layers.ServerQuicLayer(ctx), - ] - assert isinstance( - nl._next_layer(ctx, b"notahandshakebutignore", b""), - layers.ClientQuicLayer, - ) - assert len(ctx.layers) == 3 - decision = nl._next_layer(ctx, b"", b"") - assert isinstance(decision, layers.HttpLayer) - assert decision.mode is HTTPMode.transparent - - def test_next_layer_reverse_invalid_mode(self): - nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "invalidscheme" - ctx.layers = [layers.modes.ReverseProxy(ctx)] - with pytest.raises(AssertionError, match="invalidscheme"): - nl._next_layer(ctx, b"", b"") +reverse_proxy_configs.extend( + [ + pytest.param( + TestConf( + before=[modes.ReverseProxy], + after=[modes.ReverseProxy, DNSLayer], + proxy_mode="reverse:dns://example.com:53", + ), + id="reverse proxy: dns", + ), + pytest.param( + TestConf( + before=[modes.ReverseProxy], + after=[modes.ReverseProxy, ServerQuicLayer, ClientQuicLayer, HttpLayer], + proxy_mode="reverse:http3://example.com", + ), + id="reverse proxy: http3", + ), + pytest.param( + TestConf( + before=[modes.ReverseProxy], + after=[ + modes.ReverseProxy, + ServerQuicLayer, + ClientQuicLayer, + RawQuicLayer, + ], + proxy_mode="reverse:quic://example.com", + ), + id="reverse proxy: quic", + ), + ] +) - def test_next_layer_reverse_dtls_mode(self): - nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "dtls" - ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx)] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) - ctx.layers = [layers.modes.ReverseProxy(ctx), layers.ServerTLSLayer(ctx)] - assert isinstance( - nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), - layers.ClientTLSLayer, +transparent_proxy_configs = [ + pytest.param( + TestConf( + before=[modes.TransparentProxy], + after=[modes.TransparentProxy, ServerTLSLayer, ClientTLSLayer], + data_client=client_hello_no_extensions, + ), + id=f"transparent proxy: tls", + ), + pytest.param( + TestConf( + before=[modes.TransparentProxy], + after=[modes.TransparentProxy, ServerTLSLayer, ClientTLSLayer], + data_client=dtls_client_hello_with_extensions, + transport_protocol="udp", + ), + id=f"transparent proxy: dtls", + ), + pytest.param( + TestConf( + before=[modes.TransparentProxy], + after=[modes.TransparentProxy, ServerQuicLayer, ClientQuicLayer], + data_client=quic_client_hello, + transport_protocol="udp", + ), + id="transparent proxy: quic", + ), + pytest.param( + TestConf( + before=[modes.TransparentProxy], + after=[modes.TransparentProxy, TCPLayer], + data_server=b"220 service ready", + ), + id="transparent proxy: raw tcp", + ), + pytest.param( + http := TestConf( + before=[modes.TransparentProxy], + after=[modes.TransparentProxy, HttpLayer], + data_client=b"GET / HTTP/1.1\r\n", + ), + id="transparent proxy: http", + ), + pytest.param( + dataclasses.replace( + http, + tcp_hosts=["example.com"], + after=[modes.TransparentProxy, TCPLayer], + ), + id="transparent proxy: tcp_hosts", + ), + pytest.param( + dataclasses.replace( + http, + ignore_hosts=["example.com"], + after=[modes.TransparentProxy, TCPLayer], + ignore_conn=True, + ), + id="transparent proxy: ignore_hosts", + ), + pytest.param( + dns := TestConf( + before=[modes.TransparentProxy], + after=[modes.TransparentProxy, DNSLayer], + transport_protocol="udp", + data_client=dns_query, + ), + id="transparent proxy: dns", + ), + pytest.param( + TestConf( + before=[modes.TransparentProxy], + after=[modes.TransparentProxy, UDPLayer], + transport_protocol="udp", + data_client=b"\xFF", + ), + id="transparent proxy: raw udp", + ), + pytest.param( + dataclasses.replace( + dns, + udp_hosts=["example.com"], + after=[modes.TransparentProxy, UDPLayer], + ), + id="transparent proxy: udp_hosts", + ), +] + + +@pytest.mark.parametrize( + "test_conf", + [ + *explicit_proxy_configs, + *reverse_proxy_configs, + *transparent_proxy_configs, + ], +) +def test_next_layer( + test_conf: TestConf, +): + nl = NextLayer() + with taddons.context(nl) as tctx: + tctx.configure( + nl, + ignore_hosts=test_conf.ignore_hosts, + tcp_hosts=test_conf.tcp_hosts, + udp_hosts=test_conf.udp_hosts, ) - assert len(ctx.layers) == 3 - assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) - def test_next_layer_reverse_udp_mode(self): - nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "udp" - ctx.layers = [layers.modes.ReverseProxy(ctx)] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) - ctx.layers = [layers.modes.ReverseProxy(ctx)] - assert isinstance( - nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), - layers.ClientTLSLayer, + ctx = Context( + Client(peername=("192.168.0.42", 51234), sockname=("0.0.0.0", 8080)), + tctx.options, ) - assert len(ctx.layers) == 2 - assert isinstance(nl._next_layer(ctx, b"", b""), layers.UDPLayer) - - def test_next_layer_reverse_dns_mode(self): - nl = NextLayer() - ctx = MagicMock() - ctx.client.alpn = None - ctx.server.address = ("example.com", 443) - ctx.client.transport_protocol = "udp" - ctx.client.proxy_mode.scheme = "dns" - ctx.layers = [layers.modes.ReverseProxy(ctx)] - assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) - ctx.layers = [layers.modes.ReverseProxy(ctx)] - assert isinstance( - nl._next_layer(ctx, dtls_client_hello_with_extensions, b""), - layers.ClientTLSLayer, + ctx.server.address = ("example.com", 42) + # these aren't properly set up, but this does not matter here. + ctx.client.transport_protocol = test_conf.transport_protocol + ctx.client.proxy_mode = ProxyMode.parse(test_conf.proxy_mode) + ctx.layers = [x(ctx) for x in test_conf.before] + nl._next_layer( + ctx, + data_client=test_conf.data_client, + data_server=test_conf.data_server, ) - assert len(ctx.layers) == 2 - assert isinstance(nl._next_layer(ctx, b"", b""), layers.DNSLayer) + assert stack_match(ctx, test_conf.after) - def test_next_layer_invalid_proto(self): - nl = NextLayer() - ctx = MagicMock() - ctx.client.transport_protocol = "invalid" - with taddons.context(nl): - with pytest.raises(AssertionError): - nl._next_layer(ctx, b"", b"") + last_layer = ctx.layers[-1] + if isinstance(last_layer, (UDPLayer, TCPLayer)): + assert bool(last_layer.flow) ^ test_conf.ignore_conn diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index beca215e4c..45acafe877 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -27,6 +27,7 @@ from mitmproxy import dns from mitmproxy import exceptions from mitmproxy.addons import dns_resolver +from mitmproxy.addons import next_layer from mitmproxy.addons.next_layer import NextLayer from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.addons.tlsconfig import TlsConfig @@ -50,10 +51,6 @@ class HelperAddon: def __init__(self): self.flows = [] - self.layers = [ - lambda ctx: layers.HttpLayer(ctx, HTTPMode.regular), - lambda ctx: layers.TCPLayer(ctx), - ] def request(self, f): self.flows.append(f) @@ -61,9 +58,6 @@ def request(self, f): def tcp_start(self, f): self.flows.append(f) - def next_layer(self, nl): - nl.layer = self.layers.pop(0)(nl.context) - @asynccontextmanager async def tcp_server(handle_conn) -> Address: @@ -87,9 +81,10 @@ async def server_handler( writer.close() ps = Proxyserver() - with taddons.context(ps) as tctx: - state = HelperAddon() - tctx.master.addons.add(state) + nl = NextLayer() + state = HelperAddon() + + with taddons.context(ps, nl, state) as tctx: async with tcp_server(server_handler) as addr: tctx.configure(ps, listen_host="127.0.0.1", listen_port=0) assert not ps.servers @@ -142,9 +137,10 @@ async def server_handler( writer.write(s.upper()) ps = Proxyserver() - with taddons.context(ps) as tctx: - state = HelperAddon() - tctx.master.addons.add(state) + nl = NextLayer() + state = HelperAddon() + + with taddons.context(ps, nl, state) as tctx: async with tcp_server(server_handler) as addr: tctx.configure(ps, listen_host="127.0.0.1", listen_port=0) assert await ps.setup_servers() @@ -347,7 +343,7 @@ async def udp_server(handle_conn) -> Address: server.close() -async def test_dtls(monkeypatch, caplog_async) -> None: +async def test_udp(caplog_async) -> None: caplog_async.set_level("INFO") def server_handler( @@ -360,20 +356,16 @@ def server_handler( transport.sendto(b"\x01", remote_addr) ps = Proxyserver() + nl = NextLayer() - # We just want to relay the messages and skip the handshake. - monkeypatch.setattr(tls, "ServerTLSLayer", layers.UDPLayer) - - with taddons.context(ps) as tctx: - state = HelperAddon() - tctx.master.addons.add(state) + with taddons.context(ps, nl) as tctx: async with udp_server(server_handler) as server_addr: - mode = f"reverse:dtls://{server_addr[0]}:{server_addr[1]}@127.0.0.1:0" + mode = f"reverse:udp://{server_addr[0]}:{server_addr[1]}@127.0.0.1:0" tctx.configure(ps, mode=[mode]) assert await ps.setup_servers() ps.running() await caplog_async.await_log( - f"reverse proxy to dtls://{server_addr[0]}:{server_addr[1]} listening" + f"reverse proxy to udp://{server_addr[0]}:{server_addr[1]} listening" ) assert ps.servers addr = ps.servers[mode].listen_addrs[0] @@ -605,9 +597,9 @@ def quic_event_received(self, event: quic_events.QuicEvent) -> None: def request( self, - headers: h3_events.H3Event, + headers: h3_events.Headers, data: bytes | None = None, - trailers: h3_events.H3Event | None = None, + trailers: h3_events.Headers | None = None, end_stream: bool = True, ) -> H3Response: stream_id = self._quic.get_next_available_stream_id() diff --git a/test/mitmproxy/contentviews/test_http3.py b/test/mitmproxy/contentviews/test_http3.py index d1b9fc6413..0ffc9c1115 100644 --- a/test/mitmproxy/contentviews/test_http3.py +++ b/test/mitmproxy/contentviews/test_http3.py @@ -5,9 +5,6 @@ from mitmproxy.tcp import TCPMessage from mitmproxy.test import tflow -if http3 is None: - pytest.skip("HTTP/3 not available.", allow_module_level=True) - @pytest.mark.parametrize( "data", diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index 60f1287b16..c6cd5a5534 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -72,19 +72,20 @@ def test_sslkeylogfile(tdata, monkeypatch): def test_is_record_magic(): assert not tls.starts_like_tls_record(b"POST /") assert not tls.starts_like_tls_record(b"\x16\x03\x04") - assert tls.starts_like_tls_record(b"") - assert tls.starts_like_tls_record(b"\x16") - assert tls.starts_like_tls_record(b"\x16\x03") + assert not tls.starts_like_tls_record(b"") + assert not tls.starts_like_tls_record(b"\x16") + assert not tls.starts_like_tls_record(b"\x16\x03") assert tls.starts_like_tls_record(b"\x16\x03\x00") assert tls.starts_like_tls_record(b"\x16\x03\x01") assert tls.starts_like_tls_record(b"\x16\x03\x02") assert tls.starts_like_tls_record(b"\x16\x03\x03") + assert not tls.starts_like_tls_record(bytes.fromhex("16fefe")) def test_is_dtls_record_magic(): - assert tls.starts_like_dtls_record(bytes.fromhex("")) - assert tls.starts_like_dtls_record(bytes.fromhex("16")) - assert tls.starts_like_dtls_record(bytes.fromhex("16fe")) + assert not tls.starts_like_dtls_record(bytes.fromhex("")) + assert not tls.starts_like_dtls_record(bytes.fromhex("16")) + assert not tls.starts_like_dtls_record(bytes.fromhex("16fe")) assert tls.starts_like_dtls_record(bytes.fromhex("16fefd")) assert tls.starts_like_dtls_record(bytes.fromhex("16fefe")) assert not tls.starts_like_dtls_record(bytes.fromhex("160300")) diff --git a/test/mitmproxy/proxy/layers/test_modes.py b/test/mitmproxy/proxy/layers/test_modes.py index 6d8040e3b0..0b5aa93ebf 100644 --- a/test/mitmproxy/proxy/layers/test_modes.py +++ b/test/mitmproxy/proxy/layers/test_modes.py @@ -207,6 +207,9 @@ def set_settings(data: quic.QuicTlsData): Playbook(modes.ReverseProxy(tctx)) << OpenConnection(tctx.server) >> reply(None) + >> DataReceived(tctx.client, b"\x00") + << NextLayerHook(Placeholder(NextLayer)) + >> reply_next_layer(layers.ServerQuicLayer) << quic.QuicStartServerHook(Placeholder(quic.QuicTlsData)) >> reply(side_effect=set_settings) << SendData(tctx.server, client_hello) @@ -250,9 +253,6 @@ def test_reverse_proxy_tcp_over_tls( reverse proxying. """ - if patch: - monkeypatch.setattr(tls, "ServerTLSLayer", tls.MockTLSLayer) - flow = Placeholder(TCPFlow) data = Placeholder(bytes) tctx.client.proxy_mode = ProxyMode.parse("reverse:https://localhost:8000") @@ -287,6 +287,11 @@ def test_reverse_proxy_tcp_over_tls( ) assert data() == b"\x01\x02\x03" else: + ( + playbook + << NextLayerHook(Placeholder(NextLayer)) + >> reply_next_layer(tls.ServerTLSLayer) + ) if connection_strategy == "lazy": ( playbook diff --git a/test/mitmproxy/proxy/layers/test_tls.py b/test/mitmproxy/proxy/layers/test_tls.py index 2cc36a8484..9960f8b748 100644 --- a/test/mitmproxy/proxy/layers/test_tls.py +++ b/test/mitmproxy/proxy/layers/test_tls.py @@ -765,15 +765,6 @@ def test_unsupported_protocol(self, tctx: context.Context): assert tls_hook_data().conn.error -def test_is_dtls_handshake_record(): - assert tls.is_dtls_handshake_record(bytes.fromhex("16fefd")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("160300")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("16fefe")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("160304")) - assert not tls.is_dtls_handshake_record(bytes.fromhex("150301")) - - def test_dtls_record_contents(): data = bytes.fromhex( "16fefd00000000000000000002beef" "16fefd00000000000000000001ff" From f9d808562d83dd05335c0e208b7a1b449b8ea8d9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Mar 2023 18:30:51 +0100 Subject: [PATCH 207/695] add option to disable HTTP/3 --- mitmproxy/options.py | 6 ++++++ mitmproxy/proxy/layers/quic.py | 10 +++++++++- test/mitmproxy/proxy/layers/test_quic.py | 12 ++++++++++++ web/src/js/ducks/_options_gen.ts | 2 ++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 1e9f2404be..6ef97db44a 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -143,6 +143,12 @@ def __init__(self, **kwargs) -> None: Set to 0 to disable this feature. """, ) + self.add_option( + "http3", + bool, + True, + "Enable/disable support for QUIC and HTTP/3. Enabled by default.", + ) self.add_option( "websocket", bool, diff --git a/mitmproxy/proxy/layers/quic.py b/mitmproxy/proxy/layers/quic.py index d20924c769..d4434ca00d 100644 --- a/mitmproxy/proxy/layers/quic.py +++ b/mitmproxy/proxy/layers/quic.py @@ -1119,7 +1119,9 @@ def __init__( context.client.cipher_list = [] super().__init__(context, context.client, time) - self.server_tls_available = isinstance(self.context.layers[-2], ServerQuicLayer) + self.server_tls_available = len(self.context.layers) >= 2 and isinstance( + self.context.layers[-2], ServerQuicLayer + ) def start_handshake(self) -> layer.CommandGenerator[None]: yield from () @@ -1127,6 +1129,12 @@ def start_handshake(self) -> layer.CommandGenerator[None]: def receive_handshake_data( self, data: bytes ) -> layer.CommandGenerator[tuple[bool, str | None]]: + if not self.context.options.http3: + yield commands.Log( + f"Swallowing QUIC handshake because HTTP/3 is disabled.", DEBUG + ) + return False, None + # if we already had a valid client hello, don't process further packets if self.tls: return (yield from super().receive_handshake_data(data)) diff --git a/test/mitmproxy/proxy/layers/test_quic.py b/test/mitmproxy/proxy/layers/test_quic.py index e68222ff8d..83177d8a1f 100644 --- a/test/mitmproxy/proxy/layers/test_quic.py +++ b/test/mitmproxy/proxy/layers/test_quic.py @@ -889,6 +889,18 @@ def make_client_tls_layer( class TestClientTLS: + def test_http3_disabled(self, tctx: context.Context): + """Test that we swallow QUIC packets if QUIC and HTTP/3 are disabled.""" + tctx.options.http3 = False + assert ( + tutils.Playbook(quic.ClientQuicLayer(tctx, time=time.time), logs=True) + >> events.DataReceived(tctx.client, client_hello) + << commands.Log( + "Swallowing QUIC handshake because HTTP/3 is disabled.", DEBUG + ) + << None + ) + def test_client_only(self, tctx: context.Context): """Test TLS with client only""" playbook, client_layer, tssl_client = make_client_tls_layer(tctx) diff --git a/web/src/js/ducks/_options_gen.ts b/web/src/js/ducks/_options_gen.ts index 8c93d91dbf..3e9d3a4106 100644 --- a/web/src/js/ducks/_options_gen.ts +++ b/web/src/js/ducks/_options_gen.ts @@ -24,6 +24,7 @@ export interface OptionsState { export_preserve_original_ip: boolean http2: boolean http2_ping_keepalive: number + http3: boolean ignore_hosts: string[] intercept: string | undefined intercept_active: boolean @@ -114,6 +115,7 @@ export const defaultState: OptionsState = { export_preserve_original_ip: false, http2: true, http2_ping_keepalive: 58, + http3: true, ignore_hosts: [], intercept: undefined, intercept_active: false, From aa61f70b5388063b20011ad518fd12977101dfae Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Mar 2023 18:35:37 +0100 Subject: [PATCH 208/695] update CHANGELOG --- CHANGELOG.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 823cc356bb..5924609963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,15 +2,12 @@ ## Unreleased: mitmproxy next -* mitmproxy now requires Python 3.10 or above. - ([#5954](https://github.com/mitmproxy/mitmproxy/pull/5954), @mhils) -* Fix a bug where the direction indicator in the message stream view would be in the wrong direction. - ([#5921](https://github.com/mitmproxy/mitmproxy/issues/5921), @konradh) -* Fix a bug where peername would be None in tls_passthrough script, which would make it not working. - ([#5904](https://github.com/mitmproxy/mitmproxy/pull/5904), @truebit) -* Add experimental QUIC support. +* Add experimental support for HTTP/3 and QUIC. ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) +* You can now set the `http3` to false to block all QUIC and HTTP/3 traffic + in transparent modes. + ([#5967](https://github.com/mitmproxy/mitmproxy/pull/5967), @mhils) * ASGI/WSGI apps can now listen on all ports for a specific hostname. This makes it simpler to accept both HTTP and HTTPS. ([#5725](https://github.com/mitmproxy/mitmproxy/pull/5725), @mhils) @@ -25,6 +22,12 @@ ([#5148](https://github.com/mitmproxy/mitmproxy/issues/5148), @mhils) * Added documentation on using Magisk module for intercepting traffic in Android production builds. ([#5924](https://github.com/mitmproxy/mitmproxy/pull/5924), @Jurrie) +* Fix a bug where the direction indicator in the message stream view would be in the wrong direction. + ([#5921](https://github.com/mitmproxy/mitmproxy/issues/5921), @konradh) +* Fix a bug where peername would be None in tls_passthrough script, which would make it not working. + ([#5904](https://github.com/mitmproxy/mitmproxy/pull/5904), @truebit) +* mitmproxy now requires Python 3.10 or above. + ([#5954](https://github.com/mitmproxy/mitmproxy/pull/5954), @mhils) ### Breaking Changes From ed740e72bf4f305ca6fc0753538cb1378635beb4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 5 Mar 2023 18:53:51 +0100 Subject: [PATCH 209/695] fix Python 3.10 compat --- mitmproxy/addons/next_layer.py | 15 ++++++++++----- mitmproxy/connection.py | 5 ----- mitmproxy/proxy/layers/modes.py | 14 +++++++------- setup.py | 2 +- test/mitmproxy/addons/test_next_layer.py | 4 ++-- test/mitmproxy/addons/test_proxyserver.py | 3 --- 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index c26e3f6bdc..81244b2c18 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -19,10 +19,10 @@ import logging import re import struct +import sys from collections.abc import Iterable from collections.abc import Sequence from typing import Any -from typing import assert_never from typing import cast from mitmproxy import ctx @@ -40,19 +40,24 @@ from mitmproxy.proxy.layers import ClientTLSLayer from mitmproxy.proxy.layers import DNSLayer from mitmproxy.proxy.layers import HttpLayer +from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers import RawQuicLayer from mitmproxy.proxy.layers import ServerQuicLayer from mitmproxy.proxy.layers import ServerTLSLayer from mitmproxy.proxy.layers import TCPLayer from mitmproxy.proxy.layers import UDPLayer -from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy.layers.quic import quic_parse_client_hello -from mitmproxy.proxy.layers.tls import HTTP_ALPNS from mitmproxy.proxy.layers.tls import dtls_parse_client_hello +from mitmproxy.proxy.layers.tls import HTTP_ALPNS from mitmproxy.proxy.layers.tls import parse_client_hello from mitmproxy.tls import ClientHello +if sys.version_info < (3, 11): + from typing_extensions import assert_never +else: + from typing import assert_never + logger = logging.getLogger(__name__) @@ -273,7 +278,7 @@ def _get_client_hello( raise NeedsMoreData return ch return None - case _: + case _: # pragma: no cover assert_never(context.client.transport_protocol) def _setup_reverse_proxy(self, context: Context, data_client: bytes) -> Layer: @@ -331,7 +336,7 @@ def _setup_reverse_proxy(self, context: Context, data_client: bytes) -> Layer: stack /= ClientQuicLayer(context) stack /= RawQuicLayer(context) - case _: + case _: # pragma: no cover assert_never(spec.scheme) return stack[0] diff --git a/mitmproxy/connection.py b/mitmproxy/connection.py index 46bb63b327..0ced6b8a63 100644 --- a/mitmproxy/connection.py +++ b/mitmproxy/connection.py @@ -1,5 +1,4 @@ import dataclasses -import sys import time import uuid import warnings @@ -260,10 +259,6 @@ class Server(Connection): address: Address | None # type: ignore """The server's `(host, port)` address tuple. The host can either be a domain or a plain IP address.""" - if sys.version_info < (3, 10): # pragma: no cover - # no keyword-only arguments here. - address: Address | None = None - peername: Address | None = None """ The server's resolved `(ip, port)` tuple. Will be set during connection establishment. diff --git a/mitmproxy/proxy/layers/modes.py b/mitmproxy/proxy/layers/modes.py index 4499296a5f..0f917cd78b 100644 --- a/mitmproxy/proxy/layers/modes.py +++ b/mitmproxy/proxy/layers/modes.py @@ -2,24 +2,24 @@ import socket import struct +import sys from abc import ABCMeta from collections.abc import Callable from dataclasses import dataclass -from typing import assert_never from mitmproxy import connection from mitmproxy.proxy import commands from mitmproxy.proxy import events from mitmproxy.proxy import layer -from mitmproxy.proxy import tunnel from mitmproxy.proxy.commands import StartHook -from mitmproxy.proxy.layers import dns -from mitmproxy.proxy.layers import quic -from mitmproxy.proxy.layers import tls -from mitmproxy.proxy.layers import http from mitmproxy.proxy.mode_specs import ReverseMode from mitmproxy.proxy.utils import expect +if sys.version_info < (3, 11): + from typing_extensions import assert_never +else: + from typing import assert_never + class HttpProxy(layer.Layer): @expect(events.Start) @@ -78,7 +78,7 @@ def _handle_event(self, event: events.Event) -> layer.CommandGenerator[None]: self.context.server.sni = spec.address[0] case "tcp" | "http" | "udp" | "dns": pass - case _: + case _: # pragma: no cover assert_never(spec.scheme) err = yield from self.finish_start() diff --git a/setup.py b/setup.py index 0859da179e..110d05dce4 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ "wsproto>=1.0,<1.3", "publicsuffix2>=2.20190812,<3", "zstandard>=0.11,<0.21", - "typing-extensions>=4.3,<4.5; python_version<'3.10'", + "typing-extensions>=4.3,<4.6; python_version<'3.11'", ], extras_require={ ':sys_platform == "win32"': [ diff --git a/test/mitmproxy/addons/test_next_layer.py b/test/mitmproxy/addons/test_next_layer.py index 1698dcd892..6bd9c85115 100644 --- a/test/mitmproxy/addons/test_next_layer.py +++ b/test/mitmproxy/addons/test_next_layer.py @@ -2,9 +2,9 @@ import dataclasses import logging +from collections.abc import Sequence from dataclasses import dataclass from functools import partial -from typing import Sequence from unittest.mock import MagicMock import pytest @@ -20,12 +20,12 @@ from mitmproxy.proxy.layers import ClientTLSLayer from mitmproxy.proxy.layers import DNSLayer from mitmproxy.proxy.layers import HttpLayer +from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers import RawQuicLayer from mitmproxy.proxy.layers import ServerQuicLayer from mitmproxy.proxy.layers import ServerTLSLayer from mitmproxy.proxy.layers import TCPLayer from mitmproxy.proxy.layers import UDPLayer -from mitmproxy.proxy.layers import modes from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.proxy.layers.http import HttpStream from mitmproxy.proxy.mode_specs import ProxyMode diff --git a/test/mitmproxy/addons/test_proxyserver.py b/test/mitmproxy/addons/test_proxyserver.py index 45acafe877..fdf749962e 100644 --- a/test/mitmproxy/addons/test_proxyserver.py +++ b/test/mitmproxy/addons/test_proxyserver.py @@ -27,7 +27,6 @@ from mitmproxy import dns from mitmproxy import exceptions from mitmproxy.addons import dns_resolver -from mitmproxy.addons import next_layer from mitmproxy.addons.next_layer import NextLayer from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.addons.tlsconfig import TlsConfig @@ -35,8 +34,6 @@ from mitmproxy.net import udp from mitmproxy.proxy import layers from mitmproxy.proxy import server_hooks -from mitmproxy.proxy.layers import tls -from mitmproxy.proxy.layers.http import HTTPMode from mitmproxy.test import taddons from mitmproxy.test import tflow from mitmproxy.test.tflow import tclient_conn From 2b10e3f3a79aef4e4c077a4ea056ebb3bb493ddb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Mar 2023 13:14:12 +0100 Subject: [PATCH 210/695] cleanups and mypy 1.1.1 fixes (#5977) --- mitmproxy/certs.py | 13 +++++-------- mitmproxy/coretypes/serializable.py | 2 +- mitmproxy/hooks.py | 2 +- mitmproxy/proxy/mode_servers.py | 12 +++++++----- mitmproxy/proxy/mode_specs.py | 10 ++++++---- setup.cfg | 2 +- 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index ce6d9443c4..7477c61f78 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -6,6 +6,7 @@ import sys from dataclasses import dataclass from pathlib import Path +from typing import cast from typing import NewType from typing import Optional from typing import Union @@ -134,7 +135,7 @@ def keyinfo(self) -> tuple[str, int]: def cn(self) -> str | None: attrs = self._cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME) if attrs: - return attrs[0].value + return cast(str, attrs[0].value) return None @property @@ -143,7 +144,7 @@ def organization(self) -> str | None: x509.NameOID.ORGANIZATION_NAME ) if attrs: - return attrs[0].value + return cast(str, attrs[0].value) return None @property @@ -166,12 +167,8 @@ def altnames(self) -> list[str]: def _name_to_keyval(name: x509.Name) -> list[tuple[str, str]]: parts = [] for attr in name: - # pyca cryptography <35.0.0 backwards compatiblity - if hasattr(name, "rfc4514_attribute_name"): # pragma: no cover - k = attr.rfc4514_attribute_name # type: ignore - else: # pragma: no cover - k = attr.rfc4514_string().partition("=")[0] - v = attr.value + k = attr.rfc4514_string().partition("=")[0] + v = cast(str, attr.value) parts.append((k, v)) return parts diff --git a/mitmproxy/coretypes/serializable.py b/mitmproxy/coretypes/serializable.py index f91d202c8b..fc185e3520 100644 --- a/mitmproxy/coretypes/serializable.py +++ b/mitmproxy/coretypes/serializable.py @@ -69,7 +69,7 @@ def __fields(cls) -> tuple[dataclasses.Field, ...]: hints = typing.get_type_hints(cls) fields = [] # noinspection PyDataclass - for field in dataclasses.fields(cls): + for field in dataclasses.fields(cls): # type: ignore[arg-type] if field.metadata.get("serialize", True) is False: continue if isinstance(field.type, str): diff --git a/mitmproxy/hooks.py b/mitmproxy/hooks.py index 4acb926691..870c6c208f 100644 --- a/mitmproxy/hooks.py +++ b/mitmproxy/hooks.py @@ -20,7 +20,7 @@ class Hook: def args(self) -> list[Any]: args = [] - for field in fields(self): + for field in fields(self): # type: ignore[arg-type] args.append(getattr(self, field.name)) return args diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index 405348e6b8..43213f91b2 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -17,6 +17,7 @@ import logging import os import socket +import sys import textwrap import typing from abc import ABCMeta @@ -46,6 +47,11 @@ from mitmproxy.proxy.layer import Layer from mitmproxy.utils import human +if sys.version_info < (3, 11): + from typing_extensions import Self # pragma: no cover +else: + from typing import Self + logger = logging.getLogger(__name__) @@ -79,10 +85,6 @@ def register_connection( ... # pragma: no cover -# Python 3.11: Use typing.Self -Self = TypeVar("Self", bound="ServerInstance") - - class ServerInstance(Generic[M], metaclass=ABCMeta): __modes: ClassVar[dict[str, type[ServerInstance]]] = {} @@ -103,7 +105,7 @@ def __init_subclass__(cls, **kwargs): @classmethod def make( - cls: type[Self], + cls, mode: mode_specs.ProxyMode | str, manager: ServerManager, ) -> Self: diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index 8bffb3b532..281c81d2e5 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -22,21 +22,23 @@ from __future__ import annotations import dataclasses +import sys from abc import ABCMeta from abc import abstractmethod from dataclasses import dataclass from functools import cache from typing import ClassVar from typing import Literal -from typing import TypeVar import mitmproxy_rs from mitmproxy.coretypes.serializable import Serializable from mitmproxy.net import server_spec -# Python 3.11: Use typing.Self -Self = TypeVar("Self", bound="ProxyMode") +if sys.version_info < (3, 11): + from typing_extensions import Self # pragma: no cover +else: + from typing import Self @dataclass(frozen=True) # type: ignore @@ -92,7 +94,7 @@ def transport_protocol(self) -> Literal["tcp", "udp"] | None: @classmethod @cache - def parse(cls: type[Self], spec: str) -> Self: + def parse(cls, spec: str) -> Self: """ Parse a proxy mode specification and return the corresponding `ProxyMode` instance. """ diff --git a/setup.cfg b/setup.cfg index 46f9890247..df4764b938 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ exclude_lines = [mypy] check_untyped_defs = True ignore_missing_imports = True -files = mitmproxy,examples/addons,release +files = mitmproxy,examples/addons,release/*.py [mypy-mitmproxy.contrib.*] ignore_errors = True From 884fd60e26b8beaa3785ea3d708c2d4386707f93 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 9 Mar 2023 16:48:26 +0100 Subject: [PATCH 211/695] fix #5972 (#5982) --- mitmproxy/addons/next_layer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitmproxy/addons/next_layer.py b/mitmproxy/addons/next_layer.py index 81244b2c18..2687e94749 100644 --- a/mitmproxy/addons/next_layer.py +++ b/mitmproxy/addons/next_layer.py @@ -349,7 +349,10 @@ def _setup_explicit_http_proxy(self, context: Context, data_client: bytes) -> La elif starts_like_tls_record(data_client): stack /= layers.ClientTLSLayer(context) - stack /= layers.HttpLayer(context, HTTPMode.regular) + if isinstance(context.layers[0], modes.HttpUpstreamProxy): + stack /= layers.HttpLayer(context, HTTPMode.upstream) + else: + stack /= layers.HttpLayer(context, HTTPMode.regular) return stack[0] From 8f1329377147538afdf06344179c2fd90795e93a Mon Sep 17 00:00:00 2001 From: Alex Gershberg <108593771+alexgershberg@users.noreply.github.com> Date: Sun, 12 Mar 2023 16:50:15 +0000 Subject: [PATCH 212/695] Add prettier to mitmweb (#5985) Co-authored-by: Maximilian Hils Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 6 + CHANGELOG.md | 2 + test/mitmproxy/tools/web/test_app.py | 6 +- web/.prettierignore | 7 + web/README.md | 3 + web/gulpfile.js | 82 +- web/jest.config.js | 45 +- web/package-lock.json | 36 +- web/package.json | 1 + web/src/css/app.less | 4 +- web/src/css/command.less | 4 +- web/src/css/contentview.less | 23 +- web/src/css/dropdown.less | 1 - web/src/css/eventlog.less | 3 +- web/src/css/flowdetail.less | 32 +- web/src/css/flowtable.less | 30 +- web/src/css/flowview.less | 5 +- web/src/css/header.less | 13 +- web/src/css/layout.less | 13 +- web/src/css/modal.less | 1 - web/src/css/tabs.less | 2 - web/src/css/vendor-bootstrap-variables.less | 7 +- web/src/css/vendor.less | 4 +- web/src/fonts/README | 1 - web/src/fonts/font-awesome.css | 1574 ++--- web/src/js/__tests__/backends/staticSpec.tsx | 26 +- .../js/__tests__/backends/websocketSpec.tsx | 93 +- .../__tests__/components/CaptureSetupSpec.tsx | 9 +- .../__tests__/components/CommandBarSpec.tsx | 91 +- .../components/EventLog/EventListSpec.tsx | 41 +- .../js/__tests__/components/EventLogSpec.tsx | 105 +- .../components/FlowTable/FlowColumnsSpec.tsx | 167 +- .../components/FlowTable/FlowRowSpec.tsx | 38 +- .../FlowTable/FlowTableHeadSpec.tsx | 31 +- .../js/__tests__/components/FlowTableSpec.tsx | 77 +- .../js/__tests__/components/FlowViewSpec.tsx | 12 +- .../Header/ConnectionIndicatorSpec.tsx | 29 +- .../components/Header/FileMenuSpec.tsx | 25 +- .../components/Header/FilterDocsSpec.tsx | 26 +- .../components/Header/FilterInputSpec.tsx | 161 +- .../components/Header/FlowMenuSpec.tsx | 8 +- .../components/Header/MainMenuSpec.tsx | 10 +- .../components/Header/MenuToggleSpec.tsx | 51 +- .../components/Header/OptionMenuSpec.tsx | 24 +- .../__snapshots__/OptionMenuSpec.tsx.snap | 3 +- .../js/__tests__/components/HeaderSpec.tsx | 12 +- .../__tests__/components/Modal/ModalSpec.tsx | 13 +- .../components/Modal/OptionModalSpec.tsx | 84 +- .../__tests__/components/Modal/OptionSpec.tsx | 145 +- .../js/__tests__/components/ProxyAppSpec.tsx | 20 +- .../components/common/ButtonSpec.tsx | 50 +- .../components/common/DocsLinkSpec.tsx | 32 +- .../components/common/DropdownSpec.tsx | 67 +- .../components/common/FileChooserSpec.tsx | 18 +- .../components/common/SplitterSpec.tsx | 155 +- .../components/common/ToggleButtonSpec.tsx | 36 +- .../contentviews/CodeEditorSpec.tsx | 11 +- .../contentviews/HttpMessageSpec.tsx | 40 +- .../contentviews/LineRendererSpec.tsx | 23 +- .../contentviews/ViewSelectorSpec.tsx | 12 +- .../contentviews/useContentSpec.tsx | 25 +- .../components/editors/ValidateEditorSpec.tsx | 20 +- .../components/editors/ValueEditorSpec.tsx | 19 +- .../components/helpers/AutoScrollSpec.tsx | 64 +- .../components/helpers/VirtualScrollSpec.tsx | 102 +- web/src/js/__tests__/ducks/commandBarSpec.tsx | 14 +- web/src/js/__tests__/ducks/connectionSpec.tsx | 65 +- web/src/js/__tests__/ducks/eventLogSpec.tsx | 70 +- web/src/js/__tests__/ducks/flowsSpec.tsx | 351 +- web/src/js/__tests__/ducks/indexSpec.tsx | 20 +- web/src/js/__tests__/ducks/optionsSpec.tsx | 73 +- .../js/__tests__/ducks/options_metaSpec.tsx | 24 +- web/src/js/__tests__/ducks/tutils.ts | 110 +- web/src/js/__tests__/ducks/ui/flowSpec.tsx | 30 +- web/src/js/__tests__/ducks/ui/indexSpec.tsx | 14 +- .../js/__tests__/ducks/ui/keyboardSpec.tsx | 334 +- web/src/js/__tests__/ducks/ui/modalSpec.tsx | 35 +- .../__tests__/ducks/ui/optionEditorSpec.tsx | 81 +- .../js/__tests__/ducks/utils/storeSpec.tsx | 308 +- web/src/js/__tests__/flow/utilsSpec.tsx | 118 +- web/src/js/__tests__/test-utils.tsx | 35 +- web/src/js/__tests__/urlStateSpec.tsx | 165 +- web/src/js/__tests__/utilsSpec.tsx | 144 +- web/src/js/app.tsx | 39 +- web/src/js/backends/static.tsx | 30 +- web/src/js/backends/websocket.tsx | 100 +- web/src/js/components/CaptureSetup.tsx | 127 +- web/src/js/components/CommandBar.tsx | 337 +- web/src/js/components/EventLog.jsx | 68 +- web/src/js/components/EventLog/EventList.tsx | 100 +- web/src/js/components/FlowTable.jsx | 115 +- .../js/components/FlowTable/FlowColumns.tsx | 301 +- web/src/js/components/FlowTable/FlowRow.tsx | 75 +- .../js/components/FlowTable/FlowTableHead.tsx | 51 +- web/src/js/components/FlowView.tsx | 96 +- web/src/js/components/FlowView/Connection.tsx | 248 +- .../js/components/FlowView/DnsMessages.tsx | 136 +- web/src/js/components/FlowView/Error.tsx | 12 +- .../js/components/FlowView/HttpMessages.tsx | 193 +- web/src/js/components/FlowView/Messages.tsx | 97 +- .../js/components/FlowView/TcpMessages.tsx | 11 +- web/src/js/components/FlowView/Timing.tsx | 113 +- .../js/components/FlowView/UdpMessages.tsx | 11 +- web/src/js/components/FlowView/WebSocket.tsx | 51 +- web/src/js/components/Footer.tsx | 82 +- web/src/js/components/Header.tsx | 50 +- .../components/Header/ConnectionIndicator.tsx | 39 +- web/src/js/components/Header/FileMenu.tsx | 55 +- web/src/js/components/Header/FilterDocs.tsx | 70 +- web/src/js/components/Header/FilterInput.tsx | 134 +- web/src/js/components/Header/FlowMenu.tsx | 247 +- web/src/js/components/Header/MenuToggle.tsx | 48 +- web/src/js/components/Header/OptionMenu.tsx | 30 +- web/src/js/components/Header/StartMenu.tsx | 89 +- web/src/js/components/MainView.tsx | 24 +- web/src/js/components/Modal/Modal.tsx | 17 +- web/src/js/components/Modal/ModalLayout.tsx | 28 +- web/src/js/components/Modal/ModalList.tsx | 12 +- web/src/js/components/Modal/Option.jsx | 160 +- web/src/js/components/Modal/OptionModal.jsx | 127 +- web/src/js/components/ProxyApp.tsx | 95 +- web/src/js/components/common/Button.tsx | 42 +- web/src/js/components/common/DocsLink.tsx | 10 +- web/src/js/components/common/Dropdown.tsx | 189 +- web/src/js/components/common/FileChooser.tsx | 51 +- web/src/js/components/common/HideInStatic.tsx | 8 +- web/src/js/components/common/Splitter.tsx | 115 +- web/src/js/components/common/ToggleButton.tsx | 32 +- .../js/components/contentviews/CodeEditor.tsx | 32 +- .../components/contentviews/HttpMessage.tsx | 132 +- .../components/contentviews/LineRenderer.tsx | 61 +- .../components/contentviews/ViewSelector.tsx | 35 +- .../js/components/contentviews/useContent.tsx | 69 +- .../components/editors/KeyValueListEditor.tsx | 190 +- .../js/components/editors/ValidateEditor.tsx | 29 +- web/src/js/components/editors/ValueEditor.tsx | 119 +- web/src/js/components/helpers/AutoScroll.tsx | 42 +- .../js/components/helpers/VirtualScroll.tsx | 45 +- web/src/js/contrib/CodeMirror.tsx | 105 +- web/src/js/ducks/_options_gen.ts | 176 +- web/src/js/ducks/backendState.ts | 32 +- web/src/js/ducks/commandBar.ts | 12 +- web/src/js/ducks/connection.ts | 21 +- web/src/js/ducks/eventLog.ts | 70 +- web/src/js/ducks/flows.ts | 212 +- web/src/js/ducks/index.ts | 47 +- web/src/js/ducks/options.ts | 54 +- web/src/js/ducks/options_meta.ts | 34 +- web/src/js/ducks/ui/flow.ts | 33 +- web/src/js/ducks/ui/index.ts | 10 +- web/src/js/ducks/ui/keyboard.tsx | 123 +- web/src/js/ducks/ui/modal.ts | 25 +- web/src/js/ducks/ui/optionsEditor.ts | 36 +- web/src/js/ducks/utils/store.ts | 259 +- web/src/js/filt/command.js | 1607 ++--- web/src/js/filt/filt.js | 5172 ++++++++++------- web/src/js/flow.ts | 193 +- web/src/js/flow/export.ts | 11 +- web/src/js/flow/utils.ts | 79 +- web/src/js/urlState.ts | 96 +- web/src/js/utils.ts | 99 +- web/src/templates/index.html | 28 +- web/tsconfig.json | 2 +- 163 files changed, 10877 insertions(+), 8349 deletions(-) create mode 100644 web/.prettierignore diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index a7fcf0b505..9429bb476e 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -42,6 +42,12 @@ jobs: - uses: install-pinned/black@eb57934f28e8d533bbcb4caa88d00b613b6ddd00 - run: black --extend-exclude mitmproxy/contrib . + - name: Run prettier + run: | + npm ci + npx prettier --write . + working-directory: web + - uses: mhils/add-pr-ref-in-changelog@main - uses: autofix-ci/action@8bc06253bec489732e5f9c52884c7cace15c0160 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5924609963..34591de21d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## Unreleased: mitmproxy next +* Add "Prettier" code linting tool to mitmweb. + ([#5985](https://github.com/mitmproxy/mitmproxy/pull/5985), @alexgershberg) * Add experimental support for HTTP/3 and QUIC. ([#5435](https://github.com/mitmproxy/mitmproxy/issues/5435), @meitinger) * You can now set the `http3` to false to block all QUIC and HTTP/3 traffic diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 249ed00d64..f5b274b139 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -135,10 +135,10 @@ def ts_type(t): print("export interface OptionsState {") for _, opt in sorted(m.options.items()): - print(f" {opt.name}: {ts_type(opt.typespec)}") + print(f" {opt.name}: {ts_type(opt.typespec)};") print("}") print("") - print("export type Option = keyof OptionsState") + print("export type Option = keyof OptionsState;") print("") print("export const defaultState: OptionsState = {") for _, opt in sorted(m.options.items()): @@ -147,7 +147,7 @@ def ts_type(t): ": null", ": undefined" ) ) - print("}") + print("};") ( Path(__file__).parent / "../../../../web/src/js/ducks/_options_gen.ts" diff --git a/web/.prettierignore b/web/.prettierignore new file mode 100644 index 0000000000..bd599e40ef --- /dev/null +++ b/web/.prettierignore @@ -0,0 +1,7 @@ +node_modules +coverage +*.md +.tox +src/js/filt/*.js +src/js/__tests__/ducks/_tflow.ts +src/js/__tests__/ducks/_tbackendstate.ts \ No newline at end of file diff --git a/web/README.md b/web/README.md index 0c08dbda79..f9f38b6d12 100644 --- a/web/README.md +++ b/web/README.md @@ -13,6 +13,9 @@ - Run `npm test` to run the test suite. +## Linting +- Run `npx prettier --write .` to lint your code. + ## Architecture There are two components: diff --git a/web/gulpfile.js b/web/gulpfile.js index b7dafce778..e322e818c8 100644 --- a/web/gulpfile.js +++ b/web/gulpfile.js @@ -1,59 +1,64 @@ const gulp = require("gulp"); -const gulpEsbuild = require('gulp-esbuild'); +const gulpEsbuild = require("gulp-esbuild"); const less = require("gulp-less"); const livereload = require("gulp-livereload"); -const cleanCSS = require('gulp-clean-css'); +const cleanCSS = require("gulp-clean-css"); const notify = require("gulp-notify"); const compilePeg = require("gulp-peg"); const plumber = require("gulp-plumber"); -const replace = require('gulp-replace'); -const sourcemaps = require('gulp-sourcemaps'); +const replace = require("gulp-replace"); +const sourcemaps = require("gulp-sourcemaps"); const through = require("through2"); const noop = () => through.obj(); -var handleError = {errorHandler: notify.onError("Error: <%= error.message %>")}; +var handleError = { + errorHandler: notify.onError("Error: <%= error.message %>"), +}; function styles(files, dev) { - return gulp.src(files) + return gulp + .src(files) .pipe(dev ? plumber(handleError) : noop()) .pipe(sourcemaps.init()) .pipe(less()) .pipe(dev ? noop() : cleanCSS()) - .pipe(sourcemaps.write(".", {sourceRoot: '/src/css'})) + .pipe(sourcemaps.write(".", { sourceRoot: "/src/css" })) .pipe(gulp.dest("../mitmproxy/tools/web/static")) - .pipe(livereload({auto: false})); + .pipe(livereload({ auto: false })); } function styles_vendor_prod() { - return styles("src/css/vendor.less", false) + return styles("src/css/vendor.less", false); } function styles_vendor_dev() { - return styles("src/css/vendor.less", true) + return styles("src/css/vendor.less", true); } function styles_app_prod() { - return styles("src/css/app.less", false) + return styles("src/css/app.less", false); } function styles_app_dev() { - return styles("src/css/app.less", true) + return styles("src/css/app.less", true); } - function esbuild(dev) { - return gulp.src('src/js/app.tsx').pipe( - gulpEsbuild({ - outfile: 'app.js', - sourcemap: true, - sourceRoot: "/", - minify: !dev, - keepNames: true, - bundle: true, - })) + return gulp + .src("src/js/app.tsx") + .pipe( + gulpEsbuild({ + outfile: "app.js", + sourcemap: true, + sourceRoot: "/", + minify: !dev, + keepNames: true, + bundle: true, + }) + ) .pipe(gulp.dest("../mitmproxy/tools/web/static")) - .pipe(livereload({auto: false})); + .pipe(livereload({ auto: false })); } function scripts_dev() { @@ -64,29 +69,40 @@ function scripts_prod() { return esbuild(false); } -const copy_src = ["src/images/**", "src/fonts/fontawesome-webfont.*", "!**/*.psd"]; +const copy_src = [ + "src/images/**", + "src/fonts/fontawesome-webfont.*", + "!**/*.psd", +]; function copy() { - return gulp.src(copy_src, {base: "src/"}) + return gulp + .src(copy_src, { base: "src/" }) .pipe(gulp.dest("../mitmproxy/tools/web/static")); } const template_src = "src/templates/*"; function templates() { - return gulp.src(template_src, {base: "src/"}) + return gulp + .src(template_src, { base: "src/" }) .pipe(gulp.dest("../mitmproxy/tools/web")); } const peg_src = "src/js/filt/*.peg"; function peg() { - return gulp.src(peg_src, {base: "src/"}) + return gulp + .src(peg_src, { base: "src/" }) .pipe(plumber(handleError)) .pipe(compilePeg()) - .pipe(replace('module.exports = ', - 'import * as flowutils from "../flow/utils"\n' + - 'export default ')) + .pipe( + replace( + "module.exports = ", + 'import * as flowutils from "../flow/utils"\n' + + "export default " + ) + ) .pipe(gulp.dest("src/")); } @@ -111,12 +127,12 @@ const prod = gulp.parallel( exports.dev = dev; exports.prod = prod; exports.default = function watch() { - const opts = {ignoreInitial: false}; - livereload.listen({auto: true}); + const opts = { ignoreInitial: false }; + livereload.listen({ auto: true }); gulp.watch(["src/css/vendor*"], opts, styles_vendor_dev); gulp.watch(["src/css/**"], opts, styles_app_dev); gulp.watch(["src/js/**"], opts, scripts_dev); gulp.watch(template_src, opts, templates); gulp.watch(peg_src, opts, peg); gulp.watch(copy_src, opts, copy); -} +}; diff --git a/web/jest.config.js b/web/jest.config.js index a1e4daaed1..bedccc4c1f 100644 --- a/web/jest.config.js +++ b/web/jest.config.js @@ -1,36 +1,29 @@ module.exports = async () => { - - process.env.TZ = 'UTC'; + process.env.TZ = "UTC"; return { - "testEnvironment": "jsdom", - "testRegex": "__tests__/.*Spec.(js|ts)x?$", - "roots": [ - "/src/js" - ], - "unmockedModulePathPatterns": [ - "react" - ], - "coverageDirectory": "./coverage", - "coveragePathIgnorePatterns": [ + testEnvironment: "jsdom", + testRegex: "__tests__/.*Spec.(js|ts)x?$", + roots: ["/src/js"], + unmockedModulePathPatterns: ["react"], + coverageDirectory: "./coverage", + coveragePathIgnorePatterns: [ "/src/js/contrib/", "/src/js/filt/", - "/src/js/components/editors/" - ], - "collectCoverageFrom": [ - "src/js/**/*.{js,jsx,ts,tsx}" + "/src/js/components/editors/", ], - "transform": { + collectCoverageFrom: ["src/js/**/*.{js,jsx,ts,tsx}"], + transform: { "^.+\\.[jt]sx?$": [ "esbuild-jest", { - "loaders": { - ".js": "tsx" + loaders: { + ".js": "tsx", }, - "format": "cjs", - "sourcemap": true, - } - ] - } - } -} + format: "cjs", + sourcemap: true, + }, + ], + }, + }; +}; diff --git a/web/package-lock.json b/web/package-lock.json index 002b7e3efe..f8ba1016a4 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -2066,6 +2066,16 @@ "integrity": "sha512-nAihlQsYGyc5Bwq6+EsubvANYGExeJKHDO3RjnvwU042fawQTQfM3Kxn7IHUXQOz4bzfwsGYYHGSvXyW4zOGLg==", "dev": true }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "body": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/body/-/body-5.1.0.tgz", @@ -2293,7 +2303,11 @@ "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", "dev": true, - "optional": true + "optional": true, + "requires": { + "bindings": "^1.5.0", + "nan": "^2.12.1" + } }, "normalize-path": { "version": "3.0.0", @@ -3493,6 +3507,13 @@ "bser": "^2.0.0" } }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "optional": true + }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -7048,6 +7069,13 @@ "integrity": "sha512-kDcwXR4PS7caBpuRYYBUz9iVixUk3anO3f5OYFiIPwK/20vCzKCHyKoulbiDY1S53zD2bxUpxN/IJ+TnXjfvxg==", "dev": true }, + "nan": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.17.0.tgz", + "integrity": "sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==", + "dev": true, + "optional": true + }, "nanomatch": { "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", @@ -7718,6 +7746,12 @@ "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", "dev": true }, + "prettier": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.8.4.tgz", + "integrity": "sha512-vIS4Rlc2FNh0BySk3Wkd6xmwxB0FpOndW5fisM5H8hsZSxU2VWVB5CWIkIjWvrHjIhxk2g3bfMKM87zNTrZddw==", + "dev": true + }, "pretty-format": { "version": "27.0.2", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.0.2.tgz", diff --git a/web/package.json b/web/package.json index b2bf84c6fe..0f7440742b 100644 --- a/web/package.json +++ b/web/package.json @@ -44,6 +44,7 @@ "gulp-sourcemaps": "^3.0.0", "jest": "^27.0.6", "jest-fetch-mock": "^3.0.3", + "prettier": "2.8.4", "react-test-renderer": "^17.0.2", "redux-mock-store": "^1.5.4", "through2": "^4.0.2", diff --git a/web/src/css/app.less b/web/src/css/app.less index 57caa98d93..1f0e3a1537 100644 --- a/web/src/css/app.less +++ b/web/src/css/app.less @@ -3,7 +3,9 @@ html { box-sizing: border-box; } -*, *:before, *:after { +*, +*:before, +*:after { box-sizing: inherit; } diff --git a/web/src/css/command.less b/web/src/css/command.less index 9701e76b20..8803f6e715 100644 --- a/web/src/css/command.less +++ b/web/src/css/command.less @@ -1,5 +1,5 @@ .command-title { - background-color: #F2F2F2; + background-color: #f2f2f2; border: 1px solid #aaa; } @@ -28,4 +28,4 @@ .available-commands { overflow: auto; -} \ No newline at end of file +} diff --git a/web/src/css/contentview.less b/web/src/css/contentview.less index e483214ff4..da789730d4 100644 --- a/web/src/css/contentview.less +++ b/web/src/css/contentview.less @@ -1,29 +1,28 @@ .contentview { - .header { + .header { font-weight: bold; - } - .highlight{ + } + .highlight { font-weight: bold; } - .offset{ - color: blue + .offset { + color: blue; } - .text{ - + .text { } - .codeeditor{ + .codeeditor { margin-bottom: 12px; } - .Token_Name_Tag{ + .Token_Name_Tag { color: darkgreen; } - .Token_Literal_String{ + .Token_Literal_String { color: firebrick; } - .Token_Literal_Number{ + .Token_Literal_Number { color: purple; } - .Token_Keyword_Constant{ + .Token_Keyword_Constant { color: blue; } } diff --git a/web/src/css/dropdown.less b/web/src/css/dropdown.less index cd27c84c07..1ed8ad8e5f 100644 --- a/web/src/css/dropdown.less +++ b/web/src/css/dropdown.less @@ -1,5 +1,4 @@ .dropdown-menu { - // setting a margin is not compatible with popper. margin: 0 !important; diff --git a/web/src/css/eventlog.less b/web/src/css/eventlog.less index 393f75dbe2..bc38101c86 100644 --- a/web/src/css/eventlog.less +++ b/web/src/css/eventlog.less @@ -1,5 +1,4 @@ .eventlog { - height: 200px; flex: 0 0 auto; @@ -7,7 +6,7 @@ flex-direction: column; > div { - background-color: #F2F2F2; + background-color: #f2f2f2; padding: 0 5px; flex: 0 0 auto; border-top: 1px solid #aaa; diff --git a/web/src/css/flowdetail.less b/web/src/css/flowdetail.less index ec38bb5f01..b7f5b8f51a 100644 --- a/web/src/css/flowdetail.less +++ b/web/src/css/flowdetail.less @@ -5,7 +5,7 @@ flex-direction: column; nav { - background-color: #F2F2F2; + background-color: #f2f2f2; } section { @@ -21,7 +21,6 @@ } } - .first-line { font-family: @font-family-monospace; background-color: #428bca; @@ -59,7 +58,6 @@ } } - .inline-input { display: inline; margin: 0 -3px; @@ -68,7 +66,8 @@ border: solid transparent 1px; &:hover { - box-shadow: 0 0 0 1px rgba(0, 0, 0, 1.25%), 0 2px 4px rgba(0, 0, 0, 5%), 0 2px 6px rgba(0, 0, 0, 2.5%); + box-shadow: 0 0 0 1px rgba(0, 0, 0, 1.25%), 0 2px 4px rgba(0, 0, 0, 5%), + 0 2px 6px rgba(0, 0, 0, 2.5%); background-color: rgba(255, 255, 255, 0.1); } @@ -80,10 +79,10 @@ &[contenteditable] { outline-width: 0; - box-shadow: 0 0 0 1px rgba(0, 0, 0, 5%), 0 2px 4px rgba(0, 0, 0, 20%), 0 2px 6px rgba(0, 0, 0, 10%); + box-shadow: 0 0 0 1px rgba(0, 0, 0, 5%), 0 2px 4px rgba(0, 0, 0, 20%), + 0 2px 6px rgba(0, 0, 0, 10%); background-color: rgba(255, 255, 255, 0.2); - &.has-warning { color: rgb(255, 184, 184); } @@ -94,7 +93,9 @@ } } -.connection-table, .timing-table, .certificate-table { +.connection-table, +.timing-table, +.certificate-table { td:nth-child(2) { font-family: @font-family-monospace; width: 70%; @@ -125,9 +126,10 @@ } } -.headers, .trailers { +.headers, +.trailers { .kv-row { - margin-bottom: .3em; + margin-bottom: 0.3em; max-height: 12.4ex; overflow-y: auto; } @@ -162,7 +164,8 @@ overflow-wrap: break-word; } -.connection-table, .timing-table { +.connection-table, +.timing-table { td { overflow: hidden; text-overflow: ellipsis; @@ -176,7 +179,8 @@ dl.cert-attributes { flex-wrap: wrap; margin-bottom: 0; - dt, dd { + dt, + dd { text-overflow: ellipsis; overflow: hidden; } @@ -190,8 +194,10 @@ dl.cert-attributes { } } -.dns-request table, .dns-response table { - th, td { +.dns-request table, +.dns-response table { + th, + td { padding-right: 1rem; } } diff --git a/web/src/css/flowtable.less b/web/src/css/flowtable.less index b3c21aad64..9c53e7ff1c 100644 --- a/web/src/css/flowtable.less +++ b/web/src/css/flowtable.less @@ -18,7 +18,7 @@ } thead tr { - background-color: #F2F2F2; + background-color: #f2f2f2; border-bottom: solid #bebebe 1px; line-height: 23px; } @@ -29,17 +29,19 @@ padding-left: 1px; .user-select(none); - &.sort-asc, &.sort-desc { - background-color: lighten(#F2F2F2, 3%); + &.sort-asc, + &.sort-desc { + background-color: lighten(#f2f2f2, 3%); } - &.sort-asc:after, &.sort-desc:after { + &.sort-asc:after, + &.sort-desc:after { font: normal normal normal 14px/1 FontAwesome; position: absolute; right: 3px; top: 3px; padding: 2px; - background-color: fadeout(lighten(#F2F2F2, 3%), 20%); + background-color: fadeout(lighten(#f2f2f2, 3%), 20%); } &.sort-asc:after { @@ -49,7 +51,6 @@ &.sort-desc:after { content: "\f0dd"; } - } tr { @@ -86,13 +87,16 @@ @interceptorange: hsl(30, 100%, 50%); tr.intercepted:not(.has-response) { - .col-path, .col-method { + .col-path, + .col-method { color: @interceptorange; } } tr.intercepted.has-response { - .col-status, .col-size, .col-time { + .col-status, + .col-size, + .col-time { color: @interceptorange; } } @@ -130,7 +134,8 @@ color: @interceptorange; } - .fa-exclamation, .fa-times { + .fa-exclamation, + .fa-times { color: darkred; } } @@ -155,7 +160,9 @@ width: 170px; } - td.col-time, td.col-size, td.col-timestamp { + td.col-time, + td.col-size, + td.col-timestamp { text-align: right; } @@ -170,7 +177,8 @@ font-size: 20px; } - tr:hover .col-quickactions, .col-quickactions.hover { + tr:hover .col-quickactions, + .col-quickactions.hover { overflow: visible; } diff --git a/web/src/css/flowview.less b/web/src/css/flowview.less index 0f031c5609..7180c270c5 100644 --- a/web/src/css/flowview.less +++ b/web/src/css/flowview.less @@ -1,5 +1,4 @@ .flowview-image { - text-align: center; padding: 10px 0; @@ -14,7 +13,7 @@ right: 20px; } -.edit-flow { +.edit-flow { cursor: pointer; position: absolute; right: 0; @@ -36,6 +35,6 @@ .edit-flow:hover { background-color: rgba(239, 108, 0, 0.7); - color: rgba(0,0,0,0.8); + color: rgba(0, 0, 0, 0.8); border: solid 2px transparent; } diff --git a/web/src/css/header.less b/web/src/css/header.less index a94773d33a..b890e0b06e 100644 --- a/web/src/css/header.less +++ b/web/src/css/header.less @@ -1,5 +1,5 @@ -@import (reference) '../../node_modules/bootstrap/less/variables.less'; -@import (reference) '../../node_modules/bootstrap/less/mixins/grid.less'; +@import (reference) "../../node_modules/bootstrap/less/variables.less"; +@import (reference) "../../node_modules/bootstrap/less/mixins/grid.less"; @import (reference) "../../node_modules/bootstrap/less/mixins/labels.less"; @import (reference) "../../node_modules/bootstrap/less/labels.less"; @@ -34,7 +34,8 @@ header { > a { display: inline-block; } - > .btn, > a > .btn { + > .btn, + > a > .btn { height: @menu-height - @menu-legend-height; text-align: center; margin: 0 1px; @@ -70,7 +71,7 @@ header { font-weight: normal; margin: 0; } - input[type=checkbox] { + input[type="checkbox"] { margin: 0 2px; vertical-align: middle; } @@ -117,7 +118,6 @@ header { opacity: 0.9; @media (max-width: @screen-xs-max) { - top: 16px; left: 29px; right: 2px; @@ -143,7 +143,8 @@ header { opacity: 1; transition: all 1s linear; - &.init, &.fetching { + &.init, + &.fetching { background-color: @label-info-bg; } &.established { diff --git a/web/src/css/layout.less b/web/src/css/layout.less index ed4adb6986..4b94031397 100644 --- a/web/src/css/layout.less +++ b/web/src/css/layout.less @@ -1,4 +1,7 @@ -html, body, #container, #mitmproxy { +html, +body, +#container, +#mitmproxy { height: 100%; margin: 0; overflow: hidden; @@ -10,7 +13,9 @@ html, body, #container, #mitmproxy { outline: none; // our root element is focused by default. - > header, > footer, > .eventlog { + > header, + > footer, + > .eventlog { flex: 0 0 auto; } } @@ -30,10 +35,10 @@ html, body, #container, #mitmproxy { flex-direction: column; } - .flow-detail, .flow-table { + .flow-detail, + .flow-table { flex: 1 1 auto; } - } .splitter { diff --git a/web/src/css/modal.less b/web/src/css/modal.less index 30e98f9c23..13ef997f77 100644 --- a/web/src/css/modal.less +++ b/web/src/css/modal.less @@ -2,7 +2,6 @@ display: block; } - .modal-dialog { overflow-y: initial !important; } diff --git a/web/src/css/tabs.less b/web/src/css/tabs.less index a66d30ed6d..5a19705e14 100644 --- a/web/src/css/tabs.less +++ b/web/src/css/tabs.less @@ -1,5 +1,4 @@ .nav-tabs { - @separator-color: lighten(grey, 15%); border-bottom: solid @separator-color 1px; @@ -26,7 +25,6 @@ } } } - } .nav-tabs-lg { diff --git a/web/src/css/vendor-bootstrap-variables.less b/web/src/css/vendor-bootstrap-variables.less index 668fec45da..663a87b5f6 100644 --- a/web/src/css/vendor-bootstrap-variables.less +++ b/web/src/css/vendor-bootstrap-variables.less @@ -3,5 +3,8 @@ @navbar-default-color: #303030; @navbar-default-bg: #ffffff; @navbar-default-border: #e0e0e0; -@font-family-sans-serif: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", "Liberation Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; -@font-family-monospace: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; +@font-family-sans-serif: system-ui, -apple-system, "Segoe UI", Roboto, + "Helvetica Neue", Arial, "Noto Sans", "Liberation Sans", sans-serif, + "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; +@font-family-monospace: SFMono-Regular, Menlo, Monaco, Consolas, + "Liberation Mono", "Courier New", monospace; diff --git a/web/src/css/vendor.less b/web/src/css/vendor.less index e91ae3a843..eb6208842a 100644 --- a/web/src/css/vendor.less +++ b/web/src/css/vendor.less @@ -1,3 +1,3 @@ // Bootstrap -@import 'vendor-bootstrap.less'; -@import (less) '../fonts/font-awesome.css'; +@import "vendor-bootstrap.less"; +@import (less) "../fonts/font-awesome.css"; diff --git a/web/src/fonts/README b/web/src/fonts/README index 218a78e1a7..e922ce1e37 100644 --- a/web/src/fonts/README +++ b/web/src/fonts/README @@ -1,2 +1 @@ - From a rendered version of the FontAwesome github repo. diff --git a/web/src/fonts/font-awesome.css b/web/src/fonts/font-awesome.css index 9510290049..ce2ca1c34b 100644 --- a/web/src/fonts/font-awesome.css +++ b/web/src/fonts/font-awesome.css @@ -5,2333 +5,2339 @@ /* FONT PATH * -------------------------- */ @font-face { - font-family: 'FontAwesome'; - src: url('./fonts/fontawesome-webfont.eot?v=4.7.0'); - src: url('./fonts/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'), url('./fonts/fontawesome-webfont.woff2?v=4.7.0') format('woff2'), url('./fonts/fontawesome-webfont.woff?v=4.7.0') format('woff'), url('./fonts/fontawesome-webfont.ttf?v=4.7.0') format('truetype'), url('./fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg'); - font-weight: normal; - font-style: normal; + font-family: "FontAwesome"; + src: url("./fonts/fontawesome-webfont.eot?v=4.7.0"); + src: url("./fonts/fontawesome-webfont.eot?#iefix&v=4.7.0") + format("embedded-opentype"), + url("./fonts/fontawesome-webfont.woff2?v=4.7.0") format("woff2"), + url("./fonts/fontawesome-webfont.woff?v=4.7.0") format("woff"), + url("./fonts/fontawesome-webfont.ttf?v=4.7.0") format("truetype"), + url("./fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular") + format("svg"); + font-weight: normal; + font-style: normal; } .fa { - display: inline-block; - font: normal normal normal 14px/1 FontAwesome; - font-size: inherit; - text-rendering: auto; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; + display: inline-block; + font: normal normal normal 14px/1 FontAwesome; + font-size: inherit; + text-rendering: auto; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; } /* makes the font 33% larger relative to the icon container */ .fa-lg { - font-size: 1.33333333em; - line-height: 0.75em; - vertical-align: -15%; + font-size: 1.33333333em; + line-height: 0.75em; + vertical-align: -15%; } .fa-2x { - font-size: 2em; + font-size: 2em; } .fa-3x { - font-size: 3em; + font-size: 3em; } .fa-4x { - font-size: 4em; + font-size: 4em; } .fa-5x { - font-size: 5em; + font-size: 5em; } .fa-fw { - width: 1.28571429em; - text-align: center; + width: 1.28571429em; + text-align: center; } .fa-ul { - padding-left: 0; - margin-left: 2.14285714em; - list-style-type: none; + padding-left: 0; + margin-left: 2.14285714em; + list-style-type: none; } .fa-ul > li { - position: relative; + position: relative; } .fa-li { - position: absolute; - left: -2.14285714em; - width: 2.14285714em; - top: 0.14285714em; - text-align: center; + position: absolute; + left: -2.14285714em; + width: 2.14285714em; + top: 0.14285714em; + text-align: center; } .fa-li.fa-lg { - left: -1.85714286em; + left: -1.85714286em; } .fa-border { - padding: .2em .25em .15em; - border: solid 0.08em #eeeeee; - border-radius: .1em; + padding: 0.2em 0.25em 0.15em; + border: solid 0.08em #eeeeee; + border-radius: 0.1em; } .fa-pull-left { - float: left; + float: left; } .fa-pull-right { - float: right; + float: right; } .fa.fa-pull-left { - margin-right: .3em; + margin-right: 0.3em; } .fa.fa-pull-right { - margin-left: .3em; + margin-left: 0.3em; } /* Deprecated as of 4.4.0 */ .pull-right { - float: right; + float: right; } .pull-left { - float: left; + float: left; } .fa.pull-left { - margin-right: .3em; + margin-right: 0.3em; } .fa.pull-right { - margin-left: .3em; + margin-left: 0.3em; } .fa-spin { - -webkit-animation: fa-spin 2s infinite linear; - animation: fa-spin 2s infinite linear; + -webkit-animation: fa-spin 2s infinite linear; + animation: fa-spin 2s infinite linear; } .fa-pulse { - -webkit-animation: fa-spin 1s infinite steps(8); - animation: fa-spin 1s infinite steps(8); + -webkit-animation: fa-spin 1s infinite steps(8); + animation: fa-spin 1s infinite steps(8); } @-webkit-keyframes fa-spin { - 0% { - -webkit-transform: rotate(0deg); - transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(359deg); - transform: rotate(359deg); - } + 0% { + -webkit-transform: rotate(0deg); + transform: rotate(0deg); + } + 100% { + -webkit-transform: rotate(359deg); + transform: rotate(359deg); + } } @keyframes fa-spin { - 0% { - -webkit-transform: rotate(0deg); - transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(359deg); - transform: rotate(359deg); - } + 0% { + -webkit-transform: rotate(0deg); + transform: rotate(0deg); + } + 100% { + -webkit-transform: rotate(359deg); + transform: rotate(359deg); + } } .fa-rotate-90 { - -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=1)"; - -webkit-transform: rotate(90deg); - -ms-transform: rotate(90deg); - transform: rotate(90deg); + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=1)"; + -webkit-transform: rotate(90deg); + -ms-transform: rotate(90deg); + transform: rotate(90deg); } .fa-rotate-180 { - -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2)"; - -webkit-transform: rotate(180deg); - -ms-transform: rotate(180deg); - transform: rotate(180deg); + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2)"; + -webkit-transform: rotate(180deg); + -ms-transform: rotate(180deg); + transform: rotate(180deg); } .fa-rotate-270 { - -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=3)"; - -webkit-transform: rotate(270deg); - -ms-transform: rotate(270deg); - transform: rotate(270deg); + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=3)"; + -webkit-transform: rotate(270deg); + -ms-transform: rotate(270deg); + transform: rotate(270deg); } .fa-flip-horizontal { - -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)"; - -webkit-transform: scale(-1, 1); - -ms-transform: scale(-1, 1); - transform: scale(-1, 1); + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)"; + -webkit-transform: scale(-1, 1); + -ms-transform: scale(-1, 1); + transform: scale(-1, 1); } .fa-flip-vertical { - -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)"; - -webkit-transform: scale(1, -1); - -ms-transform: scale(1, -1); - transform: scale(1, -1); + -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)"; + -webkit-transform: scale(1, -1); + -ms-transform: scale(1, -1); + transform: scale(1, -1); } :root .fa-rotate-90, :root .fa-rotate-180, :root .fa-rotate-270, :root .fa-flip-horizontal, :root .fa-flip-vertical { - filter: none; + filter: none; } .fa-stack { - position: relative; - display: inline-block; - width: 2em; - height: 2em; - line-height: 2em; - vertical-align: middle; + position: relative; + display: inline-block; + width: 2em; + height: 2em; + line-height: 2em; + vertical-align: middle; } .fa-stack-1x, .fa-stack-2x { - position: absolute; - left: 0; - width: 100%; - text-align: center; + position: absolute; + left: 0; + width: 100%; + text-align: center; } .fa-stack-1x { - line-height: inherit; + line-height: inherit; } .fa-stack-2x { - font-size: 2em; + font-size: 2em; } .fa-inverse { - color: #ffffff; + color: #ffffff; } /* Font Awesome uses the Unicode Private Use Area (PUA) to ensure screen readers do not read off random characters that represent icons */ .fa-glass:before { - content: "\f000"; + content: "\f000"; } .fa-music:before { - content: "\f001"; + content: "\f001"; } .fa-search:before { - content: "\f002"; + content: "\f002"; } .fa-envelope-o:before { - content: "\f003"; + content: "\f003"; } .fa-heart:before { - content: "\f004"; + content: "\f004"; } .fa-star:before { - content: "\f005"; + content: "\f005"; } .fa-star-o:before { - content: "\f006"; + content: "\f006"; } .fa-user:before { - content: "\f007"; + content: "\f007"; } .fa-film:before { - content: "\f008"; + content: "\f008"; } .fa-th-large:before { - content: "\f009"; + content: "\f009"; } .fa-th:before { - content: "\f00a"; + content: "\f00a"; } .fa-th-list:before { - content: "\f00b"; + content: "\f00b"; } .fa-check:before { - content: "\f00c"; + content: "\f00c"; } .fa-remove:before, .fa-close:before, .fa-times:before { - content: "\f00d"; + content: "\f00d"; } .fa-search-plus:before { - content: "\f00e"; + content: "\f00e"; } .fa-search-minus:before { - content: "\f010"; + content: "\f010"; } .fa-power-off:before { - content: "\f011"; + content: "\f011"; } .fa-signal:before { - content: "\f012"; + content: "\f012"; } .fa-gear:before, .fa-cog:before { - content: "\f013"; + content: "\f013"; } .fa-trash-o:before { - content: "\f014"; + content: "\f014"; } .fa-home:before { - content: "\f015"; + content: "\f015"; } .fa-file-o:before { - content: "\f016"; + content: "\f016"; } .fa-clock-o:before { - content: "\f017"; + content: "\f017"; } .fa-road:before { - content: "\f018"; + content: "\f018"; } .fa-download:before { - content: "\f019"; + content: "\f019"; } .fa-arrow-circle-o-down:before { - content: "\f01a"; + content: "\f01a"; } .fa-arrow-circle-o-up:before { - content: "\f01b"; + content: "\f01b"; } .fa-inbox:before { - content: "\f01c"; + content: "\f01c"; } .fa-play-circle-o:before { - content: "\f01d"; + content: "\f01d"; } .fa-rotate-right:before, .fa-repeat:before { - content: "\f01e"; + content: "\f01e"; } .fa-refresh:before { - content: "\f021"; + content: "\f021"; } .fa-list-alt:before { - content: "\f022"; + content: "\f022"; } .fa-lock:before { - content: "\f023"; + content: "\f023"; } .fa-flag:before { - content: "\f024"; + content: "\f024"; } .fa-headphones:before { - content: "\f025"; + content: "\f025"; } .fa-volume-off:before { - content: "\f026"; + content: "\f026"; } .fa-volume-down:before { - content: "\f027"; + content: "\f027"; } .fa-volume-up:before { - content: "\f028"; + content: "\f028"; } .fa-qrcode:before { - content: "\f029"; + content: "\f029"; } .fa-barcode:before { - content: "\f02a"; + content: "\f02a"; } .fa-tag:before { - content: "\f02b"; + content: "\f02b"; } .fa-tags:before { - content: "\f02c"; + content: "\f02c"; } .fa-book:before { - content: "\f02d"; + content: "\f02d"; } .fa-bookmark:before { - content: "\f02e"; + content: "\f02e"; } .fa-print:before { - content: "\f02f"; + content: "\f02f"; } .fa-camera:before { - content: "\f030"; + content: "\f030"; } .fa-font:before { - content: "\f031"; + content: "\f031"; } .fa-bold:before { - content: "\f032"; + content: "\f032"; } .fa-italic:before { - content: "\f033"; + content: "\f033"; } .fa-text-height:before { - content: "\f034"; + content: "\f034"; } .fa-text-width:before { - content: "\f035"; + content: "\f035"; } .fa-align-left:before { - content: "\f036"; + content: "\f036"; } .fa-align-center:before { - content: "\f037"; + content: "\f037"; } .fa-align-right:before { - content: "\f038"; + content: "\f038"; } .fa-align-justify:before { - content: "\f039"; + content: "\f039"; } .fa-list:before { - content: "\f03a"; + content: "\f03a"; } .fa-dedent:before, .fa-outdent:before { - content: "\f03b"; + content: "\f03b"; } .fa-indent:before { - content: "\f03c"; + content: "\f03c"; } .fa-video-camera:before { - content: "\f03d"; + content: "\f03d"; } .fa-photo:before, .fa-image:before, .fa-picture-o:before { - content: "\f03e"; + content: "\f03e"; } .fa-pencil:before { - content: "\f040"; + content: "\f040"; } .fa-map-marker:before { - content: "\f041"; + content: "\f041"; } .fa-adjust:before { - content: "\f042"; + content: "\f042"; } .fa-tint:before { - content: "\f043"; + content: "\f043"; } .fa-edit:before, .fa-pencil-square-o:before { - content: "\f044"; + content: "\f044"; } .fa-share-square-o:before { - content: "\f045"; + content: "\f045"; } .fa-check-square-o:before { - content: "\f046"; + content: "\f046"; } .fa-arrows:before { - content: "\f047"; + content: "\f047"; } .fa-step-backward:before { - content: "\f048"; + content: "\f048"; } .fa-fast-backward:before { - content: "\f049"; + content: "\f049"; } .fa-backward:before { - content: "\f04a"; + content: "\f04a"; } .fa-play:before { - content: "\f04b"; + content: "\f04b"; } .fa-pause:before { - content: "\f04c"; + content: "\f04c"; } .fa-stop:before { - content: "\f04d"; + content: "\f04d"; } .fa-forward:before { - content: "\f04e"; + content: "\f04e"; } .fa-fast-forward:before { - content: "\f050"; + content: "\f050"; } .fa-step-forward:before { - content: "\f051"; + content: "\f051"; } .fa-eject:before { - content: "\f052"; + content: "\f052"; } .fa-chevron-left:before { - content: "\f053"; + content: "\f053"; } .fa-chevron-right:before { - content: "\f054"; + content: "\f054"; } .fa-plus-circle:before { - content: "\f055"; + content: "\f055"; } .fa-minus-circle:before { - content: "\f056"; + content: "\f056"; } .fa-times-circle:before { - content: "\f057"; + content: "\f057"; } .fa-check-circle:before { - content: "\f058"; + content: "\f058"; } .fa-question-circle:before { - content: "\f059"; + content: "\f059"; } .fa-info-circle:before { - content: "\f05a"; + content: "\f05a"; } .fa-crosshairs:before { - content: "\f05b"; + content: "\f05b"; } .fa-times-circle-o:before { - content: "\f05c"; + content: "\f05c"; } .fa-check-circle-o:before { - content: "\f05d"; + content: "\f05d"; } .fa-ban:before { - content: "\f05e"; + content: "\f05e"; } .fa-arrow-left:before { - content: "\f060"; + content: "\f060"; } .fa-arrow-right:before { - content: "\f061"; + content: "\f061"; } .fa-arrow-up:before { - content: "\f062"; + content: "\f062"; } .fa-arrow-down:before { - content: "\f063"; + content: "\f063"; } .fa-mail-forward:before, .fa-share:before { - content: "\f064"; + content: "\f064"; } .fa-expand:before { - content: "\f065"; + content: "\f065"; } .fa-compress:before { - content: "\f066"; + content: "\f066"; } .fa-plus:before { - content: "\f067"; + content: "\f067"; } .fa-minus:before { - content: "\f068"; + content: "\f068"; } .fa-asterisk:before { - content: "\f069"; + content: "\f069"; } .fa-exclamation-circle:before { - content: "\f06a"; + content: "\f06a"; } .fa-gift:before { - content: "\f06b"; + content: "\f06b"; } .fa-leaf:before { - content: "\f06c"; + content: "\f06c"; } .fa-fire:before { - content: "\f06d"; + content: "\f06d"; } .fa-eye:before { - content: "\f06e"; + content: "\f06e"; } .fa-eye-slash:before { - content: "\f070"; + content: "\f070"; } .fa-warning:before, .fa-exclamation-triangle:before { - content: "\f071"; + content: "\f071"; } .fa-plane:before { - content: "\f072"; + content: "\f072"; } .fa-calendar:before { - content: "\f073"; + content: "\f073"; } .fa-random:before { - content: "\f074"; + content: "\f074"; } .fa-comment:before { - content: "\f075"; + content: "\f075"; } .fa-magnet:before { - content: "\f076"; + content: "\f076"; } .fa-chevron-up:before { - content: "\f077"; + content: "\f077"; } .fa-chevron-down:before { - content: "\f078"; + content: "\f078"; } .fa-retweet:before { - content: "\f079"; + content: "\f079"; } .fa-shopping-cart:before { - content: "\f07a"; + content: "\f07a"; } .fa-folder:before { - content: "\f07b"; + content: "\f07b"; } .fa-folder-open:before { - content: "\f07c"; + content: "\f07c"; } .fa-arrows-v:before { - content: "\f07d"; + content: "\f07d"; } .fa-arrows-h:before { - content: "\f07e"; + content: "\f07e"; } .fa-bar-chart-o:before, .fa-bar-chart:before { - content: "\f080"; + content: "\f080"; } .fa-twitter-square:before { - content: "\f081"; + content: "\f081"; } .fa-facebook-square:before { - content: "\f082"; + content: "\f082"; } .fa-camera-retro:before { - content: "\f083"; + content: "\f083"; } .fa-key:before { - content: "\f084"; + content: "\f084"; } .fa-gears:before, .fa-cogs:before { - content: "\f085"; + content: "\f085"; } .fa-comments:before { - content: "\f086"; + content: "\f086"; } .fa-thumbs-o-up:before { - content: "\f087"; + content: "\f087"; } .fa-thumbs-o-down:before { - content: "\f088"; + content: "\f088"; } .fa-star-half:before { - content: "\f089"; + content: "\f089"; } .fa-heart-o:before { - content: "\f08a"; + content: "\f08a"; } .fa-sign-out:before { - content: "\f08b"; + content: "\f08b"; } .fa-linkedin-square:before { - content: "\f08c"; + content: "\f08c"; } .fa-thumb-tack:before { - content: "\f08d"; + content: "\f08d"; } .fa-external-link:before { - content: "\f08e"; + content: "\f08e"; } .fa-sign-in:before { - content: "\f090"; + content: "\f090"; } .fa-trophy:before { - content: "\f091"; + content: "\f091"; } .fa-github-square:before { - content: "\f092"; + content: "\f092"; } .fa-upload:before { - content: "\f093"; + content: "\f093"; } .fa-lemon-o:before { - content: "\f094"; + content: "\f094"; } .fa-phone:before { - content: "\f095"; + content: "\f095"; } .fa-square-o:before { - content: "\f096"; + content: "\f096"; } .fa-bookmark-o:before { - content: "\f097"; + content: "\f097"; } .fa-phone-square:before { - content: "\f098"; + content: "\f098"; } .fa-twitter:before { - content: "\f099"; + content: "\f099"; } .fa-facebook-f:before, .fa-facebook:before { - content: "\f09a"; + content: "\f09a"; } .fa-github:before { - content: "\f09b"; + content: "\f09b"; } .fa-unlock:before { - content: "\f09c"; + content: "\f09c"; } .fa-credit-card:before { - content: "\f09d"; + content: "\f09d"; } .fa-feed:before, .fa-rss:before { - content: "\f09e"; + content: "\f09e"; } .fa-hdd-o:before { - content: "\f0a0"; + content: "\f0a0"; } .fa-bullhorn:before { - content: "\f0a1"; + content: "\f0a1"; } .fa-bell:before { - content: "\f0f3"; + content: "\f0f3"; } .fa-certificate:before { - content: "\f0a3"; + content: "\f0a3"; } .fa-hand-o-right:before { - content: "\f0a4"; + content: "\f0a4"; } .fa-hand-o-left:before { - content: "\f0a5"; + content: "\f0a5"; } .fa-hand-o-up:before { - content: "\f0a6"; + content: "\f0a6"; } .fa-hand-o-down:before { - content: "\f0a7"; + content: "\f0a7"; } .fa-arrow-circle-left:before { - content: "\f0a8"; + content: "\f0a8"; } .fa-arrow-circle-right:before { - content: "\f0a9"; + content: "\f0a9"; } .fa-arrow-circle-up:before { - content: "\f0aa"; + content: "\f0aa"; } .fa-arrow-circle-down:before { - content: "\f0ab"; + content: "\f0ab"; } .fa-globe:before { - content: "\f0ac"; + content: "\f0ac"; } .fa-wrench:before { - content: "\f0ad"; + content: "\f0ad"; } .fa-tasks:before { - content: "\f0ae"; + content: "\f0ae"; } .fa-filter:before { - content: "\f0b0"; + content: "\f0b0"; } .fa-briefcase:before { - content: "\f0b1"; + content: "\f0b1"; } .fa-arrows-alt:before { - content: "\f0b2"; + content: "\f0b2"; } .fa-group:before, .fa-users:before { - content: "\f0c0"; + content: "\f0c0"; } .fa-chain:before, .fa-link:before { - content: "\f0c1"; + content: "\f0c1"; } .fa-cloud:before { - content: "\f0c2"; + content: "\f0c2"; } .fa-flask:before { - content: "\f0c3"; + content: "\f0c3"; } .fa-cut:before, .fa-scissors:before { - content: "\f0c4"; + content: "\f0c4"; } .fa-copy:before, .fa-files-o:before { - content: "\f0c5"; + content: "\f0c5"; } .fa-paperclip:before { - content: "\f0c6"; + content: "\f0c6"; } .fa-save:before, .fa-floppy-o:before { - content: "\f0c7"; + content: "\f0c7"; } .fa-square:before { - content: "\f0c8"; + content: "\f0c8"; } .fa-navicon:before, .fa-reorder:before, .fa-bars:before { - content: "\f0c9"; + content: "\f0c9"; } .fa-list-ul:before { - content: "\f0ca"; + content: "\f0ca"; } .fa-list-ol:before { - content: "\f0cb"; + content: "\f0cb"; } .fa-strikethrough:before { - content: "\f0cc"; + content: "\f0cc"; } .fa-underline:before { - content: "\f0cd"; + content: "\f0cd"; } .fa-table:before { - content: "\f0ce"; + content: "\f0ce"; } .fa-magic:before { - content: "\f0d0"; + content: "\f0d0"; } .fa-truck:before { - content: "\f0d1"; + content: "\f0d1"; } .fa-pinterest:before { - content: "\f0d2"; + content: "\f0d2"; } .fa-pinterest-square:before { - content: "\f0d3"; + content: "\f0d3"; } .fa-google-plus-square:before { - content: "\f0d4"; + content: "\f0d4"; } .fa-google-plus:before { - content: "\f0d5"; + content: "\f0d5"; } .fa-money:before { - content: "\f0d6"; + content: "\f0d6"; } .fa-caret-down:before { - content: "\f0d7"; + content: "\f0d7"; } .fa-caret-up:before { - content: "\f0d8"; + content: "\f0d8"; } .fa-caret-left:before { - content: "\f0d9"; + content: "\f0d9"; } .fa-caret-right:before { - content: "\f0da"; + content: "\f0da"; } .fa-columns:before { - content: "\f0db"; + content: "\f0db"; } .fa-unsorted:before, .fa-sort:before { - content: "\f0dc"; + content: "\f0dc"; } .fa-sort-down:before, .fa-sort-desc:before { - content: "\f0dd"; + content: "\f0dd"; } .fa-sort-up:before, .fa-sort-asc:before { - content: "\f0de"; + content: "\f0de"; } .fa-envelope:before { - content: "\f0e0"; + content: "\f0e0"; } .fa-linkedin:before { - content: "\f0e1"; + content: "\f0e1"; } .fa-rotate-left:before, .fa-undo:before { - content: "\f0e2"; + content: "\f0e2"; } .fa-legal:before, .fa-gavel:before { - content: "\f0e3"; + content: "\f0e3"; } .fa-dashboard:before, .fa-tachometer:before { - content: "\f0e4"; + content: "\f0e4"; } .fa-comment-o:before { - content: "\f0e5"; + content: "\f0e5"; } .fa-comments-o:before { - content: "\f0e6"; + content: "\f0e6"; } .fa-flash:before, .fa-bolt:before { - content: "\f0e7"; + content: "\f0e7"; } .fa-sitemap:before { - content: "\f0e8"; + content: "\f0e8"; } .fa-umbrella:before { - content: "\f0e9"; + content: "\f0e9"; } .fa-paste:before, .fa-clipboard:before { - content: "\f0ea"; + content: "\f0ea"; } .fa-lightbulb-o:before { - content: "\f0eb"; + content: "\f0eb"; } .fa-exchange:before { - content: "\f0ec"; + content: "\f0ec"; } .fa-cloud-download:before { - content: "\f0ed"; + content: "\f0ed"; } .fa-cloud-upload:before { - content: "\f0ee"; + content: "\f0ee"; } .fa-user-md:before { - content: "\f0f0"; + content: "\f0f0"; } .fa-stethoscope:before { - content: "\f0f1"; + content: "\f0f1"; } .fa-suitcase:before { - content: "\f0f2"; + content: "\f0f2"; } .fa-bell-o:before { - content: "\f0a2"; + content: "\f0a2"; } .fa-coffee:before { - content: "\f0f4"; + content: "\f0f4"; } .fa-cutlery:before { - content: "\f0f5"; + content: "\f0f5"; } .fa-file-text-o:before { - content: "\f0f6"; + content: "\f0f6"; } .fa-building-o:before { - content: "\f0f7"; + content: "\f0f7"; } .fa-hospital-o:before { - content: "\f0f8"; + content: "\f0f8"; } .fa-ambulance:before { - content: "\f0f9"; + content: "\f0f9"; } .fa-medkit:before { - content: "\f0fa"; + content: "\f0fa"; } .fa-fighter-jet:before { - content: "\f0fb"; + content: "\f0fb"; } .fa-beer:before { - content: "\f0fc"; + content: "\f0fc"; } .fa-h-square:before { - content: "\f0fd"; + content: "\f0fd"; } .fa-plus-square:before { - content: "\f0fe"; + content: "\f0fe"; } .fa-angle-double-left:before { - content: "\f100"; + content: "\f100"; } .fa-angle-double-right:before { - content: "\f101"; + content: "\f101"; } .fa-angle-double-up:before { - content: "\f102"; + content: "\f102"; } .fa-angle-double-down:before { - content: "\f103"; + content: "\f103"; } .fa-angle-left:before { - content: "\f104"; + content: "\f104"; } .fa-angle-right:before { - content: "\f105"; + content: "\f105"; } .fa-angle-up:before { - content: "\f106"; + content: "\f106"; } .fa-angle-down:before { - content: "\f107"; + content: "\f107"; } .fa-desktop:before { - content: "\f108"; + content: "\f108"; } .fa-laptop:before { - content: "\f109"; + content: "\f109"; } .fa-tablet:before { - content: "\f10a"; + content: "\f10a"; } .fa-mobile-phone:before, .fa-mobile:before { - content: "\f10b"; + content: "\f10b"; } .fa-circle-o:before { - content: "\f10c"; + content: "\f10c"; } .fa-quote-left:before { - content: "\f10d"; + content: "\f10d"; } .fa-quote-right:before { - content: "\f10e"; + content: "\f10e"; } .fa-spinner:before { - content: "\f110"; + content: "\f110"; } .fa-circle:before { - content: "\f111"; + content: "\f111"; } .fa-mail-reply:before, .fa-reply:before { - content: "\f112"; + content: "\f112"; } .fa-github-alt:before { - content: "\f113"; + content: "\f113"; } .fa-folder-o:before { - content: "\f114"; + content: "\f114"; } .fa-folder-open-o:before { - content: "\f115"; + content: "\f115"; } .fa-smile-o:before { - content: "\f118"; + content: "\f118"; } .fa-frown-o:before { - content: "\f119"; + content: "\f119"; } .fa-meh-o:before { - content: "\f11a"; + content: "\f11a"; } .fa-gamepad:before { - content: "\f11b"; + content: "\f11b"; } .fa-keyboard-o:before { - content: "\f11c"; + content: "\f11c"; } .fa-flag-o:before { - content: "\f11d"; + content: "\f11d"; } .fa-flag-checkered:before { - content: "\f11e"; + content: "\f11e"; } .fa-terminal:before { - content: "\f120"; + content: "\f120"; } .fa-code:before { - content: "\f121"; + content: "\f121"; } .fa-mail-reply-all:before, .fa-reply-all:before { - content: "\f122"; + content: "\f122"; } .fa-star-half-empty:before, .fa-star-half-full:before, .fa-star-half-o:before { - content: "\f123"; + content: "\f123"; } .fa-location-arrow:before { - content: "\f124"; + content: "\f124"; } .fa-crop:before { - content: "\f125"; + content: "\f125"; } .fa-code-fork:before { - content: "\f126"; + content: "\f126"; } .fa-unlink:before, .fa-chain-broken:before { - content: "\f127"; + content: "\f127"; } .fa-question:before { - content: "\f128"; + content: "\f128"; } .fa-info:before { - content: "\f129"; + content: "\f129"; } .fa-exclamation:before { - content: "\f12a"; + content: "\f12a"; } .fa-superscript:before { - content: "\f12b"; + content: "\f12b"; } .fa-subscript:before { - content: "\f12c"; + content: "\f12c"; } .fa-eraser:before { - content: "\f12d"; + content: "\f12d"; } .fa-puzzle-piece:before { - content: "\f12e"; + content: "\f12e"; } .fa-microphone:before { - content: "\f130"; + content: "\f130"; } .fa-microphone-slash:before { - content: "\f131"; + content: "\f131"; } .fa-shield:before { - content: "\f132"; + content: "\f132"; } .fa-calendar-o:before { - content: "\f133"; + content: "\f133"; } .fa-fire-extinguisher:before { - content: "\f134"; + content: "\f134"; } .fa-rocket:before { - content: "\f135"; + content: "\f135"; } .fa-maxcdn:before { - content: "\f136"; + content: "\f136"; } .fa-chevron-circle-left:before { - content: "\f137"; + content: "\f137"; } .fa-chevron-circle-right:before { - content: "\f138"; + content: "\f138"; } .fa-chevron-circle-up:before { - content: "\f139"; + content: "\f139"; } .fa-chevron-circle-down:before { - content: "\f13a"; + content: "\f13a"; } .fa-html5:before { - content: "\f13b"; + content: "\f13b"; } .fa-css3:before { - content: "\f13c"; + content: "\f13c"; } .fa-anchor:before { - content: "\f13d"; + content: "\f13d"; } .fa-unlock-alt:before { - content: "\f13e"; + content: "\f13e"; } .fa-bullseye:before { - content: "\f140"; + content: "\f140"; } .fa-ellipsis-h:before { - content: "\f141"; + content: "\f141"; } .fa-ellipsis-v:before { - content: "\f142"; + content: "\f142"; } .fa-rss-square:before { - content: "\f143"; + content: "\f143"; } .fa-play-circle:before { - content: "\f144"; + content: "\f144"; } .fa-ticket:before { - content: "\f145"; + content: "\f145"; } .fa-minus-square:before { - content: "\f146"; + content: "\f146"; } .fa-minus-square-o:before { - content: "\f147"; + content: "\f147"; } .fa-level-up:before { - content: "\f148"; + content: "\f148"; } .fa-level-down:before { - content: "\f149"; + content: "\f149"; } .fa-check-square:before { - content: "\f14a"; + content: "\f14a"; } .fa-pencil-square:before { - content: "\f14b"; + content: "\f14b"; } .fa-external-link-square:before { - content: "\f14c"; + content: "\f14c"; } .fa-share-square:before { - content: "\f14d"; + content: "\f14d"; } .fa-compass:before { - content: "\f14e"; + content: "\f14e"; } .fa-toggle-down:before, .fa-caret-square-o-down:before { - content: "\f150"; + content: "\f150"; } .fa-toggle-up:before, .fa-caret-square-o-up:before { - content: "\f151"; + content: "\f151"; } .fa-toggle-right:before, .fa-caret-square-o-right:before { - content: "\f152"; + content: "\f152"; } .fa-euro:before, .fa-eur:before { - content: "\f153"; + content: "\f153"; } .fa-gbp:before { - content: "\f154"; + content: "\f154"; } .fa-dollar:before, .fa-usd:before { - content: "\f155"; + content: "\f155"; } .fa-rupee:before, .fa-inr:before { - content: "\f156"; + content: "\f156"; } .fa-cny:before, .fa-rmb:before, .fa-yen:before, .fa-jpy:before { - content: "\f157"; + content: "\f157"; } .fa-ruble:before, .fa-rouble:before, .fa-rub:before { - content: "\f158"; + content: "\f158"; } .fa-won:before, .fa-krw:before { - content: "\f159"; + content: "\f159"; } .fa-bitcoin:before, .fa-btc:before { - content: "\f15a"; + content: "\f15a"; } .fa-file:before { - content: "\f15b"; + content: "\f15b"; } .fa-file-text:before { - content: "\f15c"; + content: "\f15c"; } .fa-sort-alpha-asc:before { - content: "\f15d"; + content: "\f15d"; } .fa-sort-alpha-desc:before { - content: "\f15e"; + content: "\f15e"; } .fa-sort-amount-asc:before { - content: "\f160"; + content: "\f160"; } .fa-sort-amount-desc:before { - content: "\f161"; + content: "\f161"; } .fa-sort-numeric-asc:before { - content: "\f162"; + content: "\f162"; } .fa-sort-numeric-desc:before { - content: "\f163"; + content: "\f163"; } .fa-thumbs-up:before { - content: "\f164"; + content: "\f164"; } .fa-thumbs-down:before { - content: "\f165"; + content: "\f165"; } .fa-youtube-square:before { - content: "\f166"; + content: "\f166"; } .fa-youtube:before { - content: "\f167"; + content: "\f167"; } .fa-xing:before { - content: "\f168"; + content: "\f168"; } .fa-xing-square:before { - content: "\f169"; + content: "\f169"; } .fa-youtube-play:before { - content: "\f16a"; + content: "\f16a"; } .fa-dropbox:before { - content: "\f16b"; + content: "\f16b"; } .fa-stack-overflow:before { - content: "\f16c"; + content: "\f16c"; } .fa-instagram:before { - content: "\f16d"; + content: "\f16d"; } .fa-flickr:before { - content: "\f16e"; + content: "\f16e"; } .fa-adn:before { - content: "\f170"; + content: "\f170"; } .fa-bitbucket:before { - content: "\f171"; + content: "\f171"; } .fa-bitbucket-square:before { - content: "\f172"; + content: "\f172"; } .fa-tumblr:before { - content: "\f173"; + content: "\f173"; } .fa-tumblr-square:before { - content: "\f174"; + content: "\f174"; } .fa-long-arrow-down:before { - content: "\f175"; + content: "\f175"; } .fa-long-arrow-up:before { - content: "\f176"; + content: "\f176"; } .fa-long-arrow-left:before { - content: "\f177"; + content: "\f177"; } .fa-long-arrow-right:before { - content: "\f178"; + content: "\f178"; } .fa-apple:before { - content: "\f179"; + content: "\f179"; } .fa-windows:before { - content: "\f17a"; + content: "\f17a"; } .fa-android:before { - content: "\f17b"; + content: "\f17b"; } .fa-linux:before { - content: "\f17c"; + content: "\f17c"; } .fa-dribbble:before { - content: "\f17d"; + content: "\f17d"; } .fa-skype:before { - content: "\f17e"; + content: "\f17e"; } .fa-foursquare:before { - content: "\f180"; + content: "\f180"; } .fa-trello:before { - content: "\f181"; + content: "\f181"; } .fa-female:before { - content: "\f182"; + content: "\f182"; } .fa-male:before { - content: "\f183"; + content: "\f183"; } .fa-gittip:before, .fa-gratipay:before { - content: "\f184"; + content: "\f184"; } .fa-sun-o:before { - content: "\f185"; + content: "\f185"; } .fa-moon-o:before { - content: "\f186"; + content: "\f186"; } .fa-archive:before { - content: "\f187"; + content: "\f187"; } .fa-bug:before { - content: "\f188"; + content: "\f188"; } .fa-vk:before { - content: "\f189"; + content: "\f189"; } .fa-weibo:before { - content: "\f18a"; + content: "\f18a"; } .fa-renren:before { - content: "\f18b"; + content: "\f18b"; } .fa-pagelines:before { - content: "\f18c"; + content: "\f18c"; } .fa-stack-exchange:before { - content: "\f18d"; + content: "\f18d"; } .fa-arrow-circle-o-right:before { - content: "\f18e"; + content: "\f18e"; } .fa-arrow-circle-o-left:before { - content: "\f190"; + content: "\f190"; } .fa-toggle-left:before, .fa-caret-square-o-left:before { - content: "\f191"; + content: "\f191"; } .fa-dot-circle-o:before { - content: "\f192"; + content: "\f192"; } .fa-wheelchair:before { - content: "\f193"; + content: "\f193"; } .fa-vimeo-square:before { - content: "\f194"; + content: "\f194"; } .fa-turkish-lira:before, .fa-try:before { - content: "\f195"; + content: "\f195"; } .fa-plus-square-o:before { - content: "\f196"; + content: "\f196"; } .fa-space-shuttle:before { - content: "\f197"; + content: "\f197"; } .fa-slack:before { - content: "\f198"; + content: "\f198"; } .fa-envelope-square:before { - content: "\f199"; + content: "\f199"; } .fa-wordpress:before { - content: "\f19a"; + content: "\f19a"; } .fa-openid:before { - content: "\f19b"; + content: "\f19b"; } .fa-institution:before, .fa-bank:before, .fa-university:before { - content: "\f19c"; + content: "\f19c"; } .fa-mortar-board:before, .fa-graduation-cap:before { - content: "\f19d"; + content: "\f19d"; } .fa-yahoo:before { - content: "\f19e"; + content: "\f19e"; } .fa-google:before { - content: "\f1a0"; + content: "\f1a0"; } .fa-reddit:before { - content: "\f1a1"; + content: "\f1a1"; } .fa-reddit-square:before { - content: "\f1a2"; + content: "\f1a2"; } .fa-stumbleupon-circle:before { - content: "\f1a3"; + content: "\f1a3"; } .fa-stumbleupon:before { - content: "\f1a4"; + content: "\f1a4"; } .fa-delicious:before { - content: "\f1a5"; + content: "\f1a5"; } .fa-digg:before { - content: "\f1a6"; + content: "\f1a6"; } .fa-pied-piper-pp:before { - content: "\f1a7"; + content: "\f1a7"; } .fa-pied-piper-alt:before { - content: "\f1a8"; + content: "\f1a8"; } .fa-drupal:before { - content: "\f1a9"; + content: "\f1a9"; } .fa-joomla:before { - content: "\f1aa"; + content: "\f1aa"; } .fa-language:before { - content: "\f1ab"; + content: "\f1ab"; } .fa-fax:before { - content: "\f1ac"; + content: "\f1ac"; } .fa-building:before { - content: "\f1ad"; + content: "\f1ad"; } .fa-child:before { - content: "\f1ae"; + content: "\f1ae"; } .fa-paw:before { - content: "\f1b0"; + content: "\f1b0"; } .fa-spoon:before { - content: "\f1b1"; + content: "\f1b1"; } .fa-cube:before { - content: "\f1b2"; + content: "\f1b2"; } .fa-cubes:before { - content: "\f1b3"; + content: "\f1b3"; } .fa-behance:before { - content: "\f1b4"; + content: "\f1b4"; } .fa-behance-square:before { - content: "\f1b5"; + content: "\f1b5"; } .fa-steam:before { - content: "\f1b6"; + content: "\f1b6"; } .fa-steam-square:before { - content: "\f1b7"; + content: "\f1b7"; } .fa-recycle:before { - content: "\f1b8"; + content: "\f1b8"; } .fa-automobile:before, .fa-car:before { - content: "\f1b9"; + content: "\f1b9"; } .fa-cab:before, .fa-taxi:before { - content: "\f1ba"; + content: "\f1ba"; } .fa-tree:before { - content: "\f1bb"; + content: "\f1bb"; } .fa-spotify:before { - content: "\f1bc"; + content: "\f1bc"; } .fa-deviantart:before { - content: "\f1bd"; + content: "\f1bd"; } .fa-soundcloud:before { - content: "\f1be"; + content: "\f1be"; } .fa-database:before { - content: "\f1c0"; + content: "\f1c0"; } .fa-file-pdf-o:before { - content: "\f1c1"; + content: "\f1c1"; } .fa-file-word-o:before { - content: "\f1c2"; + content: "\f1c2"; } .fa-file-excel-o:before { - content: "\f1c3"; + content: "\f1c3"; } .fa-file-powerpoint-o:before { - content: "\f1c4"; + content: "\f1c4"; } .fa-file-photo-o:before, .fa-file-picture-o:before, .fa-file-image-o:before { - content: "\f1c5"; + content: "\f1c5"; } .fa-file-zip-o:before, .fa-file-archive-o:before { - content: "\f1c6"; + content: "\f1c6"; } .fa-file-sound-o:before, .fa-file-audio-o:before { - content: "\f1c7"; + content: "\f1c7"; } .fa-file-movie-o:before, .fa-file-video-o:before { - content: "\f1c8"; + content: "\f1c8"; } .fa-file-code-o:before { - content: "\f1c9"; + content: "\f1c9"; } .fa-vine:before { - content: "\f1ca"; + content: "\f1ca"; } .fa-codepen:before { - content: "\f1cb"; + content: "\f1cb"; } .fa-jsfiddle:before { - content: "\f1cc"; + content: "\f1cc"; } .fa-life-bouy:before, .fa-life-buoy:before, .fa-life-saver:before, .fa-support:before, .fa-life-ring:before { - content: "\f1cd"; + content: "\f1cd"; } .fa-circle-o-notch:before { - content: "\f1ce"; + content: "\f1ce"; } .fa-ra:before, .fa-resistance:before, .fa-rebel:before { - content: "\f1d0"; + content: "\f1d0"; } .fa-ge:before, .fa-empire:before { - content: "\f1d1"; + content: "\f1d1"; } .fa-git-square:before { - content: "\f1d2"; + content: "\f1d2"; } .fa-git:before { - content: "\f1d3"; + content: "\f1d3"; } .fa-y-combinator-square:before, .fa-yc-square:before, .fa-hacker-news:before { - content: "\f1d4"; + content: "\f1d4"; } .fa-tencent-weibo:before { - content: "\f1d5"; + content: "\f1d5"; } .fa-qq:before { - content: "\f1d6"; + content: "\f1d6"; } .fa-wechat:before, .fa-weixin:before { - content: "\f1d7"; + content: "\f1d7"; } .fa-send:before, .fa-paper-plane:before { - content: "\f1d8"; + content: "\f1d8"; } .fa-send-o:before, .fa-paper-plane-o:before { - content: "\f1d9"; + content: "\f1d9"; } .fa-history:before { - content: "\f1da"; + content: "\f1da"; } .fa-circle-thin:before { - content: "\f1db"; + content: "\f1db"; } .fa-header:before { - content: "\f1dc"; + content: "\f1dc"; } .fa-paragraph:before { - content: "\f1dd"; + content: "\f1dd"; } .fa-sliders:before { - content: "\f1de"; + content: "\f1de"; } .fa-share-alt:before { - content: "\f1e0"; + content: "\f1e0"; } .fa-share-alt-square:before { - content: "\f1e1"; + content: "\f1e1"; } .fa-bomb:before { - content: "\f1e2"; + content: "\f1e2"; } .fa-soccer-ball-o:before, .fa-futbol-o:before { - content: "\f1e3"; + content: "\f1e3"; } .fa-tty:before { - content: "\f1e4"; + content: "\f1e4"; } .fa-binoculars:before { - content: "\f1e5"; + content: "\f1e5"; } .fa-plug:before { - content: "\f1e6"; + content: "\f1e6"; } .fa-slideshare:before { - content: "\f1e7"; + content: "\f1e7"; } .fa-twitch:before { - content: "\f1e8"; + content: "\f1e8"; } .fa-yelp:before { - content: "\f1e9"; + content: "\f1e9"; } .fa-newspaper-o:before { - content: "\f1ea"; + content: "\f1ea"; } .fa-wifi:before { - content: "\f1eb"; + content: "\f1eb"; } .fa-calculator:before { - content: "\f1ec"; + content: "\f1ec"; } .fa-paypal:before { - content: "\f1ed"; + content: "\f1ed"; } .fa-google-wallet:before { - content: "\f1ee"; + content: "\f1ee"; } .fa-cc-visa:before { - content: "\f1f0"; + content: "\f1f0"; } .fa-cc-mastercard:before { - content: "\f1f1"; + content: "\f1f1"; } .fa-cc-discover:before { - content: "\f1f2"; + content: "\f1f2"; } .fa-cc-amex:before { - content: "\f1f3"; + content: "\f1f3"; } .fa-cc-paypal:before { - content: "\f1f4"; + content: "\f1f4"; } .fa-cc-stripe:before { - content: "\f1f5"; + content: "\f1f5"; } .fa-bell-slash:before { - content: "\f1f6"; + content: "\f1f6"; } .fa-bell-slash-o:before { - content: "\f1f7"; + content: "\f1f7"; } .fa-trash:before { - content: "\f1f8"; + content: "\f1f8"; } .fa-copyright:before { - content: "\f1f9"; + content: "\f1f9"; } .fa-at:before { - content: "\f1fa"; + content: "\f1fa"; } .fa-eyedropper:before { - content: "\f1fb"; + content: "\f1fb"; } .fa-paint-brush:before { - content: "\f1fc"; + content: "\f1fc"; } .fa-birthday-cake:before { - content: "\f1fd"; + content: "\f1fd"; } .fa-area-chart:before { - content: "\f1fe"; + content: "\f1fe"; } .fa-pie-chart:before { - content: "\f200"; + content: "\f200"; } .fa-line-chart:before { - content: "\f201"; + content: "\f201"; } .fa-lastfm:before { - content: "\f202"; + content: "\f202"; } .fa-lastfm-square:before { - content: "\f203"; + content: "\f203"; } .fa-toggle-off:before { - content: "\f204"; + content: "\f204"; } .fa-toggle-on:before { - content: "\f205"; + content: "\f205"; } .fa-bicycle:before { - content: "\f206"; + content: "\f206"; } .fa-bus:before { - content: "\f207"; + content: "\f207"; } .fa-ioxhost:before { - content: "\f208"; + content: "\f208"; } .fa-angellist:before { - content: "\f209"; + content: "\f209"; } .fa-cc:before { - content: "\f20a"; + content: "\f20a"; } .fa-shekel:before, .fa-sheqel:before, .fa-ils:before { - content: "\f20b"; + content: "\f20b"; } .fa-meanpath:before { - content: "\f20c"; + content: "\f20c"; } .fa-buysellads:before { - content: "\f20d"; + content: "\f20d"; } .fa-connectdevelop:before { - content: "\f20e"; + content: "\f20e"; } .fa-dashcube:before { - content: "\f210"; + content: "\f210"; } .fa-forumbee:before { - content: "\f211"; + content: "\f211"; } .fa-leanpub:before { - content: "\f212"; + content: "\f212"; } .fa-sellsy:before { - content: "\f213"; + content: "\f213"; } .fa-shirtsinbulk:before { - content: "\f214"; + content: "\f214"; } .fa-simplybuilt:before { - content: "\f215"; + content: "\f215"; } .fa-skyatlas:before { - content: "\f216"; + content: "\f216"; } .fa-cart-plus:before { - content: "\f217"; + content: "\f217"; } .fa-cart-arrow-down:before { - content: "\f218"; + content: "\f218"; } .fa-diamond:before { - content: "\f219"; + content: "\f219"; } .fa-ship:before { - content: "\f21a"; + content: "\f21a"; } .fa-user-secret:before { - content: "\f21b"; + content: "\f21b"; } .fa-motorcycle:before { - content: "\f21c"; + content: "\f21c"; } .fa-street-view:before { - content: "\f21d"; + content: "\f21d"; } .fa-heartbeat:before { - content: "\f21e"; + content: "\f21e"; } .fa-venus:before { - content: "\f221"; + content: "\f221"; } .fa-mars:before { - content: "\f222"; + content: "\f222"; } .fa-mercury:before { - content: "\f223"; + content: "\f223"; } .fa-intersex:before, .fa-transgender:before { - content: "\f224"; + content: "\f224"; } .fa-transgender-alt:before { - content: "\f225"; + content: "\f225"; } .fa-venus-double:before { - content: "\f226"; + content: "\f226"; } .fa-mars-double:before { - content: "\f227"; + content: "\f227"; } .fa-venus-mars:before { - content: "\f228"; + content: "\f228"; } .fa-mars-stroke:before { - content: "\f229"; + content: "\f229"; } .fa-mars-stroke-v:before { - content: "\f22a"; + content: "\f22a"; } .fa-mars-stroke-h:before { - content: "\f22b"; + content: "\f22b"; } .fa-neuter:before { - content: "\f22c"; + content: "\f22c"; } .fa-genderless:before { - content: "\f22d"; + content: "\f22d"; } .fa-facebook-official:before { - content: "\f230"; + content: "\f230"; } .fa-pinterest-p:before { - content: "\f231"; + content: "\f231"; } .fa-whatsapp:before { - content: "\f232"; + content: "\f232"; } .fa-server:before { - content: "\f233"; + content: "\f233"; } .fa-user-plus:before { - content: "\f234"; + content: "\f234"; } .fa-user-times:before { - content: "\f235"; + content: "\f235"; } .fa-hotel:before, .fa-bed:before { - content: "\f236"; + content: "\f236"; } .fa-viacoin:before { - content: "\f237"; + content: "\f237"; } .fa-train:before { - content: "\f238"; + content: "\f238"; } .fa-subway:before { - content: "\f239"; + content: "\f239"; } .fa-medium:before { - content: "\f23a"; + content: "\f23a"; } .fa-yc:before, .fa-y-combinator:before { - content: "\f23b"; + content: "\f23b"; } .fa-optin-monster:before { - content: "\f23c"; + content: "\f23c"; } .fa-opencart:before { - content: "\f23d"; + content: "\f23d"; } .fa-expeditedssl:before { - content: "\f23e"; + content: "\f23e"; } .fa-battery-4:before, .fa-battery:before, .fa-battery-full:before { - content: "\f240"; + content: "\f240"; } .fa-battery-3:before, .fa-battery-three-quarters:before { - content: "\f241"; + content: "\f241"; } .fa-battery-2:before, .fa-battery-half:before { - content: "\f242"; + content: "\f242"; } .fa-battery-1:before, .fa-battery-quarter:before { - content: "\f243"; + content: "\f243"; } .fa-battery-0:before, .fa-battery-empty:before { - content: "\f244"; + content: "\f244"; } .fa-mouse-pointer:before { - content: "\f245"; + content: "\f245"; } .fa-i-cursor:before { - content: "\f246"; + content: "\f246"; } .fa-object-group:before { - content: "\f247"; + content: "\f247"; } .fa-object-ungroup:before { - content: "\f248"; + content: "\f248"; } .fa-sticky-note:before { - content: "\f249"; + content: "\f249"; } .fa-sticky-note-o:before { - content: "\f24a"; + content: "\f24a"; } .fa-cc-jcb:before { - content: "\f24b"; + content: "\f24b"; } .fa-cc-diners-club:before { - content: "\f24c"; + content: "\f24c"; } .fa-clone:before { - content: "\f24d"; + content: "\f24d"; } .fa-balance-scale:before { - content: "\f24e"; + content: "\f24e"; } .fa-hourglass-o:before { - content: "\f250"; + content: "\f250"; } .fa-hourglass-1:before, .fa-hourglass-start:before { - content: "\f251"; + content: "\f251"; } .fa-hourglass-2:before, .fa-hourglass-half:before { - content: "\f252"; + content: "\f252"; } .fa-hourglass-3:before, .fa-hourglass-end:before { - content: "\f253"; + content: "\f253"; } .fa-hourglass:before { - content: "\f254"; + content: "\f254"; } .fa-hand-grab-o:before, .fa-hand-rock-o:before { - content: "\f255"; + content: "\f255"; } .fa-hand-stop-o:before, .fa-hand-paper-o:before { - content: "\f256"; + content: "\f256"; } .fa-hand-scissors-o:before { - content: "\f257"; + content: "\f257"; } .fa-hand-lizard-o:before { - content: "\f258"; + content: "\f258"; } .fa-hand-spock-o:before { - content: "\f259"; + content: "\f259"; } .fa-hand-pointer-o:before { - content: "\f25a"; + content: "\f25a"; } .fa-hand-peace-o:before { - content: "\f25b"; + content: "\f25b"; } .fa-trademark:before { - content: "\f25c"; + content: "\f25c"; } .fa-registered:before { - content: "\f25d"; + content: "\f25d"; } .fa-creative-commons:before { - content: "\f25e"; + content: "\f25e"; } .fa-gg:before { - content: "\f260"; + content: "\f260"; } .fa-gg-circle:before { - content: "\f261"; + content: "\f261"; } .fa-tripadvisor:before { - content: "\f262"; + content: "\f262"; } .fa-odnoklassniki:before { - content: "\f263"; + content: "\f263"; } .fa-odnoklassniki-square:before { - content: "\f264"; + content: "\f264"; } .fa-get-pocket:before { - content: "\f265"; + content: "\f265"; } .fa-wikipedia-w:before { - content: "\f266"; + content: "\f266"; } .fa-safari:before { - content: "\f267"; + content: "\f267"; } .fa-chrome:before { - content: "\f268"; + content: "\f268"; } .fa-firefox:before { - content: "\f269"; + content: "\f269"; } .fa-opera:before { - content: "\f26a"; + content: "\f26a"; } .fa-internet-explorer:before { - content: "\f26b"; + content: "\f26b"; } .fa-tv:before, .fa-television:before { - content: "\f26c"; + content: "\f26c"; } .fa-contao:before { - content: "\f26d"; + content: "\f26d"; } .fa-500px:before { - content: "\f26e"; + content: "\f26e"; } .fa-amazon:before { - content: "\f270"; + content: "\f270"; } .fa-calendar-plus-o:before { - content: "\f271"; + content: "\f271"; } .fa-calendar-minus-o:before { - content: "\f272"; + content: "\f272"; } .fa-calendar-times-o:before { - content: "\f273"; + content: "\f273"; } .fa-calendar-check-o:before { - content: "\f274"; + content: "\f274"; } .fa-industry:before { - content: "\f275"; + content: "\f275"; } .fa-map-pin:before { - content: "\f276"; + content: "\f276"; } .fa-map-signs:before { - content: "\f277"; + content: "\f277"; } .fa-map-o:before { - content: "\f278"; + content: "\f278"; } .fa-map:before { - content: "\f279"; + content: "\f279"; } .fa-commenting:before { - content: "\f27a"; + content: "\f27a"; } .fa-commenting-o:before { - content: "\f27b"; + content: "\f27b"; } .fa-houzz:before { - content: "\f27c"; + content: "\f27c"; } .fa-vimeo:before { - content: "\f27d"; + content: "\f27d"; } .fa-black-tie:before { - content: "\f27e"; + content: "\f27e"; } .fa-fonticons:before { - content: "\f280"; + content: "\f280"; } .fa-reddit-alien:before { - content: "\f281"; + content: "\f281"; } .fa-edge:before { - content: "\f282"; + content: "\f282"; } .fa-credit-card-alt:before { - content: "\f283"; + content: "\f283"; } .fa-codiepie:before { - content: "\f284"; + content: "\f284"; } .fa-modx:before { - content: "\f285"; + content: "\f285"; } .fa-fort-awesome:before { - content: "\f286"; + content: "\f286"; } .fa-usb:before { - content: "\f287"; + content: "\f287"; } .fa-product-hunt:before { - content: "\f288"; + content: "\f288"; } .fa-mixcloud:before { - content: "\f289"; + content: "\f289"; } .fa-scribd:before { - content: "\f28a"; + content: "\f28a"; } .fa-pause-circle:before { - content: "\f28b"; + content: "\f28b"; } .fa-pause-circle-o:before { - content: "\f28c"; + content: "\f28c"; } .fa-stop-circle:before { - content: "\f28d"; + content: "\f28d"; } .fa-stop-circle-o:before { - content: "\f28e"; + content: "\f28e"; } .fa-shopping-bag:before { - content: "\f290"; + content: "\f290"; } .fa-shopping-basket:before { - content: "\f291"; + content: "\f291"; } .fa-hashtag:before { - content: "\f292"; + content: "\f292"; } .fa-bluetooth:before { - content: "\f293"; + content: "\f293"; } .fa-bluetooth-b:before { - content: "\f294"; + content: "\f294"; } .fa-percent:before { - content: "\f295"; + content: "\f295"; } .fa-gitlab:before { - content: "\f296"; + content: "\f296"; } .fa-wpbeginner:before { - content: "\f297"; + content: "\f297"; } .fa-wpforms:before { - content: "\f298"; + content: "\f298"; } .fa-envira:before { - content: "\f299"; + content: "\f299"; } .fa-universal-access:before { - content: "\f29a"; + content: "\f29a"; } .fa-wheelchair-alt:before { - content: "\f29b"; + content: "\f29b"; } .fa-question-circle-o:before { - content: "\f29c"; + content: "\f29c"; } .fa-blind:before { - content: "\f29d"; + content: "\f29d"; } .fa-audio-description:before { - content: "\f29e"; + content: "\f29e"; } .fa-volume-control-phone:before { - content: "\f2a0"; + content: "\f2a0"; } .fa-braille:before { - content: "\f2a1"; + content: "\f2a1"; } .fa-assistive-listening-systems:before { - content: "\f2a2"; + content: "\f2a2"; } .fa-asl-interpreting:before, .fa-american-sign-language-interpreting:before { - content: "\f2a3"; + content: "\f2a3"; } .fa-deafness:before, .fa-hard-of-hearing:before, .fa-deaf:before { - content: "\f2a4"; + content: "\f2a4"; } .fa-glide:before { - content: "\f2a5"; + content: "\f2a5"; } .fa-glide-g:before { - content: "\f2a6"; + content: "\f2a6"; } .fa-signing:before, .fa-sign-language:before { - content: "\f2a7"; + content: "\f2a7"; } .fa-low-vision:before { - content: "\f2a8"; + content: "\f2a8"; } .fa-viadeo:before { - content: "\f2a9"; + content: "\f2a9"; } .fa-viadeo-square:before { - content: "\f2aa"; + content: "\f2aa"; } .fa-snapchat:before { - content: "\f2ab"; + content: "\f2ab"; } .fa-snapchat-ghost:before { - content: "\f2ac"; + content: "\f2ac"; } .fa-snapchat-square:before { - content: "\f2ad"; + content: "\f2ad"; } .fa-pied-piper:before { - content: "\f2ae"; + content: "\f2ae"; } .fa-first-order:before { - content: "\f2b0"; + content: "\f2b0"; } .fa-yoast:before { - content: "\f2b1"; + content: "\f2b1"; } .fa-themeisle:before { - content: "\f2b2"; + content: "\f2b2"; } .fa-google-plus-circle:before, .fa-google-plus-official:before { - content: "\f2b3"; + content: "\f2b3"; } .fa-fa:before, .fa-font-awesome:before { - content: "\f2b4"; + content: "\f2b4"; } .fa-handshake-o:before { - content: "\f2b5"; + content: "\f2b5"; } .fa-envelope-open:before { - content: "\f2b6"; + content: "\f2b6"; } .fa-envelope-open-o:before { - content: "\f2b7"; + content: "\f2b7"; } .fa-linode:before { - content: "\f2b8"; + content: "\f2b8"; } .fa-address-book:before { - content: "\f2b9"; + content: "\f2b9"; } .fa-address-book-o:before { - content: "\f2ba"; + content: "\f2ba"; } .fa-vcard:before, .fa-address-card:before { - content: "\f2bb"; + content: "\f2bb"; } .fa-vcard-o:before, .fa-address-card-o:before { - content: "\f2bc"; + content: "\f2bc"; } .fa-user-circle:before { - content: "\f2bd"; + content: "\f2bd"; } .fa-user-circle-o:before { - content: "\f2be"; + content: "\f2be"; } .fa-user-o:before { - content: "\f2c0"; + content: "\f2c0"; } .fa-id-badge:before { - content: "\f2c1"; + content: "\f2c1"; } .fa-drivers-license:before, .fa-id-card:before { - content: "\f2c2"; + content: "\f2c2"; } .fa-drivers-license-o:before, .fa-id-card-o:before { - content: "\f2c3"; + content: "\f2c3"; } .fa-quora:before { - content: "\f2c4"; + content: "\f2c4"; } .fa-free-code-camp:before { - content: "\f2c5"; + content: "\f2c5"; } .fa-telegram:before { - content: "\f2c6"; + content: "\f2c6"; } .fa-thermometer-4:before, .fa-thermometer:before, .fa-thermometer-full:before { - content: "\f2c7"; + content: "\f2c7"; } .fa-thermometer-3:before, .fa-thermometer-three-quarters:before { - content: "\f2c8"; + content: "\f2c8"; } .fa-thermometer-2:before, .fa-thermometer-half:before { - content: "\f2c9"; + content: "\f2c9"; } .fa-thermometer-1:before, .fa-thermometer-quarter:before { - content: "\f2ca"; + content: "\f2ca"; } .fa-thermometer-0:before, .fa-thermometer-empty:before { - content: "\f2cb"; + content: "\f2cb"; } .fa-shower:before { - content: "\f2cc"; + content: "\f2cc"; } .fa-bathtub:before, .fa-s15:before, .fa-bath:before { - content: "\f2cd"; + content: "\f2cd"; } .fa-podcast:before { - content: "\f2ce"; + content: "\f2ce"; } .fa-window-maximize:before { - content: "\f2d0"; + content: "\f2d0"; } .fa-window-minimize:before { - content: "\f2d1"; + content: "\f2d1"; } .fa-window-restore:before { - content: "\f2d2"; + content: "\f2d2"; } .fa-times-rectangle:before, .fa-window-close:before { - content: "\f2d3"; + content: "\f2d3"; } .fa-times-rectangle-o:before, .fa-window-close-o:before { - content: "\f2d4"; + content: "\f2d4"; } .fa-bandcamp:before { - content: "\f2d5"; + content: "\f2d5"; } .fa-grav:before { - content: "\f2d6"; + content: "\f2d6"; } .fa-etsy:before { - content: "\f2d7"; + content: "\f2d7"; } .fa-imdb:before { - content: "\f2d8"; + content: "\f2d8"; } .fa-ravelry:before { - content: "\f2d9"; + content: "\f2d9"; } .fa-eercast:before { - content: "\f2da"; + content: "\f2da"; } .fa-microchip:before { - content: "\f2db"; + content: "\f2db"; } .fa-snowflake-o:before { - content: "\f2dc"; + content: "\f2dc"; } .fa-superpowers:before { - content: "\f2dd"; + content: "\f2dd"; } .fa-wpexplorer:before { - content: "\f2de"; + content: "\f2de"; } .fa-meetup:before { - content: "\f2e0"; + content: "\f2e0"; } .sr-only { - position: absolute; - width: 1px; - height: 1px; - padding: 0; - margin: -1px; - overflow: hidden; - clip: rect(0, 0, 0, 0); - border: 0; + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + border: 0; } .sr-only-focusable:active, .sr-only-focusable:focus { - position: static; - width: auto; - height: auto; - margin: 0; - overflow: visible; - clip: auto; + position: static; + width: auto; + height: auto; + margin: 0; + overflow: visible; + clip: auto; } diff --git a/web/src/js/__tests__/backends/staticSpec.tsx b/web/src/js/__tests__/backends/staticSpec.tsx index 003906b002..e892b5aded 100644 --- a/web/src/js/__tests__/backends/staticSpec.tsx +++ b/web/src/js/__tests__/backends/staticSpec.tsx @@ -1,7 +1,7 @@ -import {enableFetchMocks} from "jest-fetch-mock"; -import {TStore} from "../ducks/tutils"; +import { enableFetchMocks } from "jest-fetch-mock"; +import { TStore } from "../ducks/tutils"; import StaticBackend from "../../backends/static"; -import {waitFor} from "../test-utils"; +import { waitFor } from "../test-utils"; enableFetchMocks(); @@ -10,8 +10,20 @@ test("static backend", async () => { fetchMock.mockOnceIf("./options", "{}"); const store = TStore(); const backend = new StaticBackend(store); - await waitFor(() => expect(store.getActions()).toEqual([ - {type: "FLOWS_RECEIVE", cmd: "receive", data: [], resource: "flows"}, - {type: "OPTIONS_RECEIVE", cmd: "receive", data: {}, resource: "options"} - ])) + await waitFor(() => + expect(store.getActions()).toEqual([ + { + type: "FLOWS_RECEIVE", + cmd: "receive", + data: [], + resource: "flows", + }, + { + type: "OPTIONS_RECEIVE", + cmd: "receive", + data: {}, + resource: "options", + }, + ]) + ); }); diff --git a/web/src/js/__tests__/backends/websocketSpec.tsx b/web/src/js/__tests__/backends/websocketSpec.tsx index ba74985c2c..134de711df 100644 --- a/web/src/js/__tests__/backends/websocketSpec.tsx +++ b/web/src/js/__tests__/backends/websocketSpec.tsx @@ -1,14 +1,16 @@ -import {enableFetchMocks} from "jest-fetch-mock"; -import {TStore} from "../ducks/tutils"; +import { enableFetchMocks } from "jest-fetch-mock"; +import { TStore } from "../ducks/tutils"; import WebSocketBackend from "../../backends/websocket"; -import {waitFor} from "../test-utils"; +import { waitFor } from "../test-utils"; import * as connectionActions from "../../ducks/connection"; enableFetchMocks(); test("websocket backend", async () => { // @ts-ignore - jest.spyOn(global, 'WebSocket').mockImplementation(() => ({addEventListener: () => 0})); + jest.spyOn(global, "WebSocket").mockImplementation(() => ({ + addEventListener: () => 0, + })); fetchMock.mockOnceIf("./state", "{}"); fetchMock.mockOnceIf("./flows", "[]"); @@ -19,45 +21,78 @@ test("websocket backend", async () => { backend.onOpen(); - await waitFor(() => expect(store.getActions()).toEqual([ - connectionActions.startFetching(), - {type: "STATE_RECEIVE", cmd: "receive", data: {}, resource: "state"}, - {type: "FLOWS_RECEIVE", cmd: "receive", data: [], resource: "flows"}, - {type: "EVENTS_RECEIVE", cmd: "receive", data: [], resource: "events"}, - {type: "OPTIONS_RECEIVE", cmd: "receive", data: {}, resource: "options"}, - connectionActions.connectionEstablished(), - ])) + await waitFor(() => + expect(store.getActions()).toEqual([ + connectionActions.startFetching(), + { + type: "STATE_RECEIVE", + cmd: "receive", + data: {}, + resource: "state", + }, + { + type: "FLOWS_RECEIVE", + cmd: "receive", + data: [], + resource: "flows", + }, + { + type: "EVENTS_RECEIVE", + cmd: "receive", + data: [], + resource: "events", + }, + { + type: "OPTIONS_RECEIVE", + cmd: "receive", + data: {}, + resource: "options", + }, + connectionActions.connectionEstablished(), + ]) + ); store.clearActions(); backend.onMessage({ - "resource": "events", - "cmd": "add", - "data": {"id": "42", "message": "test", "level": "info"} + resource: "events", + cmd: "add", + data: { id: "42", message: "test", level: "info" }, }); - expect(store.getActions()).toEqual([{ - "cmd": "add", - "data": {"id": "42", "level": "info", "message": "test"}, - "resource": "events", - "type": "EVENTS_ADD" - }]); + expect(store.getActions()).toEqual([ + { + cmd: "add", + data: { id: "42", level: "info", message: "test" }, + resource: "events", + type: "EVENTS_ADD", + }, + ]); store.clearActions(); fetchMock.mockOnceIf("./events", "[]"); backend.onMessage({ - "resource": "events", - "cmd": "reset", + resource: "events", + cmd: "reset", }); - await waitFor(() => expect(store.getActions()).toEqual([ - {type: "EVENTS_RECEIVE", cmd: "receive", data: [], resource: "events"}, - connectionActions.connectionEstablished(), - ])) - store.clearActions() + await waitFor(() => + expect(store.getActions()).toEqual([ + { + type: "EVENTS_RECEIVE", + cmd: "receive", + data: [], + resource: "events", + }, + connectionActions.connectionEstablished(), + ]) + ); + store.clearActions(); expect(fetchMock.mock.calls).toHaveLength(5); console.error = jest.fn(); backend.onClose(new CloseEvent("Connection closed")); expect(console.error).toBeCalledTimes(1); - expect(store.getActions()[0].type).toEqual(connectionActions.ConnectionState.ERROR); + expect(store.getActions()[0].type).toEqual( + connectionActions.ConnectionState.ERROR + ); store.clearActions(); backend.onError(null); diff --git a/web/src/js/__tests__/components/CaptureSetupSpec.tsx b/web/src/js/__tests__/components/CaptureSetupSpec.tsx index 57ba97821f..9033ed5947 100644 --- a/web/src/js/__tests__/components/CaptureSetupSpec.tsx +++ b/web/src/js/__tests__/components/CaptureSetupSpec.tsx @@ -1,11 +1,10 @@ -import * as React from "react" -import {render} from "../test-utils"; +import * as React from "react"; +import { render } from "../test-utils"; import CaptureSetup from "../../components/CaptureSetup"; -import {TStore} from "../ducks/tutils"; - +import { TStore } from "../ducks/tutils"; test("CaptureSetup", async () => { const store = TStore(), - {asFragment} = render(, {store}); + { asFragment } = render(, { store }); expect(asFragment()).toMatchSnapshot(); }); diff --git a/web/src/js/__tests__/components/CommandBarSpec.tsx b/web/src/js/__tests__/components/CommandBarSpec.tsx index 47e6bfcea1..a418d3ddbf 100644 --- a/web/src/js/__tests__/components/CommandBarSpec.tsx +++ b/web/src/js/__tests__/components/CommandBarSpec.tsx @@ -1,55 +1,86 @@ -import * as React from "react" -import {render, screen, userEvent, waitFor} from "../test-utils"; +import * as React from "react"; +import { render, screen, userEvent, waitFor } from "../test-utils"; import CommandBar from "../../components/CommandBar"; -import fetchMock, {enableFetchMocks} from "jest-fetch-mock"; +import fetchMock, { enableFetchMocks } from "jest-fetch-mock"; enableFetchMocks(); test("CommandBar", async () => { - fetchMock.mockOnceIf("./commands", JSON.stringify({ + fetchMock.mockOnceIf( + "./commands", + JSON.stringify({ "flow.decode": { - "help": "Decode flows.", - "parameters": [ - {"name": "flows", "type": "flow[]", "kind": "POSITIONAL_OR_KEYWORD"}, - {"name": "part", "type": "str", "kind": "POSITIONAL_OR_KEYWORD"} + help: "Decode flows.", + parameters: [ + { + name: "flows", + type: "flow[]", + kind: "POSITIONAL_OR_KEYWORD", + }, + { + name: "part", + type: "str", + kind: "POSITIONAL_OR_KEYWORD", + }, ], - "return_type": null, - "signature_help": "flow.decode flows part" + return_type: null, + signature_help: "flow.decode flows part", }, "flow.encode": { - "help": "Encode flows with a specified encoding.", - "parameters": [ - {"name": "flows", "type": "flow[]", "kind": "POSITIONAL_OR_KEYWORD"}, - {"name": "part", "type": "str", "kind": "POSITIONAL_OR_KEYWORD"}, - {"name": "encoding", "type": "choice", "kind": "POSITIONAL_OR_KEYWORD"} + help: "Encode flows with a specified encoding.", + parameters: [ + { + name: "flows", + type: "flow[]", + kind: "POSITIONAL_OR_KEYWORD", + }, + { + name: "part", + type: "str", + kind: "POSITIONAL_OR_KEYWORD", + }, + { + name: "encoding", + type: "choice", + kind: "POSITIONAL_OR_KEYWORD", + }, ], - "return_type": null, - "signature_help": "flow.encode flows part encoding" - } - } - )); - fetchMock.mockOnceIf("./commands/commands.history.get", JSON.stringify({value: ["foo"]})); - fetchMock.mockOnceIf("./commands/commands.history.add", JSON.stringify({value: null})); - fetchMock.mockOnceIf("./commands/flow.encode", JSON.stringify({value: null})); + return_type: null, + signature_help: "flow.encode flows part encoding", + }, + }) + ); + fetchMock.mockOnceIf( + "./commands/commands.history.get", + JSON.stringify({ value: ["foo"] }) + ); + fetchMock.mockOnceIf( + "./commands/commands.history.add", + JSON.stringify({ value: null }) + ); + fetchMock.mockOnceIf( + "./commands/flow.encode", + JSON.stringify({ value: null }) + ); - const {asFragment} = render(); + const { asFragment } = render(); expect(asFragment()).toMatchSnapshot(); - await waitFor(() => screen.getByText('["flow.decode","flow.encode"]')) + await waitFor(() => screen.getByText('["flow.decode","flow.encode"]')); expect(asFragment()).toMatchSnapshot(); const input = screen.getByPlaceholderText("Enter command"); - userEvent.type(input, 'x'); + userEvent.type(input, "x"); expect(screen.getByText("[]")).toBeInTheDocument(); userEvent.type(input, "{backspace}"); - userEvent.type(input, 'fl'); + userEvent.type(input, "fl"); userEvent.tab(); - expect(input).toHaveValue('flow.decode'); + expect(input).toHaveValue("flow.decode"); userEvent.tab(); - expect(input).toHaveValue('flow.encode'); + expect(input).toHaveValue("flow.encode"); - fetchMock.mockOnce(JSON.stringify({value: null})); + fetchMock.mockOnce(JSON.stringify({ value: null })); userEvent.type(input, "{enter}"); await waitFor(() => screen.getByText("Command Result")); diff --git a/web/src/js/__tests__/components/EventLog/EventListSpec.tsx b/web/src/js/__tests__/components/EventLog/EventListSpec.tsx index 30e41d336d..53e5c7f0ae 100644 --- a/web/src/js/__tests__/components/EventLog/EventListSpec.tsx +++ b/web/src/js/__tests__/components/EventLog/EventListSpec.tsx @@ -1,22 +1,27 @@ -import * as React from "react" -import EventLogList from '../../../components/EventLog/EventList' -import TestUtils from 'react-dom/test-utils' +import * as React from "react"; +import EventLogList from "../../../components/EventLog/EventList"; +import TestUtils from "react-dom/test-utils"; -describe('EventList Component', () => { - let mockEventList = [ - { id: 1, level: 'info', message: 'foo' }, - { id: 2, level: 'error', message: 'bar' } +describe("EventList Component", () => { + let mockEventList = [ + { id: 1, level: "info", message: "foo" }, + { id: 2, level: "error", message: "bar" }, ], - eventLogList = TestUtils.renderIntoDocument() + eventLogList = TestUtils.renderIntoDocument( + + ); - it('should render correctly', () => { - expect(eventLogList.state).toMatchSnapshot() - expect(eventLogList.props).toMatchSnapshot() - }) + it("should render correctly", () => { + expect(eventLogList.state).toMatchSnapshot(); + expect(eventLogList.props).toMatchSnapshot(); + }); - it('should handle componentWillUnmount', () => { - window.removeEventListener = jest.fn() - eventLogList.componentWillUnmount() - expect(window.removeEventListener).toBeCalledWith('resize', eventLogList.onViewportUpdate) - }) -}) + it("should handle componentWillUnmount", () => { + window.removeEventListener = jest.fn(); + eventLogList.componentWillUnmount(); + expect(window.removeEventListener).toBeCalledWith( + "resize", + eventLogList.onViewportUpdate + ); + }); +}); diff --git a/web/src/js/__tests__/components/EventLogSpec.tsx b/web/src/js/__tests__/components/EventLogSpec.tsx index e6a74a5463..209f2dd319 100644 --- a/web/src/js/__tests__/components/EventLogSpec.tsx +++ b/web/src/js/__tests__/components/EventLogSpec.tsx @@ -1,56 +1,71 @@ -jest.mock('../../components/EventLog/EventList') +jest.mock("../../components/EventLog/EventList"); -import * as React from "react" -import renderer from 'react-test-renderer' -import EventLog, {PureEventLog} from '../../components/EventLog' -import {Provider} from 'react-redux' -import {TStore} from '../ducks/tutils' +import * as React from "react"; +import renderer from "react-test-renderer"; +import EventLog, { PureEventLog } from "../../components/EventLog"; +import { Provider } from "react-redux"; +import { TStore } from "../ducks/tutils"; -window.addEventListener = jest.fn() -window.removeEventListener = jest.fn() +window.addEventListener = jest.fn(); +window.removeEventListener = jest.fn(); -describe('EventLog Component', () => { +describe("EventLog Component", () => { let store = TStore(), provider = renderer.create( - - ), - tree = provider.toJSON() + + + ), + tree = provider.toJSON(); - it('should connect to state and render correctly', () => { - expect(tree).toMatchSnapshot() - }) + it("should connect to state and render correctly", () => { + expect(tree).toMatchSnapshot(); + }); - it('should handle toggleFilter', () => { - let debugToggleButton = tree.children[0].children[1].children[0] - debugToggleButton.props.onClick() - }) + it("should handle toggleFilter", () => { + let debugToggleButton = tree.children[0].children[1].children[0]; + debugToggleButton.props.onClick(); + }); provider = renderer.create( - ) + + + + ); let eventLog = provider.root.findByType(PureEventLog), - mockEvent = {preventDefault: jest.fn()} - - it('should handle DragStart', () => { - eventLog.instance.onDragStart(mockEvent) - expect(mockEvent.preventDefault).toBeCalled() - expect(window.addEventListener).toBeCalledWith('mousemove', eventLog.instance.onDragMove) - expect(window.addEventListener).toBeCalledWith('mouseup', eventLog.instance.onDragStop) - expect(window.addEventListener).toBeCalledWith('dragend', eventLog.instance.onDragStop) - mockEvent.preventDefault.mockClear() - }) - - it('should handle DragMove', () => { - eventLog.instance.onDragMove(mockEvent) - expect(mockEvent.preventDefault).toBeCalled() - mockEvent.preventDefault.mockClear() - }) - - console.error = jest.fn() // silent the error. - it('should handle DragStop', () => { - eventLog.instance.onDragStop(mockEvent) - expect(mockEvent.preventDefault).toBeCalled() - expect(window.removeEventListener).toBeCalledWith('mousemove', eventLog.instance.onDragMove) - }) - -}) + mockEvent = { preventDefault: jest.fn() }; + + it("should handle DragStart", () => { + eventLog.instance.onDragStart(mockEvent); + expect(mockEvent.preventDefault).toBeCalled(); + expect(window.addEventListener).toBeCalledWith( + "mousemove", + eventLog.instance.onDragMove + ); + expect(window.addEventListener).toBeCalledWith( + "mouseup", + eventLog.instance.onDragStop + ); + expect(window.addEventListener).toBeCalledWith( + "dragend", + eventLog.instance.onDragStop + ); + mockEvent.preventDefault.mockClear(); + }); + + it("should handle DragMove", () => { + eventLog.instance.onDragMove(mockEvent); + expect(mockEvent.preventDefault).toBeCalled(); + mockEvent.preventDefault.mockClear(); + }); + + console.error = jest.fn(); // silent the error. + it("should handle DragStop", () => { + eventLog.instance.onDragStop(mockEvent); + expect(mockEvent.preventDefault).toBeCalled(); + expect(window.removeEventListener).toBeCalledWith( + "mousemove", + eventLog.instance.onDragMove + ); + }); +}); diff --git a/web/src/js/__tests__/components/FlowTable/FlowColumnsSpec.tsx b/web/src/js/__tests__/components/FlowTable/FlowColumnsSpec.tsx index 27e2b3950b..cb1d3f1a7a 100644 --- a/web/src/js/__tests__/components/FlowTable/FlowColumnsSpec.tsx +++ b/web/src/js/__tests__/components/FlowTable/FlowColumnsSpec.tsx @@ -1,103 +1,112 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import FlowColumns from '../../../components/FlowTable/FlowColumns' -import {TFlow, TTCPFlow} from '../../ducks/tutils' -import {render} from "../../test-utils"; +import * as React from "react"; +import renderer from "react-test-renderer"; +import FlowColumns from "../../../components/FlowTable/FlowColumns"; +import { TFlow, TTCPFlow } from "../../ducks/tutils"; +import { render } from "../../test-utils"; test("should render columns", async () => { const tflow = TFlow(); Object.entries(FlowColumns).forEach(([name, Col]) => { - const {asFragment} = render( - - - -
    ) + const { asFragment } = render( + + + + + + +
    + ); expect(asFragment()).toMatchSnapshot(name); - }) + }); }); - -describe('Flowcolumns Components', () => { - it('should render IconColumn', () => { +describe("Flowcolumns Components", () => { + it("should render IconColumn", () => { let tcpflow = TTCPFlow(), - iconColumn = renderer.create(), - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + iconColumn = renderer.create(), + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); - let tflow = {...TFlow(), websocket: undefined}; - iconColumn = renderer.create() - tree = iconColumn.toJSON() + let tflow = { ...TFlow(), websocket: undefined }; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); // plain - expect(tree).toMatchSnapshot() + expect(tree).toMatchSnapshot(); // not modified - tflow.response.status_code = 304 - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + tflow.response.status_code = 304; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); // redirect - tflow.response.status_code = 302 - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + tflow.response.status_code = 302; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); // image - let imageFlow = {...TFlow(), websocket: undefined} - imageFlow.response.headers = [['Content-Type', 'image/jpeg']] - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + let imageFlow = { ...TFlow(), websocket: undefined }; + imageFlow.response.headers = [["Content-Type", "image/jpeg"]]; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); // javascript - let jsFlow = {...TFlow(), websocket: undefined} - jsFlow.response.headers = [['Content-Type', 'application/x-javascript']] - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + let jsFlow = { ...TFlow(), websocket: undefined }; + jsFlow.response.headers = [ + ["Content-Type", "application/x-javascript"], + ]; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); // css - let cssFlow = {...TFlow(), websocket: undefined} - cssFlow.response.headers = [['Content-Type', 'text/css']] - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + let cssFlow = { ...TFlow(), websocket: undefined }; + cssFlow.response.headers = [["Content-Type", "text/css"]]; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); // html - let htmlFlow = {...TFlow(), websocket: undefined} - htmlFlow.response.headers = [['Content-Type', 'text/html']] - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + let htmlFlow = { ...TFlow(), websocket: undefined }; + htmlFlow.response.headers = [["Content-Type", "text/html"]]; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); // default - let fooFlow = {...TFlow(), websocket: undefined} - fooFlow.response.headers = [['Content-Type', 'foo']] - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() + let fooFlow = { ...TFlow(), websocket: undefined }; + fooFlow.response.headers = [["Content-Type", "foo"]]; + iconColumn = renderer.create(); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); // no response - let noResponseFlow = {...TFlow(), response: undefined} - iconColumn = renderer.create() - tree = iconColumn.toJSON() - expect(tree).toMatchSnapshot() - }) + let noResponseFlow = { ...TFlow(), response: undefined }; + iconColumn = renderer.create( + + ); + tree = iconColumn.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should render pathColumn', () => { + it("should render pathColumn", () => { let tflow = TFlow(), - pathColumn = renderer.create(), - tree = pathColumn.toJSON() - expect(tree).toMatchSnapshot() + pathColumn = renderer.create(), + tree = pathColumn.toJSON(); + expect(tree).toMatchSnapshot(); - tflow.error.msg = 'Connection killed.' - tflow.intercepted = true - pathColumn = renderer.create() - tree = pathColumn.toJSON() - expect(tree).toMatchSnapshot() - }) + tflow.error.msg = "Connection killed."; + tflow.intercepted = true; + pathColumn = renderer.create(); + tree = pathColumn.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should render TimeColumn', () => { + it("should render TimeColumn", () => { let tflow = TFlow(), - timeColumn = renderer.create(), - tree = timeColumn.toJSON() - expect(tree).toMatchSnapshot() + timeColumn = renderer.create(), + tree = timeColumn.toJSON(); + expect(tree).toMatchSnapshot(); - let noResponseFlow = {...tflow, response: undefined} - timeColumn = renderer.create() - tree = timeColumn.toJSON() - expect(tree).toMatchSnapshot() - }) -}) + let noResponseFlow = { ...tflow, response: undefined }; + timeColumn = renderer.create( + + ); + tree = timeColumn.toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/web/src/js/__tests__/components/FlowTable/FlowRowSpec.tsx b/web/src/js/__tests__/components/FlowTable/FlowRowSpec.tsx index c1361ac3cc..6f059227f2 100644 --- a/web/src/js/__tests__/components/FlowTable/FlowRowSpec.tsx +++ b/web/src/js/__tests__/components/FlowTable/FlowRowSpec.tsx @@ -1,21 +1,27 @@ -import * as React from "react" -import FlowRow from '../../../components/FlowTable/FlowRow' -import {testState} from '../../ducks/tutils' -import {fireEvent, render, screen} from "../../test-utils"; -import {createAppStore} from "../../../ducks"; - +import * as React from "react"; +import FlowRow from "../../../components/FlowTable/FlowRow"; +import { testState } from "../../ducks/tutils"; +import { fireEvent, render, screen } from "../../test-utils"; +import { createAppStore } from "../../../ducks"; test("FlowRow", async () => { const store = createAppStore(testState), tflow2 = store.getState().flows.list[0], - {asFragment} = render( - - - -
    , {store}) - expect(asFragment()).toMatchSnapshot() + { asFragment } = render( + + + + +
    , + { store } + ); + expect(asFragment()).toMatchSnapshot(); - expect(store.getState().flows.selected[0]).not.toBe(store.getState().flows.list[0].id) - fireEvent.click(screen.getByText("http://address:22/path")) - expect(store.getState().flows.selected[0]).toBe(store.getState().flows.list[0].id) -}) + expect(store.getState().flows.selected[0]).not.toBe( + store.getState().flows.list[0].id + ); + fireEvent.click(screen.getByText("http://address:22/path")); + expect(store.getState().flows.selected[0]).toBe( + store.getState().flows.list[0].id + ); +}); diff --git a/web/src/js/__tests__/components/FlowTable/FlowTableHeadSpec.tsx b/web/src/js/__tests__/components/FlowTable/FlowTableHeadSpec.tsx index 511c2ccc26..00d1810cf1 100644 --- a/web/src/js/__tests__/components/FlowTable/FlowTableHeadSpec.tsx +++ b/web/src/js/__tests__/components/FlowTable/FlowTableHeadSpec.tsx @@ -1,29 +1,24 @@ -import * as React from "react" -import FlowTableHead from '../../../components/FlowTable/FlowTableHead' -import {Provider} from 'react-redux' -import {TStore} from '../../ducks/tutils' -import {fireEvent, render, screen} from "@testing-library/react"; -import {setSort} from "../../../ducks/flows"; - +import * as React from "react"; +import FlowTableHead from "../../../components/FlowTable/FlowTableHead"; +import { Provider } from "react-redux"; +import { TStore } from "../../ducks/tutils"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { setSort } from "../../../ducks/flows"; test("FlowTableHead Component", async () => { - const store = TStore(), - {asFragment} = render( + { asFragment } = render( - +
    - ) - expect(asFragment()).toMatchSnapshot() + ); + expect(asFragment()).toMatchSnapshot(); - fireEvent.click(screen.getByText("Size")) + fireEvent.click(screen.getByText("Size")); - expect(store.getActions()).toStrictEqual([ - setSort("size", false) - ] - ) -}) + expect(store.getActions()).toStrictEqual([setSort("size", false)]); +}); diff --git a/web/src/js/__tests__/components/FlowTableSpec.tsx b/web/src/js/__tests__/components/FlowTableSpec.tsx index 9bc67ebf43..4944fd10bf 100644 --- a/web/src/js/__tests__/components/FlowTableSpec.tsx +++ b/web/src/js/__tests__/components/FlowTableSpec.tsx @@ -1,50 +1,57 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import {PureFlowTable as FlowTable} from '../../components/FlowTable' -import TestUtils from 'react-dom/test-utils' -import { TFlow, TStore } from '../ducks/tutils' -import { Provider } from 'react-redux' +import * as React from "react"; +import renderer from "react-test-renderer"; +import { PureFlowTable as FlowTable } from "../../components/FlowTable"; +import TestUtils from "react-dom/test-utils"; +import { TFlow, TStore } from "../ducks/tutils"; +import { Provider } from "react-redux"; -window.addEventListener = jest.fn() +window.addEventListener = jest.fn(); -describe('FlowTable Component', () => { +describe("FlowTable Component", () => { let selectFn = jest.fn(), tflow = TFlow(), - store = TStore() + store = TStore(); - it('should render correctly', () => { + it("should render correctly", () => { let provider = renderer.create( - - ), - tree = provider.toJSON() - expect(tree).toMatchSnapshot() - }) + + + ), + tree = provider.toJSON(); + expect(tree).toMatchSnapshot(); + }); let provider = renderer.create( - - - ), - flowTable = provider.root.findByType(FlowTable) + + + + ), + flowTable = provider.root.findByType(FlowTable); - it('should handle componentWillUnmount', () => { - flowTable.instance.UNSAFE_componentWillUnmount() - expect(window.addEventListener).toBeCalledWith('resize', flowTable.instance.onViewportUpdate) - }) + it("should handle componentWillUnmount", () => { + flowTable.instance.UNSAFE_componentWillUnmount(); + expect(window.addEventListener).toBeCalledWith( + "resize", + flowTable.instance.onViewportUpdate + ); + }); - it('should handle componentDidUpdate', () => { + it("should handle componentDidUpdate", () => { // flowTable.shouldScrollIntoView == false - expect(flowTable.instance.componentDidUpdate()).toEqual(undefined) + expect(flowTable.instance.componentDidUpdate()).toEqual(undefined); // rowTop - headHeight < viewportTop - flowTable.instance.shouldScrollIntoView = true - flowTable.instance.componentDidUpdate() + flowTable.instance.shouldScrollIntoView = true; + flowTable.instance.componentDidUpdate(); // rowBottom > viewportTop + viewportHeight - flowTable.instance.shouldScrollIntoView = true - flowTable.instance.componentDidUpdate() - }) + flowTable.instance.shouldScrollIntoView = true; + flowTable.instance.componentDidUpdate(); + }); - it('should handle componentWillReceiveProps', () => { - flowTable.instance.UNSAFE_componentWillReceiveProps({selected: tflow}) - expect(flowTable.instance.shouldScrollIntoView).toBeTruthy() - }) -}) + it("should handle componentWillReceiveProps", () => { + flowTable.instance.UNSAFE_componentWillReceiveProps({ + selected: tflow, + }); + expect(flowTable.instance.shouldScrollIntoView).toBeTruthy(); + }); +}); diff --git a/web/src/js/__tests__/components/FlowViewSpec.tsx b/web/src/js/__tests__/components/FlowViewSpec.tsx index ab57fa1320..4cf86a3df1 100644 --- a/web/src/js/__tests__/components/FlowViewSpec.tsx +++ b/web/src/js/__tests__/components/FlowViewSpec.tsx @@ -1,16 +1,16 @@ -import * as React from "react" -import {render, screen} from "../test-utils"; +import * as React from "react"; +import { render, screen } from "../test-utils"; import FlowView from "../../components/FlowView"; -import * as flowActions from "../../ducks/flows" -import fetchMock, {enableFetchMocks} from "jest-fetch-mock"; -import {fireEvent} from "@testing-library/react"; +import * as flowActions from "../../ducks/flows"; +import fetchMock, { enableFetchMocks } from "jest-fetch-mock"; +import { fireEvent } from "@testing-library/react"; enableFetchMocks(); test("FlowView", async () => { fetchMock.mockReject(new Error("backend missing")); - const {asFragment, store} = render(); + const { asFragment, store } = render(); expect(asFragment()).toMatchSnapshot(); fireEvent.click(screen.getByText("Response")); diff --git a/web/src/js/__tests__/components/Header/ConnectionIndicatorSpec.tsx b/web/src/js/__tests__/components/Header/ConnectionIndicatorSpec.tsx index 078c1c267a..fb964f3f12 100644 --- a/web/src/js/__tests__/components/Header/ConnectionIndicatorSpec.tsx +++ b/web/src/js/__tests__/components/Header/ConnectionIndicatorSpec.tsx @@ -1,22 +1,21 @@ -import * as React from "react" -import ConnectionIndicator from '../../../components/Header/ConnectionIndicator' -import * as connectionActions from '../../../ducks/connection' -import {render} from "../../test-utils" - +import * as React from "react"; +import ConnectionIndicator from "../../../components/Header/ConnectionIndicator"; +import * as connectionActions from "../../../ducks/connection"; +import { render } from "../../test-utils"; test("ConnectionIndicator", async () => { - const {asFragment, store} = render(); - expect(asFragment()).toMatchSnapshot() + const { asFragment, store } = render(); + expect(asFragment()).toMatchSnapshot(); - store.dispatch(connectionActions.startFetching()) - expect(asFragment()).toMatchSnapshot() + store.dispatch(connectionActions.startFetching()); + expect(asFragment()).toMatchSnapshot(); - store.dispatch(connectionActions.connectionEstablished()) - expect(asFragment()).toMatchSnapshot() + store.dispatch(connectionActions.connectionEstablished()); + expect(asFragment()).toMatchSnapshot(); - store.dispatch(connectionActions.connectionError("wat")) - expect(asFragment()).toMatchSnapshot() + store.dispatch(connectionActions.connectionError("wat")); + expect(asFragment()).toMatchSnapshot(); - store.dispatch(connectionActions.setOffline()) - expect(asFragment()).toMatchSnapshot() + store.dispatch(connectionActions.setOffline()); + expect(asFragment()).toMatchSnapshot(); }); diff --git a/web/src/js/__tests__/components/Header/FileMenuSpec.tsx b/web/src/js/__tests__/components/Header/FileMenuSpec.tsx index a5ce8d634f..f25e0e9d9a 100644 --- a/web/src/js/__tests__/components/Header/FileMenuSpec.tsx +++ b/web/src/js/__tests__/components/Header/FileMenuSpec.tsx @@ -1,20 +1,19 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import FileMenu from '../../../components/Header/FileMenu' -import {Provider} from "react-redux"; -import {TStore} from "../../ducks/tutils"; - -describe('FileMenu Component', () => { +import * as React from "react"; +import renderer from "react-test-renderer"; +import FileMenu from "../../../components/Header/FileMenu"; +import { Provider } from "react-redux"; +import { TStore } from "../../ducks/tutils"; +describe("FileMenu Component", () => { let store = TStore(), fileMenu = renderer.create( - + ), - tree = fileMenu.toJSON() + tree = fileMenu.toJSON(); - it('should render correctly', () => { - expect(tree).toMatchSnapshot() - }) -}) + it("should render correctly", () => { + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/web/src/js/__tests__/components/Header/FilterDocsSpec.tsx b/web/src/js/__tests__/components/Header/FilterDocsSpec.tsx index 4feff94be1..b00d7a4313 100644 --- a/web/src/js/__tests__/components/Header/FilterDocsSpec.tsx +++ b/web/src/js/__tests__/components/Header/FilterDocsSpec.tsx @@ -1,20 +1,24 @@ -import * as React from "react" -import FilterDocs from '../../../components/Header/FilterDocs' -import {enableFetchMocks} from "jest-fetch-mock"; -import {render, screen, waitFor} from "../../test-utils"; +import * as React from "react"; +import FilterDocs from "../../../components/Header/FilterDocs"; +import { enableFetchMocks } from "jest-fetch-mock"; +import { render, screen, waitFor } from "../../test-utils"; enableFetchMocks(); test("FilterDocs Component", async () => { + fetchMock.mockOnceIf( + "./filter-help", + JSON.stringify({ + commands: [ + ["cmd1", "foo"], + ["cmd2", "bar"], + ], + }) + ); - fetchMock.mockOnceIf("./filter-help", JSON.stringify({ - commands: [['cmd1', 'foo'], ['cmd2', 'bar']] - })) - - const {asFragment} = render( 0}/>); + const { asFragment } = render( 0} />); expect(asFragment()).toMatchSnapshot(); await waitFor(() => screen.getByText("cmd1")); expect(asFragment()).toMatchSnapshot(); - -}) +}); diff --git a/web/src/js/__tests__/components/Header/FilterInputSpec.tsx b/web/src/js/__tests__/components/Header/FilterInputSpec.tsx index 9dc324cdb7..5197152613 100644 --- a/web/src/js/__tests__/components/Header/FilterInputSpec.tsx +++ b/web/src/js/__tests__/components/Header/FilterInputSpec.tsx @@ -1,94 +1,109 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import FilterInput from '../../../components/Header/FilterInput' -import FilterDocs from '../../../components/Header/FilterDocs' -import TestUtil from 'react-dom/test-utils' -import ReactDOM from 'react-dom' +import * as React from "react"; +import renderer from "react-test-renderer"; +import FilterInput from "../../../components/Header/FilterInput"; +import FilterDocs from "../../../components/Header/FilterDocs"; +import TestUtil from "react-dom/test-utils"; +import ReactDOM from "react-dom"; -describe('FilterInput Component', () => { - it('should render correctly', () => { +describe("FilterInput Component", () => { + it("should render correctly", () => { let filterInput = renderer.create( - undefined} value="42"/> + undefined} + value="42" + /> ), - tree = filterInput.toJSON() - expect(tree).toMatchSnapshot() - }) + tree = filterInput.toJSON(); + expect(tree).toMatchSnapshot(); + }); let filterInput = TestUtil.renderIntoDocument( - ) - it('should handle componentWillReceiveProps', () => { - filterInput.UNSAFE_componentWillReceiveProps({value: 'foo'}) - expect(filterInput.state.value).toEqual('foo') - }) + + ); + it("should handle componentWillReceiveProps", () => { + filterInput.UNSAFE_componentWillReceiveProps({ value: "foo" }); + expect(filterInput.state.value).toEqual("foo"); + }); - it('should handle isValid', () => { + it("should handle isValid", () => { // valid - expect(filterInput.isValid("~u foo")).toBeTruthy() - expect(filterInput.isValid("~foo bar")).toBeFalsy() - }) + expect(filterInput.isValid("~u foo")).toBeTruthy(); + expect(filterInput.isValid("~foo bar")).toBeFalsy(); + }); - it('should handle getDesc', () => { - filterInput.state.value = '' - expect(filterInput.getDesc().type).toEqual(FilterDocs) + it("should handle getDesc", () => { + filterInput.state.value = ""; + expect(filterInput.getDesc().type).toEqual(FilterDocs); - filterInput.state.value = '~u foo' - expect(filterInput.getDesc()).toEqual('url matches /foo/i') + filterInput.state.value = "~u foo"; + expect(filterInput.getDesc()).toEqual("url matches /foo/i"); - filterInput.state.value = '~foo bar' - expect(filterInput.getDesc()).toEqual('SyntaxError: Expected filter expression but \"~\" found.') - }) + filterInput.state.value = "~foo bar"; + expect(filterInput.getDesc()).toEqual( + 'SyntaxError: Expected filter expression but "~" found.' + ); + }); - it('should handle change', () => { - let mockEvent = { target: { value: '~a bar'} } - filterInput.onChange(mockEvent) - expect(filterInput.state.value).toEqual('~a bar') - expect(filterInput.props.onChange).toBeCalledWith('~a bar') - }) + it("should handle change", () => { + let mockEvent = { target: { value: "~a bar" } }; + filterInput.onChange(mockEvent); + expect(filterInput.state.value).toEqual("~a bar"); + expect(filterInput.props.onChange).toBeCalledWith("~a bar"); + }); - it('should handle focus', () => { - filterInput.onFocus() - expect(filterInput.state.focus).toBeTruthy() - }) + it("should handle focus", () => { + filterInput.onFocus(); + expect(filterInput.state.focus).toBeTruthy(); + }); - it('should handle blur', () => { - filterInput.onBlur() - expect(filterInput.state.focus).toBeFalsy() - }) + it("should handle blur", () => { + filterInput.onBlur(); + expect(filterInput.state.focus).toBeFalsy(); + }); - it('should handle mouseEnter', () => { - filterInput.onMouseEnter() - expect(filterInput.state.mousefocus).toBeTruthy() - }) + it("should handle mouseEnter", () => { + filterInput.onMouseEnter(); + expect(filterInput.state.mousefocus).toBeTruthy(); + }); - it('should handle mouseLeave', () => { - filterInput.onMouseLeave() - expect(filterInput.state.mousefocus).toBeFalsy() - }) + it("should handle mouseLeave", () => { + filterInput.onMouseLeave(); + expect(filterInput.state.mousefocus).toBeFalsy(); + }); - let input = ReactDOM.findDOMNode(filterInput.refs.input) + let input = ReactDOM.findDOMNode(filterInput.refs.input); - it('should handle keyDown', () => { - input.blur = jest.fn() + it("should handle keyDown", () => { + input.blur = jest.fn(); let mockEvent = { key: "Escape", - stopPropagation: jest.fn() - } - filterInput.onKeyDown(mockEvent) - expect(input.blur).toBeCalled() - expect(filterInput.state.mousefocus).toBeFalsy() - expect(mockEvent.stopPropagation).toBeCalled() - }) + stopPropagation: jest.fn(), + }; + filterInput.onKeyDown(mockEvent); + expect(input.blur).toBeCalled(); + expect(filterInput.state.mousefocus).toBeFalsy(); + expect(mockEvent.stopPropagation).toBeCalled(); + }); - it('should handle selectFilter', () => { - input.focus = jest.fn() - filterInput.selectFilter('bar') - expect(filterInput.state.value).toEqual('bar') - expect(input.focus).toBeCalled() - }) + it("should handle selectFilter", () => { + input.focus = jest.fn(); + filterInput.selectFilter("bar"); + expect(filterInput.state.value).toEqual("bar"); + expect(input.focus).toBeCalled(); + }); - it('should handle select', () => { - input.select = jest.fn() - filterInput.select() - expect(input.select).toBeCalled() - }) -}) + it("should handle select", () => { + input.select = jest.fn(); + filterInput.select(); + expect(input.select).toBeCalled(); + }); +}); diff --git a/web/src/js/__tests__/components/Header/FlowMenuSpec.tsx b/web/src/js/__tests__/components/Header/FlowMenuSpec.tsx index 2151df23bc..a0be1d77ba 100644 --- a/web/src/js/__tests__/components/Header/FlowMenuSpec.tsx +++ b/web/src/js/__tests__/components/Header/FlowMenuSpec.tsx @@ -1,8 +1,8 @@ -import * as React from "react" -import FlowMenu from '../../../components/Header/FlowMenu' -import {render} from "../../test-utils" +import * as React from "react"; +import FlowMenu from "../../../components/Header/FlowMenu"; +import { render } from "../../test-utils"; test("FlowMenu", async () => { - const {asFragment} = render(); + const { asFragment } = render(); expect(asFragment()).toMatchSnapshot(); }); diff --git a/web/src/js/__tests__/components/Header/MainMenuSpec.tsx b/web/src/js/__tests__/components/Header/MainMenuSpec.tsx index 5ec02f104d..723d7c9f79 100644 --- a/web/src/js/__tests__/components/Header/MainMenuSpec.tsx +++ b/web/src/js/__tests__/components/Header/MainMenuSpec.tsx @@ -1,8 +1,8 @@ -import * as React from "react" -import StartMenu from '../../../components/Header/StartMenu' -import {render} from "../../test-utils" +import * as React from "react"; +import StartMenu from "../../../components/Header/StartMenu"; +import { render } from "../../test-utils"; test("MainMenu", () => { - const {asFragment} = render(); + const { asFragment } = render(); expect(asFragment()).toMatchSnapshot(); -}) +}); diff --git a/web/src/js/__tests__/components/Header/MenuToggleSpec.tsx b/web/src/js/__tests__/components/Header/MenuToggleSpec.tsx index ce45242895..1b3d5cd6da 100644 --- a/web/src/js/__tests__/components/Header/MenuToggleSpec.tsx +++ b/web/src/js/__tests__/components/Header/MenuToggleSpec.tsx @@ -1,44 +1,49 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import {EventlogToggle, MenuToggle, OptionsToggle} from '../../../components/Header/MenuToggle' -import {Provider} from 'react-redux' -import {TStore} from '../../ducks/tutils' -import * as optionsEditorActions from "../../../ducks/ui/optionsEditor" -import {fireEvent, render, screen} from "../../test-utils" +import * as React from "react"; +import renderer from "react-test-renderer"; +import { + EventlogToggle, + MenuToggle, + OptionsToggle, +} from "../../../components/Header/MenuToggle"; +import { Provider } from "react-redux"; +import { TStore } from "../../ducks/tutils"; +import * as optionsEditorActions from "../../../ducks/ui/optionsEditor"; +import { fireEvent, render, screen } from "../../test-utils"; -describe('MenuToggle Component', () => { - it('should render correctly', () => { +describe("MenuToggle Component", () => { + it("should render correctly", () => { let changeFn = jest.fn(), menuToggle = renderer.create(

    foo children

    -     ), - tree = menuToggle.toJSON() - expect(tree).toMatchSnapshot() - }) -}) + + ), + tree = menuToggle.toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); test("OptionsToggle", async () => { const store = TStore(), - {asFragment} = render( - toggle anticache, - {store} + { asFragment } = render( + toggle anticache, + { store } ); - globalThis.fetch = jest.fn() + globalThis.fetch = jest.fn(); expect(asFragment()).toMatchSnapshot(); fireEvent.click(screen.getByText("toggle anticache")); - expect(store.getActions()).toEqual([optionsEditorActions.startUpdate("anticache", true)]) + expect(store.getActions()).toEqual([ + optionsEditorActions.startUpdate("anticache", true), + ]); }); test("EventlogToggle", async () => { - const {asFragment, store} = render( - - ); + const { asFragment, store } = render(); expect(asFragment()).toMatchSnapshot(); expect(store.getState().eventLog.visible).toBeTruthy(); fireEvent.click(screen.getByText("Display Event Log")); expect(store.getState().eventLog.visible).toBeFalsy(); -}) +}); diff --git a/web/src/js/__tests__/components/Header/OptionMenuSpec.tsx b/web/src/js/__tests__/components/Header/OptionMenuSpec.tsx index bb6a51d946..a07c8f082a 100644 --- a/web/src/js/__tests__/components/Header/OptionMenuSpec.tsx +++ b/web/src/js/__tests__/components/Header/OptionMenuSpec.tsx @@ -1,18 +1,18 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import { Provider } from 'react-redux' -import OptionMenu from '../../../components/Header/OptionMenu' -import { TStore } from '../../ducks/tutils' +import * as React from "react"; +import renderer from "react-test-renderer"; +import { Provider } from "react-redux"; +import OptionMenu from "../../../components/Header/OptionMenu"; +import { TStore } from "../../ducks/tutils"; -describe('OptionMenu Component', () => { - it('should render correctly', () => { +describe("OptionMenu Component", () => { + it("should render correctly", () => { let store = TStore(), provider = renderer.create( - + ), - tree = provider.toJSON() - expect(tree).toMatchSnapshot() - }) -}) + tree = provider.toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/web/src/js/__tests__/components/Header/__snapshots__/OptionMenuSpec.tsx.snap b/web/src/js/__tests__/components/Header/__snapshots__/OptionMenuSpec.tsx.snap index f78126abf7..f69b6c0748 100644 --- a/web/src/js/__tests__/components/Header/__snapshots__/OptionMenuSpec.tsx.snap +++ b/web/src/js/__tests__/components/Header/__snapshots__/OptionMenuSpec.tsx.snap @@ -44,7 +44,8 @@ exports[`OptionMenu Component should render correctly 1`] = ` onChange={[Function]} type="checkbox" /> - Strip cache headers + Strip cache headers + { - - const {asFragment} = render(
    ); + const { asFragment } = render(
    ); expect(asFragment()).toMatchSnapshot(); fireEvent.click(screen.getByText("Options")); @@ -18,5 +16,5 @@ test("Header", async () => { expect(screen.getByText("Open...")).toBeTruthy(); fireEvent.click(screen.getByText("File")); - expect(screen.queryByText("Open...")).toBeNull() + expect(screen.queryByText("Open...")).toBeNull(); }); diff --git a/web/src/js/__tests__/components/Modal/ModalSpec.tsx b/web/src/js/__tests__/components/Modal/ModalSpec.tsx index 21e3790dce..fea1f258bc 100644 --- a/web/src/js/__tests__/components/Modal/ModalSpec.tsx +++ b/web/src/js/__tests__/components/Modal/ModalSpec.tsx @@ -1,13 +1,12 @@ -import * as React from "react" -import Modal from '../../../components/Modal/Modal' -import {render} from "../../test-utils" -import {setActiveModal} from "../../../ducks/ui/modal"; +import * as React from "react"; +import Modal from "../../../components/Modal/Modal"; +import { render } from "../../test-utils"; +import { setActiveModal } from "../../../ducks/ui/modal"; test("Modal Component", async () => { - const {asFragment, store} = render(); + const { asFragment, store } = render(); expect(asFragment()).toMatchSnapshot(); store.dispatch(setActiveModal("OptionModal")); expect(asFragment()).toMatchSnapshot(); - -}) +}); diff --git a/web/src/js/__tests__/components/Modal/OptionModalSpec.tsx b/web/src/js/__tests__/components/Modal/OptionModalSpec.tsx index 65025cb760..4316a519d6 100644 --- a/web/src/js/__tests__/components/Modal/OptionModalSpec.tsx +++ b/web/src/js/__tests__/components/Modal/OptionModalSpec.tsx @@ -1,54 +1,54 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import { PureOptionDefault } from '../../../components/Modal/OptionModal' +import * as React from "react"; +import renderer from "react-test-renderer"; +import { PureOptionDefault } from "../../../components/Modal/OptionModal"; -describe('PureOptionDefault Component', () => { - - it('should return null when the value is default', () => { +describe("PureOptionDefault Component", () => { + it("should return null when the value is default", () => { let pureOptionDefault = renderer.create( - - ), - tree = pureOptionDefault.toJSON() - expect(tree).toMatchSnapshot() - }) + + ), + tree = pureOptionDefault.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should handle boolean type', () => { + it("should handle boolean type", () => { let pureOptionDefault = renderer.create( - - ), - tree = pureOptionDefault.toJSON() - expect(tree).toMatchSnapshot() - }) - - it('should handle array', () => { - let a = [""], b = [], c = ['c'], + + ), + tree = pureOptionDefault.toJSON(); + expect(tree).toMatchSnapshot(); + }); + + it("should handle array", () => { + let a = [""], + b = [], + c = ["c"], pureOptionDefault = renderer.create( - + ), - tree = pureOptionDefault.toJSON() - expect(tree).toMatchSnapshot() + tree = pureOptionDefault.toJSON(); + expect(tree).toMatchSnapshot(); pureOptionDefault = renderer.create( - - ) - tree = pureOptionDefault.toJSON() - expect(tree).toMatchSnapshot() - }) + + ); + tree = pureOptionDefault.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should handle string', () => { + it("should handle string", () => { let pureOptionDefault = renderer.create( - - ), - tree = pureOptionDefault.toJSON() - expect(tree).toMatchSnapshot() - }) + + ), + tree = pureOptionDefault.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should handle null value', () => { + it("should handle null value", () => { let pureOptionDefault = renderer.create( - - ), - tree = pureOptionDefault.toJSON() - expect(tree).toMatchSnapshot() - }) - -}) + + ), + tree = pureOptionDefault.toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/web/src/js/__tests__/components/Modal/OptionSpec.tsx b/web/src/js/__tests__/components/Modal/OptionSpec.tsx index cf6d84fede..47540f73db 100644 --- a/web/src/js/__tests__/components/Modal/OptionSpec.tsx +++ b/web/src/js/__tests__/components/Modal/OptionSpec.tsx @@ -1,99 +1,102 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import { Options, ChoicesOption } from '../../../components/Modal/Option' +import * as React from "react"; +import renderer from "react-test-renderer"; +import { Options, ChoicesOption } from "../../../components/Modal/Option"; -describe('BooleanOption Component', () => { - let BooleanOption = Options['bool'], +describe("BooleanOption Component", () => { + let BooleanOption = Options["bool"], onChangeFn = jest.fn(), booleanOption = renderer.create( - + ), - tree = booleanOption.toJSON() + tree = booleanOption.toJSON(); - it('should render correctly', () => { - expect(tree).toMatchSnapshot() - }) + it("should render correctly", () => { + expect(tree).toMatchSnapshot(); + }); - it('should handle onChange', () => { + it("should handle onChange", () => { let input = tree.children[0].children[0], - mockEvent = { target: { checked: true }} - input.props.onChange(mockEvent) - expect(onChangeFn).toBeCalledWith(mockEvent.target.checked) - }) -}) - -describe('StringOption Component', () => { - let StringOption = Options['str'], + mockEvent = { target: { checked: true } }; + input.props.onChange(mockEvent); + expect(onChangeFn).toBeCalledWith(mockEvent.target.checked); + }); +}); + +describe("StringOption Component", () => { + let StringOption = Options["str"], onChangeFn = jest.fn(), stringOption = renderer.create( - + ), - tree = stringOption.toJSON() - - it('should render correctly', () => { - expect(tree).toMatchSnapshot() - }) + tree = stringOption.toJSON(); - it('should handle onChange', () => { - let mockEvent = { target: { value: 'bar' }} - tree.props.onChange(mockEvent) - expect(onChangeFn).toBeCalledWith(mockEvent.target.value) - }) + it("should render correctly", () => { + expect(tree).toMatchSnapshot(); + }); -}) + it("should handle onChange", () => { + let mockEvent = { target: { value: "bar" } }; + tree.props.onChange(mockEvent); + expect(onChangeFn).toBeCalledWith(mockEvent.target.value); + }); +}); -describe('NumberOption Component', () => { - let NumberOption = Options['int'], +describe("NumberOption Component", () => { + let NumberOption = Options["int"], onChangeFn = jest.fn(), numberOption = renderer.create( - + ), - tree = numberOption.toJSON() + tree = numberOption.toJSON(); - it('should render correctly', () => { - expect(tree).toMatchSnapshot() - }) + it("should render correctly", () => { + expect(tree).toMatchSnapshot(); + }); - it('should handle onChange', () => { - let mockEvent = {target: { value: '2'}} - tree.props.onChange(mockEvent) - expect(onChangeFn).toBeCalledWith(2) - }) -}) + it("should handle onChange", () => { + let mockEvent = { target: { value: "2" } }; + tree.props.onChange(mockEvent); + expect(onChangeFn).toBeCalledWith(2); + }); +}); -describe('ChoiceOption Component', () => { +describe("ChoiceOption Component", () => { let onChangeFn = jest.fn(), choiceOption = renderer.create( - + ), - tree = choiceOption.toJSON() + tree = choiceOption.toJSON(); - it('should render correctly', () => { - expect(tree).toMatchSnapshot() - }) + it("should render correctly", () => { + expect(tree).toMatchSnapshot(); + }); - it('should handle onChange', () => { - let mockEvent = { target: {value: 'b'} } - tree.props.onChange(mockEvent) - expect(onChangeFn).toBeCalledWith(mockEvent.target.value) - }) -}) + it("should handle onChange", () => { + let mockEvent = { target: { value: "b" } }; + tree.props.onChange(mockEvent); + expect(onChangeFn).toBeCalledWith(mockEvent.target.value); + }); +}); -describe('StringOption Component', () => { +describe("StringOption Component", () => { let onChangeFn = jest.fn(), - StringSequenceOption = Options['sequence of str'], + StringSequenceOption = Options["sequence of str"], stringSequenceOption = renderer.create( - + ), - tree = stringSequenceOption.toJSON() - - it('should render correctly', () => { - expect(tree).toMatchSnapshot() - }) - - it('should handle onChange', () => { - let mockEvent = { target: {value: 'a\nb\nc\n'}} - tree.props.onChange(mockEvent) - expect(onChangeFn).toBeCalledWith(['a', 'b', 'c', '']) - }) -}) + tree = stringSequenceOption.toJSON(); + + it("should render correctly", () => { + expect(tree).toMatchSnapshot(); + }); + + it("should handle onChange", () => { + let mockEvent = { target: { value: "a\nb\nc\n" } }; + tree.props.onChange(mockEvent); + expect(onChangeFn).toBeCalledWith(["a", "b", "c", ""]); + }); +}); diff --git a/web/src/js/__tests__/components/ProxyAppSpec.tsx b/web/src/js/__tests__/components/ProxyAppSpec.tsx index 7b49410230..1b10852cd2 100644 --- a/web/src/js/__tests__/components/ProxyAppSpec.tsx +++ b/web/src/js/__tests__/components/ProxyAppSpec.tsx @@ -1,15 +1,21 @@ -import * as React from "react" -import {render, screen, waitFor} from "../test-utils"; +import * as React from "react"; +import { render, screen, waitFor } from "../test-utils"; import ProxyApp from "../../components/ProxyApp"; -import {enableFetchMocks} from "jest-fetch-mock"; -import {ContentViewData} from "../../components/contentviews/useContent"; +import { enableFetchMocks } from "jest-fetch-mock"; +import { ContentViewData } from "../../components/contentviews/useContent"; enableFetchMocks(); test("ProxyApp", async () => { - const cv: ContentViewData = {lines: [[["text", "my data"]]], description: ""} - fetchMock.doMockOnceIf("./flows/flow2/request/content/Auto.json?lines=513", JSON.stringify(cv)); - render(); + const cv: ContentViewData = { + lines: [[["text", "my data"]]], + description: "", + }; + fetchMock.doMockOnceIf( + "./flows/flow2/request/content/Auto.json?lines=513", + JSON.stringify(cv) + ); + render(); expect(screen.getByTitle("Mitmproxy Version")).toBeDefined(); await waitFor(() => screen.getByText("my data")); }); diff --git a/web/src/js/__tests__/components/common/ButtonSpec.tsx b/web/src/js/__tests__/components/common/ButtonSpec.tsx index 661b4d553a..802073565a 100644 --- a/web/src/js/__tests__/components/common/ButtonSpec.tsx +++ b/web/src/js/__tests__/components/common/ButtonSpec.tsx @@ -1,26 +1,34 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import Button from '../../../components/common/Button' +import * as React from "react"; +import renderer from "react-test-renderer"; +import Button from "../../../components/common/Button"; -describe('Button Component', () => { - - it('should render correctly', () => { +describe("Button Component", () => { + it("should render correctly", () => { let button = renderer.create( - - ), - tree = button.toJSON() - expect(tree).toMatchSnapshot() - }) + + ), + tree = button.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should be able to be disabled', () => { + it("should be able to be disabled", () => { let button = renderer.create( - + ), - tree = button.toJSON() - expect(tree).toMatchSnapshot() - }) -}) + tree = button.toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/web/src/js/__tests__/components/common/DocsLinkSpec.tsx b/web/src/js/__tests__/components/common/DocsLinkSpec.tsx index 0f78a16259..ccd8c3f993 100644 --- a/web/src/js/__tests__/components/common/DocsLinkSpec.tsx +++ b/web/src/js/__tests__/components/common/DocsLinkSpec.tsx @@ -1,17 +1,19 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import DocsLink from '../../../components/common/DocsLink' +import * as React from "react"; +import renderer from "react-test-renderer"; +import DocsLink from "../../../components/common/DocsLink"; -describe('DocsLink Component', () => { - it('should be able to be rendered with children nodes', () => { - let docsLink = renderer.create(), - tree = docsLink.toJSON() - expect(tree).toMatchSnapshot() - }) +describe("DocsLink Component", () => { + it("should be able to be rendered with children nodes", () => { + let docsLink = renderer.create( + + ), + tree = docsLink.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should be able to be rendered without children nodes', () => { - let docsLink = renderer.create(), - tree = docsLink.toJSON() - expect(tree).toMatchSnapshot() - }) -}) + it("should be able to be rendered without children nodes", () => { + let docsLink = renderer.create(), + tree = docsLink.toJSON(); + expect(tree).toMatchSnapshot(); + }); +}); diff --git a/web/src/js/__tests__/components/common/DropdownSpec.tsx b/web/src/js/__tests__/components/common/DropdownSpec.tsx index 9f430601a4..590aae2f82 100644 --- a/web/src/js/__tests__/components/common/DropdownSpec.tsx +++ b/web/src/js/__tests__/components/common/DropdownSpec.tsx @@ -1,51 +1,52 @@ import * as React from "react"; -import Dropdown, {Divider, MenuItem, SubMenu} from '../../../components/common/Dropdown' -import {fireEvent, render, screen, waitFor} from "../../test-utils"; - - -test('Dropdown', async () => { +import Dropdown, { + Divider, + MenuItem, + SubMenu, +} from "../../../components/common/Dropdown"; +import { fireEvent, render, screen, waitFor } from "../../test-utils"; + +test("Dropdown", async () => { let onOpen = jest.fn(); - const {asFragment} = render( + const { asFragment } = render( 0}>click me - + 0}>click me - ) - expect(asFragment()).toMatchSnapshot() + ); + expect(asFragment()).toMatchSnapshot(); - fireEvent.click(screen.getByText("open me")) - await waitFor(() => expect(onOpen).toBeCalledWith(true)) - expect(asFragment()).toMatchSnapshot() + fireEvent.click(screen.getByText("open me")); + await waitFor(() => expect(onOpen).toBeCalledWith(true)); + expect(asFragment()).toMatchSnapshot(); - onOpen.mockClear() - fireEvent.click(document.body) + onOpen.mockClear(); + fireEvent.click(document.body); await waitFor(() => expect(onOpen).toBeCalledWith(false)); -}) +}); -test('SubMenu', async () => { - const {asFragment} = render( +test("SubMenu", async () => { + const { asFragment } = render( 0}>click me - ) - expect(asFragment()).toMatchSnapshot() + ); + expect(asFragment()).toMatchSnapshot(); - fireEvent.mouseEnter(screen.getByText("submenu")) - await waitFor(() => screen.getByText("click me")) - expect(asFragment()).toMatchSnapshot() + fireEvent.mouseEnter(screen.getByText("submenu")); + await waitFor(() => screen.getByText("click me")); + expect(asFragment()).toMatchSnapshot(); - fireEvent.mouseLeave(screen.getByText("submenu")) - expect(screen.queryByText("click me")).toBeNull() - expect(asFragment()).toMatchSnapshot() -}) + fireEvent.mouseLeave(screen.getByText("submenu")); + expect(screen.queryByText("click me")).toBeNull(); + expect(asFragment()).toMatchSnapshot(); +}); -test('MenuItem', async () => { +test("MenuItem", async () => { let click = jest.fn(); - const {asFragment} = render( - wtf - ) - expect(asFragment()).toMatchSnapshot() - fireEvent.click(screen.getByText("wtf")) + const { asFragment } = render(wtf); + expect(asFragment()).toMatchSnapshot(); + fireEvent.click(screen.getByText("wtf")); await waitFor(() => expect(click).toBeCalled()); -}) +}); diff --git a/web/src/js/__tests__/components/common/FileChooserSpec.tsx b/web/src/js/__tests__/components/common/FileChooserSpec.tsx index bf7f196b3f..77fc2af6a6 100644 --- a/web/src/js/__tests__/components/common/FileChooserSpec.tsx +++ b/web/src/js/__tests__/components/common/FileChooserSpec.tsx @@ -1,13 +1,11 @@ -import * as React from "react" -import FileChooser from '../../../components/common/FileChooser' -import {render} from '@testing-library/react' - +import * as React from "react"; +import FileChooser from "../../../components/common/FileChooser"; +import { render } from "@testing-library/react"; test("FileChooser", async () => { - const {asFragment} = render( - 0}/> - ); - - expect(asFragment()).toMatchSnapshot() + const { asFragment } = render( + 0} /> + ); -}) + expect(asFragment()).toMatchSnapshot(); +}); diff --git a/web/src/js/__tests__/components/common/SplitterSpec.tsx b/web/src/js/__tests__/components/common/SplitterSpec.tsx index f2adffe7a9..488656d91e 100644 --- a/web/src/js/__tests__/components/common/SplitterSpec.tsx +++ b/web/src/js/__tests__/components/common/SplitterSpec.tsx @@ -1,84 +1,105 @@ -import * as React from "react" -import ReactDOM from 'react-dom' -import renderer from 'react-test-renderer' -import Splitter from '../../../components/common/Splitter' -import TestUtils from 'react-dom/test-utils'; +import * as React from "react"; +import ReactDOM from "react-dom"; +import renderer from "react-test-renderer"; +import Splitter from "../../../components/common/Splitter"; +import TestUtils from "react-dom/test-utils"; -describe('Splitter Component', () => { +describe("Splitter Component", () => { + it("should render correctly", () => { + let splitter = renderer.create(), + tree = splitter.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should render correctly', () => { - let splitter = renderer.create(), - tree = splitter.toJSON() - expect(tree).toMatchSnapshot() - }) - - let splitter = TestUtils.renderIntoDocument(), + let splitter = TestUtils.renderIntoDocument(), dom = ReactDOM.findDOMNode(splitter), previousElementSibling = { offsetHeight: 0, offsetWidth: 0, - style: {flex: ''} + style: { flex: "" }, }, nextElementSibling = { - style: {flex: ''} - } - - it('should handle mouseDown ', () => { - window.addEventListener = jest.fn() - splitter.onMouseDown({pageX: 1, pageY: 2}) - expect(splitter.state.startX).toEqual(1) - expect(splitter.state.startY).toEqual(2) - expect(window.addEventListener).toBeCalledWith('mousemove', splitter.onMouseMove) - expect(window.addEventListener).toBeCalledWith('mouseup', splitter.onMouseUp) - expect(window.addEventListener).toBeCalledWith('dragend', splitter.onDragEnd) - }) - - it('should handle dragEnd', () => { - window.removeEventListener = jest.fn() - splitter.onDragEnd() - expect(dom.style.transform).toEqual('') - expect(window.removeEventListener).toBeCalledWith('dragend', splitter.onDragEnd) - expect(window.removeEventListener).toBeCalledWith('mouseup', splitter.onMouseUp) - expect(window.removeEventListener).toBeCalledWith('mousemove', splitter.onMouseMove) - }) + style: { flex: "" }, + }; - it('should handle mouseUp', () => { + it("should handle mouseDown ", () => { + window.addEventListener = jest.fn(); + splitter.onMouseDown({ pageX: 1, pageY: 2 }); + expect(splitter.state.startX).toEqual(1); + expect(splitter.state.startY).toEqual(2); + expect(window.addEventListener).toBeCalledWith( + "mousemove", + splitter.onMouseMove + ); + expect(window.addEventListener).toBeCalledWith( + "mouseup", + splitter.onMouseUp + ); + expect(window.addEventListener).toBeCalledWith( + "dragend", + splitter.onDragEnd + ); + }); - Object.defineProperty(dom, 'previousElementSibling', {value: previousElementSibling}) - Object.defineProperty(dom, 'nextElementSibling', {value: nextElementSibling}) - splitter.onMouseUp({pageX: 3, pageY: 4}) - expect(splitter.state.applied).toBeTruthy() - expect(nextElementSibling.style.flex).toEqual('1 1 auto') - expect(previousElementSibling.style.flex).toEqual('0 0 2px') - }) + it("should handle dragEnd", () => { + window.removeEventListener = jest.fn(); + splitter.onDragEnd(); + expect(dom.style.transform).toEqual(""); + expect(window.removeEventListener).toBeCalledWith( + "dragend", + splitter.onDragEnd + ); + expect(window.removeEventListener).toBeCalledWith( + "mouseup", + splitter.onMouseUp + ); + expect(window.removeEventListener).toBeCalledWith( + "mousemove", + splitter.onMouseMove + ); + }); - it('should handle mouseMove', () => { - splitter.onMouseMove({pageX: 10, pageY: 10}) - expect(dom.style.transform).toEqual("translate(9px, 0px)") + it("should handle mouseUp", () => { + Object.defineProperty(dom, "previousElementSibling", { + value: previousElementSibling, + }); + Object.defineProperty(dom, "nextElementSibling", { + value: nextElementSibling, + }); + splitter.onMouseUp({ pageX: 3, pageY: 4 }); + expect(splitter.state.applied).toBeTruthy(); + expect(nextElementSibling.style.flex).toEqual("1 1 auto"); + expect(previousElementSibling.style.flex).toEqual("0 0 2px"); + }); - let splitterY = TestUtils.renderIntoDocument() - splitterY.onMouseMove({pageX: 10, pageY: 10}) - expect(ReactDOM.findDOMNode(splitterY).style.transform).toEqual("translate(0px, 10px)") - }) + it("should handle mouseMove", () => { + splitter.onMouseMove({ pageX: 10, pageY: 10 }); + expect(dom.style.transform).toEqual("translate(9px, 0px)"); - it('should handle resize', () => { - let x = jest.spyOn(window, 'setTimeout'); - splitter.onResize() - expect(x).toHaveBeenCalled() - }) + let splitterY = TestUtils.renderIntoDocument(); + splitterY.onMouseMove({ pageX: 10, pageY: 10 }); + expect(ReactDOM.findDOMNode(splitterY).style.transform).toEqual( + "translate(0px, 10px)" + ); + }); - it('should handle componentWillUnmount', () => { - splitter.componentWillUnmount() - expect(previousElementSibling.style.flex).toEqual('') - expect(nextElementSibling.style.flex).toEqual('') - expect(splitter.state.applied).toBeTruthy() - }) + it("should handle resize", () => { + let x = jest.spyOn(window, "setTimeout"); + splitter.onResize(); + expect(x).toHaveBeenCalled(); + }); - it('should handle reset', () => { - splitter.reset(false) - expect(splitter.state.applied).toBeFalsy() + it("should handle componentWillUnmount", () => { + splitter.componentWillUnmount(); + expect(previousElementSibling.style.flex).toEqual(""); + expect(nextElementSibling.style.flex).toEqual(""); + expect(splitter.state.applied).toBeTruthy(); + }); - expect(splitter.reset(true)).toEqual(undefined) - }) + it("should handle reset", () => { + splitter.reset(false); + expect(splitter.state.applied).toBeFalsy(); -}) + expect(splitter.reset(true)).toEqual(undefined); + }); +}); diff --git a/web/src/js/__tests__/components/common/ToggleButtonSpec.tsx b/web/src/js/__tests__/components/common/ToggleButtonSpec.tsx index 23378f5397..62e0d0e08d 100644 --- a/web/src/js/__tests__/components/common/ToggleButtonSpec.tsx +++ b/web/src/js/__tests__/components/common/ToggleButtonSpec.tsx @@ -1,22 +1,24 @@ -import * as React from "react" -import renderer from 'react-test-renderer' -import ToggleButton from '../../../components/common/ToggleButton' +import * as React from "react"; +import renderer from "react-test-renderer"; +import ToggleButton from "../../../components/common/ToggleButton"; -describe('ToggleButton Component', () => { - let mockFunc = jest.fn() +describe("ToggleButton Component", () => { + let mockFunc = jest.fn(); - it('should render correctly', () => { + it("should render correctly", () => { let checkedButton = renderer.create( - ), - tree = checkedButton.toJSON() - expect(tree).toMatchSnapshot() - }) + + ), + tree = checkedButton.toJSON(); + expect(tree).toMatchSnapshot(); + }); - it('should handle click action', () => { + it("should handle click action", () => { let uncheckButton = renderer.create( - ), - tree = uncheckButton.toJSON() - tree.props.onClick() - expect(mockFunc).toBeCalled() - }) -}) + + ), + tree = uncheckButton.toJSON(); + tree.props.onClick(); + expect(mockFunc).toBeCalled(); + }); +}); diff --git a/web/src/js/__tests__/components/contentviews/CodeEditorSpec.tsx b/web/src/js/__tests__/components/contentviews/CodeEditorSpec.tsx index 1d66c2b860..4a11de9d95 100644 --- a/web/src/js/__tests__/components/contentviews/CodeEditorSpec.tsx +++ b/web/src/js/__tests__/components/contentviews/CodeEditorSpec.tsx @@ -1,10 +1,9 @@ -jest.mock("../../../contrib/CodeMirror") -import * as React from 'react'; -import CodeEditor from '../../../components/contentviews/CodeEditor' -import {render} from "../../test-utils" - +jest.mock("../../../contrib/CodeMirror"); +import * as React from "react"; +import CodeEditor from "../../../components/contentviews/CodeEditor"; +import { render } from "../../test-utils"; test("CodeEditor", async () => { - const {asFragment} = render(); + const { asFragment } = render(); expect(asFragment()).toMatchSnapshot(); }); diff --git a/web/src/js/__tests__/components/contentviews/HttpMessageSpec.tsx b/web/src/js/__tests__/components/contentviews/HttpMessageSpec.tsx index fbb9d5ee84..74ed7d015b 100644 --- a/web/src/js/__tests__/components/contentviews/HttpMessageSpec.tsx +++ b/web/src/js/__tests__/components/contentviews/HttpMessageSpec.tsx @@ -1,26 +1,30 @@ -import {TFlow} from "../../ducks/tutils"; -import * as React from 'react'; -import HttpMessage, {ViewImage} from '../../../components/contentviews/HttpMessage' -import {fireEvent, render, screen, waitFor} from "../../test-utils" -import fetchMock, {enableFetchMocks} from "jest-fetch-mock"; +import { TFlow } from "../../ducks/tutils"; +import * as React from "react"; +import HttpMessage, { + ViewImage, +} from "../../../components/contentviews/HttpMessage"; +import { fireEvent, render, screen, waitFor } from "../../test-utils"; +import fetchMock, { enableFetchMocks } from "jest-fetch-mock"; -jest.mock("../../../contrib/CodeMirror") +jest.mock("../../../contrib/CodeMirror"); enableFetchMocks(); test("HttpMessage", async () => { - const lines = Array(512).fill([["text", "data"]]).concat( - Array(512).fill([["text", "additional"]]) - ); + const lines = Array(512) + .fill([["text", "data"]]) + .concat(Array(512).fill([["text", "additional"]])); fetchMock.mockResponses( JSON.stringify({ lines: lines.slice(0, 512 + 1), - description: "Auto" - }), JSON.stringify({ + description: "Auto", + }), + JSON.stringify({ lines, - description: "Auto" - }), JSON.stringify({ + description: "Auto", + }), + JSON.stringify({ lines: Array(5).fill([["text", "rawdata"]]), description: "Raw", }), @@ -32,9 +36,11 @@ test("HttpMessage", async () => { ); const tflow = TFlow(); - const {asFragment} = render(); + const { asFragment } = render( + + ); await waitFor(() => screen.getAllByText("data")); - expect(screen.queryByText('additional')).toBeNull(); + expect(screen.queryByText("additional")).toBeNull(); fireEvent.click(screen.getByText("Show more")); await waitFor(() => screen.getAllByText("additional")); @@ -54,6 +60,8 @@ test("HttpMessage", async () => { test("ViewImage", async () => { const flow = TFlow(); - const {asFragment} = render() + const { asFragment } = render( + + ); expect(asFragment()).toMatchSnapshot(); }); diff --git a/web/src/js/__tests__/components/contentviews/LineRendererSpec.tsx b/web/src/js/__tests__/components/contentviews/LineRendererSpec.tsx index 60ff4528b4..551402a7f9 100644 --- a/web/src/js/__tests__/components/contentviews/LineRendererSpec.tsx +++ b/web/src/js/__tests__/components/contentviews/LineRendererSpec.tsx @@ -1,28 +1,31 @@ -import * as React from 'react'; -import LineRenderer from '../../../components/contentviews/LineRenderer' -import {fireEvent, render, screen} from "../../test-utils" - +import * as React from "react"; +import LineRenderer from "../../../components/contentviews/LineRenderer"; +import { fireEvent, render, screen } from "../../test-utils"; test("LineRenderer", async () => { const lines: [style: string, text: string][][] = [ [ ["header", "foo: "], - ["text", "42"] + ["text", "42"], ], [ ["header", "bar: "], - ["text", "43"] + ["text", "43"], ], - ] + ]; const showMore = jest.fn(); - const {asFragment} = render(); + const { asFragment } = render( + + ); expect(asFragment()).toMatchSnapshot(); fireEvent.click(screen.getByText("Show more")); expect(showMore).toBeCalled(); }); test("No lines", async () => { - const {asFragment} = render( 0}/>); + const { asFragment } = render( + 0} /> + ); expect(asFragment()).toMatchSnapshot(); -}) +}); diff --git a/web/src/js/__tests__/components/contentviews/ViewSelectorSpec.tsx b/web/src/js/__tests__/components/contentviews/ViewSelectorSpec.tsx index f87e67855b..8e36a4e7c5 100644 --- a/web/src/js/__tests__/components/contentviews/ViewSelectorSpec.tsx +++ b/web/src/js/__tests__/components/contentviews/ViewSelectorSpec.tsx @@ -1,12 +1,12 @@ -import * as React from 'react'; -import ViewSelector from '../../../components/contentviews/ViewSelector' -import {fireEvent, render, screen} from "../../test-utils" - +import * as React from "react"; +import ViewSelector from "../../../components/contentviews/ViewSelector"; +import { fireEvent, render, screen } from "../../test-utils"; test("ViewSelector", async () => { - const onChange = jest.fn(); - const {asFragment} = render(); + const { asFragment } = render( + + ); expect(asFragment()).toMatchSnapshot(); fireEvent.click(screen.getByText("auto")); expect(asFragment()).toMatchSnapshot(); diff --git a/web/src/js/__tests__/components/contentviews/useContentSpec.tsx b/web/src/js/__tests__/components/contentviews/useContentSpec.tsx index f03af5f576..bf4bfcea7c 100644 --- a/web/src/js/__tests__/components/contentviews/useContentSpec.tsx +++ b/web/src/js/__tests__/components/contentviews/useContentSpec.tsx @@ -1,31 +1,32 @@ -import * as React from 'react'; -import {render, screen, waitFor} from "../../test-utils" -import {useContent} from "../../../components/contentviews/useContent"; -import fetchMock, {enableFetchMocks} from 'jest-fetch-mock' +import * as React from "react"; +import { render, screen, waitFor } from "../../test-utils"; +import { useContent } from "../../../components/contentviews/useContent"; +import fetchMock, { enableFetchMocks } from "jest-fetch-mock"; -enableFetchMocks() +enableFetchMocks(); - -function TComp({url, hash}: { url: string, hash: string }) { +function TComp({ url, hash }: { url: string; hash: string }) { const content = useContent(url, hash); return
    {content}
    ; } test("caching", async () => { fetchMock.mockResponses("hello", "world"); - const {rerender} = render(); + const { rerender } = render(); await waitFor(() => screen.getByText("hello")); - rerender(); + rerender(); expect(fetchMock.mock.calls).toHaveLength(1); - rerender(); + rerender(); await waitFor(() => screen.getByText("world")); expect(fetchMock.mock.calls).toHaveLength(2); }); test("network error", async () => { fetchMock.mockRejectOnce(new Error("I/O error")); - render(); - await waitFor(() => screen.getByText("Error getting content: Error: I/O error.")); + render(); + await waitFor(() => + screen.getByText("Error getting content: Error: I/O error.") + ); }); diff --git a/web/src/js/__tests__/components/editors/ValidateEditorSpec.tsx b/web/src/js/__tests__/components/editors/ValidateEditorSpec.tsx index ae07b96396..0caf9c6328 100644 --- a/web/src/js/__tests__/components/editors/ValidateEditorSpec.tsx +++ b/web/src/js/__tests__/components/editors/ValidateEditorSpec.tsx @@ -1,11 +1,21 @@ -import * as React from "react" -import ValidateEditor from '../../../components/editors/ValidateEditor' -import {fireEvent, render, screen, userEvent, waitFor} from "../../test-utils"; +import * as React from "react"; +import ValidateEditor from "../../../components/editors/ValidateEditor"; +import { + fireEvent, + render, + screen, + userEvent, + waitFor, +} from "../../test-utils"; test("ValidateEditor", async () => { const onEditDone = jest.fn(); - const {asFragment} = render( - x.includes("ok")} onEditDone={onEditDone}/> + const { asFragment } = render( + x.includes("ok")} + onEditDone={onEditDone} + /> ); expect(asFragment()).toMatchSnapshot(); diff --git a/web/src/js/__tests__/components/editors/ValueEditorSpec.tsx b/web/src/js/__tests__/components/editors/ValueEditorSpec.tsx index 1d2513f543..50cfa6e57d 100644 --- a/web/src/js/__tests__/components/editors/ValueEditorSpec.tsx +++ b/web/src/js/__tests__/components/editors/ValueEditorSpec.tsx @@ -1,17 +1,20 @@ -import * as React from "react" -import ValueEditor from '../../../components/editors/ValueEditor' -import {render, waitFor} from "../../test-utils"; +import * as React from "react"; +import ValueEditor from "../../../components/editors/ValueEditor"; +import { render, waitFor } from "../../test-utils"; test("ValueEditor", async () => { const onEditDone = jest.fn(); - let editor: { current?: ValueEditor | null } = {} - const {asFragment} = render( - editor.current = x} content="hello world" onEditDone={onEditDone}/> + let editor: { current?: ValueEditor | null } = {}; + const { asFragment } = render( + (editor.current = x)} + content="hello world" + onEditDone={onEditDone} + /> ); expect(asFragment()).toMatchSnapshot(); - if (!editor.current) - throw "err"; + if (!editor.current) throw "err"; editor.current.startEditing(); await waitFor(() => expect(editor.current?.isEditing()).toBeTruthy()); diff --git a/web/src/js/__tests__/components/helpers/AutoScrollSpec.tsx b/web/src/js/__tests__/components/helpers/AutoScrollSpec.tsx index 31f570463f..59b8af462d 100644 --- a/web/src/js/__tests__/components/helpers/AutoScrollSpec.tsx +++ b/web/src/js/__tests__/components/helpers/AutoScrollSpec.tsx @@ -1,41 +1,47 @@ import * as React from "react"; -import ReactDOM from "react-dom" -import AutoScroll from '../../../components/helpers/AutoScroll' -import { calcVScroll } from '../../../components/helpers/VirtualScroll' -import TestUtils from 'react-dom/test-utils' +import ReactDOM from "react-dom"; +import AutoScroll from "../../../components/helpers/AutoScroll"; +import { calcVScroll } from "../../../components/helpers/VirtualScroll"; +import TestUtils from "react-dom/test-utils"; -describe('Autoscroll', () => { - let mockFn = jest.fn() +describe("Autoscroll", () => { + let mockFn = jest.fn(); class tComponent extends React.Component { - constructor(props, context){ - super(props, context) - this.state = { vScroll: calcVScroll() } + constructor(props, context) { + super(props, context); + this.state = { vScroll: calcVScroll() }; } - UNSAFE_componentWillUpdate() { - mockFn("foo") - } + UNSAFE_componentWillUpdate() { + mockFn("foo"); + } - componentDidUpdate() { - mockFn("bar") - } + componentDidUpdate() { + mockFn("bar"); + } - render() { - return (

    foo

    ) - } + render() { + return

    foo

    ; + } } - it('should update component', () => { + it("should update component", () => { let Foo = AutoScroll(tComponent), autoScroll = TestUtils.renderIntoDocument(), - viewport = ReactDOM.findDOMNode(autoScroll) - viewport.scrollTop = 10 - Object.defineProperty(viewport, "scrollHeight", { value: 10, writable: true }) - autoScroll.UNSAFE_componentWillUpdate() - expect(mockFn).toBeCalledWith("foo") + viewport = ReactDOM.findDOMNode(autoScroll); + viewport.scrollTop = 10; + Object.defineProperty(viewport, "scrollHeight", { + value: 10, + writable: true, + }); + autoScroll.UNSAFE_componentWillUpdate(); + expect(mockFn).toBeCalledWith("foo"); - Object.defineProperty(viewport, "scrollHeight", { value: 0, writable: true }) - autoScroll.componentDidUpdate() - expect(mockFn).toBeCalledWith("bar") - }) -}) + Object.defineProperty(viewport, "scrollHeight", { + value: 0, + writable: true, + }); + autoScroll.componentDidUpdate(); + expect(mockFn).toBeCalledWith("bar"); + }); +}); diff --git a/web/src/js/__tests__/components/helpers/VirtualScrollSpec.tsx b/web/src/js/__tests__/components/helpers/VirtualScrollSpec.tsx index 7a6239379f..605aa32ec0 100644 --- a/web/src/js/__tests__/components/helpers/VirtualScrollSpec.tsx +++ b/web/src/js/__tests__/components/helpers/VirtualScrollSpec.tsx @@ -1,34 +1,78 @@ -import { calcVScroll } from '../../../components/helpers/VirtualScroll' +import { calcVScroll } from "../../../components/helpers/VirtualScroll"; -describe('VirtualScroll', () => { +describe("VirtualScroll", () => { + it("should return default state without options", () => { + expect(calcVScroll()).toEqual({ + start: 0, + end: 0, + paddingTop: 0, + paddingBottom: 0, + }); + }); - it('should return default state without options', () => { - expect(calcVScroll()).toEqual({start: 0, end: 0, paddingTop: 0, paddingBottom: 0}) - }) + it("should calculate position without itemHeights", () => { + expect( + calcVScroll({ + itemCount: 0, + rowHeight: 32, + viewportHeight: 400, + viewportTop: 0, + }) + ).toEqual({ + start: 0, + end: 0, + paddingTop: 0, + paddingBottom: 0, + }); + }); - it('should calculate position without itemHeights', () => { - expect(calcVScroll({itemCount: 0, rowHeight: 32, viewportHeight: 400, viewportTop: 0})).toEqual({ - start: 0, end: 0, paddingTop: 0, paddingBottom: 0 - }) - }) + it("should calculate position with itemHeights", () => { + expect( + calcVScroll({ + itemCount: 5, + itemHeights: [100, 100, 100, 100, 100], + viewportHeight: 300, + viewportTop: 0, + rowHeight: 100, + }) + ).toEqual({ + start: 0, + end: 4, + paddingTop: 0, + paddingBottom: 100, + }); + }); - it('should calculate position with itemHeights', () => { - expect(calcVScroll({itemCount: 5, itemHeights: [100, 100, 100, 100, 100], - viewportHeight: 300, viewportTop: 0, rowHeight: 100})).toEqual({ - start: 0, end: 4, paddingTop: 0, paddingBottom: 100 - }) - }) + it("should handle the case where lots of existing rows are removed without itemHeights", () => { + expect( + calcVScroll({ + itemCount: 10, + rowHeight: 32, + viewportHeight: 400, + viewportTop: 12_000, + }) + ).toEqual({ + start: 0, + end: 10, + paddingTop: 0, + paddingBottom: 0, + }); + }); - it('should handle the case where lots of existing rows are removed without itemHeights', () => { - expect(calcVScroll({itemCount: 10, rowHeight: 32, viewportHeight: 400, viewportTop: 12_000})).toEqual({ - start: 0, end: 10, paddingTop: 0, paddingBottom: 0 - }) - }) - - it('should handle the case where lots of existing rows are removed with itemHeights', () => { - expect(calcVScroll({itemCount: 4, itemHeights: [100, 100, 100, 100], - viewportHeight: 400, viewportTop: 12_000, rowHeight: 32})).toEqual({ - start: 0, end: 4, paddingTop: 0, paddingBottom: 0 - }) - }) -}) + it("should handle the case where lots of existing rows are removed with itemHeights", () => { + expect( + calcVScroll({ + itemCount: 4, + itemHeights: [100, 100, 100, 100], + viewportHeight: 400, + viewportTop: 12_000, + rowHeight: 32, + }) + ).toEqual({ + start: 0, + end: 4, + paddingTop: 0, + paddingBottom: 0, + }); + }); +}); diff --git a/web/src/js/__tests__/ducks/commandBarSpec.tsx b/web/src/js/__tests__/ducks/commandBarSpec.tsx index 5262cbcb68..3480bf5980 100644 --- a/web/src/js/__tests__/ducks/commandBarSpec.tsx +++ b/web/src/js/__tests__/ducks/commandBarSpec.tsx @@ -1,10 +1,12 @@ -import reduceCommandBar, * as commandBarActions from '../../ducks/commandBar' +import reduceCommandBar, * as commandBarActions from "../../ducks/commandBar"; test("CommandBar", async () => { - expect(reduceCommandBar(undefined, {type: "other"})).toEqual({ - visible: false - }) - expect(reduceCommandBar(undefined, commandBarActions.toggleVisibility())).toEqual({ - visible: true + expect(reduceCommandBar(undefined, { type: "other" })).toEqual({ + visible: false, + }); + expect( + reduceCommandBar(undefined, commandBarActions.toggleVisibility()) + ).toEqual({ + visible: true, }); }); diff --git a/web/src/js/__tests__/ducks/connectionSpec.tsx b/web/src/js/__tests__/ducks/connectionSpec.tsx index 68631d9ed6..0254411321 100644 --- a/web/src/js/__tests__/ducks/connectionSpec.tsx +++ b/web/src/js/__tests__/ducks/connectionSpec.tsx @@ -1,41 +1,54 @@ -import reduceConnection from "../../ducks/connection" -import * as ConnectionActions from "../../ducks/connection" -import { ConnectionState } from "../../ducks/connection" +import reduceConnection from "../../ducks/connection"; +import * as ConnectionActions from "../../ducks/connection"; +import { ConnectionState } from "../../ducks/connection"; -describe('connection reducer', () => { - it('should return initial state', () => { - expect(reduceConnection(undefined, {type: "other"})).toEqual({ +describe("connection reducer", () => { + it("should return initial state", () => { + expect(reduceConnection(undefined, { type: "other" })).toEqual({ state: ConnectionState.INIT, message: undefined, - }) - }) + }); + }); - it('should handle start fetch', () => { - expect(reduceConnection(undefined, ConnectionActions.startFetching())).toEqual({ + it("should handle start fetch", () => { + expect( + reduceConnection(undefined, ConnectionActions.startFetching()) + ).toEqual({ state: ConnectionState.FETCHING, message: undefined, - }) - }) + }); + }); - it('should handle connection established', () => { - expect(reduceConnection(undefined, ConnectionActions.connectionEstablished())).toEqual({ + it("should handle connection established", () => { + expect( + reduceConnection( + undefined, + ConnectionActions.connectionEstablished() + ) + ).toEqual({ state: ConnectionState.ESTABLISHED, message: undefined, - }) - }) + }); + }); - it('should handle connection error', () => { - expect(reduceConnection(undefined, ConnectionActions.connectionError("no internet"))).toEqual({ + it("should handle connection error", () => { + expect( + reduceConnection( + undefined, + ConnectionActions.connectionError("no internet") + ) + ).toEqual({ state: ConnectionState.ERROR, message: "no internet", - }) - }) + }); + }); - it('should handle offline mode', () => { - expect(reduceConnection(undefined, ConnectionActions.setOffline())).toEqual({ + it("should handle offline mode", () => { + expect( + reduceConnection(undefined, ConnectionActions.setOffline()) + ).toEqual({ state: ConnectionState.OFFLINE, message: undefined, - }) - }) - -}) + }); + }); +}); diff --git a/web/src/js/__tests__/ducks/eventLogSpec.tsx b/web/src/js/__tests__/ducks/eventLogSpec.tsx index 2d232b5a5f..4560f85daf 100644 --- a/web/src/js/__tests__/ducks/eventLogSpec.tsx +++ b/web/src/js/__tests__/ducks/eventLogSpec.tsx @@ -1,38 +1,52 @@ -import reduceEventLog, * as eventLogActions from '../../ducks/eventLog' -import {reduce} from '../../ducks/utils/store' +import reduceEventLog, * as eventLogActions from "../../ducks/eventLog"; +import { reduce } from "../../ducks/utils/store"; -describe('event log reducer', () => { - it('should return initial state', () => { +describe("event log reducer", () => { + it("should return initial state", () => { expect(reduceEventLog(undefined, {})).toEqual({ visible: false, - filters: { debug: false, info: true, web: true, warn: true, error: true }, + filters: { + debug: false, + info: true, + web: true, + warn: true, + error: true, + }, ...reduce(undefined, {}), - }) - }) + }); + }); - it('should be possible to toggle filter', () => { - let state = reduceEventLog(undefined, eventLogActions.add('foo')) - expect(reduceEventLog(state, eventLogActions.toggleFilter('info'))).toEqual({ + it("should be possible to toggle filter", () => { + let state = reduceEventLog(undefined, eventLogActions.add("foo")); + expect( + reduceEventLog(state, eventLogActions.toggleFilter("info")) + ).toEqual({ visible: false, - filters: { ...state.filters, info: false}, - ...reduce(state, {}) - }) - }) + filters: { ...state.filters, info: false }, + ...reduce(state, {}), + }); + }); - it('should be possible to toggle visibility', () => { - let state = reduceEventLog(undefined, {}) - expect(reduceEventLog(state, eventLogActions.toggleVisibility())).toEqual({ + it("should be possible to toggle visibility", () => { + let state = reduceEventLog(undefined, {}); + expect( + reduceEventLog(state, eventLogActions.toggleVisibility()) + ).toEqual({ visible: true, - filters: {...state.filters}, - ...reduce(undefined, {}) - }) - }) + filters: { ...state.filters }, + ...reduce(undefined, {}), + }); + }); - it('should be possible to add message', () => { - let state = reduceEventLog(undefined, eventLogActions.add('foo')) - expect(state.visible).toBeFalsy() + it("should be possible to add message", () => { + let state = reduceEventLog(undefined, eventLogActions.add("foo")); + expect(state.visible).toBeFalsy(); expect(state.filters).toEqual({ - debug: false, info: true, web: true, warn: true, error: true - }) - }) -}) + debug: false, + info: true, + web: true, + warn: true, + error: true, + }); + }); +}); diff --git a/web/src/js/__tests__/ducks/flowsSpec.tsx b/web/src/js/__tests__/ducks/flowsSpec.tsx index fe4db34b93..03dc030ef8 100644 --- a/web/src/js/__tests__/ducks/flowsSpec.tsx +++ b/web/src/js/__tests__/ducks/flowsSpec.tsx @@ -1,187 +1,225 @@ import reduceFlows, * as flowActions from "../../ducks/flows"; -import {reduce} from "../../ducks/utils/store" -import {fetchApi} from "../../utils" -import {TFlow, TStore} from "./tutils" -import FlowColumns from "../../components/FlowTable/FlowColumns" +import { reduce } from "../../ducks/utils/store"; +import { fetchApi } from "../../utils"; +import { TFlow, TStore } from "./tutils"; +import FlowColumns from "../../components/FlowTable/FlowColumns"; -jest.mock('../../utils') - -describe('flow reducer', () => { +jest.mock("../../utils"); +describe("flow reducer", () => { let s; for (let i of ["1", "2", "3", "4"]) { - s = reduceFlows(s, {type: flowActions.ADD, data: {id: i}, cmd: 'add'}) + s = reduceFlows(s, { + type: flowActions.ADD, + data: { id: i }, + cmd: "add", + }); } let state = s; - it('should return initial state', () => { + it("should return initial state", () => { expect(reduceFlows(undefined, {})).toEqual({ highlight: undefined, filter: undefined, - sort: {column: undefined, desc: false}, + sort: { column: undefined, desc: false }, selected: [], - ...reduce(undefined, {}) - }) - }) - - describe('selections', () => { - it('should be possible to select a single flow', () => { - expect(reduceFlows(state, flowActions.select("2"))).toEqual( - { - ...state, - selected: ["2"], - } - ) - }) - - it('should be possible to deselect a flow', () => { - expect(reduceFlows({...state, selected: ["1"]}, flowActions.select())).toEqual( - { - ...state, - selected: [], - } - ) - }) - - it('should be possible to select relative', () => { - // haven't selected any flow + ...reduce(undefined, {}), + }); + }); + + describe("selections", () => { + it("should be possible to select a single flow", () => { + expect(reduceFlows(state, flowActions.select("2"))).toEqual({ + ...state, + selected: ["2"], + }); + }); + + it("should be possible to deselect a flow", () => { expect( - flowActions.selectRelative(state, 1) - ).toEqual( + reduceFlows({ ...state, selected: ["1"] }, flowActions.select()) + ).toEqual({ + ...state, + selected: [], + }); + }); + + it("should be possible to select relative", () => { + // haven't selected any flow + expect(flowActions.selectRelative(state, 1)).toEqual( flowActions.select("4") - ) + ); // already selected some flows expect( - flowActions.selectRelative({...state, selected: [2]}, 1) - ).toEqual( - flowActions.select("3") - ) - }) - - it('should update state.selected on remove', () => { - let next - next = reduceFlows({...state, selected: ["2"]}, { - type: flowActions.REMOVE, - data: "2", - cmd: 'remove' - }) - expect(next.selected).toEqual(["3"]) + flowActions.selectRelative({ ...state, selected: [2] }, 1) + ).toEqual(flowActions.select("3")); + }); + + it("should update state.selected on remove", () => { + let next; + next = reduceFlows( + { ...state, selected: ["2"] }, + { + type: flowActions.REMOVE, + data: "2", + cmd: "remove", + } + ); + expect(next.selected).toEqual(["3"]); //last row - next = reduceFlows({...state, selected: ["4"]}, { - type: flowActions.REMOVE, - data: "4", - cmd: 'remove' - }) - expect(next.selected).toEqual(["3"]) + next = reduceFlows( + { ...state, selected: ["4"] }, + { + type: flowActions.REMOVE, + data: "4", + cmd: "remove", + } + ); + expect(next.selected).toEqual(["3"]); //multiple selection - next = reduceFlows({...state, selected: ["2", "3", "4"]}, { - type: flowActions.REMOVE, - data: "3", - cmd: 'remove' - }) - expect(next.selected).toEqual(["2", "4"]) - }) - }) - - it('should be possible to set filter', () => { - let filt = "~u 123" - expect(reduceFlows(undefined, flowActions.setFilter(filt)).filter).toEqual(filt) - }) - - it('should be possible to set highlight', () => { - let key = "foo" - expect(reduceFlows(undefined, flowActions.setHighlight(key)).highlight).toEqual(key) - }) - - it('should be possible to set sort', () => { - let sort = {column: "tls", desc: true} - expect(reduceFlows(undefined, flowActions.setSort(sort.column, sort.desc)).sort).toEqual(sort) - }) - -}) - -describe('flows actions', () => { + next = reduceFlows( + { ...state, selected: ["2", "3", "4"] }, + { + type: flowActions.REMOVE, + data: "3", + cmd: "remove", + } + ); + expect(next.selected).toEqual(["2", "4"]); + }); + }); + + it("should be possible to set filter", () => { + let filt = "~u 123"; + expect( + reduceFlows(undefined, flowActions.setFilter(filt)).filter + ).toEqual(filt); + }); + + it("should be possible to set highlight", () => { + let key = "foo"; + expect( + reduceFlows(undefined, flowActions.setHighlight(key)).highlight + ).toEqual(key); + }); + + it("should be possible to set sort", () => { + let sort = { column: "tls", desc: true }; + expect( + reduceFlows(undefined, flowActions.setSort(sort.column, sort.desc)) + .sort + ).toEqual(sort); + }); +}); +describe("flows actions", () => { let store = TStore(); let tflow = TFlow(); - it('should handle resume action', () => { - store.dispatch(flowActions.resume(tflow)) - expect(fetchApi).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/resume', {method: 'POST'}) - }) - - it('should handle resumeAll action', () => { - store.dispatch(flowActions.resumeAll()) - expect(fetchApi).toBeCalledWith('/flows/resume', {method: 'POST'}) - }) - - it('should handle kill action', () => { - store.dispatch(flowActions.kill(tflow)) - expect(fetchApi).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/kill', {method: 'POST'}) - - }) - - it('should handle killAll action', () => { - store.dispatch(flowActions.killAll()) - expect(fetchApi).toBeCalledWith('/flows/kill', {method: 'POST'}) - }) - - it('should handle remove action', () => { - store.dispatch(flowActions.remove(tflow)) - expect(fetchApi).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29', {method: 'DELETE'}) - }) - - it('should handle duplicate action', () => { - store.dispatch(flowActions.duplicate(tflow)) - expect(fetchApi).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/duplicate', {method: 'POST'}) - }) - - it('should handle replay action', () => { - store.dispatch(flowActions.replay(tflow)) - expect(fetchApi).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/replay', {method: 'POST'}) - }) - - it('should handle revert action', () => { - store.dispatch(flowActions.revert(tflow)) - expect(fetchApi).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/revert', {method: 'POST'}) - }) - - it('should handle update action', () => { - store.dispatch(flowActions.update(tflow, 'foo')) - expect(fetchApi.put).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29', 'foo') - }) - - it('should handle uploadContent action', () => { + it("should handle resume action", () => { + store.dispatch(flowActions.resume(tflow)); + expect(fetchApi).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/resume", + { method: "POST" } + ); + }); + + it("should handle resumeAll action", () => { + store.dispatch(flowActions.resumeAll()); + expect(fetchApi).toBeCalledWith("/flows/resume", { method: "POST" }); + }); + + it("should handle kill action", () => { + store.dispatch(flowActions.kill(tflow)); + expect(fetchApi).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/kill", + { method: "POST" } + ); + }); + + it("should handle killAll action", () => { + store.dispatch(flowActions.killAll()); + expect(fetchApi).toBeCalledWith("/flows/kill", { method: "POST" }); + }); + + it("should handle remove action", () => { + store.dispatch(flowActions.remove(tflow)); + expect(fetchApi).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29", + { method: "DELETE" } + ); + }); + + it("should handle duplicate action", () => { + store.dispatch(flowActions.duplicate(tflow)); + expect(fetchApi).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/duplicate", + { method: "POST" } + ); + }); + + it("should handle replay action", () => { + store.dispatch(flowActions.replay(tflow)); + expect(fetchApi).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/replay", + { method: "POST" } + ); + }); + + it("should handle revert action", () => { + store.dispatch(flowActions.revert(tflow)); + expect(fetchApi).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/revert", + { method: "POST" } + ); + }); + + it("should handle update action", () => { + store.dispatch(flowActions.update(tflow, "foo")); + expect(fetchApi.put).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29", + "foo" + ); + }); + + it("should handle uploadContent action", () => { let body = new FormData(), - file = new window.Blob(['foo'], {type: 'plain/text'}) - body.append('file', file) - store.dispatch(flowActions.uploadContent(tflow, 'foo', 'foo')) + file = new window.Blob(["foo"], { type: "plain/text" }); + body.append("file", file); + store.dispatch(flowActions.uploadContent(tflow, "foo", "foo")); // window.Blob's lastModified is always the current time, // which causes flaky tests on comparison. - expect(fetchApi).toBeCalledWith('/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/foo/content.data', { - method: 'POST', - body: expect.anything() - }) - }) - - it('should handle clear action', () => { - store.dispatch(flowActions.clear()) - expect(fetchApi).toBeCalledWith('/clear', {method: 'POST'}) - }) - - it('should handle upload action', () => { - let body = new FormData() - body.append('file', 'foo') - store.dispatch(flowActions.upload('foo')) - expect(fetchApi).toBeCalledWith('/flows/dump', {method: 'POST', body}) - }) -}) + expect(fetchApi).toBeCalledWith( + "/flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/foo/content.data", + { + method: "POST", + body: expect.anything(), + } + ); + }); + + it("should handle clear action", () => { + store.dispatch(flowActions.clear()); + expect(fetchApi).toBeCalledWith("/clear", { method: "POST" }); + }); + + it("should handle upload action", () => { + let body = new FormData(); + body.append("file", "foo"); + store.dispatch(flowActions.upload("foo")); + expect(fetchApi).toBeCalledWith("/flows/dump", { + method: "POST", + body, + }); + }); +}); test("makeSort", () => { - const a = TFlow(), b = TFlow(); + const a = TFlow(), + b = TFlow(); a.request.scheme = "https"; a.request.method = "POST"; a.request.path = "/foo"; @@ -190,8 +228,7 @@ test("makeSort", () => { Object.keys(FlowColumns).forEach((column, i) => { // @ts-ignore - const sort = flowActions.makeSort({column, desc: i % 2 == 0}); + const sort = flowActions.makeSort({ column, desc: i % 2 == 0 }); expect(sort(a, b)).toBeDefined(); - }) - + }); }); diff --git a/web/src/js/__tests__/ducks/indexSpec.tsx b/web/src/js/__tests__/ducks/indexSpec.tsx index efe6ce7f89..c72900fbaa 100644 --- a/web/src/js/__tests__/ducks/indexSpec.tsx +++ b/web/src/js/__tests__/ducks/indexSpec.tsx @@ -1,11 +1,11 @@ -import {rootReducer} from '../../ducks/index' +import { rootReducer } from "../../ducks/index"; -describe('reduceState in js/ducks/index.js', () => { - it('should combine flow and header', () => { - let state = rootReducer(undefined, {type: "other"}) - expect(state.hasOwnProperty('eventLog')).toBeTruthy() - expect(state.hasOwnProperty('flows')).toBeTruthy() - expect(state.hasOwnProperty('connection')).toBeTruthy() - expect(state.hasOwnProperty('ui')).toBeTruthy() - }) -}) +describe("reduceState in js/ducks/index.js", () => { + it("should combine flow and header", () => { + let state = rootReducer(undefined, { type: "other" }); + expect(state.hasOwnProperty("eventLog")).toBeTruthy(); + expect(state.hasOwnProperty("flows")).toBeTruthy(); + expect(state.hasOwnProperty("connection")).toBeTruthy(); + expect(state.hasOwnProperty("ui")).toBeTruthy(); + }); +}); diff --git a/web/src/js/__tests__/ducks/optionsSpec.tsx b/web/src/js/__tests__/ducks/optionsSpec.tsx index 4614b45bc3..b0e71b25ae 100644 --- a/web/src/js/__tests__/ducks/optionsSpec.tsx +++ b/web/src/js/__tests__/ducks/optionsSpec.tsx @@ -1,42 +1,55 @@ -import reduceOptions, * as OptionsActions from '../../ducks/options' -import * as OptionsEditorActions from '../../ducks/ui/optionsEditor' -import {enableFetchMocks} from "jest-fetch-mock"; -import {TStore} from "./tutils"; +import reduceOptions, * as OptionsActions from "../../ducks/options"; +import * as OptionsEditorActions from "../../ducks/ui/optionsEditor"; +import { enableFetchMocks } from "jest-fetch-mock"; +import { TStore } from "./tutils"; +describe("option reducer", () => { + it("should return initial state", () => { + expect(reduceOptions(undefined, { type: "other" })).toEqual( + OptionsActions.defaultState + ); + }); -describe('option reducer', () => { - it('should return initial state', () => { - expect(reduceOptions(undefined, {type: "other"})).toEqual(OptionsActions.defaultState) - }) + it("should handle receive action", () => { + let action = { + type: OptionsActions.RECEIVE, + data: { id: { value: "foo" } }, + }; + expect(reduceOptions(undefined, action)).toEqual({ id: "foo" }); + }); - it('should handle receive action', () => { - let action = {type: OptionsActions.RECEIVE, data: {id: {value: 'foo'}}} - expect(reduceOptions(undefined, action)).toEqual({id: 'foo'}) - }) - - it('should handle update action', () => { - let action = {type: OptionsActions.UPDATE, data: {id: {value: 1}}} - expect(reduceOptions(undefined, action)).toEqual({...OptionsActions.defaultState, id: 1}) - }) -}) + it("should handle update action", () => { + let action = { + type: OptionsActions.UPDATE, + data: { id: { value: 1 } }, + }; + expect(reduceOptions(undefined, action)).toEqual({ + ...OptionsActions.defaultState, + id: 1, + }); + }); +}); test("sendUpdate", async () => { enableFetchMocks(); let store = TStore(); - fetchMock.mockResponseOnce("fooerror", {status: 404}); - await store.dispatch(dispatch => OptionsActions.pureSendUpdate("intercept", "~~~", dispatch)) + fetchMock.mockResponseOnce("fooerror", { status: 404 }); + await store.dispatch((dispatch) => + OptionsActions.pureSendUpdate("intercept", "~~~", dispatch) + ); expect(store.getActions()).toEqual([ - OptionsEditorActions.updateError("intercept", "fooerror") - ]) + OptionsEditorActions.updateError("intercept", "fooerror"), + ]); store.clearActions(); - fetchMock.mockResponseOnce("", {status: 200}); - await store.dispatch(dispatch => OptionsActions.pureSendUpdate("intercept", "valid", dispatch)) + fetchMock.mockResponseOnce("", { status: 200 }); + await store.dispatch((dispatch) => + OptionsActions.pureSendUpdate("intercept", "valid", dispatch) + ); expect(store.getActions()).toEqual([ - OptionsEditorActions.updateSuccess("intercept") - ]) - + OptionsEditorActions.updateSuccess("intercept"), + ]); }); test("save", async () => { @@ -60,7 +73,7 @@ test("addInterceptFilter", async () => { expect(fetchMock.mock.calls).toHaveLength(1); await store.dispatch(OptionsActions.addInterceptFilter("~u bar")); - expect(fetchMock.mock.calls[1][1]?.body).toEqual('{"intercept":"~u foo | ~u bar"}'); - - + expect(fetchMock.mock.calls[1][1]?.body).toEqual( + '{"intercept":"~u foo | ~u bar"}' + ); }); diff --git a/web/src/js/__tests__/ducks/options_metaSpec.tsx b/web/src/js/__tests__/ducks/options_metaSpec.tsx index e7d1ac4b48..82d1933862 100644 --- a/web/src/js/__tests__/ducks/options_metaSpec.tsx +++ b/web/src/js/__tests__/ducks/options_metaSpec.tsx @@ -2,15 +2,21 @@ import reduceOptionsMeta, * as OptionsMetaActions from "../../ducks/options_meta import * as OptionsActions from "../../ducks/options"; test("options_meta", async () => { - expect(reduceOptionsMeta(undefined, {type: "other"})).toEqual(OptionsMetaActions.defaultState); + expect(reduceOptionsMeta(undefined, { type: "other" })).toEqual( + OptionsMetaActions.defaultState + ); - expect(reduceOptionsMeta(undefined, { - type: OptionsActions.RECEIVE, - data: {id: {value: 'foo'}} - })).toEqual({id: {value: 'foo'}}) + expect( + reduceOptionsMeta(undefined, { + type: OptionsActions.RECEIVE, + data: { id: { value: "foo" } }, + }) + ).toEqual({ id: { value: "foo" } }); - expect(reduceOptionsMeta(undefined, { - type: OptionsActions.UPDATE, - data: {id: {value: 1}} - })).toEqual({...OptionsMetaActions.defaultState, id: {value: 1}}) + expect( + reduceOptionsMeta(undefined, { + type: OptionsActions.UPDATE, + data: { id: { value: 1 } }, + }) + ).toEqual({ ...OptionsMetaActions.defaultState, id: { value: 1 } }); }); diff --git a/web/src/js/__tests__/ducks/tutils.ts b/web/src/js/__tests__/ducks/tutils.ts index b555246146..868f236075 100644 --- a/web/src/js/__tests__/ducks/tutils.ts +++ b/web/src/js/__tests__/ducks/tutils.ts @@ -1,23 +1,27 @@ -import thunk from 'redux-thunk' -import configureStore, {MockStoreCreator, MockStoreEnhanced} from 'redux-mock-store' -import {ConnectionState} from '../../ducks/connection' -import {TDNSFlow, THTTPFlow, TTCPFlow, TUDPFlow} from './_tflow' -import {AppDispatch, RootState} from "../../ducks"; -import {DNSFlow, HTTPFlow, TCPFlow, UDPFlow} from "../../flow"; -import {defaultState as defaultOptions} from "../../ducks/options" -import {TBackendState} from "./_tbackendstate" +import thunk from "redux-thunk"; +import configureStore, { + MockStoreCreator, + MockStoreEnhanced, +} from "redux-mock-store"; +import { ConnectionState } from "../../ducks/connection"; +import { TDNSFlow, THTTPFlow, TTCPFlow, TUDPFlow } from "./_tflow"; +import { AppDispatch, RootState } from "../../ducks"; +import { DNSFlow, HTTPFlow, TCPFlow, UDPFlow } from "../../flow"; +import { defaultState as defaultOptions } from "../../ducks/options"; +import { TBackendState } from "./_tbackendstate"; -const mockStoreCreator: MockStoreCreator = configureStore([thunk]) +const mockStoreCreator: MockStoreCreator = + configureStore([thunk]); -export {THTTPFlow as TFlow, TTCPFlow, TUDPFlow} +export { THTTPFlow as TFlow, TTCPFlow, TUDPFlow }; const tflow0: HTTPFlow = THTTPFlow(); const tflow1: HTTPFlow = THTTPFlow(); const tflow2: TCPFlow = TTCPFlow(); const tflow3: DNSFlow = TDNSFlow(); const tflow4: UDPFlow = TUDPFlow(); -tflow0.modified = true -tflow0.intercepted = true +tflow0.modified = true; +tflow0.intercepted = true; tflow1.id = "flow2"; tflow1.request.path = "/second"; @@ -25,51 +29,48 @@ export const testState: RootState = { backendState: TBackendState(), options_meta: { anticache: { - "type": "bool", - "default": false, - "value": false, - "help": "Strip out request headers that might cause the server to return 304-not-modified.", - "choices": undefined + type: "bool", + default: false, + value: false, + help: "Strip out request headers that might cause the server to return 304-not-modified.", + choices: undefined, }, body_size_limit: { - "type": "optional str", - "default": undefined, - "value": undefined, - "help": "Byte size limit of HTTP request and response bodies. Understands k/m/g suffixes, i.e. 3m for 3 megabytes.", - "choices": undefined, + type: "optional str", + default: undefined, + value: undefined, + help: "Byte size limit of HTTP request and response bodies. Understands k/m/g suffixes, i.e. 3m for 3 megabytes.", + choices: undefined, }, connection_strategy: { - "type": "str", - "default": "eager", - "value": "eager", - "help": "Determine when server connections should be established. When set to lazy, mitmproxy tries to defer establishing an upstream connection as long as possible. This makes it possible to use server replay while being offline. When set to eager, mitmproxy can detect protocols with server-side greetings, as well as accurately mirror TLS ALPN negotiation.", - "choices": [ - "eager", - "lazy" - ] + type: "str", + default: "eager", + value: "eager", + help: "Determine when server connections should be established. When set to lazy, mitmproxy tries to defer establishing an upstream connection as long as possible. This makes it possible to use server replay while being offline. When set to eager, mitmproxy can detect protocols with server-side greetings, as well as accurately mirror TLS ALPN negotiation.", + choices: ["eager", "lazy"], }, listen_port: { - "type": "int", - "default": 8080, - "value": 8080, - "help": "Proxy service port.", - "choices": undefined - } + type: "int", + default: 8080, + value: 8080, + help: "Proxy service port.", + choices: undefined, + }, }, ui: { flow: { contentViewFor: {}, - tab: 'request' + tab: "request", }, modal: { - activeModal: undefined + activeModal: undefined, }, optionsEditor: { - booleanOption: {isUpdating: true, error: false}, - strOption: {error: true}, + booleanOption: { isUpdating: true, error: false }, + strOption: { error: true }, intOption: {}, choiceOption: {}, - } + }, }, options: defaultOptions, flows: { @@ -81,11 +82,11 @@ export const testState: RootState = { [tflow3.id]: tflow3, [tflow4.id]: tflow4, }, - filter: '~u /second | ~tcp | ~dns | ~udp', - highlight: '~u /path', + filter: "~u /second | ~tcp | ~dns | ~udp", + highlight: "~u /path", sort: { desc: true, - column: "path" + column: "path", }, view: [tflow1, tflow2, tflow3, tflow4], list: [tflow0, tflow1, tflow2, tflow3, tflow4], @@ -104,7 +105,7 @@ export const testState: RootState = { }, }, connection: { - state: ConnectionState.ESTABLISHED + state: ConnectionState.ESTABLISHED, }, eventLog: { visible: true, @@ -113,23 +114,22 @@ export const testState: RootState = { info: true, web: false, warn: true, - error: true + error: true, }, view: [ - {id: "1", level: 'info', message: 'foo'}, - {id: "2", level: 'error', message: 'bar'} + { id: "1", level: "info", message: "foo" }, + { id: "2", level: "error", message: "bar" }, ], byId: {}, // TODO: incomplete - list: [], // TODO: incomplete - listIndex: {}, // TODO: incomplete - viewIndex: {}, // TODO: incomplete + list: [], // TODO: incomplete + listIndex: {}, // TODO: incomplete + viewIndex: {}, // TODO: incomplete }, commandBar: { visible: false, - } -} - + }, +}; export function TStore(): MockStoreEnhanced { - return mockStoreCreator(testState) + return mockStoreCreator(testState); } diff --git a/web/src/js/__tests__/ducks/ui/flowSpec.tsx b/web/src/js/__tests__/ducks/ui/flowSpec.tsx index 7b1bbdeab6..ad397004f6 100644 --- a/web/src/js/__tests__/ducks/ui/flowSpec.tsx +++ b/web/src/js/__tests__/ducks/ui/flowSpec.tsx @@ -1,20 +1,22 @@ -import reduceFlow, * as FlowActions from '../../../ducks/ui/flow' +import reduceFlow, * as FlowActions from "../../../ducks/ui/flow"; +describe("option reducer", () => { + it("should return initial state", () => { + expect(reduceFlow(undefined, { type: "other" })).toEqual( + FlowActions.defaultState + ); + }); -describe('option reducer', () => { - it('should return initial state', () => { - expect(reduceFlow(undefined, {type: "other"})).toEqual(FlowActions.defaultState) - }) - - it('should handle set tab', () => { + it("should handle set tab", () => { expect( reduceFlow(undefined, FlowActions.selectTab("response")).tab - ).toEqual("response") - }) + ).toEqual("response"); + }); - it('should handle set content view', () => { + it("should handle set content view", () => { expect( - reduceFlow(undefined, FlowActions.setContentViewFor("foo", "Raw")).contentViewFor["foo"] - ).toEqual("Raw") - }) -}) + reduceFlow(undefined, FlowActions.setContentViewFor("foo", "Raw")) + .contentViewFor["foo"] + ).toEqual("Raw"); + }); +}); diff --git a/web/src/js/__tests__/ducks/ui/indexSpec.tsx b/web/src/js/__tests__/ducks/ui/indexSpec.tsx index 515d1b3196..3db54fe7b3 100644 --- a/web/src/js/__tests__/ducks/ui/indexSpec.tsx +++ b/web/src/js/__tests__/ducks/ui/indexSpec.tsx @@ -1,8 +1,8 @@ -import reduceUI from '../../../ducks/ui/index' +import reduceUI from "../../../ducks/ui/index"; -describe('reduceUI in js/ducks/ui/index.js', () => { - it('should combine flow and header', () => { - let state = reduceUI(undefined, {type: "other"}) - expect(state.hasOwnProperty('flow')).toBeTruthy() - }) -}) +describe("reduceUI in js/ducks/ui/index.js", () => { + it("should combine flow and header", () => { + let state = reduceUI(undefined, { type: "other" }); + expect(state.hasOwnProperty("flow")).toBeTruthy(); + }); +}); diff --git a/web/src/js/__tests__/ducks/ui/keyboardSpec.tsx b/web/src/js/__tests__/ducks/ui/keyboardSpec.tsx index 514cfb98cf..b1803ee9a0 100644 --- a/web/src/js/__tests__/ducks/ui/keyboardSpec.tsx +++ b/web/src/js/__tests__/ducks/ui/keyboardSpec.tsx @@ -1,20 +1,20 @@ import reduceFlows, * as flowsActions from "../../../ducks/flows"; -import {onKeyDown} from '../../../ducks/ui/keyboard' -import * as UIActions from '../../../ducks/ui/flow' -import * as modalActions from '../../../ducks/ui/modal' -import {fetchApi, runCommand} from '../../../utils' -import {TStore} from "../tutils"; +import { onKeyDown } from "../../../ducks/ui/keyboard"; +import * as UIActions from "../../../ducks/ui/flow"; +import * as modalActions from "../../../ducks/ui/modal"; +import { fetchApi, runCommand } from "../../../utils"; +import { TStore } from "../tutils"; -jest.mock('../../../utils') +jest.mock("../../../utils"); -describe('onKeyDown', () => { +describe("onKeyDown", () => { let flows = flowsActions.defaultState; for (let i = 1; i <= 12; i++) { flows = reduceFlows(flows, { type: flowsActions.ADD, - data: {id: i + "", request: true, response: true, type: "http"}, - cmd: 'add' - }) + data: { id: i + "", request: true, response: true, type: "http" }, + cmd: "add", + }); } const store = TStore(); @@ -22,150 +22,184 @@ describe('onKeyDown', () => { let createKeyEvent = (key, ctrlKey = false) => { // @ts-ignore - return onKeyDown({key, ctrlKey, preventDefault: jest.fn()}) - } + return onKeyDown({ key, ctrlKey, preventDefault: jest.fn() }); + }; afterEach(() => { - store.clearActions() + store.clearActions(); // @ts-ignore - fetchApi.mockClear() - }); - - it('should handle cursor up', () => { - store.getState().flows = reduceFlows(flows, flowsActions.select("2")) - store.dispatch(createKeyEvent("k")) - expect(store.getActions()).toEqual([{flowIds: ["1"], type: flowsActions.SELECT}]) - - store.clearActions() - store.dispatch(createKeyEvent("ArrowUp")) - expect(store.getActions()).toEqual([{flowIds: ["1"], type: flowsActions.SELECT}]) - }) - - it('should handle cursor down', () => { - store.dispatch(createKeyEvent("j")) - expect(store.getActions()).toEqual([{flowIds: ["3"], type: flowsActions.SELECT}]) - - store.clearActions() - store.dispatch(createKeyEvent("ArrowDown")) - expect(store.getActions()).toEqual([{flowIds: ["3"], type: flowsActions.SELECT}]) - }) - - it('should handle page down', () => { - store.dispatch(createKeyEvent(" ")) - expect(store.getActions()).toEqual([{flowIds: ["12"], type: flowsActions.SELECT}]) - - store.getState().flows = reduceFlows(flows, flowsActions.select("1")) - store.clearActions() - store.dispatch(createKeyEvent("PageDown")) - expect(store.getActions()).toEqual([{flowIds: ["11"], type: flowsActions.SELECT}]) - }) - - it('should handle page up', () => { - store.getState().flows = reduceFlows(flows, flowsActions.select("11")) - store.dispatch(createKeyEvent("PageUp")) - expect(store.getActions()).toEqual([{flowIds: ["1"], type: flowsActions.SELECT}]) - }) - - it('should handle select first', () => { - store.dispatch(createKeyEvent("Home")) - expect(store.getActions()).toEqual([{flowIds: ["1"], type: flowsActions.SELECT}]) - }) - - it('should handle select last', () => { - store.getState().flows = reduceFlows(flows, flowsActions.select("1")) - store.dispatch(createKeyEvent("End")) - expect(store.getActions()).toEqual([{flowIds: ["12"], type: flowsActions.SELECT}]) - }) - - it('should handle deselect', () => { - store.dispatch(createKeyEvent("Escape")) - expect(store.getActions()).toEqual([{flowIds: [], type: flowsActions.SELECT}]) - }) - - it('should handle switch to left tab', () => { - store.dispatch(createKeyEvent("ArrowLeft")) - expect(store.getActions()).toEqual([{tab: 'timing', type: UIActions.SET_TAB}]) - }) - - it('should handle switch to right tab', () => { - store.dispatch(createKeyEvent("Tab")) - expect(store.getActions()).toEqual([{tab: 'response', type: UIActions.SET_TAB}]) - - store.clearActions() - store.dispatch(createKeyEvent("ArrowRight")) - expect(store.getActions()).toEqual([{tab: 'response', type: UIActions.SET_TAB}]) - }) - - it('should handle delete action', () => { - store.dispatch(createKeyEvent("d")) - expect(fetchApi).toBeCalledWith('/flows/1', {method: 'DELETE'}) - - }) - - it('should handle create action', () => { - store.dispatch(createKeyEvent("n")) - expect(runCommand).toBeCalledWith('view.flows.create', "get", "https://example.com/") - }) - - it('should handle duplicate action', () => { - store.dispatch(createKeyEvent("D")) - expect(fetchApi).toBeCalledWith('/flows/1/duplicate', {method: 'POST'}) - }) - - it('should handle resume action', () => { + fetchApi.mockClear(); + }); + + it("should handle cursor up", () => { + store.getState().flows = reduceFlows(flows, flowsActions.select("2")); + store.dispatch(createKeyEvent("k")); + expect(store.getActions()).toEqual([ + { flowIds: ["1"], type: flowsActions.SELECT }, + ]); + + store.clearActions(); + store.dispatch(createKeyEvent("ArrowUp")); + expect(store.getActions()).toEqual([ + { flowIds: ["1"], type: flowsActions.SELECT }, + ]); + }); + + it("should handle cursor down", () => { + store.dispatch(createKeyEvent("j")); + expect(store.getActions()).toEqual([ + { flowIds: ["3"], type: flowsActions.SELECT }, + ]); + + store.clearActions(); + store.dispatch(createKeyEvent("ArrowDown")); + expect(store.getActions()).toEqual([ + { flowIds: ["3"], type: flowsActions.SELECT }, + ]); + }); + + it("should handle page down", () => { + store.dispatch(createKeyEvent(" ")); + expect(store.getActions()).toEqual([ + { flowIds: ["12"], type: flowsActions.SELECT }, + ]); + + store.getState().flows = reduceFlows(flows, flowsActions.select("1")); + store.clearActions(); + store.dispatch(createKeyEvent("PageDown")); + expect(store.getActions()).toEqual([ + { flowIds: ["11"], type: flowsActions.SELECT }, + ]); + }); + + it("should handle page up", () => { + store.getState().flows = reduceFlows(flows, flowsActions.select("11")); + store.dispatch(createKeyEvent("PageUp")); + expect(store.getActions()).toEqual([ + { flowIds: ["1"], type: flowsActions.SELECT }, + ]); + }); + + it("should handle select first", () => { + store.dispatch(createKeyEvent("Home")); + expect(store.getActions()).toEqual([ + { flowIds: ["1"], type: flowsActions.SELECT }, + ]); + }); + + it("should handle select last", () => { + store.getState().flows = reduceFlows(flows, flowsActions.select("1")); + store.dispatch(createKeyEvent("End")); + expect(store.getActions()).toEqual([ + { flowIds: ["12"], type: flowsActions.SELECT }, + ]); + }); + + it("should handle deselect", () => { + store.dispatch(createKeyEvent("Escape")); + expect(store.getActions()).toEqual([ + { flowIds: [], type: flowsActions.SELECT }, + ]); + }); + + it("should handle switch to left tab", () => { + store.dispatch(createKeyEvent("ArrowLeft")); + expect(store.getActions()).toEqual([ + { tab: "timing", type: UIActions.SET_TAB }, + ]); + }); + + it("should handle switch to right tab", () => { + store.dispatch(createKeyEvent("Tab")); + expect(store.getActions()).toEqual([ + { tab: "response", type: UIActions.SET_TAB }, + ]); + + store.clearActions(); + store.dispatch(createKeyEvent("ArrowRight")); + expect(store.getActions()).toEqual([ + { tab: "response", type: UIActions.SET_TAB }, + ]); + }); + + it("should handle delete action", () => { + store.dispatch(createKeyEvent("d")); + expect(fetchApi).toBeCalledWith("/flows/1", { method: "DELETE" }); + }); + + it("should handle create action", () => { + store.dispatch(createKeyEvent("n")); + expect(runCommand).toBeCalledWith( + "view.flows.create", + "get", + "https://example.com/" + ); + }); + + it("should handle duplicate action", () => { + store.dispatch(createKeyEvent("D")); + expect(fetchApi).toBeCalledWith("/flows/1/duplicate", { + method: "POST", + }); + }); + + it("should handle resume action", () => { // resume all - store.dispatch(createKeyEvent("A")) - expect(fetchApi).toBeCalledWith('/flows/resume', {method: 'POST'}) + store.dispatch(createKeyEvent("A")); + expect(fetchApi).toBeCalledWith("/flows/resume", { method: "POST" }); // resume - store.getState().flows.byId[store.getState().flows.selected[0]].intercepted = true - store.dispatch(createKeyEvent("a")) - expect(fetchApi).toBeCalledWith('/flows/1/resume', {method: 'POST'}) - }) - - it('should handle replay action', () => { - store.dispatch(createKeyEvent("r")) - expect(fetchApi).toBeCalledWith('/flows/1/replay', {method: 'POST'}) - }) - - it('should handle revert action', () => { - store.getState().flows.byId[store.getState().flows.selected[0]].modified = true - store.dispatch(createKeyEvent("v")) - expect(fetchApi).toBeCalledWith('/flows/1/revert', {method: 'POST'}) - }) - - it('should handle kill action', () => { + store.getState().flows.byId[ + store.getState().flows.selected[0] + ].intercepted = true; + store.dispatch(createKeyEvent("a")); + expect(fetchApi).toBeCalledWith("/flows/1/resume", { method: "POST" }); + }); + + it("should handle replay action", () => { + store.dispatch(createKeyEvent("r")); + expect(fetchApi).toBeCalledWith("/flows/1/replay", { method: "POST" }); + }); + + it("should handle revert action", () => { + store.getState().flows.byId[ + store.getState().flows.selected[0] + ].modified = true; + store.dispatch(createKeyEvent("v")); + expect(fetchApi).toBeCalledWith("/flows/1/revert", { method: "POST" }); + }); + + it("should handle kill action", () => { // kill all - store.dispatch(createKeyEvent("X")) - expect(fetchApi).toBeCalledWith('/flows/kill', {method: 'POST'}) + store.dispatch(createKeyEvent("X")); + expect(fetchApi).toBeCalledWith("/flows/kill", { method: "POST" }); // kill - store.dispatch(createKeyEvent("x")) - expect(fetchApi).toBeCalledWith('/flows/1/kill', {method: 'POST'}) - }) - - it('should handle clear action', () => { - store.dispatch(createKeyEvent("z")) - expect(fetchApi).toBeCalledWith('/clear', {method: 'POST'}) - }) - - it('should stop on some action with no flow is selected', () => { - store.getState().flows = reduceFlows(undefined, {}) - store.dispatch(createKeyEvent("ArrowLeft")) - store.dispatch(createKeyEvent("Tab")) - store.dispatch(createKeyEvent("ArrowRight")) - store.dispatch(createKeyEvent("D")) - expect(fetchApi).not.toBeCalled() - }) - - it('should do nothing when Ctrl and undefined key is pressed ', () => { - store.dispatch(createKeyEvent("Backspace", true)) - store.dispatch(createKeyEvent(0)) - expect(fetchApi).not.toBeCalled() - }) - - it('should close modal', () => { - store.getState().ui.modal.activeModal = true - store.dispatch(createKeyEvent("Escape")) - expect(store.getActions()).toEqual([{type: modalActions.HIDE_MODAL}]) - }) - -}) + store.dispatch(createKeyEvent("x")); + expect(fetchApi).toBeCalledWith("/flows/1/kill", { method: "POST" }); + }); + + it("should handle clear action", () => { + store.dispatch(createKeyEvent("z")); + expect(fetchApi).toBeCalledWith("/clear", { method: "POST" }); + }); + + it("should stop on some action with no flow is selected", () => { + store.getState().flows = reduceFlows(undefined, {}); + store.dispatch(createKeyEvent("ArrowLeft")); + store.dispatch(createKeyEvent("Tab")); + store.dispatch(createKeyEvent("ArrowRight")); + store.dispatch(createKeyEvent("D")); + expect(fetchApi).not.toBeCalled(); + }); + + it("should do nothing when Ctrl and undefined key is pressed ", () => { + store.dispatch(createKeyEvent("Backspace", true)); + store.dispatch(createKeyEvent(0)); + expect(fetchApi).not.toBeCalled(); + }); + + it("should close modal", () => { + store.getState().ui.modal.activeModal = true; + store.dispatch(createKeyEvent("Escape")); + expect(store.getActions()).toEqual([{ type: modalActions.HIDE_MODAL }]); + }); +}); diff --git a/web/src/js/__tests__/ducks/ui/modalSpec.tsx b/web/src/js/__tests__/ducks/ui/modalSpec.tsx index b6ef5e6924..7773ee561f 100644 --- a/web/src/js/__tests__/ducks/ui/modalSpec.tsx +++ b/web/src/js/__tests__/ducks/ui/modalSpec.tsx @@ -1,24 +1,17 @@ -import reduceModal, * as ModalActions from '../../../ducks/ui/modal' +import reduceModal, * as ModalActions from "../../../ducks/ui/modal"; -describe('modal reducer', () => { +describe("modal reducer", () => { + it("should return the initial state", () => { + expect(reduceModal(undefined, {})).toEqual({ activeModal: undefined }); + }); - it('should return the initial state', () => { - expect(reduceModal(undefined, {})).toEqual( - { activeModal: undefined } - ) - }) + it("should handle setActiveModal action", () => { + let state = reduceModal(undefined, ModalActions.setActiveModal("foo")); + expect(state).toEqual({ activeModal: "foo" }); + }); - it('should handle setActiveModal action', () => { - let state = reduceModal(undefined, ModalActions.setActiveModal('foo')) - expect(state).toEqual( - { activeModal: 'foo' } - ) - }) - - it('should handle hideModal action', () => { - let state = reduceModal(undefined, ModalActions.hideModal()) - expect(state).toEqual( - { activeModal: undefined } - ) - }) -}) + it("should handle hideModal action", () => { + let state = reduceModal(undefined, ModalActions.hideModal()); + expect(state).toEqual({ activeModal: undefined }); + }); +}); diff --git a/web/src/js/__tests__/ducks/ui/optionEditorSpec.tsx b/web/src/js/__tests__/ducks/ui/optionEditorSpec.tsx index 4d9f46d168..814e782368 100644 --- a/web/src/js/__tests__/ducks/ui/optionEditorSpec.tsx +++ b/web/src/js/__tests__/ducks/ui/optionEditorSpec.tsx @@ -1,33 +1,60 @@ -import reduceOptionsEditor, * as optionsEditorActions from '../../../ducks/ui/optionsEditor' -import { HIDE_MODAL } from '../../../ducks/ui/modal' -import {OptionsState} from "../../../ducks/_options_gen"; +import reduceOptionsEditor, * as optionsEditorActions from "../../../ducks/ui/optionsEditor"; +import { HIDE_MODAL } from "../../../ducks/ui/modal"; +import { OptionsState } from "../../../ducks/_options_gen"; -describe('optionsEditor reducer', () => { +describe("optionsEditor reducer", () => { + it("should return initial state", () => { + expect(reduceOptionsEditor(undefined, {})).toEqual({}); + }); - it('should return initial state', () => { - expect(reduceOptionsEditor(undefined, {})).toEqual({}) - }) + it("should handle option update start", () => { + let state = reduceOptionsEditor( + undefined, + optionsEditorActions.startUpdate("foo", "bar") + ); + expect(state).toEqual({ + foo: { error: false, isUpdating: true, value: "bar" }, + }); + }); - it('should handle option update start', () => { - let state = reduceOptionsEditor(undefined, optionsEditorActions.startUpdate('foo', 'bar')) - expect(state).toEqual({ foo: {error: false, isUpdating: true, value: 'bar'}}) - }) + it("should handle option update success", () => { + expect( + reduceOptionsEditor( + undefined, + optionsEditorActions.updateSuccess("foo") + ) + ).toEqual({ foo: undefined }); + }); - it('should handle option update success', () => { - expect(reduceOptionsEditor(undefined, optionsEditorActions.updateSuccess('foo'))).toEqual({foo: undefined}) - }) - - it('should handle option update error', () => { - let state = reduceOptionsEditor(undefined, optionsEditorActions.startUpdate('foo', 'bar')) - state = reduceOptionsEditor(state, optionsEditorActions.updateError('foo', 'errorMsg')) - expect(state).toEqual({ foo: {error: 'errorMsg', isUpdating: false, value: 'bar'}}) + it("should handle option update error", () => { + let state = reduceOptionsEditor( + undefined, + optionsEditorActions.startUpdate("foo", "bar") + ); + state = reduceOptionsEditor( + state, + optionsEditorActions.updateError("foo", "errorMsg") + ); + expect(state).toEqual({ + foo: { error: "errorMsg", isUpdating: false, value: "bar" }, + }); // boolean type - state = reduceOptionsEditor(undefined, optionsEditorActions.startUpdate('foo', true)) - state = reduceOptionsEditor(state, optionsEditorActions.updateError('foo', 'errorMsg')) - expect(state).toEqual({ foo: {error: 'errorMsg', isUpdating: false, value: false}}) - }) + state = reduceOptionsEditor( + undefined, + optionsEditorActions.startUpdate("foo", true) + ); + state = reduceOptionsEditor( + state, + optionsEditorActions.updateError("foo", "errorMsg") + ); + expect(state).toEqual({ + foo: { error: "errorMsg", isUpdating: false, value: false }, + }); + }); - it('should handle hide modal', () => { - expect(reduceOptionsEditor(undefined, {type: HIDE_MODAL})).toEqual({}) - }) -}) + it("should handle hide modal", () => { + expect(reduceOptionsEditor(undefined, { type: HIDE_MODAL })).toEqual( + {} + ); + }); +}); diff --git a/web/src/js/__tests__/ducks/utils/storeSpec.tsx b/web/src/js/__tests__/ducks/utils/storeSpec.tsx index e156315958..6b4d6d6b9a 100644 --- a/web/src/js/__tests__/ducks/utils/storeSpec.tsx +++ b/web/src/js/__tests__/ducks/utils/storeSpec.tsx @@ -1,188 +1,214 @@ -import * as storeActions from '../../../ducks/utils/store' -import {Item, reduce} from '../../../ducks/utils/store' +import * as storeActions from "../../../ducks/utils/store"; +import { Item, reduce } from "../../../ducks/utils/store"; -describe('store reducer', () => { - it('should return initial state', () => { +describe("store reducer", () => { + it("should return initial state", () => { expect(reduce(undefined, {})).toEqual({ byId: {}, list: [], listIndex: {}, view: [], viewIndex: {}, - }) - }) - - it('should handle add action', () => { - let a = {id: "1"}, - b = {id: "9"}, - state = reduce(undefined, {}) - expect(state = reduce(state, storeActions.add(a))).toEqual({ - byId: {"1": a}, - listIndex: {"1": 0}, + }); + }); + + it("should handle add action", () => { + let a = { id: "1" }, + b = { id: "9" }, + state = reduce(undefined, {}); + expect((state = reduce(state, storeActions.add(a)))).toEqual({ + byId: { "1": a }, + listIndex: { "1": 0 }, list: [a], view: [a], - viewIndex: {"1": 0}, - }) + viewIndex: { "1": 0 }, + }); - expect(state = reduce(state, storeActions.add(b))).toEqual({ - byId: {"1": a, 9: b}, - listIndex: {"1": 0, "9": 1}, + expect((state = reduce(state, storeActions.add(b)))).toEqual({ + byId: { "1": a, 9: b }, + listIndex: { "1": 0, "9": 1 }, list: [a, b], view: [a, b], - viewIndex: {"1": 0, "9": 1}, - }) + viewIndex: { "1": 0, "9": 1 }, + }); // add item and sort them - let c = {id: "0"} - expect(reduce(state, storeActions.add(c, undefined, - (a, b) => { - return a.id > b.id ? 1 : -1 - }))).toEqual({ - byId: {...state.byId, "0": c}, + let c = { id: "0" }; + expect( + reduce( + state, + storeActions.add(c, undefined, (a, b) => { + return a.id > b.id ? 1 : -1; + }) + ) + ).toEqual({ + byId: { ...state.byId, "0": c }, list: [...state.list, c], - listIndex: {...state.listIndex, "0": 2}, + listIndex: { ...state.listIndex, "0": 2 }, view: [c, ...state.view], - viewIndex: {"0": 0, "1": 1, "9": 2} - - }) - }) + viewIndex: { "0": 0, "1": 1, "9": 2 }, + }); + }); - it('should not add the item with duplicated id', () => { - let a = {id: "1"}, - state = reduce(undefined, storeActions.add(a)) - expect(reduce(state, storeActions.add(a))).toEqual(state) - }) + it("should not add the item with duplicated id", () => { + let a = { id: "1" }, + state = reduce(undefined, storeActions.add(a)); + expect(reduce(state, storeActions.add(a))).toEqual(state); + }); - it('should handle update action', () => { + it("should handle update action", () => { interface TItem extends Item { - foo: string + foo: string; } - let a: TItem = {id: "1", foo: "foo"}, - updated = {...a, foo: "bar"}, - state = reduce(undefined, storeActions.add(a)) + let a: TItem = { id: "1", foo: "foo" }, + updated = { ...a, foo: "bar" }, + state = reduce(undefined, storeActions.add(a)); expect(reduce(state, storeActions.update(updated))).toEqual({ - byId: {1: updated}, + byId: { 1: updated }, list: [updated], - listIndex: {1: 0}, + listIndex: { 1: 0 }, view: [updated], - viewIndex: {1: 0}, - }) - }) - - it('should handle update action with filter', () => { - let a = {id: "0"}, b = {id: "1"}, - state = reduce(undefined, storeActions.receive([a, b])) - state = reduce(state, storeActions.update(b, - item => { - return item.id !== "1" - })) + viewIndex: { 1: 0 }, + }); + }); + + it("should handle update action with filter", () => { + let a = { id: "0" }, + b = { id: "1" }, + state = reduce(undefined, storeActions.receive([a, b])); + state = reduce( + state, + storeActions.update(b, (item) => { + return item.id !== "1"; + }) + ); expect(state).toEqual({ - byId: {"0": a, "1": b}, + byId: { "0": a, "1": b }, list: [a, b], - listIndex: {"0": 0, "1": 1}, + listIndex: { "0": 0, "1": 1 }, view: [a], - viewIndex: {"0": 0} - }) - expect(reduce(state, storeActions.update(b, - item => { - return item.id !== "0" - }))).toEqual({ - byId: {"0": a, "1": b}, + viewIndex: { "0": 0 }, + }); + expect( + reduce( + state, + storeActions.update(b, (item) => { + return item.id !== "0"; + }) + ) + ).toEqual({ + byId: { "0": a, "1": b }, list: [a, b], - listIndex: {"0": 0, "1": 1}, + listIndex: { "0": 0, "1": 1 }, view: [a, b], - viewIndex: {"0": 0, "1": 1} - }) - }) - - it('should handle update action with sort', () => { - let a = {id: "2"}, - b = {id: "3"}, - state = reduce(undefined, storeActions.receive([a, b])) - expect(reduce(state, storeActions.update(b, undefined, - (a, b) => { - return b.id > a.id ? 1 : -1 - }))).toEqual({ + viewIndex: { "0": 0, "1": 1 }, + }); + }); + + it("should handle update action with sort", () => { + let a = { id: "2" }, + b = { id: "3" }, + state = reduce(undefined, storeActions.receive([a, b])); + expect( + reduce( + state, + storeActions.update(b, undefined, (a, b) => { + return b.id > a.id ? 1 : -1; + }) + ) + ).toEqual({ // sort by id in descending order - byId: {"2": a, "3": b}, + byId: { "2": a, "3": b }, list: [a, b], - listIndex: {"2": 0, "3": 1}, + listIndex: { "2": 0, "3": 1 }, view: [b, a], - viewIndex: {"2": 1, "3": 0}, - }) - - let state1 = reduce(undefined, storeActions.receive([b, a])) - expect(reduce(state1, storeActions.update(b, undefined, - (a, b) => { - return a.id > b.id ? 1 : -1 - }))).toEqual({ + viewIndex: { "2": 1, "3": 0 }, + }); + + let state1 = reduce(undefined, storeActions.receive([b, a])); + expect( + reduce( + state1, + storeActions.update(b, undefined, (a, b) => { + return a.id > b.id ? 1 : -1; + }) + ) + ).toEqual({ // sort by id in ascending order - byId: {"2": a, "3": b}, + byId: { "2": a, "3": b }, list: [b, a], - listIndex: {"2": 1, "3": 0}, + listIndex: { "2": 1, "3": 0 }, view: [a, b], - viewIndex: {"2": 0, "3": 1}, - }) - }) - - it('should set filter', () => { - let a = {id: "1"}, - b = {id: "2"}, - state = reduce(undefined, storeActions.receive([a, b])) - expect(reduce(state, storeActions.setFilter( - item => { - return item.id !== "1" - } - ))).toEqual({ - byId: {"1": a, "2": b}, + viewIndex: { "2": 0, "3": 1 }, + }); + }); + + it("should set filter", () => { + let a = { id: "1" }, + b = { id: "2" }, + state = reduce(undefined, storeActions.receive([a, b])); + expect( + reduce( + state, + storeActions.setFilter((item) => { + return item.id !== "1"; + }) + ) + ).toEqual({ + byId: { "1": a, "2": b }, list: [a, b], - listIndex: {"1": 0, "2": 1}, + listIndex: { "1": 0, "2": 1 }, view: [b], - viewIndex: {"2": 0}, - }) - }) - - it('should set sort', () => { - let a = {id: "1"}, - b = {id: "2"}, - state = reduce(undefined, storeActions.receive([a, b])) - expect(reduce(state, storeActions.setSort( - (a, b) => { - return b.id > a.id ? 1 : -1 - } - ))).toEqual({ - byId: {1: a, 2: b}, + viewIndex: { "2": 0 }, + }); + }); + + it("should set sort", () => { + let a = { id: "1" }, + b = { id: "2" }, + state = reduce(undefined, storeActions.receive([a, b])); + expect( + reduce( + state, + storeActions.setSort((a, b) => { + return b.id > a.id ? 1 : -1; + }) + ) + ).toEqual({ + byId: { 1: a, 2: b }, list: [a, b], - listIndex: {1: 0, 2: 1}, + listIndex: { 1: 0, 2: 1 }, view: [b, a], - viewIndex: {1: 1, 2: 0}, - }) - }) - - it('should handle remove action', () => { - let a = {id: "1"}, b = {id: "2"}, - state = reduce(undefined, storeActions.receive([a, b])) + viewIndex: { 1: 1, 2: 0 }, + }); + }); + + it("should handle remove action", () => { + let a = { id: "1" }, + b = { id: "2" }, + state = reduce(undefined, storeActions.receive([a, b])); expect(reduce(state, storeActions.remove("1"))).toEqual({ - byId: {"2": b}, + byId: { "2": b }, list: [b], - listIndex: {"2": 0}, + listIndex: { "2": 0 }, view: [b], - viewIndex: {"2": 0}, - }) + viewIndex: { "2": 0 }, + }); - expect(reduce(state, storeActions.remove("3"))).toEqual(state) - }) + expect(reduce(state, storeActions.remove("3"))).toEqual(state); + }); - it('should handle receive list', () => { - let a = {id: "1"}, b = {id: "2"}, - list = [a, b] + it("should handle receive list", () => { + let a = { id: "1" }, + b = { id: "2" }, + list = [a, b]; expect(reduce(undefined, storeActions.receive(list))).toEqual({ - byId: {"1": a, "2": b}, + byId: { "1": a, "2": b }, list: [a, b], - listIndex: {"1": 0, "2": 1}, + listIndex: { "1": 0, "2": 1 }, view: [a, b], - viewIndex: {"1": 0, "2": 1}, - }) - }) -}) + viewIndex: { "1": 0, "2": 1 }, + }); + }); +}); diff --git a/web/src/js/__tests__/flow/utilsSpec.tsx b/web/src/js/__tests__/flow/utilsSpec.tsx index e8b5b00052..271aa04936 100644 --- a/web/src/js/__tests__/flow/utilsSpec.tsx +++ b/web/src/js/__tests__/flow/utilsSpec.tsx @@ -1,83 +1,95 @@ -import * as utils from '../../flow/utils' -import {TFlow, TTCPFlow, TUDPFlow} from "../ducks/tutils"; -import {TDNSFlow, THTTPFlow} from "../ducks/_tflow"; -import {HTTPFlow} from "../../flow"; +import * as utils from "../../flow/utils"; +import { TFlow, TTCPFlow, TUDPFlow } from "../ducks/tutils"; +import { TDNSFlow, THTTPFlow } from "../ducks/_tflow"; +import { HTTPFlow } from "../../flow"; -describe('MessageUtils', () => { - it('should be possible to get first header', () => { +describe("MessageUtils", () => { + it("should be possible to get first header", () => { let tflow = TFlow(); - expect(utils.MessageUtils.get_first_header(tflow.request, /header/)).toEqual("qvalue") - expect(utils.MessageUtils.get_first_header(tflow.request, /123/)).toEqual(undefined) - }) + expect( + utils.MessageUtils.get_first_header(tflow.request, /header/) + ).toEqual("qvalue"); + expect( + utils.MessageUtils.get_first_header(tflow.request, /123/) + ).toEqual(undefined); + }); - it('should be possible to get Content-Type', () => { + it("should be possible to get Content-Type", () => { let tflow = TFlow(); tflow.request.headers = [["Content-Type", "text/html"]]; - expect(utils.MessageUtils.getContentType(tflow.request)).toEqual("text/html"); - }) + expect(utils.MessageUtils.getContentType(tflow.request)).toEqual( + "text/html" + ); + }); - it('should be possible to match header', () => { + it("should be possible to match header", () => { let h1 = ["foo", "bar"], - msg = {headers : [h1]} - expect(utils.MessageUtils.match_header(msg, /foo/i)).toEqual(h1) - expect(utils.MessageUtils.match_header(msg, /123/i)).toBeFalsy() - }) + msg = { headers: [h1] }; + expect(utils.MessageUtils.match_header(msg, /foo/i)).toEqual(h1); + expect(utils.MessageUtils.match_header(msg, /123/i)).toBeFalsy(); + }); - it('should be possible to get content URL', () => { + it("should be possible to get content URL", () => { const flow = TFlow(); // request let view = "bar"; - expect(utils.MessageUtils.getContentURL(flow, flow.request, view)).toEqual( + expect( + utils.MessageUtils.getContentURL(flow, flow.request, view) + ).toEqual( "./flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/request/content/bar.json" - ) - expect(utils.MessageUtils.getContentURL(flow, flow.request, '')).toEqual( + ); + expect( + utils.MessageUtils.getContentURL(flow, flow.request, "") + ).toEqual( "./flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/request/content.data" - ) + ); // response - expect(utils.MessageUtils.getContentURL(flow, flow.response, view)).toEqual( + expect( + utils.MessageUtils.getContentURL(flow, flow.response, view) + ).toEqual( "./flows/d91165be-ca1f-4612-88a9-c0f8696f3e29/response/content/bar.json" - ) - }) -}) + ); + }); +}); -describe('RequestUtils', () => { - it('should be possible prettify url', () => { +describe("RequestUtils", () => { + it("should be possible prettify url", () => { let flow = TFlow(); expect(utils.RequestUtils.pretty_url(flow.request)).toEqual( "http://address:22/path" - ) - }) -}) + ); + }); +}); -describe('parseUrl', () => { - it('should be possible to parse url', () => { - let url = "http://foo:4444/bar" +describe("parseUrl", () => { + it("should be possible to parse url", () => { + let url = "http://foo:4444/bar"; expect(utils.parseUrl(url)).toEqual({ port: 4444, - scheme: 'http', - host: 'foo', - path: '/bar' - }) + scheme: "http", + host: "foo", + path: "/bar", + }); - expect(utils.parseUrl("foo:foo")).toBeFalsy() - }) -}) + expect(utils.parseUrl("foo:foo")).toBeFalsy(); + }); +}); -describe('isValidHttpVersion', () => { - it('should be possible to validate http version', () => { - expect(utils.isValidHttpVersion("HTTP/1.1")).toBeTruthy() - expect(utils.isValidHttpVersion("HTTP//1")).toBeFalsy() - }) -}) +describe("isValidHttpVersion", () => { + it("should be possible to validate http version", () => { + expect(utils.isValidHttpVersion("HTTP/1.1")).toBeTruthy(); + expect(utils.isValidHttpVersion("HTTP//1")).toBeFalsy(); + }); +}); -it('should be possible to get a start time', () => { +it("should be possible to get a start time", () => { expect(utils.startTime(THTTPFlow())).toEqual(946681200); expect(utils.startTime(TTCPFlow())).toEqual(946681200); expect(utils.startTime(TUDPFlow())).toEqual(946681200); expect(utils.startTime(TDNSFlow())).toEqual(946681200); -}) +}); -it('should be possible to get an end time', () => { +it("should be possible to get an end time", () => { let f: HTTPFlow = THTTPFlow(); expect(utils.endTime(f)).toEqual(946681205); f.websocket = undefined; @@ -85,11 +97,11 @@ it('should be possible to get an end time', () => { expect(utils.endTime(TTCPFlow())).toEqual(946681205); expect(utils.endTime(TUDPFlow())).toEqual(946681204.5); expect(utils.endTime(TDNSFlow())).toEqual(946681201); -}) +}); -it('should be possible to get a total size', () => { +it("should be possible to get a total size", () => { expect(utils.getTotalSize(THTTPFlow())).toEqual(43); expect(utils.getTotalSize(TTCPFlow())).toEqual(12); expect(utils.getTotalSize(TUDPFlow())).toEqual(12); expect(utils.getTotalSize(TDNSFlow())).toEqual(8); -}) +}); diff --git a/web/src/js/__tests__/test-utils.tsx b/web/src/js/__tests__/test-utils.tsx index b479b089ef..fc5736fdc4 100644 --- a/web/src/js/__tests__/test-utils.tsx +++ b/web/src/js/__tests__/test-utils.tsx @@ -1,31 +1,24 @@ -import * as React from "react" -import {render as rtlRender} from '@testing-library/react' -import userEvent from '@testing-library/user-event' -import "@testing-library/jest-dom" -import {Provider} from 'react-redux' +import * as React from "react"; +import { render as rtlRender } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import "@testing-library/jest-dom"; +import { Provider } from "react-redux"; // Import your own reducer -import {createAppStore} from '../ducks' -import {testState} from "./ducks/tutils"; +import { createAppStore } from "../ducks"; +import { testState } from "./ducks/tutils"; // re-export everything -export { - waitFor, fireEvent, act, screen -} from '@testing-library/react' -export { - userEvent -} +export { waitFor, fireEvent, act, screen } from "@testing-library/react"; +export { userEvent }; export function render( ui, - { - store = createAppStore(testState), - ...renderOptions - } = {} + { store = createAppStore(testState), ...renderOptions } = {} ) { - function Wrapper({children}) { - return {children} + function Wrapper({ children }) { + return {children}; } - const ret = rtlRender(ui, {wrapper: Wrapper, ...renderOptions}) - return {...ret, store} + const ret = rtlRender(ui, { wrapper: Wrapper, ...renderOptions }); + return { ...ret, store }; } diff --git a/web/src/js/__tests__/urlStateSpec.tsx b/web/src/js/__tests__/urlStateSpec.tsx index 31a0d8c539..e530acc954 100644 --- a/web/src/js/__tests__/urlStateSpec.tsx +++ b/web/src/js/__tests__/urlStateSpec.tsx @@ -1,103 +1,114 @@ -import initialize from '../urlState' -import { updateStoreFromUrl, updateUrlFromStore } from '../urlState' +import initialize from "../urlState"; +import { updateStoreFromUrl, updateUrlFromStore } from "../urlState"; -import reduceFlows from '../ducks/flows' -import reduceUI from '../ducks/ui/index' -import reduceEventLog from '../ducks/eventLog' -import reduceCommandBar from '../ducks/commandBar' -import * as flowsActions from '../ducks/flows' +import reduceFlows from "../ducks/flows"; +import reduceUI from "../ducks/ui/index"; +import reduceEventLog from "../ducks/eventLog"; +import reduceCommandBar from "../ducks/commandBar"; +import * as flowsActions from "../ducks/flows"; -import configureStore from 'redux-mock-store' +import configureStore from "redux-mock-store"; -const mockStore = configureStore() -history.replaceState = jest.fn() +const mockStore = configureStore(); +history.replaceState = jest.fn(); -describe('updateStoreFromUrl', () => { - - it('should handle search query', () => { - window.location.hash = "#/flows?s=foo" - let store = mockStore() - updateStoreFromUrl(store) - expect(store.getActions()).toEqual([{ filter: "foo", type: "FLOWS_SET_FILTER" }]) - }) +describe("updateStoreFromUrl", () => { + it("should handle search query", () => { + window.location.hash = "#/flows?s=foo"; + let store = mockStore(); + updateStoreFromUrl(store); + expect(store.getActions()).toEqual([ + { filter: "foo", type: "FLOWS_SET_FILTER" }, + ]); + }); - it('should handle highlight query', () => { - window.location.hash = "#/flows?h=foo" - let store = mockStore() - updateStoreFromUrl(store) - expect(store.getActions()).toEqual([{ highlight: "foo", type: "FLOWS_SET_HIGHLIGHT" }]) - }) + it("should handle highlight query", () => { + window.location.hash = "#/flows?h=foo"; + let store = mockStore(); + updateStoreFromUrl(store); + expect(store.getActions()).toEqual([ + { highlight: "foo", type: "FLOWS_SET_HIGHLIGHT" }, + ]); + }); - it('should handle show event log', () => { - window.location.hash = "#/flows?e=true" + it("should handle show event log", () => { + window.location.hash = "#/flows?e=true"; let initialState = { eventLog: reduceEventLog(undefined, {}) }, - store = mockStore(initialState) - updateStoreFromUrl(store) - expect(store.getActions()).toEqual([{ type: "EVENTS_TOGGLE_VISIBILITY" }]) - }) + store = mockStore(initialState); + updateStoreFromUrl(store); + expect(store.getActions()).toEqual([ + { type: "EVENTS_TOGGLE_VISIBILITY" }, + ]); + }); - it('should handle unimplemented query argument', () => { - window.location.hash = "#/flows?foo=bar" - console.error = jest.fn() - let store = mockStore() - updateStoreFromUrl(store) - expect(console.error).toBeCalledWith("unimplemented query arg: foo=bar") - }) + it("should handle unimplemented query argument", () => { + window.location.hash = "#/flows?foo=bar"; + console.error = jest.fn(); + let store = mockStore(); + updateStoreFromUrl(store); + expect(console.error).toBeCalledWith( + "unimplemented query arg: foo=bar" + ); + }); - it('should select flow and tab', () => { - window.location.hash = "#/flows/123/request" - let store = mockStore() - updateStoreFromUrl(store) + it("should select flow and tab", () => { + window.location.hash = "#/flows/123/request"; + let store = mockStore(); + updateStoreFromUrl(store); expect(store.getActions()).toEqual([ { flowIds: ["123"], - type: "FLOWS_SELECT" + type: "FLOWS_SELECT", }, { tab: "request", - type: "UI_FLOWVIEW_SET_TAB" - } - ]) - }) -}) + type: "UI_FLOWVIEW_SET_TAB", + }, + ]); + }); +}); -describe('updateUrlFromStore', () => { +describe("updateUrlFromStore", () => { let initialState = { - flows: reduceFlows(undefined, {type: "other"}), - ui: reduceUI(undefined, {type: "other"}), - eventLog: reduceEventLog(undefined, {type: "other"}), - commandBar: reduceCommandBar(undefined, {type: "other"}), - } + flows: reduceFlows(undefined, { type: "other" }), + ui: reduceUI(undefined, { type: "other" }), + eventLog: reduceEventLog(undefined, { type: "other" }), + commandBar: reduceCommandBar(undefined, { type: "other" }), + }; - it('should update initial url', () => { - let store = mockStore(initialState) - updateUrlFromStore(store) - expect(history.replaceState).toBeCalledWith(undefined, '', '/#/flows') - }) + it("should update initial url", () => { + let store = mockStore(initialState); + updateUrlFromStore(store); + expect(history.replaceState).toBeCalledWith(undefined, "", "/#/flows"); + }); - it('should update url', () => { + it("should update url", () => { let flows = reduceFlows(undefined, flowsActions.select("123")), state = { ...initialState, - flows: reduceFlows(flows, flowsActions.setFilter('~u foo')) + flows: reduceFlows(flows, flowsActions.setFilter("~u foo")), }, - store = mockStore(state) - updateUrlFromStore(store) - expect(history.replaceState).toBeCalledWith(undefined, '', '/#/flows/123/request?s=~u%20foo') - }) -}) + store = mockStore(state); + updateUrlFromStore(store); + expect(history.replaceState).toBeCalledWith( + undefined, + "", + "/#/flows/123/request?s=~u%20foo" + ); + }); +}); -describe('initialize', () => { +describe("initialize", () => { let initialState = { - flows: reduceFlows(undefined, {type: "other"}), - ui: reduceUI(undefined, {type: "other"}), - eventLog: reduceEventLog(undefined, {type: "other"}), - commandBar: reduceCommandBar(undefined, {type: "other"}), - } + flows: reduceFlows(undefined, { type: "other" }), + ui: reduceUI(undefined, { type: "other" }), + eventLog: reduceEventLog(undefined, { type: "other" }), + commandBar: reduceCommandBar(undefined, { type: "other" }), + }; - it('should handle initial state', () => { - let store = mockStore(initialState) - initialize(store) - store.dispatch({ type: "foo" }) - }) -}) + it("should handle initial state", () => { + let store = mockStore(initialState); + initialize(store); + store.dispatch({ type: "foo" }); + }); +}); diff --git a/web/src/js/__tests__/utilsSpec.tsx b/web/src/js/__tests__/utilsSpec.tsx index 124954af32..b91fa84114 100644 --- a/web/src/js/__tests__/utilsSpec.tsx +++ b/web/src/js/__tests__/utilsSpec.tsx @@ -1,85 +1,87 @@ -import * as utils from '../utils' -import {enableFetchMocks} from "jest-fetch-mock"; +import * as utils from "../utils"; +import { enableFetchMocks } from "jest-fetch-mock"; enableFetchMocks(); -describe('formatSize', () => { - it('should return 0 when 0 byte', () => { - expect(utils.formatSize(0)).toEqual('0') - }) +describe("formatSize", () => { + it("should return 0 when 0 byte", () => { + expect(utils.formatSize(0)).toEqual("0"); + }); - it('should return formatted size', () => { - expect(utils.formatSize(27104011)).toEqual("25.8mb") - expect(utils.formatSize(1023)).toEqual("1023b") - }) -}) + it("should return formatted size", () => { + expect(utils.formatSize(27104011)).toEqual("25.8mb"); + expect(utils.formatSize(1023)).toEqual("1023b"); + }); +}); -describe('formatTimeDelta', () => { - it('should return formatted time', () => { - expect(utils.formatTimeDelta(3600100)).toEqual("1h") - }) -}) +describe("formatTimeDelta", () => { + it("should return formatted time", () => { + expect(utils.formatTimeDelta(3600100)).toEqual("1h"); + }); +}); -describe('formatTimeStamp', () => { - it('should return formatted time', () => { - expect(utils.formatTimeStamp(1483228800, {milliseconds: false})).toEqual("2017-01-01 00:00:00") - expect(utils.formatTimeStamp(1483228800, {milliseconds: true})).toEqual("2017-01-01 00:00:00.000") - }) -}) +describe("formatTimeStamp", () => { + it("should return formatted time", () => { + expect( + utils.formatTimeStamp(1483228800, { milliseconds: false }) + ).toEqual("2017-01-01 00:00:00"); + expect( + utils.formatTimeStamp(1483228800, { milliseconds: true }) + ).toEqual("2017-01-01 00:00:00.000"); + }); +}); -describe('formatAddress', () => { - it('should return formatted addresses', () => { - expect(utils.formatAddress(["127.0.0.1", 8080])).toEqual("127.0.0.1:8080"); +describe("formatAddress", () => { + it("should return formatted addresses", () => { + expect(utils.formatAddress(["127.0.0.1", 8080])).toEqual( + "127.0.0.1:8080" + ); expect(utils.formatAddress(["::1", 8080])).toEqual("[::1]:8080"); - }) -}) + }); +}); -describe('reverseString', () => { - it('should return reversed string', () => { - let str1 = "abc", str2 = "xyz" - expect(utils.reverseString(str1) > utils.reverseString(str2)).toBeTruthy() - }) -}) +describe("reverseString", () => { + it("should return reversed string", () => { + let str1 = "abc", + str2 = "xyz"; + expect( + utils.reverseString(str1) > utils.reverseString(str2) + ).toBeTruthy(); + }); +}); -describe('fetchApi', () => { - it('should handle fetch operation', () => { - utils.fetchApi('http://foo/bar', {method: "POST"}) - expect(fetchMock.mock.calls[0][0]).toEqual( - "http://foo/bar" - ) - fetchMock.mockClear() +describe("fetchApi", () => { + it("should handle fetch operation", () => { + utils.fetchApi("http://foo/bar", { method: "POST" }); + expect(fetchMock.mock.calls[0][0]).toEqual("http://foo/bar"); + fetchMock.mockClear(); - utils.fetchApi('http://foo?bar=1', {method: "POST"}) - expect(fetchMock.mock.calls[0][0]).toEqual( - "http://foo?bar=1" - ) + utils.fetchApi("http://foo?bar=1", { method: "POST" }); + expect(fetchMock.mock.calls[0][0]).toEqual("http://foo?bar=1"); + }); - }) - - it('should be possible to do put request', () => { - fetchMock.mockClear() - utils.fetchApi.put("http://foo", [1, 2, 3], {}) - expect(fetchMock.mock.calls[0]).toEqual( - [ - "http://foo", - { - body: "[1,2,3]", - credentials: "same-origin", - headers: { - "Content-Type": "application/json", - "X-XSRFToken": undefined, - }, - method: "PUT" + it("should be possible to do put request", () => { + fetchMock.mockClear(); + utils.fetchApi.put("http://foo", [1, 2, 3], {}); + expect(fetchMock.mock.calls[0]).toEqual([ + "http://foo", + { + body: "[1,2,3]", + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + "X-XSRFToken": undefined, }, - ] - ) - }) -}) + method: "PUT", + }, + ]); + }); +}); -describe('getDiff', () => { - it('should return json object including only the changed keys value pairs', () => { - let obj1 = {a: 1, b: {foo: 1}, c: [3]}, - obj2 = {a: 1, b: {foo: 2}, c: [4]} - expect(utils.getDiff(obj1, obj2)).toEqual({b: {foo: 2}, c: [4]}) - }) -}) +describe("getDiff", () => { + it("should return json object including only the changed keys value pairs", () => { + let obj1 = { a: 1, b: { foo: 1 }, c: [3] }, + obj2 = { a: 1, b: { foo: 2 }, c: [4] }; + expect(utils.getDiff(obj1, obj2)).toEqual({ b: { foo: 2 }, c: [4] }); + }); +}); diff --git a/web/src/js/app.tsx b/web/src/js/app.tsx index 69fd66788a..ae1e4697f6 100644 --- a/web/src/js/app.tsx +++ b/web/src/js/app.tsx @@ -1,34 +1,33 @@ -import * as React from "react" -import {render} from 'react-dom' -import {Provider} from 'react-redux' +import * as React from "react"; +import { render } from "react-dom"; +import { Provider } from "react-redux"; -import ProxyApp from './components/ProxyApp' -import {add as addLog} from './ducks/eventLog' -import useUrlState from './urlState' -import WebSocketBackend from './backends/websocket' -import StaticBackend from './backends/static' -import {store} from "./ducks"; +import ProxyApp from "./components/ProxyApp"; +import { add as addLog } from "./ducks/eventLog"; +import useUrlState from "./urlState"; +import WebSocketBackend from "./backends/websocket"; +import StaticBackend from "./backends/static"; +import { store } from "./ducks"; - -useUrlState(store) +useUrlState(store); // @ts-ignore if (window.MITMWEB_STATIC) { // @ts-ignore - window.backend = new StaticBackend(store) + window.backend = new StaticBackend(store); } else { // @ts-ignore - window.backend = new WebSocketBackend(store) + window.backend = new WebSocketBackend(store); } -window.addEventListener('error', (e: ErrorEvent) => { - store.dispatch(addLog(`${e.message}\n${e.error.stack}`)) -}) +window.addEventListener("error", (e: ErrorEvent) => { + store.dispatch(addLog(`${e.message}\n${e.error.stack}`)); +}); -document.addEventListener('DOMContentLoaded', () => { +document.addEventListener("DOMContentLoaded", () => { render( - + , document.getElementById("mitmproxy") - ) -}) + ); +}); diff --git a/web/src/js/backends/static.tsx b/web/src/js/backends/static.tsx index 79d537ba77..bb40bc43c5 100644 --- a/web/src/js/backends/static.tsx +++ b/web/src/js/backends/static.tsx @@ -2,36 +2,34 @@ * This backend uses the REST API only to host static instances, * without any Websocket connection. */ -import {fetchApi} from "../utils" -import {Store} from "redux"; -import {RootState} from "../ducks"; +import { fetchApi } from "../utils"; +import { Store } from "redux"; +import { RootState } from "../ducks"; export default class StaticBackend { - - store: Store + store: Store; constructor(store) { - this.store = store - this.onOpen() + this.store = store; + this.onOpen(); } onOpen() { - this.fetchData("flows") - this.fetchData("options") + this.fetchData("flows"); + this.fetchData("options"); // this.fetchData("events") # TODO: Add events log to static viewer. } fetchData(resource) { fetchApi(`./${resource}`) - .then(res => res.json()) - .then(json => { - this.receive(resource, json) - }) + .then((res) => res.json()) + .then((json) => { + this.receive(resource, json); + }); } receive(resource, data) { - let type = `${resource}_RECEIVE`.toUpperCase() - this.store.dispatch({type, cmd: "receive", resource, data}) + let type = `${resource}_RECEIVE`.toUpperCase(); + this.store.dispatch({ type, cmd: "receive", resource, data }); } - } diff --git a/web/src/js/backends/websocket.tsx b/web/src/js/backends/websocket.tsx index 64cf284841..c38362dede 100644 --- a/web/src/js/backends/websocket.tsx +++ b/web/src/js/backends/websocket.tsx @@ -3,92 +3,98 @@ * from the REST API and live updates delivered via a WebSocket connection. * An alternative backend may use the REST API only to host static instances. */ -import {fetchApi} from "../utils" -import * as connectionActions from "../ducks/connection" -import {Store} from "redux"; -import {RootState} from "../ducks"; +import { fetchApi } from "../utils"; +import * as connectionActions from "../ducks/connection"; +import { Store } from "redux"; +import { RootState } from "../ducks"; -const CMD_RESET = 'reset' +const CMD_RESET = "reset"; export default class WebsocketBackend { - activeFetches: { - flows?: [] - events?: [] - options?: [] - } - store: Store - socket: WebSocket + flows?: []; + events?: []; + options?: []; + }; + store: Store; + socket: WebSocket; constructor(store) { - this.activeFetches = {} - this.store = store - this.connect() + this.activeFetches = {}; + this.store = store; + this.connect(); } connect() { - this.socket = new WebSocket(location.origin.replace('http', 'ws') + '/updates') - this.socket.addEventListener('open', () => this.onOpen()) - this.socket.addEventListener('close', event => this.onClose(event)) - this.socket.addEventListener('message', msg => this.onMessage(JSON.parse(msg.data))) - this.socket.addEventListener('error', error => this.onError(error)) + this.socket = new WebSocket( + location.origin.replace("http", "ws") + "/updates" + ); + this.socket.addEventListener("open", () => this.onOpen()); + this.socket.addEventListener("close", (event) => this.onClose(event)); + this.socket.addEventListener("message", (msg) => + this.onMessage(JSON.parse(msg.data)) + ); + this.socket.addEventListener("error", (error) => this.onError(error)); } onOpen() { - this.fetchData("state") - this.fetchData("flows") - this.fetchData("events") - this.fetchData("options") - this.store.dispatch(connectionActions.startFetching()) + this.fetchData("state"); + this.fetchData("flows"); + this.fetchData("events"); + this.fetchData("options"); + this.store.dispatch(connectionActions.startFetching()); } fetchData(resource) { - let queue = [] - this.activeFetches[resource] = queue + let queue = []; + this.activeFetches[resource] = queue; fetchApi(`./${resource}`) - .then(res => res.json()) - .then(json => { + .then((res) => res.json()) + .then((json) => { // Make sure that we are not superseded yet by the server sending a RESET. if (this.activeFetches[resource] === queue) - this.receive(resource, json) - }) + this.receive(resource, json); + }); } onMessage(msg) { - if (msg.cmd === CMD_RESET) { - return this.fetchData(msg.resource) + return this.fetchData(msg.resource); } if (msg.resource in this.activeFetches) { - this.activeFetches[msg.resource].push(msg) + this.activeFetches[msg.resource].push(msg); } else { - let type = `${msg.resource}_${msg.cmd}`.toUpperCase() - this.store.dispatch({type, ...msg}) + let type = `${msg.resource}_${msg.cmd}`.toUpperCase(); + this.store.dispatch({ type, ...msg }); } } receive(resource, data) { - let type = `${resource}_RECEIVE`.toUpperCase() - this.store.dispatch({type, cmd: "receive", resource, data}) - let queue = this.activeFetches[resource] - delete this.activeFetches[resource] - queue.forEach(msg => this.onMessage(msg)) + let type = `${resource}_RECEIVE`.toUpperCase(); + this.store.dispatch({ type, cmd: "receive", resource, data }); + let queue = this.activeFetches[resource]; + delete this.activeFetches[resource]; + queue.forEach((msg) => this.onMessage(msg)); if (Object.keys(this.activeFetches).length === 0) { // We have fetched the last resource - this.store.dispatch(connectionActions.connectionEstablished()) + this.store.dispatch(connectionActions.connectionEstablished()); } } onClose(closeEvent) { - this.store.dispatch(connectionActions.connectionError( - `Connection closed at ${new Date().toUTCString()} with error code ${closeEvent.code}.` - )) - console.error("websocket connection closed", closeEvent) + this.store.dispatch( + connectionActions.connectionError( + `Connection closed at ${new Date().toUTCString()} with error code ${ + closeEvent.code + }.` + ) + ); + console.error("websocket connection closed", closeEvent); } onError(error) { // FIXME - console.error("websocket connection errored", arguments) + console.error("websocket connection errored", arguments); } } diff --git a/web/src/js/components/CaptureSetup.tsx b/web/src/js/components/CaptureSetup.tsx index 63e7a488e3..a13681919e 100644 --- a/web/src/js/components/CaptureSetup.tsx +++ b/web/src/js/components/CaptureSetup.tsx @@ -1,61 +1,74 @@ import * as React from "react"; -import {useEffect, useRef} from "react"; -import {useAppSelector} from "../ducks"; -import {ServerInfo} from "../ducks/backendState"; -import {formatAddress} from "../utils"; -import QRCode from 'qrcode'; +import { useEffect, useRef } from "react"; +import { useAppSelector } from "../ducks"; +import { ServerInfo } from "../ducks/backendState"; +import { formatAddress } from "../utils"; +import QRCode from "qrcode"; export default function CaptureSetup() { - const servers = useAppSelector(state => state.backendState.servers); + const servers = useAppSelector((state) => state.backendState.servers); let configure_action_text; if (servers.length === 0) { configure_action_text = ""; } else if (servers.length === 1) { - configure_action_text = "Configure your client to use the following proxy server:"; + configure_action_text = + "Configure your client to use the following proxy server:"; } else { - configure_action_text = "Configure your client to use one of the following proxy servers:"; + configure_action_text = + "Configure your client to use one of the following proxy servers:"; } - return
    - -

    mitmproxy is running.

    -

    - No flows have been recorded yet.
    - {configure_action_text} -

    -
      - {servers.map((server, i) =>
      • )} -
    - {/* + return ( +
    +

    mitmproxy is running.

    +

    + No flows have been recorded yet. +
    + {configure_action_text} +

    +
      + {servers.map((server, i) => ( +
    • + +
      • + ))} +
    + {/*

    You can also start additional servers:

    • TODO
    */} - -
    +
    + ); } -export function ServerDescription( - { - description, - listen_addrs, - last_exception, - is_running, - full_spec, - wireguard_conf, - }: ServerInfo -) { +export function ServerDescription({ + description, + listen_addrs, + last_exception, + is_running, + full_spec, + wireguard_conf, +}: ServerInfo) { const qrCode = useRef(null); useEffect(() => { if (wireguard_conf && qrCode.current) - QRCode.toCanvas(qrCode.current, wireguard_conf, {margin: 0, scale: 3}); + QRCode.toCanvas(qrCode.current, wireguard_conf, { + margin: 0, + scale: 3, + }); }, [wireguard_conf]); let listen_str; - const all_same_port = listen_addrs.length === 1 || (listen_addrs.length === 2 && listen_addrs[0][1] === listen_addrs[1][1]); - const unbound = listen_addrs.every(addr => ["::", "0.0.0.0"].includes(addr[0])); + const all_same_port = + listen_addrs.length === 1 || + (listen_addrs.length === 2 && + listen_addrs[0][1] === listen_addrs[1][1]); + const unbound = listen_addrs.every((addr) => + ["::", "0.0.0.0"].includes(addr[0]) + ); if (all_same_port && unbound) { listen_str = formatAddress(["*", listen_addrs[0][1]]); } else { @@ -64,25 +77,41 @@ export function ServerDescription( description = description[0].toUpperCase() + description.substr(1); let desc, icon; if (last_exception) { - icon = "fa-exclamation text-error" - desc = <>{description} ({full_spec}):
    {last_exception}; + icon = "fa-exclamation text-error"; + desc = ( + <> + {description} ({full_spec}): +
    + {last_exception} + + ); } else if (!is_running) { - icon = "fa-pause text-warning" - desc = <>{description} ({full_spec}) + icon = "fa-pause text-warning"; + desc = ( + <> + {description} ({full_spec}) + + ); } else { - icon = "fa-check text-success" - desc = `${description} listening at ${listen_str}.` + icon = "fa-check text-success"; + desc = `${description} listening at ${listen_str}.`; if (wireguard_conf) { - desc = <> - {desc} -
    - 
    {wireguard_conf}
    - -
    - ; + desc = ( + <> + {desc} +
    + 
    {wireguard_conf}
    + +
    + + ); } - } - return <>{desc}; + return ( + <> + + {desc} + + ); } diff --git a/web/src/js/components/CommandBar.tsx b/web/src/js/components/CommandBar.tsx index e6e3cd9019..42f3ef915a 100644 --- a/web/src/js/components/CommandBar.tsx +++ b/web/src/js/components/CommandBar.tsx @@ -1,154 +1,196 @@ -import React, {useEffect, useRef, useState} from 'react' -import classnames from 'classnames' -import {fetchApi, runCommand} from '../utils' -import Filt from '../filt/command' +import React, { useEffect, useRef, useState } from "react"; +import classnames from "classnames"; +import { fetchApi, runCommand } from "../utils"; +import Filt from "../filt/command"; type CommandParameter = { - name: string - type: string - kind: string -} + name: string; + type: string; + kind: string; +}; type Command = { - help?: string - parameters: CommandParameter[] - return_type: string | undefined - signature_help: string -} + help?: string; + parameters: CommandParameter[]; + return_type: string | undefined; + signature_help: string; +}; type AllCommands = { - [name: string]: Command -} + [name: string]: Command; +}; type CommandHelpProps = { - nextArgs: string[], - currentArg: number, - help: string, - description: string, - availableCommands: string[], -} + nextArgs: string[]; + currentArg: number; + help: string; + description: string; + availableCommands: string[]; +}; type CommandResult = { - command: string, - result: string, -} + command: string; + result: string; +}; type ResultProps = { - results: CommandResult[], -} + results: CommandResult[]; +}; -function getAvailableCommands(commands: AllCommands, input: string = ""): string[] { - if (!commands) return [] - let availableCommands: string[] = [] +function getAvailableCommands( + commands: AllCommands, + input: string = "" +): string[] { + if (!commands) return []; + let availableCommands: string[] = []; for (const [command, args] of Object.entries(commands)) { if (command.startsWith(input)) { - availableCommands.push(command) + availableCommands.push(command); } } - return availableCommands + return availableCommands; } -export function Results({results}: ResultProps) { +export function Results({ results }: ResultProps) { const resultElement = useRef(null!); useEffect(() => { if (resultElement) { - resultElement.current.addEventListener('DOMNodeInserted', (event) => { - const target = event.currentTarget as Element; - target.scroll({top: target.scrollHeight, behavior: 'auto'}); - }); + resultElement.current.addEventListener( + "DOMNodeInserted", + (event) => { + const target = event.currentTarget as Element; + target.scroll({ + top: target.scrollHeight, + behavior: "auto", + }); + } + ); } - }, []) + }, []); return (
    {results.map((result, i) => (
    -
    $ {result.command}
    +
    + $ {result.command} +
    {result.result}
    ))}
    - ) + ); } -export function CommandHelp({nextArgs, currentArg, help, description, availableCommands}: CommandHelpProps) { - let argumentSuggestion: JSX.Element[] = [] +export function CommandHelp({ + nextArgs, + currentArg, + help, + description, + availableCommands, +}: CommandHelpProps) { + let argumentSuggestion: JSX.Element[] = []; for (let i: number = 0; i < nextArgs.length; i++) { if (i == currentArg) { - argumentSuggestion.push({nextArgs[i]}) - continue + argumentSuggestion.push({nextArgs[i]}); + continue; } - argumentSuggestion.push({nextArgs[i]} ) + argumentSuggestion.push({nextArgs[i]} ); } - return (
    -
    -
    - {argumentSuggestion.length > 0 &&
    Argument suggestion: {argumentSuggestion}
    } - {help?.includes("->") &&
    Signature help: {help}
    } - {description &&
    # {description}
    } -
    Available Commands:

    {JSON.stringify(availableCommands)}

    + return ( +
    +
    +
    + {argumentSuggestion.length > 0 && ( +
    + Argument suggestion:{" "} + {argumentSuggestion} +
    + )} + {help?.includes("->") && ( +
    + Signature help: + {help} +
    + )} + {description &&
    # {description}
    } +
    + Available Commands: +

    + {JSON.stringify(availableCommands)} +

    +
    +
    -
    ) + ); } export default function CommandBar() { - const [input, setInput] = useState("") - const [originalInput, setOriginalInput] = useState("") - const [currentCompletion, setCurrentCompletion] = useState(0) - const [completionCandidate, setCompletionCandidate] = useState([]) - - const [availableCommands, setAvailableCommands] = useState([]) - const [allCommands, setAllCommands] = useState({}) - const [nextArgs, setNextArgs] = useState([]) - const [currentArg, setCurrentArg] = useState(0) - const [signatureHelp, setSignatureHelp] = useState("") - const [description, setDescription] = useState("") - - const [results, setResults] = useState([]) - const [history, setHistory] = useState([]) + const [input, setInput] = useState(""); + const [originalInput, setOriginalInput] = useState(""); + const [currentCompletion, setCurrentCompletion] = useState(0); + const [completionCandidate, setCompletionCandidate] = useState( + [] + ); + + const [availableCommands, setAvailableCommands] = useState([]); + const [allCommands, setAllCommands] = useState({}); + const [nextArgs, setNextArgs] = useState([]); + const [currentArg, setCurrentArg] = useState(0); + const [signatureHelp, setSignatureHelp] = useState(""); + const [description, setDescription] = useState(""); + + const [results, setResults] = useState([]); + const [history, setHistory] = useState([]); const [currentPos, setCurrentPos] = useState(undefined); useEffect(() => { - fetchApi('/commands', {method: 'GET'}) - .then(response => response.json()) + fetchApi("/commands", { method: "GET" }) + .then((response) => response.json()) .then((data: AllCommands) => { - setAllCommands(data) - setCompletionCandidate(getAvailableCommands(data)) - setAvailableCommands(Object.keys(data)) - }).catch(e => console.error(e)) - }, []) + setAllCommands(data); + setCompletionCandidate(getAvailableCommands(data)); + setAvailableCommands(Object.keys(data)); + }) + .catch((e) => console.error(e)); + }, []); useEffect(() => { - runCommand("commands.history.get").then((ret) => { - setHistory(ret.value); - }).catch(e => console.error(e)) - }, []) + runCommand("commands.history.get") + .then((ret) => { + setHistory(ret.value); + }) + .catch((e) => console.error(e)); + }, []); const parseCommand = (originalInput: string, input: string) => { - const parts: string[] = Filt.parse(input) - const originalParts: string[] = Filt.parse(originalInput) + const parts: string[] = Filt.parse(input); + const originalParts: string[] = Filt.parse(originalInput); - setSignatureHelp(allCommands[parts[0]]?.signature_help) - setDescription(allCommands[parts[0]]?.help || "") + setSignatureHelp(allCommands[parts[0]]?.signature_help); + setDescription(allCommands[parts[0]]?.help || ""); - setCompletionCandidate(getAvailableCommands(allCommands, originalParts[0])) - setAvailableCommands(getAvailableCommands(allCommands, parts[0])) + setCompletionCandidate( + getAvailableCommands(allCommands, originalParts[0]) + ); + setAvailableCommands(getAvailableCommands(allCommands, parts[0])); - const nextArgs: string[] = allCommands[parts[0]]?.parameters.map(p => p.name) + const nextArgs: string[] = allCommands[parts[0]]?.parameters.map( + (p) => p.name + ); if (nextArgs) { - setNextArgs([parts[0], ...nextArgs]) - setCurrentArg(parts.length - 1) + setNextArgs([parts[0], ...nextArgs]); + setCurrentArg(parts.length - 1); } - } + }; const onChange = (e) => { - setInput(e.target.value) - setOriginalInput(e.target.value) - setCurrentCompletion(0) - } + setInput(e.target.value); + setOriginalInput(e.target.value); + setCurrentCompletion(0); + }; const onKeyDown = (e) => { if (e.key === "Enter") { @@ -157,32 +199,40 @@ export default function CommandBar() { setHistory([...history, input]); runCommand("commands.history.add", input).catch(() => 0); - fetchApi.post(`/commands/${cmd}`, {arguments: args}) - .then(response => response.json()) - .then(data => { - setCurrentPos(undefined) - setNextArgs([]) - setResults([...results, { - command: input, - result: JSON.stringify(data.value || data.error) - }]) - }).catch(e => { - setCurrentPos(undefined) - setNextArgs([]) - setResults([...results, { - command: input, - result: e.toString() - }]); - }) + fetchApi + .post(`/commands/${cmd}`, { arguments: args }) + .then((response) => response.json()) + .then((data) => { + setCurrentPos(undefined); + setNextArgs([]); + setResults([ + ...results, + { + command: input, + result: JSON.stringify(data.value || data.error), + }, + ]); + }) + .catch((e) => { + setCurrentPos(undefined); + setNextArgs([]); + setResults([ + ...results, + { + command: input, + result: e.toString(), + }, + ]); + }); - setSignatureHelp("") - setDescription("") + setSignatureHelp(""); + setDescription(""); - setInput("") - setOriginalInput("") + setInput(""); + setOriginalInput(""); - setCurrentCompletion(0) - setCompletionCandidate(availableCommands) + setCurrentCompletion(0); + setCompletionCandidate(availableCommands); } if (e.key === "ArrowUp") { let nextPos; @@ -191,52 +241,57 @@ export default function CommandBar() { } else { nextPos = Math.max(0, currentPos - 1); } - setInput(history[nextPos]) - setOriginalInput(history[nextPos]) - setCurrentPos(nextPos) + setInput(history[nextPos]); + setOriginalInput(history[nextPos]); + setCurrentPos(nextPos); } if (e.key === "ArrowDown") { if (currentPos === undefined) { - return + return; } else if (currentPos == history.length - 1) { setInput(""); setOriginalInput(""); setCurrentPos(undefined); } else { const nextPos = currentPos + 1; - setInput(history[nextPos]) - setOriginalInput(history[nextPos]) - setCurrentPos(nextPos) + setInput(history[nextPos]); + setOriginalInput(history[nextPos]); + setCurrentPos(nextPos); } } if (e.key === "Tab") { - setInput(completionCandidate[currentCompletion]) - setCurrentCompletion((currentCompletion + 1) % completionCandidate.length) - e.preventDefault() + setInput(completionCandidate[currentCompletion]); + setCurrentCompletion( + (currentCompletion + 1) % completionCandidate.length + ); + e.preventDefault(); } - e.stopPropagation() - } + e.stopPropagation(); + }; const onKeyUp = (e) => { if (!input) { - setAvailableCommands(Object.keys(allCommands)) - return + setAvailableCommands(Object.keys(allCommands)); + return; } - parseCommand(originalInput, input) - e.stopPropagation() - } + parseCommand(originalInput, input); + e.stopPropagation(); + }; return (
    -
    - Command Result -
    - - -
    +
    Command Result
    + + +
    - +
    - ) + ); } diff --git a/web/src/js/components/EventLog.jsx b/web/src/js/components/EventLog.jsx index 40fe900ed3..c07cff9bcc 100644 --- a/web/src/js/components/EventLog.jsx +++ b/web/src/js/components/EventLog.jsx @@ -1,75 +1,81 @@ -import React, { Component } from 'react' -import PropTypes from 'prop-types' -import { connect } from 'react-redux' -import { toggleFilter, toggleVisibility } from '../ducks/eventLog' -import ToggleButton from './common/ToggleButton' -import EventList from './EventLog/EventList' +import React, { Component } from "react"; +import PropTypes from "prop-types"; +import { connect } from "react-redux"; +import { toggleFilter, toggleVisibility } from "../ducks/eventLog"; +import ToggleButton from "./common/ToggleButton"; +import EventList from "./EventLog/EventList"; export class PureEventLog extends Component { - static propTypes = { filters: PropTypes.object.isRequired, events: PropTypes.array.isRequired, toggleFilter: PropTypes.func.isRequired, close: PropTypes.func.isRequired, defaultHeight: PropTypes.number, - } + }; static defaultProps = { defaultHeight: 200, - } + }; constructor(props, context) { - super(props, context) + super(props, context); - this.state = { height: this.props.defaultHeight } + this.state = { height: this.props.defaultHeight }; - this.onDragStart = this.onDragStart.bind(this) - this.onDragMove = this.onDragMove.bind(this) - this.onDragStop = this.onDragStop.bind(this) + this.onDragStart = this.onDragStart.bind(this); + this.onDragMove = this.onDragMove.bind(this); + this.onDragStop = this.onDragStop.bind(this); } onDragStart(event) { - event.preventDefault() - this.dragStart = this.state.height + event.pageY - window.addEventListener('mousemove', this.onDragMove) - window.addEventListener('mouseup', this.onDragStop) - window.addEventListener('dragend', this.onDragStop) + event.preventDefault(); + this.dragStart = this.state.height + event.pageY; + window.addEventListener("mousemove", this.onDragMove); + window.addEventListener("mouseup", this.onDragStop); + window.addEventListener("dragend", this.onDragStop); } onDragMove(event) { - event.preventDefault() - this.setState({ height: this.dragStart - event.pageY }) + event.preventDefault(); + this.setState({ height: this.dragStart - event.pageY }); } onDragStop(event) { - event.preventDefault() - window.removeEventListener('mousemove', this.onDragMove) + event.preventDefault(); + window.removeEventListener("mousemove", this.onDragMove); } render() { - const { height } = this.state - const { filters, events, toggleFilter, close } = this.props + const { height } = this.state; + const { filters, events, toggleFilter, close } = this.props; return (
    Eventlog
    - {['debug', 'info', 'web', 'warn', 'error'].map(type => ( - toggleFilter(type)}/> - ))} + {["debug", "info", "web", "warn", "error"].map( + (type) => ( + toggleFilter(type)} + /> + ) + )}
    - ) + ); } } export default connect( - state => ({ + (state) => ({ filters: state.eventLog.filters, events: state.eventLog.view, }), @@ -77,4 +83,4 @@ export default connect( close: toggleVisibility, toggleFilter: toggleFilter, } -)(PureEventLog) +)(PureEventLog); diff --git a/web/src/js/components/EventLog/EventList.tsx b/web/src/js/components/EventLog/EventList.tsx index 2b25a31aee..f52f656835 100644 --- a/web/src/js/components/EventLog/EventList.tsx +++ b/web/src/js/components/EventLog/EventList.tsx @@ -1,108 +1,112 @@ -import React, { Component } from 'react' -import PropTypes from 'prop-types' -import ReactDOM from 'react-dom' -import shallowEqual from 'shallowequal' -import AutoScroll from '../helpers/AutoScroll' -import {calcVScroll, VScroll} from '../helpers/VirtualScroll' -import {EventLogItem} from "../../ducks/eventLog"; - +import React, { Component } from "react"; +import PropTypes from "prop-types"; +import ReactDOM from "react-dom"; +import shallowEqual from "shallowequal"; +import AutoScroll from "../helpers/AutoScroll"; +import { calcVScroll, VScroll } from "../helpers/VirtualScroll"; +import { EventLogItem } from "../../ducks/eventLog"; type EventLogListProps = { - events: EventLogItem[] - rowHeight: number -} + events: EventLogItem[]; + rowHeight: number; +}; type EventLogListState = { - vScroll: VScroll -} + vScroll: VScroll; +}; class EventLogList extends Component { - static propTypes = { events: PropTypes.array.isRequired, rowHeight: PropTypes.number, - } + }; static defaultProps = { rowHeight: 18, - } + }; - heights: {[id: string]: number} + heights: { [id: string]: number }; constructor(props) { - super(props) + super(props); - this.heights = {} - this.state = { vScroll: calcVScroll() } + this.heights = {}; + this.state = { vScroll: calcVScroll() }; - this.onViewportUpdate = this.onViewportUpdate.bind(this) + this.onViewportUpdate = this.onViewportUpdate.bind(this); } componentDidMount() { - window.addEventListener('resize', this.onViewportUpdate) - this.onViewportUpdate() + window.addEventListener("resize", this.onViewportUpdate); + this.onViewportUpdate(); } componentWillUnmount() { - window.removeEventListener('resize', this.onViewportUpdate) + window.removeEventListener("resize", this.onViewportUpdate); } componentDidUpdate() { - this.onViewportUpdate() + this.onViewportUpdate(); } onViewportUpdate() { - const viewport = ReactDOM.findDOMNode(this) + const viewport = ReactDOM.findDOMNode(this); const vScroll = calcVScroll({ itemCount: this.props.events.length, rowHeight: this.props.rowHeight, viewportTop: viewport.scrollTop, viewportHeight: viewport.offsetHeight, - itemHeights: this.props.events.map(entry => this.heights[entry.id]), - }) + itemHeights: this.props.events.map( + (entry) => this.heights[entry.id] + ), + }); if (!shallowEqual(this.state.vScroll, vScroll)) { - this.setState({vScroll}) + this.setState({ vScroll }); } } setHeight(id, node) { if (node && !this.heights[id]) { - const height = node.offsetHeight + const height = node.offsetHeight; if (this.heights[id] !== height) { - this.heights[id] = height - this.onViewportUpdate() + this.heights[id] = height; + this.onViewportUpdate(); } } } render() { - const { vScroll } = this.state - const { events } = this.props + const { vScroll } = this.state; + const { events } = this.props; return ( 
    -                
    
-                {events.slice(vScroll.start, vScroll.end).map(event => (
-                    
     this.setHeight(event.id, node)}>
-                        
+                
    
+                {events.slice(vScroll.start, vScroll.end).map((event) => (
+                    
     this.setHeight(event.id, node)}
+                    >
+                        
                         {event.message}
                     
    
                 ))}
-                
    
+
    - ) + ); } } function LogIcon({ event }) { - const icon = { - web: 'html5', - debug: 'bug', - warn: 'exclamation-triangle', - error: 'ban' - }[event.level] || 'info' - return + const icon = + { + web: "html5", + debug: "bug", + warn: "exclamation-triangle", + error: "ban", + }[event.level] || "info"; + return ; } -export default AutoScroll(EventLogList) +export default AutoScroll(EventLogList); diff --git a/web/src/js/components/FlowTable.jsx b/web/src/js/components/FlowTable.jsx index eb5d4adb96..379c67e3a3 100644 --- a/web/src/js/components/FlowTable.jsx +++ b/web/src/js/components/FlowTable.jsx @@ -1,37 +1,35 @@ -import * as React from "react" -import PropTypes from 'prop-types' -import ReactDOM from 'react-dom' -import { connect } from 'react-redux' -import shallowEqual from 'shallowequal' -import AutoScroll from './helpers/AutoScroll' -import { calcVScroll } from './helpers/VirtualScroll' -import FlowTableHead from './FlowTable/FlowTableHead' -import FlowRow from './FlowTable/FlowRow' -import Filt from "../filt/filt" - +import * as React from "react"; +import PropTypes from "prop-types"; +import ReactDOM from "react-dom"; +import { connect } from "react-redux"; +import shallowEqual from "shallowequal"; +import AutoScroll from "./helpers/AutoScroll"; +import { calcVScroll } from "./helpers/VirtualScroll"; +import FlowTableHead from "./FlowTable/FlowTableHead"; +import FlowRow from "./FlowTable/FlowRow"; +import Filt from "../filt/filt"; class FlowTable extends React.Component { - static propTypes = { flows: PropTypes.array.isRequired, rowHeight: PropTypes.number, highlight: PropTypes.string, selected: PropTypes.object, - } + }; static defaultProps = { rowHeight: 32, - } + }; constructor(props, context) { - super(props, context) + super(props, context); - this.state = { vScroll: calcVScroll() } - this.onViewportUpdate = this.onViewportUpdate.bind(this) + this.state = { vScroll: calcVScroll() }; + this.onViewportUpdate = this.onViewportUpdate.bind(this); } UNSAFE_componentWillMount() { - window.addEventListener('resize', this.onViewportUpdate) + window.addEventListener("resize", this.onViewportUpdate); } componentDidMount() { @@ -39,81 +37,90 @@ class FlowTable extends React.Component { } UNSAFE_componentWillUnmount() { - window.removeEventListener('resize', this.onViewportUpdate) + window.removeEventListener("resize", this.onViewportUpdate); } componentDidUpdate() { - this.onViewportUpdate() + this.onViewportUpdate(); if (!this.shouldScrollIntoView) { - return + return; } - this.shouldScrollIntoView = false + this.shouldScrollIntoView = false; - const { rowHeight, flows, selected } = this.props - const viewport = ReactDOM.findDOMNode(this) - const head = ReactDOM.findDOMNode(this.refs.head) + const { rowHeight, flows, selected } = this.props; + const viewport = ReactDOM.findDOMNode(this); + const head = ReactDOM.findDOMNode(this.refs.head); - const headHeight = head ? head.offsetHeight : 0 + const headHeight = head ? head.offsetHeight : 0; - const rowTop = (flows.indexOf(selected) * rowHeight) + headHeight - const rowBottom = rowTop + rowHeight + const rowTop = flows.indexOf(selected) * rowHeight + headHeight; + const rowBottom = rowTop + rowHeight; - const viewportTop = viewport.scrollTop - const viewportHeight = viewport.offsetHeight + const viewportTop = viewport.scrollTop; + const viewportHeight = viewport.offsetHeight; // Account for pinned thead if (rowTop - headHeight < viewportTop) { - viewport.scrollTop = rowTop - headHeight + viewport.scrollTop = rowTop - headHeight; } else if (rowBottom > viewportTop + viewportHeight) { - viewport.scrollTop = rowBottom - viewportHeight + viewport.scrollTop = rowBottom - viewportHeight; } } UNSAFE_componentWillReceiveProps(nextProps) { if (nextProps.selected && nextProps.selected !== this.props.selected) { - this.shouldScrollIntoView = true + this.shouldScrollIntoView = true; } } onViewportUpdate() { - const viewport = ReactDOM.findDOMNode(this) - const viewportTop = viewport.scrollTop || 0 + const viewport = ReactDOM.findDOMNode(this); + const viewportTop = viewport.scrollTop || 0; const vScroll = calcVScroll({ viewportTop, viewportHeight: viewport.offsetHeight || 0, itemCount: this.props.flows.length, rowHeight: this.props.rowHeight, - }) + }); - if (this.state.viewportTop !== viewportTop || !shallowEqual(this.state.vScroll, vScroll)) { + if ( + this.state.viewportTop !== viewportTop || + !shallowEqual(this.state.vScroll, vScroll) + ) { // the next rendered state may only have much lower number of rows compared to what the current // viewportHeight anticipates. To make sure that we update (almost) at once, we already constrain // the maximum viewportTop value. See https://github.com/mitmproxy/mitmproxy/pull/5658 for details. - let newViewportTop = Math.min(viewportTop, vScroll.end * this.props.rowHeight); + let newViewportTop = Math.min( + viewportTop, + vScroll.end * this.props.rowHeight + ); this.setState({ vScroll, - viewportTop: newViewportTop + viewportTop: newViewportTop, }); } } render() { - const { vScroll, viewportTop } = this.state - const { flows, selected, highlight } = this.props - const isHighlighted = highlight ? Filt.parse(highlight) : () => false + const { vScroll, viewportTop } = this.state; + const { flows, selected, highlight } = this.props; + const isHighlighted = highlight ? Filt.parse(highlight) : () => false; return (
    - + - - {flows.slice(vScroll.start, vScroll.end).map(flow => ( + + {flows.slice(vScroll.start, vScroll.end).map((flow) => ( ))} - +
    - ) + ); } } -export const PureFlowTable = AutoScroll(FlowTable) +export const PureFlowTable = AutoScroll(FlowTable); -export default connect( - state => ({ - flows: state.flows.view, - highlight: state.flows.highlight, - selected: state.flows.byId[state.flows.selected[0]], - }) -)(PureFlowTable) +export default connect((state) => ({ + flows: state.flows.view, + highlight: state.flows.highlight, + selected: state.flows.byId[state.flows.selected[0]], +}))(PureFlowTable); diff --git a/web/src/js/components/FlowTable/FlowColumns.tsx b/web/src/js/components/FlowTable/FlowColumns.tsx index 9613c90066..460334231e 100644 --- a/web/src/js/components/FlowTable/FlowColumns.tsx +++ b/web/src/js/components/FlowTable/FlowColumns.tsx @@ -1,15 +1,21 @@ -import React, {ReactElement, useState} from 'react' -import {useDispatch} from 'react-redux' -import classnames from 'classnames' -import {canReplay, endTime, getTotalSize, RequestUtils, ResponseUtils, startTime} from '../../flow/utils' -import {formatSize, formatTimeDelta, formatTimeStamp} from '../../utils' +import React, { ReactElement, useState } from "react"; +import { useDispatch } from "react-redux"; +import classnames from "classnames"; +import { + canReplay, + endTime, + getTotalSize, + RequestUtils, + ResponseUtils, + startTime, +} from "../../flow/utils"; +import { formatSize, formatTimeDelta, formatTimeStamp } from "../../utils"; import * as flowActions from "../../ducks/flows"; -import {Flow} from "../../flow"; - +import { Flow } from "../../flow"; type FlowColumnProps = { - flow: Flow -} + flow: Flow; +}; interface FlowColumn { (props: FlowColumnProps): JSX.Element; @@ -18,182 +24,212 @@ interface FlowColumn { sortKey: (flow: Flow) => any; } -export const tls: FlowColumn = ({flow}) => { +export const tls: FlowColumn = ({ flow }) => { return ( - - ) -} -tls.headerName = '' -tls.sortKey = flow => flow.type === "http" && flow.request.scheme + + ); +}; +tls.headerName = ""; +tls.sortKey = (flow) => flow.type === "http" && flow.request.scheme; -export const icon: FlowColumn = ({flow}) => { +export const icon: FlowColumn = ({ flow }) => { return ( -
    +
    - ) -} -icon.headerName = '' -icon.sortKey = flow => getIcon(flow) + ); +}; +icon.headerName = ""; +icon.sortKey = (flow) => getIcon(flow); const getIcon = (flow: Flow): string => { if (flow.type !== "http") { if (flow.client_conn.tls_version === "QUIC") { return `resource-icon-quic`; } - return `resource-icon-${flow.type}` + return `resource-icon-${flow.type}`; } if (flow.websocket) { - return 'resource-icon-websocket' + return "resource-icon-websocket"; } if (!flow.response) { - return 'resource-icon-plain' + return "resource-icon-plain"; } - var contentType = ResponseUtils.getContentType(flow.response) || '' + var contentType = ResponseUtils.getContentType(flow.response) || ""; if (flow.response.status_code === 304) { - return 'resource-icon-not-modified' + return "resource-icon-not-modified"; } if (300 <= flow.response.status_code && flow.response.status_code < 400) { - return 'resource-icon-redirect' + return "resource-icon-redirect"; } - if (contentType.indexOf('image') >= 0) { - return 'resource-icon-image' + if (contentType.indexOf("image") >= 0) { + return "resource-icon-image"; } - if (contentType.indexOf('javascript') >= 0) { - return 'resource-icon-js' + if (contentType.indexOf("javascript") >= 0) { + return "resource-icon-js"; } - if (contentType.indexOf('css') >= 0) { - return 'resource-icon-css' + if (contentType.indexOf("css") >= 0) { + return "resource-icon-css"; } - if (contentType.indexOf('html') >= 0) { - return 'resource-icon-document' + if (contentType.indexOf("html") >= 0) { + return "resource-icon-document"; } - return 'resource-icon-plain' -} + return "resource-icon-plain"; +}; const mainPath = (flow: Flow): string => { switch (flow.type) { case "http": - return RequestUtils.pretty_url(flow.request) + return RequestUtils.pretty_url(flow.request); case "tcp": case "udp": - return `${flow.client_conn.peername.join(':')} ↔ ${flow.server_conn?.address?.join(':')}` + return `${flow.client_conn.peername.join( + ":" + )} ↔ ${flow.server_conn?.address?.join(":")}`; case "dns": - return `${flow.request.questions.map(q => `${q.name} ${q.type}`).join(", ")} = ${(flow.response?.answers.map(q => q.data).join(", ") ?? "...") || "?"}` + return `${flow.request.questions + .map((q) => `${q.name} ${q.type}`) + .join(", ")} = ${ + (flow.response?.answers.map((q) => q.data).join(", ") ?? + "...") || + "?" + }`; } -} +}; -export const path: FlowColumn = ({flow}) => { +export const path: FlowColumn = ({ flow }) => { let err; if (flow.error) { if (flow.error.msg === "Connection killed.") { - err = + err = ; } else { - err = + err = ; } } return ( {flow.is_replay === "request" && ( - - )} - {flow.intercepted && ( - + )} + {flow.intercepted && } {err} {flow.marked} {mainPath(flow)} - ) + ); }; -path.headerName = 'Path' -path.sortKey = flow => mainPath(flow) +path.headerName = "Path"; +path.sortKey = (flow) => mainPath(flow); -export const method: FlowColumn = ({flow}) => {method.sortKey(flow)} -method.headerName = 'Method' -method.sortKey = flow => { +export const method: FlowColumn = ({ flow }) => ( + {method.sortKey(flow)} +); +method.headerName = "Method"; +method.sortKey = (flow) => { switch (flow.type) { - case "http": return flow.websocket ? (flow.client_conn.tls_established ? "WSS" : "WS") : flow.request.method - case "dns": return flow.request.op_code - default: return flow.type.toUpperCase() + case "http": + return flow.websocket + ? flow.client_conn.tls_established + ? "WSS" + : "WS" + : flow.request.method; + case "dns": + return flow.request.op_code; + default: + return flow.type.toUpperCase(); } -} +}; -export const status: FlowColumn = ({flow}) => { - let color = 'darkred' +export const status: FlowColumn = ({ flow }) => { + let color = "darkred"; if ((flow.type !== "http" && flow.type != "dns") || !flow.response) - return + return ; if (100 <= flow.response.status_code && flow.response.status_code < 200) { - color = 'green' - } else if (200 <= flow.response.status_code && flow.response.status_code < 300) { - color = 'darkgreen' - } else if (300 <= flow.response.status_code && flow.response.status_code < 400) { - color = 'lightblue' - } else if (400 <= flow.response.status_code && flow.response.status_code < 500) { - color = 'red' - } else if (500 <= flow.response.status_code && flow.response.status_code < 600) { - color = 'red' + color = "green"; + } else if ( + 200 <= flow.response.status_code && + flow.response.status_code < 300 + ) { + color = "darkgreen"; + } else if ( + 300 <= flow.response.status_code && + flow.response.status_code < 400 + ) { + color = "lightblue"; + } else if ( + 400 <= flow.response.status_code && + flow.response.status_code < 500 + ) { + color = "red"; + } else if ( + 500 <= flow.response.status_code && + flow.response.status_code < 600 + ) { + color = "red"; } return ( - {status.sortKey(flow)} - ) -} -status.headerName = 'Status' -status.sortKey = flow => { + + {status.sortKey(flow)} + + ); +}; +status.headerName = "Status"; +status.sortKey = (flow) => { switch (flow.type) { - case "http": return flow.response?.status_code - case "dns": return flow.response?.response_code - default: return undefined + case "http": + return flow.response?.status_code; + case "dns": + return flow.response?.response_code; + default: + return undefined; } -} - -export const size: FlowColumn = ({flow}) => { - return ( - {formatSize(getTotalSize(flow))} - ) }; -size.headerName = 'Size' -size.sortKey = flow => getTotalSize(flow) +export const size: FlowColumn = ({ flow }) => { + return {formatSize(getTotalSize(flow))}; +}; +size.headerName = "Size"; +size.sortKey = (flow) => getTotalSize(flow); -export const time: FlowColumn = ({flow}) => { - const start = startTime(flow), end = endTime(flow); +export const time: FlowColumn = ({ flow }) => { + const start = startTime(flow), + end = endTime(flow); return ( - {start && end ? ( - formatTimeDelta(1000 * (end - start)) - ) : ( - '...' - )} + {start && end ? formatTimeDelta(1000 * (end - start)) : "..."} - ) -} -time.headerName = 'Time' -time.sortKey = flow => { - const start = startTime(flow), end = endTime(flow); + ); +}; +time.headerName = "Time"; +time.sortKey = (flow) => { + const start = startTime(flow), + end = endTime(flow); return start && end && end - start; -} +}; -export const timestamp: FlowColumn = ({flow}) => { +export const timestamp: FlowColumn = ({ flow }) => { const start = startTime(flow); return ( - {start ? ( - formatTimeStamp(start) - ) : ( - '...' - )} + {start ? formatTimeStamp(start) : "..."} - ) -} -timestamp.headerName = 'Start time' -timestamp.sortKey = flow => startTime(flow) + ); +}; +timestamp.headerName = "Start time"; +timestamp.sortKey = (flow) => startTime(flow); const markers = { ":red_circle:": "🔴", @@ -203,32 +239,47 @@ const markers = { ":large_blue_circle:": "🔵", ":purple_circle:": "🟣", ":brown_circle:": "🟤", -} +}; -export const quickactions: FlowColumn = ({flow}) => { - const dispatch = useDispatch() - let [open, setOpen] = useState(false) +export const quickactions: FlowColumn = ({ flow }) => { + const dispatch = useDispatch(); + let [open, setOpen] = useState(false); let resume_or_replay: ReactElement | null = null; if (flow.intercepted) { - resume_or_replay = dispatch(flowActions.resume(flow))}> - - ; + resume_or_replay = ( + dispatch(flowActions.resume(flow))} + > + + + ); } else if (canReplay(flow)) { - resume_or_replay = dispatch(flowActions.replay(flow))}> - - ; + resume_or_replay = ( + dispatch(flowActions.replay(flow))} + > + + + ); } return ( - 0}> + 0} + > {resume_or_replay ?
    {resume_or_replay}
    : <>} - ) -} + ); +}; -quickactions.headerName = '' -quickactions.sortKey = flow => 0; +quickactions.headerName = ""; +quickactions.sortKey = (flow) => 0; export default { icon, @@ -239,5 +290,5 @@ export default { status, time, timestamp, - tls + tls, }; diff --git a/web/src/js/components/FlowTable/FlowRow.tsx b/web/src/js/components/FlowTable/FlowRow.tsx index 8e2d6ac0d4..84ac55a889 100644 --- a/web/src/js/components/FlowTable/FlowRow.tsx +++ b/web/src/js/components/FlowTable/FlowRow.tsx @@ -1,45 +1,56 @@ -import React, {useCallback} from 'react' -import classnames from 'classnames' -import {Flow} from "../../flow"; -import {useAppDispatch, useAppSelector} from "../../ducks"; -import {select} from '../../ducks/flows' +import React, { useCallback } from "react"; +import classnames from "classnames"; +import { Flow } from "../../flow"; +import { useAppDispatch, useAppSelector } from "../../ducks"; +import { select } from "../../ducks/flows"; import * as columns from "./FlowColumns"; type FlowRowProps = { - flow: Flow - selected: boolean - highlighted: boolean -} + flow: Flow; + selected: boolean; + highlighted: boolean; +}; -export default React.memo(function FlowRow({flow, selected, highlighted}: FlowRowProps) { +export default React.memo(function FlowRow({ + flow, + selected, + highlighted, +}: FlowRowProps) { const dispatch = useAppDispatch(), - displayColumnNames = useAppSelector(state => state.options.web_columns), + displayColumnNames = useAppSelector( + (state) => state.options.web_columns + ), className = classnames({ - 'selected': selected, - 'highlighted': highlighted, - 'intercepted': flow.intercepted, - 'has-request': flow.type === "http" && flow.request, - 'has-response': flow.type === "http" && flow.response, - }) + selected: selected, + highlighted: highlighted, + intercepted: flow.intercepted, + "has-request": flow.type === "http" && flow.request, + "has-response": flow.type === "http" && flow.response, + }); - const onClick = useCallback(e => { - // a bit of a hack to disable row selection for quickactions. - let node = e.target; - while (node.parentNode) { - if (node.classList.contains("col-quickactions")) - return - node = node.parentNode; - } - dispatch(select(flow.id)); - }, [flow]); + const onClick = useCallback( + (e) => { + // a bit of a hack to disable row selection for quickactions. + let node = e.target; + while (node.parentNode) { + if (node.classList.contains("col-quickactions")) return; + node = node.parentNode; + } + dispatch(select(flow.id)); + }, + [flow] + ); - const displayColumns = displayColumnNames.map(x => columns[x]).filter(x => x).concat(columns.quickactions); + const displayColumns = displayColumnNames + .map((x) => columns[x]) + .filter((x) => x) + .concat(columns.quickactions); return ( - {displayColumns.map(Column => ( - + {displayColumns.map((Column) => ( + ))} - ) -}) + ); +}); diff --git a/web/src/js/components/FlowTable/FlowTableHead.tsx b/web/src/js/components/FlowTable/FlowTableHead.tsx index e06dbd0f09..9f566ce62d 100644 --- a/web/src/js/components/FlowTable/FlowTableHead.tsx +++ b/web/src/js/components/FlowTable/FlowTableHead.tsx @@ -1,30 +1,47 @@ -import * as React from "react" -import classnames from 'classnames' -import * as columns from './FlowColumns' +import * as React from "react"; +import classnames from "classnames"; +import * as columns from "./FlowColumns"; -import {setSort} from '../../ducks/flows' -import {useAppDispatch, useAppSelector} from "../../ducks"; +import { setSort } from "../../ducks/flows"; +import { useAppDispatch, useAppSelector } from "../../ducks"; export default React.memo(function FlowTableHead() { const dispatch = useAppDispatch(), - sortDesc = useAppSelector(state => state.flows.sort.desc), - sortColumn = useAppSelector(state => state.flows.sort.column), - displayColumnNames = useAppSelector(state => state.options.web_columns); + sortDesc = useAppSelector((state) => state.flows.sort.desc), + sortColumn = useAppSelector((state) => state.flows.sort.column), + displayColumnNames = useAppSelector( + (state) => state.options.web_columns + ); - const sortType = sortDesc ? 'sort-desc' : 'sort-asc' - const displayColumns = displayColumnNames.map(x => columns[x]).filter(x => x).concat(columns.quickactions); + const sortType = sortDesc ? "sort-desc" : "sort-asc"; + const displayColumns = displayColumnNames + .map((x) => columns[x]) + .filter((x) => x) + .concat(columns.quickactions); return ( - {displayColumns.map(Column => ( - ( + dispatch(setSort( - Column.name === sortColumn && sortDesc ? undefined : Column.name, - Column.name !== sortColumn ? false : !sortDesc))}> + onClick={() => + dispatch( + setSort( + Column.name === sortColumn && sortDesc + ? undefined + : Column.name, + Column.name !== sortColumn ? false : !sortDesc + ) + ) + } + > {Column.headerName} ))} - ) -}) + ); +}); diff --git a/web/src/js/components/FlowView.tsx b/web/src/js/components/FlowView.tsx index 4c69efa3e5..15153e9b13 100644 --- a/web/src/js/components/FlowView.tsx +++ b/web/src/js/components/FlowView.tsx @@ -1,24 +1,29 @@ -import * as React from "react" -import {FunctionComponent} from "react" -import {Request, Response} from './FlowView/HttpMessages' -import {Request as DnsRequest, Response as DnsResponse} from './FlowView/DnsMessages' -import Connection from './FlowView/Connection' -import Error from "./FlowView/Error" -import Timing from "./FlowView/Timing" -import WebSocket from "./FlowView/WebSocket" +import * as React from "react"; +import { FunctionComponent } from "react"; +import { Request, Response } from "./FlowView/HttpMessages"; +import { + Request as DnsRequest, + Response as DnsResponse, +} from "./FlowView/DnsMessages"; +import Connection from "./FlowView/Connection"; +import Error from "./FlowView/Error"; +import Timing from "./FlowView/Timing"; +import WebSocket from "./FlowView/WebSocket"; -import {selectTab} from '../ducks/ui/flow' -import {useAppDispatch, useAppSelector} from "../ducks"; -import {Flow} from "../flow"; +import { selectTab } from "../ducks/ui/flow"; +import { useAppDispatch, useAppSelector } from "../ducks"; +import { Flow } from "../flow"; import classnames from "classnames"; import TcpMessages from "./FlowView/TcpMessages"; import UdpMessages from "./FlowView/UdpMessages"; type TabProps = { - flow: Flow -} + flow: Flow; +}; -export const allTabs: { [name: string]: FunctionComponent & { displayName: string } } = { +export const allTabs: { + [name: string]: FunctionComponent & { displayName: string }; +} = { request: Request, response: Response, error: Error, @@ -29,45 +34,48 @@ export const allTabs: { [name: string]: FunctionComponent & { displayN udpmessages: UdpMessages, dnsrequest: DnsRequest, dnsresponse: DnsResponse, -} +}; export function tabsForFlow(flow: Flow): string[] { let tabs; switch (flow.type) { case "http": - tabs = ['request', 'response', 'websocket'].filter(k => flow[k]) - break + tabs = ["request", "response", "websocket"].filter((k) => flow[k]); + break; case "tcp": - tabs = ["tcpmessages"] - break + tabs = ["tcpmessages"]; + break; case "udp": - tabs = ["udpmessages"] - break + tabs = ["udpmessages"]; + break; case "dns": - tabs = ['request', 'response'].filter(k => flow[k]).map(s => "dns" + s) - break + tabs = ["request", "response"] + .filter((k) => flow[k]) + .map((s) => "dns" + s); + break; } - if (flow.error) - tabs.push("error") - tabs.push("connection") - tabs.push("timing") + if (flow.error) tabs.push("error"); + tabs.push("connection"); + tabs.push("timing"); return tabs; } export default function FlowView() { const dispatch = useAppDispatch(), - flow = useAppSelector(state => state.flows.byId[state.flows.selected[0]]), + flow = useAppSelector( + (state) => state.flows.byId[state.flows.selected[0]] + ), tabs = tabsForFlow(flow); - let active = useAppSelector(state => state.ui.flow.tab) + let active = useAppSelector((state) => state.ui.flow.tab); if (tabs.indexOf(active) < 0) { - if (active === 'response' && flow.error) { - active = 'error' - } else if (active === 'error' && "response" in flow) { - active = 'response' + if (active === "response" && flow.error) { + active = "error"; + } else if (active === "error" && "response" in flow) { + active = "response"; } else { - active = tabs[0] + active = tabs[0]; } } const Tab = allTabs[active]; @@ -75,17 +83,21 @@ export default function FlowView() { return ( - ) + ); } diff --git a/web/src/js/components/FlowView/Connection.tsx b/web/src/js/components/FlowView/Connection.tsx index 48493ca12e..63a99753e5 100644 --- a/web/src/js/components/FlowView/Connection.tsx +++ b/web/src/js/components/FlowView/Connection.tsx @@ -1,152 +1,170 @@ -import * as React from "react" -import {formatTimeStamp} from '../../utils' -import {Client, Flow, Server} from '../../flow' - +import * as React from "react"; +import { formatTimeStamp } from "../../utils"; +import { Client, Flow, Server } from "../../flow"; type ConnectionInfoProps = { - conn: Client | Server -} + conn: Client | Server; +}; -export function ConnectionInfo({conn}: ConnectionInfoProps) { +export function ConnectionInfo({ conn }: ConnectionInfoProps) { let address_info: JSX.Element | null = null; if ("address" in conn) { // Server - address_info = <> - - Address: - {conn.address?.join(':')} - - {conn.peername && ( - - Resolved address: - {conn.peername.join(':')} - - )} - {conn.sockname && ( + address_info = ( + <> - Source address: - {conn.sockname.join(':')} + Address: + {conn.address?.join(":")} - )} - ; + {conn.peername && ( + + Resolved address: + {conn.peername.join(":")} + + )} + {conn.sockname && ( + + Source address: + {conn.sockname.join(":")} + + )} + + ); } else { // Client if (conn.peername?.[0]) { - address_info = <> - - Address: - {conn.peername?.join(':')} - - + address_info = ( + <> + + Address: + {conn.peername?.join(":")} + + + ); } - } return ( - {address_info} - {conn.sni ? ( - - - - - ): null} - {conn.alpn ? ( - - - - - ) : null} - {conn.tls_version ? ( - - - - - ): null} - {conn.cipher ? ( - - - - - ): null} + {address_info} + {conn.sni ? ( + + + + + ) : null} + {conn.alpn ? ( + + + + + ) : null} + {conn.tls_version ? ( + + + + + ) : null} + {conn.cipher ? ( + + + + + ) : null}
    SNI:{conn.sni}
    ALPN:{conn.alpn}
    TLS Version:{conn.tls_version}
    TLS Cipher:{conn.cipher}
    + SNI: + {conn.sni}
    + ALPN: + {conn.alpn}
    TLS Version:{conn.tls_version}
    TLS Cipher:{conn.cipher}
    - ) + ); } function attrList(data: [string, string][]): JSX.Element { - return
    - {data.map(([k, v]) => - -
    {k}
    -
    {v}
    -     - )} -
    + return ( +
    + {data.map(([k, v]) => ( + +
    {k}
    +
    {v}
    +     + ))} +
    + ); } -export function CertificateInfo({flow}: { flow: Flow }): JSX.Element { +export function CertificateInfo({ flow }: { flow: Flow }): JSX.Element { const cert = flow.server_conn?.cert; - if (!cert) - return <>; + if (!cert) return <>; - return <> -

    Server Certificate

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Type{cert.keyinfo[0]}, {cert.keyinfo[1]} bits
    SHA256 digest{cert.sha256}
    Valid from{formatTimeStamp(cert.notbefore, {milliseconds: false})}
    Valid to{formatTimeStamp(cert.notafter, {milliseconds: false})}
    Subject Alternative Names{cert.altnames.join(", ")}
    Subject{attrList(cert.subject)}
    Issuer{attrList(cert.issuer)}
    Serial{cert.serial}
    - + return ( + <> +

    Server Certificate

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Type + {cert.keyinfo[0]}, {cert.keyinfo[1]} bits +
    SHA256 digest{cert.sha256}
    Valid from + {formatTimeStamp(cert.notbefore, { + milliseconds: false, + })} +
    Valid to + {formatTimeStamp(cert.notafter, { + milliseconds: false, + })} +
    Subject Alternative Names{cert.altnames.join(", ")}
    Subject{attrList(cert.subject)}
    Issuer{attrList(cert.issuer)}
    Serial{cert.serial}
    + + ); } -export default function Connection({flow}: { flow: Flow }) { +export default function Connection({ flow }: { flow: Flow }) { return (

    Client Connection

    - + - { - flow.server_conn?.address && + {flow.server_conn?.address && ( <>

    Server Connection

    - + - } + )} - +
    - ) + ); } -Connection.displayName = "Connection" +Connection.displayName = "Connection"; diff --git a/web/src/js/components/FlowView/DnsMessages.tsx b/web/src/js/components/FlowView/DnsMessages.tsx index 8cc51ca1bc..51483d2627 100644 --- a/web/src/js/components/FlowView/DnsMessages.tsx +++ b/web/src/js/components/FlowView/DnsMessages.tsx @@ -1,108 +1,116 @@ -import * as React from "react" +import * as React from "react"; -import {useAppSelector} from "../../ducks"; -import {DNSFlow, DNSMessage, DNSResourceRecord} from '../../flow' +import { useAppSelector } from "../../ducks"; +import { DNSFlow, DNSMessage, DNSResourceRecord } from "../../flow"; const Summary: React.FC<{ - message: DNSMessage -}> = ({message}) => ( + message: DNSMessage; +}> = ({ message }) => (
    {message.query ? message.op_code : message.response_code}   {message.truncation ? "(Truncated)" : ""}
    -) +); const Questions: React.FC<{ - message: DNSMessage -}> = ({message}) => ( + message: DNSMessage; +}> = ({ message }) => ( <>
    {message.recursion_desired ? "Recursive " : ""}Question
    - - - - - + + + + + - {message.questions.map((question, index) => ( - - - - - - ))} + {message.questions.map((question, index) => ( + + + + + + ))}
    NameTypeClass
    NameTypeClass
    {question.name}{question.type}{question.class}
    {question.name}{question.type}{question.class}
    -) +); const ResourceRecords: React.FC<{ - name: string - values: DNSResourceRecord[] -}> = ({name, values}) => ( + name: string; + values: DNSResourceRecord[]; +}> = ({ name, values }) => ( <>
    {name}
    - {values.length > 0 - ? + {values.length > 0 ? ( +
    - - - - - - - + + + + + + + - {values.map((rr, index) => ( - - - - - - - - ))} + {values.map((rr, index) => ( + + + + + + + + ))}
    NameTypeClassTTLData
    NameTypeClassTTLData
    {rr.name}{rr.type}{rr.class}{rr.ttl}{rr.data}
    {rr.name}{rr.type}{rr.class}{rr.ttl}{rr.data}
    - : "—" - } + ) : ( + "—" + )} -) +); const Message: React.FC<{ - type: "request" | "response" - message: DNSMessage -}> = ({type, message}) => ( + type: "request" | "response"; + message: DNSMessage; +}> = ({ type, message }) => (
    - +
    - -
    + +
    -
    - -
    - + name={`${message.authoritative_answer ? "Authoritative " : ""}${ + message.recursion_available ? "Recursive " : "" + }Answer`} + values={message.answers} + /> +
    + +
    +
    -) +); export function Request() { - const flow = useAppSelector(state => state.flows.byId[state.flows.selected[0]]) as DNSFlow; - return ; + const flow = useAppSelector( + (state) => state.flows.byId[state.flows.selected[0]] + ) as DNSFlow; + return ; } -Request.displayName = "Request" +Request.displayName = "Request"; export function Response() { - const flow = useAppSelector(state => state.flows.byId[state.flows.selected[0]]) as DNSFlow & { response: DNSMessage } - return ; + const flow = useAppSelector( + (state) => state.flows.byId[state.flows.selected[0]] + ) as DNSFlow & { response: DNSMessage }; + return ; } -Response.displayName = "Response" +Response.displayName = "Response"; diff --git a/web/src/js/components/FlowView/Error.tsx b/web/src/js/components/FlowView/Error.tsx index e99e2fa327..9eb4589dc3 100644 --- a/web/src/js/components/FlowView/Error.tsx +++ b/web/src/js/components/FlowView/Error.tsx @@ -1,12 +1,12 @@ -import {HTTPFlow} from "../../flow"; -import {formatTimeStamp} from "../../utils"; +import { HTTPFlow } from "../../flow"; +import { formatTimeStamp } from "../../utils"; import * as React from "react"; type ErrorProps = { - flow: HTTPFlow & { error: Error } -} + flow: HTTPFlow & { error: Error }; +}; -export default function Error({flow}: ErrorProps) { +export default function Error({ flow }: ErrorProps) { return (
    @@ -16,6 +16,6 @@ export default function Error({flow}: ErrorProps) {
    - ) + ); } Error.displayName = "Error"; diff --git a/web/src/js/components/FlowView/HttpMessages.tsx b/web/src/js/components/FlowView/HttpMessages.tsx index 1eaab2f478..c4e7873c0e 100644 --- a/web/src/js/components/FlowView/HttpMessages.tsx +++ b/web/src/js/components/FlowView/HttpMessages.tsx @@ -1,21 +1,25 @@ -import * as React from "react" - -import {isValidHttpVersion, MessageUtils, parseUrl, RequestUtils} from '../../flow/utils' -import ValidateEditor from '../editors/ValidateEditor' -import ValueEditor from '../editors/ValueEditor' - -import {useAppDispatch, useAppSelector} from "../../ducks"; -import {HTTPFlow, HTTPMessage, HTTPResponse} from '../../flow' -import * as flowActions from '../../ducks/flows' +import * as React from "react"; + +import { + isValidHttpVersion, + MessageUtils, + parseUrl, + RequestUtils, +} from "../../flow/utils"; +import ValidateEditor from "../editors/ValidateEditor"; +import ValueEditor from "../editors/ValueEditor"; + +import { useAppDispatch, useAppSelector } from "../../ducks"; +import { HTTPFlow, HTTPMessage, HTTPResponse } from "../../flow"; +import * as flowActions from "../../ducks/flows"; import KeyValueListEditor from "../editors/KeyValueListEditor"; import HttpMessage from "../contentviews/HttpMessage"; - type RequestLineProps = { - flow: HTTPFlow, -} + flow: HTTPFlow; +}; -function RequestLine({flow}: RequestLineProps) { +function RequestLine({ flow }: RequestLineProps) { const dispatch = useAppDispatch(); return ( @@ -23,67 +27,97 @@ function RequestLine({flow}: RequestLineProps) {
    dispatch(flowActions.update(flow, {request: {method}}))} - isValid={method => method.length > 0} + onEditDone={(method) => + dispatch( + flowActions.update(flow, { request: { method } }) + ) + } + isValid={(method) => method.length > 0} />   dispatch(flowActions.update(flow, {request: {path: '', ...parseUrl(url)}}))} - isValid={url => !!parseUrl(url)?.host} + onEditDone={(url) => + dispatch( + flowActions.update(flow, { + request: { path: "", ...parseUrl(url) }, + }) + ) + } + isValid={(url) => !!parseUrl(url)?.host} />   dispatch(flowActions.update(flow, {request: {http_version}}))} + onEditDone={(http_version) => + dispatch( + flowActions.update(flow, { + request: { http_version }, + }) + ) + } isValid={isValidHttpVersion} />
    - ) + ); } - type ResponseLineProps = { - flow: HTTPFlow & { response: HTTPResponse }, -} + flow: HTTPFlow & { response: HTTPResponse }; +}; -function ResponseLine({flow}: ResponseLineProps) { +function ResponseLine({ flow }: ResponseLineProps) { const dispatch = useAppDispatch(); return (
    dispatch(flowActions.update(flow, {response: {http_version: nextVer}}))} + onEditDone={(nextVer) => + dispatch( + flowActions.update(flow, { + response: { http_version: nextVer }, + }) + ) + } isValid={isValidHttpVersion} />   dispatch(flowActions.update(flow, {response: {code: parseInt(code)}}))} - isValid={code => /^\d+$/.test(code)} + content={flow.response.status_code + ""} + onEditDone={(code) => + dispatch( + flowActions.update(flow, { + response: { code: parseInt(code) }, + }) + ) + } + isValid={(code) => /^\d+$/.test(code)} /> - {flow.response.http_version !== "HTTP/2.0" && - <>  - dispatch(flowActions.update(flow, {response: {msg}}))} - /> - - } - + {flow.response.http_version !== "HTTP/2.0" && ( + <> +   + + dispatch( + flowActions.update(flow, { response: { msg } }) + ) + } + /> + + )}
    - ) + ); } - type HeadersProps = { - flow: HTTPFlow, - message: HTTPMessage -} + flow: HTTPFlow; + message: HTTPMessage; +}; -function Headers({flow, message}: HeadersProps) { +function Headers({ flow, message }: HeadersProps) { const dispatch = useAppDispatch(); const part = flow.request === message ? "request" : "response"; @@ -91,58 +125,73 @@ function Headers({flow, message}: HeadersProps) { dispatch(flowActions.update(flow, {[part]: {headers}}))} + onChange={(headers) => + dispatch(flowActions.update(flow, { [part]: { headers } })) + } /> ); } type TrailersProps = { - flow: HTTPFlow, - message: HTTPMessage -} + flow: HTTPFlow; + message: HTTPMessage; +}; -function Trailers({flow, message}: TrailersProps) { +function Trailers({ flow, message }: TrailersProps) { const dispatch = useAppDispatch(); const part = flow.request === message ? "request" : "response"; - const hasTrailers = !!MessageUtils.get_first_header(message, /^trailer$/i) + const hasTrailers = !!MessageUtils.get_first_header(message, /^trailer$/i); - if (!hasTrailers) - return null; + if (!hasTrailers) return null; - return <> -
    -
    HTTP Trailers
    - dispatch(flowActions.update(flow, {[part]: {trailers}}))} - /> - ; + return ( + <> +
    +
    HTTP Trailers
    + + dispatch(flowActions.update(flow, { [part]: { trailers } })) + } + /> + + ); } -const Message = React.memo(function Message({flow, message}: { flow: HTTPFlow, message: HTTPMessage }) { +const Message = React.memo(function Message({ + flow, + message, +}: { + flow: HTTPFlow; + message: HTTPMessage; +}) { const part = flow.request === message ? "request" : "response"; const FirstLine = flow.request === message ? RequestLine : ResponseLine; return (
    - - -
    - - + + +
    + +
    - ) + ); }); export function Request() { - const flow = useAppSelector(state => state.flows.byId[state.flows.selected[0]]) as HTTPFlow; - return ; + const flow = useAppSelector( + (state) => state.flows.byId[state.flows.selected[0]] + ) as HTTPFlow; + return ; } -Request.displayName = "Request" +Request.displayName = "Request"; export function Response() { - const flow = useAppSelector(state => state.flows.byId[state.flows.selected[0]]) as HTTPFlow & { response: HTTPResponse } - return ; + const flow = useAppSelector( + (state) => state.flows.byId[state.flows.selected[0]] + ) as HTTPFlow & { response: HTTPResponse }; + return ; } -Response.displayName = "Response" +Response.displayName = "Response"; diff --git a/web/src/js/components/FlowView/Messages.tsx b/web/src/js/components/FlowView/Messages.tsx index 1f5f5a8c91..e380cb85cb 100644 --- a/web/src/js/components/FlowView/Messages.tsx +++ b/web/src/js/components/FlowView/Messages.tsx @@ -1,61 +1,86 @@ -import {Flow, MessagesMeta} from "../../flow"; -import {useAppDispatch, useAppSelector} from "../../ducks"; +import { Flow, MessagesMeta } from "../../flow"; +import { useAppDispatch, useAppSelector } from "../../ducks"; import * as React from "react"; -import {useCallback, useMemo, useState} from "react"; -import {ContentViewData, useContent} from "../contentviews/useContent"; -import {MessageUtils} from "../../flow/utils"; +import { useCallback, useMemo, useState } from "react"; +import { ContentViewData, useContent } from "../contentviews/useContent"; +import { MessageUtils } from "../../flow/utils"; import ViewSelector from "../contentviews/ViewSelector"; -import {setContentViewFor} from "../../ducks/ui/flow"; -import {formatTimeStamp} from "../../utils"; +import { setContentViewFor } from "../../ducks/ui/flow"; +import { formatTimeStamp } from "../../utils"; import LineRenderer from "../contentviews/LineRenderer"; type MessagesPropTypes = { - flow: Flow - messages_meta: MessagesMeta -} + flow: Flow; + messages_meta: MessagesMeta; +}; -export default function Messages({flow, messages_meta}: MessagesPropTypes) { +export default function Messages({ flow, messages_meta }: MessagesPropTypes) { const dispatch = useAppDispatch(); - const contentView = useAppSelector(state => state.ui.flow.contentViewFor[flow.id + "messages"] || "Auto"); - let [maxLines, setMaxLines] = useState(useAppSelector(state => state.options.content_view_lines_cutoff)); - const showMore = useCallback(() => setMaxLines(Math.max(1024, maxLines * 2)), [maxLines]); + const contentView = useAppSelector( + (state) => state.ui.flow.contentViewFor[flow.id + "messages"] || "Auto" + ); + let [maxLines, setMaxLines] = useState( + useAppSelector((state) => state.options.content_view_lines_cutoff) + ); + const showMore = useCallback( + () => setMaxLines(Math.max(1024, maxLines * 2)), + [maxLines] + ); const content = useContent( MessageUtils.getContentURL(flow, "messages", contentView, maxLines + 1), flow.id + messages_meta.count ); - const messages = useMemo(() => { - if (content) { - try { - return JSON.parse(content) - } catch (e) { - const err: ContentViewData[] = [ - {"description": "Network Error", lines: [[["error", `${content}`]]]} - ]; - return err; + const messages = + useMemo(() => { + if (content) { + try { + return JSON.parse(content); + } catch (e) { + const err: ContentViewData[] = [ + { + description: "Network Error", + lines: [[["error", `${content}`]]], + }, + ]; + return err; + } } - } - }, [content]) || []; + }, [content]) || []; return (
    {messages_meta.count} Messages
    - dispatch(setContentViewFor(flow.id + "messages", cv))}/> + + dispatch(setContentViewFor(flow.id + "messages", cv)) + } + />
    {messages.map((d: ContentViewData, i) => { - const className = `fa fa-fw fa-arrow-${d.from_client ? "right text-primary" : "left text-danger"}`; - const renderer =
    - - - {d.timestamp && formatTimeStamp(d.timestamp)} - - -
    ; + const className = `fa fa-fw fa-arrow-${ + d.from_client ? "right text-primary" : "left text-danger" + }`; + const renderer = ( +
    + + + + {d.timestamp && formatTimeStamp(d.timestamp)} + + + +
    + ); maxLines -= d.lines.length; return renderer; })}
    - ) + ); } diff --git a/web/src/js/components/FlowView/TcpMessages.tsx b/web/src/js/components/FlowView/TcpMessages.tsx index 115caf7d3d..9391dd2bc0 100644 --- a/web/src/js/components/FlowView/TcpMessages.tsx +++ b/web/src/js/components/FlowView/TcpMessages.tsx @@ -1,13 +1,12 @@ -import {TCPFlow} from "../../flow"; +import { TCPFlow } from "../../flow"; import * as React from "react"; import Messages from "./Messages"; - -export default function TcpMessages({flow}: { flow: TCPFlow }) { +export default function TcpMessages({ flow }: { flow: TCPFlow }) { return (
    - +
    - ) + ); } -TcpMessages.displayName = "Stream Data" +TcpMessages.displayName = "Stream Data"; diff --git a/web/src/js/components/FlowView/Timing.tsx b/web/src/js/components/FlowView/Timing.tsx index 864d3a4943..7657295435 100644 --- a/web/src/js/components/FlowView/Timing.tsx +++ b/web/src/js/components/FlowView/Timing.tsx @@ -1,14 +1,14 @@ -import {Flow} from "../../flow"; +import { Flow } from "../../flow"; import * as React from "react"; -import {formatTimeDelta, formatTimeStamp} from "../../utils"; +import { formatTimeDelta, formatTimeStamp } from "../../utils"; export type TimeStampProps = { - t: number, - deltaTo?: number, - title: string, -} + t: number; + deltaTo?: number; + title: string; +}; -export function TimeStamp({t, deltaTo, title}: TimeStampProps) { +export function TimeStamp({ t, deltaTo, title }: TimeStampProps) { return t ? ( {title}: @@ -22,68 +22,79 @@ export function TimeStamp({t, deltaTo, title}: TimeStampProps) { ) : ( - - ) + + ); } -export default function Timing({flow}: { flow: Flow }) { +export default function Timing({ flow }: { flow: Flow }) { let ref: number; if (flow.type === "http") { - ref = flow.request.timestamp_start + ref = flow.request.timestamp_start; } else { - ref = flow.client_conn.timestamp_start + ref = flow.client_conn.timestamp_start; } const timestamps: Partial[] = [ { title: "Server conn. initiated", t: flow.server_conn?.timestamp_start, - deltaTo: ref - }, { + deltaTo: ref, + }, + { title: "Server conn. TCP handshake", t: flow.server_conn?.timestamp_tcp_setup, - deltaTo: ref - }, { + deltaTo: ref, + }, + { title: "Server conn. TLS handshake", t: flow.server_conn?.timestamp_tls_setup, - deltaTo: ref - }, { + deltaTo: ref, + }, + { title: "Server conn. closed", t: flow.server_conn?.timestamp_end, - deltaTo: ref - }, { + deltaTo: ref, + }, + { title: "Client conn. established", t: flow.client_conn.timestamp_start, - deltaTo: flow.type === "http" ? ref : undefined - }, { + deltaTo: flow.type === "http" ? ref : undefined, + }, + { title: "Client conn. TLS handshake", t: flow.client_conn.timestamp_tls_setup, - deltaTo: ref - }, { + deltaTo: ref, + }, + { title: "Client conn. closed", t: flow.client_conn.timestamp_end, - deltaTo: ref + deltaTo: ref, }, - ] + ]; if (flow.type === "http") { - timestamps.push(...[ - { - title: "First request byte", - t: flow.request.timestamp_start - }, { - title: "Request complete", - t: flow.request.timestamp_end, - deltaTo: ref - }, { - title: "First response byte", - t: flow.response?.timestamp_start, - deltaTo: ref - }, { - title: "Response complete", - t: flow.response?.timestamp_end, - deltaTo: ref - } - ]); + timestamps.push( + ...[ + { + title: "First request byte", + t: flow.request.timestamp_start, + }, + { + title: "Request complete", + t: flow.request.timestamp_end, + deltaTo: ref, + }, + { + title: "First response byte", + t: flow.response?.timestamp_start, + deltaTo: ref, + }, + { + title: "Response complete", + t: flow.response?.timestamp_end, + deltaTo: ref, + }, + ] + ); } return ( @@ -91,13 +102,15 @@ export default function Timing({flow}: { flow: Flow }) {

    Timing

    - {timestamps - .filter((v): v is TimeStampProps => !!v.t) - .sort((a, b) => a.t - b.t) - .map(props => )} + {timestamps + .filter((v): v is TimeStampProps => !!v.t) + .sort((a, b) => a.t - b.t) + .map((props) => ( + + ))}
    - ) + ); } -Timing.displayName = "Timing" +Timing.displayName = "Timing"; diff --git a/web/src/js/components/FlowView/UdpMessages.tsx b/web/src/js/components/FlowView/UdpMessages.tsx index 0dafda693e..79e7c3b79c 100644 --- a/web/src/js/components/FlowView/UdpMessages.tsx +++ b/web/src/js/components/FlowView/UdpMessages.tsx @@ -1,13 +1,12 @@ -import {UDPFlow} from "../../flow"; +import { UDPFlow } from "../../flow"; import * as React from "react"; import Messages from "./Messages"; - -export default function UdpMessages({flow}: { flow: UDPFlow }) { +export default function UdpMessages({ flow }: { flow: UDPFlow }) { return (
    - +
    - ) + ); } -UdpMessages.displayName = "Datagrams" +UdpMessages.displayName = "Datagrams"; diff --git a/web/src/js/components/FlowView/WebSocket.tsx b/web/src/js/components/FlowView/WebSocket.tsx index eecea1db14..67ea6917b4 100644 --- a/web/src/js/components/FlowView/WebSocket.tsx +++ b/web/src/js/components/FlowView/WebSocket.tsx @@ -1,32 +1,39 @@ -import {HTTPFlow, WebSocketData} from "../../flow"; +import { HTTPFlow, WebSocketData } from "../../flow"; import * as React from "react"; -import {formatTimeStamp} from "../../utils"; +import { formatTimeStamp } from "../../utils"; import Messages from "./Messages"; - -export default function WebSocket({flow}: { flow: HTTPFlow & { websocket: WebSocketData } }) { +export default function WebSocket({ + flow, +}: { + flow: HTTPFlow & { websocket: WebSocketData }; +}) { return (

    WebSocket

    - - + +
    - ) + ); } -WebSocket.displayName = "WebSocket" - +WebSocket.displayName = "WebSocket"; -function CloseSummary({websocket}: { websocket: WebSocketData }) { - if (!websocket.timestamp_end) - return null; - const reason = websocket.close_reason ? `(${websocket.close_reason})` : "" - return
    - -   - Closed by {websocket.closed_by_client ? "client" : "server"} with code {websocket.close_code} {reason}. - - - {formatTimeStamp(websocket.timestamp_end)} - -
    +function CloseSummary({ websocket }: { websocket: WebSocketData }) { + if (!websocket.timestamp_end) return null; + const reason = websocket.close_reason ? `(${websocket.close_reason})` : ""; + return ( +
    + +   Closed by {websocket.closed_by_client + ? "client" + : "server"}{" "} + with code {websocket.close_code} {reason}. + + {formatTimeStamp(websocket.timestamp_end)} + +
    + ); } diff --git a/web/src/js/components/Footer.tsx b/web/src/js/components/Footer.tsx index a6f91a537a..dfc364bf13 100644 --- a/web/src/js/components/Footer.tsx +++ b/web/src/js/components/Footer.tsx @@ -1,14 +1,28 @@ -import * as React from 'react' -import {formatSize} from '../utils' -import HideInStatic from '../components/common/HideInStatic' -import {useAppSelector} from "../ducks"; +import * as React from "react"; +import { formatSize } from "../utils"; +import HideInStatic from "../components/common/HideInStatic"; +import { useAppSelector } from "../ducks"; export default function Footer() { - const version = useAppSelector(state => state.backendState.version); + const version = useAppSelector((state) => state.backendState.version); let { - mode, intercept, showhost, upstream_cert, rawtcp, http2, websocket, anticache, anticomp, - stickyauth, stickycookie, stream_large_bodies, listen_host, listen_port, server, ssl_insecure - } = useAppSelector(state => state.options); + mode, + intercept, + showhost, + upstream_cert, + rawtcp, + http2, + websocket, + anticache, + anticomp, + stickyauth, + stickycookie, + stream_large_bodies, + listen_host, + listen_port, + server, + ssl_insecure, + } = useAppSelector((state) => state.options); return (
    @@ -16,54 +30,56 @@ export default function Footer() { {mode.join(",")} )} {intercept && ( - Intercept: {intercept} + + Intercept: {intercept} + )} {ssl_insecure && ( ssl_insecure )} - {showhost && ( - showhost - )} + {showhost && showhost} {!upstream_cert && ( no-upstream-cert )} - {!rawtcp && ( - no-raw-tcp - )} - {!http2 && ( - no-http2 - )} + {!rawtcp && no-raw-tcp} + {!http2 && no-http2} {!websocket && ( no-websocket )} {anticache && ( anticache )} - {anticomp && ( - anticomp - )} + {anticomp && anticomp} {stickyauth && ( - stickyauth: {stickyauth} + + stickyauth: {stickyauth} + )} {stickycookie && ( - stickycookie: {stickycookie} + + stickycookie: {stickycookie} + )} {stream_large_bodies && ( - stream: {formatSize(stream_large_bodies)} + + stream: {formatSize(stream_large_bodies)} + )}
    - { - server && ( - - {listen_host || "*"}:{listen_port || 8080} - ) - } + {server && ( + + {listen_host || "*"}:{listen_port || 8080} + + )} - mitmproxy {version} - + mitmproxy {version} +
    - ) + ); } diff --git a/web/src/js/components/Header.tsx b/web/src/js/components/Header.tsx index c00d55ed88..c25c269679 100644 --- a/web/src/js/components/Header.tsx +++ b/web/src/js/components/Header.tsx @@ -1,12 +1,12 @@ -import React, {useState} from 'react' -import classnames from 'classnames' -import StartMenu from './Header/StartMenu' -import OptionMenu from './Header/OptionMenu' -import FileMenu from './Header/FileMenu' -import FlowMenu from './Header/FlowMenu' -import ConnectionIndicator from "./Header/ConnectionIndicator" -import HideInStatic from './common/HideInStatic' -import {useAppSelector} from "../ducks"; +import React, { useState } from "react"; +import classnames from "classnames"; +import StartMenu from "./Header/StartMenu"; +import OptionMenu from "./Header/OptionMenu"; +import FileMenu from "./Header/FileMenu"; +import FlowMenu from "./Header/FlowMenu"; +import ConnectionIndicator from "./Header/ConnectionIndicator"; +import HideInStatic from "./common/HideInStatic"; +import { useAppSelector } from "../ducks"; interface Menu { (): JSX.Element; @@ -15,7 +15,9 @@ interface Menu { } export default function Header() { - const selectedFlows = useAppSelector(state => state.flows.selected.filter(id => id in state.flows.byId)), + const selectedFlows = useAppSelector((state) => + state.flows.selected.filter((id) => id in state.flows.byId) + ), [ActiveMenu, setActiveMenu] = useState(() => StartMenu), [wasFlowSelected, setWasFlowSelected] = useState(false); @@ -25,40 +27,42 @@ export default function Header() { setActiveMenu(() => FlowMenu); setWasFlowSelected(true); } - entries.push(FlowMenu) + entries.push(FlowMenu); } else { if (wasFlowSelected) { setWasFlowSelected(false); } if (ActiveMenu === FlowMenu) { - setActiveMenu(() => StartMenu) + setActiveMenu(() => StartMenu); } } function handleClick(active: Menu, e) { - e.preventDefault() - setActiveMenu(() => active) + e.preventDefault(); + setActiveMenu(() => active); } return (
    - +
    - ) + ); } diff --git a/web/src/js/components/Header/ConnectionIndicator.tsx b/web/src/js/components/Header/ConnectionIndicator.tsx index e3246d6784..29823481d2 100644 --- a/web/src/js/components/Header/ConnectionIndicator.tsx +++ b/web/src/js/components/Header/ConnectionIndicator.tsx @@ -1,27 +1,40 @@ import * as React from "react"; -import {ConnectionState} from "../../ducks/connection" -import {useAppSelector} from "../../ducks"; - +import { ConnectionState } from "../../ducks/connection"; +import { useAppSelector } from "../../ducks"; export default React.memo(function ConnectionIndicator() { - - const connState = useAppSelector(state => state.connection.state), - message = useAppSelector(state => state.connection.message) + const connState = useAppSelector((state) => state.connection.state), + message = useAppSelector((state) => state.connection.message); switch (connState) { case ConnectionState.INIT: - return connecting…; + return ( + connecting… + ); case ConnectionState.FETCHING: - return fetching data…; + return ( + + fetching data… + + ); case ConnectionState.ESTABLISHED: - return connected; + return ( + + connected + + ); case ConnectionState.ERROR: - return connection lost; + return ( + + connection lost + + ); case ConnectionState.OFFLINE: - return offline; + return ( + offline + ); default: const exhaustiveCheck: never = connState; throw "unknown connection state"; } -}) +}); diff --git a/web/src/js/components/Header/FileMenu.tsx b/web/src/js/components/Header/FileMenu.tsx index e5fdbe9ba1..f7e5bc7c1f 100644 --- a/web/src/js/components/Header/FileMenu.tsx +++ b/web/src/js/components/Header/FileMenu.tsx @@ -1,47 +1,62 @@ -import * as React from "react" -import {useDispatch} from 'react-redux' -import FileChooser from '../common/FileChooser' -import Dropdown, {Divider, MenuItem} from '../common/Dropdown' -import * as flowsActions from '../../ducks/flows' +import * as React from "react"; +import { useDispatch } from "react-redux"; +import FileChooser from "../common/FileChooser"; +import Dropdown, { Divider, MenuItem } from "../common/Dropdown"; +import * as flowsActions from "../../ducks/flows"; import HideInStatic from "../common/HideInStatic"; -import {useAppSelector} from "../../ducks"; - +import { useAppSelector } from "../../ducks"; export default React.memo(function FileMenu() { - const dispatch = useDispatch(), filter = useAppSelector(state => state.flows.filter); + const dispatch = useDispatch(), + filter = useAppSelector((state) => state.flows.filter); return ( - +
  • e.stopPropagation() + (e) => e.stopPropagation() } - onOpenFile={file => { + onOpenFile={(file) => { dispatch(flowsActions.upload(file)); document.body.click(); // "restart" event propagation }} />
    • - location.replace('/flows/dump')}> -  Save + location.replace("/flows/dump")}> + +  Save - location.replace('/flows/dump?filter=' + filter)}> -  Save filtered + location.replace("/flows/dump?filter=" + filter)} + > + +  Save filtered - confirm('Delete all flows?') && dispatch(flowsActions.clear())}> -  Clear All + + confirm("Delete all flows?") && + dispatch(flowsActions.clear()) + } + > + +  Clear All - +
  • -  Install Certificates... + +  Install Certificates...
    • - ) + ); }); diff --git a/web/src/js/components/Header/FilterDocs.tsx b/web/src/js/components/Header/FilterDocs.tsx index 5f34b05993..26fb94667e 100644 --- a/web/src/js/components/Header/FilterDocs.tsx +++ b/web/src/js/components/Header/FilterDocs.tsx @@ -1,64 +1,78 @@ -import React, { Component } from 'react' +import React, { Component } from "react"; import { fetchApi } from "../../utils"; type FilterDocsProps = { - selectHandler: (cmd: string) => void, -} + selectHandler: (cmd: string) => void; +}; type FilterDocsStates = { - doc: {commands: string[][]} -} - -export default class FilterDocs extends Component { + doc: { commands: string[][] }; +}; +export default class FilterDocs extends Component< + FilterDocsProps, + FilterDocsStates +> { // @todo move to redux - static xhr: Promise | null - static doc: {commands: string[][]} + static xhr: Promise | null; + static doc: { commands: string[][] }; constructor(props, context) { - super(props, context) - this.state = { doc: FilterDocs.doc } + super(props, context); + this.state = { doc: FilterDocs.doc }; } componentDidMount() { if (!FilterDocs.xhr) { - FilterDocs.xhr = fetchApi('/filter-help').then(response => response.json()) + FilterDocs.xhr = fetchApi("/filter-help").then((response) => + response.json() + ); FilterDocs.xhr.catch(() => { - FilterDocs.xhr = null - }) + FilterDocs.xhr = null; + }); } if (!this.state.doc) { - FilterDocs.xhr.then(doc => { - FilterDocs.doc = doc - this.setState({ doc }) - }) + FilterDocs.xhr.then((doc) => { + FilterDocs.doc = doc; + this.setState({ doc }); + }); } } render() { - const { doc } = this.state + const { doc } = this.state; return !doc ? ( - + ) : ( - {doc.commands.map(cmd => ( - this.props.selectHandler(cmd[0].split(" ")[0] + " ")}> - + {doc.commands.map((cmd) => ( + + this.props.selectHandler( + cmd[0].split(" ")[0] + " " + ) + } + > + ))}
    {cmd[0].replace(' ', '\u00a0')}
    {cmd[0].replace(" ", "\u00a0")} {cmd[1]}
    - - -   mitmproxy docs + + +   mitmproxy docs +
    - ) + ); } } diff --git a/web/src/js/components/Header/FilterInput.tsx b/web/src/js/components/Header/FilterInput.tsx index ecafe29dd6..64f17feced 100644 --- a/web/src/js/components/Header/FilterInput.tsx +++ b/web/src/js/components/Header/FilterInput.tsx @@ -1,123 +1,135 @@ -import React, {Component} from 'react' -import ReactDOM from 'react-dom' -import classnames from 'classnames' -import Filt from '../../filt/filt' -import FilterDocs from './FilterDocs' +import React, { Component } from "react"; +import ReactDOM from "react-dom"; +import classnames from "classnames"; +import Filt from "../../filt/filt"; +import FilterDocs from "./FilterDocs"; type FilterInputProps = { - type: string - color: any - placeholder: string - value: string - onChange: (value) => { type: string, filter?: string, highlight?: string } | void -} + type: string; + color: any; + placeholder: string; + value: string; + onChange: ( + value + ) => { type: string; filter?: string; highlight?: string } | void; +}; type FilterInputState = { - value: string - focus: boolean - mousefocus: boolean -} - -export default class FilterInput extends Component { - + value: string; + focus: boolean; + mousefocus: boolean; +}; + +export default class FilterInput extends Component< + FilterInputProps, + FilterInputState +> { constructor(props, context) { - super(props, context) + super(props, context); // Consider both focus and mouseover for showing/hiding the tooltip, // because onBlur of the input is triggered before the click on the tooltip // finalized, hiding the tooltip just as the user clicks on it. - this.state = {value: this.props.value, focus: false, mousefocus: false} - - this.onChange = this.onChange.bind(this) - this.onFocus = this.onFocus.bind(this) - this.onBlur = this.onBlur.bind(this) - this.onKeyDown = this.onKeyDown.bind(this) - this.onMouseEnter = this.onMouseEnter.bind(this) - this.onMouseLeave = this.onMouseLeave.bind(this) - this.selectFilter = this.selectFilter.bind(this) + this.state = { + value: this.props.value, + focus: false, + mousefocus: false, + }; + + this.onChange = this.onChange.bind(this); + this.onFocus = this.onFocus.bind(this); + this.onBlur = this.onBlur.bind(this); + this.onKeyDown = this.onKeyDown.bind(this); + this.onMouseEnter = this.onMouseEnter.bind(this); + this.onMouseLeave = this.onMouseLeave.bind(this); + this.selectFilter = this.selectFilter.bind(this); } UNSAFE_componentWillReceiveProps(nextProps) { - this.setState({value: nextProps.value}) + this.setState({ value: nextProps.value }); } isValid(filt) { try { if (filt) { - Filt.parse(filt) + Filt.parse(filt); } - return true + return true; } catch (e) { - return false + return false; } } getDesc() { if (!this.state.value) { - return + return ; } try { - return Filt.parse(this.state.value).desc + return Filt.parse(this.state.value).desc; } catch (e) { - return '' + e + return "" + e; } } onChange(e) { - const value = e.target.value - this.setState({value}) + const value = e.target.value; + this.setState({ value }); // Only propagate valid filters upwards. if (this.isValid(value)) { - this.props.onChange(value) + this.props.onChange(value); } } onFocus() { - this.setState({focus: true}) + this.setState({ focus: true }); } onBlur() { - this.setState({focus: false}) + this.setState({ focus: false }); } onMouseEnter() { - this.setState({mousefocus: true}) + this.setState({ mousefocus: true }); } onMouseLeave() { - this.setState({mousefocus: false}) + this.setState({ mousefocus: false }); } onKeyDown(e) { if (e.key === "Escape" || e.key === "Enter") { - this.blur() + this.blur(); // If closed using ESC/ENTER, hide the tooltip. - this.setState({mousefocus: false}) + this.setState({ mousefocus: false }); } - e.stopPropagation() + e.stopPropagation(); } selectFilter(cmd) { - this.setState({value: cmd}) - ReactDOM.findDOMNode(this.refs.input).focus() + this.setState({ value: cmd }); + ReactDOM.findDOMNode(this.refs.input).focus(); } blur() { - ReactDOM.findDOMNode(this.refs.input).blur() + ReactDOM.findDOMNode(this.refs.input).blur(); } select() { - ReactDOM.findDOMNode(this.refs.input).select() + ReactDOM.findDOMNode(this.refs.input).select(); } render() { - const {type, color, placeholder} = this.props - const {value, focus, mousefocus} = this.state + const { type, color, placeholder } = this.props; + const { value, focus, mousefocus } = this.state; return ( -
    +
    - + {(focus || mousefocus) && ( -
    -
    -
    - {this.getDesc()} -
    +
    +
    +
    {this.getDesc()}
    )}
    - ) + ); } } diff --git a/web/src/js/components/Header/FlowMenu.tsx b/web/src/js/components/Header/FlowMenu.tsx index 9d6e6cceb1..cbfa7b456a 100644 --- a/web/src/js/components/Header/FlowMenu.tsx +++ b/web/src/js/components/Header/FlowMenu.tsx @@ -1,53 +1,66 @@ import * as React from "react"; -import Button from "../common/Button" -import {canReplay, MessageUtils} from "../../flow/utils" +import Button from "../common/Button"; +import { canReplay, MessageUtils } from "../../flow/utils"; import HideInStatic from "../common/HideInStatic"; -import {useAppDispatch, useAppSelector} from "../../ducks"; -import * as flowActions from "../../ducks/flows" +import { useAppDispatch, useAppSelector } from "../../ducks"; +import * as flowActions from "../../ducks/flows"; import { duplicate as duplicateFlow, kill as killFlow, remove as removeFlow, replay as replayFlow, resume as resumeFlow, - revert as revertFlow -} from "../../ducks/flows" -import Dropdown, {MenuItem} from "../common/Dropdown"; -import {copy} from "../../flow/export"; -import {Flow} from "../../flow"; + revert as revertFlow, +} from "../../ducks/flows"; +import Dropdown, { MenuItem } from "../common/Dropdown"; +import { copy } from "../../flow/export"; +import { Flow } from "../../flow"; -FlowMenu.title = 'Flow' +FlowMenu.title = "Flow"; export default function FlowMenu(): JSX.Element { const dispatch = useAppDispatch(), - flow = useAppSelector(state => state.flows.byId[state.flows.selected[0]]) + flow = useAppSelector( + (state) => state.flows.byId[state.flows.selected[0]] + ); - if (!flow) - return
    + if (!flow) return
    ; return (
    - - - - - +
    Flow Modification
    @@ -55,8 +68,8 @@ export default function FlowMenu(): JSX.Element {
    - - + +
    Export
    @@ -64,12 +77,20 @@ export default function FlowMenu(): JSX.Element {
    - -
    @@ -77,60 +98,118 @@ export default function FlowMenu(): JSX.Element {
    - ) + ); } // Reference: https://stackoverflow.com/a/63627688/9921431 const openInNewTab = (url) => { - const newWindow = window.open(url, '_blank', 'noopener,noreferrer') - if (newWindow) newWindow.opener = null -} + const newWindow = window.open(url, "_blank", "noopener,noreferrer"); + if (newWindow) newWindow.opener = null; +}; -function DownloadButton({flow}: { flow: Flow }) { +function DownloadButton({ flow }: { flow: Flow }) { if (flow.type !== "http") - return ; + return ( + + ); if (flow.request.contentLength && !flow.response?.contentLength) { - return + return ( + + ); } if (flow.response) { const response = flow.response; if (!flow.request.contentLength && flow.response.contentLength) { - return + return ( + + ); } if (flow.request.contentLength && flow.response.contentLength) { - return 1}>Download▾ - } options={{"placement": "bottom-start"}}> - openInNewTab(MessageUtils.getContentURL(flow, flow.request))}>Download - request - openInNewTab(MessageUtils.getContentURL(flow, response))}>Download - response - + return ( + 1}> + Download▾ + + } + options={{ placement: "bottom-start" }} + > + + openInNewTab( + MessageUtils.getContentURL(flow, flow.request) + ) + } + > + Download request + + + openInNewTab( + MessageUtils.getContentURL(flow, response) + ) + } + > + Download response + + + ); } } return null; } -function ExportButton({flow}: { flow: Flow }) { - return 1} - disabled={flow.type !== "http"}>Export▾ - } options={{"placement": "bottom-start"}}> - copy(flow, "raw_request")}>Copy raw request - copy(flow, "raw_response")}>Copy raw response - copy(flow, "raw")}>Copy raw request and response - copy(flow, "curl")}>Copy as cURL - copy(flow, "httpie")}>Copy as HTTPie - +function ExportButton({ flow }: { flow: Flow }) { + return ( + 1} + disabled={flow.type !== "http"} + > + Export▾ + + } + options={{ placement: "bottom-start" }} + > + copy(flow, "raw_request")}> + Copy raw request + + copy(flow, "raw_response")}> + Copy raw response + + copy(flow, "raw")}> + Copy raw request and response + + copy(flow, "curl")}>Copy as cURL + copy(flow, "httpie")}> + Copy as HTTPie + + + ); } - const markers = { ":red_circle:": "🔴", ":orange_circle:": "🟠", @@ -139,21 +218,41 @@ const markers = { ":large_blue_circle:": "🔵", ":purple_circle:": "🟣", ":brown_circle:": "🟤", -} +}; -function MarkButton({flow}: { flow: Flow }) { +function MarkButton({ flow }: { flow: Flow }) { const dispatch = useAppDispatch(); - return 1}>Mark▾ - } options={{"placement": "bottom-start"}}> - dispatch(flowActions.update(flow, {marked: ""}))}>⚪ (no - marker) - {Object.entries(markers).map(([name, sym]) => + return ( + 1} + > + Mark▾ + + } + options={{ placement: "bottom-start" }} + > dispatch(flowActions.update(flow, {marked: name}))}> - {sym} {name.replace(/[:_]/g, " ")} + onClick={() => + dispatch(flowActions.update(flow, { marked: "" })) + } + > + ⚪ (no marker) - )} - + {Object.entries(markers).map(([name, sym]) => ( + + dispatch(flowActions.update(flow, { marked: name })) + } + > + {sym} {name.replace(/[:_]/g, " ")} + + ))} + + ); } diff --git a/web/src/js/components/Header/MenuToggle.tsx b/web/src/js/components/Header/MenuToggle.tsx index da0b89539a..4a6b3b3b0d 100644 --- a/web/src/js/components/Header/MenuToggle.tsx +++ b/web/src/js/components/Header/MenuToggle.tsx @@ -1,38 +1,35 @@ import * as React from "react"; -import {useDispatch} from "react-redux" -import * as eventLogActions from "../../ducks/eventLog" -import * as commandBarActions from "../../ducks/commandBar" -import {useAppDispatch, useAppSelector} from "../../ducks" -import * as optionsActions from "../../ducks/options" - +import { useDispatch } from "react-redux"; +import * as eventLogActions from "../../ducks/eventLog"; +import * as commandBarActions from "../../ducks/commandBar"; +import { useAppDispatch, useAppSelector } from "../../ducks"; +import * as optionsActions from "../../ducks/options"; type MenuToggleProps = { - value: boolean - onChange: (e: React.ChangeEvent) => void - children: React.ReactNode -} + value: boolean; + onChange: (e: React.ChangeEvent) => void; + children: React.ReactNode; +}; -export function MenuToggle({value, onChange, children}: MenuToggleProps) { +export function MenuToggle({ value, onChange, children }: MenuToggleProps) { return (
    - ) + ); } type OptionsToggleProps = { - name: optionsActions.Option, - children: React.ReactNode -} + name: optionsActions.Option; + children: React.ReactNode; +}; -export function OptionsToggle({name, children}: OptionsToggleProps) { +export function OptionsToggle({ name, children }: OptionsToggleProps) { const dispatch = useAppDispatch(), - value = useAppSelector(state => state.options[name]); + value = useAppSelector((state) => state.options[name]); return ( {children} - ) + ); } - export function EventlogToggle() { const dispatch = useDispatch(), - visible = useAppSelector(state => state.eventLog.visible); + visible = useAppSelector((state) => state.eventLog.visible); return ( Display Event Log - ) + ); } export function CommandBarToggle() { const dispatch = useDispatch(), - visible = useAppSelector(state => state.commandBar.visible); + visible = useAppSelector((state) => state.commandBar.visible); return ( Display Command Bar - ) + ); } diff --git a/web/src/js/components/Header/OptionMenu.tsx b/web/src/js/components/Header/OptionMenu.tsx index 2932a1254c..7fdc3466f2 100644 --- a/web/src/js/components/Header/OptionMenu.tsx +++ b/web/src/js/components/Header/OptionMenu.tsx @@ -1,24 +1,27 @@ import * as React from "react"; -import {CommandBarToggle, EventlogToggle, OptionsToggle} from "./MenuToggle" -import Button from "../common/Button" -import DocsLink from "../common/DocsLink" +import { CommandBarToggle, EventlogToggle, OptionsToggle } from "./MenuToggle"; +import Button from "../common/Button"; +import DocsLink from "../common/DocsLink"; import HideInStatic from "../common/HideInStatic"; -import * as modalActions from "../../ducks/ui/modal" +import * as modalActions from "../../ducks/ui/modal"; import { useAppDispatch } from "../../ducks"; -OptionMenu.title = 'Options' +OptionMenu.title = "Options"; export default function OptionMenu() { - const dispatch = useAppDispatch() - const openOptions = () => modalActions.setActiveModal('OptionModal') + const dispatch = useAppDispatch(); + const openOptions = () => modalActions.setActiveModal("OptionModal"); return (
    -
    @@ -28,7 +31,8 @@ export default function OptionMenu() {
    - Strip cache headers + Strip cache headers{" "} + Use host header for display @@ -43,11 +47,11 @@ export default function OptionMenu() {
    - - + +
    View Options
    - ) + ); } diff --git a/web/src/js/components/Header/StartMenu.tsx b/web/src/js/components/Header/StartMenu.tsx index 0cc363482e..58f57e3e3d 100644 --- a/web/src/js/components/Header/StartMenu.tsx +++ b/web/src/js/components/Header/StartMenu.tsx @@ -1,78 +1,87 @@ import * as React from "react"; -import FilterInput from "./FilterInput" -import * as flowsActions from "../../ducks/flows" -import {setFilter, setHighlight} from "../../ducks/flows" -import Button from "../common/Button" -import {update as updateOptions} from "../../ducks/options"; -import {useAppDispatch, useAppSelector} from "../../ducks"; +import FilterInput from "./FilterInput"; +import * as flowsActions from "../../ducks/flows"; +import { setFilter, setHighlight } from "../../ducks/flows"; +import Button from "../common/Button"; +import { update as updateOptions } from "../../ducks/options"; +import { useAppDispatch, useAppSelector } from "../../ducks"; -StartMenu.title = "Start" +StartMenu.title = "Start"; export default function StartMenu() { return (
    - - + +
    Find
    - - + +
    Intercept
    - ) + ); } function InterceptInput() { const dispatch = useAppDispatch(), - value = useAppSelector(state => state.options.intercept) - return dispatch(updateOptions("intercept", val))} - /> + value = useAppSelector((state) => state.options.intercept); + return ( + dispatch(updateOptions("intercept", val))} + /> + ); } function FlowFilterInput() { const dispatch = useAppDispatch(), - value = useAppSelector(state => state.flows.filter) - return dispatch(setFilter(value))} - /> + value = useAppSelector((state) => state.flows.filter); + return ( + dispatch(setFilter(value))} + /> + ); } function HighlightInput() { const dispatch = useAppDispatch(), - value = useAppSelector(state => state.flows.highlight) - return dispatch(setHighlight(value))} - /> + value = useAppSelector((state) => state.flows.highlight); + return ( + dispatch(setHighlight(value))} + /> + ); } - export function ResumeAll() { const dispatch = useAppDispatch(); return ( - - ) + ); } diff --git a/web/src/js/components/MainView.tsx b/web/src/js/components/MainView.tsx index 7c8644af70..c430156ec3 100644 --- a/web/src/js/components/MainView.tsx +++ b/web/src/js/components/MainView.tsx @@ -1,19 +1,21 @@ -import * as React from "react" -import Splitter from './common/Splitter' -import FlowTable from './FlowTable' -import FlowView from './FlowView' -import {useAppSelector} from "../ducks"; +import * as React from "react"; +import Splitter from "./common/Splitter"; +import FlowTable from "./FlowTable"; +import FlowView from "./FlowView"; +import { useAppSelector } from "../ducks"; import CaptureSetup from "./CaptureSetup"; export default function MainView() { - const hasSelection = useAppSelector(state => !!state.flows.byId[state.flows.selected[0]]) - const hasFlows = useAppSelector(state => state.flows.list.length > 0); + const hasSelection = useAppSelector( + (state) => !!state.flows.byId[state.flows.selected[0]] + ); + const hasFlows = useAppSelector((state) => state.flows.list.length > 0); return (
    - {hasFlows ? : } + {hasFlows ? : } - {hasSelection && } - {hasSelection && } + {hasSelection && } + {hasSelection && }
    - ) + ); } diff --git a/web/src/js/components/Modal/Modal.tsx b/web/src/js/components/Modal/Modal.tsx index d9a475eb91..e5cd230346 100644 --- a/web/src/js/components/Modal/Modal.tsx +++ b/web/src/js/components/Modal/Modal.tsx @@ -1,13 +1,14 @@ -import * as React from "react" -import ModalList from './ModalList' +import * as React from "react"; +import ModalList from "./ModalList"; import { useAppSelector } from "../../ducks"; - export default function PureModal() { - const activeModal : string = useAppSelector(state => state.ui.modal.activeModal) - const ActiveModal:(() => JSX.Element) | undefined= ModalList.find(m => m.name === activeModal ) + const activeModal: string = useAppSelector( + (state) => state.ui.modal.activeModal + ); + const ActiveModal: (() => JSX.Element) | undefined = ModalList.find( + (m) => m.name === activeModal + ); - return( - activeModal&&ActiveModal!==undefined ? :
    - ) + return activeModal && ActiveModal !== undefined ? :
    ; } diff --git a/web/src/js/components/Modal/ModalLayout.tsx b/web/src/js/components/Modal/ModalLayout.tsx index dd21aed081..87fe155ae3 100644 --- a/web/src/js/components/Modal/ModalLayout.tsx +++ b/web/src/js/components/Modal/ModalLayout.tsx @@ -1,20 +1,24 @@ -import * as React from "react" +import * as React from "react"; type ModalLayoutProps = { - children: React.ReactNode, -} + children: React.ReactNode; +}; -export default function ModalLayout ({ children}: ModalLayoutProps ) { +export default function ModalLayout({ children }: ModalLayoutProps) { return (
    -
    - - ) + ); } diff --git a/web/src/js/components/Modal/ModalList.tsx b/web/src/js/components/Modal/ModalList.tsx index e1f25199a1..27209d5325 100644 --- a/web/src/js/components/Modal/ModalList.tsx +++ b/web/src/js/components/Modal/ModalList.tsx @@ -1,13 +1,13 @@ -import * as React from "react" -import ModalLayout from './ModalLayout' -import OptionContent from './OptionModal' +import * as React from "react"; +import ModalLayout from "./ModalLayout"; +import OptionContent from "./OptionModal"; function OptionModal() { return ( - + - ) + ); } -export default [ OptionModal ] +export default [OptionModal]; diff --git a/web/src/js/components/Modal/Option.jsx b/web/src/js/components/Modal/Option.jsx index 844fec8593..7ea333139e 100644 --- a/web/src/js/components/Modal/Option.jsx +++ b/web/src/js/components/Modal/Option.jsx @@ -1,150 +1,156 @@ -import React, {Component} from "react" -import PropTypes from "prop-types" -import {connect} from "react-redux" -import {update as updateOptions} from "../../ducks/options" -import classnames from 'classnames' +import React, { Component } from "react"; +import PropTypes from "prop-types"; +import { connect } from "react-redux"; +import { update as updateOptions } from "../../ducks/options"; +import classnames from "classnames"; -const stopPropagation = e => { +const stopPropagation = (e) => { if (e.key !== "Escape") { - e.stopPropagation() + e.stopPropagation(); } -} +}; BooleanOption.propTypes = { value: PropTypes.bool.isRequired, onChange: PropTypes.func.isRequired, -} +}; -function BooleanOption({value, onChange, ...props}) { +function BooleanOption({ value, onChange, ...props }) { return (
    - ) + ); } StringOption.propTypes = { value: PropTypes.string, onChange: PropTypes.func.isRequired, -} +}; -function StringOption({value, onChange, ...props}) { +function StringOption({ value, onChange, ...props }) { return ( - onChange(e.target.value)} - {...props} + onChange(e.target.value)} + {...props} /> - ) + ); } function Optional(Component) { - return function ({onChange, ...props}) { - return onChange(x ? x : null)} - {...props} - /> - } + return function ({ onChange, ...props }) { + return ( + onChange(x ? x : null)} {...props} /> + ); + }; } NumberOption.propTypes = { value: PropTypes.number.isRequired, onChange: PropTypes.func.isRequired, -} +}; -function NumberOption({value, onChange, ...props}) { +function NumberOption({ value, onChange, ...props }) { return ( - onChange(parseInt(e.target.value))} - {...props} + onChange(parseInt(e.target.value))} + {...props} /> - ) + ); } ChoicesOption.propTypes = { value: PropTypes.string.isRequired, onChange: PropTypes.func.isRequired, -} +}; -export function ChoicesOption({value, onChange, choices, ...props}) { +export function ChoicesOption({ value, onChange, choices, ...props }) { return ( - ) + ); } StringSequenceOption.propTypes = { value: PropTypes.arrayOf(PropTypes.string).isRequired, onChange: PropTypes.func.isRequired, -} +}; -function StringSequenceOption({value, onChange, ...props}) { - const height = Math.max(value.length, 1) - return