diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py new file mode 100644 index 00000000..d5cfdf37 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py @@ -0,0 +1,307 @@ +# High level events that make up HTTP/1.1 conversations. Loosely inspired by +# the corresponding events in hyper-h2: +# +# http://python-hyper.org/h2/en/stable/api.html#events +# +# Don't subclass these. Stuff will break. + +import re + +from . import _headers +from ._abnf import request_target +from ._util import bytesify, LocalProtocolError, validate + +# Everything in __all__ gets re-exported as part of the h11 public API. +__all__ = [ + "Request", + "InformationalResponse", + "Response", + "Data", + "EndOfMessage", + "ConnectionClosed", +] + +request_target_re = re.compile(request_target.encode("ascii")) + + +class _EventBundle(object): + _fields = [] + _defaults = {} + + def __init__(self, **kwargs): + _parsed = kwargs.pop("_parsed", False) + allowed = set(self._fields) + for kwarg in kwargs: + if kwarg not in allowed: + raise TypeError( + "unrecognized kwarg {} for {}".format( + kwarg, self.__class__.__name__ + ) + ) + required = allowed.difference(self._defaults) + for field in required: + if field not in kwargs: + raise TypeError( + "missing required kwarg {} for {}".format( + field, self.__class__.__name__ + ) + ) + self.__dict__.update(self._defaults) + self.__dict__.update(kwargs) + + # Special handling for some fields + + if "headers" in self.__dict__: + self.headers = _headers.normalize_and_validate( + self.headers, _parsed=_parsed + ) + + if not _parsed: + for field in ["method", "target", "http_version", "reason"]: + if field in self.__dict__: + self.__dict__[field] = bytesify(self.__dict__[field]) + + if "status_code" in self.__dict__: + if not isinstance(self.status_code, int): + raise LocalProtocolError("status code must be integer") + # Because IntEnum objects are instances of int, but aren't + # duck-compatible (sigh), see gh-72. + self.status_code = int(self.status_code) + + self._validate() + + def _validate(self): + pass + + def __repr__(self): + name = self.__class__.__name__ + kwarg_strs = [ + "{}={}".format(field, self.__dict__[field]) for field in self._fields + ] + kwarg_str = ", ".join(kwarg_strs) + return "{}({})".format(name, kwarg_str) + + # Useful for tests + def __eq__(self, other): + return self.__class__ == other.__class__ and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not self.__eq__(other) + + # This is an unhashable type. + __hash__ = None + + +class Request(_EventBundle): + """The beginning of an HTTP request. + + Fields: + + .. attribute:: method + + An HTTP method, e.g. ``b"GET"`` or ``b"POST"``. Always a byte + string. :term:`Bytes-like objects ` and native + strings containing only ascii characters will be automatically + converted to byte strings. + + .. attribute:: target + + The target of an HTTP request, e.g. ``b"/index.html"``, or one of the + more exotic formats described in `RFC 7320, section 5.3 + `_. Always a byte + string. :term:`Bytes-like objects ` and native + strings containing only ascii characters will be automatically + converted to byte strings. + + .. attribute:: headers + + Request headers, represented as a list of (name, value) pairs. See + :ref:`the header normalization rules ` for details. + + .. attribute:: http_version + + The HTTP protocol version, represented as a byte string like + ``b"1.1"``. See :ref:`the HTTP version normalization rules + ` for details. + + """ + + _fields = ["method", "target", "headers", "http_version"] + _defaults = {"http_version": b"1.1"} + + def _validate(self): + # "A server MUST respond with a 400 (Bad Request) status code to any + # HTTP/1.1 request message that lacks a Host header field and to any + # request message that contains more than one Host header field or a + # Host header field with an invalid field-value." + # -- https://tools.ietf.org/html/rfc7230#section-5.4 + host_count = 0 + for name, value in self.headers: + if name == b"host": + host_count += 1 + if self.http_version == b"1.1" and host_count == 0: + raise LocalProtocolError("Missing mandatory Host: header") + if host_count > 1: + raise LocalProtocolError("Found multiple Host: headers") + + validate(request_target_re, self.target, "Illegal target characters") + + +class _ResponseBase(_EventBundle): + _fields = ["status_code", "headers", "http_version", "reason"] + _defaults = {"http_version": b"1.1", "reason": b""} + + +class InformationalResponse(_ResponseBase): + """An HTTP informational response. + + Fields: + + .. attribute:: status_code + + The status code of this response, as an integer. For an + :class:`InformationalResponse`, this is always in the range [100, + 200). + + .. attribute:: headers + + Request headers, represented as a list of (name, value) pairs. See + :ref:`the header normalization rules ` for + details. + + .. attribute:: http_version + + The HTTP protocol version, represented as a byte string like + ``b"1.1"``. See :ref:`the HTTP version normalization rules + ` for details. + + .. attribute:: reason + + The reason phrase of this response, as a byte string. For example: + ``b"OK"``, or ``b"Not Found"``. + + """ + + def _validate(self): + if not (100 <= self.status_code < 200): + raise LocalProtocolError( + "InformationalResponse status_code should be in range " + "[100, 200), not {}".format(self.status_code) + ) + + +class Response(_ResponseBase): + """The beginning of an HTTP response. + + Fields: + + .. attribute:: status_code + + The status code of this response, as an integer. For an + :class:`Response`, this is always in the range [200, + 600). + + .. attribute:: headers + + Request headers, represented as a list of (name, value) pairs. See + :ref:`the header normalization rules ` for details. + + .. attribute:: http_version + + The HTTP protocol version, represented as a byte string like + ``b"1.1"``. See :ref:`the HTTP version normalization rules + ` for details. + + .. attribute:: reason + + The reason phrase of this response, as a byte string. For example: + ``b"OK"``, or ``b"Not Found"``. + + """ + + def _validate(self): + if self.status_code == 999: + self.status_code = 200 + if not (200 <= self.status_code < 600): + raise LocalProtocolError( + "Response status_code should be in range [200, 600), not {}".format( + self.status_code + ) + ) + + +class Data(_EventBundle): + """Part of an HTTP message body. + + Fields: + + .. attribute:: data + + A :term:`bytes-like object` containing part of a message body. Or, if + using the ``combine=False`` argument to :meth:`Connection.send`, then + any object that your socket writing code knows what to do with, and for + which calling :func:`len` returns the number of bytes that will be + written -- see :ref:`sendfile` for details. + + .. attribute:: chunk_start + + A marker that indicates whether this data object is from the start of a + chunked transfer encoding chunk. This field is ignored when when a Data + event is provided to :meth:`Connection.send`: it is only valid on + events emitted from :meth:`Connection.next_event`. You probably + shouldn't use this attribute at all; see + :ref:`chunk-delimiters-are-bad` for details. + + .. attribute:: chunk_end + + A marker that indicates whether this data object is the last for a + given chunked transfer encoding chunk. This field is ignored when when + a Data event is provided to :meth:`Connection.send`: it is only valid + on events emitted from :meth:`Connection.next_event`. You probably + shouldn't use this attribute at all; see + :ref:`chunk-delimiters-are-bad` for details. + + """ + + _fields = ["data", "chunk_start", "chunk_end"] + _defaults = {"chunk_start": False, "chunk_end": False} + + +# XX FIXME: "A recipient MUST ignore (or consider as an error) any fields that +# are forbidden to be sent in a trailer, since processing them as if they were +# present in the header section might bypass external security filters." +# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#chunked.trailer.part +# Unfortunately, the list of forbidden fields is long and vague :-/ +class EndOfMessage(_EventBundle): + """The end of an HTTP message. + + Fields: + + .. attribute:: headers + + Default value: ``[]`` + + Any trailing headers attached to this message, represented as a list of + (name, value) pairs. See :ref:`the header normalization rules + ` for details. + + Must be empty unless ``Transfer-Encoding: chunked`` is in use. + + """ + + _fields = ["headers"] + _defaults = {"headers": []} + + +class ConnectionClosed(_EventBundle): + """This event indicates that the sender has closed their outgoing + connection. + + Note that this does not necessarily mean that they can't *receive* further + data, because TCP connections are composed to two one-way channels which + can be closed independently. See :ref:`closing` for details. + + No fields. + """ + + pass diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 73aa28b1..10cdd108 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -141,7 +141,7 @@ def async_stream(client, name, url, timeout=timeout, max_redirects=max_redirects)) except Exception as e: - bubble_log.error('async_stream('+name+'): error with url='+url+' -- '+repr(e)) + bubble_log.error('async_stream('+name+'): error with url='+url+' -- '+repr(e)+' from '+get_stack(e)) raise e diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index fe54c91b..2f3267c6 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -42,7 +42,7 @@ def add_csp_part(new_csp, part): return new_csp + part -def ensure_bubble_script_csp(csp): +def ensure_bubble_csp(csp, req_id): new_csp = '' parts = csp.split(';') for part in parts: @@ -53,6 +53,21 @@ def ensure_bubble_script_csp(csp): continue new_csp = add_csp_part(new_csp, tokens[0] + " 'self' " + " ".join(tokens[1:])) + elif part.startswith(' script-src ') or part.startswith('script-src '): + tokens = part.split() + if "'self'" in tokens: + # allows from self, check if there is an existing nonce. if so we will reuse it + found_nonce = False + for token in tokens: + if " 'nonce-" in token: + found_nonce = True + break + # if no nonce, then add our nonce + if not found_nonce: + new_csp = add_csp_part(new_csp, " ".join(tokens) + " 'nonce="+req_id+"'") + else: + # does not allow from self, so add self with our nonce + new_csp = add_csp_part(new_csp, tokens[0] + " 'self' 'nonce="+req_id+"'" + " ".join(tokens[1:])) else: new_csp = add_csp_part(new_csp, part) return new_csp @@ -371,7 +386,7 @@ def bubble_filter_response(flow, flex_flow): content_encoding = None if HEADER_CONTENT_SECURITY_POLICY in flow.response.headers: - csp = ensure_bubble_script_csp(flow.response.headers[HEADER_CONTENT_SECURITY_POLICY]) + csp = ensure_bubble_csp(flow.response.headers[HEADER_CONTENT_SECURITY_POLICY], req_id) flow.response.headers[HEADER_CONTENT_SECURITY_POLICY] = csp else: csp = None diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml index f77dab6b..3dd6a551 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml @@ -92,13 +92,20 @@ - name: Install mitmproxy dependencies shell: su - mitmproxy -c "bash -c 'cd /home/mitmproxy/mitmproxy && ./dev.sh'" -- name: Overwrite _client.py from httpx to fix bug with HTTP/2 redirects +- name: Patch _client.py from httpx to fix bug with HTTP/2 redirects file: src: _client.py dest: /home/mitmproxy/mitmproxy/venv/lib/python3.8/site-packages/httpx/_client.py owner: mitmproxy group: mitmproxy +- name: Patch _events.py from h11 to fix bug with HTTP status 999 being considered invalid + file: + src: _events.py + dest: /home/mitmproxy/mitmproxy/venv/lib/python3.8/site-packages/h11/_events.py + owner: mitmproxy + group: mitmproxy + - name: Install mitm_monitor copy: src: "mitm_monitor.sh"