Преглед на файлове

better handling of script-src in csp. add h11 patch to handle HTTP 999 responses, they are really 200

tags/v1.1.4
Jonathan Cobb преди 4 години
родител
ревизия
bbbb025dda
променени са 4 файла, в които са добавени 333 реда и са изтрити 4 реда
  1. +307
    -0
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py
  2. +1
    -1
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py
  3. +17
    -2
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py
  4. +8
    -1
      bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml

+ 307
- 0
bubble-server/src/main/resources/packer/roles/mitmproxy/files/_events.py Целия файл

@@ -0,0 +1,307 @@
# High level events that make up HTTP/1.1 conversations. Loosely inspired by
# the corresponding events in hyper-h2:
#
# http://python-hyper.org/h2/en/stable/api.html#events
#
# Don't subclass these. Stuff will break.

import re

from . import _headers
from ._abnf import request_target
from ._util import bytesify, LocalProtocolError, validate

# Everything in __all__ gets re-exported as part of the h11 public API.
__all__ = [
"Request",
"InformationalResponse",
"Response",
"Data",
"EndOfMessage",
"ConnectionClosed",
]

request_target_re = re.compile(request_target.encode("ascii"))


class _EventBundle(object):
_fields = []
_defaults = {}

def __init__(self, **kwargs):
_parsed = kwargs.pop("_parsed", False)
allowed = set(self._fields)
for kwarg in kwargs:
if kwarg not in allowed:
raise TypeError(
"unrecognized kwarg {} for {}".format(
kwarg, self.__class__.__name__
)
)
required = allowed.difference(self._defaults)
for field in required:
if field not in kwargs:
raise TypeError(
"missing required kwarg {} for {}".format(
field, self.__class__.__name__
)
)
self.__dict__.update(self._defaults)
self.__dict__.update(kwargs)

# Special handling for some fields

if "headers" in self.__dict__:
self.headers = _headers.normalize_and_validate(
self.headers, _parsed=_parsed
)

if not _parsed:
for field in ["method", "target", "http_version", "reason"]:
if field in self.__dict__:
self.__dict__[field] = bytesify(self.__dict__[field])

if "status_code" in self.__dict__:
if not isinstance(self.status_code, int):
raise LocalProtocolError("status code must be integer")
# Because IntEnum objects are instances of int, but aren't
# duck-compatible (sigh), see gh-72.
self.status_code = int(self.status_code)

self._validate()

def _validate(self):
pass

def __repr__(self):
name = self.__class__.__name__
kwarg_strs = [
"{}={}".format(field, self.__dict__[field]) for field in self._fields
]
kwarg_str = ", ".join(kwarg_strs)
return "{}({})".format(name, kwarg_str)

# Useful for tests
def __eq__(self, other):
return self.__class__ == other.__class__ and self.__dict__ == other.__dict__

def __ne__(self, other):
return not self.__eq__(other)

# This is an unhashable type.
__hash__ = None


class Request(_EventBundle):
"""The beginning of an HTTP request.

Fields:

.. attribute:: method

An HTTP method, e.g. ``b"GET"`` or ``b"POST"``. Always a byte
string. :term:`Bytes-like objects <bytes-like object>` and native
strings containing only ascii characters will be automatically
converted to byte strings.

.. attribute:: target

The target of an HTTP request, e.g. ``b"/index.html"``, or one of the
more exotic formats described in `RFC 7320, section 5.3
<https://tools.ietf.org/html/rfc7230#section-5.3>`_. Always a byte
string. :term:`Bytes-like objects <bytes-like object>` and native
strings containing only ascii characters will be automatically
converted to byte strings.

.. attribute:: headers

Request headers, represented as a list of (name, value) pairs. See
:ref:`the header normalization rules <headers-format>` for details.

.. attribute:: http_version

The HTTP protocol version, represented as a byte string like
``b"1.1"``. See :ref:`the HTTP version normalization rules
<http_version-format>` for details.

"""

_fields = ["method", "target", "headers", "http_version"]
_defaults = {"http_version": b"1.1"}

def _validate(self):
# "A server MUST respond with a 400 (Bad Request) status code to any
# HTTP/1.1 request message that lacks a Host header field and to any
# request message that contains more than one Host header field or a
# Host header field with an invalid field-value."
# -- https://tools.ietf.org/html/rfc7230#section-5.4
host_count = 0
for name, value in self.headers:
if name == b"host":
host_count += 1
if self.http_version == b"1.1" and host_count == 0:
raise LocalProtocolError("Missing mandatory Host: header")
if host_count > 1:
raise LocalProtocolError("Found multiple Host: headers")

validate(request_target_re, self.target, "Illegal target characters")


class _ResponseBase(_EventBundle):
_fields = ["status_code", "headers", "http_version", "reason"]
_defaults = {"http_version": b"1.1", "reason": b""}


class InformationalResponse(_ResponseBase):
"""An HTTP informational response.

Fields:

.. attribute:: status_code

The status code of this response, as an integer. For an
:class:`InformationalResponse`, this is always in the range [100,
200).

.. attribute:: headers

Request headers, represented as a list of (name, value) pairs. See
:ref:`the header normalization rules <headers-format>` for
details.

.. attribute:: http_version

The HTTP protocol version, represented as a byte string like
``b"1.1"``. See :ref:`the HTTP version normalization rules
<http_version-format>` for details.

.. attribute:: reason

The reason phrase of this response, as a byte string. For example:
``b"OK"``, or ``b"Not Found"``.

"""

def _validate(self):
if not (100 <= self.status_code < 200):
raise LocalProtocolError(
"InformationalResponse status_code should be in range "
"[100, 200), not {}".format(self.status_code)
)


class Response(_ResponseBase):
"""The beginning of an HTTP response.

Fields:

.. attribute:: status_code

The status code of this response, as an integer. For an
:class:`Response`, this is always in the range [200,
600).

.. attribute:: headers

Request headers, represented as a list of (name, value) pairs. See
:ref:`the header normalization rules <headers-format>` for details.

.. attribute:: http_version

The HTTP protocol version, represented as a byte string like
``b"1.1"``. See :ref:`the HTTP version normalization rules
<http_version-format>` for details.

.. attribute:: reason

The reason phrase of this response, as a byte string. For example:
``b"OK"``, or ``b"Not Found"``.

"""

def _validate(self):
if self.status_code == 999:
self.status_code = 200
if not (200 <= self.status_code < 600):
raise LocalProtocolError(
"Response status_code should be in range [200, 600), not {}".format(
self.status_code
)
)


class Data(_EventBundle):
"""Part of an HTTP message body.

Fields:

.. attribute:: data

A :term:`bytes-like object` containing part of a message body. Or, if
using the ``combine=False`` argument to :meth:`Connection.send`, then
any object that your socket writing code knows what to do with, and for
which calling :func:`len` returns the number of bytes that will be
written -- see :ref:`sendfile` for details.

.. attribute:: chunk_start

A marker that indicates whether this data object is from the start of a
chunked transfer encoding chunk. This field is ignored when when a Data
event is provided to :meth:`Connection.send`: it is only valid on
events emitted from :meth:`Connection.next_event`. You probably
shouldn't use this attribute at all; see
:ref:`chunk-delimiters-are-bad` for details.

.. attribute:: chunk_end

A marker that indicates whether this data object is the last for a
given chunked transfer encoding chunk. This field is ignored when when
a Data event is provided to :meth:`Connection.send`: it is only valid
on events emitted from :meth:`Connection.next_event`. You probably
shouldn't use this attribute at all; see
:ref:`chunk-delimiters-are-bad` for details.

"""

_fields = ["data", "chunk_start", "chunk_end"]
_defaults = {"chunk_start": False, "chunk_end": False}


# XX FIXME: "A recipient MUST ignore (or consider as an error) any fields that
# are forbidden to be sent in a trailer, since processing them as if they were
# present in the header section might bypass external security filters."
# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#chunked.trailer.part
# Unfortunately, the list of forbidden fields is long and vague :-/
class EndOfMessage(_EventBundle):
"""The end of an HTTP message.

Fields:

.. attribute:: headers

Default value: ``[]``

Any trailing headers attached to this message, represented as a list of
(name, value) pairs. See :ref:`the header normalization rules
<headers-format>` for details.

Must be empty unless ``Transfer-Encoding: chunked`` is in use.

"""

_fields = ["headers"]
_defaults = {"headers": []}


class ConnectionClosed(_EventBundle):
"""This event indicates that the sender has closed their outgoing
connection.

Note that this does not necessarily mean that they can't *receive* further
data, because TCP connections are composed to two one-way channels which
can be closed independently. See :ref:`closing` for details.

No fields.
"""

pass

+ 1
- 1
bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py Целия файл

@@ -141,7 +141,7 @@ def async_stream(client, name, url,
timeout=timeout,
max_redirects=max_redirects))
except Exception as e:
bubble_log.error('async_stream('+name+'): error with url='+url+' -- '+repr(e))
bubble_log.error('async_stream('+name+'): error with url='+url+' -- '+repr(e)+' from '+get_stack(e))
raise e




+ 17
- 2
bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py Целия файл

@@ -42,7 +42,7 @@ def add_csp_part(new_csp, part):
return new_csp + part


def ensure_bubble_script_csp(csp):
def ensure_bubble_csp(csp, req_id):
new_csp = ''
parts = csp.split(';')
for part in parts:
@@ -53,6 +53,21 @@ def ensure_bubble_script_csp(csp):
continue
new_csp = add_csp_part(new_csp, tokens[0] + " 'self' " + " ".join(tokens[1:]))

elif part.startswith(' script-src ') or part.startswith('script-src '):
tokens = part.split()
if "'self'" in tokens:
# allows from self, check if there is an existing nonce. if so we will reuse it
found_nonce = False
for token in tokens:
if " 'nonce-" in token:
found_nonce = True
break
# if no nonce, then add our nonce
if not found_nonce:
new_csp = add_csp_part(new_csp, " ".join(tokens) + " 'nonce="+req_id+"'")
else:
# does not allow from self, so add self with our nonce
new_csp = add_csp_part(new_csp, tokens[0] + " 'self' 'nonce="+req_id+"'" + " ".join(tokens[1:]))
else:
new_csp = add_csp_part(new_csp, part)
return new_csp
@@ -371,7 +386,7 @@ def bubble_filter_response(flow, flex_flow):
content_encoding = None

if HEADER_CONTENT_SECURITY_POLICY in flow.response.headers:
csp = ensure_bubble_script_csp(flow.response.headers[HEADER_CONTENT_SECURITY_POLICY])
csp = ensure_bubble_csp(flow.response.headers[HEADER_CONTENT_SECURITY_POLICY], req_id)
flow.response.headers[HEADER_CONTENT_SECURITY_POLICY] = csp
else:
csp = None


+ 8
- 1
bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml Целия файл

@@ -92,13 +92,20 @@
- name: Install mitmproxy dependencies
shell: su - mitmproxy -c "bash -c 'cd /home/mitmproxy/mitmproxy && ./dev.sh'"

- name: Overwrite _client.py from httpx to fix bug with HTTP/2 redirects
- name: Patch _client.py from httpx to fix bug with HTTP/2 redirects
file:
src: _client.py
dest: /home/mitmproxy/mitmproxy/venv/lib/python3.8/site-packages/httpx/_client.py
owner: mitmproxy
group: mitmproxy

- name: Patch _events.py from h11 to fix bug with HTTP status 999 being considered invalid
file:
src: _events.py
dest: /home/mitmproxy/mitmproxy/venv/lib/python3.8/site-packages/h11/_events.py
owner: mitmproxy
group: mitmproxy

- name: Install mitm_monitor
copy:
src: "mitm_monitor.sh"


Зареждане…
Отказ
Запис