From fe1ab2bf905b4eb852adf79bac4e317ccc9754d5 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 11:14:52 -0400 Subject: [PATCH] fix support for flex requests with request body --- .../packer/roles/mitmproxy/files/bubble_flex.py | 14 +++++++++----- .../packer/roles/mitmproxy/files/bubble_modify.py | 11 +++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index f574d2f1..866bd0e5 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -2,8 +2,9 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # from mitmproxy import http -from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, add_flow_ctx, \ - HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH +from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, \ + get_flow_ctx, add_flow_ctx, \ + HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_FILTERED from bubble_modify import bubble_filter_response import requests @@ -51,6 +52,7 @@ def set_flex_response(client_addr, flex_host, flow): def process_flex(flex_host, flow, router): # build the request URL + method = flow.request.method scheme = flow.request.scheme url = scheme + '://' + flex_host + flow.request.path @@ -68,14 +70,15 @@ def process_flex(flex_host, flow, router): proxies = {"http": proxy_url, "https": proxy_url} # send request to flex router + request_body = flow.request.content if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: sending flex request for '+url+' to '+proxy_url+' with headers='+repr(request_headers)) + bubble_log.debug('process_flex: sending flex request for ' + method +' ' + url +' to ' + proxy_url +' with headers=' + repr(request_headers) +' and body=' + repr(request_body)) try: - response = requests.request(flow.request.method, url, + response = requests.request(method, url, headers=request_headers, timeout=(15, 15), stream=True, - data=flow.request.stream, # use the original request body, if there is one + data=request_body, # use the original request body, if there is one proxies=proxies, allow_redirects=False) if bubble_log.isEnabledFor(DEBUG): @@ -121,6 +124,7 @@ def process_flex(flex_host, flow, router): # Apply filters if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('process_flex: bubble filtering: '+url) + # flow.response.stream = lambda chunks: response.iter_content(8192) bubble_filter_response(flow, response) if bubble_log.isEnabledFor(INFO): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 7f851b67..eadbcd6e 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -25,7 +25,7 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__' REDIS_FILTER_PASSTHRU_DURATION = 600 DEBUG_STREAM_COUNTERS = {} -MIN_FILTER_CHUNK_SIZE = 4096 +MIN_FILTER_CHUNK_SIZE = 16384 bubble_log = logging.getLogger(__name__) @@ -138,7 +138,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ chunks is a generator that can be used to iterate over all chunks. """ if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_filter_chunks: starting...') + bubble_log.debug('bubble_filter_chunks: starting with content_type='+content_type) first = True last = False content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) @@ -171,7 +171,10 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ # send whatever is left in the buffer if len(buffer) > 0: # bubble_log.debug('bubble_filter_chunks(end): sending remainder buffer of size '+str(len(buffer))) - yield filter_chunk(flow, buffer, req_id, user_agent, last) + if first: + yield filter_chunk(flow, buffer, req_id, user_agent, last, content_encoding, content_type, content_length, csp) + else: + yield filter_chunk(flow, buffer, req_id, user_agent, last) if not content_length or not last: # bubble_log.debug('bubble_filter_chunks(end): sending last empty chunk') yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data @@ -184,7 +187,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ def bubble_modify(flow, flex_stream, req_id, user_agent, content_encoding, content_type, csp): if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_modify: modifying req_id='+req_id) + bubble_log.debug('bubble_modify: modifying req_id='+req_id+' with content_type='+content_type) return lambda chunks: bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_encoding, content_type, csp)