fix support for flex requests with request body

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py

@@ -2,8 +2,9 @@
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
 from mitmproxy import http
-from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, add_flow_ctx, \
-    HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH
+from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, \
+    get_flow_ctx, add_flow_ctx, \
+    HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_FILTERED
 from bubble_modify import bubble_filter_response
 
 import requests
@@ -51,6 +52,7 @@ def set_flex_response(client_addr, flex_host, flow):
 def process_flex(flex_host, flow, router):
 
     # build the request URL
+    method = flow.request.method
     scheme = flow.request.scheme
     url = scheme + '://' + flex_host + flow.request.path
 
@@ -68,14 +70,15 @@ def process_flex(flex_host, flow, router):
     proxies = {"http": proxy_url, "https": proxy_url}
 
     # send request to flex router
+    request_body = flow.request.content
     if bubble_log.isEnabledFor(DEBUG):
-        bubble_log.debug('process_flex: sending flex request for '+url+' to '+proxy_url+' with headers='+repr(request_headers))
+        bubble_log.debug('process_flex: sending flex request for ' + method +' ' + url +' to ' + proxy_url +' with headers=' + repr(request_headers) +' and body=' + repr(request_body))
     try:
-        response = requests.request(flow.request.method, url,
+        response = requests.request(method, url,
                                     headers=request_headers,
                                     timeout=(15, 15),
                                     stream=True,
-                                    data=flow.request.stream,  # use the original request body, if there is one
+                                    data=request_body,  # use the original request body, if there is one
                                     proxies=proxies,
                                     allow_redirects=False)
         if bubble_log.isEnabledFor(DEBUG):
@@ -121,6 +124,7 @@ def process_flex(flex_host, flow, router):
     # Apply filters
     if bubble_log.isEnabledFor(DEBUG):
         bubble_log.debug('process_flex: bubble filtering: '+url)
+    # flow.response.stream = lambda chunks: response.iter_content(8192)
     bubble_filter_response(flow, response)
 
     if bubble_log.isEnabledFor(INFO):

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -25,7 +25,7 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__'
 REDIS_FILTER_PASSTHRU_DURATION = 600
 
 DEBUG_STREAM_COUNTERS = {}
-MIN_FILTER_CHUNK_SIZE = 4096
+MIN_FILTER_CHUNK_SIZE = 16384
 
 bubble_log = logging.getLogger(__name__)
 
@@ -138,7 +138,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_
     chunks is a generator that can be used to iterate over all chunks.
     """
     if bubble_log.isEnabledFor(DEBUG):
-        bubble_log.debug('bubble_filter_chunks: starting...')
+        bubble_log.debug('bubble_filter_chunks: starting with content_type='+content_type)
     first = True
     last = False
     content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH)
@@ -171,7 +171,10 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_
         # send whatever is left in the buffer
         if len(buffer) > 0:
             # bubble_log.debug('bubble_filter_chunks(end): sending remainder buffer of size '+str(len(buffer)))
-            yield filter_chunk(flow, buffer, req_id, user_agent, last)
+            if first:
+                yield filter_chunk(flow, buffer, req_id, user_agent, last, content_encoding, content_type, content_length, csp)
+            else:
+                yield filter_chunk(flow, buffer, req_id, user_agent, last)
         if not content_length or not last:
             # bubble_log.debug('bubble_filter_chunks(end): sending last empty chunk')
             yield filter_chunk(flow, None, req_id, user_agent, True)  # get the last bits of data
@@ -184,7 +187,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_
 
 def bubble_modify(flow, flex_stream, req_id, user_agent, content_encoding, content_type, csp):
     if bubble_log.isEnabledFor(DEBUG):
-        bubble_log.debug('bubble_modify: modifying req_id='+req_id)
+        bubble_log.debug('bubble_modify: modifying req_id='+req_id+' with content_type='+content_type)
     return lambda chunks: bubble_filter_chunks(flow, chunks, flex_stream, req_id,
                                                user_agent, content_encoding, content_type, csp)