@@ -2,8 +2,9 @@ | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
from mitmproxy import http | |||
from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, add_flow_ctx, \ | |||
HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH | |||
from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, \ | |||
get_flow_ctx, add_flow_ctx, \ | |||
HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_FILTERED | |||
from bubble_modify import bubble_filter_response | |||
import requests | |||
@@ -51,6 +52,7 @@ def set_flex_response(client_addr, flex_host, flow): | |||
def process_flex(flex_host, flow, router): | |||
# build the request URL | |||
method = flow.request.method | |||
scheme = flow.request.scheme | |||
url = scheme + '://' + flex_host + flow.request.path | |||
@@ -68,14 +70,15 @@ def process_flex(flex_host, flow, router): | |||
proxies = {"http": proxy_url, "https": proxy_url} | |||
# send request to flex router | |||
request_body = flow.request.content | |||
if bubble_log.isEnabledFor(DEBUG): | |||
bubble_log.debug('process_flex: sending flex request for '+url+' to '+proxy_url+' with headers='+repr(request_headers)) | |||
bubble_log.debug('process_flex: sending flex request for ' + method +' ' + url +' to ' + proxy_url +' with headers=' + repr(request_headers) +' and body=' + repr(request_body)) | |||
try: | |||
response = requests.request(flow.request.method, url, | |||
response = requests.request(method, url, | |||
headers=request_headers, | |||
timeout=(15, 15), | |||
stream=True, | |||
data=flow.request.stream, # use the original request body, if there is one | |||
data=request_body, # use the original request body, if there is one | |||
proxies=proxies, | |||
allow_redirects=False) | |||
if bubble_log.isEnabledFor(DEBUG): | |||
@@ -121,6 +124,7 @@ def process_flex(flex_host, flow, router): | |||
# Apply filters | |||
if bubble_log.isEnabledFor(DEBUG): | |||
bubble_log.debug('process_flex: bubble filtering: '+url) | |||
# flow.response.stream = lambda chunks: response.iter_content(8192) | |||
bubble_filter_response(flow, response) | |||
if bubble_log.isEnabledFor(INFO): | |||
@@ -25,7 +25,7 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__' | |||
REDIS_FILTER_PASSTHRU_DURATION = 600 | |||
DEBUG_STREAM_COUNTERS = {} | |||
MIN_FILTER_CHUNK_SIZE = 4096 | |||
MIN_FILTER_CHUNK_SIZE = 16384 | |||
bubble_log = logging.getLogger(__name__) | |||
@@ -138,7 +138,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ | |||
chunks is a generator that can be used to iterate over all chunks. | |||
""" | |||
if bubble_log.isEnabledFor(DEBUG): | |||
bubble_log.debug('bubble_filter_chunks: starting...') | |||
bubble_log.debug('bubble_filter_chunks: starting with content_type='+content_type) | |||
first = True | |||
last = False | |||
content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) | |||
@@ -171,7 +171,10 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ | |||
# send whatever is left in the buffer | |||
if len(buffer) > 0: | |||
# bubble_log.debug('bubble_filter_chunks(end): sending remainder buffer of size '+str(len(buffer))) | |||
yield filter_chunk(flow, buffer, req_id, user_agent, last) | |||
if first: | |||
yield filter_chunk(flow, buffer, req_id, user_agent, last, content_encoding, content_type, content_length, csp) | |||
else: | |||
yield filter_chunk(flow, buffer, req_id, user_agent, last) | |||
if not content_length or not last: | |||
# bubble_log.debug('bubble_filter_chunks(end): sending last empty chunk') | |||
yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data | |||
@@ -184,7 +187,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ | |||
def bubble_modify(flow, flex_stream, req_id, user_agent, content_encoding, content_type, csp): | |||
if bubble_log.isEnabledFor(DEBUG): | |||
bubble_log.debug('bubble_modify: modifying req_id='+req_id) | |||
bubble_log.debug('bubble_modify: modifying req_id='+req_id+' with content_type='+content_type) | |||
return lambda chunks: bubble_filter_chunks(flow, chunks, flex_stream, req_id, | |||
user_agent, content_encoding, content_type, csp) | |||