diff --git a/automation/roles/mitmproxy/files/bubble_api.py b/automation/roles/mitmproxy/files/bubble_api.py index 382e129e..afab042a 100644 --- a/automation/roles/mitmproxy/files/bubble_api.py +++ b/automation/roles/mitmproxy/files/bubble_api.py @@ -5,7 +5,11 @@ import requests import traceback import sys import os +import time +import uuid import datetime +import redis +import json from bubble_config import bubble_network, bubble_port # Write python PID to file so that mitmdump_monitor.sh can check for excessive memory usage and restart if needed @@ -25,10 +29,33 @@ CTX_CONTENT_LENGTH='X-Bubble-Content-Length' CTX_CONTENT_LENGTH_SENT='X-Bubble-Content-Length-Sent' BUBBLE_URI_PREFIX='/__bubble/' +REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) +BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_' +BUBBLE_ACTIVITY_LOG_EXPIRATION = 600 + +def redis_set(name, value, ex): + REDIS.set(name, value, nx=True, ex=ex) + REDIS.set(name, value, xx=True, ex=ex) + + def bubble_log(message): print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr) +def bubble_activity_log(client_addr, server_addr, event, data): + key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4()) + value = json.dumps({ + 'source': 'mitmproxy', + 'client_addr': client_addr, + 'server_addr': server_addr, + 'event': event, + 'data': data + }) + bubble_log('bubble_activity_log: setting '+key+' = '+value) + redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION) + pass + + def bubble_passthru(remote_addr, addr, fqdn): headers = { 'X-Forwarded-For': remote_addr, diff --git a/automation/roles/mitmproxy/files/bubble_passthru.py b/automation/roles/mitmproxy/files/bubble_passthru.py index 4df5f274..c47c6347 100644 --- a/automation/roles/mitmproxy/files/bubble_passthru.py +++ b/automation/roles/mitmproxy/files/bubble_passthru.py @@ -26,8 +26,9 @@ from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer from mitmproxy.exceptions import TlsProtocolException -from bubble_api import bubble_log, bubble_passthru +from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set import redis +import json REDIS_DNS_PREFIX = 'bubble_dns_' REDIS_PASSTHRU_PREFIX = 'bubble_passthru_' @@ -39,10 +40,6 @@ REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) def passthru_cache_prefix(client_addr, server_addr): return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr -def redis_set(name, value, ex): - REDIS.set(name, value, nx=True, ex=ex) - REDIS.set(name, value, xx=True, ex=ex) - class TlsFeedback(TlsLayer): """ @@ -57,49 +54,59 @@ class TlsFeedback(TlsLayer): except TlsProtocolException as e: bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru') cache_key = passthru_cache_prefix(client_address, server_address) - redis_set(cache_key, str(True), ex=REDIS_PASSTHRU_DURATION) + fqdn = fqdn_for_addr(server_address) + redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) raise e -def check_bubble_passthru(remote_addr, addr): +def fqdn_for_addr(addr): fqdn = REDIS.get(REDIS_DNS_PREFIX + addr) if fqdn is None or len(fqdn) == 0: bubble_log('check_bubble_passthru: no FQDN found for addr '+repr(addr)+', checking raw addr') fqdn = b'' - fqdn = fqdn.decode() + return fqdn.decode() + + +def check_bubble_passthru(remote_addr, addr, fqdn): if bubble_passthru(remote_addr, addr, fqdn): bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True') - return True + return {'fqdn': fqdn, 'addr': addr, 'passthru': True} bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False') - return False + return {'fqdn': fqdn, 'addr': addr, 'passthru': False} def should_passthru(remote_addr, addr): - prefix = 'should_passthru: '+repr(addr) + prefix = 'should_passthru: '+repr(addr)+' ' bubble_log(prefix+'starting...') cache_key = passthru_cache_prefix(remote_addr, addr) - passthru_string = REDIS.get(cache_key) - if passthru_string is None or len(passthru_string) == 0: + passthru_json = REDIS.get(cache_key) + if passthru_json is None or len(passthru_json) == 0: bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...') - passthru = check_bubble_passthru(remote_addr, addr) - bubble_log(prefix+'check_bubble_passthru returned '+str(passthru)+", string in redis...") - redis_set(cache_key, str(passthru), ex=REDIS_PASSTHRU_DURATION) - passthru_string = str(passthru) + fqdn = fqdn_for_addr(addr) + if fqdn is None or len(fqdn) == 0: + bubble_log(prefix+' no fqdn found for addr '+addr+', returning (uncached) passthru = True') + return {'fqdn': None, 'addr': addr, 'passthru': True} + passthru = check_bubble_passthru(remote_addr, addr, fqdn) + bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...") + redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) else: - passthru_string = passthru_string.decode() - bubble_log(prefix+'found cached value, passthru_string='+str(passthru_string)+', re-setting in redis') - redis_set(cache_key, passthru_string, ex=REDIS_PASSTHRU_DURATION) - bubble_log(prefix+'returning '+str(passthru_string == 'True')) - return passthru_string == 'True' + bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis') + passthru = json.loads(passthru_json) + REDIS.touch(cache_key) + bubble_log(prefix+'returning '+repr(passthru)) + return passthru def next_layer(next_layer): if isinstance(next_layer, TlsLayer) and next_layer._client_tls: client_address = next_layer.client_conn.address[0] server_address = next_layer.server_conn.address[0] - if should_passthru(client_address, server_address): + passthru = should_passthru(client_address, server_address) + if passthru['passthru']: bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address)) + bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn']) next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) next_layer.reply.send(next_layer_replacement) else: + bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn']) next_layer.__class__ = TlsFeedback diff --git a/automation/roles/mitmproxy/files/dns_spoofing.py b/automation/roles/mitmproxy/files/dns_spoofing.py index 8df05da2..41eab031 100644 --- a/automation/roles/mitmproxy/files/dns_spoofing.py +++ b/automation/roles/mitmproxy/files/dns_spoofing.py @@ -4,7 +4,7 @@ import re import time import uuid -from bubble_api import bubble_matchers, bubble_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx +from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx from bubble_config import bubble_host, bubble_host_alias # This regex extracts splits the host header into host and port. @@ -70,6 +70,8 @@ class Rerouter: return matcher_response def request(self, flow): + client_address = flow.client_conn.address[0] + server_address = flow.server_conn.address[0] if flow.client_conn.tls_established: flow.request.scheme = "https" sni = flow.client_conn.connection.get_servername() @@ -90,11 +92,16 @@ class Rerouter: # Determine if this request should be filtered if sni or host_header: + host = str(sni or host_header) + if host.startswith("b'"): + host = host[2:-1] + log_url = flow.request.scheme + '://' + host + flow.request.path matcher_response = self.get_matchers(flow, sni or host_header) if matcher_response: if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru': bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...') add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) + bubble_activity_log(client_address, server_address, 'http_passthru', log_url) return elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'): @@ -107,10 +114,12 @@ class Rerouter: bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') abort_code = 404 add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code) + bubble_activity_log(client_address, server_address, 'http_abort' + str(abort_code), log_url) return elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match': bubble_log('dns_spoofing.request: decision was no_match, passing thru...') + bubble_activity_log(client_address, server_address, 'http_no_match', log_url) return elif ('matchers' in matcher_response @@ -120,12 +129,16 @@ class Rerouter: bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) + bubble_activity_log(client_address, server_address, 'http_match', log_url) else: bubble_log('dns_spoofing.request: no rules returned, passing thru...') + bubble_activity_log(client_address, server_address, 'http_no_rules', log_url) else: bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...') + # bubble_activity_log(client_address, server_address, 'http_no_matcher_response', log_url) else: bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path) + bubble_activity_log(client_address, server_address, 'http_no_sni_or_host', 'n/a') flow.request.host_header = host_header flow.request.host = sni or host_header