bubblev
/
bubble
9
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 130 Wiki Activity
Browse Source

use proper logging in mitm

pull/51/head
Jonathan Cobb 4 years ago
parent
85aaf50bb6
commit
70be50cd3e
7 changed files with 273 additions and 114 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -1
      bubble-server/src/main/java/bubble/rule/AppRuleDriver.java
  2. +39
    -17
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py
  3. +84
    -40
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py
  4. +34
    -0
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_debug.py
  5. +61
    -30
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py
  6. +53
    -25
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py
  7. +1
    -1
      bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh

+ 1
- 1
bubble-server/src/main/java/bubble/rule/AppRuleDriver.java View File

@@ -43,7 +43,7 @@ public interface AppRuleDriver {
String REDIS_REJECT_LISTS = "rejectLists";
String REDIS_BLOCK_LISTS = "blockLists";
String REDIS_FILTER_LISTS = "filterLists";
String REDIS_FLEX_LISTS = "flexLists";
String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy for flex routing
String REDIS_LIST_SUFFIX = "~UNION";

default Set<String> getPrimedRejectDomains () { return null; }


+ 39
- 17
bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py View File

@@ -3,6 +3,9 @@
#
import datetime
import json
import logging
from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL

import re
import requests
import redis
@@ -17,6 +20,8 @@ from bubble_vpn6 import wireguard_network_ipv6
from bubble_config import bubble_network, bubble_port, debug_capture_fqdn, \
bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host

bubble_log = logging.getLogger(__name__)

HEADER_USER_AGENT = 'User-Agent'
HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy'
HEADER_REFERER = 'Referer'
@@ -39,8 +44,8 @@ BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_'
BUBBLE_ACTIVITY_LOG_EXPIRATION = 600

LOCAL_IPS = []
for ip in subprocess.check_output(['hostname', '-I']).split():
LOCAL_IPS.append(ip.decode())
for local_ip in subprocess.check_output(['hostname', '-I']).split():
LOCAL_IPS.append(local_ip.decode())


VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4)
@@ -57,10 +62,6 @@ def redis_set(name, value, ex):
REDIS.set(name, value, xx=True, ex=ex)


def bubble_log(message):
print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr, flush=True)


def bubble_activity_log(client_addr, server_addr, event, data):
key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4())
value = json.dumps({
@@ -70,7 +71,8 @@ def bubble_activity_log(client_addr, server_addr, event, data):
'event': event,
'data': str(data)
})
bubble_log('bubble_activity_log: setting '+key+' = '+value)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('bubble_activity_log: setting '+key+' = '+value)
redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION)
pass

@@ -79,7 +81,8 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level):
if debug_capture_fqdn and fqdns:
for f in debug_capture_fqdn:
if f in fqdns:
bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f)
return 'noop'

headers = {
@@ -96,11 +99,13 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level):
response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data)
if response.ok:
return response.json()
bubble_log('bubble_conn_check API call failed: '+repr(response))
if bubble_log.isEnabledFor(ERROR):
bubble_log.error('bubble_conn_check API call failed: '+repr(response))
return None

except Exception as e:
bubble_log('bubble_conn_check API call failed: '+repr(e))
if bubble_log.isEnabledFor(ERROR):
bubble_log.error('bubble_conn_check API call failed: '+repr(e))
traceback.print_exc()
if security_level is not None and security_level['level'] == 'maximum':
return False
@@ -130,7 +135,8 @@ BLOCK_MATCHER = {

def bubble_matchers(req_id, client_addr, server_addr, flow, host):
if debug_capture_fqdn and host and host in debug_capture_fqdn:
bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host)
return DEBUG_MATCHER

headers = {
@@ -139,19 +145,22 @@ def bubble_matchers(req_id, client_addr, server_addr, flow, host):
'Content-Type': 'application/json'
}
if HEADER_USER_AGENT not in flow.request.headers:
bubble_log('bubble_matchers: no User-Agent header, setting to UNKNOWN')
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('bubble_matchers: no User-Agent header, setting to UNKNOWN')
user_agent = 'UNKNOWN'
else:
user_agent = flow.request.headers[HEADER_USER_AGENT]

if HEADER_REFERER not in flow.request.headers:
bubble_log('bubble_matchers: no Referer header, setting to NONE')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('bubble_matchers: no Referer header, setting to NONE')
referer = 'NONE'
else:
try:
referer = flow.request.headers[HEADER_REFERER].encode().decode()
except Exception as e:
bubble_log('bubble_matchers: error parsing Referer header: '+repr(e))
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('bubble_matchers: error parsing Referer header: '+repr(e))
referer = 'NONE'

try:
@@ -168,11 +177,14 @@ def bubble_matchers(req_id, client_addr, server_addr, flow, host):
if response.ok:
return response.json()
elif response.status_code == 403:
bubble_log('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text))
return BLOCK_MATCHER
bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text))
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text))
except Exception as e:
bubble_log('bubble_matchers API call failed: '+repr(e))
if bubble_log.isEnabledFor(ERROR):
bubble_log.error('bubble_matchers API call failed: '+repr(e))
traceback.print_exc()
return None

@@ -203,3 +215,13 @@ def is_sage_request(ip, fqdns):
def is_not_from_vpn(client_addr):
ip = IPAddress(client_addr)
return ip not in VPN_IP4_CIDR and ip not in VPN_IP6_CIDR


def is_flex_domain(client_addr, fqdn):
check_fqdn = fqdn
while '.' in check_fqdn:
found = REDIS.sismember("flexLists~"+client_addr+"~UNION")
if found:
return True
check_fqdn = check_fqdn[check_fqdn.index('.')+1:]
return False

+ 84
- 40
bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py View File

@@ -28,11 +28,16 @@ from mitmproxy.exceptions import TlsProtocolException
from mitmproxy.net import tls as net_tls

import json
import logging
from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL

import traceback
from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set, \
from bubble_api import bubble_conn_check, bubble_activity_log, REDIS, redis_set, \
is_bubble_request, is_sage_request, is_not_from_vpn
from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host

bubble_log = logging.getLogger(__name__)

REDIS_DNS_PREFIX = 'bubble_dns_'
REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_'
REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout
@@ -56,16 +61,19 @@ def get_device_security_level(client_addr, fqdns):
return {'level': SEC_MAX}
level = level.decode()
if level == SEC_STD:
bubble_log('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.info('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns))
if fqdns:
max_required_fqdns = REDIS.smembers(REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX+client_addr)
if max_required_fqdns is not None:
bubble_log('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.info('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns))
for max_required in max_required_fqdns:
max_required = max_required.decode()
for fqdn in fqdns:
if max_required == fqdn or (max_required.startswith('*.') and fqdn.endswith(max_required[1:])):
bubble_log('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required)
return {'level': SEC_MAX, 'pinned': True}
return {'level': level}

@@ -90,7 +98,8 @@ def fqdns_for_addr(server_addr):
prefix = REDIS_DNS_PREFIX + server_addr
keys = REDIS.keys(prefix + '_*')
if keys is None or len(keys) == 0:
bubble_log('fqdns_for_addr: no FQDN found for addr '+str(server_addr)+', checking raw addr')
if (bubble_log.isEnabledFor(DEBUG)):
bubble_log.debug('fqdns_for_addr: no FQDN found for addr '+str(server_addr)+', checking raw addr')
return ''
fqdns = []
for k in keys:
@@ -104,7 +113,8 @@ class TlsBlock(TlsLayer):
Monkey-patch __call__ to drop this connection entirely
"""
def __call__(self):
bubble_log('TlsBlock: blocking')
if bubble_log.isEnabledFor(INFO):
bubble_log.info('TlsBlock: blocking')
return


@@ -122,16 +132,19 @@ class TlsFeedback(TlsLayer):

except TlsProtocolException as e:
if self.do_block:
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address)
raise e

tb = traceback.format_exc()
if 'OpenSSL.SSL.ZeroReturnError' in tb:
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address)
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address)
raise e

elif 'SysCallError' in tb:
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SysCallError for client '+client_address)
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SysCallError for client '+client_address)
raise e

elif self.fqdns is not None and len(self.fqdns) > 0:
@@ -140,21 +153,26 @@ class TlsFeedback(TlsLayer):
if security_level['level'] == SEC_MAX:
if 'pinned' in security_level and security_level['pinned']:
redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': False, 'reason': 'tls_failure_pinned'}), ex=REDIS_CHECK_DURATION)
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
else:
redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
else:
redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
else:
cache_key = conn_check_cache_prefix(client_address, server_address)
if security_level['level'] == SEC_MAX:
redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
else:
redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
raise e


@@ -162,30 +180,37 @@ def check_bubble_connection(client_addr, server_addr, fqdns, security_level):
check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level)
if check_response is None or check_response == 'error':
if security_level['level'] == SEC_MAX:
bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block')
return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'block': True, 'reason': 'bubble_error'}
else:
bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': False, 'reason': 'bubble_error'}

elif check_response == 'passthru':
bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': False, 'reason': 'bubble_passthru'}

elif check_response == 'block':
bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block')
return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'block': True, 'reason': 'bubble_block'}

elif check_response == 'passthru_flex':
bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': True, 'reason': 'bubble_passthru_flex'}

elif check_response == 'noop_flex':
bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': True, 'reason': 'bubble_no_passthru_flex'}

else:
bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False')
return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'reason': 'bubble_no_passthru'}


@@ -198,16 +223,20 @@ def check_connection(client_addr, server_addr, fqdns, security_level):

check_json = REDIS.get(cache_key)
if check_json is None or len(check_json) == 0:
bubble_log(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns))
check_response = check_bubble_connection(client_addr, server_addr, fqdns, security_level)
bubble_log(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...")
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...")
redis_set(cache_key, json.dumps(check_response), ex=REDIS_CHECK_DURATION)

else:
bubble_log(prefix+'found check_json='+str(check_json)+', touching key in redis')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'found check_json='+str(check_json)+', touching key in redis')
check_response = json.loads(check_json)
REDIS.touch(cache_key)
bubble_log(prefix+'returning '+str(check_response))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'returning '+str(check_response))
return check_response


@@ -216,14 +245,17 @@ def next_layer(next_layer):
client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile)
client_addr = next_layer.client_conn.address[0]
server_addr = next_layer.server_conn.address[0]
bubble_log('next_layer: STARTING: client='+ client_addr+' server='+server_addr)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('next_layer: STARTING: client='+ client_addr+' server='+server_addr)
if client_hello.sni:
fqdn = client_hello.sni.decode()
bubble_log('next_layer: using fqdn in SNI: '+ fqdn)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('next_layer: using fqdn in SNI: '+ fqdn)
fqdns = [ fqdn ]
else:
fqdns = fqdns_for_addr(server_addr)
bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns))
next_layer.fqdns = fqdns
no_fqdns = fqdns is None or len(fqdns) == 0
security_level = get_device_security_level(client_addr, fqdns)
@@ -231,49 +263,59 @@ def next_layer(next_layer):
next_layer.do_block = False
called_check_api = False
if is_bubble_request(server_addr, fqdns):
bubble_log('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns))
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns))
check = FORCE_PASSTHRU

elif is_sage_request(server_addr, fqdns):
bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr)
check = FORCE_PASSTHRU

elif is_not_from_vpn(client_addr):
# todo: add to fail2ban
bubble_log('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns))
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns))
bubble_activity_log(client_addr, server_addr, 'conn_block_non_vpn', fqdns)
next_layer.__class__ = TlsBlock
return

elif security_level['level'] == SEC_OFF:
bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr)
check = FORCE_PASSTHRU

elif fqdns is not None and len(fqdns) == 1 and cert_validation_host == fqdns[0] and security_level['level'] != SEC_BASIC:
bubble_log('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr)
return

elif (security_level['level'] == SEC_STD or security_level['level'] == SEC_BASIC) and no_fqdns:
bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
check = FORCE_PASSTHRU

elif security_level['level'] == SEC_MAX and no_fqdns:
bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
check = FORCE_BLOCK

else:
bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level))
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level))
check = check_connection(client_addr, server_addr, fqdns, security_level)
called_check_api = True

if check is None or ('passthru' in check and check['passthru'] and ('flex' not in check or not check['flex'])):
bubble_log('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
next_layer.reply.send(next_layer_replacement)

elif 'block' in check and check['block']:
bubble_log('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns))
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns))
bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns)
if show_block_stats(client_addr, fqdns) and security_level['level'] != SEC_BASIC:
next_layer.do_block = True
@@ -282,12 +324,14 @@ def next_layer(next_layer):
next_layer.__class__ = TlsBlock

elif security_level['level'] == SEC_BASIC:
bubble_log('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
next_layer.reply.send(next_layer_replacement)

else:
bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns))
if bubble_log.isEnabledFor(INFO):
bubble_log.info('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns))
bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns)
next_layer.__class__ = TlsFeedback

+ 34
- 0
bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_debug.py View File

@@ -1,11 +1,42 @@
#
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
#
import logging
from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL

import os
import threading
import traceback
import signal
import sys

from pathlib import Path

BUBBLE_PORT_ENV_VAR = 'BUBBLE_PORT'
BUBBLE_PORT = os.getenv(BUBBLE_PORT_ENV_VAR)
if BUBBLE_PORT is None:
BUBBLE_PORT = '(no '+BUBBLE_PORT_ENV_VAR+' env var found)'

BUBBLE_LOG = '/var/log/bubble/mitmproxy_bubble.log'
BUBBLE_LOG_LEVEL_FILE = '/home/mitmproxy/bubble_log_level.txt'
BUBBLE_LOG_LEVEL_ENV_VAR = 'BUBBLE_LOG_LEVEL'
DEFAULT_BUBBLE_LOG_LEVEL = 'INFO'
BUBBLE_LOG_LEVEL = None
try:
BUBBLE_LOG_LEVEL = Path(BUBBLE_LOG_LEVEL_FILE).read_text().strip()
except IOError:
print('error reading log level from '+BUBBLE_LOG_LEVEL_FILE+', checking env var '+BUBBLE_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True)
BUBBLE_LOG_LEVEL = os.getenv(BUBBLE_LOG_LEVEL_ENV_VAR, DEFAULT_BUBBLE_LOG_LEVEL)

BUBBLE_NUMERIC_LOG_LEVEL = getattr(logging, BUBBLE_LOG_LEVEL.upper(), None)
if not isinstance(BUBBLE_NUMERIC_LOG_LEVEL, int):
print('Invalid log level: ' + BUBBLE_LOG_LEVEL + ' - using default '+DEFAULT_BUBBLE_LOG_LEVEL, file=sys.stderr, flush=True)
BUBBLE_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_BUBBLE_LOG_LEVEL.upper(), None)
logging.basicConfig(format='[mitm'+BUBBLE_PORT+'] %(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=BUBBLE_LOG, level=BUBBLE_NUMERIC_LOG_LEVEL)

bubble_log = logging.getLogger(__name__)


# Allow SIGUSR1 to print stack traces to stderr
def dumpstacks(signal, frame):
id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
@@ -18,5 +49,8 @@ def dumpstacks(signal, frame):
code.append(" %s" % (line.strip()))
print("\n------------------------------------- stack traces ------------------------------"+"\n".join(code), file=sys.stderr, flush=True)


signal.signal(signal.SIGUSR1, dumpstacks)

if bubble_log.isEnabledFor(INFO):
bubble_log.info('debug module initialized, default log level = '+logging.getLevelName(BUBBLE_NUMERIC_LOG_LEVEL))

+ 61
- 30
bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py View File

@@ -2,6 +2,8 @@
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
#
import json
import logging
from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
import re
import requests
import urllib
@@ -10,7 +12,7 @@ from mitmproxy.net.http import Headers
from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri
from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, BUBBLE_URI_PREFIX, \
HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, \
CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \
CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, get_flow_ctx, add_flow_ctx, \
HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header

BUFFER_SIZE = 4096
@@ -27,6 +29,8 @@ REDIS_FILTER_PASSTHRU_DURATION = 600

DEBUG_STREAM_COUNTERS = {}

bubble_log = logging.getLogger(__name__)


def add_csp_part(new_csp, part):
if len(new_csp) > 0:
@@ -53,7 +57,8 @@ def ensure_bubble_script_csp(csp):
def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, content_type=None, content_length=None, csp=None):
if debug_capture_fqdn:
if debug_capture_fqdn in req_id:
bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn)
f = open('/tmp/bubble_capture_'+req_id, mode='ab', buffering=0)
f.write(chunk)
f.close()
@@ -63,7 +68,8 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c
redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + '~~~' + user_agent + ':' + flow.request.url
do_pass = REDIS.get(redis_passthru_key)
if do_pass:
bubble_log('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk')
REDIS.touch(redis_passthru_key)
return chunk

@@ -83,14 +89,15 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c
url = url + '?last=true'

if csp:
# bubble_log('filter_chunk: url='+url+' (csp='+csp+')')
bubble_log('filter_chunk: url='+url+' (with csp) (last='+str(last)+')')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('filter_chunk: url='+url+' (csp='+csp+')')
filter_headers = {
HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY,
HEADER_CONTENT_SECURITY_POLICY: csp
}
else:
bubble_log('filter_chunk: url='+url+' (no csp) (last='+str(last)+')')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('filter_chunk: url='+url+' (no csp)')
filter_headers = STANDARD_FILTER_HEADERS

if debug_stream_fqdn and debug_stream_uri and debug_stream_fqdn in req_id and flow.request.path == debug_stream_uri:
@@ -99,7 +106,8 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c
else:
count = 0
DEBUG_STREAM_COUNTERS[req_id] = count
bubble_log('filter_chunk: debug_stream detected, capturing: '+debug_stream_fqdn)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('filter_chunk: debug_stream detected, capturing: '+debug_stream_fqdn)
f = open('/tmp/bubble_stream_'+req_id+'_chunk'+"{:04d}".format(count)+'.data', mode='wb', buffering=0)
if chunk is not None:
f.write(chunk)
@@ -114,11 +122,13 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c
response = requests.post(url, data=chunk, headers=filter_headers)
if not response.ok:
err_message = 'filter_chunk: Error fetching ' + url + ', HTTP status ' + str(response.status_code)
bubble_log(err_message)
if bubble_log.isEnabledFor(ERROR):
bubble_log.error(err_message)
return b''

elif HEADER_FILTER_PASSTHRU in response.headers:
bubble_log('filter_chunk: server returned X-Bubble-Passthru, not filtering subsequent requests')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('filter_chunk: server returned X-Bubble-Passthru, not filtering subsequent requests')
redis_set(redis_passthru_key, 'passthru', ex=REDIS_FILTER_PASSTHRU_DURATION)
return chunk

@@ -137,7 +147,8 @@ def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, con
bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT)
chunk_len = len(chunk)
last = chunk_len + bytes_sent >= content_length
bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent))
add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len)
else:
last = False
@@ -149,7 +160,8 @@ def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, con
if not content_length:
yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data
except Exception as e:
bubble_log('bubble_filter_chunks: exception='+repr(e))
if bubble_log.isEnabledFor(ERROR):
bubble_log.error('bubble_filter_chunks: exception='+repr(e))
traceback.print_exc()
yield None

@@ -183,7 +195,8 @@ def responseheaders(flow):
path = flow.request.path
if path and path.startswith(BUBBLE_URI_PREFIX):
if path.startswith(HEALTH_CHECK_URI):
# bubble_log('responseheaders: special bubble health check request, responding with OK')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('responseheaders: special bubble health check request, responding with OK')
flow.response.headers = Headers()
flow.response.headers[HEADER_HEALTH_CHECK] = 'OK'
flow.response.headers[HEADER_CONTENT_LENGTH] = '3'
@@ -191,7 +204,8 @@ def responseheaders(flow):
flow.response.stream = lambda chunks: [b'OK\n']
else:
uri = 'http://127.0.0.1:' + bubble_port + '/' + path[len(BUBBLE_URI_PREFIX):]
bubble_log('responseheaders: sending special bubble request to '+uri)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('responseheaders: sending special bubble request to '+uri)
headers = {
'Accept' : 'application/json',
'Content-Type': 'application/json'
@@ -200,13 +214,16 @@ def responseheaders(flow):
if flow.request.method == 'GET':
response = requests.get(uri, headers=headers, stream=True)
elif flow.request.method == 'POST':
bubble_log('responseheaders: special bubble request: POST content is '+str(flow.request.content))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('responseheaders: special bubble request: POST content is '+str(flow.request.content))
headers['Content-Length'] = str(len(flow.request.content))
response = requests.post(uri, data=flow.request.content, headers=headers)
else:
bubble_log('responseheaders: special bubble request: method '+flow.request.method+' not supported')
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('responseheaders: special bubble request: method '+flow.request.method+' not supported')
if response is not None:
bubble_log('responseheaders: special bubble request: response status = '+str(response.status_code))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('responseheaders: special bubble request: response status = '+str(response.status_code))
flow.response.headers = Headers()
for key, value in response.headers.items():
flow.response.headers[key] = value
@@ -218,7 +235,8 @@ def responseheaders(flow):
if abort_code is not None:
abort_location = get_flow_ctx(flow, CTX_BUBBLE_LOCATION)
if abort_location is not None:
bubble_log('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path)
flow.response.headers = Headers()
flow.response.headers[HEADER_LOCATION] = abort_location
flow.response.status_code = abort_code
@@ -228,21 +246,25 @@ def responseheaders(flow):
content_type = flow.response.headers[HEADER_CONTENT_TYPE]
else:
content_type = None
bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code)+', path was: '+path)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('responseheaders: aborting request with HTTP status '+str(abort_code)+', path was: '+path)
flow.response.headers = Headers()
flow.response.status_code = abort_code
flow.response.stream = lambda chunks: abort_data(content_type)

elif flow.response.status_code // 100 != 2:
bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path)
pass

elif flow.response.headers is None or len(flow.response.headers) == 0:
bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path)
pass

elif HEADER_CONTENT_LENGTH in flow.response.headers and flow.response.headers[HEADER_CONTENT_LENGTH] == "0":
bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path)
if bubble_log.isEnabledFor(INFO):
bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path)
pass

else:
@@ -250,7 +272,8 @@ def responseheaders(flow):
matchers = get_flow_ctx(flow, CTX_BUBBLE_MATCHERS)
prefix = 'responseheaders(req_id='+str(req_id)+'): '
if req_id is not None and matchers is not None:
bubble_log(prefix+' matchers: '+repr(matchers))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+' matchers: '+repr(matchers))
if HEADER_USER_AGENT in flow.request.headers:
user_agent = flow.request.headers[HEADER_USER_AGENT]
else:
@@ -266,10 +289,12 @@ def responseheaders(flow):
typeRegex = '^text/html.*'
if re.match(typeRegex, content_type):
any_content_type_matches = True
bubble_log(prefix+'found at least one matcher for content_type ('+content_type+'), filtering: '+path)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'found at least one matcher for content_type ('+content_type+'), filtering: '+path)
break
if not any_content_type_matches:
bubble_log(prefix+'no matchers for content_type ('+content_type+'), passing thru: '+path)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'no matchers for content_type ('+content_type+'), passing thru: '+path)
return

if HEADER_CONTENT_ENCODING in flow.response.headers:
@@ -284,7 +309,8 @@ def responseheaders(flow):
csp = None

content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None)
# bubble_log(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
flow.response.stream = bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp)
if content_length_value:
flow.response.headers['transfer-encoding'] = 'chunked'
@@ -295,10 +321,12 @@ def responseheaders(flow):
if hasattr(ctx, 'ctx'):
ctx = ctx.ctx
else:
bubble_log(prefix+'error finding server_conn for path '+path+'. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx)))
if bubble_log.isEnabledFor(ERROR):
bubble_log.error(prefix+'error finding server_conn for path '+path+'. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx)))
return
if not hasattr(ctx, 'server_conn'):
bubble_log(prefix+'error finding server_conn for path '+path+'. ctx type='+str(type(ctx))+' vars='+str(vars(ctx)))
if bubble_log.isEnabledFor(ERROR):
bubble_log.error(prefix+'error finding server_conn for path '+path+'. ctx type='+str(type(ctx))+' vars='+str(vars(ctx)))
return
content_length = int(content_length_value)
ctx.server_conn.rfile.fake_chunks = content_length
@@ -306,11 +334,14 @@ def responseheaders(flow):
add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, 0)

else:
bubble_log(prefix+'no matchers, passing thru: '+path)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'no matchers, passing thru: '+path)
pass
else:
bubble_log(prefix+'no '+HEADER_CONTENT_TYPE+' header, passing thru: '+path)
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning(prefix+'no '+HEADER_CONTENT_TYPE+' header, passing thru: '+path)
pass
else:
bubble_log(prefix+'no '+CTX_BUBBLE_MATCHERS+' in ctx, passing thru: '+path)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug(prefix+'no '+CTX_BUBBLE_MATCHERS+' in ctx, passing thru: '+path)
pass

+ 53
- 25
bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py View File

@@ -25,15 +25,21 @@
#

import re
import logging
from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL

import time
import uuid
from mitmproxy.net.http import headers as nheaders

from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, REDIS, HEALTH_CHECK_URI, \
from bubble_api import bubble_matchers, bubble_activity_log, HEALTH_CHECK_URI, \
CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \
add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn
from bubble_config import bubble_host, bubble_host_alias

bubble_log = logging.getLogger(__name__)


class Rerouter:
@staticmethod
def get_matchers(flow, host):
@@ -43,7 +49,8 @@ class Rerouter:
is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI)
if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX):
if not is_health_check:
bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path)
if bubble_log.isEnabledFor(INFO):
bubble_log.info("get_matchers: not filtering special bubble path: "+flow.request.path)
return None

client_addr = str(flow.client_conn.address[0])
@@ -54,43 +61,53 @@ class Rerouter:
try:
host = str(host)
except Exception as e:
bubble_log('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e))
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e))
return None

if host == bubble_host or host == bubble_host_alias:
bubble_log('get_matchers: request is for bubble itself ('+host+'), not matching')
if bubble_log.isEnabledFor(INFO):
bubble_log.info('get_matchers: request is for bubble itself ('+host+'), not matching')
return None

req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time())
bubble_log("get_matchers: requesting match decision for req_id="+req_id)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug("get_matchers: requesting match decision for req_id="+req_id)
resp = bubble_matchers(req_id, client_addr, server_addr, flow, host)

if not resp:
bubble_log('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host))
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host))
return None

matchers = []
if 'matchers' in resp and resp['matchers'] is not None:
for m in resp['matchers']:
if 'urlRegex' in m:
bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex'])
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex'])
else:
bubble_log('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping')
continue
if re.match(m['urlRegex'], flow.request.path):
bubble_log('get_matchers: rule matched, adding rule: '+m['rule'])
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('get_matchers: rule matched, adding rule: '+m['rule'])
matchers.append(m)
else:
bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule'])
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule'])
else:
bubble_log('get_matchers: no matchers. response='+repr(resp))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('get_matchers: no matchers. response='+repr(resp))

decision = None
if 'decision' in resp:
decision = resp['decision']

matcher_response = {'decision': decision, 'matchers': matchers, 'request_id': req_id}
bubble_log("get_matchers: returning "+repr(matcher_response))
if bubble_log.isEnabledFor(INFO):
bubble_log.info("get_matchers: returning "+repr(matcher_response))
return matcher_response

def request(self, flow):
@@ -110,7 +127,6 @@ class Rerouter:
# check if https and sni is missing but we have a host header, fill in the sni
host_header = flow.request.host_header
# bubble_log("dns_spoofing.request: host_header is "+repr(host_header))
if host_header:
m = parse_host_header.match(host_header)
if m:
@@ -133,20 +149,23 @@ class Rerouter:
if is_bubble_request(server_addr, fqdns):
is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI)
if not is_health_check:
bubble_log('dns_spoofing.request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns)+', path='+flow.request.path)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns)+', path='+flow.request.path)
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path)
return

elif is_sage_request(server_addr, fqdns):
bubble_log('dns_spoofing.request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr)
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr)
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path)
return

elif is_not_from_vpn(client_addr):
# todo: add to fail2ban
bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns))
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns))
bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns)
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
return
@@ -155,19 +174,22 @@ class Rerouter:
if matcher_response:
has_decision = 'decision' in matcher_response and matcher_response['decision'] is not None
if has_decision and matcher_response['decision'] == 'pass_thru':
bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: passthru response returned, passing thru and NOT performing TLS interception...')
add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True)
bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url)
return

elif has_decision and matcher_response['decision'].startswith('abort_'):
bubble_log('dns_spoofing.request: found abort code: ' + str(matcher_response['decision']) + ', aborting')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: found abort code: ' + str(matcher_response['decision']) + ', aborting')
if matcher_response['decision'] == 'abort_ok':
abort_code = 200
elif matcher_response['decision'] == 'abort_not_found':
abort_code = 404
else:
bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found')
abort_code = 404
flow.request.headers = nheaders.Headers([])
flow.request.content = b''
@@ -176,7 +198,8 @@ class Rerouter:
return

elif has_decision and matcher_response['decision'] == 'no_match':
bubble_log('dns_spoofing.request: decision was no_match, passing thru...')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: decision was no_match, passing thru...')
bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url)
return

@@ -184,27 +207,32 @@ class Rerouter:
and 'request_id' in matcher_response
and len(matcher_response['matchers']) > 0):
req_id = matcher_response['request_id']
bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug("request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers'])
add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id)
bubble_activity_log(client_addr, server_addr, 'http_match', log_url)
else:
bubble_log('dns_spoofing.request: no rules returned, passing thru...')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: no rules returned, passing thru...')
bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url)
else:
if not is_health_check:
bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...')
if bubble_log.isEnabledFor(DEBUG):
bubble_log.debug('request: no matcher_response returned, passing thru...')
# bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url)

elif is_http and is_not_from_vpn(client_addr):
# todo: add to fail2ban
bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr)
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr)
bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr])
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
return

else:
bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path)
if bubble_log.isEnabledFor(WARNING):
bubble_log.warning('request: no sni/host found, not applying rules to path: ' + flow.request.path)
bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr])

flow.request.host_header = host_header


+ 1
- 1
bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh View File

@@ -14,7 +14,7 @@ fi

cd /home/mitmproxy/mitmproxy && \
./dev.sh ${SETUP_VENV} && . ./venv/bin/activate && \
mitmdump \
BUBBLE_PORT=${PORT} mitmdump \
--listen-host 0.0.0.0 \
--listen-port ${PORT} \
--showhost \


Write Preview
Loading…
Cancel
Save