Merge branch 'master' into sfedoriv/APIAddSupportForAmazonEC2ComputeCloudService

automation/roles/mitmproxy/files/bubble_api.py

@@ -5,7 +5,11 @@ import requests
 import traceback
 import sys
 import os
+import time
+import uuid
 import datetime
+import redis
+import json
 from bubble_config import bubble_network, bubble_port
 
 # Write python PID to file so that mitmdump_monitor.sh can check for excessive memory usage and restart if needed
@@ -25,10 +29,33 @@ CTX_CONTENT_LENGTH='X-Bubble-Content-Length'
 CTX_CONTENT_LENGTH_SENT='X-Bubble-Content-Length-Sent'
 BUBBLE_URI_PREFIX='/__bubble/'
 
+REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
+BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_'
+BUBBLE_ACTIVITY_LOG_EXPIRATION = 600
+
+def redis_set(name, value, ex):
+    REDIS.set(name, value, nx=True, ex=ex)
+    REDIS.set(name, value, xx=True, ex=ex)
+
+
 def bubble_log(message):
     print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr)
 
 
+def bubble_activity_log(client_addr, server_addr, event, data):
+    key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4())
+    value = json.dumps({
+        'source': 'mitmproxy',
+        'client_addr': client_addr,
+        'server_addr': server_addr,
+        'event': event,
+        'data': data
+    })
+    bubble_log('bubble_activity_log: setting '+key+' = '+value)
+    redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION)
+    pass
+
+
 def bubble_passthru(remote_addr, addr, fqdn):
     headers = {
         'X-Forwarded-For': remote_addr,

automation/roles/mitmproxy/files/bubble_passthru.py

@@ -26,8 +26,9 @@
 from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
 from mitmproxy.exceptions import TlsProtocolException
 
-from bubble_api import bubble_log, bubble_passthru
+from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set
 import redis
+import json
 
 REDIS_DNS_PREFIX = 'bubble_dns_'
 REDIS_PASSTHRU_PREFIX = 'bubble_passthru_'
@@ -39,10 +40,6 @@ REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
 def passthru_cache_prefix(client_addr, server_addr):
     return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr
 
-def redis_set(name, value, ex):
-    REDIS.set(name, value, nx=True, ex=ex)
-    REDIS.set(name, value, xx=True, ex=ex)
-
 
 class TlsFeedback(TlsLayer):
     """
@@ -57,49 +54,59 @@ class TlsFeedback(TlsLayer):
         except TlsProtocolException as e:
             bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru')
             cache_key = passthru_cache_prefix(client_address, server_address)
-            redis_set(cache_key, str(True), ex=REDIS_PASSTHRU_DURATION)
+            fqdn = fqdn_for_addr(server_address)
+            redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION)
             raise e
 
 
-def check_bubble_passthru(remote_addr, addr):
+def fqdn_for_addr(addr):
     fqdn = REDIS.get(REDIS_DNS_PREFIX + addr)
     if fqdn is None or len(fqdn) == 0:
         bubble_log('check_bubble_passthru: no FQDN found for addr '+repr(addr)+', checking raw addr')
         fqdn = b''
-    fqdn = fqdn.decode()
+    return fqdn.decode()
+
+
+def check_bubble_passthru(remote_addr, addr, fqdn):
     if bubble_passthru(remote_addr, addr, fqdn):
         bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True')
-        return True
+        return {'fqdn': fqdn, 'addr': addr, 'passthru': True}
     bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False')
-    return False
+    return {'fqdn': fqdn, 'addr': addr, 'passthru': False}
 
 
 def should_passthru(remote_addr, addr):
-    prefix = 'should_passthru: '+repr(addr)
+    prefix = 'should_passthru: '+repr(addr)+' '
     bubble_log(prefix+'starting...')
     cache_key = passthru_cache_prefix(remote_addr, addr)
-    passthru_string = REDIS.get(cache_key)
-    if passthru_string is None or len(passthru_string) == 0:
+    passthru_json = REDIS.get(cache_key)
+    if passthru_json is None or len(passthru_json) == 0:
         bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...')
-        passthru = check_bubble_passthru(remote_addr, addr)
-        bubble_log(prefix+'check_bubble_passthru returned '+str(passthru)+", string in redis...")
-        redis_set(cache_key, str(passthru), ex=REDIS_PASSTHRU_DURATION)
-        passthru_string = str(passthru)
+        fqdn = fqdn_for_addr(addr)
+        if fqdn is None or len(fqdn) == 0:
+            bubble_log(prefix+' no fqdn found for addr '+addr+', returning (uncached) passthru = True')
+            return {'fqdn': None, 'addr': addr, 'passthru': True}
+        passthru = check_bubble_passthru(remote_addr, addr, fqdn)
+        bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...")
+        redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION)
     else:
-        passthru_string = passthru_string.decode()
-        bubble_log(prefix+'found cached value, passthru_string='+str(passthru_string)+', re-setting in redis')
-        redis_set(cache_key, passthru_string, ex=REDIS_PASSTHRU_DURATION)
-    bubble_log(prefix+'returning '+str(passthru_string == 'True'))
-    return passthru_string == 'True'
+        bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis')
+        passthru = json.loads(passthru_json)
+        REDIS.touch(cache_key)
+    bubble_log(prefix+'returning '+repr(passthru))
+    return passthru
 
 
 def next_layer(next_layer):
     if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
         client_address = next_layer.client_conn.address[0]
         server_address = next_layer.server_conn.address[0]
-        if should_passthru(client_address, server_address):
+        passthru = should_passthru(client_address, server_address)
+        if passthru['passthru']:
             bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address))
+            bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn'])
             next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
             next_layer.reply.send(next_layer_replacement)
         else:
+            bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn'])
             next_layer.__class__ = TlsFeedback

automation/roles/mitmproxy/files/dns_spoofing.py

@@ -4,7 +4,7 @@
 import re
 import time
 import uuid
-from bubble_api import bubble_matchers, bubble_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx
+from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx
 from bubble_config import bubble_host, bubble_host_alias
 
 # This regex extracts splits the host header into host and port.
@@ -70,6 +70,8 @@ class Rerouter:
         return matcher_response
 
     def request(self, flow):
+        client_address = flow.client_conn.address[0]
+        server_address = flow.server_conn.address[0]
         if flow.client_conn.tls_established:
             flow.request.scheme = "https"
             sni = flow.client_conn.connection.get_servername()
@@ -90,11 +92,16 @@ class Rerouter:
 
         # Determine if this request should be filtered
         if sni or host_header:
+            host = str(sni or host_header)
+            if host.startswith("b'"):
+                host = host[2:-1]
+            log_url = flow.request.scheme + '://' + host + flow.request.path
             matcher_response = self.get_matchers(flow, sni or host_header)
             if matcher_response:
                 if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru':
                     bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...')
                     add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True)
+                    bubble_activity_log(client_address, server_address, 'http_passthru', log_url)
                     return
 
                 elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'):
@@ -107,10 +114,12 @@ class Rerouter:
                         bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found')
                         abort_code = 404
                     add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code)
+                    bubble_activity_log(client_address, server_address, 'http_abort' + str(abort_code), log_url)
                     return
 
                 elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match':
                     bubble_log('dns_spoofing.request: decision was no_match, passing thru...')
+                    bubble_activity_log(client_address, server_address, 'http_no_match', log_url)
                     return
 
                 elif ('matchers' in matcher_response
@@ -120,12 +129,16 @@ class Rerouter:
                     bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
                     add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers'])
                     add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id)
+                    bubble_activity_log(client_address, server_address, 'http_match', log_url)
                 else:
                     bubble_log('dns_spoofing.request: no rules returned, passing thru...')
+                    bubble_activity_log(client_address, server_address, 'http_no_rules', log_url)
             else:
                 bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...')
+                # bubble_activity_log(client_address, server_address, 'http_no_matcher_response', log_url)
         else:
             bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path)
+            bubble_activity_log(client_address, server_address, 'http_no_sni_or_host', 'n/a')
 
         flow.request.host_header = host_header
         flow.request.host = sni or host_header