validate http connections

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -1,16 +1,21 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
-import requests
-import traceback
+import datetime
+import json
 import re
+import requests
+import redis
+import subprocess
 import sys
 import time
+import traceback
 import uuid
-import datetime
-import redis
-import json
-from bubble_config import bubble_network, bubble_port, debug_capture_fqdn
+from netaddr import IPAddress, IPNetwork
+from bubble_vpn4 import wireguard_network_ipv4
+from bubble_vpn6 import wireguard_network_ipv6
+from bubble_config import bubble_network, bubble_port, debug_capture_fqdn, \
+    bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host
 
 HEADER_USER_AGENT = 'User-Agent'
 HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy'
@@ -19,6 +24,7 @@ HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru'
 
 CTX_BUBBLE_MATCHERS='X-Bubble-Matchers'
 CTX_BUBBLE_ABORT='X-Bubble-Abort'
+CTX_BUBBLE_LOCATION='X-Bubble-Location'
 CTX_BUBBLE_PASSTHRU='X-Bubble-Passthru'
 CTX_BUBBLE_REQUEST_ID='X-Bubble-RequestId'
 CTX_CONTENT_LENGTH='X-Bubble-Content-Length'
@@ -29,6 +35,14 @@ REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
 BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_'
 BUBBLE_ACTIVITY_LOG_EXPIRATION = 600
 
+LOCAL_IPS = []
+for ip in subprocess.check_output(['hostname', '-I']).split():
+    LOCAL_IPS.append(ip.decode())
+
+
+VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4)
+VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6)
+
 # This regex extracts splits the host header into host and port.
 # Handles the edge case of IPv6 addresses containing colons.
 # https://bugzilla.mozilla.org/show_bug.cgi?id=45891
@@ -170,3 +184,17 @@ def get_flow_ctx(flow, name):
     if not name in flow.bubble_ctx:
         return None
     return flow.bubble_ctx[name]
+
+
+def is_bubble_request(ip, fqdns):
+    # return ip in LOCAL_IPS
+    return ip in LOCAL_IPS and (bubble_host in fqdns or bubble_host_alias in fqdns)
+
+
+def is_sage_request(ip, fqdns):
+    return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns
+
+
+def is_not_from_vpn(client_addr):
+    ip = IPAddress(client_addr)
+    return ip not in VPN_IP4_CIDR and ip not in VPN_IP6_CIDR

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py

@@ -27,14 +27,11 @@ from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
 from mitmproxy.exceptions import TlsProtocolException
 from mitmproxy.net import tls as net_tls
 
-from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set
-from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host
-from bubble_vpn4 import wireguard_network_ipv4
-from bubble_vpn6 import wireguard_network_ipv6
-from netaddr import IPAddress, IPNetwork
 import json
-import subprocess
 import traceback
+from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set, \
+    is_bubble_request, is_sage_request, is_not_from_vpn
+from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host
 
 REDIS_DNS_PREFIX = 'bubble_dns_'
 REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_'
@@ -52,12 +49,6 @@ SEC_STD = 'standard'
 SEC_BASIC = 'basic'
 SEC_OFF = 'disabled'
 
-local_ips = None
-
-VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4)
-VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6)
-
-
 def get_device_security_level(client_addr, fqdns):
     level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr)
     if level is None:
@@ -84,28 +75,6 @@ def show_block_stats(client_addr):
         return False
     return show.decode() == 'true'
 
-def get_local_ips():
-    global local_ips
-    if local_ips is None:
-        local_ips = []
-        for ip in subprocess.check_output(['hostname', '-I']).split():
-            local_ips.append(ip.decode())
-    return local_ips
-
-
-def is_bubble_request(ip, fqdns):
-    # return ip in get_local_ips()
-    return ip in get_local_ips() and (bubble_host in fqdns or bubble_host_alias in fqdns)
-
-
-def is_sage_request(ip, fqdns):
-    return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns
-
-
-def is_not_from_vpn(client_addr):
-    ip = IPAddress(client_addr)
-    return ip not in VPN_IP4_CIDR and ip not in VPN_IP6_CIDR
-
 
 def conn_check_cache_prefix(client_addr, server_addr):
     return REDIS_CONN_CHECK_PREFIX + client_addr + '_' + server_addr
@@ -255,6 +224,7 @@ def next_layer(next_layer):
             check = FORCE_PASSTHRU
 
         elif is_not_from_vpn(client_addr):
+            # todo: add to fail2ban
             bubble_log('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns))
             bubble_activity_log(client_addr, server_addr, 'conn_block_non_vpn', fqdns)
             next_layer.__class__ = TlsBlock

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -8,7 +8,7 @@ import uuid
 import traceback
 from mitmproxy.net.http import Headers
 from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn
-from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \
+from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, BUBBLE_URI_PREFIX, \
     CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \
     HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header
 
@@ -17,6 +17,7 @@ HEADER_CONTENT_TYPE = 'Content-Type'
 HEADER_CONTENT_LENGTH = 'Content-Length'
 HEADER_CONTENT_ENCODING = 'Content-Encoding'
 HEADER_TRANSFER_ENCODING = 'Transfer-Encoding'
+HEADER_LOCATION = 'Location'
 CONTENT_TYPE_BINARY = 'application/octet-stream'
 STANDARD_FILTER_HEADERS = {HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY}
 
@@ -200,10 +201,18 @@ def responseheaders(flow):
     else:
         abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT)
         if abort_code is not None:
-            bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code))
-            flow.response.headers = Headers()
-            flow.response.status_code = abort_code
-            flow.response.stream = lambda chunks: []
+            abort_location = get_flow_ctx(flow, CTX_BUBBLE_LOCATION)
+            if abort_location is not None:
+                bubble_log('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location)
+                flow.response.headers = Headers()
+                flow.response.headers[HEADER_LOCATION] = abort_location
+                flow.response.status_code = abort_code
+                flow.response.stream = lambda chunks: []
+            else:
+                bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code))
+                flow.response.headers = Headers()
+                flow.response.status_code = abort_code
+                flow.response.stream = lambda chunks: []
 
         else:
             req_id = get_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID)

bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py

@@ -5,8 +5,8 @@ import re
 import time
 import uuid
 from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, \
-    CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \
-    add_flow_ctx, parse_host_header
+    CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \
+    add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn
 from bubble_config import bubble_host, bubble_host_alias
 
 class Rerouter:
@@ -67,8 +67,9 @@ class Rerouter:
         return matcher_response
 
     def request(self, flow):
-        client_address = flow.client_conn.address[0]
-        server_address = flow.server_conn.address[0]
+        client_addr = flow.client_conn.address[0]
+        server_addr = flow.server_conn.address[0]
+        is_http = False
         if flow.client_conn.tls_established:
             flow.request.scheme = "https"
             sni = flow.client_conn.connection.get_servername()
@@ -77,6 +78,7 @@ class Rerouter:
             flow.request.scheme = "http"
             sni = None
             port = 80
+            is_http = True
 
         host_header = flow.request.host_header
         # bubble_log("dns_spoofing.request: host_header is "+repr(host_header))
@@ -93,12 +95,36 @@ class Rerouter:
             if host.startswith("b'"):
                 host = host[2:-1]
             log_url = flow.request.scheme + '://' + host + flow.request.path
+
+            # If https, we have already checked that the client/server are legal in bubble_conn_check.py
+            # If http, we validate client/server here
+            if is_http:
+                fqdns = [host]
+                if is_bubble_request(server_addr, fqdns):
+                    bubble_log('dns_spoofing.request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns))
+                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
+                    add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path)
+                    return
+
+                elif is_sage_request(server_addr, fqdns):
+                    bubble_log('dns_spoofing.request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr)
+                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
+                    add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path)
+                    return
+
+                elif is_not_from_vpn(client_addr):
+                    # todo: add to fail2ban
+                    bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns))
+                    bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns)
+                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
+                    return
+
             matcher_response = self.get_matchers(flow, sni or host_header)
             if matcher_response:
                 if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru':
                     bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...')
                     add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True)
-                    bubble_activity_log(client_address, server_address, 'http_passthru', log_url)
+                    bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url)
                     return
 
                 elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'):
@@ -111,12 +137,12 @@ class Rerouter:
                         bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found')
                         abort_code = 404
                     add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code)
-                    bubble_activity_log(client_address, server_address, 'http_abort' + str(abort_code), log_url)
+                    bubble_activity_log(client_addr, server_addr, 'http_abort' + str(abort_code), log_url)
                     return
 
                 elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match':
                     bubble_log('dns_spoofing.request: decision was no_match, passing thru...')
-                    bubble_activity_log(client_address, server_address, 'http_no_match', log_url)
+                    bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url)
                     return
 
                 elif ('matchers' in matcher_response
@@ -126,16 +152,24 @@ class Rerouter:
                     bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
                     add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers'])
                     add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id)
-                    bubble_activity_log(client_address, server_address, 'http_match', log_url)
+                    bubble_activity_log(client_addr, server_addr, 'http_match', log_url)
                 else:
                     bubble_log('dns_spoofing.request: no rules returned, passing thru...')
-                    bubble_activity_log(client_address, server_address, 'http_no_rules', log_url)
+                    bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url)
             else:
                 bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...')
-                # bubble_activity_log(client_address, server_address, 'http_no_matcher_response', log_url)
+                # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url)
+
+        elif is_http and is_not_from_vpn(client_addr):
+            # todo: add to fail2ban
+            bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr)
+            bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr])
+            add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
+            return
+
         else:
             bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path)
-            bubble_activity_log(client_address, server_address, 'http_no_sni_or_host', 'n/a')
+            bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr])
 
         flow.request.host_header = host_header
         flow.request.host = sni or host_header