@@ -1,16 +1,21 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
import requests | |||
import traceback | |||
import datetime | |||
import json | |||
import re | |||
import requests | |||
import redis | |||
import subprocess | |||
import sys | |||
import time | |||
import traceback | |||
import uuid | |||
import datetime | |||
import redis | |||
import json | |||
from bubble_config import bubble_network, bubble_port, debug_capture_fqdn | |||
from netaddr import IPAddress, IPNetwork | |||
from bubble_vpn4 import wireguard_network_ipv4 | |||
from bubble_vpn6 import wireguard_network_ipv6 | |||
from bubble_config import bubble_network, bubble_port, debug_capture_fqdn, \ | |||
bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host | |||
HEADER_USER_AGENT = 'User-Agent' | |||
HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy' | |||
@@ -19,6 +24,7 @@ HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru' | |||
CTX_BUBBLE_MATCHERS='X-Bubble-Matchers' | |||
CTX_BUBBLE_ABORT='X-Bubble-Abort' | |||
CTX_BUBBLE_LOCATION='X-Bubble-Location' | |||
CTX_BUBBLE_PASSTHRU='X-Bubble-Passthru' | |||
CTX_BUBBLE_REQUEST_ID='X-Bubble-RequestId' | |||
CTX_CONTENT_LENGTH='X-Bubble-Content-Length' | |||
@@ -29,6 +35,14 @@ REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) | |||
BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_' | |||
BUBBLE_ACTIVITY_LOG_EXPIRATION = 600 | |||
LOCAL_IPS = [] | |||
for ip in subprocess.check_output(['hostname', '-I']).split(): | |||
LOCAL_IPS.append(ip.decode()) | |||
VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4) | |||
VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6) | |||
# This regex extracts splits the host header into host and port. | |||
# Handles the edge case of IPv6 addresses containing colons. | |||
# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 | |||
@@ -170,3 +184,17 @@ def get_flow_ctx(flow, name): | |||
if not name in flow.bubble_ctx: | |||
return None | |||
return flow.bubble_ctx[name] | |||
def is_bubble_request(ip, fqdns): | |||
# return ip in LOCAL_IPS | |||
return ip in LOCAL_IPS and (bubble_host in fqdns or bubble_host_alias in fqdns) | |||
def is_sage_request(ip, fqdns): | |||
return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns | |||
def is_not_from_vpn(client_addr): | |||
ip = IPAddress(client_addr) | |||
return ip not in VPN_IP4_CIDR and ip not in VPN_IP6_CIDR |
@@ -27,14 +27,11 @@ from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer | |||
from mitmproxy.exceptions import TlsProtocolException | |||
from mitmproxy.net import tls as net_tls | |||
from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set | |||
from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host | |||
from bubble_vpn4 import wireguard_network_ipv4 | |||
from bubble_vpn6 import wireguard_network_ipv6 | |||
from netaddr import IPAddress, IPNetwork | |||
import json | |||
import subprocess | |||
import traceback | |||
from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set, \ | |||
is_bubble_request, is_sage_request, is_not_from_vpn | |||
from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host | |||
REDIS_DNS_PREFIX = 'bubble_dns_' | |||
REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' | |||
@@ -52,12 +49,6 @@ SEC_STD = 'standard' | |||
SEC_BASIC = 'basic' | |||
SEC_OFF = 'disabled' | |||
local_ips = None | |||
VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4) | |||
VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6) | |||
def get_device_security_level(client_addr, fqdns): | |||
level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr) | |||
if level is None: | |||
@@ -84,28 +75,6 @@ def show_block_stats(client_addr): | |||
return False | |||
return show.decode() == 'true' | |||
def get_local_ips(): | |||
global local_ips | |||
if local_ips is None: | |||
local_ips = [] | |||
for ip in subprocess.check_output(['hostname', '-I']).split(): | |||
local_ips.append(ip.decode()) | |||
return local_ips | |||
def is_bubble_request(ip, fqdns): | |||
# return ip in get_local_ips() | |||
return ip in get_local_ips() and (bubble_host in fqdns or bubble_host_alias in fqdns) | |||
def is_sage_request(ip, fqdns): | |||
return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns | |||
def is_not_from_vpn(client_addr): | |||
ip = IPAddress(client_addr) | |||
return ip not in VPN_IP4_CIDR and ip not in VPN_IP6_CIDR | |||
def conn_check_cache_prefix(client_addr, server_addr): | |||
return REDIS_CONN_CHECK_PREFIX + client_addr + '_' + server_addr | |||
@@ -255,6 +224,7 @@ def next_layer(next_layer): | |||
check = FORCE_PASSTHRU | |||
elif is_not_from_vpn(client_addr): | |||
# todo: add to fail2ban | |||
bubble_log('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns)) | |||
bubble_activity_log(client_addr, server_addr, 'conn_block_non_vpn', fqdns) | |||
next_layer.__class__ = TlsBlock | |||
@@ -8,7 +8,7 @@ import uuid | |||
import traceback | |||
from mitmproxy.net.http import Headers | |||
from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn | |||
from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \ | |||
from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, BUBBLE_URI_PREFIX, \ | |||
CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \ | |||
HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header | |||
@@ -17,6 +17,7 @@ HEADER_CONTENT_TYPE = 'Content-Type' | |||
HEADER_CONTENT_LENGTH = 'Content-Length' | |||
HEADER_CONTENT_ENCODING = 'Content-Encoding' | |||
HEADER_TRANSFER_ENCODING = 'Transfer-Encoding' | |||
HEADER_LOCATION = 'Location' | |||
CONTENT_TYPE_BINARY = 'application/octet-stream' | |||
STANDARD_FILTER_HEADERS = {HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY} | |||
@@ -200,10 +201,18 @@ def responseheaders(flow): | |||
else: | |||
abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT) | |||
if abort_code is not None: | |||
bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code)) | |||
flow.response.headers = Headers() | |||
flow.response.status_code = abort_code | |||
flow.response.stream = lambda chunks: [] | |||
abort_location = get_flow_ctx(flow, CTX_BUBBLE_LOCATION) | |||
if abort_location is not None: | |||
bubble_log('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location) | |||
flow.response.headers = Headers() | |||
flow.response.headers[HEADER_LOCATION] = abort_location | |||
flow.response.status_code = abort_code | |||
flow.response.stream = lambda chunks: [] | |||
else: | |||
bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code)) | |||
flow.response.headers = Headers() | |||
flow.response.status_code = abort_code | |||
flow.response.stream = lambda chunks: [] | |||
else: | |||
req_id = get_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID) | |||
@@ -5,8 +5,8 @@ import re | |||
import time | |||
import uuid | |||
from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, \ | |||
CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \ | |||
add_flow_ctx, parse_host_header | |||
CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \ | |||
add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn | |||
from bubble_config import bubble_host, bubble_host_alias | |||
class Rerouter: | |||
@@ -67,8 +67,9 @@ class Rerouter: | |||
return matcher_response | |||
def request(self, flow): | |||
client_address = flow.client_conn.address[0] | |||
server_address = flow.server_conn.address[0] | |||
client_addr = flow.client_conn.address[0] | |||
server_addr = flow.server_conn.address[0] | |||
is_http = False | |||
if flow.client_conn.tls_established: | |||
flow.request.scheme = "https" | |||
sni = flow.client_conn.connection.get_servername() | |||
@@ -77,6 +78,7 @@ class Rerouter: | |||
flow.request.scheme = "http" | |||
sni = None | |||
port = 80 | |||
is_http = True | |||
host_header = flow.request.host_header | |||
# bubble_log("dns_spoofing.request: host_header is "+repr(host_header)) | |||
@@ -93,12 +95,36 @@ class Rerouter: | |||
if host.startswith("b'"): | |||
host = host[2:-1] | |||
log_url = flow.request.scheme + '://' + host + flow.request.path | |||
# If https, we have already checked that the client/server are legal in bubble_conn_check.py | |||
# If http, we validate client/server here | |||
if is_http: | |||
fqdns = [host] | |||
if is_bubble_request(server_addr, fqdns): | |||
bubble_log('dns_spoofing.request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns)) | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) | |||
add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path) | |||
return | |||
elif is_sage_request(server_addr, fqdns): | |||
bubble_log('dns_spoofing.request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr) | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) | |||
add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path) | |||
return | |||
elif is_not_from_vpn(client_addr): | |||
# todo: add to fail2ban | |||
bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns)) | |||
bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns) | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) | |||
return | |||
matcher_response = self.get_matchers(flow, sni or host_header) | |||
if matcher_response: | |||
if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru': | |||
bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...') | |||
add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) | |||
bubble_activity_log(client_address, server_address, 'http_passthru', log_url) | |||
bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url) | |||
return | |||
elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'): | |||
@@ -111,12 +137,12 @@ class Rerouter: | |||
bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') | |||
abort_code = 404 | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code) | |||
bubble_activity_log(client_address, server_address, 'http_abort' + str(abort_code), log_url) | |||
bubble_activity_log(client_addr, server_addr, 'http_abort' + str(abort_code), log_url) | |||
return | |||
elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match': | |||
bubble_log('dns_spoofing.request: decision was no_match, passing thru...') | |||
bubble_activity_log(client_address, server_address, 'http_no_match', log_url) | |||
bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url) | |||
return | |||
elif ('matchers' in matcher_response | |||
@@ -126,16 +152,24 @@ class Rerouter: | |||
bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) | |||
add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) | |||
add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) | |||
bubble_activity_log(client_address, server_address, 'http_match', log_url) | |||
bubble_activity_log(client_addr, server_addr, 'http_match', log_url) | |||
else: | |||
bubble_log('dns_spoofing.request: no rules returned, passing thru...') | |||
bubble_activity_log(client_address, server_address, 'http_no_rules', log_url) | |||
bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url) | |||
else: | |||
bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...') | |||
# bubble_activity_log(client_address, server_address, 'http_no_matcher_response', log_url) | |||
# bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) | |||
elif is_http and is_not_from_vpn(client_addr): | |||
# todo: add to fail2ban | |||
bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr) | |||
bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr]) | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) | |||
return | |||
else: | |||
bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path) | |||
bubble_activity_log(client_address, server_address, 'http_no_sni_or_host', 'n/a') | |||
bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr]) | |||
flow.request.host_header = host_header | |||
flow.request.host = sni or host_header | |||