diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index d5d04fbf..1289fd10 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -56,25 +56,6 @@ public class ApiConstants { public static final ObjectMapper DB_JSON_MAPPER = COMPACT_MAPPER; - // for some reason @Getter(lazy=true) causes compilation problems when other classes try to call getter - // so we implement lombok lazy-getter logic manuall here - private static final AtomicReference debugFqdn = new AtomicReference<>(); - public static String getDebugFqdn () { - if (debugFqdn.get() == null) { - synchronized (debugFqdn) { - if (debugFqdn.get() == null) { - try { - debugFqdn.set(FileUtil.toString(HOME_DIR + "/debug_fqdn").trim()); - } catch (Exception e) { - log.debug("initDebugFqdn: " + shortError(e)); - debugFqdn.set("~debug_fqdn_disabled~"); // will never match any fqdn - } - } - } - } - return debugFqdn.get(); - } - private static String initDefaultDomain() { final File f = new File(HOME_DIR, ".BUBBLE_DEFAULT_DOMAIN"); final String domain = FileUtil.toStringOrDie(f); diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 0c684a52..44d75656 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; import static bubble.ApiConstants.*; import static bubble.resources.stream.FilterMatchersResponse.NO_MATCHERS; +import static bubble.service.stream.HttpStreamDebug.getLogFqdn; import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIMEOUT; import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY; import static java.util.Collections.emptyMap; @@ -101,7 +102,7 @@ public class FilterHttpResource { final RedisService cache = ruleEngine.getMatchersCache(); final String requestId = filterRequest.getRequestId(); - final boolean extraLog = filterRequest.getFqdn().contains(getDebugFqdn()); + final boolean extraLog = filterRequest.getFqdn().contains(getLogFqdn()); final String prefix = "getMatchersResponse("+requestId+"): "; final String cacheKey = filterRequest.cacheKey(); final String matchersJson = cache.get(cacheKey); @@ -195,7 +196,7 @@ public class FilterHttpResource { @Context ContainerRequest request, @PathParam("requestId") String requestId, FilterMatchersRequest filterRequest) { - boolean extraLog = requestId.contains(getDebugFqdn()); + boolean extraLog = requestId.contains(getLogFqdn()); if (filterRequest == null || !filterRequest.hasRequestId() || empty(requestId) || !requestId.equals(filterRequest.getRequestId())) { if (log.isDebugEnabled()) log.debug("selectMatchers: no filterRequest, missing requestId, or mismatch, returning forbidden"); else if (extraLog) log.error("selectMatchers: no filterRequest, missing requestId, or mismatch, returning forbidden"); @@ -223,6 +224,7 @@ public class FilterHttpResource { log.trace(prefix+"found device "+device.id()+" for IP "+vpnAddr); } filterRequest.setDevice(device.getUuid()); + final FilterMatchersResponse response = getMatchersResponse(filterRequest, req, request); if (log.isDebugEnabled()) log.debug(prefix+"returning response: "+json(response, COMPACT_MAPPER)); else if (extraLog) log.error(prefix+"returning response: "+json(response, COMPACT_MAPPER)); @@ -231,7 +233,7 @@ public class FilterHttpResource { private FilterMatchersResponse findMatchers(FilterMatchersRequest filterRequest, Request req, ContainerRequest request) { final String requestId = filterRequest.getRequestId(); - boolean extraLog = requestId.contains(getDebugFqdn()); + boolean extraLog = requestId.contains(getLogFqdn()); final String prefix = "findMatchers("+ requestId +"): "; final Device device = findDevice(filterRequest.getDevice()); if (device == null) { @@ -298,7 +300,7 @@ public class FilterHttpResource { } private List getEnabledMatchers(String requestId, String accountUuid, String fqdn) { - boolean extraLog = fqdn.contains(getDebugFqdn()); + boolean extraLog = fqdn.contains(getLogFqdn()); final String prefix = "getEnabledMatchers("+requestId+"): "; List matchers = matcherDAO.findByAccountAndFqdnAndEnabledAndRequestCheck(accountUuid, fqdn); if (log.isTraceEnabled()) log.trace(prefix+"checking all enabled matchers for fqdn: "+json(matchers, COMPACT_MAPPER)); diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java index b60c2ee6..882b0a01 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java @@ -10,8 +10,7 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; @NoArgsConstructor @Accessors(chain=true) public class FilterMatchersRequest { diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index ea3f281c..8ed12b96 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -36,7 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import static bubble.ApiConstants.getDebugFqdn; +import static bubble.service.stream.HttpStreamDebug.getLogFqdn; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; import static org.cobbzilla.util.http.HttpContentTypes.isHtml; @@ -161,7 +161,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { Device device, Request req, ContainerRequest request) { - final boolean extraLog = filter.getFqdn().contains(getDebugFqdn()); + final boolean extraLog = filter.getFqdn().contains(getLogFqdn()); final String app = ruleHarness.getRule().getApp(); final String site = ruleHarness.getMatcher().getSite(); final String fqdn = filter.getFqdn(); diff --git a/bubble-server/src/main/java/bubble/service/stream/HttpStreamDebug.java b/bubble-server/src/main/java/bubble/service/stream/HttpStreamDebug.java new file mode 100644 index 00000000..06565308 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/stream/HttpStreamDebug.java @@ -0,0 +1,22 @@ +package bubble.service.stream; + +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.io.FileUtil; + +import java.util.concurrent.atomic.AtomicReference; + +import static bubble.ApiConstants.HOME_DIR; +import static org.cobbzilla.util.daemon.ZillaRuntime.lazyGet; + +@Slf4j +public class HttpStreamDebug { + + // for some reason @Getter(lazy=true) causes compilation problems when other classes try to call getter + private static final AtomicReference logFqdn = new AtomicReference<>(); + public static String getLogFqdn() { + return lazyGet(logFqdn, + () -> FileUtil.toStringOrDie(HOME_DIR + "/log_fqdn").trim(), + () -> "~log_fqdn_disabled"); + } + +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java index c31f2a8e..b36d963c 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java @@ -46,7 +46,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.ws.rs.core.Response; -import java.io.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -63,7 +65,6 @@ import static org.apache.http.HttpHeaders.TRANSFER_ENCODING; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf; import static org.cobbzilla.util.http.HttpStatusCodes.OK; -import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.wizard.cache.redis.RedisService.ALL_KEYS; @@ -153,8 +154,6 @@ public class StandardRuleEngineService implements RuleEngineService { private final Map activeProcessors = new ExpirationMap<>(MINUTES.toMillis(5), ExpirationEvictionPolicy.atime); - private static final boolean DEBUG_CAPTURE = false; - public Response applyRulesToChunkAndSendResponse(ContainerRequest request, FilterHttpRequest filterRequest, Integer chunkLength, @@ -167,17 +166,6 @@ public class StandardRuleEngineService implements RuleEngineService { log.info(prefix+" applying matchers: "+filterRequest.getMatcherNames()); } - // for debugging problematic requests - if (DEBUG_CAPTURE) { - final byte[] bytes = IOUtils.toByteArray(request.getEntityStream()); - final File temp = new File("/tmp/"+filterRequest.getId()+".raw"); - try (FileOutputStream out = new FileOutputStream(temp, true)) { - log.debug(prefix+"stashed "+bytes.length+" bytes in "+abs(temp)); - IOUtils.copy(new ByteArrayInputStream(bytes), out); - } - return sendResponse(new ByteArrayInputStream(bytes)); // noop for testing - } - // have we seen this request before? final ActiveStreamState state = activeProcessors.computeIfAbsent(filterRequest.getId(), k -> new ActiveStreamState(filterRequest, initRules(filterRequest))); diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2 b/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2 index 0651a0d4..47f9d248 100644 --- a/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2 +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2 @@ -6,4 +6,5 @@ bubble_ssl_port = '{{ ssl_port }}' bubble_sage_host = '{{ sage_host }}' bubble_sage_ip4 = '{{ sage_ip4 }}' bubble_sage_ip6 = '{{ sage_ip6 }}' -cert_validation_host = '{{ cert_validation_host }}' \ No newline at end of file +cert_validation_host = '{{ cert_validation_host }}' +debug_capture_fqdn = None diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 687d1a00..f9334c10 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -3,14 +3,14 @@ # import requests import traceback +import re import sys -import os import time import uuid import datetime import redis import json -from bubble_config import bubble_network, bubble_port +from bubble_config import bubble_network, bubble_port, debug_capture_fqdn HEADER_USER_AGENT = 'User-Agent' HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy' @@ -29,6 +29,12 @@ REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_' BUBBLE_ACTIVITY_LOG_EXPIRATION = 600 +# This regex extracts splits the host header into host and port. +# Handles the edge case of IPv6 addresses containing colons. +# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 +parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") + + def redis_set(name, value, ex): REDIS.set(name, value, nx=True, ex=ex) REDIS.set(name, value, xx=True, ex=ex) @@ -53,6 +59,10 @@ def bubble_activity_log(client_addr, server_addr, event, data): def bubble_conn_check(remote_addr, addr, fqdns, security_level): + if debug_capture_fqdn and fqdns and debug_capture_fqdn in fqdns: + bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+debug_capture_fqdn) + return 'noop' + headers = { 'X-Forwarded-For': remote_addr, 'Accept' : 'application/json', @@ -78,7 +88,22 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level): return None +DEBUG_MATCHER_NAME = 'DebugCaptureMatcher' +DEBUG_MATCHER = { + 'decision': 'match', + 'matchers': [{ + 'name': DEBUG_MATCHER_NAME, + 'contentTypeRegex': '.*', + "urlRegex": ".*", + 'rule': DEBUG_MATCHER_NAME + }] +} + def bubble_matchers(req_id, remote_addr, flow, host): + if debug_capture_fqdn and host and debug_capture_fqdn == host: + bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+debug_capture_fqdn) + return DEBUG_MATCHER + headers = { 'X-Forwarded-For': remote_addr, 'Accept' : 'application/json', diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index 3498b564..adbe2d51 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -206,6 +206,7 @@ def next_layer(next_layer): next_layer.fqdns = fqdns no_fqdns = fqdns is None or len(fqdns) == 0 security_level = get_device_security_level(client_addr) + check = None if server_addr in get_local_ips(): bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) check = FORCE_PASSTHRU diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 657e47a6..de3c1d64 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -6,10 +6,10 @@ import requests import urllib import traceback from mitmproxy.net.http import Headers -from bubble_config import bubble_port, bubble_host_alias +from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \ - HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set + HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header BUFFER_SIZE = 4096 HEADER_CONTENT_TYPE = 'Content-Type' @@ -23,6 +23,31 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__' REDIS_FILTER_PASSTHRU_DURATION = 600 def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type=None, content_length=None, csp=None): + if debug_capture_fqdn: + host = None + if flow.client_conn.tls_established: + sni = flow.client_conn.connection.get_servername() + if sni: + host = str(sni) + else: + host_header = flow.request.host_header + if host_header: + m = parse_host_header.match(host_header) + if m: + host = str(m.group("host").strip("[]")) + if host: + if host.startswith("b'"): + host = host[2:-1] + if host == debug_capture_fqdn: + bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn) + f = open('/tmp/bubble_capture_'+req_id, mode='ab', buffering=0) + f.write(chunk) + f.close() + return chunk + else: + bubble_log('filter_chunk: debug_capture_fqdn detected but host='+repr(host)+', NOT capturing: '+debug_capture_fqdn) + else: + bubble_log('filter_chunk: debug_capture_fqdn detected but no host could be detected, NOT capturing: '+debug_capture_fqdn) # should we just passthru? redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + ':' + flow.request.url @@ -161,10 +186,10 @@ def responseheaders(flow): typeRegex = '^text/html.*' if re.match(typeRegex, content_type): any_content_type_matches = True - bubble_log(prefix+': found at least one matcher for content_type ('+content_type+'), filtering') + bubble_log(prefix+'found at least one matcher for content_type ('+content_type+'), filtering') break if not any_content_type_matches: - bubble_log(prefix+': no matchers for content_type ('+content_type+'), passing thru') + bubble_log(prefix+'no matchers for content_type ('+content_type+'), passing thru') return if HEADER_CONTENT_ENCODING in flow.response.headers: @@ -178,7 +203,7 @@ def responseheaders(flow): csp = None content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) - bubble_log(prefix+': content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) + bubble_log(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type, csp) if content_length_value: flow.response.headers['transfer-encoding'] = 'chunked' diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index 74990d86..78753473 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -4,15 +4,11 @@ import re import time import uuid -from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx +from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, \ + CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \ + add_flow_ctx, parse_host_header from bubble_config import bubble_host, bubble_host_alias -# This regex extracts splits the host header into host and port. -# Handles the edge case of IPv6 addresses containing colons. -# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 -parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") - - class Rerouter: @staticmethod def get_matchers(flow, host): diff --git a/bubble-web b/bubble-web index 6c5c3f03..a5f611e0 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 6c5c3f03f325972a0c6ea7c6f6d3a479f2140a99 +Subproject commit a5f611e010290965b1d2e1b100493fccc672579d diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index cbd3cec3..a39467db 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit cbd3cec3fc2d393cef9f251d05b4e3afb40bc945 +Subproject commit a39467dbcd062ed1471e44450d495ba7a9bd8942