move debugFqdn from ApiConstants to HttpStreamDebug, rename to logFqdn. move captureFqdn functionality to mitm. update web

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -56,25 +56,6 @@ public class ApiConstants {
 
     public static final ObjectMapper DB_JSON_MAPPER = COMPACT_MAPPER;
 
-    // for some reason @Getter(lazy=true) causes compilation problems when other classes try to call getter
-    // so we implement lombok lazy-getter logic manuall here
-    private static final AtomicReference<String> debugFqdn = new AtomicReference<>();
-    public static String getDebugFqdn () {
-        if (debugFqdn.get() == null) {
-            synchronized (debugFqdn) {
-                if (debugFqdn.get() == null) {
-                    try {
-                        debugFqdn.set(FileUtil.toString(HOME_DIR + "/debug_fqdn").trim());
-                    } catch (Exception e) {
-                        log.debug("initDebugFqdn: " + shortError(e));
-                        debugFqdn.set("~debug_fqdn_disabled~");  // will never match any fqdn
-                    }
-                }
-            }
-        }
-        return debugFqdn.get();
-    }
-
     private static String initDefaultDomain() {
         final File f = new File(HOME_DIR, ".BUBBLE_DEFAULT_DOMAIN");
         final String domain = FileUtil.toStringOrDie(f);

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -43,6 +43,7 @@ import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.*;
 import static bubble.resources.stream.FilterMatchersResponse.NO_MATCHERS;
+import static bubble.service.stream.HttpStreamDebug.getLogFqdn;
 import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIMEOUT;
 import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY;
 import static java.util.Collections.emptyMap;
@@ -101,7 +102,7 @@ public class FilterHttpResource {
         final RedisService cache = ruleEngine.getMatchersCache();
 
         final String requestId = filterRequest.getRequestId();
-        final boolean extraLog = filterRequest.getFqdn().contains(getDebugFqdn());
+        final boolean extraLog = filterRequest.getFqdn().contains(getLogFqdn());
         final String prefix = "getMatchersResponse("+requestId+"): ";
         final String cacheKey = filterRequest.cacheKey();
         final String matchersJson = cache.get(cacheKey);
@@ -195,7 +196,7 @@ public class FilterHttpResource {
                                    @Context ContainerRequest request,
                                    @PathParam("requestId") String requestId,
                                    FilterMatchersRequest filterRequest) {
-        boolean extraLog = requestId.contains(getDebugFqdn());
+        boolean extraLog = requestId.contains(getLogFqdn());
         if (filterRequest == null || !filterRequest.hasRequestId() || empty(requestId) || !requestId.equals(filterRequest.getRequestId())) {
             if (log.isDebugEnabled()) log.debug("selectMatchers: no filterRequest, missing requestId, or mismatch, returning forbidden");
             else if (extraLog) log.error("selectMatchers: no filterRequest, missing requestId, or mismatch, returning forbidden");
@@ -223,6 +224,7 @@ public class FilterHttpResource {
             log.trace(prefix+"found device "+device.id()+" for IP "+vpnAddr);
         }
         filterRequest.setDevice(device.getUuid());
+
         final FilterMatchersResponse response = getMatchersResponse(filterRequest, req, request);
         if (log.isDebugEnabled()) log.debug(prefix+"returning response: "+json(response, COMPACT_MAPPER));
         else if (extraLog) log.error(prefix+"returning response: "+json(response, COMPACT_MAPPER));
@@ -231,7 +233,7 @@ public class FilterHttpResource {
 
     private FilterMatchersResponse findMatchers(FilterMatchersRequest filterRequest, Request req, ContainerRequest request) {
         final String requestId = filterRequest.getRequestId();
-        boolean extraLog = requestId.contains(getDebugFqdn());
+        boolean extraLog = requestId.contains(getLogFqdn());
         final String prefix = "findMatchers("+ requestId +"): ";
         final Device device = findDevice(filterRequest.getDevice());
         if (device == null) {
@@ -298,7 +300,7 @@ public class FilterHttpResource {
     }
 
     private List<AppMatcher> getEnabledMatchers(String requestId, String accountUuid, String fqdn) {
-        boolean extraLog = fqdn.contains(getDebugFqdn());
+        boolean extraLog = fqdn.contains(getLogFqdn());
         final String prefix = "getEnabledMatchers("+requestId+"): ";
         List<AppMatcher> matchers = matcherDAO.findByAccountAndFqdnAndEnabledAndRequestCheck(accountUuid, fqdn);
         if (log.isTraceEnabled()) log.trace(prefix+"checking all enabled matchers for fqdn: "+json(matchers, COMPACT_MAPPER));

bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java

@@ -10,8 +10,7 @@ import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.Accessors;
 
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 
 @NoArgsConstructor @Accessors(chain=true)
 public class FilterMatchersRequest {

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -36,7 +36,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static bubble.ApiConstants.getDebugFqdn;
+import static bubble.service.stream.HttpStreamDebug.getLogFqdn;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 import static org.cobbzilla.util.http.HttpContentTypes.isHtml;
@@ -161,7 +161,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
                                                     Device device,
                                                     Request req,
                                                     ContainerRequest request) {
-        final boolean extraLog = filter.getFqdn().contains(getDebugFqdn());
+        final boolean extraLog = filter.getFqdn().contains(getLogFqdn());
         final String app = ruleHarness.getRule().getApp();
         final String site = ruleHarness.getMatcher().getSite();
         final String fqdn = filter.getFqdn();

bubble-server/src/main/java/bubble/service/stream/HttpStreamDebug.java

@@ -0,0 +1,22 @@
+package bubble.service.stream;
+
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.io.FileUtil;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import static bubble.ApiConstants.HOME_DIR;
+import static org.cobbzilla.util.daemon.ZillaRuntime.lazyGet;
+
+@Slf4j
+public class HttpStreamDebug {
+
+    // for some reason @Getter(lazy=true) causes compilation problems when other classes try to call getter
+    private static final AtomicReference<String> logFqdn = new AtomicReference<>();
+    public static String getLogFqdn() {
+        return lazyGet(logFqdn,
+                () -> FileUtil.toStringOrDie(HOME_DIR + "/log_fqdn").trim(),
+                () -> "~log_fqdn_disabled");
+    }
+
+}

bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java

@@ -46,7 +46,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.ws.rs.core.Response;
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -63,7 +65,6 @@ import static org.apache.http.HttpHeaders.TRANSFER_ENCODING;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf;
 import static org.cobbzilla.util.http.HttpStatusCodes.OK;
-import static org.cobbzilla.util.io.FileUtil.abs;
 import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.wizard.cache.redis.RedisService.ALL_KEYS;
@@ -153,8 +154,6 @@ public class StandardRuleEngineService implements RuleEngineService {
 
     private final Map<String, ActiveStreamState> activeProcessors = new ExpirationMap<>(MINUTES.toMillis(5), ExpirationEvictionPolicy.atime);
 
-    private static final boolean DEBUG_CAPTURE = false;
-
     public Response applyRulesToChunkAndSendResponse(ContainerRequest request,
                                                      FilterHttpRequest filterRequest,
                                                      Integer chunkLength,
@@ -167,17 +166,6 @@ public class StandardRuleEngineService implements RuleEngineService {
             log.info(prefix+" applying matchers: "+filterRequest.getMatcherNames());
         }
 
-        // for debugging problematic requests
-        if (DEBUG_CAPTURE) {
-            final byte[] bytes = IOUtils.toByteArray(request.getEntityStream());
-            final File temp = new File("/tmp/"+filterRequest.getId()+".raw");
-            try (FileOutputStream out = new FileOutputStream(temp, true)) {
-                log.debug(prefix+"stashed "+bytes.length+" bytes in "+abs(temp));
-                IOUtils.copy(new ByteArrayInputStream(bytes), out);
-            }
-            return sendResponse(new ByteArrayInputStream(bytes)); // noop for testing
-        }
-
         // have we seen this request before?
         final ActiveStreamState state = activeProcessors.computeIfAbsent(filterRequest.getId(),
                 k -> new ActiveStreamState(filterRequest, initRules(filterRequest)));

bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2

@@ -6,4 +6,5 @@ bubble_ssl_port = '{{ ssl_port }}'
 bubble_sage_host = '{{ sage_host }}'
 bubble_sage_ip4 = '{{ sage_ip4 }}'
 bubble_sage_ip6 = '{{ sage_ip6 }}'
-cert_validation_host = '{{ cert_validation_host }}'
+cert_validation_host = '{{ cert_validation_host }}'
+debug_capture_fqdn = None

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -3,14 +3,14 @@
 #
 import requests
 import traceback
+import re
 import sys
-import os
 import time
 import uuid
 import datetime
 import redis
 import json
-from bubble_config import bubble_network, bubble_port
+from bubble_config import bubble_network, bubble_port, debug_capture_fqdn
 
 HEADER_USER_AGENT = 'User-Agent'
 HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy'
@@ -29,6 +29,12 @@ REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
 BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_'
 BUBBLE_ACTIVITY_LOG_EXPIRATION = 600
 
+# This regex extracts splits the host header into host and port.
+# Handles the edge case of IPv6 addresses containing colons.
+# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
+parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
+
+
 def redis_set(name, value, ex):
     REDIS.set(name, value, nx=True, ex=ex)
     REDIS.set(name, value, xx=True, ex=ex)
@@ -53,6 +59,10 @@ def bubble_activity_log(client_addr, server_addr, event, data):
 
 
 def bubble_conn_check(remote_addr, addr, fqdns, security_level):
+    if debug_capture_fqdn and fqdns and debug_capture_fqdn in fqdns:
+        bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+debug_capture_fqdn)
+        return 'noop'
+
     headers = {
         'X-Forwarded-For': remote_addr,
         'Accept' : 'application/json',
@@ -78,7 +88,22 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level):
         return None
 
 
+DEBUG_MATCHER_NAME = 'DebugCaptureMatcher'
+DEBUG_MATCHER = {
+    'decision': 'match',
+    'matchers': [{
+        'name': DEBUG_MATCHER_NAME,
+        'contentTypeRegex': '.*',
+        "urlRegex": ".*",
+        'rule': DEBUG_MATCHER_NAME
+    }]
+}
+
 def bubble_matchers(req_id, remote_addr, flow, host):
+    if debug_capture_fqdn and host and debug_capture_fqdn == host:
+        bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+debug_capture_fqdn)
+        return DEBUG_MATCHER
+
     headers = {
         'X-Forwarded-For': remote_addr,
         'Accept' : 'application/json',

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py

@@ -206,6 +206,7 @@ def next_layer(next_layer):
         next_layer.fqdns = fqdns
         no_fqdns = fqdns is None or len(fqdns) == 0
         security_level = get_device_security_level(client_addr)
+        check = None
         if server_addr in get_local_ips():
             bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
             check = FORCE_PASSTHRU

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -6,10 +6,10 @@ import requests
 import urllib
 import traceback
 from mitmproxy.net.http import Headers
-from bubble_config import bubble_port, bubble_host_alias
+from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn
 from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \
     CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \
-    HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set
+    HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header
 
 BUFFER_SIZE = 4096
 HEADER_CONTENT_TYPE = 'Content-Type'
@@ -23,6 +23,31 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__'
 REDIS_FILTER_PASSTHRU_DURATION = 600
 
 def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type=None, content_length=None, csp=None):
+    if debug_capture_fqdn:
+        host = None
+        if flow.client_conn.tls_established:
+            sni = flow.client_conn.connection.get_servername()
+            if sni:
+                host = str(sni)
+        else:
+            host_header = flow.request.host_header
+            if host_header:
+                m = parse_host_header.match(host_header)
+                if m:
+                    host = str(m.group("host").strip("[]"))
+        if host:
+            if host.startswith("b'"):
+                host = host[2:-1]
+            if host == debug_capture_fqdn:
+                bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn)
+                f = open('/tmp/bubble_capture_'+req_id, mode='ab', buffering=0)
+                f.write(chunk)
+                f.close()
+                return chunk
+            else:
+                bubble_log('filter_chunk: debug_capture_fqdn detected but host='+repr(host)+', NOT capturing: '+debug_capture_fqdn)
+        else:
+            bubble_log('filter_chunk: debug_capture_fqdn detected but no host could be detected, NOT capturing: '+debug_capture_fqdn)
 
     # should we just passthru?
     redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + ':' + flow.request.url
@@ -161,10 +186,10 @@ def responseheaders(flow):
                                     typeRegex = '^text/html.*'
                                 if re.match(typeRegex, content_type):
                                     any_content_type_matches = True
-                                    bubble_log(prefix+': found at least one matcher for content_type ('+content_type+'), filtering')
+                                    bubble_log(prefix+'found at least one matcher for content_type ('+content_type+'), filtering')
                                     break
                         if not any_content_type_matches:
-                            bubble_log(prefix+': no matchers for content_type ('+content_type+'), passing thru')
+                            bubble_log(prefix+'no matchers for content_type ('+content_type+'), passing thru')
                             return
 
                         if HEADER_CONTENT_ENCODING in flow.response.headers:
@@ -178,7 +203,7 @@ def responseheaders(flow):
                             csp = None
 
                         content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None)
-                        bubble_log(prefix+': content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
+                        bubble_log(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
                         flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type, csp)
                         if content_length_value:
                             flow.response.headers['transfer-encoding'] = 'chunked'

bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py

@@ -4,15 +4,11 @@
 import re
 import time
 import uuid
-from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx
+from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, \
+    CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \
+    add_flow_ctx, parse_host_header
 from bubble_config import bubble_host, bubble_host_alias
 
-# This regex extracts splits the host header into host and port.
-# Handles the edge case of IPv6 addresses containing colons.
-# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
-parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
-
-
 class Rerouter:
     @staticmethod
     def get_matchers(flow, host):

bubble-web

@@ -1 +1 @@
-Subproject commit 6c5c3f03f325972a0c6ea7c6f6d3a479f2140a99
+Subproject commit a5f611e010290965b1d2e1b100493fccc672579d

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit cbd3cec3fc2d393cef9f251d05b4e3afb40bc945
+Subproject commit a39467dbcd062ed1471e44450d495ba7a9bd8942