change passthru to conn_check, now supports blocking

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -207,7 +207,7 @@ public class ApiConstants {
     public static final String BUBBLE_MAGIC_ENDPOINT = "/.bubble";
 
     public static final String FILTER_HTTP_ENDPOINT = "/filter";
-    public static final String EP_PASSTHRU = "/passthru";
+    public static final String EP_CHECK = "/check";
     public static final String EP_APPLY = "/apply";
     public static final String EP_ASSETS = "/assets";
 

bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java

@@ -39,7 +39,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
                 and(
                         eq("account", account),
                         eq("enabled", true),
-                        eq("passthru", false),
+                        eq("connCheck", false),
                         or(
                                 eq("fqdn", fqdn),
                                 eq("fqdn", WILDCARD_FQDN)
@@ -47,8 +47,8 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
         ).addOrder(PRIORITY_ASC));
     }
 
-    public List<AppMatcher> findByAccountAndEnabledAndPassthru(String account) {
-        return findByFields("account", account, "enabled", true, "passthru", true);
+    public List<AppMatcher> findByAccountAndEnabledAndConnCheck(String account) {
+        return findByFields("account", account, "enabled", true, "connCheck", true);
     }
 
     public List<AppMatcher> findAllChangesSince(Long lastMod) {
@@ -56,7 +56,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
     }
 
     @Override public Object preCreate(AppMatcher matcher) {
-        if (matcher.getPassthru() == null) matcher.setPassthru(false);
+        if (matcher.getConnCheck() == null) matcher.setConnCheck(false);
         return super.preCreate(matcher);
     }
 

bubble-server/src/main/java/bubble/model/app/AppMatcher.java

@@ -45,7 +45,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
 public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, HasPriority {
 
     public static final String[] VALUE_FIELDS = {"fqdn", "urlRegex", "template", "enabled", "priority"};
-    public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "passthru");
+    public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "connCheck");
 
     public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE);
     public static final String WILDCARD_FQDN = "*";
@@ -131,8 +131,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H
 
     @ECSearchable @ECField(index=120, required=EntityFieldRequired.optional)
     @ECIndex @Column(nullable=false)
-    @Getter @Setter private Boolean passthru;
-    public boolean passthru () { return bool(passthru); }
+    @Getter @Setter private Boolean connCheck;
+    public boolean connCheck () { return bool(connCheck); }
 
     @ECSearchable @ECField(index=130)
     @Column(nullable=false)

bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java

@@ -21,8 +21,8 @@ public enum BubbleDeviceType {
     uninitialized (null, false),
     windows       (CertType.cer, true, DeviceSecurityLevel.maximum),
     macosx        (CertType.pem, true, DeviceSecurityLevel.maximum),
-    ios           (CertType.pem, true, DeviceSecurityLevel.standard),
-    android       (CertType.cer, true, DeviceSecurityLevel.basic),
+    ios           (CertType.pem, true, DeviceSecurityLevel.maximum),
+    android       (CertType.cer, true, DeviceSecurityLevel.standard),
     linux         (CertType.crt, true, DeviceSecurityLevel.maximum),
     firefox       (CertType.crt, false),
     other         (null, true, DeviceSecurityLevel.basic);
@@ -32,7 +32,7 @@ public enum BubbleDeviceType {
     @Getter private final DeviceSecurityLevel defaultSecurityLevel;
     public boolean hasDefaultSecurityLevel () { return defaultSecurityLevel != null; }
 
-    BubbleDeviceType (CertType certType, boolean selectable) { this(CertType.cer, selectable, null); }
+    BubbleDeviceType (CertType certType, boolean selectable) { this(certType, selectable, null); }
 
     @JsonCreator public static BubbleDeviceType fromString (String v) { return enumFromString(BubbleDeviceType.class, v); }
 

bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java → bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java

@@ -9,7 +9,7 @@ import lombok.Setter;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
-public class FilterPassthruRequest {
+public class FilterConnCheckRequest {
 
     @Getter @Setter private String addr;
     public boolean hasAddr() { return !empty(addr); }

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -21,6 +21,7 @@ import bubble.rule.FilterMatchDecision;
 import bubble.server.BubbleConfiguration;
 import bubble.service.boot.SelfNodeService;
 import bubble.service.cloud.DeviceIdService;
+import bubble.service.stream.ConnectionCheckResponse;
 import bubble.service.stream.StandardRuleEngineService;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -124,23 +125,23 @@ public class FilterHttpResource {
         return null;
     }
 
-    @POST @Path(EP_PASSTHRU)
+    @POST @Path(EP_CHECK)
     @Consumes(APPLICATION_JSON)
     @Produces(APPLICATION_JSON)
-    public Response isTlsPassthru(@Context Request req,
-                                  @Context ContainerRequest request,
-                                  FilterPassthruRequest passthruRequest) {
-        final String prefix = "isPassthru: ";
-        if (passthruRequest == null || !passthruRequest.hasAddr() || !passthruRequest.hasRemoteAddr()) {
-            if (log.isDebugEnabled()) log.debug(prefix+"invalid passthruRequest, returning forbidden");
+    public Response checkConnection(@Context Request req,
+                                    @Context ContainerRequest request,
+                                    FilterConnCheckRequest connCheckRequest) {
+        final String prefix = "checkConnection: ";
+        if (connCheckRequest == null || !connCheckRequest.hasAddr() || !connCheckRequest.hasRemoteAddr()) {
+            if (log.isDebugEnabled()) log.debug(prefix+"invalid connCheckRequest, returning forbidden");
             return forbidden();
         }
         validateMitmCall(req);
 
         // if the requested IP is the same as our IP, then always passthru
-        if (isForUs(passthruRequest)) return ok();
+        if (isForUs(connCheckRequest)) return ok();
 
-        final String vpnAddr = passthruRequest.getRemoteAddr();
+        final String vpnAddr = connCheckRequest.getRemoteAddr();
         final Device device = deviceIdService.findDeviceByIp(vpnAddr);
         if (device == null) {
             if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning not found");
@@ -154,7 +155,7 @@ public class FilterHttpResource {
             return notFound();
         }
 
-        final List<AppMatcher> matchers = matcherDAO.findByAccountAndEnabledAndPassthru(device.getAccount());
+        final List<AppMatcher> matchers = matcherDAO.findByAccountAndEnabledAndConnCheck(device.getAccount());
         final List<AppMatcher> retained = new ArrayList<>();
         for (AppMatcher matcher : matchers) {
             final BubbleApp app = appDAO.findByUuid(matcher.getApp());
@@ -164,23 +165,23 @@ public class FilterHttpResource {
             retained.add(matcher);
         }
 
-        final String[] fqdns = passthruRequest.getFqdns();
+        final String[] fqdns = connCheckRequest.getFqdns();
         for (String fqdn : fqdns) {
-            final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), fqdn);
-            if (passthru) {
-                if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+fqdn+"/"+passthruRequest.getAddr());
-                return ok();
+            final ConnectionCheckResponse checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn);
+            if (checkResponse != ConnectionCheckResponse.noop) {
+                if (log.isDebugEnabled()) log.debug(prefix + "returning "+checkResponse+" for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr());
+                return ok(checkResponse);
             }
         }
-        if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdns/addr="+Arrays.toString(fqdns)+"/"+passthruRequest.getAddr());
-        return notFound();
+        if (log.isDebugEnabled()) log.debug(prefix+"returning noop for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr());
+        return ok(ConnectionCheckResponse.noop);
     }
 
-    private boolean isForUs(FilterPassthruRequest passthruRequest) {
+    private boolean isForUs(FilterConnCheckRequest connCheckRequest) {
         final BubbleNode thisNode = selfNodeService.getThisNode();
-        return passthruRequest.hasAddr()
-                && (thisNode.hasIp4() && thisNode.getIp4().equals(passthruRequest.getAddr())
-                || thisNode.hasIp6() && thisNode.getIp6().equals(passthruRequest.getAddr()));
+        return connCheckRequest.hasAddr()
+                && (thisNode.hasIp4() && thisNode.getIp4().equals(connCheckRequest.getAddr())
+                || thisNode.hasIp6() && thisNode.getIp6().equals(connCheckRequest.getAddr()));
     }
 
     @POST @Path(EP_MATCHERS+"/{requestId}")

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -11,6 +11,7 @@ import bubble.model.device.Device;
 import bubble.resources.stream.FilterHttpRequest;
 import bubble.resources.stream.FilterMatchersRequest;
 import bubble.service.stream.AppRuleHarness;
+import bubble.service.stream.ConnectionCheckResponse;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jknack.handlebars.Handlebars;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
@@ -145,6 +146,12 @@ public interface AppRuleDriver {
         return null;
     }
 
-    default boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { return false; }
+    default ConnectionCheckResponse checkConnection(AppRuleHarness harness,
+                                                    Account account,
+                                                    Device device,
+                                                    String addr,
+                                                    String fqdn) {
+        return ConnectionCheckResponse.noop;
+    }
 
 }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -15,6 +15,7 @@ import bubble.rule.AppRuleDriver;
 import bubble.rule.FilterMatchDecision;
 import bubble.rule.analytics.TrafficAnalyticsRuleDriver;
 import bubble.service.stream.AppRuleHarness;
+import bubble.service.stream.ConnectionCheckResponse;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.ReaderInputStream;
@@ -108,14 +109,11 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         }
         blockList.set(newBlockList);
 
-        boolean doPrime = false;
         if (!newBlockList.getFullyBlockedDomains().equals(fullyBlockedDomains.get())) {
             fullyBlockedDomains.set(newBlockList.getFullyBlockedDomains());
-            doPrime = true;
         }
         if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) {
             partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains());
-            doPrime = true;
         }
 
         log.info("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size());
@@ -123,6 +121,24 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         log.info("refreshBlockLists: refreshed "+refreshed.size()+" block lists: "+StringUtil.toString(refreshed));
     }
 
+    @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness,
+                                                             Account account,
+                                                             Device device,
+                                                             String addr,
+                                                             String fqdn) {
+        final BlockDecision decision = getBlockList().getFqdnDecision(fqdn);
+        final BlockDecisionType decisionType = decision.getDecisionType();
+        switch (decisionType) {
+            case allow:
+                return ConnectionCheckResponse.noop;
+            case block:
+                return ConnectionCheckResponse.block;
+            default:
+                if (log.isWarnEnabled()) log.warn("checkConnection: unexpected decision: "+decisionType+", returning noop");
+                return ConnectionCheckResponse.noop;
+        }
+    }
+
     @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness,
                                                     FilterMatchersRequest filter,
                                                     Account account,

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java

@@ -8,6 +8,7 @@ import com.amazonaws.util.IOUtils;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.ToString;
 import lombok.experimental.Accessors;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.cache.AutoRefreshingReference;
@@ -21,9 +22,11 @@ import java.util.stream.Collectors;
 
 import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_FEEDS;
 import static java.util.concurrent.TimeUnit.HOURS;
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 import static org.cobbzilla.util.http.HttpUtil.getUrlInputStream;
+import static org.cobbzilla.util.string.ValidationRegexes.HOST;
 import static org.cobbzilla.wizard.server.RestServerBase.reportError;
 
 @Slf4j @Accessors(chain=true)
@@ -69,6 +72,7 @@ public class TlsPassthruConfig {
         return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
     }
 
+    @ToString
     private static class TlsPassthruMatcher {
         @Getter @Setter private String fqdn;
         @Getter @Setter private Pattern fqdnPattern;
@@ -76,7 +80,9 @@ public class TlsPassthruConfig {
         public TlsPassthruMatcher (String fqdn) {
             this.fqdn = fqdn;
             if (fqdn.startsWith("/") && fqdn.endsWith("/")) {
-                this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), Pattern.CASE_INSENSITIVE);
+                this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), CASE_INSENSITIVE);
+            } else if (fqdn.startsWith("*.")) {
+                this.fqdnPattern = Pattern.compile("("+HOST+"\\.)?"+Pattern.quote(fqdn.substring(2)), CASE_INSENSITIVE);
             }
         }
         public boolean matches (String val) {

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java

@@ -8,6 +8,7 @@ import bubble.model.account.Account;
 import bubble.model.device.Device;
 import bubble.rule.AbstractAppRuleDriver;
 import bubble.service.stream.AppRuleHarness;
+import bubble.service.stream.ConnectionCheckResponse;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -15,14 +16,14 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
 
     @Override public <C> Class<C> getConfigClass() { return (Class<C>) TlsPassthruConfig.class; }
 
-    @Override public boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) {
+    @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) {
         final TlsPassthruConfig passthruConfig = getRuleConfig();
         if (passthruConfig.isPassthru(fqdn) || passthruConfig.isPassthru(addr)) {
-            if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning true for fqdn/addr="+fqdn+"/"+addr);
-            return true;
+            if (log.isDebugEnabled()) log.debug("checkConnection: returning passthru for fqdn/addr="+fqdn+"/"+addr);
+            return ConnectionCheckResponse.passthru;
         }
-        if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning false for fqdn/addr="+fqdn+"/"+addr);
-        return false;
+        if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr);
+        return ConnectionCheckResponse.noop;
     }
 
 }

bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java

@@ -0,0 +1,13 @@
+package bubble.service.stream;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+import static bubble.ApiConstants.enumFromString;
+
+public enum ConnectionCheckResponse {
+
+    noop, passthru, block, error;
+
+    @JsonCreator public static ConnectionCheckResponse fromString (String v) { return enumFromString(ConnectionCheckResponse.class, v); }
+
+}

bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java

@@ -50,7 +50,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -64,6 +63,7 @@ import static org.apache.http.HttpHeaders.TRANSFER_ENCODING;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf;
 import static org.cobbzilla.util.http.HttpStatusCodes.OK;
+import static org.cobbzilla.wizard.cache.redis.RedisService.ALL_KEYS;
 import static org.cobbzilla.wizard.resources.ResourceUtil.send;
 
 @Service @Slf4j
@@ -179,18 +179,18 @@ public class StandardRuleEngineService implements RuleEngineService {
     public Map<Object, Object> flushCaches() {
         final int ruleEngineCacheSize = ruleCache.size();
         ruleCache.clear();
-        log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries");
+        if (log.isInfoEnabled()) log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries");
 
         final RedisService matchersCache = getMatchersCache();
-        final Collection<String> keys = matchersCache.keys("*");
-        for (String key : keys) {
-            if (log.isTraceEnabled()) log.trace("flushCaches: deleting key: "+key);
-            matchersCache.del_withPrefix(key);
-        }
-        log.info("flushCaches: flushed "+keys.size()+" matchersCache entries");
+        final Long matcherCount = matchersCache.del_matching(ALL_KEYS);
+        if (log.isInfoEnabled()) log.info("flushCaches: flushed "+matcherCount+" matchersCache entries");
+
+        final Long connCheckDeletions = redis.del_matching("bubble_conn_check_*");
+        if (log.isInfoEnabled()) log.info("flushCaches: removed "+connCheckDeletions+" conn_check cache entries");
 
         return MapBuilder.build(new Object[][] {
-                {"matchersCache", keys.size()},
+                {"connCheckCache", connCheckDeletions},
+                {"matchersCache", matcherCount},
                 {"ruleEngineCache", ruleEngineCacheSize}
         });
     }
@@ -275,11 +275,13 @@ public class StandardRuleEngineService implements RuleEngineService {
     @Getter(lazy=true) private final HttpClientBuilder httpClientBuilder = newHttpClientBuilder(1000, 50);
     public CloseableHttpClient newHttpConn() { return getHttpClientBuilder().build(); }
 
-    public boolean isTlsPassthru(Account account, Device device, List<AppMatcher> matchers, String addr, String fqdn) {
+    public ConnectionCheckResponse checkConnection(Account account, Device device, List<AppMatcher> matchers, String addr, String fqdn) {
         final List<AppRuleHarness> ruleHarnesses = initRules(account, device, matchers);
         for (AppRuleHarness harness : ruleHarnesses) {
-            if (harness.getDriver().isTlsPassthru(harness, account, device, addr, fqdn)) return true;
+            final ConnectionCheckResponse checkResponse = harness.getDriver().checkConnection(harness, account, device, addr, fqdn);
+            if (checkResponse != ConnectionCheckResponse.noop) return checkResponse;
         }
-        return false;
+        return ConnectionCheckResponse.noop;
     }
+
 }

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json

@@ -4,6 +4,7 @@
     "AppMatcher": [{
       "name": "BubbleBlockMatcher",
       "template": true,
+      "connCheck": true,
       "site": "All_Sites",
       "fqdn": "*",
       "urlRegex": ".*",

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json

@@ -4,7 +4,7 @@
     "AppMatcher": [{
       "name": "TlsPassthruMatcher",
       "template": true,
-      "passthru": true,
+      "connCheck": true,
       "site": "All_Sites",
       "fqdn": "*",
       "urlRegex": ".*",

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -56,7 +56,7 @@ def bubble_activity_log(client_addr, server_addr, event, data):
     pass
 
 
-def bubble_passthru(remote_addr, addr, fqdns):
+def bubble_conn_check(remote_addr, addr, fqdns, security_level):
     headers = {
         'X-Forwarded-For': remote_addr,
         'Accept' : 'application/json',
@@ -68,11 +68,17 @@ def bubble_passthru(remote_addr, addr, fqdns):
             'fqdns': fqdns,
             'remoteAddr': remote_addr
         }
-        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data)
-        return response.ok
+        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data)
+        if response.ok:
+            return response.json()
+        bubble_log('bubble_conn_check API call failed: '+repr(response))
+        return None
+
     except Exception as e:
-        bubble_log('bubble_passthru API call failed: '+repr(e))
+        bubble_log('bubble_conn_check API call failed: '+repr(e))
         traceback.print_exc()
+        if security_level is not None and security_level == 'maximum':
+            return False
         return None
 
 

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py

@@ -0,0 +1,227 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here:
+#
+# Copyright (c) 2013, Aldo Cortesi. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.exceptions import TlsProtocolException
+from mitmproxy.net import tls as net_tls
+
+from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, redis_set
+from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6
+import redis
+import json
+import subprocess
+
+REDIS_DNS_PREFIX = 'bubble_dns_'
+REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_'
+REDIS_CHECK_DURATION = 60 * 60  # 1 hour timeout
+REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_'  # defined in StandardDeviceIdService
+
+REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
+
+FORCE_PASSTHRU = {'passthru': True}
+FORCE_BLOCK = {'block': True}
+
+local_ips = None
+
+
+def get_device_security_level(client_addr):
+    level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr)
+    if level is None:
+        return 'maximum'
+    return level.decode()
+
+
+def get_local_ips():
+    global local_ips
+    if local_ips is None:
+        local_ips = []
+        for ip in subprocess.check_output(['hostname', '-I']).split():
+            local_ips.append(ip.decode())
+    return local_ips
+
+
+def is_sage_request(ip, fqdns):
+    return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns
+
+
+def conn_check_cache_prefix(client_addr, server_addr):
+    return REDIS_CONN_CHECK_PREFIX + client_addr + '_' + server_addr
+
+
+def fqdns_for_addr(addr):
+    prefix = REDIS_DNS_PREFIX + addr
+    keys = REDIS.keys(prefix + '_*')
+    if keys is None or len(keys) == 0:
+        bubble_log('fqdns_for_addr: no FQDN found for addr '+str(addr)+', checking raw addr')
+        return ''
+    fqdns = []
+    for k in keys:
+        fqdn = k.decode()[len(prefix)+1:]
+        fqdns.append(fqdn)
+    return fqdns
+
+
+class TlsBlock(TlsLayer):
+    """
+    Monkey-patch __call__ to drop this connection entirely
+    """
+    def __call__(self):
+        bubble_log('TlsBlock: blocking')
+        return
+
+
+class TlsFeedback(TlsLayer):
+    """
+    Monkey-patch _establish_tls_with_client to get feedback if TLS could be established
+    successfully on the client connection (which may fail due to cert pinning).
+    """
+    def _establish_tls_with_client(self):
+        client_address = self.client_conn.address[0]
+        server_address = self.server_conn.address[0]
+        security_level = get_device_security_level(client_address)
+        try:
+            super(TlsFeedback, self)._establish_tls_with_client()
+
+        except TlsProtocolException as e:
+            if self.fqdns is not None and len(self.fqdns) > 0:
+                for fqdn in self.fqdns:
+                    cache_key = conn_check_cache_prefix(client_address, fqdn)
+                    if security_level == 'maximum':
+                        redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                        bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn)
+                    else:
+                        redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                        bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn)
+            else:
+                cache_key = conn_check_cache_prefix(client_address, server_address)
+                if security_level == 'maximum':
+                    redis_set(cache_key, json.dumps({'fqdn': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                    bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address)
+                else:
+                    redis_set(cache_key, json.dumps({'fqdn': None, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                    bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address)
+            raise e
+
+
+def check_bubble_connection(client_addr, server_addr, fqdns, security_level):
+    check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level)
+    if check_response is None or check_response == 'error':
+        if security_level == 'maximum':
+            bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block')
+            return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'}
+        else:
+            bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
+            return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_error'}
+
+    elif check_response == 'passthru':
+        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
+        return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_passthru'}
+
+    elif check_response == 'block':
+        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block')
+        return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_block'}
+
+    else:
+        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False')
+        return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'reason': 'bubble_no_passthru'}
+
+
+def check_connection(client_addr, server_addr, fqdns, security_level):
+    if fqdns and len(fqdns) == 1:
+        cache_key = conn_check_cache_prefix(client_addr, fqdns[0])
+    else:
+        cache_key = conn_check_cache_prefix(client_addr, server_addr)
+    prefix = 'check_connection: ip=' + str(server_addr) + ' (fqdns=' + str(fqdns) + ') cache_key=' + cache_key + ': '
+
+    check_json = REDIS.get(cache_key)
+    if check_json is None or len(check_json) == 0:
+        bubble_log(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns))
+        check_response = check_bubble_connection(client_addr, server_addr, fqdns, security_level)
+        bubble_log(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...")
+        redis_set(cache_key, json.dumps(check_response), ex=REDIS_CHECK_DURATION)
+
+    else:
+        bubble_log(prefix+'found check_json='+str(check_json)+', touching key in redis')
+        check_response = json.loads(check_json)
+        REDIS.touch(cache_key)
+    bubble_log(prefix+'returning '+str(check_response))
+    return check_response
+
+
+def next_layer(next_layer):
+    if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
+        client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile)
+        client_addr = next_layer.client_conn.address[0]
+        server_addr = next_layer.server_conn.address[0]
+
+        if client_hello.sni:
+            fqdn = client_hello.sni.decode()
+            bubble_log('next_layer: using fqdn in SNI: '+ fqdn)
+            fqdns = [ fqdn ]
+        else:
+            fqdns = fqdns_for_addr(server_addr)
+            bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns))
+        next_layer.fqdns = fqdns
+        no_fqdns = fqdns is None or len(fqdns) == 0
+        security_level = get_device_security_level(client_addr)
+        if server_addr in get_local_ips():
+            bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif is_sage_request(server_addr, fqdns):
+            bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif security_level == 'disabled' or security_level == 'basic':
+            bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif security_level == 'standard' and no_fqdns:
+            bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif security_level == 'maximum' and no_fqdns:
+            bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
+            check = FORCE_BLOCK
+
+        else:
+            bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+security_level)
+            check = check_connection(client_addr, server_addr, fqdns, security_level)
+
+        if check is None or ('passthru' in check and check['passthru']):
+            bubble_log('next_layer: enabling passthru for server_addr' + server_addr+', fqdns='+str(fqdns))
+            bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
+            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
+            next_layer.reply.send(next_layer_replacement)
+
+        elif 'block' in check and check['block']:
+            bubble_log('next_layer: enabling block for server_addr' + server_addr+', fqdns='+str(fqdns))
+            bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns)
+            next_layer.__class__ = TlsBlock
+
+        else:
+            bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns))
+            bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns)
+            next_layer.__class__ = TlsFeedback

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py

@@ -1,171 +0,0 @@
-#
-# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
-#
-# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here:
-#
-# Copyright (c) 2013, Aldo Cortesi. All rights reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
-from mitmproxy.exceptions import TlsProtocolException
-
-from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set
-from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6
-import redis
-import json
-import subprocess
-
-REDIS_DNS_PREFIX = 'bubble_dns_'
-REDIS_PASSTHRU_PREFIX = 'bubble_passthru_'
-REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_'  # defined in StandardDeviceIdService
-REDIS_PASSTHRU_DURATION = 60 * 60  # 1 hour timeout on passthru
-
-REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
-
-FORCE_PASSTHRU = {'passthru': True}
-
-local_ips = None
-
-
-def get_device_security_level(client_addr):
-    level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr)
-    if level is None:
-        return 'maximum'
-    return level.decode()
-
-
-def get_local_ips():
-    global local_ips
-    if local_ips is None:
-        local_ips = []
-        for ip in subprocess.check_output(['hostname', '-I']).split():
-            local_ips.append(ip.decode())
-    return local_ips
-
-
-def is_sage_request(ip, fqdns):
-    return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns
-
-
-def passthru_cache_prefix(client_addr, server_addr):
-    return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr
-
-
-def fqdns_for_addr(addr):
-    prefix = REDIS_DNS_PREFIX + addr
-    keys = REDIS.keys(prefix + '_*')
-    if keys is None or len(keys) == 0:
-        bubble_log('fqdns_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr')
-        return ''
-    fqdns = []
-    for k in keys:
-        fqdn = k.decode()[len(prefix)+1:]
-        fqdns.append(fqdn)
-    return fqdns
-
-
-class TlsFeedback(TlsLayer):
-    """
-    Monkey-patch _establish_tls_with_client to get feedback if TLS could be established
-    successfully on the client connection (which may fail due to cert pinning).
-    """
-    def _establish_tls_with_client(self):
-        client_address = self.client_conn.address[0]
-        server_address = self.server_conn.address[0]
-        fqdns = fqdns_for_addr(server_address)
-        try:
-            super(TlsFeedback, self)._establish_tls_with_client()
-
-        except TlsProtocolException as e:
-            cache_key = passthru_cache_prefix(client_address, server_address)
-            bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key)
-            redis_set(cache_key, json.dumps({'fqdns': fqdns, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION)
-            raise e
-
-
-def check_bubble_passthru(client_addr, addr, fqdns):
-    passthru = bubble_passthru(client_addr, addr, fqdns)
-    if passthru is None or passthru:
-        bubble_log('check_bubble_passthru: bubble_passthru returned '+repr(passthru)+' for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning True')
-        return {'fqdns': fqdns, 'addr': addr, 'passthru': True}
-    bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning False')
-    return {'fqdns': fqdns, 'addr': addr, 'passthru': False}
-
-
-def should_passthru(client_addr, addr, fqdns):
-    cache_key = passthru_cache_prefix(client_addr, addr)
-    prefix = 'should_passthru: ip='+repr(addr)+' (fqdns='+repr(fqdns)+') cache_key='+cache_key+': '
-
-    passthru_json = REDIS.get(cache_key)
-    if passthru_json is None or len(passthru_json) == 0:
-        bubble_log(prefix+'not in redis or empty, calling check_bubble_passthru against fqdns='+repr(fqdns))
-        passthru = check_bubble_passthru(client_addr, addr, fqdns)
-        bubble_log(prefix+'check_bubble_passthru('+repr(fqdns)+') returned '+repr(passthru)+", storing in redis...")
-        redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION)
-
-    else:
-        bubble_log(prefix+'found passthru_json='+str(passthru_json)+', touching key in redis')
-        passthru = json.loads(passthru_json)
-        REDIS.touch(cache_key)
-    bubble_log(prefix+'returning '+repr(passthru))
-    return passthru
-
-
-def next_layer(next_layer):
-    if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
-        client_addr = next_layer.client_conn.address[0]
-        server_addr = next_layer.server_conn.address[0]
-
-        fqdns = fqdns_for_addr(server_addr)
-        no_fqdns = fqdns is None or len(fqdns) == 0
-        security_level = get_device_security_level(client_addr)
-        if server_addr in get_local_ips():
-            bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif is_sage_request(server_addr, fqdns):
-            bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif security_level == 'disabled' or security_level == 'basic':
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif security_level == 'standard' and no_fqdns:
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif security_level == 'maximum' and no_fqdns:
-            bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
-            return
-
-        else:
-            bubble_log('next_layer: checking should_passthru for server='+server_addr+', client='+client_addr+' with security_level='+security_level)
-            passthru = should_passthru(client_addr, server_addr, fqdns)
-
-        if passthru is None or passthru['passthru']:
-            bubble_log('next_layer: enabling passthru for ' + repr(next_layer.server_conn.address))
-            bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
-            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
-            next_layer.reply.send(next_layer_replacement)
-        else:
-            bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr)
-            bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns)
-            next_layer.__class__ = TlsFeedback

bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh

@@ -16,6 +16,6 @@ mitmdump \
   --set stream_large_bodies=5m \
   --set keep_host_header \
   -s ./dns_spoofing.py \
-  -s ./bubble_passthru.py \
+  -s ./bubble_conn_check.py \
   -s ./bubble_modify.py \
   --mode transparent

utils/abp-parser

@@ -1 +1 @@
-Subproject commit df343fc4b3e1f123b3caa025a493861edf46b81a
+Subproject commit e3d05733525eb15a7106cbca0d585a3104681925

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit e5d7abc4b58a339a5da90fcfe53ba21c20e40c75
+Subproject commit cbd62f426d8544cc625dfb0ac4e5b6ec6a049b8f

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit d450510d6f1be5b328b919ae73d8c1450f008d91
+Subproject commit 9aa42648fcc5694dfcd1778fa7dc4cb0f25455e8