Merge branch 'master' into kris/add_support_for_restore_ui

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -207,7 +207,7 @@ public class ApiConstants {
     public static final String BUBBLE_MAGIC_ENDPOINT = "/.bubble";
 
     public static final String FILTER_HTTP_ENDPOINT = "/filter";
-    public static final String EP_PASSTHRU = "/passthru";
+    public static final String EP_CHECK = "/check";
     public static final String EP_APPLY = "/apply";
     public static final String EP_ASSETS = "/assets";
 

bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java

@@ -34,12 +34,12 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
 
     @Override public Order getDefaultSortOrder() { return PRIORITY_ASC; }
 
-    public List<AppMatcher> findByAccountAndFqdnAndEnabled(String account, String fqdn) {
+    public List<AppMatcher> findByAccountAndFqdnAndEnabledAndRequestCheck(String account, String fqdn) {
         return list(criteria().add(
                 and(
                         eq("account", account),
                         eq("enabled", true),
-                        eq("passthru", false),
+                        eq("requestCheck", true),
                         or(
                                 eq("fqdn", fqdn),
                                 eq("fqdn", WILDCARD_FQDN)
@@ -47,8 +47,8 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
         ).addOrder(PRIORITY_ASC));
     }
 
-    public List<AppMatcher> findByAccountAndEnabledAndPassthru(String account) {
-        return findByFields("account", account, "enabled", true, "passthru", true);
+    public List<AppMatcher> findByAccountAndEnabledAndConnCheck(String account) {
+        return findByFields("account", account, "enabled", true, "connCheck", true);
     }
 
     public List<AppMatcher> findAllChangesSince(Long lastMod) {
@@ -56,7 +56,8 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
     }
 
     @Override public Object preCreate(AppMatcher matcher) {
-        if (matcher.getPassthru() == null) matcher.setPassthru(false);
+        if (matcher.getConnCheck() == null) matcher.setConnCheck(false);
+        if (matcher.getRequestCheck() == null) matcher.setRequestCheck(true);
         return super.preCreate(matcher);
     }
 

bubble-server/src/main/java/bubble/model/app/AppMatcher.java

@@ -45,7 +45,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
 public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, HasPriority {
 
     public static final String[] VALUE_FIELDS = {"fqdn", "urlRegex", "template", "enabled", "priority"};
-    public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "passthru");
+    public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "connCheck");
 
     public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE);
     public static final String WILDCARD_FQDN = "*";
@@ -131,8 +131,13 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H
 
     @ECSearchable @ECField(index=120, required=EntityFieldRequired.optional)
     @ECIndex @Column(nullable=false)
-    @Getter @Setter private Boolean passthru;
-    public boolean passthru () { return bool(passthru); }
+    @Getter @Setter private Boolean connCheck;
+    public boolean connCheck () { return bool(connCheck); }
+
+    @ECSearchable @ECField(index=130, required=EntityFieldRequired.optional)
+    @ECIndex @Column(nullable=false)
+    @Getter @Setter private Boolean requestCheck;
+    public boolean requestCheck () { return bool(requestCheck); }
 
     @ECSearchable @ECField(index=130)
     @Column(nullable=false)

bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java

@@ -19,11 +19,11 @@ import static bubble.ApiConstants.enumFromString;
 public enum BubbleDeviceType {
 
     uninitialized (null, false),
-    windows       (CertType.cer, true, DeviceSecurityLevel.maximum),
-    macosx        (CertType.pem, true, DeviceSecurityLevel.maximum),
+    windows       (CertType.cer, true, DeviceSecurityLevel.standard),
+    macosx        (CertType.pem, true, DeviceSecurityLevel.standard),
     ios           (CertType.pem, true, DeviceSecurityLevel.standard),
-    android       (CertType.cer, true, DeviceSecurityLevel.basic),
-    linux         (CertType.crt, true, DeviceSecurityLevel.maximum),
+    android       (CertType.cer, true, DeviceSecurityLevel.standard),
+    linux         (CertType.crt, true, DeviceSecurityLevel.standard),
     firefox       (CertType.crt, false),
     other         (null, true, DeviceSecurityLevel.basic);
 
@@ -32,7 +32,7 @@ public enum BubbleDeviceType {
     @Getter private final DeviceSecurityLevel defaultSecurityLevel;
     public boolean hasDefaultSecurityLevel () { return defaultSecurityLevel != null; }
 
-    BubbleDeviceType (CertType certType, boolean selectable) { this(CertType.cer, selectable, null); }
+    BubbleDeviceType (CertType certType, boolean selectable) { this(certType, selectable, null); }
 
     @JsonCreator public static BubbleDeviceType fromString (String v) { return enumFromString(BubbleDeviceType.class, v); }
 

bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java → bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java

@@ -9,7 +9,7 @@ import lombok.Setter;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
-public class FilterPassthruRequest {
+public class FilterConnCheckRequest {
 
     @Getter @Setter private String addr;
     public boolean hasAddr() { return !empty(addr); }

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -21,6 +21,7 @@ import bubble.rule.FilterMatchDecision;
 import bubble.server.BubbleConfiguration;
 import bubble.service.boot.SelfNodeService;
 import bubble.service.cloud.DeviceIdService;
+import bubble.service.stream.ConnectionCheckResponse;
 import bubble.service.stream.StandardRuleEngineService;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -124,23 +125,23 @@ public class FilterHttpResource {
         return null;
     }
 
-    @POST @Path(EP_PASSTHRU)
+    @POST @Path(EP_CHECK)
     @Consumes(APPLICATION_JSON)
     @Produces(APPLICATION_JSON)
-    public Response isTlsPassthru(@Context Request req,
-                                  @Context ContainerRequest request,
-                                  FilterPassthruRequest passthruRequest) {
-        final String prefix = "isPassthru: ";
-        if (passthruRequest == null || !passthruRequest.hasAddr() || !passthruRequest.hasRemoteAddr()) {
-            if (log.isDebugEnabled()) log.debug(prefix+"invalid passthruRequest, returning forbidden");
+    public Response checkConnection(@Context Request req,
+                                    @Context ContainerRequest request,
+                                    FilterConnCheckRequest connCheckRequest) {
+        final String prefix = "checkConnection: ";
+        if (connCheckRequest == null || !connCheckRequest.hasAddr() || !connCheckRequest.hasRemoteAddr()) {
+            if (log.isDebugEnabled()) log.debug(prefix+"invalid connCheckRequest, returning forbidden");
             return forbidden();
         }
         validateMitmCall(req);
 
         // if the requested IP is the same as our IP, then always passthru
-        if (isForUs(passthruRequest)) return ok();
+        if (isForUs(connCheckRequest)) return ok();
 
-        final String vpnAddr = passthruRequest.getRemoteAddr();
+        final String vpnAddr = connCheckRequest.getRemoteAddr();
         final Device device = deviceIdService.findDeviceByIp(vpnAddr);
         if (device == null) {
             if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning not found");
@@ -154,7 +155,7 @@ public class FilterHttpResource {
             return notFound();
         }
 
-        final List<AppMatcher> matchers = matcherDAO.findByAccountAndEnabledAndPassthru(device.getAccount());
+        final List<AppMatcher> matchers = matcherDAO.findByAccountAndEnabledAndConnCheck(device.getAccount());
         final List<AppMatcher> retained = new ArrayList<>();
         for (AppMatcher matcher : matchers) {
             final BubbleApp app = appDAO.findByUuid(matcher.getApp());
@@ -164,23 +165,23 @@ public class FilterHttpResource {
             retained.add(matcher);
         }
 
-        final String[] fqdns = passthruRequest.getFqdns();
+        final String[] fqdns = connCheckRequest.getFqdns();
         for (String fqdn : fqdns) {
-            final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), fqdn);
-            if (passthru) {
-                if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+fqdn+"/"+passthruRequest.getAddr());
-                return ok();
+            final ConnectionCheckResponse checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn);
+            if (checkResponse != ConnectionCheckResponse.noop) {
+                if (log.isDebugEnabled()) log.debug(prefix + "returning "+checkResponse+" for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr());
+                return ok(checkResponse);
             }
         }
-        if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdns/addr="+Arrays.toString(fqdns)+"/"+passthruRequest.getAddr());
-        return notFound();
+        if (log.isDebugEnabled()) log.debug(prefix+"returning noop for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr());
+        return ok(ConnectionCheckResponse.noop);
     }
 
-    private boolean isForUs(FilterPassthruRequest passthruRequest) {
+    private boolean isForUs(FilterConnCheckRequest connCheckRequest) {
         final BubbleNode thisNode = selfNodeService.getThisNode();
-        return passthruRequest.hasAddr()
-                && (thisNode.hasIp4() && thisNode.getIp4().equals(passthruRequest.getAddr())
-                || thisNode.hasIp6() && thisNode.getIp6().equals(passthruRequest.getAddr()));
+        return connCheckRequest.hasAddr()
+                && (thisNode.hasIp4() && thisNode.getIp4().equals(connCheckRequest.getAddr())
+                || thisNode.hasIp6() && thisNode.getIp6().equals(connCheckRequest.getAddr()));
     }
 
     @POST @Path(EP_MATCHERS+"/{requestId}")
@@ -281,7 +282,7 @@ public class FilterHttpResource {
 
     private List<AppMatcher> getEnabledMatchers(String requestId, String accountUuid, String fqdn) {
         final String prefix = "getEnabledMatchers("+requestId+"): ";
-        List<AppMatcher> matchers = matcherDAO.findByAccountAndFqdnAndEnabled(accountUuid, fqdn);
+        List<AppMatcher> matchers = matcherDAO.findByAccountAndFqdnAndEnabledAndRequestCheck(accountUuid, fqdn);
         if (log.isTraceEnabled()) log.trace(prefix+"checking all enabled matchers for fqdn: "+json(matchers, COMPACT_MAPPER));
         matchers = matchers.stream()
                 .filter(m -> appDAO.findByAccountAndId(accountUuid, m.getApp()).enabled()).collect(Collectors.toList());

bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java

@@ -64,7 +64,7 @@ public class ReverseProxyResource {
         if (device == null) return ruleEngine.passthru(request);
 
         final URIBean ub = getUriBean(request);
-        final List<AppMatcher> matchers = matcherDAO.findByAccountAndFqdnAndEnabled(account.getUuid(), ub.getHost());
+        final List<AppMatcher> matchers = matcherDAO.findByAccountAndFqdnAndEnabledAndRequestCheck(account.getUuid(), ub.getHost());
         if (empty(matchers)) {
             // no matchers, pass-thru
             return ruleEngine.passthru(ub, request);

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -11,6 +11,7 @@ import bubble.model.device.Device;
 import bubble.resources.stream.FilterHttpRequest;
 import bubble.resources.stream.FilterMatchersRequest;
 import bubble.service.stream.AppRuleHarness;
+import bubble.service.stream.ConnectionCheckResponse;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jknack.handlebars.Handlebars;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
@@ -145,6 +146,12 @@ public interface AppRuleDriver {
         return null;
     }
 
-    default boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { return false; }
+    default ConnectionCheckResponse checkConnection(AppRuleHarness harness,
+                                                    Account account,
+                                                    Device device,
+                                                    String addr,
+                                                    String fqdn) {
+        return ConnectionCheckResponse.noop;
+    }
 
 }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -15,6 +15,7 @@ import bubble.rule.AppRuleDriver;
 import bubble.rule.FilterMatchDecision;
 import bubble.rule.analytics.TrafficAnalyticsRuleDriver;
 import bubble.service.stream.AppRuleHarness;
+import bubble.service.stream.ConnectionCheckResponse;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.ReaderInputStream;
@@ -108,14 +109,11 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         }
         blockList.set(newBlockList);
 
-        boolean doPrime = false;
         if (!newBlockList.getFullyBlockedDomains().equals(fullyBlockedDomains.get())) {
             fullyBlockedDomains.set(newBlockList.getFullyBlockedDomains());
-            doPrime = true;
         }
         if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) {
             partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains());
-            doPrime = true;
         }
 
         log.info("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size());
@@ -123,6 +121,24 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         log.info("refreshBlockLists: refreshed "+refreshed.size()+" block lists: "+StringUtil.toString(refreshed));
     }
 
+    @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness,
+                                                             Account account,
+                                                             Device device,
+                                                             String addr,
+                                                             String fqdn) {
+        final BlockDecision decision = getBlockList().getFqdnDecision(fqdn);
+        final BlockDecisionType decisionType = decision.getDecisionType();
+        switch (decisionType) {
+            case allow:
+                return ConnectionCheckResponse.noop;
+            case block:
+                return ConnectionCheckResponse.block;
+            default:
+                if (log.isWarnEnabled()) log.warn("checkConnection: unexpected decision: "+decisionType+", returning noop");
+                return ConnectionCheckResponse.noop;
+        }
+    }
+
     @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness,
                                                     FilterMatchersRequest filter,
                                                     Account account,

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java

@@ -8,6 +8,7 @@ import com.amazonaws.util.IOUtils;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.ToString;
 import lombok.experimental.Accessors;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.cache.AutoRefreshingReference;
@@ -21,9 +22,11 @@ import java.util.stream.Collectors;
 
 import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_FEEDS;
 import static java.util.concurrent.TimeUnit.HOURS;
+import static java.util.regex.Pattern.CASE_INSENSITIVE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 import static org.cobbzilla.util.http.HttpUtil.getUrlInputStream;
+import static org.cobbzilla.util.string.ValidationRegexes.HOST;
 import static org.cobbzilla.wizard.server.RestServerBase.reportError;
 
 @Slf4j @Accessors(chain=true)
@@ -69,6 +72,7 @@ public class TlsPassthruConfig {
         return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
     }
 
+    @ToString
     private static class TlsPassthruMatcher {
         @Getter @Setter private String fqdn;
         @Getter @Setter private Pattern fqdnPattern;
@@ -76,7 +80,9 @@ public class TlsPassthruConfig {
         public TlsPassthruMatcher (String fqdn) {
             this.fqdn = fqdn;
             if (fqdn.startsWith("/") && fqdn.endsWith("/")) {
-                this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), Pattern.CASE_INSENSITIVE);
+                this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), CASE_INSENSITIVE);
+            } else if (fqdn.startsWith("*.")) {
+                this.fqdnPattern = Pattern.compile("("+HOST+"\\.)?"+Pattern.quote(fqdn.substring(2)), CASE_INSENSITIVE);
             }
         }
         public boolean matches (String val) {

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java

@@ -8,6 +8,7 @@ import bubble.model.account.Account;
 import bubble.model.device.Device;
 import bubble.rule.AbstractAppRuleDriver;
 import bubble.service.stream.AppRuleHarness;
+import bubble.service.stream.ConnectionCheckResponse;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -15,14 +16,14 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
 
     @Override public <C> Class<C> getConfigClass() { return (Class<C>) TlsPassthruConfig.class; }
 
-    @Override public boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) {
+    @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) {
         final TlsPassthruConfig passthruConfig = getRuleConfig();
         if (passthruConfig.isPassthru(fqdn) || passthruConfig.isPassthru(addr)) {
-            if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning true for fqdn/addr="+fqdn+"/"+addr);
-            return true;
+            if (log.isDebugEnabled()) log.debug("checkConnection: returning passthru for fqdn/addr="+fqdn+"/"+addr);
+            return ConnectionCheckResponse.passthru;
         }
-        if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning false for fqdn/addr="+fqdn+"/"+addr);
-        return false;
+        if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr);
+        return ConnectionCheckResponse.noop;
     }
 
 }

bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java

@@ -0,0 +1,13 @@
+package bubble.service.stream;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+import static bubble.ApiConstants.enumFromString;
+
+public enum ConnectionCheckResponse {
+
+    noop, passthru, block, error;
+
+    @JsonCreator public static ConnectionCheckResponse fromString (String v) { return enumFromString(ConnectionCheckResponse.class, v); }
+
+}

bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java

@@ -50,7 +50,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -64,6 +63,7 @@ import static org.apache.http.HttpHeaders.TRANSFER_ENCODING;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf;
 import static org.cobbzilla.util.http.HttpStatusCodes.OK;
+import static org.cobbzilla.wizard.cache.redis.RedisService.ALL_KEYS;
 import static org.cobbzilla.wizard.resources.ResourceUtil.send;
 
 @Service @Slf4j
@@ -179,18 +179,18 @@ public class StandardRuleEngineService implements RuleEngineService {
     public Map<Object, Object> flushCaches() {
         final int ruleEngineCacheSize = ruleCache.size();
         ruleCache.clear();
-        log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries");
+        if (log.isInfoEnabled()) log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries");
 
         final RedisService matchersCache = getMatchersCache();
-        final Collection<String> keys = matchersCache.keys("*");
-        for (String key : keys) {
-            if (log.isTraceEnabled()) log.trace("flushCaches: deleting key: "+key);
-            matchersCache.del_withPrefix(key);
-        }
-        log.info("flushCaches: flushed "+keys.size()+" matchersCache entries");
+        final Long matcherCount = matchersCache.del_matching(ALL_KEYS);
+        if (log.isInfoEnabled()) log.info("flushCaches: flushed "+matcherCount+" matchersCache entries");
+
+        final Long connCheckDeletions = redis.del_matching("bubble_conn_check_*");
+        if (log.isInfoEnabled()) log.info("flushCaches: removed "+connCheckDeletions+" conn_check cache entries");
 
         return MapBuilder.build(new Object[][] {
-                {"matchersCache", keys.size()},
+                {"connCheckCache", connCheckDeletions},
+                {"matchersCache", matcherCount},
                 {"ruleEngineCache", ruleEngineCacheSize}
         });
     }
@@ -275,11 +275,13 @@ public class StandardRuleEngineService implements RuleEngineService {
     @Getter(lazy=true) private final HttpClientBuilder httpClientBuilder = newHttpClientBuilder(1000, 50);
     public CloseableHttpClient newHttpConn() { return getHttpClientBuilder().build(); }
 
-    public boolean isTlsPassthru(Account account, Device device, List<AppMatcher> matchers, String addr, String fqdn) {
+    public ConnectionCheckResponse checkConnection(Account account, Device device, List<AppMatcher> matchers, String addr, String fqdn) {
         final List<AppRuleHarness> ruleHarnesses = initRules(account, device, matchers);
         for (AppRuleHarness harness : ruleHarnesses) {
-            if (harness.getDriver().isTlsPassthru(harness, account, device, addr, fqdn)) return true;
+            final ConnectionCheckResponse checkResponse = harness.getDriver().checkConnection(harness, account, device, addr, fqdn);
+            if (checkResponse != ConnectionCheckResponse.noop) return checkResponse;
         }
-        return false;
+        return ConnectionCheckResponse.noop;
     }
+
 }

bubble-server/src/main/resources/META-INF/bubble/bubble.properties

@@ -1 +1 @@
-bubble.version=0.12.4
+bubble.version=0.12.5

bubble-server/src/main/resources/logback.xml

@@ -61,6 +61,7 @@
     <logger name="bubble.rule.passthru" level="DEBUG" />
 <!--    <logger name="bubble.service.cloud.StandardNetworkService" level="INFO" />-->
 <!--    <logger name="bubble.service.cloud.AnsiblePrepService" level="DEBUG" />-->
+    <logger name="bubble.abp.BlockSpec" level="DEBUG" />
     <logger name="bubble.resources.notify" level="WARN" />
     <logger name="bubble.client" level="WARN" />
     <logger name="bubble.main.rekey" level="INFO" />

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json

@@ -134,6 +134,13 @@
             "description": "EasyList is the primary filter list that removes most adverts from international web pages, including unwanted frames, images, and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists.",
             "tags": ["ads"]
           },
+          {
+            "name": "Peter Lowe's Ad and tracking server list",
+            "id": "peter_lowes_ad_and_tracking_list",
+            "url": "https://pgl.yoyo.org/adservers/serverlist.php?showintro=0;hostformat=raw",
+            "description": "A comprehensive list of ad and tracking servers",
+            "tags": ["ads", "privacy"]
+          },
           {
             "name": "Dandelion Sprout's Anti-Malware List",
             "id": "dandelion_sprouts_anti_malware_list",

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json

@@ -4,6 +4,7 @@
     "AppMatcher": [{
       "name": "BubbleBlockMatcher",
       "template": true,
+      "connCheck": true,
       "site": "All_Sites",
       "fqdn": "*",
       "urlRegex": ".*",

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json

@@ -4,7 +4,8 @@
     "AppMatcher": [{
       "name": "TlsPassthruMatcher",
       "template": true,
-      "passthru": true,
+      "connCheck": true,
+      "requestCheck": false,
       "site": "All_Sites",
       "fqdn": "*",
       "urlRegex": ".*",

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -56,7 +56,7 @@ def bubble_activity_log(client_addr, server_addr, event, data):
     pass
 
 
-def bubble_passthru(remote_addr, addr, fqdns):
+def bubble_conn_check(remote_addr, addr, fqdns, security_level):
     headers = {
         'X-Forwarded-For': remote_addr,
         'Accept' : 'application/json',
@@ -68,11 +68,17 @@ def bubble_passthru(remote_addr, addr, fqdns):
             'fqdns': fqdns,
             'remoteAddr': remote_addr
         }
-        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data)
-        return response.ok
+        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data)
+        if response.ok:
+            return response.json()
+        bubble_log('bubble_conn_check API call failed: '+repr(response))
+        return None
+
     except Exception as e:
-        bubble_log('bubble_passthru API call failed: '+repr(e))
+        bubble_log('bubble_conn_check API call failed: '+repr(e))
         traceback.print_exc()
+        if security_level is not None and security_level == 'maximum':
+            return False
         return None
 
 

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py

@@ -0,0 +1,239 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here:
+#
+# Copyright (c) 2013, Aldo Cortesi. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.exceptions import TlsProtocolException
+from mitmproxy.net import tls as net_tls
+
+from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, redis_set
+from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6
+import redis
+import json
+import subprocess
+import traceback
+
+REDIS_DNS_PREFIX = 'bubble_dns_'
+REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_'
+REDIS_CHECK_DURATION = 60 * 60  # 1 hour timeout
+REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_'  # defined in StandardDeviceIdService
+
+REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
+
+FORCE_PASSTHRU = {'passthru': True}
+FORCE_BLOCK = {'block': True}
+
+# Matches enums in DeviceSecurityLevel
+SEC_MAX = 'maximum'
+SEC_STD = 'standard'
+SEC_BASIC = 'basic'
+SEC_OFF = 'disabled'
+
+local_ips = None
+
+
+def get_device_security_level(client_addr):
+    level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr)
+    if level is None:
+        return SEC_MAX
+    return level.decode()
+
+
+def get_local_ips():
+    global local_ips
+    if local_ips is None:
+        local_ips = []
+        for ip in subprocess.check_output(['hostname', '-I']).split():
+            local_ips.append(ip.decode())
+    return local_ips
+
+
+def is_sage_request(ip, fqdns):
+    return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns
+
+
+def conn_check_cache_prefix(client_addr, server_addr):
+    return REDIS_CONN_CHECK_PREFIX + client_addr + '_' + server_addr
+
+
+def fqdns_for_addr(addr):
+    prefix = REDIS_DNS_PREFIX + addr
+    keys = REDIS.keys(prefix + '_*')
+    if keys is None or len(keys) == 0:
+        bubble_log('fqdns_for_addr: no FQDN found for addr '+str(addr)+', checking raw addr')
+        return ''
+    fqdns = []
+    for k in keys:
+        fqdn = k.decode()[len(prefix)+1:]
+        fqdns.append(fqdn)
+    return fqdns
+
+
+class TlsBlock(TlsLayer):
+    """
+    Monkey-patch __call__ to drop this connection entirely
+    """
+    def __call__(self):
+        bubble_log('TlsBlock: blocking')
+        return
+
+
+class TlsFeedback(TlsLayer):
+    """
+    Monkey-patch _establish_tls_with_client to get feedback if TLS could be established
+    successfully on the client connection (which may fail due to cert pinning).
+    """
+    def _establish_tls_with_client(self):
+        client_address = self.client_conn.address[0]
+        server_address = self.server_conn.address[0]
+        security_level = get_device_security_level(client_address)
+        try:
+            super(TlsFeedback, self)._establish_tls_with_client()
+
+        except TlsProtocolException as e:
+            tb = traceback.format_exc()
+            if 'OpenSSL.SSL.ZeroReturnError' in tb:
+                bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', ignoring SSL zero return error for client '+client_address)
+                return
+
+            elif self.fqdns is not None and len(self.fqdns) > 0:
+                for fqdn in self.fqdns:
+                    cache_key = conn_check_cache_prefix(client_address, fqdn)
+                    if security_level == SEC_MAX:
+                        redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                        bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
+                    else:
+                        redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                        bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
+            else:
+                cache_key = conn_check_cache_prefix(client_address, server_address)
+                if security_level == SEC_MAX:
+                    redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                    bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
+                else:
+                    redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                    bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
+            raise e
+
+
+def check_bubble_connection(client_addr, server_addr, fqdns, security_level):
+    check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level)
+    if check_response is None or check_response == 'error':
+        if security_level == SEC_MAX:
+            bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block')
+            return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'}
+        else:
+            bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
+            return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_error'}
+
+    elif check_response == 'passthru':
+        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
+        return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_passthru'}
+
+    elif check_response == 'block':
+        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block')
+        return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_block'}
+
+    else:
+        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False')
+        return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'reason': 'bubble_no_passthru'}
+
+
+def check_connection(client_addr, server_addr, fqdns, security_level):
+    if fqdns and len(fqdns) == 1:
+        cache_key = conn_check_cache_prefix(client_addr, fqdns[0])
+    else:
+        cache_key = conn_check_cache_prefix(client_addr, server_addr)
+    prefix = 'check_connection: ip=' + str(server_addr) + ' (fqdns=' + str(fqdns) + ') cache_key=' + cache_key + ': '
+
+    check_json = REDIS.get(cache_key)
+    if check_json is None or len(check_json) == 0:
+        bubble_log(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns))
+        check_response = check_bubble_connection(client_addr, server_addr, fqdns, security_level)
+        bubble_log(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...")
+        redis_set(cache_key, json.dumps(check_response), ex=REDIS_CHECK_DURATION)
+
+    else:
+        bubble_log(prefix+'found check_json='+str(check_json)+', touching key in redis')
+        check_response = json.loads(check_json)
+        REDIS.touch(cache_key)
+    bubble_log(prefix+'returning '+str(check_response))
+    return check_response
+
+
+def next_layer(next_layer):
+    if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
+        client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile)
+        client_addr = next_layer.client_conn.address[0]
+        server_addr = next_layer.server_conn.address[0]
+
+        if client_hello.sni:
+            fqdn = client_hello.sni.decode()
+            bubble_log('next_layer: using fqdn in SNI: '+ fqdn)
+            fqdns = [ fqdn ]
+        else:
+            fqdns = fqdns_for_addr(server_addr)
+            bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns))
+        next_layer.fqdns = fqdns
+        no_fqdns = fqdns is None or len(fqdns) == 0
+        security_level = get_device_security_level(client_addr)
+        if server_addr in get_local_ips():
+            bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif is_sage_request(server_addr, fqdns):
+            bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif security_level == SEC_OFF or security_level == SEC_BASIC:
+            bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif security_level == SEC_STD and no_fqdns:
+            bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
+            check = FORCE_PASSTHRU
+
+        elif security_level == SEC_MAX and no_fqdns:
+            bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
+            check = FORCE_BLOCK
+
+        else:
+            bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+security_level)
+            check = check_connection(client_addr, server_addr, fqdns, security_level)
+
+        if check is None or ('passthru' in check and check['passthru']):
+            bubble_log('next_layer: enabling passthru for server_addr' + server_addr+', fqdns='+str(fqdns))
+            bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
+            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
+            next_layer.reply.send(next_layer_replacement)
+
+        elif 'block' in check and check['block']:
+            bubble_log('next_layer: enabling block for server_addr' + server_addr+', fqdns='+str(fqdns))
+            bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns)
+            next_layer.__class__ = TlsBlock
+
+        else:
+            bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns))
+            bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns)
+            next_layer.__class__ = TlsFeedback

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py

@@ -1,171 +0,0 @@
-#
-# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
-#
-# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here:
-#
-# Copyright (c) 2013, Aldo Cortesi. All rights reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-#
-from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
-from mitmproxy.exceptions import TlsProtocolException
-
-from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set
-from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6
-import redis
-import json
-import subprocess
-
-REDIS_DNS_PREFIX = 'bubble_dns_'
-REDIS_PASSTHRU_PREFIX = 'bubble_passthru_'
-REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_'  # defined in StandardDeviceIdService
-REDIS_PASSTHRU_DURATION = 60 * 60  # 1 hour timeout on passthru
-
-REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
-
-FORCE_PASSTHRU = {'passthru': True}
-
-local_ips = None
-
-
-def get_device_security_level(client_addr):
-    level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr)
-    if level is None:
-        return 'maximum'
-    return level.decode()
-
-
-def get_local_ips():
-    global local_ips
-    if local_ips is None:
-        local_ips = []
-        for ip in subprocess.check_output(['hostname', '-I']).split():
-            local_ips.append(ip.decode())
-    return local_ips
-
-
-def is_sage_request(ip, fqdns):
-    return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns
-
-
-def passthru_cache_prefix(client_addr, server_addr):
-    return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr
-
-
-def fqdns_for_addr(addr):
-    prefix = REDIS_DNS_PREFIX + addr
-    keys = REDIS.keys(prefix + '_*')
-    if keys is None or len(keys) == 0:
-        bubble_log('fqdns_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr')
-        return ''
-    fqdns = []
-    for k in keys:
-        fqdn = k.decode()[len(prefix)+1:]
-        fqdns.append(fqdn)
-    return fqdns
-
-
-class TlsFeedback(TlsLayer):
-    """
-    Monkey-patch _establish_tls_with_client to get feedback if TLS could be established
-    successfully on the client connection (which may fail due to cert pinning).
-    """
-    def _establish_tls_with_client(self):
-        client_address = self.client_conn.address[0]
-        server_address = self.server_conn.address[0]
-        fqdns = fqdns_for_addr(server_address)
-        try:
-            super(TlsFeedback, self)._establish_tls_with_client()
-
-        except TlsProtocolException as e:
-            cache_key = passthru_cache_prefix(client_address, server_address)
-            bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key)
-            redis_set(cache_key, json.dumps({'fqdns': fqdns, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION)
-            raise e
-
-
-def check_bubble_passthru(client_addr, addr, fqdns):
-    passthru = bubble_passthru(client_addr, addr, fqdns)
-    if passthru is None or passthru:
-        bubble_log('check_bubble_passthru: bubble_passthru returned '+repr(passthru)+' for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning True')
-        return {'fqdns': fqdns, 'addr': addr, 'passthru': True}
-    bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning False')
-    return {'fqdns': fqdns, 'addr': addr, 'passthru': False}
-
-
-def should_passthru(client_addr, addr, fqdns):
-    cache_key = passthru_cache_prefix(client_addr, addr)
-    prefix = 'should_passthru: ip='+repr(addr)+' (fqdns='+repr(fqdns)+') cache_key='+cache_key+': '
-
-    passthru_json = REDIS.get(cache_key)
-    if passthru_json is None or len(passthru_json) == 0:
-        bubble_log(prefix+'not in redis or empty, calling check_bubble_passthru against fqdns='+repr(fqdns))
-        passthru = check_bubble_passthru(client_addr, addr, fqdns)
-        bubble_log(prefix+'check_bubble_passthru('+repr(fqdns)+') returned '+repr(passthru)+", storing in redis...")
-        redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION)
-
-    else:
-        bubble_log(prefix+'found passthru_json='+str(passthru_json)+', touching key in redis')
-        passthru = json.loads(passthru_json)
-        REDIS.touch(cache_key)
-    bubble_log(prefix+'returning '+repr(passthru))
-    return passthru
-
-
-def next_layer(next_layer):
-    if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
-        client_addr = next_layer.client_conn.address[0]
-        server_addr = next_layer.server_conn.address[0]
-
-        fqdns = fqdns_for_addr(server_addr)
-        no_fqdns = fqdns is None or len(fqdns) == 0
-        security_level = get_device_security_level(client_addr)
-        if server_addr in get_local_ips():
-            bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif is_sage_request(server_addr, fqdns):
-            bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif security_level == 'disabled' or security_level == 'basic':
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif security_level == 'standard' and no_fqdns:
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
-            passthru = FORCE_PASSTHRU
-
-        elif security_level == 'maximum' and no_fqdns:
-            bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
-            return
-
-        else:
-            bubble_log('next_layer: checking should_passthru for server='+server_addr+', client='+client_addr+' with security_level='+security_level)
-            passthru = should_passthru(client_addr, server_addr, fqdns)
-
-        if passthru is None or passthru['passthru']:
-            bubble_log('next_layer: enabling passthru for ' + repr(next_layer.server_conn.address))
-            bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
-            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
-            next_layer.reply.send(next_layer_replacement)
-        else:
-            bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr)
-            bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns)
-            next_layer.__class__ = TlsFeedback

bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh

@@ -16,6 +16,6 @@ mitmdump \
   --set stream_large_bodies=5m \
   --set keep_host_header \
   -s ./dns_spoofing.py \
-  -s ./bubble_passthru.py \
+  -s ./bubble_conn_check.py \
   -s ./bubble_modify.py \
   --mode transparent

bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml

@@ -40,7 +40,7 @@
   get_url:
     url: https://github.com/getbubblenow/bubble-dist/raw/master/mitmproxy/mitmproxy.zip
     dest: /tmp/mitmproxy.zip
-    checksum: sha256:a8c4447197376eb1d59113a29419654ab7340d98437965592358ec947031b85b
+    checksum: sha256:1466520af16761b22f4802506cc2d81defdbe759e83bc626aec6a27a4376527a
 
 - name: Unzip mitmproxy.zip
   unarchive:
@@ -58,7 +58,7 @@
   with_items:
     - bubble_api.py
     - dns_spoofing.py
-    - bubble_passthru.py
+    - bubble_conn_check.py
     - bubble_modify.py
     - run_mitmdump.sh
 

utils/abp-parser

@@ -1 +1 @@
-Subproject commit df343fc4b3e1f123b3caa025a493861edf46b81a
+Subproject commit e3d05733525eb15a7106cbca0d585a3104681925

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit e5d7abc4b58a339a5da90fcfe53ba21c20e40c75
+Subproject commit cbd62f426d8544cc625dfb0ac4e5b6ec6a049b8f

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit d450510d6f1be5b328b919ae73d8c1450f008d91
+Subproject commit 9aa42648fcc5694dfcd1778fa7dc4cb0f25455e8