diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index e5bee707..8dfc437e 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -207,7 +207,7 @@ public class ApiConstants { public static final String BUBBLE_MAGIC_ENDPOINT = "/.bubble"; public static final String FILTER_HTTP_ENDPOINT = "/filter"; - public static final String EP_PASSTHRU = "/passthru"; + public static final String EP_CHECK = "/check"; public static final String EP_APPLY = "/apply"; public static final String EP_ASSETS = "/assets"; diff --git a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java index f29ac991..9508846a 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java @@ -34,12 +34,12 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { @Override public Order getDefaultSortOrder() { return PRIORITY_ASC; } - public List findByAccountAndFqdnAndEnabled(String account, String fqdn) { + public List findByAccountAndFqdnAndEnabledAndRequestCheck(String account, String fqdn) { return list(criteria().add( and( eq("account", account), eq("enabled", true), - eq("passthru", false), + eq("requestCheck", true), or( eq("fqdn", fqdn), eq("fqdn", WILDCARD_FQDN) @@ -47,8 +47,8 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { ).addOrder(PRIORITY_ASC)); } - public List findByAccountAndEnabledAndPassthru(String account) { - return findByFields("account", account, "enabled", true, "passthru", true); + public List findByAccountAndEnabledAndConnCheck(String account) { + return findByFields("account", account, "enabled", true, "connCheck", true); } public List findAllChangesSince(Long lastMod) { @@ -56,7 +56,8 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { } @Override public Object preCreate(AppMatcher matcher) { - if (matcher.getPassthru() == null) matcher.setPassthru(false); + if (matcher.getConnCheck() == null) matcher.setConnCheck(false); + if (matcher.getRequestCheck() == null) matcher.setRequestCheck(true); return super.preCreate(matcher); } diff --git a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java index f319784b..c6e6704b 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java +++ b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java @@ -45,7 +45,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, HasPriority { public static final String[] VALUE_FIELDS = {"fqdn", "urlRegex", "template", "enabled", "priority"}; - public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "passthru"); + public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "connCheck"); public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE); public static final String WILDCARD_FQDN = "*"; @@ -131,8 +131,13 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H @ECSearchable @ECField(index=120, required=EntityFieldRequired.optional) @ECIndex @Column(nullable=false) - @Getter @Setter private Boolean passthru; - public boolean passthru () { return bool(passthru); } + @Getter @Setter private Boolean connCheck; + public boolean connCheck () { return bool(connCheck); } + + @ECSearchable @ECField(index=130, required=EntityFieldRequired.optional) + @ECIndex @Column(nullable=false) + @Getter @Setter private Boolean requestCheck; + public boolean requestCheck () { return bool(requestCheck); } @ECSearchable @ECField(index=130) @Column(nullable=false) diff --git a/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java b/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java index e45c8841..fbb41f77 100644 --- a/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java +++ b/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java @@ -19,11 +19,11 @@ import static bubble.ApiConstants.enumFromString; public enum BubbleDeviceType { uninitialized (null, false), - windows (CertType.cer, true, DeviceSecurityLevel.maximum), - macosx (CertType.pem, true, DeviceSecurityLevel.maximum), + windows (CertType.cer, true, DeviceSecurityLevel.standard), + macosx (CertType.pem, true, DeviceSecurityLevel.standard), ios (CertType.pem, true, DeviceSecurityLevel.standard), - android (CertType.cer, true, DeviceSecurityLevel.basic), - linux (CertType.crt, true, DeviceSecurityLevel.maximum), + android (CertType.cer, true, DeviceSecurityLevel.standard), + linux (CertType.crt, true, DeviceSecurityLevel.standard), firefox (CertType.crt, false), other (null, true, DeviceSecurityLevel.basic); @@ -32,7 +32,7 @@ public enum BubbleDeviceType { @Getter private final DeviceSecurityLevel defaultSecurityLevel; public boolean hasDefaultSecurityLevel () { return defaultSecurityLevel != null; } - BubbleDeviceType (CertType certType, boolean selectable) { this(CertType.cer, selectable, null); } + BubbleDeviceType (CertType certType, boolean selectable) { this(certType, selectable, null); } @JsonCreator public static BubbleDeviceType fromString (String v) { return enumFromString(BubbleDeviceType.class, v); } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java similarity index 94% rename from bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java rename to bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java index 8684ca35..83a9f0e2 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java @@ -9,7 +9,7 @@ import lombok.Setter; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -public class FilterPassthruRequest { +public class FilterConnCheckRequest { @Getter @Setter private String addr; public boolean hasAddr() { return !empty(addr); } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 2abbaba0..293238a2 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -21,6 +21,7 @@ import bubble.rule.FilterMatchDecision; import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.DeviceIdService; +import bubble.service.stream.ConnectionCheckResponse; import bubble.service.stream.StandardRuleEngineService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -124,23 +125,23 @@ public class FilterHttpResource { return null; } - @POST @Path(EP_PASSTHRU) + @POST @Path(EP_CHECK) @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON) - public Response isTlsPassthru(@Context Request req, - @Context ContainerRequest request, - FilterPassthruRequest passthruRequest) { - final String prefix = "isPassthru: "; - if (passthruRequest == null || !passthruRequest.hasAddr() || !passthruRequest.hasRemoteAddr()) { - if (log.isDebugEnabled()) log.debug(prefix+"invalid passthruRequest, returning forbidden"); + public Response checkConnection(@Context Request req, + @Context ContainerRequest request, + FilterConnCheckRequest connCheckRequest) { + final String prefix = "checkConnection: "; + if (connCheckRequest == null || !connCheckRequest.hasAddr() || !connCheckRequest.hasRemoteAddr()) { + if (log.isDebugEnabled()) log.debug(prefix+"invalid connCheckRequest, returning forbidden"); return forbidden(); } validateMitmCall(req); // if the requested IP is the same as our IP, then always passthru - if (isForUs(passthruRequest)) return ok(); + if (isForUs(connCheckRequest)) return ok(); - final String vpnAddr = passthruRequest.getRemoteAddr(); + final String vpnAddr = connCheckRequest.getRemoteAddr(); final Device device = deviceIdService.findDeviceByIp(vpnAddr); if (device == null) { if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning not found"); @@ -154,7 +155,7 @@ public class FilterHttpResource { return notFound(); } - final List matchers = matcherDAO.findByAccountAndEnabledAndPassthru(device.getAccount()); + final List matchers = matcherDAO.findByAccountAndEnabledAndConnCheck(device.getAccount()); final List retained = new ArrayList<>(); for (AppMatcher matcher : matchers) { final BubbleApp app = appDAO.findByUuid(matcher.getApp()); @@ -164,23 +165,23 @@ public class FilterHttpResource { retained.add(matcher); } - final String[] fqdns = passthruRequest.getFqdns(); + final String[] fqdns = connCheckRequest.getFqdns(); for (String fqdn : fqdns) { - final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), fqdn); - if (passthru) { - if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+fqdn+"/"+passthruRequest.getAddr()); - return ok(); + final ConnectionCheckResponse checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); + if (checkResponse != ConnectionCheckResponse.noop) { + if (log.isDebugEnabled()) log.debug(prefix + "returning "+checkResponse+" for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); + return ok(checkResponse); } } - if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdns/addr="+Arrays.toString(fqdns)+"/"+passthruRequest.getAddr()); - return notFound(); + if (log.isDebugEnabled()) log.debug(prefix+"returning noop for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.noop); } - private boolean isForUs(FilterPassthruRequest passthruRequest) { + private boolean isForUs(FilterConnCheckRequest connCheckRequest) { final BubbleNode thisNode = selfNodeService.getThisNode(); - return passthruRequest.hasAddr() - && (thisNode.hasIp4() && thisNode.getIp4().equals(passthruRequest.getAddr()) - || thisNode.hasIp6() && thisNode.getIp6().equals(passthruRequest.getAddr())); + return connCheckRequest.hasAddr() + && (thisNode.hasIp4() && thisNode.getIp4().equals(connCheckRequest.getAddr()) + || thisNode.hasIp6() && thisNode.getIp6().equals(connCheckRequest.getAddr())); } @POST @Path(EP_MATCHERS+"/{requestId}") @@ -281,7 +282,7 @@ public class FilterHttpResource { private List getEnabledMatchers(String requestId, String accountUuid, String fqdn) { final String prefix = "getEnabledMatchers("+requestId+"): "; - List matchers = matcherDAO.findByAccountAndFqdnAndEnabled(accountUuid, fqdn); + List matchers = matcherDAO.findByAccountAndFqdnAndEnabledAndRequestCheck(accountUuid, fqdn); if (log.isTraceEnabled()) log.trace(prefix+"checking all enabled matchers for fqdn: "+json(matchers, COMPACT_MAPPER)); matchers = matchers.stream() .filter(m -> appDAO.findByAccountAndId(accountUuid, m.getApp()).enabled()).collect(Collectors.toList()); diff --git a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java index 15f18a66..f1c742e4 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java @@ -64,7 +64,7 @@ public class ReverseProxyResource { if (device == null) return ruleEngine.passthru(request); final URIBean ub = getUriBean(request); - final List matchers = matcherDAO.findByAccountAndFqdnAndEnabled(account.getUuid(), ub.getHost()); + final List matchers = matcherDAO.findByAccountAndFqdnAndEnabledAndRequestCheck(account.getUuid(), ub.getHost()); if (empty(matchers)) { // no matchers, pass-thru return ruleEngine.passthru(ub, request); diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 163d0d6b..aaeec6bf 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -11,6 +11,7 @@ import bubble.model.device.Device; import bubble.resources.stream.FilterHttpRequest; import bubble.resources.stream.FilterMatchersRequest; import bubble.service.stream.AppRuleHarness; +import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; import org.cobbzilla.util.handlebars.HandlebarsUtil; @@ -145,6 +146,12 @@ public interface AppRuleDriver { return null; } - default boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { return false; } + default ConnectionCheckResponse checkConnection(AppRuleHarness harness, + Account account, + Device device, + String addr, + String fqdn) { + return ConnectionCheckResponse.noop; + } } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index ea483b20..b6c45d05 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -15,6 +15,7 @@ import bubble.rule.AppRuleDriver; import bubble.rule.FilterMatchDecision; import bubble.rule.analytics.TrafficAnalyticsRuleDriver; import bubble.service.stream.AppRuleHarness; +import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.input.ReaderInputStream; @@ -108,14 +109,11 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { } blockList.set(newBlockList); - boolean doPrime = false; if (!newBlockList.getFullyBlockedDomains().equals(fullyBlockedDomains.get())) { fullyBlockedDomains.set(newBlockList.getFullyBlockedDomains()); - doPrime = true; } if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) { partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains()); - doPrime = true; } log.info("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size()); @@ -123,6 +121,24 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { log.info("refreshBlockLists: refreshed "+refreshed.size()+" block lists: "+StringUtil.toString(refreshed)); } + @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, + Account account, + Device device, + String addr, + String fqdn) { + final BlockDecision decision = getBlockList().getFqdnDecision(fqdn); + final BlockDecisionType decisionType = decision.getDecisionType(); + switch (decisionType) { + case allow: + return ConnectionCheckResponse.noop; + case block: + return ConnectionCheckResponse.block; + default: + if (log.isWarnEnabled()) log.warn("checkConnection: unexpected decision: "+decisionType+", returning noop"); + return ConnectionCheckResponse.noop; + } + } + @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness, FilterMatchersRequest filter, Account account, diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java index f576bdcc..5788ca09 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java @@ -8,6 +8,7 @@ import com.amazonaws.util.IOUtils; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.cache.AutoRefreshingReference; @@ -21,9 +22,11 @@ import java.util.stream.Collectors; import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_FEEDS; import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.regex.Pattern.CASE_INSENSITIVE; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; import static org.cobbzilla.util.http.HttpUtil.getUrlInputStream; +import static org.cobbzilla.util.string.ValidationRegexes.HOST; import static org.cobbzilla.wizard.server.RestServerBase.reportError; @Slf4j @Accessors(chain=true) @@ -69,6 +72,7 @@ public class TlsPassthruConfig { return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); } + @ToString private static class TlsPassthruMatcher { @Getter @Setter private String fqdn; @Getter @Setter private Pattern fqdnPattern; @@ -76,7 +80,9 @@ public class TlsPassthruConfig { public TlsPassthruMatcher (String fqdn) { this.fqdn = fqdn; if (fqdn.startsWith("/") && fqdn.endsWith("/")) { - this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), Pattern.CASE_INSENSITIVE); + this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), CASE_INSENSITIVE); + } else if (fqdn.startsWith("*.")) { + this.fqdnPattern = Pattern.compile("("+HOST+"\\.)?"+Pattern.quote(fqdn.substring(2)), CASE_INSENSITIVE); } } public boolean matches (String val) { diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index 1b4cb50b..9241ebd0 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -8,6 +8,7 @@ import bubble.model.account.Account; import bubble.model.device.Device; import bubble.rule.AbstractAppRuleDriver; import bubble.service.stream.AppRuleHarness; +import bubble.service.stream.ConnectionCheckResponse; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -15,14 +16,14 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { @Override public Class getConfigClass() { return (Class) TlsPassthruConfig.class; } - @Override public boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { + @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { final TlsPassthruConfig passthruConfig = getRuleConfig(); if (passthruConfig.isPassthru(fqdn) || passthruConfig.isPassthru(addr)) { - if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning true for fqdn/addr="+fqdn+"/"+addr); - return true; + if (log.isDebugEnabled()) log.debug("checkConnection: returning passthru for fqdn/addr="+fqdn+"/"+addr); + return ConnectionCheckResponse.passthru; } - if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning false for fqdn/addr="+fqdn+"/"+addr); - return false; + if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr); + return ConnectionCheckResponse.noop; } } diff --git a/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java new file mode 100644 index 00000000..9c663898 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java @@ -0,0 +1,13 @@ +package bubble.service.stream; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import static bubble.ApiConstants.enumFromString; + +public enum ConnectionCheckResponse { + + noop, passthru, block, error; + + @JsonCreator public static ConnectionCheckResponse fromString (String v) { return enumFromString(ConnectionCheckResponse.class, v); } + +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java index 94db1fed..778b5864 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java @@ -50,7 +50,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -64,6 +63,7 @@ import static org.apache.http.HttpHeaders.TRANSFER_ENCODING; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf; import static org.cobbzilla.util.http.HttpStatusCodes.OK; +import static org.cobbzilla.wizard.cache.redis.RedisService.ALL_KEYS; import static org.cobbzilla.wizard.resources.ResourceUtil.send; @Service @Slf4j @@ -179,18 +179,18 @@ public class StandardRuleEngineService implements RuleEngineService { public Map flushCaches() { final int ruleEngineCacheSize = ruleCache.size(); ruleCache.clear(); - log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries"); + if (log.isInfoEnabled()) log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries"); final RedisService matchersCache = getMatchersCache(); - final Collection keys = matchersCache.keys("*"); - for (String key : keys) { - if (log.isTraceEnabled()) log.trace("flushCaches: deleting key: "+key); - matchersCache.del_withPrefix(key); - } - log.info("flushCaches: flushed "+keys.size()+" matchersCache entries"); + final Long matcherCount = matchersCache.del_matching(ALL_KEYS); + if (log.isInfoEnabled()) log.info("flushCaches: flushed "+matcherCount+" matchersCache entries"); + + final Long connCheckDeletions = redis.del_matching("bubble_conn_check_*"); + if (log.isInfoEnabled()) log.info("flushCaches: removed "+connCheckDeletions+" conn_check cache entries"); return MapBuilder.build(new Object[][] { - {"matchersCache", keys.size()}, + {"connCheckCache", connCheckDeletions}, + {"matchersCache", matcherCount}, {"ruleEngineCache", ruleEngineCacheSize} }); } @@ -275,11 +275,13 @@ public class StandardRuleEngineService implements RuleEngineService { @Getter(lazy=true) private final HttpClientBuilder httpClientBuilder = newHttpClientBuilder(1000, 50); public CloseableHttpClient newHttpConn() { return getHttpClientBuilder().build(); } - public boolean isTlsPassthru(Account account, Device device, List matchers, String addr, String fqdn) { + public ConnectionCheckResponse checkConnection(Account account, Device device, List matchers, String addr, String fqdn) { final List ruleHarnesses = initRules(account, device, matchers); for (AppRuleHarness harness : ruleHarnesses) { - if (harness.getDriver().isTlsPassthru(harness, account, device, addr, fqdn)) return true; + final ConnectionCheckResponse checkResponse = harness.getDriver().checkConnection(harness, account, device, addr, fqdn); + if (checkResponse != ConnectionCheckResponse.noop) return checkResponse; } - return false; + return ConnectionCheckResponse.noop; } + } diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index af78b40d..463bfc38 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=0.12.4 +bubble.version=0.12.5 diff --git a/bubble-server/src/main/resources/logback.xml b/bubble-server/src/main/resources/logback.xml index 3714e9cb..4ebaeb60 100644 --- a/bubble-server/src/main/resources/logback.xml +++ b/bubble-server/src/main/resources/logback.xml @@ -61,6 +61,7 @@ + diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json index 9f69be13..f8664293 100644 --- a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json +++ b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json @@ -134,6 +134,13 @@ "description": "EasyList is the primary filter list that removes most adverts from international web pages, including unwanted frames, images, and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists.", "tags": ["ads"] }, + { + "name": "Peter Lowe's Ad and tracking server list", + "id": "peter_lowes_ad_and_tracking_list", + "url": "https://pgl.yoyo.org/adservers/serverlist.php?showintro=0;hostformat=raw", + "description": "A comprehensive list of ad and tracking servers", + "tags": ["ads", "privacy"] + }, { "name": "Dandelion Sprout's Anti-Malware List", "id": "dandelion_sprouts_anti_malware_list", diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json index ca1ff704..40634288 100644 --- a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json +++ b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json @@ -4,6 +4,7 @@ "AppMatcher": [{ "name": "BubbleBlockMatcher", "template": true, + "connCheck": true, "site": "All_Sites", "fqdn": "*", "urlRegex": ".*", diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json index 80cc5aad..a8dd1a72 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json @@ -4,7 +4,8 @@ "AppMatcher": [{ "name": "TlsPassthruMatcher", "template": true, - "passthru": true, + "connCheck": true, + "requestCheck": false, "site": "All_Sites", "fqdn": "*", "urlRegex": ".*", diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 47ef2d44..02419dee 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -56,7 +56,7 @@ def bubble_activity_log(client_addr, server_addr, event, data): pass -def bubble_passthru(remote_addr, addr, fqdns): +def bubble_conn_check(remote_addr, addr, fqdns, security_level): headers = { 'X-Forwarded-For': remote_addr, 'Accept' : 'application/json', @@ -68,11 +68,17 @@ def bubble_passthru(remote_addr, addr, fqdns): 'fqdns': fqdns, 'remoteAddr': remote_addr } - response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data) - return response.ok + response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data) + if response.ok: + return response.json() + bubble_log('bubble_conn_check API call failed: '+repr(response)) + return None + except Exception as e: - bubble_log('bubble_passthru API call failed: '+repr(e)) + bubble_log('bubble_conn_check API call failed: '+repr(e)) traceback.print_exc() + if security_level is not None and security_level == 'maximum': + return False return None diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py new file mode 100644 index 00000000..ad4909af --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -0,0 +1,239 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here: +# +# Copyright (c) 2013, Aldo Cortesi. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer +from mitmproxy.exceptions import TlsProtocolException +from mitmproxy.net import tls as net_tls + +from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, redis_set +from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6 +import redis +import json +import subprocess +import traceback + +REDIS_DNS_PREFIX = 'bubble_dns_' +REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' +REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout +REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService + +REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) + +FORCE_PASSTHRU = {'passthru': True} +FORCE_BLOCK = {'block': True} + +# Matches enums in DeviceSecurityLevel +SEC_MAX = 'maximum' +SEC_STD = 'standard' +SEC_BASIC = 'basic' +SEC_OFF = 'disabled' + +local_ips = None + + +def get_device_security_level(client_addr): + level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr) + if level is None: + return SEC_MAX + return level.decode() + + +def get_local_ips(): + global local_ips + if local_ips is None: + local_ips = [] + for ip in subprocess.check_output(['hostname', '-I']).split(): + local_ips.append(ip.decode()) + return local_ips + + +def is_sage_request(ip, fqdns): + return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns + + +def conn_check_cache_prefix(client_addr, server_addr): + return REDIS_CONN_CHECK_PREFIX + client_addr + '_' + server_addr + + +def fqdns_for_addr(addr): + prefix = REDIS_DNS_PREFIX + addr + keys = REDIS.keys(prefix + '_*') + if keys is None or len(keys) == 0: + bubble_log('fqdns_for_addr: no FQDN found for addr '+str(addr)+', checking raw addr') + return '' + fqdns = [] + for k in keys: + fqdn = k.decode()[len(prefix)+1:] + fqdns.append(fqdn) + return fqdns + + +class TlsBlock(TlsLayer): + """ + Monkey-patch __call__ to drop this connection entirely + """ + def __call__(self): + bubble_log('TlsBlock: blocking') + return + + +class TlsFeedback(TlsLayer): + """ + Monkey-patch _establish_tls_with_client to get feedback if TLS could be established + successfully on the client connection (which may fail due to cert pinning). + """ + def _establish_tls_with_client(self): + client_address = self.client_conn.address[0] + server_address = self.server_conn.address[0] + security_level = get_device_security_level(client_address) + try: + super(TlsFeedback, self)._establish_tls_with_client() + + except TlsProtocolException as e: + tb = traceback.format_exc() + if 'OpenSSL.SSL.ZeroReturnError' in tb: + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', ignoring SSL zero return error for client '+client_address) + return + + elif self.fqdns is not None and len(self.fqdns) > 0: + for fqdn in self.fqdns: + cache_key = conn_check_cache_prefix(client_address, fqdn) + if security_level == SEC_MAX: + redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) + else: + redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) + else: + cache_key = conn_check_cache_prefix(client_address, server_address) + if security_level == SEC_MAX: + redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e)) + else: + redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e)) + raise e + + +def check_bubble_connection(client_addr, server_addr, fqdns, security_level): + check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level) + if check_response is None or check_response == 'error': + if security_level == SEC_MAX: + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'} + else: + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_error'} + + elif check_response == 'passthru': + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_passthru'} + + elif check_response == 'block': + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_block'} + + else: + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'reason': 'bubble_no_passthru'} + + +def check_connection(client_addr, server_addr, fqdns, security_level): + if fqdns and len(fqdns) == 1: + cache_key = conn_check_cache_prefix(client_addr, fqdns[0]) + else: + cache_key = conn_check_cache_prefix(client_addr, server_addr) + prefix = 'check_connection: ip=' + str(server_addr) + ' (fqdns=' + str(fqdns) + ') cache_key=' + cache_key + ': ' + + check_json = REDIS.get(cache_key) + if check_json is None or len(check_json) == 0: + bubble_log(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns)) + check_response = check_bubble_connection(client_addr, server_addr, fqdns, security_level) + bubble_log(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...") + redis_set(cache_key, json.dumps(check_response), ex=REDIS_CHECK_DURATION) + + else: + bubble_log(prefix+'found check_json='+str(check_json)+', touching key in redis') + check_response = json.loads(check_json) + REDIS.touch(cache_key) + bubble_log(prefix+'returning '+str(check_response)) + return check_response + + +def next_layer(next_layer): + if isinstance(next_layer, TlsLayer) and next_layer._client_tls: + client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile) + client_addr = next_layer.client_conn.address[0] + server_addr = next_layer.server_conn.address[0] + + if client_hello.sni: + fqdn = client_hello.sni.decode() + bubble_log('next_layer: using fqdn in SNI: '+ fqdn) + fqdns = [ fqdn ] + else: + fqdns = fqdns_for_addr(server_addr) + bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns)) + next_layer.fqdns = fqdns + no_fqdns = fqdns is None or len(fqdns) == 0 + security_level = get_device_security_level(client_addr) + if server_addr in get_local_ips(): + bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif is_sage_request(server_addr, fqdns): + bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif security_level == SEC_OFF or security_level == SEC_BASIC: + bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif security_level == SEC_STD and no_fqdns: + bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif security_level == SEC_MAX and no_fqdns: + bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) + check = FORCE_BLOCK + + else: + bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+security_level) + check = check_connection(client_addr, server_addr, fqdns, security_level) + + if check is None or ('passthru' in check and check['passthru']): + bubble_log('next_layer: enabling passthru for server_addr' + server_addr+', fqdns='+str(fqdns)) + bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) + next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) + next_layer.reply.send(next_layer_replacement) + + elif 'block' in check and check['block']: + bubble_log('next_layer: enabling block for server_addr' + server_addr+', fqdns='+str(fqdns)) + bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns) + next_layer.__class__ = TlsBlock + + else: + bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) + bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns) + next_layer.__class__ = TlsFeedback diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py deleted file mode 100644 index 1bf8a176..00000000 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py +++ /dev/null @@ -1,171 +0,0 @@ -# -# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ -# -# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here: -# -# Copyright (c) 2013, Aldo Cortesi. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer -from mitmproxy.exceptions import TlsProtocolException - -from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set -from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6 -import redis -import json -import subprocess - -REDIS_DNS_PREFIX = 'bubble_dns_' -REDIS_PASSTHRU_PREFIX = 'bubble_passthru_' -REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService -REDIS_PASSTHRU_DURATION = 60 * 60 # 1 hour timeout on passthru - -REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) - -FORCE_PASSTHRU = {'passthru': True} - -local_ips = None - - -def get_device_security_level(client_addr): - level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr) - if level is None: - return 'maximum' - return level.decode() - - -def get_local_ips(): - global local_ips - if local_ips is None: - local_ips = [] - for ip in subprocess.check_output(['hostname', '-I']).split(): - local_ips.append(ip.decode()) - return local_ips - - -def is_sage_request(ip, fqdns): - return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns - - -def passthru_cache_prefix(client_addr, server_addr): - return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr - - -def fqdns_for_addr(addr): - prefix = REDIS_DNS_PREFIX + addr - keys = REDIS.keys(prefix + '_*') - if keys is None or len(keys) == 0: - bubble_log('fqdns_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr') - return '' - fqdns = [] - for k in keys: - fqdn = k.decode()[len(prefix)+1:] - fqdns.append(fqdn) - return fqdns - - -class TlsFeedback(TlsLayer): - """ - Monkey-patch _establish_tls_with_client to get feedback if TLS could be established - successfully on the client connection (which may fail due to cert pinning). - """ - def _establish_tls_with_client(self): - client_address = self.client_conn.address[0] - server_address = self.server_conn.address[0] - fqdns = fqdns_for_addr(server_address) - try: - super(TlsFeedback, self)._establish_tls_with_client() - - except TlsProtocolException as e: - cache_key = passthru_cache_prefix(client_address, server_address) - bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key) - redis_set(cache_key, json.dumps({'fqdns': fqdns, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) - raise e - - -def check_bubble_passthru(client_addr, addr, fqdns): - passthru = bubble_passthru(client_addr, addr, fqdns) - if passthru is None or passthru: - bubble_log('check_bubble_passthru: bubble_passthru returned '+repr(passthru)+' for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning True') - return {'fqdns': fqdns, 'addr': addr, 'passthru': True} - bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning False') - return {'fqdns': fqdns, 'addr': addr, 'passthru': False} - - -def should_passthru(client_addr, addr, fqdns): - cache_key = passthru_cache_prefix(client_addr, addr) - prefix = 'should_passthru: ip='+repr(addr)+' (fqdns='+repr(fqdns)+') cache_key='+cache_key+': ' - - passthru_json = REDIS.get(cache_key) - if passthru_json is None or len(passthru_json) == 0: - bubble_log(prefix+'not in redis or empty, calling check_bubble_passthru against fqdns='+repr(fqdns)) - passthru = check_bubble_passthru(client_addr, addr, fqdns) - bubble_log(prefix+'check_bubble_passthru('+repr(fqdns)+') returned '+repr(passthru)+", storing in redis...") - redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) - - else: - bubble_log(prefix+'found passthru_json='+str(passthru_json)+', touching key in redis') - passthru = json.loads(passthru_json) - REDIS.touch(cache_key) - bubble_log(prefix+'returning '+repr(passthru)) - return passthru - - -def next_layer(next_layer): - if isinstance(next_layer, TlsLayer) and next_layer._client_tls: - client_addr = next_layer.client_conn.address[0] - server_addr = next_layer.server_conn.address[0] - - fqdns = fqdns_for_addr(server_addr) - no_fqdns = fqdns is None or len(fqdns) == 0 - security_level = get_device_security_level(client_addr) - if server_addr in get_local_ips(): - bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif is_sage_request(server_addr, fqdns): - bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif security_level == 'disabled' or security_level == 'basic': - bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif security_level == 'standard' and no_fqdns: - bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif security_level == 'maximum' and no_fqdns: - bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) - return - - else: - bubble_log('next_layer: checking should_passthru for server='+server_addr+', client='+client_addr+' with security_level='+security_level) - passthru = should_passthru(client_addr, server_addr, fqdns) - - if passthru is None or passthru['passthru']: - bubble_log('next_layer: enabling passthru for ' + repr(next_layer.server_conn.address)) - bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) - next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) - next_layer.reply.send(next_layer_replacement) - else: - bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr) - bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns) - next_layer.__class__ = TlsFeedback diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh index 4ab69597..0f4bc1e6 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh @@ -16,6 +16,6 @@ mitmdump \ --set stream_large_bodies=5m \ --set keep_host_header \ -s ./dns_spoofing.py \ - -s ./bubble_passthru.py \ + -s ./bubble_conn_check.py \ -s ./bubble_modify.py \ --mode transparent diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml index f1eca30f..fcee8a4c 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml @@ -40,7 +40,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/mitmproxy/mitmproxy.zip dest: /tmp/mitmproxy.zip - checksum: sha256:a8c4447197376eb1d59113a29419654ab7340d98437965592358ec947031b85b + checksum: sha256:1466520af16761b22f4802506cc2d81defdbe759e83bc626aec6a27a4376527a - name: Unzip mitmproxy.zip unarchive: @@ -58,7 +58,7 @@ with_items: - bubble_api.py - dns_spoofing.py - - bubble_passthru.py + - bubble_conn_check.py - bubble_modify.py - run_mitmdump.sh diff --git a/utils/abp-parser b/utils/abp-parser index df343fc4..e3d05733 160000 --- a/utils/abp-parser +++ b/utils/abp-parser @@ -1 +1 @@ -Subproject commit df343fc4b3e1f123b3caa025a493861edf46b81a +Subproject commit e3d05733525eb15a7106cbca0d585a3104681925 diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index e5d7abc4..cbd62f42 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit e5d7abc4b58a339a5da90fcfe53ba21c20e40c75 +Subproject commit cbd62f426d8544cc625dfb0ac4e5b6ec6a049b8f diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index d450510d..9aa42648 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit d450510d6f1be5b328b919ae73d8c1450f008d91 +Subproject commit 9aa42648fcc5694dfcd1778fa7dc4cb0f25455e8