diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index e5bee707..8dfc437e 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -207,7 +207,7 @@ public class ApiConstants { public static final String BUBBLE_MAGIC_ENDPOINT = "/.bubble"; public static final String FILTER_HTTP_ENDPOINT = "/filter"; - public static final String EP_PASSTHRU = "/passthru"; + public static final String EP_CHECK = "/check"; public static final String EP_APPLY = "/apply"; public static final String EP_ASSETS = "/assets"; diff --git a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java index f29ac991..0391d823 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java @@ -39,7 +39,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { and( eq("account", account), eq("enabled", true), - eq("passthru", false), + eq("connCheck", false), or( eq("fqdn", fqdn), eq("fqdn", WILDCARD_FQDN) @@ -47,8 +47,8 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { ).addOrder(PRIORITY_ASC)); } - public List findByAccountAndEnabledAndPassthru(String account) { - return findByFields("account", account, "enabled", true, "passthru", true); + public List findByAccountAndEnabledAndConnCheck(String account) { + return findByFields("account", account, "enabled", true, "connCheck", true); } public List findAllChangesSince(Long lastMod) { @@ -56,7 +56,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { } @Override public Object preCreate(AppMatcher matcher) { - if (matcher.getPassthru() == null) matcher.setPassthru(false); + if (matcher.getConnCheck() == null) matcher.setConnCheck(false); return super.preCreate(matcher); } diff --git a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java index f319784b..e4e01dcf 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java +++ b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java @@ -45,7 +45,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, HasPriority { public static final String[] VALUE_FIELDS = {"fqdn", "urlRegex", "template", "enabled", "priority"}; - public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "passthru"); + public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "connCheck"); public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE); public static final String WILDCARD_FQDN = "*"; @@ -131,8 +131,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H @ECSearchable @ECField(index=120, required=EntityFieldRequired.optional) @ECIndex @Column(nullable=false) - @Getter @Setter private Boolean passthru; - public boolean passthru () { return bool(passthru); } + @Getter @Setter private Boolean connCheck; + public boolean connCheck () { return bool(connCheck); } @ECSearchable @ECField(index=130) @Column(nullable=false) diff --git a/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java b/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java index e45c8841..138c7ae1 100644 --- a/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java +++ b/bubble-server/src/main/java/bubble/model/device/BubbleDeviceType.java @@ -21,8 +21,8 @@ public enum BubbleDeviceType { uninitialized (null, false), windows (CertType.cer, true, DeviceSecurityLevel.maximum), macosx (CertType.pem, true, DeviceSecurityLevel.maximum), - ios (CertType.pem, true, DeviceSecurityLevel.standard), - android (CertType.cer, true, DeviceSecurityLevel.basic), + ios (CertType.pem, true, DeviceSecurityLevel.maximum), + android (CertType.cer, true, DeviceSecurityLevel.standard), linux (CertType.crt, true, DeviceSecurityLevel.maximum), firefox (CertType.crt, false), other (null, true, DeviceSecurityLevel.basic); @@ -32,7 +32,7 @@ public enum BubbleDeviceType { @Getter private final DeviceSecurityLevel defaultSecurityLevel; public boolean hasDefaultSecurityLevel () { return defaultSecurityLevel != null; } - BubbleDeviceType (CertType certType, boolean selectable) { this(CertType.cer, selectable, null); } + BubbleDeviceType (CertType certType, boolean selectable) { this(certType, selectable, null); } @JsonCreator public static BubbleDeviceType fromString (String v) { return enumFromString(BubbleDeviceType.class, v); } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java similarity index 94% rename from bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java rename to bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java index 8684ca35..83a9f0e2 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java @@ -9,7 +9,7 @@ import lombok.Setter; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -public class FilterPassthruRequest { +public class FilterConnCheckRequest { @Getter @Setter private String addr; public boolean hasAddr() { return !empty(addr); } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 2abbaba0..9e59dbc7 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -21,6 +21,7 @@ import bubble.rule.FilterMatchDecision; import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.DeviceIdService; +import bubble.service.stream.ConnectionCheckResponse; import bubble.service.stream.StandardRuleEngineService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -124,23 +125,23 @@ public class FilterHttpResource { return null; } - @POST @Path(EP_PASSTHRU) + @POST @Path(EP_CHECK) @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON) - public Response isTlsPassthru(@Context Request req, - @Context ContainerRequest request, - FilterPassthruRequest passthruRequest) { - final String prefix = "isPassthru: "; - if (passthruRequest == null || !passthruRequest.hasAddr() || !passthruRequest.hasRemoteAddr()) { - if (log.isDebugEnabled()) log.debug(prefix+"invalid passthruRequest, returning forbidden"); + public Response checkConnection(@Context Request req, + @Context ContainerRequest request, + FilterConnCheckRequest connCheckRequest) { + final String prefix = "checkConnection: "; + if (connCheckRequest == null || !connCheckRequest.hasAddr() || !connCheckRequest.hasRemoteAddr()) { + if (log.isDebugEnabled()) log.debug(prefix+"invalid connCheckRequest, returning forbidden"); return forbidden(); } validateMitmCall(req); // if the requested IP is the same as our IP, then always passthru - if (isForUs(passthruRequest)) return ok(); + if (isForUs(connCheckRequest)) return ok(); - final String vpnAddr = passthruRequest.getRemoteAddr(); + final String vpnAddr = connCheckRequest.getRemoteAddr(); final Device device = deviceIdService.findDeviceByIp(vpnAddr); if (device == null) { if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning not found"); @@ -154,7 +155,7 @@ public class FilterHttpResource { return notFound(); } - final List matchers = matcherDAO.findByAccountAndEnabledAndPassthru(device.getAccount()); + final List matchers = matcherDAO.findByAccountAndEnabledAndConnCheck(device.getAccount()); final List retained = new ArrayList<>(); for (AppMatcher matcher : matchers) { final BubbleApp app = appDAO.findByUuid(matcher.getApp()); @@ -164,23 +165,23 @@ public class FilterHttpResource { retained.add(matcher); } - final String[] fqdns = passthruRequest.getFqdns(); + final String[] fqdns = connCheckRequest.getFqdns(); for (String fqdn : fqdns) { - final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), fqdn); - if (passthru) { - if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+fqdn+"/"+passthruRequest.getAddr()); - return ok(); + final ConnectionCheckResponse checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); + if (checkResponse != ConnectionCheckResponse.noop) { + if (log.isDebugEnabled()) log.debug(prefix + "returning "+checkResponse+" for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); + return ok(checkResponse); } } - if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdns/addr="+Arrays.toString(fqdns)+"/"+passthruRequest.getAddr()); - return notFound(); + if (log.isDebugEnabled()) log.debug(prefix+"returning noop for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.noop); } - private boolean isForUs(FilterPassthruRequest passthruRequest) { + private boolean isForUs(FilterConnCheckRequest connCheckRequest) { final BubbleNode thisNode = selfNodeService.getThisNode(); - return passthruRequest.hasAddr() - && (thisNode.hasIp4() && thisNode.getIp4().equals(passthruRequest.getAddr()) - || thisNode.hasIp6() && thisNode.getIp6().equals(passthruRequest.getAddr())); + return connCheckRequest.hasAddr() + && (thisNode.hasIp4() && thisNode.getIp4().equals(connCheckRequest.getAddr()) + || thisNode.hasIp6() && thisNode.getIp6().equals(connCheckRequest.getAddr())); } @POST @Path(EP_MATCHERS+"/{requestId}") diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 163d0d6b..aaeec6bf 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -11,6 +11,7 @@ import bubble.model.device.Device; import bubble.resources.stream.FilterHttpRequest; import bubble.resources.stream.FilterMatchersRequest; import bubble.service.stream.AppRuleHarness; +import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; import org.cobbzilla.util.handlebars.HandlebarsUtil; @@ -145,6 +146,12 @@ public interface AppRuleDriver { return null; } - default boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { return false; } + default ConnectionCheckResponse checkConnection(AppRuleHarness harness, + Account account, + Device device, + String addr, + String fqdn) { + return ConnectionCheckResponse.noop; + } } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index ea483b20..b6c45d05 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -15,6 +15,7 @@ import bubble.rule.AppRuleDriver; import bubble.rule.FilterMatchDecision; import bubble.rule.analytics.TrafficAnalyticsRuleDriver; import bubble.service.stream.AppRuleHarness; +import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.input.ReaderInputStream; @@ -108,14 +109,11 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { } blockList.set(newBlockList); - boolean doPrime = false; if (!newBlockList.getFullyBlockedDomains().equals(fullyBlockedDomains.get())) { fullyBlockedDomains.set(newBlockList.getFullyBlockedDomains()); - doPrime = true; } if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) { partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains()); - doPrime = true; } log.info("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size()); @@ -123,6 +121,24 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { log.info("refreshBlockLists: refreshed "+refreshed.size()+" block lists: "+StringUtil.toString(refreshed)); } + @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, + Account account, + Device device, + String addr, + String fqdn) { + final BlockDecision decision = getBlockList().getFqdnDecision(fqdn); + final BlockDecisionType decisionType = decision.getDecisionType(); + switch (decisionType) { + case allow: + return ConnectionCheckResponse.noop; + case block: + return ConnectionCheckResponse.block; + default: + if (log.isWarnEnabled()) log.warn("checkConnection: unexpected decision: "+decisionType+", returning noop"); + return ConnectionCheckResponse.noop; + } + } + @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness, FilterMatchersRequest filter, Account account, diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java index f576bdcc..5788ca09 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java @@ -8,6 +8,7 @@ import com.amazonaws.util.IOUtils; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.cache.AutoRefreshingReference; @@ -21,9 +22,11 @@ import java.util.stream.Collectors; import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_FEEDS; import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.regex.Pattern.CASE_INSENSITIVE; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; import static org.cobbzilla.util.http.HttpUtil.getUrlInputStream; +import static org.cobbzilla.util.string.ValidationRegexes.HOST; import static org.cobbzilla.wizard.server.RestServerBase.reportError; @Slf4j @Accessors(chain=true) @@ -69,6 +72,7 @@ public class TlsPassthruConfig { return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); } + @ToString private static class TlsPassthruMatcher { @Getter @Setter private String fqdn; @Getter @Setter private Pattern fqdnPattern; @@ -76,7 +80,9 @@ public class TlsPassthruConfig { public TlsPassthruMatcher (String fqdn) { this.fqdn = fqdn; if (fqdn.startsWith("/") && fqdn.endsWith("/")) { - this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), Pattern.CASE_INSENSITIVE); + this.fqdnPattern = Pattern.compile(fqdn.substring(1, fqdn.length()-1), CASE_INSENSITIVE); + } else if (fqdn.startsWith("*.")) { + this.fqdnPattern = Pattern.compile("("+HOST+"\\.)?"+Pattern.quote(fqdn.substring(2)), CASE_INSENSITIVE); } } public boolean matches (String val) { diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index 1b4cb50b..9241ebd0 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -8,6 +8,7 @@ import bubble.model.account.Account; import bubble.model.device.Device; import bubble.rule.AbstractAppRuleDriver; import bubble.service.stream.AppRuleHarness; +import bubble.service.stream.ConnectionCheckResponse; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -15,14 +16,14 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { @Override public Class getConfigClass() { return (Class) TlsPassthruConfig.class; } - @Override public boolean isTlsPassthru(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { + @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { final TlsPassthruConfig passthruConfig = getRuleConfig(); if (passthruConfig.isPassthru(fqdn) || passthruConfig.isPassthru(addr)) { - if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning true for fqdn/addr="+fqdn+"/"+addr); - return true; + if (log.isDebugEnabled()) log.debug("checkConnection: returning passthru for fqdn/addr="+fqdn+"/"+addr); + return ConnectionCheckResponse.passthru; } - if (log.isDebugEnabled()) log.debug("isTlsPassthru: returning false for fqdn/addr="+fqdn+"/"+addr); - return false; + if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr); + return ConnectionCheckResponse.noop; } } diff --git a/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java new file mode 100644 index 00000000..9c663898 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java @@ -0,0 +1,13 @@ +package bubble.service.stream; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import static bubble.ApiConstants.enumFromString; + +public enum ConnectionCheckResponse { + + noop, passthru, block, error; + + @JsonCreator public static ConnectionCheckResponse fromString (String v) { return enumFromString(ConnectionCheckResponse.class, v); } + +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java index 94db1fed..778b5864 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java @@ -50,7 +50,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -64,6 +63,7 @@ import static org.apache.http.HttpHeaders.TRANSFER_ENCODING; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf; import static org.cobbzilla.util.http.HttpStatusCodes.OK; +import static org.cobbzilla.wizard.cache.redis.RedisService.ALL_KEYS; import static org.cobbzilla.wizard.resources.ResourceUtil.send; @Service @Slf4j @@ -179,18 +179,18 @@ public class StandardRuleEngineService implements RuleEngineService { public Map flushCaches() { final int ruleEngineCacheSize = ruleCache.size(); ruleCache.clear(); - log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries"); + if (log.isInfoEnabled()) log.info("flushCaches: flushed "+ruleEngineCacheSize+" ruleCache entries"); final RedisService matchersCache = getMatchersCache(); - final Collection keys = matchersCache.keys("*"); - for (String key : keys) { - if (log.isTraceEnabled()) log.trace("flushCaches: deleting key: "+key); - matchersCache.del_withPrefix(key); - } - log.info("flushCaches: flushed "+keys.size()+" matchersCache entries"); + final Long matcherCount = matchersCache.del_matching(ALL_KEYS); + if (log.isInfoEnabled()) log.info("flushCaches: flushed "+matcherCount+" matchersCache entries"); + + final Long connCheckDeletions = redis.del_matching("bubble_conn_check_*"); + if (log.isInfoEnabled()) log.info("flushCaches: removed "+connCheckDeletions+" conn_check cache entries"); return MapBuilder.build(new Object[][] { - {"matchersCache", keys.size()}, + {"connCheckCache", connCheckDeletions}, + {"matchersCache", matcherCount}, {"ruleEngineCache", ruleEngineCacheSize} }); } @@ -275,11 +275,13 @@ public class StandardRuleEngineService implements RuleEngineService { @Getter(lazy=true) private final HttpClientBuilder httpClientBuilder = newHttpClientBuilder(1000, 50); public CloseableHttpClient newHttpConn() { return getHttpClientBuilder().build(); } - public boolean isTlsPassthru(Account account, Device device, List matchers, String addr, String fqdn) { + public ConnectionCheckResponse checkConnection(Account account, Device device, List matchers, String addr, String fqdn) { final List ruleHarnesses = initRules(account, device, matchers); for (AppRuleHarness harness : ruleHarnesses) { - if (harness.getDriver().isTlsPassthru(harness, account, device, addr, fqdn)) return true; + final ConnectionCheckResponse checkResponse = harness.getDriver().checkConnection(harness, account, device, addr, fqdn); + if (checkResponse != ConnectionCheckResponse.noop) return checkResponse; } - return false; + return ConnectionCheckResponse.noop; } + } diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json index ca1ff704..40634288 100644 --- a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json +++ b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_matchers.json @@ -4,6 +4,7 @@ "AppMatcher": [{ "name": "BubbleBlockMatcher", "template": true, + "connCheck": true, "site": "All_Sites", "fqdn": "*", "urlRegex": ".*", diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json index 80cc5aad..f730e872 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru_matchers.json @@ -4,7 +4,7 @@ "AppMatcher": [{ "name": "TlsPassthruMatcher", "template": true, - "passthru": true, + "connCheck": true, "site": "All_Sites", "fqdn": "*", "urlRegex": ".*", diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 47ef2d44..02419dee 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -56,7 +56,7 @@ def bubble_activity_log(client_addr, server_addr, event, data): pass -def bubble_passthru(remote_addr, addr, fqdns): +def bubble_conn_check(remote_addr, addr, fqdns, security_level): headers = { 'X-Forwarded-For': remote_addr, 'Accept' : 'application/json', @@ -68,11 +68,17 @@ def bubble_passthru(remote_addr, addr, fqdns): 'fqdns': fqdns, 'remoteAddr': remote_addr } - response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data) - return response.ok + response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data) + if response.ok: + return response.json() + bubble_log('bubble_conn_check API call failed: '+repr(response)) + return None + except Exception as e: - bubble_log('bubble_passthru API call failed: '+repr(e)) + bubble_log('bubble_conn_check API call failed: '+repr(e)) traceback.print_exc() + if security_level is not None and security_level == 'maximum': + return False return None diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py new file mode 100644 index 00000000..c661f3c3 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -0,0 +1,227 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here: +# +# Copyright (c) 2013, Aldo Cortesi. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer +from mitmproxy.exceptions import TlsProtocolException +from mitmproxy.net import tls as net_tls + +from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, redis_set +from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6 +import redis +import json +import subprocess + +REDIS_DNS_PREFIX = 'bubble_dns_' +REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' +REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout +REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService + +REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) + +FORCE_PASSTHRU = {'passthru': True} +FORCE_BLOCK = {'block': True} + +local_ips = None + + +def get_device_security_level(client_addr): + level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr) + if level is None: + return 'maximum' + return level.decode() + + +def get_local_ips(): + global local_ips + if local_ips is None: + local_ips = [] + for ip in subprocess.check_output(['hostname', '-I']).split(): + local_ips.append(ip.decode()) + return local_ips + + +def is_sage_request(ip, fqdns): + return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns + + +def conn_check_cache_prefix(client_addr, server_addr): + return REDIS_CONN_CHECK_PREFIX + client_addr + '_' + server_addr + + +def fqdns_for_addr(addr): + prefix = REDIS_DNS_PREFIX + addr + keys = REDIS.keys(prefix + '_*') + if keys is None or len(keys) == 0: + bubble_log('fqdns_for_addr: no FQDN found for addr '+str(addr)+', checking raw addr') + return '' + fqdns = [] + for k in keys: + fqdn = k.decode()[len(prefix)+1:] + fqdns.append(fqdn) + return fqdns + + +class TlsBlock(TlsLayer): + """ + Monkey-patch __call__ to drop this connection entirely + """ + def __call__(self): + bubble_log('TlsBlock: blocking') + return + + +class TlsFeedback(TlsLayer): + """ + Monkey-patch _establish_tls_with_client to get feedback if TLS could be established + successfully on the client connection (which may fail due to cert pinning). + """ + def _establish_tls_with_client(self): + client_address = self.client_conn.address[0] + server_address = self.server_conn.address[0] + security_level = get_device_security_level(client_address) + try: + super(TlsFeedback, self)._establish_tls_with_client() + + except TlsProtocolException as e: + if self.fqdns is not None and len(self.fqdns) > 0: + for fqdn in self.fqdns: + cache_key = conn_check_cache_prefix(client_address, fqdn) + if security_level == 'maximum': + redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn) + else: + redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn) + else: + cache_key = conn_check_cache_prefix(client_address, server_address) + if security_level == 'maximum': + redis_set(cache_key, json.dumps({'fqdn': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address) + else: + redis_set(cache_key, json.dumps({'fqdn': None, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address) + raise e + + +def check_bubble_connection(client_addr, server_addr, fqdns, security_level): + check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level) + if check_response is None or check_response == 'error': + if security_level == 'maximum': + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'} + else: + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_error'} + + elif check_response == 'passthru': + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_passthru'} + + elif check_response == 'block': + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_block'} + + else: + bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False') + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'reason': 'bubble_no_passthru'} + + +def check_connection(client_addr, server_addr, fqdns, security_level): + if fqdns and len(fqdns) == 1: + cache_key = conn_check_cache_prefix(client_addr, fqdns[0]) + else: + cache_key = conn_check_cache_prefix(client_addr, server_addr) + prefix = 'check_connection: ip=' + str(server_addr) + ' (fqdns=' + str(fqdns) + ') cache_key=' + cache_key + ': ' + + check_json = REDIS.get(cache_key) + if check_json is None or len(check_json) == 0: + bubble_log(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns)) + check_response = check_bubble_connection(client_addr, server_addr, fqdns, security_level) + bubble_log(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...") + redis_set(cache_key, json.dumps(check_response), ex=REDIS_CHECK_DURATION) + + else: + bubble_log(prefix+'found check_json='+str(check_json)+', touching key in redis') + check_response = json.loads(check_json) + REDIS.touch(cache_key) + bubble_log(prefix+'returning '+str(check_response)) + return check_response + + +def next_layer(next_layer): + if isinstance(next_layer, TlsLayer) and next_layer._client_tls: + client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile) + client_addr = next_layer.client_conn.address[0] + server_addr = next_layer.server_conn.address[0] + + if client_hello.sni: + fqdn = client_hello.sni.decode() + bubble_log('next_layer: using fqdn in SNI: '+ fqdn) + fqdns = [ fqdn ] + else: + fqdns = fqdns_for_addr(server_addr) + bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns)) + next_layer.fqdns = fqdns + no_fqdns = fqdns is None or len(fqdns) == 0 + security_level = get_device_security_level(client_addr) + if server_addr in get_local_ips(): + bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif is_sage_request(server_addr, fqdns): + bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif security_level == 'disabled' or security_level == 'basic': + bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif security_level == 'standard' and no_fqdns: + bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) + check = FORCE_PASSTHRU + + elif security_level == 'maximum' and no_fqdns: + bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) + check = FORCE_BLOCK + + else: + bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+security_level) + check = check_connection(client_addr, server_addr, fqdns, security_level) + + if check is None or ('passthru' in check and check['passthru']): + bubble_log('next_layer: enabling passthru for server_addr' + server_addr+', fqdns='+str(fqdns)) + bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) + next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) + next_layer.reply.send(next_layer_replacement) + + elif 'block' in check and check['block']: + bubble_log('next_layer: enabling block for server_addr' + server_addr+', fqdns='+str(fqdns)) + bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns) + next_layer.__class__ = TlsBlock + + else: + bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) + bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns) + next_layer.__class__ = TlsFeedback diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py deleted file mode 100644 index 1bf8a176..00000000 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py +++ /dev/null @@ -1,171 +0,0 @@ -# -# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ -# -# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here: -# -# Copyright (c) 2013, Aldo Cortesi. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer -from mitmproxy.exceptions import TlsProtocolException - -from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set -from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6 -import redis -import json -import subprocess - -REDIS_DNS_PREFIX = 'bubble_dns_' -REDIS_PASSTHRU_PREFIX = 'bubble_passthru_' -REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService -REDIS_PASSTHRU_DURATION = 60 * 60 # 1 hour timeout on passthru - -REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) - -FORCE_PASSTHRU = {'passthru': True} - -local_ips = None - - -def get_device_security_level(client_addr): - level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr) - if level is None: - return 'maximum' - return level.decode() - - -def get_local_ips(): - global local_ips - if local_ips is None: - local_ips = [] - for ip in subprocess.check_output(['hostname', '-I']).split(): - local_ips.append(ip.decode()) - return local_ips - - -def is_sage_request(ip, fqdns): - return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns - - -def passthru_cache_prefix(client_addr, server_addr): - return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr - - -def fqdns_for_addr(addr): - prefix = REDIS_DNS_PREFIX + addr - keys = REDIS.keys(prefix + '_*') - if keys is None or len(keys) == 0: - bubble_log('fqdns_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr') - return '' - fqdns = [] - for k in keys: - fqdn = k.decode()[len(prefix)+1:] - fqdns.append(fqdn) - return fqdns - - -class TlsFeedback(TlsLayer): - """ - Monkey-patch _establish_tls_with_client to get feedback if TLS could be established - successfully on the client connection (which may fail due to cert pinning). - """ - def _establish_tls_with_client(self): - client_address = self.client_conn.address[0] - server_address = self.server_conn.address[0] - fqdns = fqdns_for_addr(server_address) - try: - super(TlsFeedback, self)._establish_tls_with_client() - - except TlsProtocolException as e: - cache_key = passthru_cache_prefix(client_address, server_address) - bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key) - redis_set(cache_key, json.dumps({'fqdns': fqdns, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) - raise e - - -def check_bubble_passthru(client_addr, addr, fqdns): - passthru = bubble_passthru(client_addr, addr, fqdns) - if passthru is None or passthru: - bubble_log('check_bubble_passthru: bubble_passthru returned '+repr(passthru)+' for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning True') - return {'fqdns': fqdns, 'addr': addr, 'passthru': True} - bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning False') - return {'fqdns': fqdns, 'addr': addr, 'passthru': False} - - -def should_passthru(client_addr, addr, fqdns): - cache_key = passthru_cache_prefix(client_addr, addr) - prefix = 'should_passthru: ip='+repr(addr)+' (fqdns='+repr(fqdns)+') cache_key='+cache_key+': ' - - passthru_json = REDIS.get(cache_key) - if passthru_json is None or len(passthru_json) == 0: - bubble_log(prefix+'not in redis or empty, calling check_bubble_passthru against fqdns='+repr(fqdns)) - passthru = check_bubble_passthru(client_addr, addr, fqdns) - bubble_log(prefix+'check_bubble_passthru('+repr(fqdns)+') returned '+repr(passthru)+", storing in redis...") - redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) - - else: - bubble_log(prefix+'found passthru_json='+str(passthru_json)+', touching key in redis') - passthru = json.loads(passthru_json) - REDIS.touch(cache_key) - bubble_log(prefix+'returning '+repr(passthru)) - return passthru - - -def next_layer(next_layer): - if isinstance(next_layer, TlsLayer) and next_layer._client_tls: - client_addr = next_layer.client_conn.address[0] - server_addr = next_layer.server_conn.address[0] - - fqdns = fqdns_for_addr(server_addr) - no_fqdns = fqdns is None or len(fqdns) == 0 - security_level = get_device_security_level(client_addr) - if server_addr in get_local_ips(): - bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif is_sage_request(server_addr, fqdns): - bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif security_level == 'disabled' or security_level == 'basic': - bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif security_level == 'standard' and no_fqdns: - bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) - passthru = FORCE_PASSTHRU - - elif security_level == 'maximum' and no_fqdns: - bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) - return - - else: - bubble_log('next_layer: checking should_passthru for server='+server_addr+', client='+client_addr+' with security_level='+security_level) - passthru = should_passthru(client_addr, server_addr, fqdns) - - if passthru is None or passthru['passthru']: - bubble_log('next_layer: enabling passthru for ' + repr(next_layer.server_conn.address)) - bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) - next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) - next_layer.reply.send(next_layer_replacement) - else: - bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr) - bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns) - next_layer.__class__ = TlsFeedback diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh index 4ab69597..0f4bc1e6 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh @@ -16,6 +16,6 @@ mitmdump \ --set stream_large_bodies=5m \ --set keep_host_header \ -s ./dns_spoofing.py \ - -s ./bubble_passthru.py \ + -s ./bubble_conn_check.py \ -s ./bubble_modify.py \ --mode transparent diff --git a/utils/abp-parser b/utils/abp-parser index df343fc4..e3d05733 160000 --- a/utils/abp-parser +++ b/utils/abp-parser @@ -1 +1 @@ -Subproject commit df343fc4b3e1f123b3caa025a493861edf46b81a +Subproject commit e3d05733525eb15a7106cbca0d585a3104681925 diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index e5d7abc4..cbd62f42 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit e5d7abc4b58a339a5da90fcfe53ba21c20e40c75 +Subproject commit cbd62f426d8544cc625dfb0ac4e5b6ec6a049b8f diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index d450510d..9aa42648 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit d450510d6f1be5b328b919ae73d8c1450f008d91 +Subproject commit 9aa42648fcc5694dfcd1778fa7dc4cb0f25455e8