From 36e06cf31dd4bea5802e3f89703cf828060f4a88 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 16:56:42 -0400 Subject: [PATCH] add support for flex exclude domains --- .../src/main/java/bubble/rule/AppRuleDriver.java | 14 ++++++++++++-- .../rule/passthru/TlsPassthruRuleDriver.java | 13 ++++++++++++- .../service/stream/StandardAppPrimerService.java | 12 +++++++++++- .../models/apps/passthru/bubbleApp_passthru.json | 2 +- .../resources/packer/roles/algo/tasks/main.yml | 2 +- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index e0290c83..3d3e2fb4 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -44,7 +44,8 @@ public interface AppRuleDriver { String REDIS_BLOCK_LISTS = "blockLists"; String REDIS_WHITE_LISTS = "whiteLists"; String REDIS_FILTER_LISTS = "filterLists"; - String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy for flex routing + String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy and dnscrypt-proxy for flex routing + String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists"; // used in mitmproxy and dnscrypt-proxy for flex routing String REDIS_LIST_SUFFIX = "~UNION"; default Set getPrimedRejectDomains () { return null; } @@ -52,6 +53,7 @@ public interface AppRuleDriver { default Set getPrimedWhiteListDomains() { return null; } default Set getPrimedFilterDomains () { return null; } default Set getPrimedFlexDomains () { return null; } + default Set getPrimedFlexExcludeDomains () { return null; } static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) { defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains); @@ -73,6 +75,10 @@ public interface AppRuleDriver { defineRedisSet(redis, ip, REDIS_FLEX_LISTS, list, flexDomains); } + static void defineRedisFlexExcludeSet(RedisService redis, String ip, String list, String[] flexExcludeDomains) { + defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains); + } + static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { final String listOfListsForIp = listOfListsName + "~" + ip; final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX; @@ -94,10 +100,14 @@ public interface AppRuleDriver { static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) { final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX; + final String excludeKey = REDIS_FLEX_EXCLUDE_LISTS + "~" + ip + REDIS_LIST_SUFFIX; String check = fqdn; while (true) { final boolean found = redis.sismember_plaintext(key, check); - if (found) return true; + if (found) { + final boolean excluded = redis.sismember_plaintext(excludeKey, check); + if (!excluded) return true; + } final int dotPos = check.indexOf('.'); if (dotPos == check.length()) return false; check = check.substring(dotPos+1); diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index 56977ccd..ece93a1d 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -17,6 +17,7 @@ import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ArrayUtil; import java.util.Set; +import java.util.stream.Collectors; import static org.cobbzilla.util.json.JsonUtil.json; @@ -27,7 +28,17 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { @Override public Set getPrimedFlexDomains() { final TlsPassthruConfig passthruConfig = getRuleConfig(); - return passthruConfig.getFlexDomains(); + return passthruConfig.getFlexDomains().stream() + .filter(d -> !d.startsWith("!")) + .collect(Collectors.toSet()); + } + + @Override public Set getPrimedFlexExcludeDomains() { + final TlsPassthruConfig passthruConfig = getRuleConfig(); + return passthruConfig.getFlexDomains().stream() + .filter(d -> d.startsWith("!")) + .map(d -> d.substring(1)) + .collect(Collectors.toSet()); } @Override public void init(JsonNode config, diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index 2e5af4e3..a17f3fe7 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -147,6 +147,7 @@ public class StandardAppPrimerService implements AppPrimerService { final Set whiteListDomains = new HashSet<>(); final Set filterDomains = new HashSet<>(); final Set flexDomains = new HashSet<>(); + final Set flexExcludeDomains = new HashSet<>(); for (AppMatcher matcher : matchers) { final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); final Set rejects = appRuleDriver.getPrimedRejectDomains(); @@ -179,8 +180,14 @@ public class StandardAppPrimerService implements AppPrimerService { } else { flexDomains.addAll(flexes); } + final Set flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains(); + if (empty(flexExcludes)) { + log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + flexExcludeDomains.addAll(flexExcludes); + } } - if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains)) { + if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) { for (String ip : accountDeviceIps.get(device.getUuid())) { if (!empty(rejectDomains)) { AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new)); @@ -197,6 +204,9 @@ public class StandardAppPrimerService implements AppPrimerService { if (!empty(flexDomains)) { AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new)); } + if (!empty(flexExcludeDomains)) { + AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new)); + } } } } diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json index c112e5fa..1619d202 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json @@ -120,7 +120,7 @@ {"name": "config.view.manageFlexDomains", "value": "Manage Flex Routing Domains"}, {"name": "config.view.manageFlexFeeds", "value": "Manage Flex Routing Domain Feeds"}, {"name": "config.field.flexFqdn", "value": "Domain"}, - {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this hostname"}, + {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this domain and all subdomains. Prefix with ! to exclude from flex routing."}, {"name": "config.field.flexFeedName", "value": "Name"}, {"name": "config.field.flexFeedUrl", "value": "Flex Routing Domains List URL"}, {"name": "config.field.flexFeedUrl.description", "value": "URL returning a list of domains and/or hostnames to flex route, one per line"}, diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index ac83c14b..d2a0f8a5 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:895d28911907d8f7f79cca6b70a6eda6ca4c892553cd02c0fd95060f392970a3 + checksum: sha256:1be58465d27dd8b40bc8ef9fe33c4c1dbad8dec6abb0b0c68d19754786562add - name: Unzip algo master.zip unarchive: