add support for flex exclude domains

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -44,7 +44,8 @@ public interface AppRuleDriver {
     String REDIS_BLOCK_LISTS = "blockLists";
     String REDIS_WHITE_LISTS = "whiteLists";
     String REDIS_FILTER_LISTS = "filterLists";
-    String REDIS_FLEX_LISTS = "flexLists";  // used in mitmproxy for flex routing
+    String REDIS_FLEX_LISTS = "flexLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
+    String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
     String REDIS_LIST_SUFFIX = "~UNION";
 
     default Set<String> getPrimedRejectDomains () { return null; }
@@ -52,6 +53,7 @@ public interface AppRuleDriver {
     default Set<String> getPrimedWhiteListDomains() { return null; }
     default Set<String> getPrimedFilterDomains () { return null; }
     default Set<String> getPrimedFlexDomains () { return null; }
+    default Set<String> getPrimedFlexExcludeDomains () { return null; }
 
     static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) {
         defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains);
@@ -73,6 +75,10 @@ public interface AppRuleDriver {
         defineRedisSet(redis, ip, REDIS_FLEX_LISTS, list, flexDomains);
     }
 
+    static void defineRedisFlexExcludeSet(RedisService redis, String ip, String list, String[] flexExcludeDomains) {
+        defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains);
+    }
+
     static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) {
         final String listOfListsForIp = listOfListsName + "~" + ip;
         final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX;
@@ -94,10 +100,14 @@ public interface AppRuleDriver {
 
     static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) {
         final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX;
+        final String excludeKey = REDIS_FLEX_EXCLUDE_LISTS + "~" + ip + REDIS_LIST_SUFFIX;
         String check = fqdn;
         while (true) {
             final boolean found = redis.sismember_plaintext(key, check);
-            if (found) return true;
+            if (found) {
+                final boolean excluded = redis.sismember_plaintext(excludeKey, check);
+                if (!excluded) return true;
+            }
             final int dotPos = check.indexOf('.');
             if (dotPos == check.length()) return false;
             check = check.substring(dotPos+1);

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java

@@ -17,6 +17,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ArrayUtil;
 
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.cobbzilla.util.json.JsonUtil.json;
 
@@ -27,7 +28,17 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
 
     @Override public Set<String> getPrimedFlexDomains() {
         final TlsPassthruConfig passthruConfig = getRuleConfig();
-        return passthruConfig.getFlexDomains();
+        return passthruConfig.getFlexDomains().stream()
+                .filter(d -> !d.startsWith("!"))
+                .collect(Collectors.toSet());
+    }
+
+    @Override public Set<String> getPrimedFlexExcludeDomains() {
+        final TlsPassthruConfig passthruConfig = getRuleConfig();
+        return passthruConfig.getFlexDomains().stream()
+                .filter(d -> d.startsWith("!"))
+                .map(d -> d.substring(1))
+                .collect(Collectors.toSet());
     }
 
     @Override public void init(JsonNode config,

bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java

@@ -147,6 +147,7 @@ public class StandardAppPrimerService implements AppPrimerService {
                         final Set<String> whiteListDomains = new HashSet<>();
                         final Set<String> filterDomains = new HashSet<>();
                         final Set<String> flexDomains = new HashSet<>();
+                        final Set<String> flexExcludeDomains = new HashSet<>();
                         for (AppMatcher matcher : matchers) {
                             final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device);
                             final Set<String> rejects = appRuleDriver.getPrimedRejectDomains();
@@ -179,8 +180,14 @@ public class StandardAppPrimerService implements AppPrimerService {
                             } else {
                                 flexDomains.addAll(flexes);
                             }
+                            final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
+                            if (empty(flexExcludes)) {
+                                log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                            } else {
+                                flexExcludeDomains.addAll(flexExcludes);
+                            }
                         }
-                        if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains)) {
+                        if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) {
                             for (String ip : accountDeviceIps.get(device.getUuid())) {
                                 if (!empty(rejectDomains)) {
                                     AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new));
@@ -197,6 +204,9 @@ public class StandardAppPrimerService implements AppPrimerService {
                                 if (!empty(flexDomains)) {
                                     AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new));
                                 }
+                                if (!empty(flexExcludeDomains)) {
+                                    AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new));
+                                }
                             }
                         }
                     }

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json

@@ -120,7 +120,7 @@
         {"name": "config.view.manageFlexDomains", "value": "Manage Flex Routing Domains"},
         {"name": "config.view.manageFlexFeeds", "value": "Manage Flex Routing Domain Feeds"},
         {"name": "config.field.flexFqdn", "value": "Domain"},
-        {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this hostname"},
+        {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this domain and all subdomains. Prefix with ! to exclude from flex routing."},
         {"name": "config.field.flexFeedName", "value": "Name"},
         {"name": "config.field.flexFeedUrl", "value": "Flex Routing Domains List URL"},
         {"name": "config.field.flexFeedUrl.description", "value": "URL returning a list of domains and/or hostnames to flex route, one per line"},

bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml

@@ -13,7 +13,7 @@
   get_url:
     url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip
     dest: /tmp/algo.zip
-    checksum: sha256:895d28911907d8f7f79cca6b70a6eda6ca4c892553cd02c0fd95060f392970a3
+    checksum: sha256:1be58465d27dd8b40bc8ef9fe33c4c1dbad8dec6abb0b0c68d19754786562add
 
 - name: Unzip algo master.zip
   unarchive: