bin/mitm_pid

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Print PID of currently-active mitmproxy
+# Note: this command only works on a running bubble node
+#
+ps auxwww | grep /home/mitmproxy/mitmproxy/venv/bin/mitmdump | grep $(cat /home/mitmproxy/mitmproxy_port) | grep -v grep | awk '{print $2}'

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -12,7 +12,7 @@ import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.RandomUtils;
 import org.cobbzilla.util.daemon.ZillaRuntime;
-import org.cobbzilla.util.io.FileUtil;
+import org.cobbzilla.util.string.StringUtil;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
 
@@ -36,6 +36,7 @@ import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.network.NetworkUtil.*;
 import static org.cobbzilla.util.string.StringUtil.splitAndTrim;
+import static org.cobbzilla.util.system.CommandShell.execScript;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 
 @Slf4j
@@ -58,7 +59,7 @@ public class ApiConstants {
 
     private static String initDefaultDomain() {
         final File f = new File(HOME_DIR, ".BUBBLE_DEFAULT_DOMAIN");
-        final String domain = FileUtil.toStringOrDie(f);
+        final String domain = toStringOrDie(f);
         return domain != null ? domain.trim() : die("initDefaultDomain: "+abs(f)+" not found");
     }
 
@@ -75,6 +76,27 @@ public class ApiConstants {
 
     public static final GoogleAuthenticator G_AUTH = new GoogleAuthenticator();
 
+    private static final AtomicReference<String> knownHostKey = new AtomicReference<>();
+    public static String getKnownHostKey () {
+        return lazyGet(knownHostKey,
+                () -> execScript("ssh-keyscan -t rsa $(hostname -d) 2>&1 | grep -v \"^#\""),
+                () -> die("getKnownHostKey"));
+    }
+
+    private static final AtomicReference<String> privateIp = new AtomicReference<>();
+    public static String getPrivateIp() {
+        return lazyGet(privateIp,
+                () -> configuredIps().stream()
+                        .filter(addr -> addr.startsWith("10."))
+                        .findFirst()
+                        .orElse(null),
+                () -> {
+                    final String msg = "getPrivateIp: no system private IP found, configuredIps=" + StringUtil.toString(configuredIps());
+                    log.error(msg);
+                    return die(msg);
+                });
+    }
+
     public static final Predicate ALWAYS_TRUE = m -> true;
     public static final String HOME_DIR;
     static {
@@ -176,6 +198,7 @@ public class ApiConstants {
     public static final String EP_NODES = "/nodes";
     public static final String EP_DEVICES = "/devices";
     public static final String EP_DEVICE_TYPES = "/deviceTypes";
+    public static final String EP_FLEX_ROUTERS = "/flexRouters";
     public static final String EP_MODEL = "/model";
     public static final String EP_VPN = "/vpn";
     public static final String EP_IPS = "/ips";
@@ -276,8 +299,7 @@ public class ApiConstants {
     }
 
     public static String getRemoteHost(Request req) {
-        final String xff = req.getHeader("X-Forwarded-For");
-        final String remoteHost = xff == null ? req.getRemoteAddr() : xff;
+        final String remoteHost = getRemoteAddr(req);
         if (isPublicIpv4(remoteHost)) return remoteHost;
         final String publicIp = getFirstPublicIpv4();
         if (publicIp != null) return publicIp;
@@ -285,6 +307,11 @@ public class ApiConstants {
         return isPublicIpv4(externalIp) ? externalIp : remoteHost;
     }
 
+    public static String getRemoteAddr(Request req) {
+        final String xff = req.getHeader("X-Forwarded-For");
+        return xff == null ? req.getRemoteAddr() : xff;
+    }
+
     public static String getUserAgent(ContainerRequest ctx) { return ctx.getHeaderString(USER_AGENT); }
     public static String getReferer(ContainerRequest ctx) { return ctx.getHeaderString(REFERER); }
 

bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java

@@ -9,10 +9,7 @@ import bubble.model.account.Account;
 import bubble.model.app.AppRule;
 import bubble.model.app.BubbleApp;
 import bubble.model.app.config.AppConfigDriverBase;
-import bubble.rule.passthru.TlsPassthruConfig;
-import bubble.rule.passthru.TlsPassthruFeed;
-import bubble.rule.passthru.TlsPassthruFqdn;
-import bubble.rule.passthru.TlsPassthruRuleDriver;
+import bubble.rule.passthru.*;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -30,86 +27,159 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 @Slf4j
 public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
 
-    public static final String VIEW_manageDomains = "manageDomains";
-    public static final String VIEW_manageFeeds = "manageFeeds";
+    public static final String VIEW_managePassthruDomains = "managePassthruDomains";
+    public static final String VIEW_managePassthruFeeds = "managePassthruFeeds";
+    public static final String VIEW_manageFlexDomains = "manageFlexDomains";
+    public static final String VIEW_manageFlexFeeds = "manageFlexFeeds";
 
     @Autowired @Getter private AppRuleDAO ruleDAO;
 
     @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
         switch (view) {
-            case VIEW_manageDomains:
-                return loadManageDomains(account, app);
-            case VIEW_manageFeeds:
-                return loadManageFeeds(account, app);
+            case VIEW_managePassthruDomains:
+                return loadManagePassthruDomains(account, app);
+            case VIEW_managePassthruFeeds:
+                return loadManagePassthuFeeds(account, app);
+            case VIEW_manageFlexDomains:
+                return loadManageFlexDomains(account, app);
+            case VIEW_manageFlexFeeds:
+                return loadManageFlexFeeds(account, app);
         }
         throw notFoundEx(view);
     }
 
-    private Set<TlsPassthruFeed> loadManageFeeds(Account account, BubbleApp app) {
+    private Set<TlsPassthruFeed> loadManagePassthuFeeds(Account account, BubbleApp app) {
         final TlsPassthruConfig config = getConfig(account, app);
         config.getPassthruSet(); // ensure names are initialized
-        return config.getFeedSet();
+        return config.getPassthruFeedSet();
     }
 
-    private Set<TlsPassthruFqdn> loadManageDomains(Account account, BubbleApp app) {
+    private Set<TlsPassthruFqdn> loadManagePassthruDomains(Account account, BubbleApp app) {
         final TlsPassthruConfig config = getConfig(account, app);
-        return !config.hasFqdnList() ? Collections.emptySet() :
-                Arrays.stream(config.getFqdnList())
+        return !config.hasPassthruFqdnList() ? Collections.emptySet() :
+                Arrays.stream(config.getPassthruFqdnList())
                         .map(TlsPassthruFqdn::new)
                         .collect(Collectors.toCollection(TreeSet::new));
     }
 
+    private Set<FlexFeed> loadManageFlexFeeds(Account account, BubbleApp app) {
+        final TlsPassthruConfig config = getConfig(account, app);
+        config.getFlexSet(); // ensure names are initialized
+        return config.getFlexFeedSet();
+    }
+
+    private Set<FlexFqdn> loadManageFlexDomains(Account account, BubbleApp app) {
+        final TlsPassthruConfig config = getConfig(account, app);
+        return !config.hasFlexFqdnList() ? Collections.emptySet() :
+                Arrays.stream(config.getFlexFqdnList())
+                        .map(FlexFqdn::new)
+                        .collect(Collectors.toCollection(TreeSet::new));
+    }
+
     private TlsPassthruConfig getConfig(Account account, BubbleApp app) {
         return getConfig(account, app, TlsPassthruRuleDriver.class, TlsPassthruConfig.class);
     }
 
-    public static final String ACTION_addFqdn = "addFqdn";
-    public static final String ACTION_removeFqdn = "removeFqdn";
-    public static final String ACTION_addFeed = "addFeed";
-    public static final String ACTION_removeFeed = "removeFeed";
+    public static final String ACTION_addPassthruFqdn = "addPassthruFqdn";
+    public static final String ACTION_addPassthruFeed = "addPassthruFeed";
+    public static final String ACTION_removePassthruFqdn = "removePassthruFqdn";
+    public static final String ACTION_removePassthruFeed = "removePassthruFeed";
+
+    public static final String ACTION_addFlexFqdn = "addFlexFqdn";
+    public static final String ACTION_addFlexFeed = "addFlexFeed";
+    public static final String ACTION_removeFlexFqdn = "removeFlexFqdn";
+    public static final String ACTION_removeFlexFeed = "removeFlexFeed";
 
-    public static final String PARAM_FQDN = "passthruFqdn";
-    public static final String PARAM_FEED_URL = "feedUrl";
+    public static final String PARAM_PASSTHRU_FQDN = "passthruFqdn";
+    public static final String PARAM_PASSTHRU_FEED_URL = "passthruFeedUrl";
+    public static final String PARAM_FLEX_FQDN = "flexFqdn";
+    public static final String PARAM_FLEX_FEED_URL = "flexFeedUrl";
 
     @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action, Map<String, String> params, JsonNode data) {
         switch (action) {
-            case ACTION_addFqdn:
-                return addFqdn(account, app, data);
-            case ACTION_addFeed:
-                return addFeed(account, app, params, data);
+            case ACTION_addPassthruFqdn:
+                return addPassthruFqdn(account, app, data);
+            case ACTION_addPassthruFeed:
+                return addPassthruFeed(account, app, params, data);
+            case ACTION_addFlexFqdn:
+                return addFlexFqdn(account, app, data);
+            case ACTION_addFlexFeed:
+                return addFlexFeed(account, app, params, data);
         }
-        log.debug("takeAppAction: action not found: "+action);
+        if (log.isWarnEnabled()) log.warn("takeAppAction: action not found: "+action);
         throw notFoundEx(action);
     }
 
-    private List<TlsPassthruFqdn> addFqdn(Account account, BubbleApp app, JsonNode data) {
-        final JsonNode fqdnNode = data.get(PARAM_FQDN);
+    private List<TlsPassthruFqdn> addPassthruFqdn(Account account, BubbleApp app, JsonNode data) {
+        final JsonNode fqdnNode = data.get(PARAM_PASSTHRU_FQDN);
         if (fqdnNode == null || fqdnNode.textValue() == null || empty(fqdnNode.textValue().trim())) {
-            throw invalidEx("err.addFqdn.passthruFqdnRequired");
+            throw invalidEx("err.passthruFqdn.passthruFqdnRequired");
         }
 
         final String fqdn = fqdnNode.textValue().trim().toLowerCase();
+        final TlsPassthruConfig config = getConfig(account, app).addPassthruFqdn(fqdn);
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        if (log.isDebugEnabled()) log.debug("addPassthruFqdn: updating rule: "+rule.getName()+", adding fqdn: "+fqdn);
+        ruleDAO.update(rule.setConfigJson(json(config)));
+
+        return getPassthruFqdnList(config);
+    }
+
+    private List<TlsPassthruFqdn> getPassthruFqdnList(TlsPassthruConfig config) {
+        return Arrays.stream(config.getPassthruFqdnList())
+                .map(TlsPassthruFqdn::new)
+                .collect(Collectors.toList());
+    }
+
+    private Set<TlsPassthruFeed> addPassthruFeed(Account account, BubbleApp app, Map<String, String> params, JsonNode data) {
+        final JsonNode urlNode = data.get(PARAM_PASSTHRU_FEED_URL);
+        if (urlNode == null || urlNode.textValue() == null || empty(urlNode.textValue().trim())) {
+            throw invalidEx("err.passthruFeedUrl.feedUrlRequired");
+        }
+
+        final String url = urlNode.textValue().trim().toLowerCase();
+        final TlsPassthruConfig config = getConfig(account, app);
 
-        final TlsPassthruConfig config = getConfig(account, app)
-                .addFqdn(fqdn);
+        final TlsPassthruFeed feed = config.loadFeed(url);
+        if (!feed.hasFqdnList()) throw invalidEx("err.passthruFeedUrl.emptyFqdnList");
+        config.addPassthruFeed(feed);
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        ruleDAO.update(rule.setConfigJson(json(config)));
+
+        return config.getPassthruFeedSet();
+    }
+
+    private List<TlsPassthruFqdn> addFlexFqdn(Account account, BubbleApp app, JsonNode data) {
+        final JsonNode fqdnNode = data.get(PARAM_FLEX_FQDN);
+        if (fqdnNode == null || fqdnNode.textValue() == null || empty(fqdnNode.textValue().trim())) {
+            throw invalidEx("err.flexFqdn.flexFqdnRequired");
+        }
+
+        final String fqdn = fqdnNode.textValue().trim().toLowerCase();
+        final TlsPassthruConfig config = getConfig(account, app).addFlexFqdn(fqdn);
 
         final AppRule rule = loadRule(account, app);
         loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        if (log.isDebugEnabled()) log.debug("addFlexFqdn: updating rule: "+rule.getName()+", adding fqdn: "+fqdn);
         ruleDAO.update(rule.setConfigJson(json(config)));
 
-        return getFqdnList(config);
+        return getFlexFqdnList(config);
     }
 
-    private List<TlsPassthruFqdn> getFqdnList(TlsPassthruConfig config) {
-        return Arrays.stream(config.getFqdnList())
+    private List<TlsPassthruFqdn> getFlexFqdnList(TlsPassthruConfig config) {
+        return Arrays.stream(config.getFlexFqdnList())
                 .map(TlsPassthruFqdn::new)
                 .collect(Collectors.toList());
     }
 
-    private Set<TlsPassthruFeed> addFeed(Account account, BubbleApp app, Map<String, String> params, JsonNode data) {
-        final JsonNode urlNode = data.get(PARAM_FEED_URL);
+    private Set<TlsPassthruFeed> addFlexFeed(Account account, BubbleApp app, Map<String, String> params, JsonNode data) {
+        final JsonNode urlNode = data.get(PARAM_FLEX_FEED_URL);
         if (urlNode == null || urlNode.textValue() == null || empty(urlNode.textValue().trim())) {
-            throw invalidEx("err.addFeed.feedUrlRequired");
+            throw invalidEx("err.flexFeedUrl.feedUrlRequired");
         }
 
         final String url = urlNode.textValue().trim().toLowerCase();
@@ -117,45 +187,69 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
         final TlsPassthruConfig config = getConfig(account, app);
 
         final TlsPassthruFeed feed = config.loadFeed(url);
-        if (!feed.hasFqdnList()) throw invalidEx("err.addFeed.emptyFqdnList");
-        config.addFeed(feed);
+        if (!feed.hasFqdnList()) throw invalidEx("err.flexFeedUrl.emptyFqdnList");
+        config.addPassthruFeed(feed);
 
         final AppRule rule = loadRule(account, app);
         loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
         ruleDAO.update(rule.setConfigJson(json(config)));
 
-        return config.getFeedSet();
+        return config.getPassthruFeedSet();
     }
 
     @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id, Map<String, String> params, JsonNode data) {
         switch (action) {
-            case ACTION_removeFqdn:
-                return removeFqdn(account, app, id);
-            case ACTION_removeFeed:
-                return removeFeed(account, app, id);
+            case ACTION_removePassthruFqdn:
+                return removePassthruFqdn(account, app, id);
+            case ACTION_removePassthruFeed:
+                return removePassthruFeed(account, app, id);
+            case ACTION_removeFlexFqdn:
+                return removeFlexFqdn(account, app, id);
+            case ACTION_removeFlexFeed:
+                return removeFlexFeed(account, app, id);
         }
-        log.debug("takeItemAction: action not found: "+action);
+        if (log.isWarnEnabled()) log.warn("takeItemAction: action not found: "+action);
         throw notFoundEx(action);
     }
 
-    private List<TlsPassthruFqdn> removeFqdn(Account account, BubbleApp app, String id) {
+    private List<TlsPassthruFqdn> removePassthruFqdn(Account account, BubbleApp app, String id) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        final TlsPassthruConfig config = getConfig(account, app);
+        if (log.isDebugEnabled()) log.debug("removePassthruFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getPassthruFqdnList()));
+
+        final TlsPassthruConfig updated = config.removePassthruFqdn(id);
+        if (log.isDebugEnabled()) log.debug("removePassthruFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getPassthruFqdnList()));
+        ruleDAO.update(rule.setConfigJson(json(updated)));
+        return getPassthruFqdnList(updated);
+    }
+
+    public Set<TlsPassthruFeed> removePassthruFeed(Account account, BubbleApp app, String id) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        final TlsPassthruConfig config = getConfig(account, app).removePassthruFeed(id);
+        ruleDAO.update(rule.setConfigJson(json(config)));
+        return config.getPassthruFeedSet();
+    }
+
+    private List<TlsPassthruFqdn> removeFlexFqdn(Account account, BubbleApp app, String id) {
         final AppRule rule = loadRule(account, app);
         loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
         final TlsPassthruConfig config = getConfig(account, app);
-        log.debug("removeFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getFqdnList()));
+        if (log.isDebugEnabled()) log.debug("removeFlexFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getPassthruFqdnList()));
 
-        final TlsPassthruConfig updated = config.removeFqdn(id);
-        log.debug("removeFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getFqdnList()));
+        final TlsPassthruConfig updated = config.removeFlexFqdn(id);
+        if (log.isDebugEnabled()) log.debug("removeFlexFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getPassthruFqdnList()));
         ruleDAO.update(rule.setConfigJson(json(updated)));
-        return getFqdnList(updated);
+        return getFlexFqdnList(updated);
     }
 
-    public Set<TlsPassthruFeed> removeFeed(Account account, BubbleApp app, String id) {
+    public Set<TlsPassthruFeed> removeFlexFeed(Account account, BubbleApp app, String id) {
         final AppRule rule = loadRule(account, app);
         loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
-        final TlsPassthruConfig config = getConfig(account, app).removeFeed(id);
+        final TlsPassthruConfig config = getConfig(account, app).removeFlexFeed(id);
         ruleDAO.update(rule.setConfigJson(json(config)));
-        return config.getFeedSet();
+        return config.getPassthruFeedSet();
     }
 
 }

bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java

@@ -95,7 +95,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase {
     @Getter(lazy=true) private final RedisService imageCache = redis.prefixNamespace(getClass().getSimpleName()+".ec2_ubuntu_image");
     public static final long IMAGE_CACHE_TIME = DAYS.toSeconds(30);
 
-    @Getter(lazy=true) private final ExecutorService perRegionExecutor = fixedPool(getRegions().size());
+    @Getter(lazy=true) private final ExecutorService perRegionExecutor = fixedPool(getRegions().size(), "AmazonEC2Driver.perRegionExecutor");
 
     @Getter(lazy=true) private final List<OsImage> cloudOsImages = initImages();
     private List<OsImage> initImages() {

bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java

@@ -9,6 +9,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import lombok.ToString;
 import lombok.experimental.Accessors;
 import org.cobbzilla.util.math.Haversine;
 
@@ -16,7 +17,7 @@ import javax.persistence.Transient;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 
-@NoArgsConstructor @Accessors(chain=true)
+@NoArgsConstructor @Accessors(chain=true) @ToString(of={"lat", "lon"})
 public class GeoLocation {
 
     @Getter @Setter private String country;

bubble-server/src/main/java/bubble/dao/account/AccountDAO.java

@@ -23,7 +23,7 @@ import bubble.server.BubbleConfiguration;
 import bubble.service.SearchService;
 import bubble.service.account.SyncAccountService;
 import bubble.service.boot.SelfNodeService;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import bubble.service.stream.RuleEngineService;
 import lombok.Getter;
 import lombok.NonNull;
@@ -80,7 +80,7 @@ public class AccountDAO extends AbstractCRUDDAO<Account> implements SqlViewSearc
     @Autowired private SearchService searchService;
     @Autowired private SyncAccountService syncAccountService;
     @Autowired private ReferralCodeDAO referralCodeDAO;
-    @Autowired private DeviceIdService deviceService;
+    @Autowired private DeviceService deviceService;
     @Autowired private RuleEngineService ruleEngineService;
 
     public Account newAccount(Request req, Account caller, AccountRegistration request, Account parent) {
@@ -198,7 +198,7 @@ public class AccountDAO extends AbstractCRUDDAO<Account> implements SqlViewSearc
                 syncAccountService.syncAccount(account);
             }
             if (previousState.isRefreshShowBlockStats()) {
-                deviceService.initBlockStats(account);
+                deviceService.initBlocksAndFlexRoutes(account);
                 ruleEngineService.flushCaches();
             }
         }

bubble-server/src/main/java/bubble/dao/app/AppRuleDAO.java

@@ -5,20 +5,36 @@
 package bubble.dao.app;
 
 import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
+import bubble.service.stream.AppPrimerService;
 import bubble.service.stream.RuleEngineService;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
-@Repository public class AppRuleDAO extends AppTemplateEntityDAO<AppRule> {
+@Repository @Slf4j
+public class AppRuleDAO extends AppTemplateEntityDAO<AppRule> {
 
     @Autowired private RuleEngineService ruleEngineService;
+    @Autowired private BubbleAppDAO appDAO;
+    @Autowired private AppPrimerService appPrimerService;
 
-    @Override public AppRule postUpdate(AppRule entity, Object context) {
+    @Override public Object preUpdate(AppRule rule) {
+        final AppRule existing = findByUuid(rule.getUuid());
+        rule.setPreviousConfigJson(existing.getConfigJson());
+        return super.preUpdate(rule);
+    }
+
+    @Override public AppRule postUpdate(AppRule rule, Object context) {
 
-        ruleEngineService.flushCaches();
+        if (rule.configJsonChanged()) {
+            final BubbleApp app = appDAO.findByUuid(rule.getApp());
+            appPrimerService.prime(app);
+        }
+        ruleEngineService.flushCaches(false);
 
         // todo: update entities based on this template if account has updates enabled
-        return super.postUpdate(entity, context);
+        return super.postUpdate(rule, context);
     }
 
     @Override public void delete(String uuid) {

bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java

@@ -5,7 +5,7 @@
 package bubble.dao.app;
 
 import bubble.model.app.AppSite;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import bubble.service.stream.RuleEngineService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
@@ -14,7 +14,7 @@ import org.springframework.stereotype.Repository;
 public class AppSiteDAO extends AppTemplateEntityDAO<AppSite> {
 
     @Autowired private RuleEngineService ruleEngineService;
-    @Autowired private DeviceIdService deviceService;
+    @Autowired private DeviceService deviceService;
 
     @Override public AppSite postCreate(AppSite site, Object context) {
         // todo: update entities based on this template if account has updates enabled

bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java

@@ -164,7 +164,7 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO<AccountPlan> {
             background(() -> {
                 sleep(PURCHASE_DELAY, "AccountPlanDAO.postCreate: waiting to finalize purchase");
                 paymentDriver.purchase(accountPlanUuid, paymentMethodUuid, billUuid);
-            });
+            }, "AccountPlanDAO.postCreate");
         }
         return super.postCreate(accountPlan, context);
     }

bubble-server/src/main/java/bubble/dao/device/DeviceDAO.java

@@ -11,7 +11,7 @@ import bubble.dao.app.AppDataDAO;
 import bubble.model.device.BubbleDeviceType;
 import bubble.model.device.Device;
 import bubble.server.BubbleConfiguration;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.hibernate.criterion.Order;
@@ -46,7 +46,7 @@ public class DeviceDAO extends AccountOwnedEntityDAO<Device> {
     @Autowired private BubbleConfiguration configuration;
     @Autowired private AppDataDAO dataDAO;
     @Autowired private TrustedClientDAO trustDAO;
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
 
     @Override public Order getDefaultSortOrder() { return ORDER_CTIME_ASC; }
 
@@ -113,7 +113,7 @@ public class DeviceDAO extends AccountOwnedEntityDAO<Device> {
                 result = super.update(uninitialized);
             }
 
-            deviceIdService.setDeviceSecurityLevel(result);
+            deviceService.setDeviceSecurityLevel(result);
             return result;
         }
     }
@@ -125,7 +125,7 @@ public class DeviceDAO extends AccountOwnedEntityDAO<Device> {
 
         toUpdate.update(updateRequest);
         final var updated = super.update(toUpdate);
-        deviceIdService.setDeviceSecurityLevel(updated);
+        deviceService.setDeviceSecurityLevel(updated);
         refreshVpnUsers();
         return updated;
     }

bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java

@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.dao.device;
+
+import bubble.dao.account.AccountOwnedEntityDAO;
+import bubble.model.device.FlexRouter;
+import bubble.service.device.FlexRouterService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+import static bubble.ApiConstants.getKnownHostKey;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
+import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
+import static org.hibernate.criterion.Restrictions.*;
+
+@Repository @Slf4j
+public class FlexRouterDAO extends AccountOwnedEntityDAO<FlexRouter> {
+
+    @Autowired private FlexRouterService flexRouterService;
+
+    @Override protected String getNameField() { return "ip"; }
+
+    @Override public Object preCreate(FlexRouter router) {
+        return super.preCreate(router.setInitialized(false));
+    }
+
+    @Override public FlexRouter postCreate(FlexRouter router, Object context) {
+        flexRouterService.register(router);
+        router.setHost_key(getKnownHostKey());
+        return super.postCreate(router, context);
+    }
+
+    @Override public Object preUpdate(FlexRouter router) {
+        final FlexRouter existing = findByUuid(router.getUuid());
+        if (!existing.getIp().equals(router.getIp())) throw invalidEx("err.ip.cannotChange");
+        if (!existing.getPort().equals(router.getPort())) throw invalidEx("err.port.cannotSetOrChange");
+        if (!existing.getKeyHash().equals(router.getKeyHash())) throw invalidEx("err.sshPublicKey.cannotChange");
+        return super.preUpdate(router);
+    }
+
+    @Override public FlexRouter postUpdate(FlexRouter router, Object context) {
+        flexRouterService.register(router);
+        router.setHost_key(getKnownHostKey());
+        return super.postUpdate(router, context);
+    }
+
+    @Override public void delete(String uuid) {
+        final FlexRouter router = findByUuid(uuid);
+        if (router == null) return;
+        flexRouterService.unregister(router);
+        super.delete(uuid);
+    }
+
+    public List<FlexRouter> findEnabledAndRegistered() {
+        return list(criteria().add(and(
+                eq("enabled", true),
+                gt("port", 1024),
+                le("port", 65535),
+                isNotNull("token"))));
+    }
+
+    public List<FlexRouter> findActive(long maxAge) {
+        return list(criteria().add(and(
+                eq("active", true),
+                eq("initialized", true),
+                eq("enabled", true),
+                gt("port", 1024),
+                le("port", 65535),
+                ge("lastSeen", now()-maxAge),
+                isNotNull("token"))));
+    }
+
+    public FlexRouter findByPort(int port) { return findByUniqueField("port", port); }
+
+    public FlexRouter findByKeyHash(String keyHash) { return findByUniqueField("keyHash", keyHash); }
+
+}

bubble-server/src/main/java/bubble/main/RekeyDatabaseMain.java

@@ -34,7 +34,7 @@ public class RekeyDatabaseMain extends BaseMain<RekeyDatabaseOptions> {
             } catch (Exception e) {
                 die("READ ERROR: " + e);
             }
-        });
+        }, "RekeyDatabaseMain.run.reader");
 
         final AtomicReference<CommandResult> writeResult = new AtomicReference<>();
         final Thread writer = runWriter(options, writeResult, options.getEnv());
@@ -58,7 +58,7 @@ public class RekeyDatabaseMain extends BaseMain<RekeyDatabaseOptions> {
             } catch (Exception e) {
                 writeResult.set(new CommandResult(e).setExitStatus(-1));
             }
-        }, e -> writeResult.set(new CommandResult(e).setExitStatus(-1)));
+        }, "RekeyDatabaseMain.runWriter", e -> writeResult.set(new CommandResult(e).setExitStatus(-1)));
     }
 
     public static Command readerCommand(RekeyDatabaseOptions options, Map<String, String> env) {

bubble-server/src/main/java/bubble/model/account/AccountSshKey.java

@@ -57,8 +57,8 @@ public class AccountSshKey extends IdentifiableBase implements HasAccount {
     @Type(type=ENCRYPTED_STRING)  @Column(updatable=false, columnDefinition="varchar("+(10000+ENC_PAD)+") NOT NULL")
     @Getter private String sshPublicKey;
     public AccountSshKey setSshPublicKey(String k) {
-        this.sshPublicKey = k;
-        if (!empty(k)) this.sshPublicKeyHash = sha256_hex(k);
+        this.sshPublicKey = k.trim();
+        if (!empty(sshPublicKey)) this.sshPublicKeyHash = sha256_hex(sshPublicKey);
         return this;
     }
     public boolean hasSshPublicKey () { return !empty(sshPublicKey); }

bubble-server/src/main/java/bubble/model/app/AppRule.java

@@ -107,6 +107,11 @@ public class AppRule extends IdentifiableBaseParentEntity implements AppTemplate
     @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(500000+ENC_PAD)+")")
     @JsonIgnore @Getter @Setter private String configJson;
 
+    @JsonIgnore @Transient @Getter @Setter private String previousConfigJson;
+    public boolean configJsonChanged () {
+        return (previousConfigJson == null && configJson != null) || (previousConfigJson != null && !previousConfigJson.equals(configJson));
+    }
+
     public AppRule(AppRule other) {
         copy(this, other, CREATE_FIELDS);
         setUuid(null);

bubble-server/src/main/java/bubble/model/device/DeviceStatus.java

@@ -9,6 +9,7 @@ import bubble.service.cloud.GeoService;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import lombok.ToString;
 import lombok.experimental.Accessors;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.cache.redis.RedisService;
@@ -17,7 +18,7 @@ import static java.util.concurrent.TimeUnit.*;
 import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 
-@NoArgsConstructor @Accessors(chain=true) @Slf4j
+@NoArgsConstructor @Accessors(chain=true) @ToString(of={"ip", "location"}) @Slf4j
 public class DeviceStatus {
 
     public static final DeviceStatus NO_DEVICE_STATUS = new DeviceStatus();
@@ -97,7 +98,8 @@ public class DeviceStatus {
                 } else {
                     for (int i=0; i<parts.length-1; i+=2) {
                         final int count = Integer.parseInt(parts[i]);
-                        final String unit = parts[i+1];
+                        String unit = parts[i+1].trim();
+                        if (unit.endsWith(",")) unit = unit.substring(0, unit.length()-1);
                         switch (unit) {
                             case "hour": case "hours": setLastHandshakeHours(count); break;
                             case "minute": case "minutes": setLastHandshakeMinutes(count); break;

bubble-server/src/main/java/bubble/model/device/FlexRouter.java

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.model.device;
+
+import bubble.model.account.Account;
+import bubble.model.account.HasAccount;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.collection.ArrayUtil;
+import org.cobbzilla.wizard.model.Identifiable;
+import org.cobbzilla.wizard.model.IdentifiableBase;
+import org.cobbzilla.wizard.model.entityconfig.annotations.*;
+import org.cobbzilla.wizard.validation.HasValue;
+import org.hibernate.annotations.Type;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Transient;
+
+import static bubble.ApiConstants.EP_FLEX_ROUTERS;
+import static org.cobbzilla.util.daemon.ZillaRuntime.bool;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING;
+import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
+
+@Entity
+@ECType(root=true) @ToString(of={"ip", "port"})
+@ECTypeURIs(baseURI=EP_FLEX_ROUTERS, listFields={"name", "enabled"})
+@NoArgsConstructor @Accessors(chain=true) @Slf4j
+@ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) })
+public class FlexRouter extends IdentifiableBase implements HasAccount {
+
+    public static final String[] UPDATE_FIELDS = { "enabled", "active", "auth_token", "token", "key", "host_key" };
+    public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip", "port");
+
+    public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); }
+
+    @Override public Identifiable update(Identifiable other) { copy(this, other, UPDATE_FIELDS); return this; }
+
+    @ECSearchable @ECField(index=10)
+    @ECForeignKey(entity=Account.class)
+    @Column(nullable=false, updatable=false, length=UUID_MAXLEN)
+    @Getter @Setter private String account;
+
+    @ECSearchable @ECField(index=20)
+    @ECForeignKey(entity=Device.class)
+    @Column(nullable=false, updatable=false, length=UUID_MAXLEN)
+    @Getter @Setter private String device;
+
+    @ECSearchable(filter=true) @ECField(index=30)
+    @ECIndex @Column(nullable=false, updatable=false, length=50)
+    @Getter @Setter private String ip;
+
+    @JsonIgnore @Transient public String getName () { return getIp(); }
+
+    @ECField(index=40) @HasValue(message="err.sshPublicKey.required")
+    @Type(type=ENCRYPTED_STRING)  @Column(updatable=false, columnDefinition="varchar("+(10000+ENC_PAD)+") NOT NULL")
+    @Getter private String key;
+    public boolean hasKey () { return !empty(key); }
+    public FlexRouter setKey(String k) {
+        this.key = k.trim();
+        if (!empty(key)) this.keyHash = sha256_hex(key);
+        return this;
+    }
+
+    @ECField(index=50)
+    @ECIndex(unique=true) @Column(length=100, updatable=false, nullable=false)
+    @Getter @Setter private String keyHash;
+    public boolean hasKeyHash () { return !empty(keyHash); };
+
+    @ECSearchable(filter=true) @ECField(index=60)
+    @ECIndex(unique=true) @Column(nullable=false, updatable=false)
+    @Getter @Setter private Integer port;
+    public boolean hasPort () { return port != null && port > 1024; }
+
+    public String id () { return getIp() + "/" + getUuid(); }
+
+    @ECSearchable @ECField(index=70)
+    @ECIndex @Column(nullable=false)
+    @Getter @Setter private Boolean enabled = true;
+    public boolean enabled () { return bool(enabled); }
+
+    @ECSearchable @ECField(index=80)
+    @ECIndex @Column(nullable=false)
+    @Getter @Setter private Boolean initialized = true;
+    public boolean initialized() { return bool(initialized); }
+    public boolean uninitialized() { return !initialized(); }
+
+    @ECSearchable @ECField(index=90)
+    @ECIndex @Column(nullable=false)
+    @Getter @Setter private Boolean active = true;
+    public boolean active() { return bool(active); }
+    public boolean inactive() { return !active(); }
+
+    @ECSearchable(filter=true) @ECField(index=100)
+    @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(100+ENC_PAD)+") NOT NULL")
+    @JsonIgnore @Getter @Setter private String token;
+    public boolean hasToken () { return !empty(token); }
+
+    // used for sending the token, we never send it back
+    @Transient @Getter @Setter private String auth_token;
+    public boolean hasAuthToken () { return !empty(auth_token); }
+
+    // used for sending the SSH host key to flexrouter
+    @Transient @Getter @Setter private String host_key;
+
+    public FlexRouterPing pingObject() { return new FlexRouterPing(this); }
+    public String proxyBaseUri() { return "http://127.0.0.1:" + getPort(); }
+
+}

bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.model.device;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class FlexRouterPing {
+
+    public static final long MAX_PING_AGE = SECONDS.toMillis(30);
+    public static final long MIN_PING_AGE = -1 * SECONDS.toMillis(5);
+
+    @Getter @Setter private long time;
+    @Getter @Setter private String salt;
+    @Getter @Setter private String hash;
+
+    public FlexRouterPing (FlexRouter router) {
+        time = now();
+        salt = randomAlphanumeric(50);
+        hash = sha256_hex(data(router));
+    }
+
+    public boolean validate(FlexRouter router) {
+        if (empty(salt) || salt.length() < 50) return false;
+        final long age = now() - time;
+        if (age > MAX_PING_AGE || age < MIN_PING_AGE) return false;
+        return sha256_hex(data(router)).equals(hash);
+    }
+
+    private String data(FlexRouter router) { return salt + ":" + time + ":" + router.getToken(); }
+
+}

bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java

@@ -91,16 +91,26 @@ public class AccountOwnedResource<E extends HasAccount, DAO extends AccountOwned
         return getDao().findByAccount(getAccountUuid(ctx));
     }
 
+    protected E find(Request req, ContainerRequest ctx, String id) { return find(ctx, id); }
+
     protected E find(ContainerRequest ctx, String id) {
         return getDao().findByAccountAndId(getAccountUuid(ctx), id);
     }
 
     protected E findAlternate(ContainerRequest ctx, E request) { return null; }
+    protected E findAlternate(Request req, ContainerRequest ctx, E request) { return findAlternate(ctx, request); }
+
     protected E findAlternate(ContainerRequest ctx, String id) { return null; }
+    protected E findAlternate(Request req, ContainerRequest ctx, String id) { return findAlternate(ctx, id); }
 
     protected E findAlternateForCreate(ContainerRequest ctx, E request) { return findAlternate(ctx, request); }
+    protected E findAlternateForCreate(Request req, ContainerRequest ctx, E request) { return findAlternateForCreate(ctx, request); }
+
     protected E findAlternateForUpdate(ContainerRequest ctx, String id) { return findAlternate(ctx, id); }
+    protected E findAlternateForUpdate(Request req, ContainerRequest ctx, String id) { return findAlternateForUpdate(ctx, id); }
+
     protected E findAlternateForDelete(ContainerRequest ctx, String id) { return findAlternate(ctx, id); }
+    protected E findAlternateForDelete(Request req, ContainerRequest ctx, String id) { return findAlternateForDelete(ctx, id); }
 
     protected List<E> populate(ContainerRequest ctx, List<E> entities) {
         for (E e : entities) populate(ctx, e);
@@ -110,14 +120,15 @@ public class AccountOwnedResource<E extends HasAccount, DAO extends AccountOwned
     protected E populate(ContainerRequest ctx, E entity) { return entity; }
 
     @GET @Path("/{id}")
-    public Response view(@Context ContainerRequest ctx,
+    public Response view(@Context Request req,
+                         @Context ContainerRequest ctx,
                          @PathParam("id") String id) {
 
         final Account caller = getAccountForViewById(ctx);
-        E found = find(ctx, id);
+        E found = find(req, ctx, id);
 
         if (found == null) {
-            found = findAlternate(ctx, id);
+            found = findAlternate(req, ctx, id);
             if (found == null) return notFound(id);
         }
         if (caller != null && !found.getAccount().equals(caller.getUuid()) && !caller.admin()) return notFound(id);
@@ -138,15 +149,15 @@ public class AccountOwnedResource<E extends HasAccount, DAO extends AccountOwned
                            E request) {
         if (request == null) return invalid("err.request.invalid");
         final Account caller = checkEditable(ctx);
-        E found = find(ctx, request.getName());
+        E found = find(req, ctx, request.getName());
         if (found == null) {
-            found = findAlternateForCreate(ctx, request);
+            found = findAlternateForCreate(req, ctx, request);
         }
         if (found != null) {
             if (!canUpdate(ctx, caller, found, request)) return ok(found);
-            setReferences(ctx, caller, request);
+            setReferences(ctx, req, caller, request);
             found.update(request);
-            return ok(getDao().update(found));
+            return ok(daoUpdate(found));
         }
 
         if (!canCreate(req, ctx, caller, request)) return invalid("err.cannotCreate", "Create entity not allowed", request.getName());
@@ -156,19 +167,21 @@ public class AccountOwnedResource<E extends HasAccount, DAO extends AccountOwned
     }
 
     protected Object daoCreate(E toCreate) { return getDao().create(toCreate); }
+    protected Object daoUpdate(E toUpdate) { return getDao().update(toUpdate); }
 
     protected E setReferences(ContainerRequest ctx, Account caller, E e) { return e; }
     protected E setReferences(ContainerRequest ctx, Request req, Account caller, E e) { return setReferences(ctx, caller, e); }
 
     @POST @Path("/{id}")
-    public Response update(@Context ContainerRequest ctx,
+    public Response update(@Context Request req,
+                           @Context ContainerRequest ctx,
                            @PathParam("id") String id,
                            E request) {
         if (request == null) return invalid("err.request.invalid");
         final Account caller = checkEditable(ctx);
-        E found = find(ctx, id);
+        E found = find(req, ctx, id);
         if (found == null) {
-            found = findAlternateForUpdate(ctx, id);
+            found = findAlternateForUpdate(req, ctx, id);
             if (found == null) return notFound(id);
         }
         if (!(found instanceof HasAccountNoName) && !canChangeName() && request.hasName() && !request.getName().equals(found.getName())) {
@@ -177,18 +190,19 @@ public class AccountOwnedResource<E extends HasAccount, DAO extends AccountOwned
 
         if (!canUpdate(ctx, caller, found, request)) return invalid("err.cannotUpdate", "Update entity not allowed", request.getName());
         found.update(request);
-        return ok(getDao().update(found.setAccount(getAccountUuid(ctx))));
+        return ok(daoUpdate(found.setAccount(getAccountUuid(ctx))));
     }
 
     protected boolean canChangeName() { return false; }
 
     @DELETE @Path("/{id}")
-    public Response delete(@Context ContainerRequest ctx,
+    public Response delete(@Context Request req,
+                           @Context ContainerRequest ctx,
                            @PathParam("id") String id) {
         final Account caller = checkEditable(ctx);
-        E found = find(ctx, id);
+        E found = find(req, ctx, id);
         if (found == null) {
-            found = findAlternateForDelete(ctx, id);
+            found = findAlternateForDelete(req, ctx, id);
             if (found == null) return notFound(id);
         }
 

bubble-server/src/main/java/bubble/resources/account/AccountsResource.java

@@ -19,6 +19,7 @@ import bubble.model.device.BubbleDeviceType;
 import bubble.resources.app.AppsResource;
 import bubble.resources.bill.*;
 import bubble.resources.cloud.*;
+import bubble.resources.device.DevicesResource;
 import bubble.resources.driver.DriversResource;
 import bubble.resources.notify.ReceivedNotificationsResource;
 import bubble.resources.notify.SentNotificationsResource;

bubble-server/src/main/java/bubble/resources/account/AuthResource.java

@@ -32,7 +32,7 @@ import bubble.service.bill.PromotionService;
 import bubble.service.boot.ActivationService;
 import bubble.service.boot.NodeManagerService;
 import bubble.service.boot.SageHelloService;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import bubble.service.cloud.GeoService;
 import bubble.service.notify.NotificationService;
 import bubble.service.upgrade.BubbleJarUpgradeService;
@@ -102,7 +102,7 @@ public class AuthResource {
     @Autowired private BubbleConfiguration configuration;
     @Autowired private StandardAuthenticatorService authenticatorService;
     @Autowired private PromotionService promoService;
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
     @Autowired private DeviceDAO deviceDAO;
     @Autowired private BubbleNodeKeyDAO nodeKeyDAO;
     @Autowired private NodeManagerService nodeManagerService;
@@ -707,7 +707,7 @@ public class AuthResource {
             } else {
                 final String remoteHost = getRemoteHost(req);
                 if (!empty(remoteHost)) {
-                    final Device device = deviceIdService.findDeviceByIp(remoteHost);
+                    final Device device = deviceService.findDeviceByIp(remoteHost);
                     if (device != null) {
                         type = device.getDeviceType().getCertType();
                     }

bubble-server/src/main/java/bubble/resources/account/MeResource.java

@@ -19,6 +19,8 @@ import bubble.model.device.BubbleDeviceType;
 import bubble.resources.app.AppsResource;
 import bubble.resources.bill.*;
 import bubble.resources.cloud.*;
+import bubble.resources.device.DevicesResource;
+import bubble.resources.device.FlexRoutersResource;
 import bubble.resources.driver.DriversResource;
 import bubble.resources.notify.ReceivedNotificationsResource;
 import bubble.resources.notify.SentNotificationsResource;
@@ -371,6 +373,12 @@ public class MeResource {
         return ok(BubbleDeviceType.getSelectableTypes());
     }
 
+    @Path(EP_FLEX_ROUTERS)
+    public FlexRoutersResource getFlexRouters(@Context ContainerRequest ctx) {
+        final Account caller = userPrincipal(ctx);
+        return configuration.subResource(FlexRoutersResource.class, caller);
+    }
+
     @Path(EP_REFERRAL_CODES)
     public ReferralCodesResource getReferralCodes(@Context ContainerRequest ctx) {
         final Account caller = userPrincipal(ctx);
@@ -444,7 +452,7 @@ public class MeResource {
         if (!caller.admin()) return forbidden();
         authenticatorService.ensureAuthenticated(ctx);
 
-        background(() -> jarUpgradeService.upgrade());
+        background(() -> jarUpgradeService.upgrade(), "MeResource.upgrade");
         return ok(configuration.getPublicSystemConfigs());
     }
 

bubble-server/src/main/java/bubble/resources/app/AppSitesResource.java

@@ -12,7 +12,7 @@ import bubble.model.app.config.AppDataDriver;
 import bubble.model.app.config.AppDataView;
 import bubble.model.device.Device;
 import bubble.resources.account.AccountOwnedTemplateResource;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import org.cobbzilla.wizard.model.search.SearchQuery;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
@@ -32,7 +32,7 @@ public class AppSitesResource extends AccountOwnedTemplateResource<AppSite, AppS
 
     private final BubbleApp app;
 
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
 
     public AppSitesResource(Account account, BubbleApp app) {
         super(account);
@@ -102,7 +102,7 @@ public class AppSitesResource extends AccountOwnedTemplateResource<AppSite, AppS
         if (view == null) return notFound(viewName);
 
         final String remoteHost = getRemoteHost(req);
-        final Device device = deviceIdService.findDeviceByIp(remoteHost);
+        final Device device = deviceService.findDeviceByIp(remoteHost);
 
         final AppDataDriver driver = app.getDataConfig().getDataDriver(configuration);
         return ok(driver.query(caller, device, app, site, view, query));

bubble-server/src/main/java/bubble/resources/app/AppsResource.java

@@ -9,7 +9,7 @@ import bubble.model.app.BubbleApp;
 import bubble.model.app.config.AppDataDriver;
 import bubble.model.app.config.AppDataView;
 import bubble.model.device.Device;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.model.search.SearchQuery;
 import org.glassfish.grizzly.http.server.Request;
@@ -29,7 +29,7 @@ public class AppsResource extends AppsResourceBase {
 
     public AppsResource(Account account) { super(account); }
 
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
 
     @GET @Path("/{id}"+EP_VIEW+"/{view}")
     public Response search(@Context Request req,
@@ -58,7 +58,7 @@ public class AppsResource extends AppsResourceBase {
         if (view == null) return notFound(viewName);
 
         final String remoteHost = getRemoteHost(req);
-        final Device device = deviceIdService.findDeviceByIp(remoteHost);
+        final Device device = deviceService.findDeviceByIp(remoteHost);
 
         final AppDataDriver driver = app.getDataConfig().getDataDriver(configuration);
         return ok(driver.query(caller, device, app, null, view, query));

bubble-server/src/main/java/bubble/resources/app/AppsResourceBase.java

@@ -63,8 +63,9 @@ public abstract class AppsResourceBase extends AccountOwnedTemplateResource<Bubb
     }
 
     @DELETE @Path("/{id}")
-    public Response delete(@Context ContainerRequest ctx,
-                           @PathParam("id") String id) {
+    @Override public Response delete(@Context Request req,
+                                     @Context ContainerRequest ctx,
+                                     @PathParam("id") String id) {
 
         if (isReadOnly(ctx)) return forbidden();
 

bubble-server/src/main/java/bubble/resources/app/DataResourceBase.java

@@ -16,6 +16,7 @@ import bubble.model.device.Device;
 import bubble.resources.account.AccountOwnedTemplateResource;
 import bubble.server.BubbleConfiguration;
 import lombok.extern.slf4j.Slf4j;
+import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -86,14 +87,15 @@ public abstract class DataResourceBase extends AccountOwnedTemplateResource<AppD
     }
 
     @POST @Path("/{id}"+EP_ACTIONS+"/{action}")
-    public Response takeAction(@Context ContainerRequest ctx,
+    public Response takeAction(@Context Request req,
+                               @Context ContainerRequest ctx,
                                @PathParam("id") String id,
                                @PathParam("action") String action) {
         if (isReadOnly(ctx)) return forbidden();
         switch (action) {
             case "enable": return enable(ctx, id);
             case "disable": return disable(ctx, id);
-            case "delete": return delete(ctx, id);
+            case "delete": return delete(req, ctx, id);
             default:
                 app.getDataConfig().getDataDriver(configuration).takeAction(id, action);
                 return ok();

bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java

@@ -306,7 +306,8 @@ public class AccountPlansResource extends AccountOwnedResource<AccountPlan, Acco
 
     // If the accountPlan is not found, look for an orphaned network
     @DELETE @Path("/{id}")
-    @Override public Response delete(@Context ContainerRequest ctx,
+    @Override public Response delete(@Context Request req,
+                                     @Context ContainerRequest ctx,
                                      @PathParam("id") String id) {
         final Account caller = checkEditable(ctx);
         AccountPlan found = find(ctx, id);
@@ -326,7 +327,7 @@ public class AccountPlansResource extends AccountOwnedResource<AccountPlan, Acco
         getDao().update(found.setDeleting(true));
 
         final String planUuid = found.getUuid();
-        background(() -> getDao().delete(planUuid));
+        background(() -> getDao().delete(planUuid), "AccountPlansResource.delete");
 
         return ok(found.setDeletedNetwork(found.getNetwork()));
     }

bubble-server/src/main/java/bubble/resources/account/DevicesResource.java → bubble-server/src/main/java/bubble/resources/device/DevicesResource.java

@@ -2,14 +2,16 @@
  * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
  * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
  */
-package bubble.resources.account;
+package bubble.resources.device;
 
 import bubble.dao.device.DeviceDAO;
 import bubble.model.account.Account;
 import bubble.model.device.Device;
 import bubble.model.device.DeviceSecurityLevel;
+import bubble.resources.account.AccountOwnedResource;
+import bubble.resources.account.VpnConfigResource;
 import bubble.server.BubbleConfiguration;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import lombok.extern.slf4j.Slf4j;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
@@ -53,7 +55,7 @@ public class DevicesResource extends AccountOwnedResource<Device, DeviceDAO> {
     }
 
     @Override protected Device populate(ContainerRequest ctx, Device device) {
-        return device.setStatus(deviceIdService.getDeviceStatus(device.getUuid()));
+        return device.setStatus(deviceService.getDeviceStatus(device.getUuid()));
     }
 
     @Override protected List<Device> sort(List<Device> list, Request req, ContainerRequest ctx) {
@@ -106,14 +108,14 @@ public class DevicesResource extends AccountOwnedResource<Device, DeviceDAO> {
         return configuration.subResource(VpnConfigResource.class, device);
     }
 
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
 
     @GET @Path("/{id}"+EP_IPS)
     public Response getIps(@Context ContainerRequest ctx,
                            @PathParam("id") String id) {
         final Device device = getDao().findByAccountAndId(getAccountUuid(ctx), id);
         if (device == null) return notFound(id);
-        return ok(deviceIdService.findIpsByDevice(device.getUuid()));
+        return ok(deviceService.findIpsByDevice(device.getUuid()));
     }
 
     @GET @Path("/{id}"+EP_STATUS)
@@ -121,7 +123,7 @@ public class DevicesResource extends AccountOwnedResource<Device, DeviceDAO> {
                               @PathParam("id") String id) {
         final Device device = getDao().findByAccountAndId(getAccountUuid(ctx), id);
         if (device == null) return notFound(id);
-        return ok(deviceIdService.getLiveDeviceStatus(device.getUuid()));
+        return ok(deviceService.getLiveDeviceStatus(device.getUuid()));
     }
 
 }

bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java

@@ -0,0 +1,115 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.resources.device;
+
+import bubble.dao.device.FlexRouterDAO;
+import bubble.model.account.Account;
+import bubble.model.device.Device;
+import bubble.model.device.FlexRouter;
+import bubble.resources.account.AccountOwnedResource;
+import bubble.service.device.DeviceService;
+import bubble.service.device.FlexRouterStatus;
+import bubble.service.device.StandardFlexRouterService;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.network.PortPicker;
+import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.jersey.server.ContainerRequest;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import java.math.BigInteger;
+import java.net.InetAddress;
+
+import static bubble.ApiConstants.EP_STATUS;
+import static org.cobbzilla.util.network.PortPicker.portIsAvailable;
+import static org.cobbzilla.wizard.resources.ResourceUtil.*;
+
+@Slf4j
+public class FlexRoutersResource extends AccountOwnedResource<FlexRouter, FlexRouterDAO> {
+
+    @Autowired private DeviceService deviceService;
+    @Autowired private StandardFlexRouterService flexRouterService;
+
+    public FlexRoutersResource(Account account) { super(account); }
+
+    @Override protected boolean isReadOnly(ContainerRequest ctx) {
+        final Account caller = userPrincipal(ctx);
+        return !caller.admin();
+    }
+
+    @Override protected FlexRouter findAlternate(ContainerRequest ctx, FlexRouter request) {
+        return getDao().findByKeyHash(request.getKeyHash());
+    }
+
+    @GET @Path("/{id}"+EP_STATUS)
+    public Response update(@Context Request req,
+                           @Context ContainerRequest ctx,
+                           @PathParam("id") String id) {
+        FlexRouter router = find(req, ctx, id);
+        if (router == null) router = findAlternate(req, ctx, id);
+        if (router == null) return ok(FlexRouterStatus.deleted);
+        return ok(flexRouterService.status(router.getUuid()));
+    }
+
+    @Override protected Object daoCreate(FlexRouter toCreate) {
+        final Object router = super.daoCreate(toCreate);
+        flexRouterService.interruptSoon();
+        return router;
+    }
+
+    @Override protected Object daoUpdate(FlexRouter toUpdate) {
+        final Object router = super.daoUpdate(toUpdate);
+        flexRouterService.interruptSoon();
+        return router;
+    }
+
+    @Override protected FlexRouter setReferences(ContainerRequest ctx, Request req, Account caller, FlexRouter router) {
+
+        if (!router.hasKey()) throw invalidEx("err.sshPublicKey.required");
+
+        final String ip = router.getIp();
+        final Device device = deviceService.findDeviceByIp(ip);
+        if (device == null) throw invalidEx("err.device.notFound");
+        router.setDevice(device.getUuid());
+
+        if (!router.hasAuthToken()) throw invalidEx("err.token.required");
+        router.setToken(router.getAuth_token());
+
+        if (!router.hasPort()) {
+            // choose a port that no other FlexRouter is using, and that is available
+            final InetAddress inetAddress;
+            try {
+                inetAddress = InetAddress.getByName(ip);
+            } catch (Exception e) {
+                throw invalidEx("err.ip.invalid");
+            }
+            final BigInteger addrInt = new BigInteger(inetAddress.getAddress());
+            final int portOffset = addrInt.mod(new BigInteger("10000", 10)).intValueExact();
+            boolean foundPort = false;
+            for (int base = 20000; base < 65535; base += 10000) {
+                final int port = base + portOffset;
+                if (getDao().findByPort(port) == null && portIsAvailable(port)) {
+                    router.setPort(port);
+                    foundPort = true;
+                    break;
+                }
+            }
+            if (!foundPort) {
+                final int port = PortPicker.pickOrDie();
+                log.warn("setReferences: standard port could not be found, using port " + port + " from PortPicker");
+                router.setPort(port);
+            }
+        } else if (!router.hasUuid()) {
+            throw invalidEx("err.port.cannotSetOrChange");
+        }
+        router.setActive(false);
+        return super.setReferences(ctx, req, caller, router);
+    }
+
+}

bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java

@@ -12,14 +12,14 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
 public class FilterConnCheckRequest {
 
-    @Getter @Setter private String addr;
-    public boolean hasAddr() { return !empty(addr); }
+    @Getter @Setter private String clientAddr;
+    public boolean hasClientAddr() { return !empty(clientAddr); }
+
+    @Getter @Setter private String serverAddr;
+    public boolean hasServerAddr() { return !empty(serverAddr); }
 
     @Getter @Setter private String[] fqdns;
     public boolean hasFqdns() { return !empty(fqdns); }
     public boolean hasFqdn(String f) { return hasFqdns() && ArrayUtils.contains(fqdns, f); }
 
-    @Getter @Setter private String remoteAddr;
-    public boolean hasRemoteAddr() { return !empty(remoteAddr); }
-
 }

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -19,12 +19,16 @@ import bubble.model.cloud.BubbleNetwork;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.device.Device;
 import bubble.model.device.DeviceSecurityLevel;
+import bubble.model.device.DeviceStatus;
 import bubble.rule.FilterMatchDecision;
 import bubble.server.BubbleConfiguration;
 import bubble.service.block.BlockStatsService;
 import bubble.service.block.BlockStatsSummary;
 import bubble.service.boot.SelfNodeService;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
+import bubble.service.device.FlexRouterInfo;
+import bubble.service.device.StandardFlexRouterService;
+import bubble.service.message.MessageService;
 import bubble.service.stream.ConnectionCheckResponse;
 import bubble.service.stream.StandardRuleEngineService;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -51,6 +55,8 @@ import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.*;
 import static bubble.resources.stream.FilterMatchersResponse.NO_MATCHERS;
+import static bubble.rule.AppRuleDriver.isFlexRouteFqdn;
+import static bubble.service.device.FlexRouterInfo.missingFlexRouter;
 import static bubble.service.stream.HttpStreamDebug.getLogFqdn;
 import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIMEOUT;
 import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY;
@@ -60,6 +66,7 @@ import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
 import static org.cobbzilla.util.collection.ArrayUtil.arrayToString;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
+import static org.cobbzilla.util.http.HttpContentTypes.TEXT_PLAIN;
 import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.network.NetworkUtil.isLocalIpv4;
@@ -79,11 +86,13 @@ public class FilterHttpResource {
     @Autowired private AppSiteDAO siteDAO;
     @Autowired private AppRuleDAO ruleDAO;
     @Autowired private DeviceDAO deviceDAO;
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
     @Autowired private RedisService redis;
     @Autowired private BubbleConfiguration configuration;
     @Autowired private SelfNodeService selfNodeService;
     @Autowired private BlockStatsService blockStats;
+    @Autowired private StandardFlexRouterService flexRouterService;
+    @Autowired private MessageService messageService;
 
     private static final long ACTIVE_REQUEST_TIMEOUT = HOURS.toSeconds(12);
 
@@ -146,7 +155,7 @@ public class FilterHttpResource {
                                     @Context ContainerRequest request,
                                     FilterConnCheckRequest connCheckRequest) {
         final String prefix = "checkConnection: ";
-        if (connCheckRequest == null || !connCheckRequest.hasAddr() || !connCheckRequest.hasRemoteAddr()) {
+        if (connCheckRequest == null || !connCheckRequest.hasServerAddr() || !connCheckRequest.hasClientAddr()) {
             if (log.isDebugEnabled()) log.debug(prefix+"invalid connCheckRequest, returning forbidden");
             return forbidden();
         }
@@ -157,13 +166,13 @@ public class FilterHttpResource {
         if (isLocalIp) {
             // if it is for our host or net name, passthru
             if (connCheckRequest.hasFqdns() && (connCheckRequest.hasFqdn(getThisNode().getFqdn()) || connCheckRequest.hasFqdn(getThisNetwork().getNetworkDomain()))) {
-                if (log.isDebugEnabled()) log.debug(prefix + "returning passthru for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr());
+                if (log.isDebugEnabled()) log.debug(prefix + "returning passthru for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr());
                 return ok(ConnectionCheckResponse.passthru);
             }
         }
 
-        final String vpnAddr = connCheckRequest.getRemoteAddr();
-        final Device device = deviceIdService.findDeviceByIp(vpnAddr);
+        final String vpnAddr = connCheckRequest.getClientAddr();
+        final Device device = deviceService.findDeviceByIp(vpnAddr);
         if (device == null) {
             if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning not found");
             return notFound();
@@ -177,16 +186,21 @@ public class FilterHttpResource {
             return notFound();
         }
 
+        // if this is for a local ip, it's either a flex route or an automatic block
+        // legitimate local requests would have otherwise never reached here
         if (isLocalIp) {
-            final boolean showStats = showStats(accountUuid, connCheckRequest.getAddr(), connCheckRequest.getFqdns());
-            final DeviceSecurityLevel secLevel = device.getSecurityLevel();
-            if (showStats && secLevel.supportsRequestModification()) {
-                // allow it for now
-                if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showStats=true, secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr());
-                return ok(ConnectionCheckResponse.noop);
+            if (isFlexRouteFqdn(redis, vpnAddr, connCheckRequest.getFqdns())) {
+                if (log.isDebugEnabled()) log.debug(prefix + "detected flex route, allowing processing to continue");
             } else {
-                if (log.isDebugEnabled()) log.debug(prefix + "returning block (showStats="+showStats+", secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr());
-                return ok(ConnectionCheckResponse.block);
+                final boolean showStats = showStats(accountUuid, connCheckRequest.getServerAddr(), connCheckRequest.getFqdns());
+                final DeviceSecurityLevel secLevel = device.getSecurityLevel();
+                if (showStats && secLevel.supportsRequestModification()) {
+                    // allow it for now
+                    if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showStats=true, secLevel=" + secLevel + ") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr());
+                } else {
+                    if (log.isDebugEnabled()) log.debug(prefix + "returning block (showStats=" + showStats + ", secLevel=" + secLevel + ") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr());
+                    return ok(ConnectionCheckResponse.block);
+                }
             }
         }
 
@@ -204,17 +218,17 @@ public class FilterHttpResource {
         if (connCheckRequest.hasFqdns()) {
             final String[] fqdns = connCheckRequest.getFqdns();
             for (String fqdn : fqdns) {
-                checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn);
+                checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getServerAddr(), fqdn);
                 if (checkResponse != ConnectionCheckResponse.noop) {
-                    if (log.isDebugEnabled()) log.debug(prefix + "found " + checkResponse + " (breaking) for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr());
+                    if (log.isDebugEnabled()) log.debug(prefix + "found " + checkResponse + " (breaking) for fqdn/addr=" + fqdn + "/" + connCheckRequest.getServerAddr());
                     break;
                 }
             }
-            if (log.isDebugEnabled()) log.debug(prefix+"returning "+checkResponse+" for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr());
+            if (log.isDebugEnabled()) log.debug(prefix+"returning "+checkResponse+" for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getServerAddr());
             return ok(checkResponse);
 
         } else {
-            if (log.isDebugEnabled()) log.debug(prefix+"returning noop for NO fqdns,  addr="+connCheckRequest.getAddr());
+            if (log.isDebugEnabled()) log.debug(prefix+"returning noop for NO fqdns,  addr="+connCheckRequest.getServerAddr());
             return ok(ConnectionCheckResponse.noop);
         }
     }
@@ -225,7 +239,7 @@ public class FilterHttpResource {
     }
 
     private boolean isForLocalIp(FilterConnCheckRequest connCheckRequest) {
-        return connCheckRequest.hasAddr() && getConfiguredIps().contains(connCheckRequest.getAddr());
+        return connCheckRequest.hasServerAddr() && getConfiguredIps().contains(connCheckRequest.getServerAddr());
     }
 
     private boolean isForLocalIp(FilterMatchersRequest matchersRequest) {
@@ -237,17 +251,17 @@ public class FilterHttpResource {
     @Getter(lazy=true) private final BubbleNetwork thisNetwork = selfNodeService.getThisNetwork();
 
     public boolean showStats(String accountUuid, String ip, String[] fqdns) {
-        if (!deviceIdService.doShowBlockStats(accountUuid)) return false;
+        if (!deviceService.doShowBlockStats(accountUuid)) return false;
         for (String fqdn : fqdns) {
-            final Boolean show = deviceIdService.doShowBlockStatsForIpAndFqdn(ip, fqdn);
+            final Boolean show = deviceService.doShowBlockStatsForIpAndFqdn(ip, fqdn);
             if (show != null) return show;
         }
         return true;
     }
 
     public boolean showStats(String accountUuid, String ip, String fqdn) {
-        if (!deviceIdService.doShowBlockStats(accountUuid)) return false;
-        final Boolean show = deviceIdService.doShowBlockStatsForIpAndFqdn(ip, fqdn);
+        if (!deviceService.doShowBlockStats(accountUuid)) return false;
+        final Boolean show = deviceService.doShowBlockStatsForIpAndFqdn(ip, fqdn);
         return show == null || show;
     }
 
@@ -277,7 +291,7 @@ public class FilterHttpResource {
         }
 
         final String vpnAddr = filterRequest.getClientAddr();
-        final Device device = deviceIdService.findDeviceByIp(vpnAddr);
+        final Device device = deviceService.findDeviceByIp(vpnAddr);
         if (device == null) {
             if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning no matchers");
             else if (extraLog) log.error(prefix+"device not found for IP "+vpnAddr+", returning no matchers");
@@ -287,16 +301,20 @@ public class FilterHttpResource {
         }
         filterRequest.setDevice(device.getUuid());
 
-        // if this is for a local ip, it's an automatic block
-        // legitimate local requests would have been passthru and never reached here
+        // if this is for a local ip, it's either a flex route or an automatic block
+        // legitimate local requests would have otherwise been "passthru" and never reached here
         final boolean isLocalIp = isForLocalIp(filterRequest);
         final boolean showStats = showStats(device.getAccount(), filterRequest.getClientAddr(), filterRequest.getFqdn());
         if (isLocalIp) {
-            if (filterRequest.isBrowser() && showStats) {
-                blockStats.record(filterRequest, FilterMatchDecision.abort_not_found);
+            if (isFlexRouteFqdn(redis, vpnAddr, filterRequest.getFqdn())) {
+                if (log.isDebugEnabled()) log.debug(prefix + "detected flex route, not blocking");
+            } else {
+                if (filterRequest.isBrowser() && showStats) {
+                    blockStats.record(filterRequest, FilterMatchDecision.abort_not_found);
+                }
+                if (log.isDebugEnabled()) log.debug(prefix + "returning FORBIDDEN (showBlockStats==" + showStats + ")");
+                return forbidden();
             }
-            if (log.isDebugEnabled()) log.debug(prefix + "returning FORBIDDEN (showBlockStats=="+ showStats +")");
-            return forbidden();
         }
 
         final FilterMatchersResponse response = getMatchersResponse(filterRequest, req, request);
@@ -619,6 +637,7 @@ public class FilterHttpResource {
     }
 
     @GET @Path(EP_STATUS+"/{requestId}")
+    @Produces(APPLICATION_JSON)
     public Response getRequestStatus(@Context Request req,
                                      @Context ContainerRequest ctx,
                                      @PathParam("requestId") String requestId) {
@@ -628,7 +647,38 @@ public class FilterHttpResource {
         return ok(summary);
     }
 
+    @GET @Path(EP_FLEX_ROUTERS+"/{fqdn}")
+    @Produces(APPLICATION_JSON)
+    public Response getFlexRouter(@Context Request req,
+                                  @Context ContainerRequest ctx,
+                                  @PathParam("fqdn") String fqdn) {
+        final String publicIp = getRemoteAddr(req);
+        final Device device = deviceService.findDeviceByIp(publicIp);
+        if (device == null) {
+            log.warn("getFlexRouter: device not found with IP: "+publicIp);
+            return notFound();
+        }
+
+        final DeviceStatus deviceStatus = deviceService.getDeviceStatus(device.getUuid());
+        if (!deviceStatus.hasIp()) {
+            log.error("getFlexRouter: no device status for device: "+device);
+            return notFound();
+        }
+        final String vpnIp = deviceStatus.getIp();
+
+        if (log.isDebugEnabled()) log.debug("getFlexRouter: finding routers for vpnIp="+vpnIp);
+        Collection<FlexRouterInfo> routers = flexRouterService.selectClosestRouter(device.getAccount(), vpnIp, publicIp);
+
+        if (log.isDebugEnabled()) log.debug("getFlexRouter: found router(s) for vpnIp="+vpnIp+": "+json(routers, COMPACT_MAPPER));
+        if (routers.isEmpty()) {
+            final Account account = accountDAO.findByUuid(device.getAccount());
+            return ok(missingFlexRouter(account, device, fqdn, messageService, configuration.getHandlebars()));
+        }
+        return ok(routers.iterator().next().initAuth());
+    }
+
     @POST @Path(EP_LOGS+"/{requestId}")
+    @Produces(APPLICATION_JSON)
     public Response requestLog(@Context Request req,
                                @Context ContainerRequest ctx,
                                @PathParam("requestId") String requestId,
@@ -642,6 +692,7 @@ public class FilterHttpResource {
             = new ExpirationMap<>(1000, DAYS.toMillis(3), ExpirationEvictionPolicy.atime);
 
     @POST @Path(EP_FOLLOW+"/{requestId}")
+    @Produces(TEXT_PLAIN)
     public Response followLink(@Context Request req,
                                @Context ContainerRequest ctx,
                                @PathParam("requestId") String requestId,

bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java

@@ -11,7 +11,7 @@ import bubble.model.app.AppMatcher;
 import bubble.model.device.Device;
 import bubble.rule.FilterMatchDecision;
 import bubble.server.BubbleConfiguration;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import bubble.service.stream.StandardRuleEngineService;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -46,7 +46,7 @@ public class ReverseProxyResource {
     @Autowired private AppMatcherDAO matcherDAO;
     @Autowired private AppRuleDAO ruleDAO;
     @Autowired private StandardRuleEngineService ruleEngine;
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
     @Autowired private FilterHttpResource filterHttpResource;
 
     @Getter(lazy=true) private final int prefixLength = configuration.getHttp().getBaseUri().length() + PROXY_ENDPOINT.length() + 1;
@@ -60,7 +60,7 @@ public class ReverseProxyResource {
                         @PathParam("path") String path) throws URISyntaxException, IOException {
         final Account account = userPrincipal(request);
         final String remoteHost = getRemoteHost(req);
-        final Device device = deviceIdService.findDeviceByIp(remoteHost);
+        final Device device = deviceService.findDeviceByIp(remoteHost);
         if (device == null) return ruleEngine.passthru(request);
 
         final URIBean ub = getUriBean(request);

bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java

@@ -16,7 +16,7 @@ import bubble.model.device.Device;
 import bubble.resources.stream.FilterHttpRequest;
 import bubble.resources.stream.FilterMatchersRequest;
 import bubble.server.BubbleConfiguration;
-import bubble.service.cloud.StandardDeviceIdService;
+import bubble.service.device.StandardDeviceService;
 import bubble.service.stream.AppPrimerService;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jknack.handlebars.Handlebars;
@@ -64,7 +64,7 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver {
     @Autowired protected BubbleNetworkDAO networkDAO;
     @Autowired protected DeviceDAO deviceDAO;
     @Autowired protected AppPrimerService appPrimerService;
-    @Autowired protected StandardDeviceIdService deviceService;
+    @Autowired protected StandardDeviceService deviceService;
 
     @Getter @Setter private AppRuleDriver next;
 

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -5,7 +5,6 @@
 package bubble.rule;
 
 import bubble.model.account.Account;
-import bubble.model.app.AppData;
 import bubble.model.app.AppMatcher;
 import bubble.model.app.AppRule;
 import bubble.model.app.BubbleApp;
@@ -27,7 +26,6 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.util.Map;
 import java.util.Set;
-import java.util.function.Function;
 
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
 import static org.cobbzilla.util.daemon.ZillaRuntime.now;
@@ -44,12 +42,18 @@ public interface AppRuleDriver {
     // also used in dnscrypt-proxy/plugin_reverse_resolve_cache.go
     String REDIS_REJECT_LISTS = "rejectLists";
     String REDIS_BLOCK_LISTS = "blockLists";
+    String REDIS_WHITE_LISTS = "whiteLists";
     String REDIS_FILTER_LISTS = "filterLists";
+    String REDIS_FLEX_LISTS = "flexLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
+    String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
     String REDIS_LIST_SUFFIX = "~UNION";
 
     default Set<String> getPrimedRejectDomains () { return null; }
     default Set<String> getPrimedBlockDomains () { return null; }
+    default Set<String> getPrimedWhiteListDomains() { return null; }
     default Set<String> getPrimedFilterDomains () { return null; }
+    default Set<String> getPrimedFlexDomains () { return null; }
+    default Set<String> getPrimedFlexExcludeDomains () { return null; }
 
     static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) {
         defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains);
@@ -59,10 +63,22 @@ public interface AppRuleDriver {
         defineRedisSet(redis, ip, REDIS_BLOCK_LISTS, list, fullyBlockedDomains);
     }
 
+    static void defineRedisWhiteListSet(RedisService redis, String ip, String list, String[] fullyBlockedDomains) {
+        defineRedisSet(redis, ip, REDIS_WHITE_LISTS, list, fullyBlockedDomains);
+    }
+
     static void defineRedisFilterSet(RedisService redis, String ip, String list, String[] filterDomains) {
         defineRedisSet(redis, ip, REDIS_FILTER_LISTS, list, filterDomains);
     }
 
+    static void defineRedisFlexSet(RedisService redis, String ip, String list, String[] flexDomains) {
+        defineRedisSet(redis, ip, REDIS_FLEX_LISTS, list, flexDomains);
+    }
+
+    static void defineRedisFlexExcludeSet(RedisService redis, String ip, String list, String[] flexExcludeDomains) {
+        defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains);
+    }
+
     static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) {
         final String listOfListsForIp = listOfListsName + "~" + ip;
         final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX;
@@ -72,7 +88,33 @@ public interface AppRuleDriver {
         redis.rename(tempList, ipList);
         redis.sadd_plaintext(listOfListsForIp, ipList);
         final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp));
-        log.debug("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count);
+        if (log.isDebugEnabled()) log.debug("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count);
+    }
+
+    static boolean isFlexRouteFqdn(RedisService redis, String ip, String[] fqdns) {
+        for (String fqdn : fqdns) {
+            if (isFlexRouteFqdn(redis, ip, fqdn)) return true;
+        }
+        return false;
+    }
+
+    static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) {
+
+        final String excludeKey = REDIS_FLEX_EXCLUDE_LISTS + "~" + ip + REDIS_LIST_SUFFIX;
+        if (redis.sismember_plaintext(excludeKey, fqdn)) {
+            return false;
+        }
+
+        final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX;
+        String check = fqdn;
+        while (true) {
+            final boolean found = redis.sismember_plaintext(key, check);
+            if (found) return true;
+            final int dotPos = check.indexOf('.');
+            if (dotPos == check.length()) return false;
+            check = check.substring(dotPos+1);
+            if (!check.contains(".")) return false;
+        }
     }
 
     AppRuleDriver getNext();

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -16,7 +16,7 @@ import bubble.rule.RequestModifierConfig;
 import bubble.rule.RequestModifierRule;
 import bubble.rule.analytics.TrafficAnalyticsRuleDriver;
 import bubble.server.BubbleConfiguration;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import bubble.service.stream.AppRuleHarness;
 import bubble.service.stream.ConnectionCheckResponse;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -58,6 +58,9 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver
     private final AtomicReference<Set<String>> fullyBlockedDomains = new AtomicReference<>(Collections.emptySet());
     @Override public Set<String> getPrimedBlockDomains() { return fullyBlockedDomains.get(); }
 
+    private final AtomicReference<Set<String>> whiteListDomains = new AtomicReference<>(Collections.emptySet());
+    @Override public Set<String> getPrimedWhiteListDomains() { return whiteListDomains.get(); }
+
     private final AtomicReference<Set<String>> rejectDomains = new AtomicReference<>(Collections.emptySet());
     @Override public Set<String> getPrimedRejectDomains() { return rejectDomains.get(); }
 
@@ -165,6 +168,9 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver
         if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) {
             partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains());
         }
+        if (!newBlockList.getWhitelistDomains().equals(whiteListDomains.get())) {
+            whiteListDomains.set(newBlockList.getWhitelistDomainNames());
+        }
 
         log.debug("refreshBlockLists: rejectDomains="+rejectDomains.get().size());
         log.debug("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size());
@@ -419,11 +425,11 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver
     }
 
     @Override public void prime(Account account, BubbleApp app, BubbleConfiguration configuration) {
-        final DeviceIdService deviceIdService = configuration.getBean(DeviceIdService.class);
+        final DeviceService deviceService = configuration.getBean(DeviceService.class);
         final AppDataDAO dataDAO = configuration.getBean(AppDataDAO.class);
         log.info("priming app="+app.getName());
         dataDAO.findByAccountAndAppAndAndKeyPrefix(account.getUuid(), app.getUuid(), PREFIX_APPDATA_HIDE_STATS)
-                .forEach(data -> deviceIdService.setBlockStatsForFqdn(account, fqdnFromKey(data.getKey()), !Boolean.parseBoolean(data.getData())));
+                .forEach(data -> deviceService.setBlockStatsForFqdn(account, fqdnFromKey(data.getKey()), !Boolean.parseBoolean(data.getData())));
     }
 
     @Override public Function<AppData, AppData> createCallback(Account account,
@@ -433,15 +439,15 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver
             final String prefix = "createCallbackB("+data.getKey()+"="+data.getData()+"): ";
             log.info(prefix+"starting");
             if (data.getKey().startsWith(PREFIX_APPDATA_HIDE_STATS)) {
-                final DeviceIdService deviceIdService = configuration.getBean(DeviceIdService.class);
+                final DeviceService deviceService = configuration.getBean(DeviceService.class);
                 final String fqdn = fqdnFromKey(data.getKey());
                 if (validateRegexMatches(HOST_PATTERN, fqdn)) {
                     if (data.deleting()) {
                         log.info(prefix+"unsetting fqdn: "+fqdn);
-                        deviceIdService.unsetBlockStatsForFqdn(account, fqdn);
+                        deviceService.unsetBlockStatsForFqdn(account, fqdn);
                     } else {
                         log.info(prefix+"setting fqdn: "+fqdn);
-                        deviceIdService.setBlockStatsForFqdn(account, fqdn, !Boolean.parseBoolean(data.getData()));
+                        deviceService.setBlockStatsForFqdn(account, fqdn, !Boolean.parseBoolean(data.getData()));
                     }
                 } else {
                     throw invalidEx("err.fqdn.invalid", "not a valid FQDN: "+fqdn, fqdn);

bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.passthru;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+import java.util.Set;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+
+@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of="feedUrl")
+public class BasePassthruFeed implements Comparable<BasePassthruFeed> {
+
+    public BasePassthruFeed (String url) { setFeedUrl(url); }
+
+    public String getId() { return sha256_hex(getFeedUrl()); }
+    public void setId(String id) {} // noop
+
+    @JsonIgnore @Getter @Setter private String feedName;
+    public boolean hasFeedName() { return !empty(feedName); }
+
+    @JsonIgnore @Getter @Setter private String feedUrl;
+
+    @JsonIgnore @Getter @Setter private Set<String> fqdnList;
+    public boolean hasFqdnList() { return !empty(fqdnList); }
+
+    @Override public int compareTo(BasePassthruFeed o) {
+        return getFeedUrl().toLowerCase().compareTo(o.getFeedUrl().toLowerCase());
+    }
+
+}

bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.passthru;
+
+import lombok.NoArgsConstructor;
+import lombok.experimental.Accessors;
+
+import java.util.Set;
+
+import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class FlexFeed extends BasePassthruFeed {
+
+    public static final FlexFeed[] EMPTY_FLEX_FEEDS = new FlexFeed[0];
+
+    public String getFlexFeedName () { return getFeedName(); }
+    public FlexFeed setFlexFeedName (String name) { return (FlexFeed) setFeedName(name); }
+
+    public String getFlexFeedUrl () { return getFeedUrl(); }
+    public FlexFeed setFlexFeedUrl (String url) { return (FlexFeed) setFeedUrl(url); }
+
+    public Set<String> getFlexFqdnList () { return getFqdnList(); }
+    public FlexFeed setFlexFqdnList (Set<String> set) { return (FlexFeed) setFqdnList(set); }
+
+    public FlexFeed(String url) { super(url); }
+
+    public FlexFeed(FlexFeed feed) { copy(this, feed); }
+
+}

bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java

@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.passthru;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class FlexFqdn implements Comparable<FlexFqdn> {
+
+    public String getId() { return flexFqdn; }
+    public void setId(String id) {} // noop
+
+    @Getter @Setter private String flexFqdn;
+
+    public FlexFqdn(String fqdn) { setFlexFqdn(fqdn); }
+
+    @Override public int compareTo(FlexFqdn o) {
+        return getFlexFqdn().toLowerCase().compareTo(o.getFlexFqdn().toLowerCase());
+    }
+
+}

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java

@@ -21,7 +21,8 @@ import java.util.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_FEEDS;
+import static bubble.rule.passthru.FlexFeed.EMPTY_FLEX_FEEDS;
+import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_PASSTHRU_FEEDS;
 import static java.util.concurrent.TimeUnit.HOURS;
 import static java.util.regex.Pattern.CASE_INSENSITIVE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
@@ -34,47 +35,90 @@ import static org.cobbzilla.wizard.server.RestServerBase.reportError;
 public class TlsPassthruConfig {
 
     public static final long DEFAULT_TLS_FEED_REFRESH_INTERVAL = HOURS.toMillis(1);
+    public static final long DEFAULT_FLEX_FEED_REFRESH_INTERVAL = HOURS.toMillis(1);
     public static final String FEED_NAME_PREFIX = "# Name:";
 
-    @Getter @Setter private String[] fqdnList;
-    public boolean hasFqdnList () { return !empty(fqdnList); }
-    public boolean hasFqdn(String fqdn) { return hasFqdnList() && ArrayUtils.indexOf(fqdnList, fqdn) != -1; }
+    @Getter @Setter private String[] passthruFqdnList;
+    public boolean hasPassthruFqdnList() { return !empty(passthruFqdnList); }
+    public boolean hasPassthruFqdn(String fqdn) { return hasPassthruFqdnList() && ArrayUtils.indexOf(passthruFqdnList, fqdn) != -1; }
 
-    public TlsPassthruConfig addFqdn(String fqdn) {
-        return setFqdnList(Arrays.stream(ArrayUtil.append(fqdnList, fqdn)).collect(Collectors.toSet()).toArray(String[]::new));
+    public TlsPassthruConfig addPassthruFqdn(String fqdn) {
+        return setPassthruFqdnList(Arrays.stream(ArrayUtil.append(passthruFqdnList, fqdn)).collect(Collectors.toSet()).toArray(String[]::new));
     }
 
-    public TlsPassthruConfig removeFqdn(String id) {
-        return !hasFqdnList() ? this :
-                setFqdnList(Arrays.stream(getFqdnList())
+    public TlsPassthruConfig removePassthruFqdn(String id) {
+        return !hasPassthruFqdnList() ? this :
+                setPassthruFqdnList(Arrays.stream(getPassthruFqdnList())
                         .filter(fqdn -> !fqdn.equalsIgnoreCase(id.trim()))
                         .toArray(String[]::new));
     }
 
-    @Getter @Setter private TlsPassthruFeed[] feedList;
-    public boolean hasFeedList () { return !empty(feedList); }
-    public boolean hasFeed (TlsPassthruFeed feed) {
-        return hasFeedList() && Arrays.stream(feedList).anyMatch(f -> f.getFeedUrl().equals(feed.getFeedUrl()));
+    @Getter @Setter private TlsPassthruFeed[] passthruFeedList;
+    public boolean hasPassthruFeedList() { return !empty(passthruFeedList); }
+    public boolean hasPassthruFeed(TlsPassthruFeed feed) {
+        return hasPassthruFeedList() && Arrays.stream(passthruFeedList).anyMatch(f -> f.getPassthruFeedUrl().equals(feed.getPassthruFeedUrl()));
     }
 
-    public TlsPassthruConfig addFeed(TlsPassthruFeed feed) {
-        final Set<TlsPassthruFeed> feeds = getFeedSet();
-        if (empty(feeds)) return setFeedList(new TlsPassthruFeed[] {feed});
+    public TlsPassthruConfig addPassthruFeed(TlsPassthruFeed feed) {
+        final Set<TlsPassthruFeed> feeds = getPassthruFeedSet();
+        if (empty(feeds)) return setPassthruFeedList(new TlsPassthruFeed[] {feed});
         feeds.add(feed);
-        return setFeedList(feeds.toArray(EMPTY_FEEDS));
+        return setPassthruFeedList(feeds.toArray(EMPTY_PASSTHRU_FEEDS));
     }
 
-    public TlsPassthruConfig removeFeed(String id) {
-        return setFeedList(getFeedSet().stream()
+    public TlsPassthruConfig removePassthruFeed(String id) {
+        return setPassthruFeedList(getPassthruFeedSet().stream()
                 .filter(feed -> !feed.getId().equals(id))
                 .toArray(TlsPassthruFeed[]::new));
     }
 
-    private Map<String, Set<String>> recentFeedValues = new HashMap<>();
+    private final Map<String, Set<String>> recentFeedValues = new HashMap<>();
 
-    @JsonIgnore public Set<TlsPassthruFeed> getFeedSet() {
-        final TlsPassthruFeed[] feedList = getFeedList();
-        return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
+    @JsonIgnore public Set<TlsPassthruFeed> getPassthruFeedSet() {
+        final TlsPassthruFeed[] feedList = getPassthruFeedList();
+        return !empty(feedList) ? Arrays.stream(feedList)
+                .collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
+    }
+
+    @Getter @Setter private String[] flexFqdnList;
+    public boolean hasFlexFqdnList () { return !empty(flexFqdnList); }
+    public boolean hasFlexFqdn(String flexFqdn) { return hasFlexFqdnList() && ArrayUtils.indexOf(flexFqdnList, flexFqdn) != -1; }
+
+    public TlsPassthruConfig addFlexFqdn(String flexFqdn) {
+        return setFlexFqdnList(Arrays.stream(ArrayUtil.append(flexFqdnList, flexFqdn)).collect(Collectors.toSet()).toArray(String[]::new));
+    }
+
+    public TlsPassthruConfig removeFlexFqdn(String id) {
+        return !hasFlexFqdnList() ? this :
+                setFlexFqdnList(Arrays.stream(getFlexFqdnList())
+                        .filter(flexFqdn -> !flexFqdn.equalsIgnoreCase(id.trim()))
+                        .toArray(String[]::new));
+    }
+
+    @Getter @Setter private FlexFeed[] flexFeedList;
+    public boolean hasFlexFeedList () { return !empty(flexFeedList); }
+    public boolean hasFlexFeed (FlexFeed flexFeed) {
+        return hasFlexFeedList() && Arrays.stream(flexFeedList).anyMatch(f -> f.getFlexFeedUrl().equals(flexFeed.getFlexFeedUrl()));
+    }
+
+    public TlsPassthruConfig addFlexFeed(FlexFeed flexFeed) {
+        final Set<FlexFeed> flexFeeds = getFlexFeedSet();
+        if (empty(flexFeeds)) return setFlexFeedList(new FlexFeed[] {flexFeed});
+        flexFeeds.add(flexFeed);
+        return setFlexFeedList(flexFeeds.toArray(EMPTY_FLEX_FEEDS));
+    }
+
+    public TlsPassthruConfig removeFlexFeed(String id) {
+        return setFlexFeedList(getFlexFeedSet().stream()
+                .filter(flexFeed -> !flexFeed.getId().equals(id))
+                .toArray(FlexFeed[]::new));
+    }
+
+    private final Map<String, Set<String>> recentFlexFeedValues = new HashMap<>();
+
+    @JsonIgnore public Set<FlexFeed> getFlexFeedSet() {
+        final FlexFeed[] flexFeedList = getFlexFeedList();
+        return !empty(flexFeedList) ? Arrays.stream(flexFeedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
     }
 
     @ToString
@@ -82,6 +126,7 @@ public class TlsPassthruConfig {
         @Getter @Setter private String fqdn;
         @Getter @Setter private Pattern fqdnPattern;
         public boolean hasPattern () { return fqdnPattern != null; }
+        public boolean fqdnOnly () { return !hasPattern(); }
         public TlsPassthruMatcher (String fqdn) {
             this.fqdn = fqdn;
             if (fqdn.startsWith("/") && fqdn.endsWith("/")) {
@@ -103,33 +148,58 @@ public class TlsPassthruConfig {
     @JsonIgnore public Set<TlsPassthruMatcher> getPassthruSet() { return getPassthruSetRef().get(); }
 
     private Set<TlsPassthruMatcher> loadPassthruSet() {
+        final Set<TlsPassthruMatcher> set = loadFeeds(this.passthruFeedList, this.passthruFqdnList, this.recentFeedValues);
+        if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning set: "+StringUtil.toString(set, ", ")+" -- fqdnList="+Arrays.toString(this.passthruFqdnList));
+        return set;
+    }
+
+    @JsonIgnore @Getter(lazy=true) private final AutoRefreshingReference<Set<TlsPassthruMatcher>> flexSetRef = new AutoRefreshingReference<>() {
+        @Override public Set<TlsPassthruMatcher> refresh() { return loadFlexSet(); }
+        // todo: load refresh interval from config. implement a config view with an action to set it
+        @Override public long getTimeout() { return DEFAULT_FLEX_FEED_REFRESH_INTERVAL; }
+    };
+    @JsonIgnore public Set<TlsPassthruMatcher> getFlexSet() { return getFlexSetRef().get(); }
+
+    @JsonIgnore public Set<String> getFlexDomains() {
+        return getFlexSetRef().get().stream()
+                .filter(TlsPassthruMatcher::fqdnOnly)
+                .map(TlsPassthruMatcher::getFqdn)
+                .collect(Collectors.toSet());
+    }
+
+    private Set<TlsPassthruMatcher> loadFlexSet() {
+        final Set<TlsPassthruMatcher> set = loadFeeds(this.flexFeedList, this.flexFqdnList, this.recentFlexFeedValues);
+        if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", "));
+        return set;
+    }
+
+    private Set<TlsPassthruMatcher> loadFeeds(BasePassthruFeed[] feedList, String[] fqdnList, Map<String, Set<String>> recentValues) {
         final Set<TlsPassthruMatcher> set = new HashSet<>();
-        if (hasFqdnList()) {
-            for (String val : getFqdnList()) {
+        if (!empty(fqdnList)) {
+            for (String val : fqdnList) {
                 set.add(new TlsPassthruMatcher(val));
             }
         }
-        if (hasFeedList()) {
+        if (!empty(feedList)) {
             // put in a set to avoid duplicate URLs
-            for (TlsPassthruFeed feed : new HashSet<>(Arrays.asList(feedList))) {
+            for (BasePassthruFeed feed : new HashSet<>(Arrays.asList(feedList))) {
                 final TlsPassthruFeed loaded = loadFeed(feed.getFeedUrl());
 
                 // set name if found in special comment
-                if (!feed.hasFeedName() && loaded.hasFeedName()) feed.setFeedName(loaded.getFeedName());
+                if (!feed.hasFeedName() && loaded.hasFeedName()) feed.setFeedName(loaded.getPassthruFeedName());
 
                 // add to set if anything was found
-                if (loaded.hasFqdnList()) recentFeedValues.put(feed.getFeedUrl(), loaded.getFqdnList());
+                if (loaded.hasFqdnList()) recentValues.put(feed.getFeedUrl(), loaded.getPassthruFqdnList());
             }
         }
-        for (String val : recentFeedValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet())) {
+        for (String val : recentValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet())) {
             set.add(new TlsPassthruMatcher(val));
         }
-        if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", "));
         return set;
     }
 
     public TlsPassthruFeed loadFeed(String url) {
-        final TlsPassthruFeed loaded = new TlsPassthruFeed().setFeedUrl(url);
+        final TlsPassthruFeed loaded = new TlsPassthruFeed().setPassthruFeedUrl(url);
         try (final InputStream in = getUrlInputStream(url)) {
             final List<String> lines = StringUtil.split(IOUtils.toString(in), "\r\n");
             final Set<String> fqdnList = new HashSet<>();
@@ -139,8 +209,8 @@ public class TlsPassthruConfig {
                 if (trimmed.startsWith("#")) {
                     if (!loaded.hasFeedName() && trimmed.toLowerCase().startsWith(FEED_NAME_PREFIX.toLowerCase())) {
                         final String name = trimmed.substring(FEED_NAME_PREFIX.length()).trim();
-                        if (log.isDebugEnabled()) log.debug("loadFeed("+url+"): setting name="+name+" from special comment: "+trimmed);
-                        loaded.setFeedName(name);
+                        if (log.isTraceEnabled()) log.trace("loadFeed("+url+"): setting name="+name+" from special comment: "+trimmed);
+                        loaded.setPassthruFeedName(name);
                     } else {
                         if (log.isDebugEnabled()) log.debug("loadFeed("+url+"): ignoring comment: "+trimmed);
                     }
@@ -148,7 +218,7 @@ public class TlsPassthruConfig {
                     fqdnList.add(trimmed);
                 }
             }
-            loaded.setFqdnList(fqdnList);
+            loaded.setPassthruFqdnList(fqdnList);
         } catch (Exception e) {
             reportError("loadFeed("+url+"): "+shortError(e), e);
         }
@@ -162,4 +232,11 @@ public class TlsPassthruConfig {
         return false;
     }
 
+    public boolean isFlex(String fqdn) {
+        for (TlsPassthruMatcher match : getFlexSet()) {
+            if (match.matches(fqdn)) return true;
+        }
+        return false;
+    }
+
 }

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruFeed.java

@@ -4,41 +4,29 @@
  */
 package bubble.rule.passthru;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import lombok.EqualsAndHashCode;
-import lombok.Getter;
 import lombok.NoArgsConstructor;
-import lombok.Setter;
 import lombok.experimental.Accessors;
 
 import java.util.Set;
 
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
-import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
 
-@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of="feedUrl")
-public class TlsPassthruFeed implements Comparable<TlsPassthruFeed> {
+@NoArgsConstructor @Accessors(chain=true)
+public class TlsPassthruFeed extends BasePassthruFeed {
 
-    public static final TlsPassthruFeed[] EMPTY_FEEDS = new TlsPassthruFeed[0];
+    public static final TlsPassthruFeed[] EMPTY_PASSTHRU_FEEDS = new TlsPassthruFeed[0];
 
-    public String getId() { return sha256_hex(getFeedUrl()); }
-    public void setId(String id) {} // noop
+    public String getPassthruFeedName () { return getFeedName(); }
+    public TlsPassthruFeed setPassthruFeedName (String name) { return (TlsPassthruFeed) setFeedName(name); }
 
-    @Getter @Setter private String feedName;
-    public boolean hasFeedName() { return !empty(feedName); }
+    public String getPassthruFeedUrl () { return getFeedUrl(); }
+    public TlsPassthruFeed setPassthruFeedUrl (String url) { return (TlsPassthruFeed) setFeedUrl(url); }
 
-    @Getter @Setter private String feedUrl;
+    public Set<String> getPassthruFqdnList () { return getFqdnList(); }
+    public TlsPassthruFeed setPassthruFqdnList (Set<String> set) { return (TlsPassthruFeed) setFqdnList(set); }
 
-    @JsonIgnore @Getter @Setter private Set<String> fqdnList;
-    public boolean hasFqdnList () { return !empty(fqdnList); }
-
-    public TlsPassthruFeed(String url) { setFeedUrl(url); }
+    public TlsPassthruFeed(String url) { super(url); }
 
     public TlsPassthruFeed(TlsPassthruFeed feed) { copy(this, feed); }
 
-    @Override public int compareTo(TlsPassthruFeed o) {
-        return getFeedUrl().toLowerCase().compareTo(o.getFeedUrl().toLowerCase());
-    }
-
 }

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java

@@ -5,6 +5,9 @@
 package bubble.rule.passthru;
 
 import bubble.model.account.Account;
+import bubble.model.app.AppMatcher;
+import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
 import bubble.model.device.Device;
 import bubble.rule.AbstractAppRuleDriver;
 import bubble.service.stream.AppRuleHarness;
@@ -13,6 +16,9 @@ import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ArrayUtil;
 
+import java.util.Set;
+import java.util.stream.Collectors;
+
 import static org.cobbzilla.util.json.JsonUtil.json;
 
 @Slf4j
@@ -20,10 +26,40 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
 
     @Override public <C> Class<C> getConfigClass() { return (Class<C>) TlsPassthruConfig.class; }
 
+    @Override public Set<String> getPrimedFlexDomains() {
+        final TlsPassthruConfig passthruConfig = getRuleConfig();
+        return passthruConfig.getFlexDomains().stream()
+                .filter(d -> !d.startsWith("!"))
+                .collect(Collectors.toSet());
+    }
+
+    @Override public Set<String> getPrimedFlexExcludeDomains() {
+        final TlsPassthruConfig passthruConfig = getRuleConfig();
+        return passthruConfig.getFlexDomains().stream()
+                .filter(d -> d.startsWith("!"))
+                .map(d -> d.substring(1))
+                .collect(Collectors.toSet());
+    }
+
+    @Override public void init(JsonNode config,
+                               JsonNode userConfig,
+                               BubbleApp app,
+                               AppRule rule,
+                               AppMatcher matcher,
+                               Account account,
+                               Device device) {
+        super.init(config, userConfig, app, rule, matcher, account, device);
+
+        // refresh lists
+        final TlsPassthruConfig passthruConfig = getRuleConfig();
+        passthruConfig.getPassthruSet();
+        passthruConfig.getFlexSet();
+    }
+
     @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) {
         final TlsPassthruConfig passthruConfig = getRuleConfig();
         if (passthruConfig.isPassthru(fqdn) || passthruConfig.isPassthru(addr)) {
-            if (log.isDebugEnabled()) log.debug("checkConnection: returning passthru for fqdn/addr="+fqdn+"/"+addr);
+            if (log.isDebugEnabled()) log.debug("checkConnection: detected passthru for fqdn/addr="+fqdn+"/"+addr);
             return ConnectionCheckResponse.passthru;
         }
         if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr);
@@ -33,17 +69,17 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
     @Override public JsonNode upgradeRuleConfig(JsonNode sageRuleConfig, JsonNode localRuleConfig) {
         final TlsPassthruConfig sageConfig = json(sageRuleConfig, getConfigClass());
         final TlsPassthruConfig localConfig = json(sageRuleConfig, getConfigClass());
-        if (sageConfig.hasFqdnList()) {
-            for (String fqdn : sageConfig.getFqdnList()) {
-                if (!localConfig.hasFqdnList() || localConfig.hasFqdn(fqdn)) {
-                    localConfig.setFqdnList(ArrayUtil.append(localConfig.getFqdnList(), fqdn));
+        if (sageConfig.hasPassthruFqdnList()) {
+            for (String fqdn : sageConfig.getPassthruFqdnList()) {
+                if (!localConfig.hasPassthruFqdnList() || localConfig.hasPassthruFqdn(fqdn)) {
+                    localConfig.setPassthruFqdnList(ArrayUtil.append(localConfig.getPassthruFqdnList(), fqdn));
                 }
             }
         }
-        if (sageConfig.hasFeedList()) {
-            for (TlsPassthruFeed feed : sageConfig.getFeedList()) {
-                if (!localConfig.hasFeed(feed)) {
-                    localConfig.setFeedList(ArrayUtil.append(localConfig.getFeedList(), feed));
+        if (sageConfig.hasPassthruFeedList()) {
+            for (TlsPassthruFeed feed : sageConfig.getPassthruFeedList()) {
+                if (!localConfig.hasPassthruFeed(feed)) {
+                    localConfig.setPassthruFeedList(ArrayUtil.append(localConfig.getPassthruFeedList(), feed));
                 }
             }
         }

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -385,7 +385,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration
     // called after activation, because now thisNetwork will be defined
     public void refreshPublicSystemConfigs () {
         synchronized (publicSystemConfigs) { publicSystemConfigs.set(null); }
-        background(this::getPublicSystemConfigs);
+        background(this::getPublicSystemConfigs, "BubbleConfiguration.refreshPublicSystemConfigs");
     }
 
     public boolean paymentsEnabled () {

bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java

@@ -13,8 +13,9 @@ import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.CloudService;
 import bubble.server.BubbleConfiguration;
 import bubble.service.boot.SelfNodeService;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import bubble.service.cloud.NetworkMonitorService;
+import bubble.service.device.StandardFlexRouterService;
 import bubble.service.stream.AppDataCleaner;
 import bubble.service.stream.AppPrimerService;
 import lombok.extern.slf4j.Slf4j;
@@ -102,7 +103,7 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase<Bub
                 } catch (Exception e) {
                     die("onStart: error initializing driver for cloud: "+cloud.getName()+"/"+cloud.getUuid()+": "+shortError(e), e);
                 }
-//            background(() -> cloud.wireAndSetup(c));
+//            background(() -> cloud.wireAndSetup(c), "NodeInitializerListener.onStart.cloudInit);
             }
         }
 
@@ -112,7 +113,8 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase<Bub
             final BubbleNetwork thisNetwork = c.getThisNetwork();
             if (thisNetwork != null && thisNetwork.getInstallType() == AnsibleInstallType.node) {
                 c.getBean(AppPrimerService.class).primeApps();
-                c.getBean(DeviceIdService.class).initDeviceSecurityLevels();
+                c.getBean(StandardFlexRouterService.class).start();
+                c.getBean(DeviceService.class).initDeviceSecurityLevels();
                 c.getBean(AppDataCleaner.class).start();
             }
         }

bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java

@@ -102,7 +102,7 @@ public class NetworkKeysService {
                     log.error("Cannot delete tmp backup folder " + backupDir, e);
                 }
             }
-        });
+        }, "NetworkKeysService.startBackupDownload");
     }
 
     @NonNull public BackupPackagingStatus backupDownloadStatus(@NonNull final String keysCode) {

bubble-server/src/main/java/bubble/service/boot/ActivationService.java

@@ -226,7 +226,7 @@ public class ActivationService {
                 final Map<CrudOperation, Collection<Identifiable>> objects
                         = modelSetupService.setupModel(api, account, "manifest-defaults");
                 log.info("bootstrapThisNode: created default objects\n"+json(objects));
-            });
+            }, "ActivationService.bootstrapThisNode.createDefaultObjects");
         }
 
         return node;

bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java

@@ -161,7 +161,7 @@ public class StandardSelfNodeService implements SelfNodeService {
                               .booleanValue()) {
                     deviceDAO.refreshVpnUsers();
                 }
-            });
+            }, "StandardSelfNodeService.onStart.spareDevices");
         }
 
         // start RefundService if payments are enabled and this is a SageLauncher

bubble-server/src/main/java/bubble/service/cloud/GeoService.java

@@ -72,7 +72,7 @@ public class GeoService {
     }
 
     private final Map<String, Future<GeoLocation>> backgroundLookups = new ConcurrentHashMap<>();
-    private final ExecutorService backgroundLookupExec = fixedPool(5);
+    private final ExecutorService backgroundLookupExec = fixedPool(5, "GeoService.backgroundLookupExec");
 
     public GeoLocation locate (String accountUuid, String ip, boolean cacheOnly) {
         final String cacheKey = hashOf(accountUuid, ip);

bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java

@@ -209,7 +209,7 @@ public class NodeProgressMeter extends PipedOutputStream implements Runnable {
                 .setAccount(nn.getAccount())
                 .setMessageKey(METER_COMPLETED_OK)
                 .setPercent(100));
-        background(this::close);
+        background(this::close, "NodeProgressMeter.completed");
     }
 
     public NodeProgressMeter uncloseable() throws IOException {
@@ -225,7 +225,7 @@ public class NodeProgressMeter extends PipedOutputStream implements Runnable {
                 .setAccount(nn.getAccount())
                 .setMessageKey(METER_ERROR_CANCELED)
                 .setPercent(0));
-        background(this::close);
+        background(this::close, "NodeProgressMeter.cancel");
     }
 
     private class UncloseableNodeProgressMeter extends NodeProgressMeter {

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -150,7 +150,7 @@ public class StandardNetworkService implements NetworkService {
         String lock = nn.getLock();
         NodeProgressMeter progressMeter = null;
         final BubbleNetwork network = nn.getNetworkObject();
-        final ExecutorService backgroundJobs = DaemonThreadFactory.fixedPool(3);
+        final ExecutorService backgroundJobs = DaemonThreadFactory.fixedPool(3, "StandardNetworkService.backgroundJobs");
         boolean killNode = false;
         try {
             progressMeter = launchMonitor.getProgressMeter(nn);
@@ -883,7 +883,7 @@ public class StandardNetworkService implements NetworkService {
             } finally {
                 if (lock != null) unlockNetwork(networkUuid, lock);
             }
-        });
+        }, "StandardNetworkService.stopNetwork");
         return true;
     }
 

bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java

@@ -114,7 +114,7 @@ public class DatabaseFilterService {
                             ? new FullEntityIterator(configuration, network, readerError)
                             : new FilteredEntityIterator(configuration, account, network, node, planApps, readerError);
                 }
-            }.runInBackground(readerError::set);
+            }.runInBackground("RekeyReaderMain.reader", readerError::set);
 
             // start a RekeyWriter to pull objects from RekeyReader
             final AtomicReference<CommandResult> writeResult = new AtomicReference<>();

bubble-server/src/main/java/bubble/service/dbfilter/EntityIterator.java

@@ -51,7 +51,7 @@ public abstract class EntityIterator implements Iterator<Identifiable> {
 
     public EntityIterator(AtomicReference<Exception> error) {
         this.error = error;
-        this.thread = background(this::_iterate, this.error::set);
+        this.thread = background(this::_iterate, "EntityIterator", this.error::set);
     }
 
     @Override public boolean hasNext() {

bubble-server/src/main/java/bubble/service/dbfilter/FilteredEntityIterator.java

@@ -24,6 +24,7 @@ import bubble.model.cloud.BubbleNodeKey;
 import bubble.model.cloud.notify.ReceivedNotification;
 import bubble.model.cloud.notify.SentNotification;
 import bubble.model.device.Device;
+import bubble.model.device.FlexRouter;
 import bubble.server.BubbleConfiguration;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.dao.DAO;
@@ -39,14 +40,14 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 @Slf4j
 public class FilteredEntityIterator extends EntityIterator {
 
-    private static final List<Class<? extends Identifiable>> POST_COPY_ENTITIES = Arrays.asList(new Class[] {
+    private static final List<Class<? extends Identifiable>> NO_DEFAULT_COPY_ENTITIES = Arrays.asList(new Class[] {
         BubbleNode.class, BubbleNodeKey.class, Device.class, AccountMessage.class,
         ReferralCode.class, AccountPayment.class, Bill.class, Promotion.class,
-        ReceivedNotification.class, SentNotification.class, TrustedClient.class
+        ReceivedNotification.class, SentNotification.class, TrustedClient.class, FlexRouter.class
     });
 
-    private static boolean isPostCopyEntity(Class<? extends Identifiable> clazz) {
-        return POST_COPY_ENTITIES.stream().anyMatch(c -> c.isAssignableFrom(clazz));
+    private static boolean isNotDefaultCopyEntity(Class<? extends Identifiable> clazz) {
+        return NO_DEFAULT_COPY_ENTITIES.stream().anyMatch(c -> c.isAssignableFrom(clazz));
     }
 
     private final BubbleConfiguration configuration;
@@ -82,10 +83,10 @@ public class FilteredEntityIterator extends EntityIterator {
         configuration.getEntityClasses().forEach(c -> {
             final DAO dao = configuration.getDaoForEntityClass(c);
             if (!AccountOwnedEntityDAO.class.isAssignableFrom(dao.getClass())) {
-                log.debug("iterate: skipping entity: " + c.getSimpleName());
-            } else if (isPostCopyEntity(c)) {
+                log.debug("iterate: skipping entity, not an AccountOwnedEntityDAO: " + c.getSimpleName());
+            } else if (isNotDefaultCopyEntity(c)) {
                 log.debug("iterate: skipping " + c.getSimpleName()
-                          + ", will copy some of these after other objects are copied");
+                          + ", may copy some of these after default objects are copied");
             } else {
                 // copy entities. this is how the re-keying works (decrypt using current spring config,
                 // encrypt using new config)

bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java → bubble-server/src/main/java/bubble/service/device/DeviceService.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
  * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
  */
-package bubble.service.cloud;
+package bubble.service.device;
 
 import bubble.model.account.Account;
 import bubble.model.device.Device;
@@ -10,7 +10,7 @@ import bubble.model.device.DeviceStatus;
 
 import java.util.List;
 
-public interface DeviceIdService {
+public interface DeviceService {
 
     Device findDeviceByIp(String ip);
 
@@ -19,7 +19,7 @@ public interface DeviceIdService {
     void initDeviceSecurityLevels();
     void setDeviceSecurityLevel(Device device);
 
-    void initBlockStats (Account account);
+    void initBlocksAndFlexRoutes(Account account);
     default boolean doShowBlockStats(String accountUuid) { return false; }
     default Boolean doShowBlockStatsForIpAndFqdn(String ip, String fqdn) { return false; }
     default void setBlockStatsForFqdn (Account account, String fqdn, boolean value) {}

bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java

@@ -0,0 +1,95 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.service.device;
+
+import bubble.cloud.geoLocation.GeoLocation;
+import bubble.model.account.Account;
+import bubble.model.device.Device;
+import bubble.model.device.DeviceStatus;
+import bubble.model.device.FlexRouter;
+import bubble.service.message.MessageService;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.github.jknack.handlebars.Handlebars;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+import org.cobbzilla.util.handlebars.HandlebarsUtil;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static bubble.model.device.DeviceStatus.NO_DEVICE_STATUS;
+import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
+import static org.cobbzilla.util.json.JsonUtil.json;
+
+@Accessors(chain=true) @ToString
+public class FlexRouterInfo {
+
+    @JsonIgnore @Getter private final FlexRouter router;
+    @JsonIgnore @Getter private final DeviceStatus deviceStatus;
+    @Getter @Setter private String auth;
+
+    // set by missingFlexRouter method when there is no flex router but there should be one
+    @Getter @Setter private String error_html;
+
+    public FlexRouterInfo (FlexRouter router, DeviceStatus deviceStatus) {
+        this.router = router;
+        this.deviceStatus = deviceStatus;
+    }
+
+    @JsonIgnore public String getVpnIp () { return router.getIp(); }
+
+    public int getPort () { return router == null ? -1 : router.getPort(); }
+    public String getProxyUrl () { return router == null ? null  : router.proxyBaseUri(); }
+
+    public boolean hasGeoLocation () { return hasDeviceStatus() && deviceStatus.getLocation() != null && deviceStatus.getLocation().hasLatLon(); }
+    public boolean hasNoGeoLocation () { return !hasGeoLocation(); }
+
+    public boolean hasDeviceStatus () { return deviceStatus != NO_DEVICE_STATUS && deviceStatus.hasIp(); }
+    public boolean hasNoDeviceStatus () { return !hasDeviceStatus(); }
+
+    public double distance(GeoLocation geoLocation) {
+        return hasNoGeoLocation() ? Double.MAX_VALUE : deviceStatus.getLocation().distance(geoLocation);
+    }
+
+    public boolean hasIp () { return hasDeviceStatus() && deviceStatus.hasIp(); }
+    public String ip () { return hasIp() ? deviceStatus.getIp() : null; }
+
+    public FlexRouterInfo initAuth () { auth = json(router.pingObject(), COMPACT_MAPPER); return this; }
+
+    @Override public int hashCode() { return getPort(); }
+
+    @Override public boolean equals(Object obj) {
+        return obj instanceof FlexRouterInfo && ((FlexRouterInfo) obj).getPort() == getPort();
+    }
+
+    public static final String CTX_ACCOUNT = "account";
+    public static final String CTX_DEVICE = "device";
+    public static final String CTX_MESSAGES = "messages";
+    public static final String CTX_FLEX_FQDN = "flex_fqdn";
+    public static final String CTX_DEVICE_TYPE_LABEL = "device_type_label";
+
+    public static FlexRouterInfo missingFlexRouter(Account account,
+                                                   Device device,
+                                                   String fqdn,
+                                                   MessageService messageService,
+                                                   Handlebars handlebars) {
+        final String locale = account.getLocale();
+        final String template = messageService.loadPageTemplate(locale, "no_flex_router");
+        final Map<String, Object> ctx = new HashMap<>();
+        ctx.put(CTX_ACCOUNT, account);
+        ctx.put(CTX_DEVICE, device);
+        ctx.put(CTX_FLEX_FQDN, fqdn);
+
+        final Map<String, String> messages = messageService.formatStandardMessages(locale);
+        ctx.put(CTX_MESSAGES, messages);
+        ctx.put(CTX_DEVICE_TYPE_LABEL, messages.get("device_type_"+device.getDeviceType().name()));
+
+        final String html = HandlebarsUtil.apply(handlebars, template, ctx);
+        return new FlexRouterInfo(null, null).setError_html(html);
+    }
+
+}

bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.service.device;
+
+import bubble.cloud.geoLocation.GeoLocation;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Comparator;
+
+@AllArgsConstructor @Slf4j
+public class FlexRouterProximityComparator implements Comparator<FlexRouterInfo> {
+
+    private final GeoLocation geoLocation;
+    private final String preferredIp;
+
+    @Override public int compare(FlexRouterInfo r1, FlexRouterInfo r2) {
+
+        // if preferred ip matches, that takes precedence over everything
+        if (r1.getVpnIp().equals(preferredIp)) return Integer.MIN_VALUE;
+        if (r2.getVpnIp().equals(preferredIp)) return Integer.MAX_VALUE;
+
+        // if a router has no location info, it goes last
+        if (r1.hasNoGeoLocation()) return Integer.MAX_VALUE;
+        if (r2.hasNoGeoLocation()) return Integer.MIN_VALUE;
+
+        // if WE have no location info, just compare ports (we choose randomly)
+        if (geoLocation == null) return r1.getPort() - r2.getPort();
+
+        // compare distances. if they are equals, just compare ports (we choose randomly)
+        final double distance1 = r1.distance(geoLocation);
+        final double distance2 = r2.distance(geoLocation);
+        final int delta = (int) (1000.0d * (distance1 - distance2));
+        return delta != 0 ? delta : r1.getPort() - r2.getPort();
+    }
+
+}

bubble-server/src/main/java/bubble/service/device/FlexRouterService.java

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.service.device;
+
+import bubble.model.device.FlexRouter;
+
+public interface FlexRouterService {
+
+    default void register (FlexRouter router) {}
+    default void unregister (FlexRouter router) {}
+    default void interruptSoon () {}
+
+}

bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.service.device;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+import static bubble.ApiConstants.enumFromString;
+
+public enum FlexRouterStatus {
+
+    none, active, unreachable, deleted;
+
+    @JsonCreator public static FlexRouterStatus fromString (String v) { return enumFromString(FlexRouterStatus.class, v); }
+
+}

bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java → bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java

@@ -2,7 +2,7 @@
  * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
  * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
  */
-package bubble.service.cloud;
+package bubble.service.device;
 
 import bubble.dao.account.AccountDAO;
 import bubble.dao.app.AppSiteDAO;
@@ -12,14 +12,13 @@ import bubble.model.app.AppSite;
 import bubble.model.device.Device;
 import bubble.model.device.DeviceStatus;
 import bubble.server.BubbleConfiguration;
+import bubble.service.cloud.GeoService;
 import bubble.service.stream.StandardRuleEngineService;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ExpirationMap;
 import org.cobbzilla.util.collection.SingletonList;
 import org.cobbzilla.util.io.FileUtil;
 import org.cobbzilla.util.io.FilenamePrefixFilter;
-import org.cobbzilla.util.network.NetworkUtil;
-import org.cobbzilla.util.string.StringUtil;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -30,9 +29,9 @@ import java.net.InetAddress;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static bubble.ApiConstants.HOME_DIR;
+import static bubble.ApiConstants.getPrivateIp;
 import static bubble.model.device.DeviceStatus.NO_DEVICE_STATUS;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
@@ -40,7 +39,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 import static org.cobbzilla.wizard.server.RestServerBase.reportError;
 
 @Service @Slf4j
-public class StandardDeviceIdService implements DeviceIdService {
+public class StandardDeviceService implements DeviceService {
 
     public static final File WG_DEVICES_DIR = new File(HOME_DIR, "wg_devices");
 
@@ -57,6 +56,9 @@ public class StandardDeviceIdService implements DeviceIdService {
     // used in dnscrypt-proxy to determine how to respond to blocked requests
     public static final String REDIS_KEY_DEVICE_REJECT_WITH = "bubble_device_reject_with_";
 
+    // used in dnscrypt-proxy to determine how to respond to flex routed requests
+    public static final String REDIS_KEY_DEVICE_FLEX_WITH = "bubble_device_flex_with_";
+
     // used in mitmproxy to determine how to respond to blocked requests
     public static final String REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = "bubble_device_showBlockStats_";
 
@@ -161,7 +163,7 @@ public class StandardDeviceIdService implements DeviceIdService {
         }
     }
 
-    @Override public void initBlockStats (Account account) {
+    @Override public void initBlocksAndFlexRoutes(Account account) {
         final boolean showBlockStats = configuration.showBlockStatsSupported() && account.showBlockStats();
         redis.set_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS+account.getUuid(), Boolean.toString(showBlockStats));
         redis.del_matching_withPrefix(REDIS_KEY_CHUNK_FILTER_PASS+"*");
@@ -171,6 +173,7 @@ public class StandardDeviceIdService implements DeviceIdService {
             } else {
                 hideBlockStats(device);
             }
+            initFlexRoutes(device);
         }
     }
 
@@ -192,15 +195,8 @@ public class StandardDeviceIdService implements DeviceIdService {
     }
 
     public void showBlockStats (Device device) {
-        final Set<String> configuredIps = NetworkUtil.configuredIps();
-        final String privateIp = configuredIps.stream()
-                .filter(addr -> addr.startsWith("10."))
-                .findFirst()
-                .orElse(null);
-        if (privateIp == null) {
-            log.error("showBlockStats: no system private IP found, configuredIps="+StringUtil.toString(configuredIps));
-            return;
-        }
+        final String privateIp = getPrivateIp();
+        if (privateIp == null) return;
         for (String ip : findIpsByDevice(device.getUuid())) {
             redis.set_plaintext(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip, Boolean.toString(true));
             redis.set_plaintext(REDIS_KEY_DEVICE_REJECT_WITH + ip, privateIp);
@@ -214,6 +210,14 @@ public class StandardDeviceIdService implements DeviceIdService {
         }
     }
 
+    public void initFlexRoutes (Device device) {
+        final String privateIp = getPrivateIp();
+        if (privateIp == null) return;
+        for (String ip : findIpsByDevice(device.getUuid())) {
+            redis.set_plaintext(REDIS_KEY_DEVICE_FLEX_WITH + ip, privateIp);
+        }
+    }
+
     @Override public void setBlockStatsForFqdn(Account account, String fqdn, boolean value) {
         for (Device device : deviceDAO.findByAccount(account.getUuid())) {
             for (String ip : findIpsByDevice(device.getUuid())) {

bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java

@@ -0,0 +1,294 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.service.device;
+
+import bubble.cloud.geoLocation.GeoLocation;
+import bubble.dao.device.FlexRouterDAO;
+import bubble.model.device.DeviceStatus;
+import bubble.model.device.FlexRouter;
+import bubble.model.device.FlexRouterPing;
+import bubble.service.cloud.GeoService;
+import lombok.Cleanup;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.cobbzilla.util.collection.SingletonSet;
+import org.cobbzilla.util.daemon.AwaitResult;
+import org.cobbzilla.util.daemon.SimpleDaemon;
+import org.cobbzilla.util.http.HttpRequestBean;
+import org.cobbzilla.util.http.HttpResponseBean;
+import org.cobbzilla.util.http.HttpUtil;
+import org.cobbzilla.util.io.FileUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static bubble.ApiConstants.HOME_DIR;
+import static bubble.model.device.FlexRouterPing.MAX_PING_AGE;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.cobbzilla.util.daemon.Await.awaitAll;
+import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.http.HttpMethods.POST;
+import static org.cobbzilla.util.io.FileUtil.*;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.string.StringUtil.EMPTY_ARRAY;
+import static org.cobbzilla.util.system.CommandShell.chmod;
+import static org.cobbzilla.util.system.Sleep.sleep;
+
+@Service @Slf4j
+public class StandardFlexRouterService extends SimpleDaemon implements FlexRouterService {
+
+    public static final int MAX_PING_TRIES = 5;
+
+    private static final long PING_SLEEP_FACTOR = SECONDS.toMillis(2);
+
+    // HttpClient timeouts are in seconds
+    public static final int DEFAULT_PING_TIMEOUT = (int) SECONDS.toSeconds(MAX_PING_AGE/2);
+    public static final RequestConfig DEFAULT_PING_REQUEST_CONFIG = RequestConfig.custom()
+            .setConnectTimeout(DEFAULT_PING_TIMEOUT)
+            .setSocketTimeout(DEFAULT_PING_TIMEOUT)
+            .setConnectionRequestTimeout(DEFAULT_PING_TIMEOUT).build();
+
+    // wait for ssh key to be written
+    private static final long FIRST_TIME_WAIT = SECONDS.toMillis(10);
+    private static final long INTERRUPT_WAIT = FIRST_TIME_WAIT/2;
+
+    public static final long PING_ALL_TIMEOUT
+            = (SECONDS.toMillis(1) * DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + FIRST_TIME_WAIT;
+
+    // thread pool size
+    public static final int DEFAULT_MAX_TUNNELS = 5;
+
+    private static CloseableHttpClient getHttpClient() {
+        return HttpClientBuilder.create()
+                .setDefaultRequestConfig(DEFAULT_PING_REQUEST_CONFIG)
+                .build();
+    }
+
+    public static final long DEFAULT_SLEEP_TIME = MINUTES.toMillis(2);
+
+    @Autowired private FlexRouterDAO flexRouterDAO;
+    @Autowired private GeoService geoService;
+    @Autowired private DeviceService deviceService;
+
+    private final AtomicBoolean interrupted = new AtomicBoolean(false);
+
+    @Override public void onStart() {
+        flexRouterDAO.findEnabledAndRegistered().forEach(this::register);
+        super.onStart();
+    }
+
+    @Override protected boolean canInterruptSleep() { return true; }
+
+    @Override protected long getSleepTime() { return interrupted.get() ? 0 : DEFAULT_SLEEP_TIME; }
+
+    @Override public void interruptSoon() {
+        if (log.isTraceEnabled()) log.trace("interruptSoon: will interrupt in "+INTERRUPT_WAIT+" millis");
+        synchronized (interrupted) {
+            if (interrupted.get()) {
+                if (log.isTraceEnabled()) log.trace("interruptSoon: interrupt flag already set, not setting again");
+            }
+            interrupted.set(true);
+        }
+        background(() -> {
+            sleep(INTERRUPT_WAIT);
+            if (log.isTraceEnabled()) log.trace("interruptSoon: interrupting...");
+            interrupt();
+        }, "StandardFlexRouterService.interruptSoon");
+    }
+
+    public void register (FlexRouter router) { allowFlexKey(router.getKey()); }
+
+    public void unregister (FlexRouter router) {
+        // todo: kill tunnel process if running
+        disallowFlexKey(router.getKey());
+    }
+
+    private final Map<String, FlexRouterStatus> statusMap = new ConcurrentHashMap<>(DEFAULT_MAX_TUNNELS);
+    private final Map<String, FlexRouterInfo> activeRouters = new ConcurrentHashMap<>(DEFAULT_MAX_TUNNELS);
+
+    public FlexRouterStatus status(String uuid) {
+        final FlexRouterStatus stat = statusMap.get(uuid);
+        if (stat == FlexRouterStatus.unreachable) interruptSoon();
+        return stat == null ? FlexRouterStatus.none : stat;
+    }
+
+    public FlexRouterStatus setStatus(FlexRouter router, FlexRouterStatus status) {
+        statusMap.put(router.getUuid(), status);
+        if (status == FlexRouterStatus.active && router.initialized()) {
+            final FlexRouterInfo info = activeRouters.get(router.getUuid());
+            if (info == null || info.hasNoDeviceStatus()) {
+                try {
+                    final DeviceStatus deviceStatus = deviceService.getDeviceStatus(router.getDevice());
+                    activeRouters.put(router.getUuid(), new FlexRouterInfo(router, deviceStatus));
+                } catch (Exception e) {
+                    log.error("setStatus: error creating FlexRouterInfo: "+shortError(e));
+                }
+            }
+        } else {
+            activeRouters.remove(router.getUuid());
+        }
+        return status;
+    }
+
+    public Set<FlexRouterInfo> selectClosestRouter (String accountUuid, String publicIp, String vpnIp) {
+        final GeoLocation geoLocation = publicIp == null ? null : geoService.locate(accountUuid, publicIp);
+        final Collection<FlexRouterInfo> values = activeRouters.values();
+        switch (values.size()) {
+            case 0: return Collections.emptySet();
+            case 1: return new SingletonSet<>(values.iterator().next());
+            default:
+                final Set<FlexRouterInfo> candidates = new TreeSet<>(new FlexRouterProximityComparator(geoLocation, vpnIp));
+                candidates.addAll(values);
+                return candidates;
+        }
+    }
+
+    @Override protected void process() {
+        synchronized (interrupted) { interrupted.set(false); }
+        try {
+            @Cleanup final CloseableHttpClient httpClient = getHttpClient();
+            final List<FlexRouter> routers = flexRouterDAO.findEnabledAndRegistered();
+            if (log.isTraceEnabled()) log.trace("process: starting, will ping "+routers.size()+" routers");
+            final List<Future<?>> futures = new ArrayList<>();
+            @Cleanup("shutdownNow") final ExecutorService exec = fixedPool(DEFAULT_MAX_TUNNELS, "StandardFlexRouterService.process");
+            for (FlexRouter router : routers) {
+                futures.add(exec.submit(() -> {
+                    final long firstTimeDelay = now() - router.getCtime();
+                    if (firstTimeDelay < FIRST_TIME_WAIT) {
+                        sleep(FIRST_TIME_WAIT - firstTimeDelay, "process: waiting for flex ssh key");
+                    }
+                    boolean active = pingFlexRouter(router, httpClient);
+                    if (active != router.active() || (active && router.uninitialized())) {
+                        if (active && router.uninitialized()) {
+                            router.setInitialized(true);
+                        }
+                        router.setActive(active);
+                        flexRouterDAO.update(router);
+                    }
+                    return active;
+                }));
+            }
+            final AwaitResult<Boolean> awaitResult = awaitAll(futures, PING_ALL_TIMEOUT);
+            if (log.isTraceEnabled()) log.trace("process: awaitResult="+awaitResult);
+
+        } catch (Exception e) {
+            log.error("process: "+shortError(e));
+        }
+    }
+
+    public boolean pingFlexRouter(FlexRouter router, HttpClient httpClient) {
+        allowFlexKey(router.getKey());
+        final String pingUrl = router.proxyBaseUri() + "/ping";
+        final HttpRequestBean request = new HttpRequestBean(POST, pingUrl);
+        final String prefix = "pingRouter(" + router + "): ";
+        for (int i=0; i<MAX_PING_TRIES; i++) {
+            sleep(PING_SLEEP_FACTOR * i, "waiting to ping flexRouter");
+            if (i == 0) {
+                if (log.isTraceEnabled()) log.trace(prefix+"pinging router at "+pingUrl+" ...");
+            } else {
+                final FlexRouter existing = flexRouterDAO.findByUuid(router.getUuid());
+                if (existing == null) {
+                    log.error(prefix+"router no longer exists: "+router.getUuid());
+                    setStatus(router, FlexRouterStatus.deleted);
+                    return false;
+                } else {
+                    router = existing;
+                }
+                if (log.isWarnEnabled()) log.warn(prefix+"attempting to ping again (try="+(i+1)+"/"+MAX_PING_TRIES+")");
+            }
+            try {
+                request.setEntity(json(router.pingObject()));
+                final HttpResponseBean response = HttpUtil.getResponse(request, httpClient);
+                if (!response.isOk()) {
+                    log.error(prefix+"response not OK: "+response);
+                } else {
+                    final FlexRouterPing pong = response.getEntity(FlexRouterPing.class);
+                    if (pong.validate(router)) {
+                        // emit message if loglevel is tracing, or info message if router was previously inactive
+                        if (log.isTraceEnabled() || (router.inactive() && log.isInfoEnabled())) log.trace(prefix+"router is ok");
+                        setStatus(router, FlexRouterStatus.active);
+                        return true;
+                    } else {
+                        log.error(prefix+"pong response was invalid");
+                    }
+                }
+
+            } catch (Exception e) {
+                log.warn(prefix+"error: "+shortError(e));
+            }
+            setStatus(router, FlexRouterStatus.unreachable);
+        }
+        log.error(prefix+"error: router failed after "+MAX_PING_TRIES+" attempts, returning false");
+        return false;
+    }
+
+    public synchronized void allowFlexKey(String key) {
+        final String trimmedKey = key.trim();
+        final File authFile = getAuthFile();
+        final String authFileContents = FileUtil.toStringOrDie(authFile);
+        final String[] lines = authFileContents == null ? EMPTY_ARRAY : authFileContents.split("\n");
+        if (Arrays.stream(lines).anyMatch(line -> line.trim().equals(trimmedKey))) {
+            if (log.isDebugEnabled()) log.debug("allowKey: already present: "+trimmedKey);
+        } else {
+            @Cleanup("delete") final File temp = temp("flex_keys_", ".tmp");
+            final String dataToWrite = authFileContents != null && authFileContents.endsWith("\n") ? trimmedKey + "\n" : "\n" + trimmedKey + "\n";
+            toFileOrDie(temp, dataToWrite, true);
+            renameOrDie(temp, authFile);
+            if (log.isInfoEnabled()) log.info("allowKey: added key: "+trimmedKey);
+        }
+    }
+
+    public synchronized void disallowFlexKey(String key) {
+        final String trimmedKey = key.trim();
+        final File authFile = getAuthFile();
+        final String authFileContents = FileUtil.toStringOrDie(authFile);
+        final String[] lines = authFileContents == null ? EMPTY_ARRAY : authFileContents.split("\n");
+        final StringBuilder b = new StringBuilder();
+        boolean found = false;
+        for (String line : lines) {
+            if (b.length() > 0) b.append("\n");
+            if (line.trim().equals(trimmedKey)) {
+                found = true;
+            } else {
+                b.append(line);
+            }
+        }
+        b.append("\n");
+        @Cleanup("delete") final File temp = temp("flex_keys_", ".tmp");
+        toFileOrDie(temp, b.toString());
+        renameOrDie(temp, authFile);
+        if (found) {
+            if (log.isInfoEnabled()) log.info("disallowKey: removed key from authorized_keys file: "+trimmedKey);
+        } else {
+            if (log.isInfoEnabled()) log.info("disallowKey: key was not found in authorized_keys file: "+trimmedKey);
+        }
+    }
+
+    private static File getAuthFile() {
+        final File sshDir = new File(HOME_DIR+"/.ssh");
+        if (!sshDir.exists()) {
+            mkdirOrDie(sshDir);
+        }
+        chmod(sshDir, "700");
+        final File authFile = new File(sshDir, "flex_authorized_keys");
+        if (!authFile.exists()) {
+            touch(authFile);
+        }
+        chmod(authFile, "600");
+        return authFile;
+    }
+
+}

bubble-server/src/main/java/bubble/service/message/MessageService.java

@@ -19,13 +19,15 @@ import java.util.concurrent.ConcurrentHashMap;
 
 import static bubble.ApiConstants.MESSAGE_RESOURCE_BASE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
-import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream;
+import static org.cobbzilla.util.io.StreamUtil.*;
 import static org.cobbzilla.util.string.StringUtil.UTF8cs;
 
 @Service @Slf4j
 public class MessageService {
 
     public static final String MESSAGE_RESOURCE_PATH = "/server/";
+    public static final String PAGE_TEMPLATES_PATH = "pages/";
+    public static final String PAGE_TEMPLATES_SUFFIX = ".html.hbs";
     public static final String RESOURCE_MESSAGES_PROPS = "ResourceMessages.properties";
 
     public static final String[] PRE_AUTH_MESSAGE_GROUPS = {"pre_auth", "countries", "timezones"};
@@ -81,4 +83,13 @@ public class MessageService {
         });
     }
 
+    private final Map<String, String> pageTemplateCache = new ConcurrentHashMap<>(10);
+
+    public String loadPageTemplate(String locale, String templatePath) {
+        final String key = locale + ":" + templatePath;
+        return pageTemplateCache.computeIfAbsent(key, k -> {
+            final String path = MESSAGE_RESOURCE_BASE + locale + MESSAGE_RESOURCE_PATH + PAGE_TEMPLATES_PATH + templatePath + PAGE_TEMPLATES_SUFFIX;
+            return loadResourceAsStringOrDie(path);
+        });
+    }
 }

bubble-server/src/main/java/bubble/service/packer/PackerService.java

@@ -44,7 +44,7 @@ public class PackerService {
 
     private final Map<String, PackerJob> activeJobs = new ConcurrentHashMap<>(16);
     private final Map<String, List<PackerImage>> completedJobs = new ConcurrentHashMap<>(16);
-    private final ExecutorService pool = DaemonThreadFactory.fixedPool(5);
+    private final ExecutorService pool = DaemonThreadFactory.fixedPool(5, "PackerService.pool");
 
     @Autowired private BubbleConfiguration configuration;
 

bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java

@@ -11,5 +11,6 @@ import static java.util.Collections.emptyMap;
 public interface RuleEngineService {
 
     default Map<Object, Object> flushCaches() { return emptyMap(); }
+    default Map<Object, Object> flushCaches(boolean prime) { return emptyMap(); }
 
 }

bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java

@@ -14,7 +14,7 @@ import bubble.model.cloud.BubbleNetwork;
 import bubble.model.device.Device;
 import bubble.rule.AppRuleDriver;
 import bubble.server.BubbleConfiguration;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.SingletonList;
@@ -34,7 +34,7 @@ public class StandardAppPrimerService implements AppPrimerService {
 
     @Autowired private AccountDAO accountDAO;
     @Autowired private DeviceDAO deviceDAO;
-    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private DeviceService deviceService;
     @Autowired private BubbleAppDAO appDAO;
     @Autowired private AppMatcherDAO matcherDAO;
     @Autowired private AppRuleDAO ruleDAO;
@@ -77,7 +77,7 @@ public class StandardAppPrimerService implements AppPrimerService {
     }
 
     public void prime(Account account) {
-        deviceIdService.initBlockStats(account);
+        deviceService.initBlocksAndFlexRoutes(account);
         prime(account, (BubbleApp) null);
     }
 
@@ -99,7 +99,7 @@ public class StandardAppPrimerService implements AppPrimerService {
         prime(account, singleApp);
     }
 
-    @Getter(lazy=true) private final ExecutorService primerThread = fixedPool(1);
+    @Getter(lazy=true) private final ExecutorService primerThread = fixedPool(1, "StandardAppPrimerService.primerThread");
 
     private void prime(Account account, BubbleApp singleApp) {
         if (!isPrimingEnabled()) {
@@ -114,7 +114,7 @@ public class StandardAppPrimerService implements AppPrimerService {
             final Map<String, List<String>> accountDeviceIps = new HashMap<>();
             final List<Device> devices = deviceDAO.findByAccount(account.getUuid());
             for (Device device : devices) {
-                accountDeviceIps.put(device.getUuid(), deviceIdService.findIpsByDevice(device.getUuid()));
+                accountDeviceIps.put(device.getUuid(), deviceService.findIpsByDevice(device.getUuid()));
             }
             if (accountDeviceIps.isEmpty()) return;
 
@@ -144,7 +144,10 @@ public class StandardAppPrimerService implements AppPrimerService {
                     for (Device device : devices) {
                         final Set<String> rejectDomains = new HashSet<>();
                         final Set<String> blockDomains = new HashSet<>();
+                        final Set<String> whiteListDomains = new HashSet<>();
                         final Set<String> filterDomains = new HashSet<>();
+                        final Set<String> flexDomains = new HashSet<>();
+                        final Set<String> flexExcludeDomains = new HashSet<>();
                         for (AppMatcher matcher : matchers) {
                             final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device);
                             final Set<String> rejects = appRuleDriver.getPrimedRejectDomains();
@@ -159,14 +162,32 @@ public class StandardAppPrimerService implements AppPrimerService {
                             } else {
                                 blockDomains.addAll(blocks);
                             }
+                            final Set<String> whiteList = appRuleDriver.getPrimedWhiteListDomains();
+                            if (empty(whiteList)) {
+                                log.debug("_prime: no whiteListDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                            } else {
+                                whiteListDomains.addAll(whiteList);
+                            }
                             final Set<String> filters = appRuleDriver.getPrimedFilterDomains();
                             if (empty(filters)) {
                                 log.debug("_prime: no filterDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
                             } else {
                                 filterDomains.addAll(filters);
                             }
+                            final Set<String> flexes = appRuleDriver.getPrimedFlexDomains();
+                            if (empty(flexes)) {
+                                log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                            } else {
+                                flexDomains.addAll(flexes);
+                            }
+                            final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
+                            if (empty(flexExcludes)) {
+                                log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                            } else {
+                                flexExcludeDomains.addAll(flexExcludes);
+                            }
                         }
-                        if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains)) {
+                        if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) {
                             for (String ip : accountDeviceIps.get(device.getUuid())) {
                                 if (!empty(rejectDomains)) {
                                     AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new));
@@ -174,9 +195,18 @@ public class StandardAppPrimerService implements AppPrimerService {
                                 if (!empty(blockDomains)) {
                                     AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new));
                                 }
+                                if (!empty(whiteListDomains)) {
+                                    AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), whiteListDomains.toArray(String[]::new));
+                                }
                                 if (!empty(filterDomains)) {
                                     AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new));
                                 }
+                                if (!empty(flexDomains)) {
+                                    AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new));
+                                }
+                                if (!empty(flexExcludeDomains)) {
+                                    AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new));
+                                }
                             }
                         }
                     }

bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java

@@ -193,7 +193,11 @@ public class StandardRuleEngineService implements RuleEngineService {
             = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime);
 
     private final AtomicBoolean cachedFlushingEnabled = new AtomicBoolean(true);
-    public void enableCacheFlushing () { cachedFlushingEnabled.set(true); }
+    public void enableCacheFlushing () {
+        if (log.isDebugEnabled()) log.debug("enableCacheFlushing: caching re-enabled, flushing");
+        cachedFlushingEnabled.set(true);
+        flushCaches(false);
+    }
     public void disableCacheFlushing () { cachedFlushingEnabled.set(false); }
 
     public Map<Object, Object> flushCaches() { return flushCaches(true); }

bubble-server/src/main/java/bubble/service/upgrade/AppUpgradeService.java

@@ -114,14 +114,7 @@ public class AppUpgradeService extends SimpleDaemon {
             return;
         }
 
-        try {
-            ruleEngine.disableCacheFlushing();
-            handleAdminUpgrades(admin, sageNode);
-
-        } finally {
-            ruleEngine.enableCacheFlushing();
-            ruleEngine.flushCaches();
-        }
+        handleAdminUpgrades(admin, sageNode);
     }
 
     private void handleAdminUpgrades(Account admin, BubbleNode sageNode) {
@@ -146,6 +139,8 @@ public class AppUpgradeService extends SimpleDaemon {
 
             final List<RuleDriver> myDrivers = Arrays.asList(upgradeRequest.getDrivers());
             accountDrivers.put(admin.getUuid(), myDrivers);
+
+            ruleEngine.disableCacheFlushing();
             for (RuleDriver sageDriver : sageDrivers) {
                 log.info("handleAdminUpgrades: updating admin driver: "+sageDriver.getName());
                 updateDriver(admin, myDrivers, sageDriver);
@@ -166,6 +161,9 @@ public class AppUpgradeService extends SimpleDaemon {
             }
         } catch (Exception e) {
             log.error("handleAdminUpgrades: "+shortError(e), e);
+
+        } finally {
+            ruleEngine.enableCacheFlushing();
         }
     }
 

bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java → bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java

@@ -7,7 +7,7 @@ package bubble.service_dbfilter;
 import bubble.model.account.Account;
 import bubble.model.device.Device;
 import bubble.model.device.DeviceStatus;
-import bubble.service.cloud.DeviceIdService;
+import bubble.service.device.DeviceService;
 import org.springframework.stereotype.Service;
 
 import java.util.List;
@@ -15,7 +15,7 @@ import java.util.List;
 import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported;
 
 @Service
-public class DbFilterDeviceIdService implements DeviceIdService {
+public class DbFilterDeviceService implements DeviceService {
 
     @Override public Device findDeviceByIp(String ip) { return notSupported("findDeviceByIp"); }
 
@@ -24,7 +24,7 @@ public class DbFilterDeviceIdService implements DeviceIdService {
     @Override public void initDeviceSecurityLevels() { notSupported("initDeviceSecurityLevels"); }
     @Override public void setDeviceSecurityLevel(Device device) { notSupported("setDeviceSecurityLevel"); }
 
-    @Override public void initBlockStats(Account account) { notSupported("initBlockStats"); }
+    @Override public void initBlocksAndFlexRoutes(Account account) { notSupported("initBlocksAndFlexRoutes"); }
 
     @Override public DeviceStatus getDeviceStatus(String deviceUuid) { return notSupported("getDeviceStats"); }
     @Override public DeviceStatus getLiveDeviceStatus(String deviceUuid) { return notSupported("getLiveDeviceStatus"); }

bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.service_dbfilter;
+
+import bubble.service.device.FlexRouterService;
+import org.springframework.stereotype.Service;
+
+@Service
+public class DbFilterFlexRouterService implements FlexRouterService {}

bubble-server/src/main/resources/META-INF/bubble/bubble.properties

@@ -1 +1 @@
-bubble.version=Adventure 1.0.7
+bubble.version=Adventure 1.1.0

bubble-server/src/main/resources/ansible/bubble_scripts.txt

@@ -18,4 +18,5 @@ cleanup_bubble_databases
 install_packer.sh
 rkeys
 rmembers
-rdelkeys
+rdelkeys
+mitm_pid

bubble-server/src/main/resources/db/migration/V2020090801__add_flex_router.sql

@@ -0,0 +1,30 @@
+CREATE TABLE public.flex_router (
+    uuid character varying(100) NOT NULL,
+    ctime bigint NOT NULL,
+    mtime bigint NOT NULL,
+    account character varying(100) NOT NULL,
+    active boolean NOT NULL,
+    device character varying(100) NOT NULL,
+    enabled boolean NOT NULL,
+    initialized boolean NOT NULL,
+    ip character varying(50) NOT NULL,
+    key character varying(10100) NOT NULL,
+    key_hash character varying(100) NOT NULL,
+    port integer NOT NULL,
+    token character varying(200) NOT NULL
+);
+
+ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_pkey PRIMARY KEY (uuid);
+
+CREATE INDEX flex_router_idx_account ON flex_router USING btree (account);
+CREATE INDEX flex_router_idx_device ON flex_router USING btree (device);
+CREATE INDEX flex_router_idx_active ON flex_router USING btree (active);
+CREATE INDEX flex_router_idx_enabled ON flex_router USING btree (enabled);
+CREATE INDEX flex_router_idx_initialized ON flex_router USING btree (initialized);
+CREATE INDEX flex_router_idx_ip ON flex_router USING btree (ip);
+CREATE UNIQUE INDEX flex_router_uniq_account_ip ON flex_router USING btree (account, ip);
+CREATE UNIQUE INDEX flex_router_uniq_key_hash ON flex_router USING btree (key_hash);
+CREATE UNIQUE INDEX flex_router_uniq_port ON flex_router USING btree (port);
+
+ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_fk_account FOREIGN KEY (account) REFERENCES account(uuid);
+ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_fk_device FOREIGN KEY (device) REFERENCES device(uuid);

bubble-server/src/main/resources/logback.xml

@@ -40,8 +40,11 @@
 <!--    <logger name="org.cobbzilla.wizard.dao.SqlViewSearchHelper" level="DEBUG" />-->
     <logger name="org.cobbzilla.wizard.server.listener.BrowserLauncherListener" level="INFO" />
     <logger name="bubble.service.notify.NotificationService" level="WARN" />
+    <logger name="bubble.service.device.StandardFlexRouterService" level="ERROR" />
     <logger name="bubble.rule.AbstractAppRuleDriver" level="WARN" />
     <logger name="bubble.rule.bblock.BubbleBlockRuleDriver" level="WARN" />
+<!--    <logger name="bubble.rule.passthru.TlsPassthruRuleDriver" level="DEBUG" />-->
+<!--    <logger name="bubble.rule.passthru.TlsPassthruConfig" level="DEBUG" />-->
     <logger name="bubble.service.block.BlockStatsService" level="WARN" />
     <!--    <logger name="bubble.service.block" level="DEBUG" />-->
     <logger name="bubble.abp" level="WARN" />
@@ -70,8 +73,6 @@
 <!--    <logger name="bubble.service.cloud.StandardDeviceIdService" level="DEBUG" />-->
 <!--    <logger name="bubble.cloud.compute.vultr" level="DEBUG" />-->
     <logger name="bubble.resources.message" level="INFO" />
-    <logger name="bubble.app.analytics" level="DEBUG" />
-    <logger name="bubble.app.passthru" level="DEBUG" />
     <logger name="org.cobbzilla.util.io.regex.RegexFilterReader" level="WARN" />
     <logger name="org.cobbzilla.util.io.multi" level="INFO" />
 <!--    <logger name="bubble.service.cloud.StandardNetworkService" level="INFO" />-->

bubble-server/src/main/resources/messages

@@ -1 +1 @@
-Subproject commit 4f8345542b29228db0dba3e845f0662ab9cf6693
+Subproject commit 04db22382ddffa08b3f05c024603da75cdfb8b55

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json

@@ -2,6 +2,12 @@
   "name": "BubbleBlock",
   "children": {
     "AppData": [{
+      "site": "All_Sites",
+      "template": true,
+      "matcher": "BubbleBlockMatcher",
+      "key": "hideStats_abercrombie.com",
+      "data": "true"
+    }, {
       "site": "All_Sites",
       "template": true,
       "matcher": "BubbleBlockMatcher",

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json

@@ -5,40 +5,69 @@
   "template": true,
   "enabled": true,
   "priority": 1000000,
-  "canPrime": false,
+  "canPrime": true,
   "dataConfig": {
     "dataDriver": "bubble.app.passthru.TlsPassthruAppDataDriver",
     "presentation": "none",
     "configDriver": "bubble.app.passthru.TlsPassthruAppConfigDriver",
     "configFields": [
       {"name": "passthruFqdn", "type": "hostname", "truncate": false},
-      {"name": "feedName", "truncate": false},
-      {"name": "feedUrl", "type": "http_url"}
+      {"name": "passthruFeedName", "truncate": false},
+      {"name": "passthruFeedUrl", "type": "http_url"},
+      {"name": "flexFqdn", "type": "hostname", "truncate": false},
+      {"name": "flexFeedName", "truncate": false},
+      {"name": "flexFeedUrl", "type": "http_url"}
     ],
     "configViews": [{
-      "name": "manageDomains",
+      "name": "managePassthruDomains",
       "scope": "app",
       "root": "true",
       "fields": ["passthruFqdn"],
       "actions": [
-        {"name": "removeFqdn", "index": 10},
+        {"name": "removePassthruFqdn", "index": 10},
         {
-          "name": "addFqdn", "scope": "app", "index": 10,
+          "name": "addPassthruFqdn", "scope": "app", "index": 10,
           "params": ["passthruFqdn"],
-          "button": "addFqdn"
+          "button": "addPassthruFqdn"
         }
       ]
     }, {
-      "name": "manageFeeds",
+      "name": "managePassthruFeeds",
       "scope": "app",
       "root": "true",
-      "fields": ["feedName", "feedUrl"],
+      "fields": ["passthruFeedName", "passthruFeedUrl"],
       "actions": [
-        {"name": "removeFeed", "index": 10},
+        {"name": "removePassthruFeed", "index": 10},
         {
-          "name": "addFeed", "scope": "app", "index": 10,
-          "params": ["feedUrl"],
-          "button": "addFeed"
+          "name": "addPassthruFeed", "scope": "app", "index": 10,
+          "params": ["passthruFeedUrl"],
+          "button": "addPassthruFeed"
+        }
+      ]
+    }, {
+      "name": "manageFlexDomains",
+      "scope": "app",
+      "root": "true",
+      "fields": ["flexFqdn"],
+      "actions": [
+        {"name": "removeFlexFqdn", "index": 10},
+        {
+          "name": "addFlexFqdn", "scope": "app", "index": 10,
+          "params": ["flexFqdn"],
+          "button": "addFlexFqdn"
+        }
+      ]
+    }, {
+      "name": "manageFlexFeeds",
+      "scope": "app",
+      "root": "true",
+      "fields": ["flexFeedName", "flexFeedUrl"],
+      "actions": [
+        {"name": "removeFlexFeed", "index": 10},
+        {
+          "name": "addFlexFeed", "scope": "app", "index": 10,
+          "params": ["flexFeedUrl"],
+          "button": "addFlexFeed"
         }
       ]
     }]
@@ -56,9 +85,13 @@
       "driver": "TlsPassthruRuleDriver",
       "priority": -1000000,
       "config": {
-        "fqdnList": [],
-        "feedList": [{
-          "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/tls_passthru.txt"
+        "passthruFqdnList": [],
+        "passthruFeedList": [{
+          "passthruFeedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/tls_passthru.txt"
+        }],
+        "flexFqdnList": [],
+        "flexFeedList": [{
+          "flexFeedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/flex_routing.txt"
         }]
       }
     }],
@@ -70,19 +103,41 @@
         {"name": "summary", "value": "Network Bypass"},
         {"name": "description", "value": "Do not perform SSL interception for certificate-pinned domains"},
 
-        {"name": "config.view.manageDomains", "value": "Manage Bypass Domains"},
-        {"name": "config.view.manageFeeds", "value": "Manage Bypass Domain Feeds"},
+        {"name": "config.view.managePassthruDomains", "value": "Manage Bypass Domains"},
+        {"name": "config.view.managePassthruFeeds", "value": "Manage Bypass Domain Feeds"},
         {"name": "config.field.passthruFqdn", "value": "Domain"},
         {"name": "config.field.passthruFqdn.description", "value": "Bypass traffic interception for this hostname"},
-        {"name": "config.field.feedName", "value": "Name"},
-        {"name": "config.field.feedUrl", "value": "Bypass Domains List URL"},
-        {"name": "config.field.feedUrl.description", "value": "URL returning a list of bypass domains and/or hostnames, one per line"},
-        {"name": "config.action.addFqdn", "value": "Add New Bypass Domain"},
-        {"name": "config.button.addFqdn", "value": "Add"},
-        {"name": "config.action.removeFqdn", "value": "Remove"},
-        {"name": "config.action.addFeed", "value": "Add New Bypass Domain Feed"},
-        {"name": "config.button.addFeed", "value": "Add"},
-        {"name": "config.action.removeFeed", "value": "Remove"}
+        {"name": "config.field.passthruFeedName", "value": "Name"},
+        {"name": "config.field.passthruFeedUrl", "value": "Bypass Domains List URL"},
+        {"name": "config.field.passthruFeedUrl.description", "value": "URL returning a list of bypass domains and/or hostnames, one per line"},
+        {"name": "config.action.addPassthruFqdn", "value": "Add New Bypass Domain"},
+        {"name": "config.button.addPassthruFqdn", "value": "Add"},
+        {"name": "config.action.removePassthruFqdn", "value": "Remove"},
+        {"name": "config.action.addPassthruFeed", "value": "Add New Bypass Domain Feed"},
+        {"name": "config.button.addPassthruFeed", "value": "Add"},
+        {"name": "config.action.removePassthruFeed", "value": "Remove"},
+
+        {"name": "config.view.manageFlexDomains", "value": "Manage Flex Routing Domains"},
+        {"name": "config.view.manageFlexFeeds", "value": "Manage Flex Routing Domain Feeds"},
+        {"name": "config.field.flexFqdn", "value": "Domain"},
+        {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this domain and all subdomains. Prefix with ! to exclude from flex routing."},
+        {"name": "config.field.flexFeedName", "value": "Name"},
+        {"name": "config.field.flexFeedUrl", "value": "Flex Routing Domains List URL"},
+        {"name": "config.field.flexFeedUrl.description", "value": "URL returning a list of domains and/or hostnames to flex route, one per line"},
+        {"name": "config.action.addFlexFqdn", "value": "Add New Flex Routing Domain"},
+        {"name": "config.button.addFlexFqdn", "value": "Add"},
+        {"name": "config.action.removeFlexFqdn", "value": "Remove"},
+        {"name": "config.action.addFlexFeed", "value": "Add New Flex Routing Domain Feed"},
+        {"name": "config.button.addFlexFeed", "value": "Add"},
+        {"name": "config.action.removeFlexFeed", "value": "Remove"},
+
+        {"name": "err.passthruFqdn.passthruFqdnRequired", "value": "Domain or Hostname field is required"},
+        {"name": "err.passthruFeedUrl.feedUrlRequired", "value": "Feed URL is required"},
+        {"name": "err.passthruFeedUrl.emptyFqdnList", "value": "Feed URL was not found or contained no data"},
+
+        {"name": "err.flexFqdn.flexFqdnRequired", "value": "Domain or Hostname field is required"},
+        {"name": "err.flexFeedUrl.feedUrlRequired", "value": "Feed URL is required"},
+        {"name": "err.flexFeedUrl.emptyFqdnList", "value": "Feed URL was not found or contained no data"}
       ]
     }]
   }

bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml

@@ -13,7 +13,7 @@
   get_url:
     url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip
     dest: /tmp/algo.zip
-    checksum: sha256:44584be79375d94714f0e5c5772a76dee17eebb465f015685ff9df79f32fc809
+    checksum: sha256:af3e8856626248646ea496919b7bae5974e552e24a7603460e7eebc7f5c7f93f
 
 - name: Unzip algo master.zip
   unarchive:

bubble-server/src/main/resources/packer/roles/bubble/files/refresh_flex_keys_monitor.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+LOG=/var/log/bubble/flex_keys_monitor.log
+
+function die {
+  echo 1>&2 "${1}"
+  log "${1}"
+  exit 1
+}
+
+function log {
+  echo "$(date): ${1}" >> ${LOG}
+}
+
+SSH_KEY_BASE=/home/bubble-flex/.ssh
+if [[ ! -d ${SSH_KEY_BASE} ]] ; then
+  mkdir ${SSH_KEY_BASE}
+fi
+chown -R bubble-flex ${SSH_KEY_BASE} && chmod 700 ${SSH_KEY_BASE}
+
+BUBBLE_FLEX_KEYS=/home/bubble/.ssh/flex_authorized_keys
+AUTH_FLEX_KEYS=${SSH_KEY_BASE}/authorized_keys
+
+if [[ ! -f ${AUTH_FLEX_KEYS} ]] ; then
+  touch ${AUTH_FLEX_KEYS}
+fi
+chown bubble-flex ${AUTH_FLEX_KEYS} && chmod 600 ${AUTH_FLEX_KEYS}
+
+if [[ ! -f ${BUBBLE_FLEX_KEYS} ]] ; then
+  touch ${BUBBLE_FLEX_KEYS} && chown bubble ${BUBBLE_FLEX_KEYS} && chmod 600 ${BUBBLE_FLEX_KEYS} && sleep 2s
+fi
+
+log "Watching flex keys file ${BUBBLE_FLEX_KEYS} ..."
+while : ; do
+  if [[ $(stat -c %Y ${BUBBLE_FLEX_KEYS}) -gt $(stat -c %Y ${AUTH_FLEX_KEYS}) ]] ; then
+    cat ${BUBBLE_FLEX_KEYS} > ${AUTH_FLEX_KEYS} \
+      && log "Updated ${BUBBLE_FLEX_KEYS} > ${AUTH_FLEX_KEYS}" \
+      || log "Error overwriting ${BUBBLE_FLEX_KEYS} > ${AUTH_FLEX_KEYS}"
+    # Just for sanity's sake
+    chown -R bubble-flex ${SSH_KEY_BASE} && chmod 700 ${SSH_KEY_BASE}
+    chown bubble-flex ${AUTH_FLEX_KEYS} && chmod 600 ${AUTH_FLEX_KEYS}
+  fi
+  sleep 10s
+done

bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_flex_keys_monitor.conf

@@ -0,0 +1,5 @@
+
+[program:refresh_flex_keys_monitor]
+stdout_logfile = /dev/null
+stderr_logfile = /dev/null
+command=/usr/local/sbin/refresh_flex_keys_monitor.sh

bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml

@@ -1,7 +1,7 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
-- name: Install OpenJDK 11 JRE (headless), redis, uuid and jq
+- name: Install OpenJDK 11 JRE (headless), redis, uuid, jq, and zip
   apt:
     name: [ 'openjdk-11-jre-headless', 'redis', 'uuid', 'jq', 'zip' ]
     state: present
@@ -95,6 +95,7 @@
   with_items:
     - refresh_bubble_ssh_keys_monitor.sh
     - refresh_bubble_ssh_keys.sh
+    - refresh_flex_keys_monitor.sh
     - bubble_upgrade_monitor.sh
     - bubble_upgrade.sh
     - log_manager.sh
@@ -104,6 +105,11 @@
     src: supervisor_refresh_bubble_ssh_keys_monitor.conf
     dest: /etc/supervisor/conf.d/refresh_bubble_ssh_keys_monitor.conf
 
+- name: Install refresh_flex_keys_monitor supervisor conf file
+  copy:
+    src: supervisor_refresh_flex_keys_monitor.conf
+    dest: /etc/supervisor/conf.d/refresh_flex_keys_monitor.conf
+
 - name: Install bubble_upgrade_monitor supervisor conf file
   copy:
     src: supervisor_bubble_upgrade_monitor.conf

bubble-server/src/main/resources/packer/roles/common/tasks/main.yml

@@ -84,3 +84,12 @@
     group: bubble-log
     mode: 0770
     state: directory
+
+- name: Create bubble flexrouting user
+  user:
+    name: bubble-flex
+    comment: bubble flexrouting user
+    shell: /bin/false
+    system: yes
+    home: /home/bubble-flex
+  when: install_type == 'node'

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -1,34 +1,53 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
-import datetime
+import asyncio
 import json
+import logging
 import re
-import requests
-import redis
 import subprocess
-import sys
 import time
 import traceback
 import uuid
-from netaddr import IPAddress, IPNetwork
+from http import HTTPStatus
+from logging import INFO, DEBUG, WARNING, ERROR
+
+import httpx
+import nest_asyncio
+import redis
 from bubble_vpn4 import wireguard_network_ipv4
 from bubble_vpn6 import wireguard_network_ipv6
-from bubble_config import bubble_network, bubble_port, debug_capture_fqdn, \
-    bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host
+from netaddr import IPAddress, IPNetwork
+
+from bubble_config import bubble_port, debug_capture_fqdn, \
+    bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6
+from mitmproxy import http
+from mitmproxy.net.http import headers as nheaders
+
+bubble_log = logging.getLogger(__name__)
+
+nest_asyncio.apply()
 
 HEADER_USER_AGENT = 'User-Agent'
+HEADER_CONTENT_LENGTH = 'Content-Length'
+HEADER_CONTENT_TYPE = 'Content-Type'
+HEADER_CONTENT_ENCODING = 'Content-Encoding'
+HEADER_TRANSFER_ENCODING = 'Transfer-Encoding'
+HEADER_LOCATION = 'Location'
 HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy'
 HEADER_REFERER = 'Referer'
 HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru'
 
 CTX_BUBBLE_MATCHERS = 'X-Bubble-Matchers'
 CTX_BUBBLE_ABORT = 'X-Bubble-Abort'
+CTX_BUBBLE_SPECIAL = 'X-Bubble-Special'
 CTX_BUBBLE_LOCATION = 'X-Bubble-Location'
 CTX_BUBBLE_PASSTHRU = 'X-Bubble-Passthru'
 CTX_BUBBLE_REQUEST_ID = 'X-Bubble-RequestId'
 CTX_CONTENT_LENGTH = 'X-Bubble-Content-Length'
 CTX_CONTENT_LENGTH_SENT = 'X-Bubble-Content-Length-Sent'
+CTX_BUBBLE_FILTERED = 'X-Bubble-Filtered'
+CTX_BUBBLE_FLEX = 'X-Bubble-Flex'
 BUBBLE_URI_PREFIX = '/__bubble/'
 
 HEADER_HEALTH_CHECK = 'X-Mitm-Health'
@@ -39,8 +58,8 @@ BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_'
 BUBBLE_ACTIVITY_LOG_EXPIRATION = 600
 
 LOCAL_IPS = []
-for ip in subprocess.check_output(['hostname', '-I']).split():
-    LOCAL_IPS.append(ip.decode())
+for local_ip in subprocess.check_output(['hostname', '-I']).split():
+    LOCAL_IPS.append(local_ip.decode())
 
 
 VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4)
@@ -52,15 +71,15 @@ VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6)
 parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
 
 
+def status_reason(status_code):
+    return HTTPStatus(status_code).phrase
+
+
 def redis_set(name, value, ex):
     REDIS.set(name, value, nx=True, ex=ex)
     REDIS.set(name, value, xx=True, ex=ex)
 
 
-def bubble_log(message):
-    print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr, flush=True)
-
-
 def bubble_activity_log(client_addr, server_addr, event, data):
     key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4())
     value = json.dumps({
@@ -70,43 +89,162 @@ def bubble_activity_log(client_addr, server_addr, event, data):
         'event': event,
         'data': str(data)
     })
-    bubble_log('bubble_activity_log: setting '+key+' = '+value)
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('bubble_activity_log: setting '+key+' = '+value)
     redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION)
     pass
 
 
-def bubble_conn_check(remote_addr, addr, fqdns, security_level):
+def async_client(proxies=None,
+                 timeout=5,
+                 max_redirects=0):
+    return httpx.AsyncClient(timeout=timeout, max_redirects=max_redirects, proxies=proxies)
+
+
+async def async_response(client, name, url,
+                         headers=None,
+                         method='GET',
+                         data=None,
+                         json=None):
+    if bubble_log.isEnabledFor(INFO):
+        bubble_log.info('bubble_async_request(' + name + '): starting async: ' + method + ' ' + url)
+
+    response = await client.request(method=method, url=url, headers=headers, json=json, data=data)
+
+    if bubble_log.isEnabledFor(INFO):
+        bubble_log.info('bubble_async_request(' + name + '): async request returned HTTP status ' + str(response.status_code))
+
+    if response.status_code != 200:
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error('bubble_async_request(' + name + '): API call failed ('+url+'): ' + repr(response))
+
+    return response
+
+
+def async_stream(client, name, url,
+                 headers=None,
+                 method='GET',
+                 data=None,
+                 json=None,
+                 timeout=5,
+                 max_redirects=0,
+                 loop=asyncio.get_running_loop()):
+    try:
+        return loop.run_until_complete(_async_stream(client, name, url,
+                                                     headers=headers,
+                                                     method=method,
+                                                     data=data,
+                                                     json=json,
+                                                     timeout=timeout,
+                                                     max_redirects=max_redirects))
+    except Exception as e:
+        bubble_log.error('async_stream('+name+'): error with url='+url+' -- '+repr(e))
+
+
+async def _async_stream(client, name, url,
+                        headers=None,
+                        method='GET',
+                        data=None,
+                        json=None,
+                        timeout=5,
+                        max_redirects=0):
+    request = client.build_request(method=method, url=url, headers=headers, json=json, data=data)
+    return await client.send(request, stream=True, allow_redirects=(max_redirects > 0), timeout=timeout)
+
+
+async def _bubble_async(name, url,
+                        headers=None,
+                        method='GET',
+                        data=None,
+                        json=None,
+                        proxies=None,
+                        timeout=5,
+                        max_redirects=0):
+    async with async_client(proxies=proxies, timeout=timeout, max_redirects=max_redirects) as client:
+        return await async_response(client, name, url, headers=headers, method=method, data=data, json=json)
+
+
+def bubble_async(name, url,
+                 headers=None,
+                 method='GET',
+                 data=None,
+                 json=None,
+                 proxies=None,
+                 timeout=5,
+                 max_redirects=0,
+                 loop=asyncio.get_running_loop()):
+    try:
+        return loop.run_until_complete(_bubble_async(name, url,
+                                                     headers=headers,
+                                                     method=method,
+                                                     data=data,
+                                                     json=json,
+                                                     proxies=proxies,
+                                                     timeout=timeout,
+                                                     max_redirects=max_redirects))
+    except Exception as e:
+        bubble_log.error('bubble_async('+name+'): error: '+repr(e))
+
+
+def bubble_async_request_json(name, url, headers, method='GET', json=None):
+    response = bubble_async(name, url, headers, method=method, json=json)
+    if response and response.status_code == 200:
+        return response.json()
+    else:
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('bubble_async_request_json('+name+'): received invalid HTTP status: '+str(response.status_code))
+    return None
+
+
+def bubble_conn_check(client_addr, server_addr, fqdns, security_level):
     if debug_capture_fqdn and fqdns:
         for f in debug_capture_fqdn:
             if f in fqdns:
-                bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f)
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.debug('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f)
                 return 'noop'
 
+    name = 'bubble_conn_check'
+    url = 'http://127.0.0.1:'+bubble_port+'/api/filter/check'
     headers = {
-        'X-Forwarded-For': remote_addr,
-        'Accept' : 'application/json',
+        'X-Forwarded-For': client_addr,
+        'Accept': 'application/json',
         'Content-Type': 'application/json'
     }
+    data = {
+        'serverAddr': str(server_addr),
+        'fqdns': fqdns,
+        'clientAddr': client_addr
+    }
     try:
-        data = {
-            'addr': str(addr),
-            'fqdns': fqdns,
-            'remoteAddr': remote_addr
-        }
-        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data)
-        if response.ok:
-            return response.json()
-        bubble_log('bubble_conn_check API call failed: '+repr(response))
-        return None
+        return bubble_async_request_json(name, url, headers, method='POST', json=data)
 
     except Exception as e:
-        bubble_log('bubble_conn_check API call failed: '+repr(e))
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error('bubble_conn_check: API call failed: '+repr(e))
         traceback.print_exc()
         if security_level is not None and security_level['level'] == 'maximum':
             return False
         return None
 
 
+def bubble_get_flex_router(client_addr, host):
+    name = 'bubble_get_flex_router'
+    url = 'http://127.0.0.1:' + bubble_port + '/api/filter/flexRouters/' + host
+    headers = {
+        'X-Forwarded-For': client_addr,
+        'Accept': 'application/json'
+    }
+    try:
+        return bubble_async_request_json(name, url, headers)
+
+    except Exception as e:
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error('bubble_get_flex_routes: API call failed with exception: '+repr(e))
+        traceback.print_exc()
+        return None
+
+
 DEBUG_MATCHER_NAME = 'DebugCaptureMatcher'
 DEBUG_MATCHER = {
     'decision': 'match',
@@ -130,49 +268,59 @@ BLOCK_MATCHER = {
 
 def bubble_matchers(req_id, client_addr, server_addr, flow, host):
     if debug_capture_fqdn and host and host in debug_capture_fqdn:
-        bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host)
+        if bubble_log.isEnabledFor(INFO):
+            bubble_log.info('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host)
         return DEBUG_MATCHER
 
+    name = 'bubble_matchers'
+    url = 'http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id
     headers = {
         'X-Forwarded-For': client_addr,
-        'Accept' : 'application/json',
+        'Accept': 'application/json',
         'Content-Type': 'application/json'
     }
     if HEADER_USER_AGENT not in flow.request.headers:
-        bubble_log('bubble_matchers: no User-Agent header, setting to UNKNOWN')
+        if bubble_log.isEnabledFor(WARNING):
+            bubble_log.warning('bubble_matchers: no User-Agent header, setting to UNKNOWN')
         user_agent = 'UNKNOWN'
     else:
         user_agent = flow.request.headers[HEADER_USER_AGENT]
 
     if HEADER_REFERER not in flow.request.headers:
-        bubble_log('bubble_matchers: no Referer header, setting to NONE')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('bubble_matchers: no Referer header, setting to NONE')
         referer = 'NONE'
     else:
         try:
             referer = flow.request.headers[HEADER_REFERER].encode().decode()
         except Exception as e:
-            bubble_log('bubble_matchers: error parsing Referer header: '+repr(e))
+            if bubble_log.isEnabledFor(WARNING):
+                bubble_log.warning('bubble_matchers: error parsing Referer header: '+repr(e))
             referer = 'NONE'
 
+    data = {
+        'requestId': req_id,
+        'fqdn': host,
+        'uri': flow.request.path,
+        'userAgent': user_agent,
+        'referer': referer,
+        'clientAddr': client_addr,
+        'serverAddr': server_addr
+    }
+
     try:
-        data = {
-            'requestId': req_id,
-            'fqdn': host,
-            'uri': flow.request.path,
-            'userAgent': user_agent,
-            'referer': referer,
-            'clientAddr': client_addr,
-            'serverAddr': server_addr
-        }
-        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data)
-        if response.ok:
+        response = bubble_async(name, url, headers=headers, method='POST', json=data)
+        if response.status_code == 200:
             return response.json()
         elif response.status_code == 403:
-            bubble_log('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text))
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('bubble_matchers: response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text))
             return BLOCK_MATCHER
-        bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text))
+        if bubble_log.isEnabledFor(WARNING):
+            bubble_log.warning('bubble_matchers: response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text))
     except Exception as e:
-        bubble_log('bubble_matchers API call failed: '+repr(e))
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error('bubble_matchers: API call failed: '+repr(e))
         traceback.print_exc()
     return None
 
@@ -196,6 +344,18 @@ def is_bubble_request(ip, fqdns):
     return ip in LOCAL_IPS and (bubble_host in fqdns or bubble_host_alias in fqdns)
 
 
+def is_bubble_special_path(path):
+    return path and path.startswith(BUBBLE_URI_PREFIX)
+
+
+def make_bubble_special_path(path):
+    return 'http://127.0.0.1:' + bubble_port + '/' + path[len(BUBBLE_URI_PREFIX):]
+
+
+def is_bubble_health_check(path):
+    return path and path.startswith(HEALTH_CHECK_URI)
+
+
 def is_sage_request(ip, fqdns):
     return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns
 
@@ -203,3 +363,117 @@ def is_sage_request(ip, fqdns):
 def is_not_from_vpn(client_addr):
     ip = IPAddress(client_addr)
     return ip not in VPN_IP4_CIDR and ip not in VPN_IP6_CIDR
+
+
+def is_flex_domain(client_addr, fqdn):
+    if fqdn == bubble_host or fqdn == bubble_host_alias or fqdn == bubble_sage_host:
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('is_flex_domain: (early) returning False for: '+fqdn)
+        return False
+    check_fqdn = fqdn
+
+    exclusion_set = 'flexExcludeLists~' + client_addr + '~UNION'
+    excluded = REDIS.sismember(exclusion_set, fqdn)
+    if excluded:
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('is_flex_domain: returning False for excluded flex domain: ' + fqdn + ' (check=' + check_fqdn + ')')
+        return False
+
+    flex_set = 'flexLists~' + client_addr + '~UNION'
+    while '.' in check_fqdn:
+        found = REDIS.sismember(flex_set, check_fqdn)
+        if found:
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('is_flex_domain: returning True for: '+fqdn+' (check='+check_fqdn+')')
+            return True
+        check_fqdn = check_fqdn[check_fqdn.index('.')+1:]
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('is_flex_domain: (early) returning False for: '+fqdn)
+    return False
+
+
+def original_flex_ip(client_addr, fqdns):
+    for fqdn in fqdns:
+        ip = REDIS.get("flexOriginal~"+client_addr+"~"+fqdn)
+        if ip is not None:
+            return ip.decode()
+    return None
+
+
+def health_check_response(flow):
+    # if bubble_log.isEnabledFor(DEBUG):
+    #     bubble_log.debug('health_check_response: special bubble health check request, responding with OK')
+    response_headers = nheaders.Headers()
+    response_headers[HEADER_HEALTH_CHECK] = 'OK'
+    response_headers[HEADER_CONTENT_LENGTH] = '3'
+    if flow.response is None:
+        flow.response = http.HTTPResponse(http_version='HTTP/1.1',
+                                          status_code=200,
+                                          reason='OK',
+                                          headers=response_headers,
+                                          content=b'OK\n')
+    else:
+        flow.response.headers = nheaders.Headers()
+        flow.response.headers = response_headers
+        flow.response.status_code = 200
+        flow.response.reason = 'OK'
+        flow.response.stream = lambda chunks: [b'OK\n']
+
+
+def special_bubble_response(flow):
+    name = 'special_bubble_response'
+    path = flow.request.path
+    if is_bubble_health_check(path):
+        health_check_response(flow)
+        return
+    uri = make_bubble_special_path(path)
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('special_bubble_response: sending special bubble request to '+uri)
+    headers = {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json'
+    }
+    if flow.request.method == 'GET':
+        response = bubble_async(name, uri, headers=headers)
+
+    elif flow.request.method == 'POST':
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('special_bubble_response: special bubble request: POST content is '+str(flow.request.content))
+        if flow.request.content:
+            headers['Content-Length'] = str(len(flow.request.content))
+        response = bubble_async(name, uri, json=flow.request.content, headers=headers)
+
+    else:
+        if bubble_log.isEnabledFor(WARNING):
+            bubble_log.warning('special_bubble_response: special bubble request: method '+flow.request.method+' not supported')
+        return
+
+    if flow.response is None:
+        http_version = response.http_version
+        response_headers = collect_response_headers(response)
+        flow.response = http.HTTPResponse(http_version=http_version,
+                                          status_code=response.status_code,
+                                          reason=response.reason,
+                                          headers=response_headers,
+                                          content=None)
+    if response is not None:
+        # if bubble_log.isEnabledFor(DEBUG):
+        #     bubble_log.debug('special_bubble_response: special bubble request: response status = '+str(response.status_code))
+        flow.response.headers = collect_response_headers(response)
+        flow.response.status_code = response.status_code
+        flow.response.reason = status_reason(response.status_code)
+        flow.response.stream = lambda chunks: send_bubble_response(response)
+
+
+def send_bubble_response(response):
+    for chunk in response.iter_content(8192):
+        yield chunk
+
+
+def collect_response_headers(response, omit=None):
+    response_headers = nheaders.Headers()
+    for name in response.headers:
+        if omit and name in omit:
+            continue
+        response_headers[name] = response.headers[name]
+    return response_headers

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py

@@ -28,10 +28,16 @@ from mitmproxy.exceptions import TlsProtocolException
 from mitmproxy.net import tls as net_tls
 
 import json
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
 import traceback
-from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set, \
-    is_bubble_request, is_sage_request, is_not_from_vpn
-from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host
+from bubble_api import bubble_conn_check, bubble_activity_log, REDIS, redis_set, \
+    is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, bubble_get_flex_router
+from bubble_config import bubble_host, bubble_host_alias, cert_validation_host
+from bubble_flex_passthru import BubbleFlexPassthruLayer
+
+bubble_log = logging.getLogger(__name__)
 
 REDIS_DNS_PREFIX = 'bubble_dns_'
 REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_'
@@ -56,16 +62,19 @@ def get_device_security_level(client_addr, fqdns):
         return {'level': SEC_MAX}
     level = level.decode()
     if level == SEC_STD:
-        bubble_log('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns))
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.info('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns))
         if fqdns:
             max_required_fqdns = REDIS.smembers(REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX+client_addr)
             if max_required_fqdns is not None:
-                bubble_log('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns))
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.info('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns))
                 for max_required in max_required_fqdns:
                     max_required = max_required.decode()
                     for fqdn in fqdns:
                         if max_required == fqdn or (max_required.startswith('*.') and fqdn.endswith(max_required[1:])):
-                            bubble_log('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required)
+                            if bubble_log.isEnabledFor(INFO):
+                                bubble_log.info('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required)
                             return {'level': SEC_MAX, 'pinned': True}
     return {'level': level}
 
@@ -90,7 +99,8 @@ def fqdns_for_addr(server_addr):
     prefix = REDIS_DNS_PREFIX + server_addr
     keys = REDIS.keys(prefix + '_*')
     if keys is None or len(keys) == 0:
-        bubble_log('fqdns_for_addr: no FQDN found for addr '+str(server_addr)+', checking raw addr')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('fqdns_for_addr: no FQDN found for addr '+str(server_addr)+', checking raw addr')
         return ''
     fqdns = []
     for k in keys:
@@ -104,7 +114,8 @@ class TlsBlock(TlsLayer):
     Monkey-patch __call__ to drop this connection entirely
     """
     def __call__(self):
-        bubble_log('TlsBlock: blocking')
+        if bubble_log.isEnabledFor(INFO):
+            bubble_log.info('TlsBlock: blocking')
         return
 
 
@@ -122,16 +133,19 @@ class TlsFeedback(TlsLayer):
 
         except TlsProtocolException as e:
             if self.do_block:
-                bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address)
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address)
                 raise e
 
             tb = traceback.format_exc()
             if 'OpenSSL.SSL.ZeroReturnError' in tb:
-                bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address)
+                if bubble_log.isEnabledFor(WARNING):
+                    bubble_log.warning('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address)
                 raise e
 
             elif 'SysCallError' in tb:
-                bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SysCallError for client '+client_address)
+                if bubble_log.isEnabledFor(WARNING):
+                    bubble_log.warning('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SysCallError for client '+client_address)
                 raise e
 
             elif self.fqdns is not None and len(self.fqdns) > 0:
@@ -140,21 +154,26 @@ class TlsFeedback(TlsLayer):
                     if security_level['level'] == SEC_MAX:
                         if 'pinned' in security_level and security_level['pinned']:
                             redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': False, 'reason': 'tls_failure_pinned'}), ex=REDIS_CHECK_DURATION)
-                            bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
+                            if bubble_log.isEnabledFor(DEBUG):
+                                bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
                         else:
                             redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
-                            bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
+                            if bubble_log.isEnabledFor(DEBUG):
+                                bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
                     else:
                         redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
-                        bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
             else:
                 cache_key = conn_check_cache_prefix(client_address, server_address)
                 if security_level['level'] == SEC_MAX:
                     redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
-                    bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
                 else:
                     redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
-                    bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
             raise e
 
 
@@ -162,22 +181,27 @@ def check_bubble_connection(client_addr, server_addr, fqdns, security_level):
     check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level)
     if check_response is None or check_response == 'error':
         if security_level['level'] == SEC_MAX:
-            bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block')
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block')
             return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'}
         else:
-            bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
             return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_error'}
 
     elif check_response == 'passthru':
-        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True')
         return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_passthru'}
 
     elif check_response == 'block':
-        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block')
         return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_block'}
 
     else:
-        bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False')
         return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'reason': 'bubble_no_passthru'}
 
 
@@ -190,94 +214,147 @@ def check_connection(client_addr, server_addr, fqdns, security_level):
 
     check_json = REDIS.get(cache_key)
     if check_json is None or len(check_json) == 0:
-        bubble_log(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns))
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns))
         check_response = check_bubble_connection(client_addr, server_addr, fqdns, security_level)
-        bubble_log(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...")
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...")
         redis_set(cache_key, json.dumps(check_response), ex=REDIS_CHECK_DURATION)
 
     else:
-        bubble_log(prefix+'found check_json='+str(check_json)+', touching key in redis')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug(prefix+'found check_json='+str(check_json)+', touching key in redis')
         check_response = json.loads(check_json)
         REDIS.touch(cache_key)
-    bubble_log(prefix+'returning '+str(check_response))
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug(prefix+'returning '+str(check_response))
     return check_response
 
 
-def next_layer(next_layer):
-    if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
-        client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile)
-        client_addr = next_layer.client_conn.address[0]
-        server_addr = next_layer.server_conn.address[0]
-        bubble_log('next_layer: STARTING: client='+ client_addr+' server='+server_addr)
+def check_passthru_flex(client_addr, server_addr, fqdns):
+    if fqdns:
+        for fqdn in fqdns:
+            if is_flex_domain(client_addr, fqdn):
+                return True
+    else:
+        return is_flex_domain(client_addr, server_addr)
+
+
+def passthru_flex_port(client_addr, fqdns):
+    router = bubble_get_flex_router(client_addr)
+    if router is None or 'auth' not in router:
+        if bubble_log.isEnabledFor(INFO):
+            bubble_log.info('apply_passthru_flex: no flex router for fqdn(s): '+repr(fqdns))
+    elif 'port' in router:
+        return router['port']
+    else:
+        if bubble_log.isEnabledFor(WARNING):
+            bubble_log.warning('apply_passthru_flex: flex router found but has no port ('+repr(router)+') for fqdn(s): '+repr(fqdns))
+    return None
+
+
+def do_passthru(client_addr, server_addr, fqdns, layer):
+    flex_port = None
+    if check_passthru_flex(client_addr, server_addr, fqdns):
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('do_passthru: applying flex passthru for server=' + server_addr + ', fqdns=' + str(fqdns))
+        flex_port = passthru_flex_port(client_addr, fqdns)
+        if flex_port:
+            layer_replacement = BubbleFlexPassthruLayer(layer.ctx, ('127.0.0.1', flex_port), fqdns[0], 443)
+            layer.reply.send(layer_replacement)
+    if flex_port is None:
+        layer_replacement = RawTCPLayer(layer.ctx, ignore=True)
+        layer.reply.send(layer_replacement)
+
+
+def next_layer(layer):
+    if isinstance(layer, TlsLayer) and layer._client_tls:
+        client_hello = net_tls.ClientHello.from_file(layer.client_conn.rfile)
+        client_addr = layer.client_conn.address[0]
+        server_addr = layer.server_conn.address[0]
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('next_layer: STARTING: client='+ client_addr+' server='+server_addr)
         if client_hello.sni:
             fqdn = client_hello.sni.decode()
-            bubble_log('next_layer: using fqdn in SNI: '+ fqdn)
-            fqdns = [ fqdn ]
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('next_layer: using fqdn in SNI: '+ fqdn)
+            fqdns = [fqdn]
         else:
             fqdns = fqdns_for_addr(server_addr)
-            bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns))
-        next_layer.fqdns = fqdns
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns))
+        layer.fqdns = fqdns
         no_fqdns = fqdns is None or len(fqdns) == 0
         security_level = get_device_security_level(client_addr, fqdns)
-        next_layer.security_level = security_level
-        next_layer.do_block = False
+        layer.security_level = security_level
+        layer.do_block = False
         if is_bubble_request(server_addr, fqdns):
-            bubble_log('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns))
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns))
             check = FORCE_PASSTHRU
 
         elif is_sage_request(server_addr, fqdns):
-            bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_PASSTHRU
 
         elif is_not_from_vpn(client_addr):
             # todo: add to fail2ban
-            bubble_log('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns))
+            if bubble_log.isEnabledFor(WARNING):
+                bubble_log.warning('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns))
             bubble_activity_log(client_addr, server_addr, 'conn_block_non_vpn', fqdns)
-            next_layer.__class__ = TlsBlock
+            layer.__class__ = TlsBlock
             return
 
         elif security_level['level'] == SEC_OFF:
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_PASSTHRU
 
         elif fqdns is not None and len(fqdns) == 1 and cert_validation_host == fqdns[0] and security_level['level'] != SEC_BASIC:
-            bubble_log('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr)
             return
 
         elif (security_level['level'] == SEC_STD or security_level['level'] == SEC_BASIC) and no_fqdns:
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_PASSTHRU
 
         elif security_level['level'] == SEC_MAX and no_fqdns:
-            bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_BLOCK
 
         else:
-            bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level))
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level))
             check = check_connection(client_addr, server_addr, fqdns, security_level)
 
         if check is None or ('passthru' in check and check['passthru']):
-            bubble_log('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
             bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
-            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
-            next_layer.reply.send(next_layer_replacement)
+            do_passthru(client_addr, server_addr, fqdns, layer)
 
         elif 'block' in check and check['block']:
-            bubble_log('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns))
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns))
             bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns)
             if show_block_stats(client_addr, fqdns) and security_level['level'] != SEC_BASIC:
-                next_layer.do_block = True
-                next_layer.__class__ = TlsFeedback
+                layer.do_block = True
+                layer.__class__ = TlsFeedback
             else:
-                next_layer.__class__ = TlsBlock
+                layer.__class__ = TlsBlock
 
         elif security_level['level'] == SEC_BASIC:
-            bubble_log('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns))
             bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns)
-            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
-            next_layer.reply.send(next_layer_replacement)
+            do_passthru(client_addr, server_addr, fqdns, layer)
 
         else:
-            bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns))
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns))
             bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns)
-            next_layer.__class__ = TlsFeedback
+            layer.__class__ = TlsFeedback

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_debug.py

@@ -1,11 +1,42 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
+import os
 import threading
 import traceback
 import signal
 import sys
 
+from pathlib import Path
+
+BUBBLE_PORT_ENV_VAR = 'BUBBLE_PORT'
+BUBBLE_PORT = os.getenv(BUBBLE_PORT_ENV_VAR)
+if BUBBLE_PORT is None:
+    BUBBLE_PORT = '(no '+BUBBLE_PORT_ENV_VAR+' env var found)'
+
+BUBBLE_LOG = '/var/log/bubble/mitmproxy_bubble.log'
+BUBBLE_LOG_LEVEL_FILE = '/home/mitmproxy/bubble_log_level.txt'
+BUBBLE_LOG_LEVEL_ENV_VAR = 'BUBBLE_LOG_LEVEL'
+DEFAULT_BUBBLE_LOG_LEVEL = 'INFO'
+BUBBLE_LOG_LEVEL = None
+try:
+    BUBBLE_LOG_LEVEL = Path(BUBBLE_LOG_LEVEL_FILE).read_text().strip()
+except IOError:
+    print('error reading log level from '+BUBBLE_LOG_LEVEL_FILE+', checking env var '+BUBBLE_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True)
+    BUBBLE_LOG_LEVEL = os.getenv(BUBBLE_LOG_LEVEL_ENV_VAR, DEFAULT_BUBBLE_LOG_LEVEL)
+
+BUBBLE_NUMERIC_LOG_LEVEL = getattr(logging, BUBBLE_LOG_LEVEL.upper(), None)
+if not isinstance(BUBBLE_NUMERIC_LOG_LEVEL, int):
+    print('Invalid log level: ' + BUBBLE_LOG_LEVEL + ' - using default '+DEFAULT_BUBBLE_LOG_LEVEL, file=sys.stderr, flush=True)
+    BUBBLE_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_BUBBLE_LOG_LEVEL.upper(), None)
+logging.basicConfig(format='[mitm'+BUBBLE_PORT+'] %(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=BUBBLE_LOG, level=BUBBLE_NUMERIC_LOG_LEVEL)
+
+bubble_log = logging.getLogger(__name__)
+
+
 # Allow SIGUSR1 to print stack traces to stderr
 def dumpstacks(signal, frame):
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
@@ -18,5 +49,8 @@ def dumpstacks(signal, frame):
                 code.append("  %s" % (line.strip()))
     print("\n------------------------------------- stack traces ------------------------------"+"\n".join(code), file=sys.stderr, flush=True)
 
+
 signal.signal(signal.SIGUSR1, dumpstacks)
 
+if bubble_log.isEnabledFor(INFO):
+    bubble_log.info('debug module initialized, default log level = '+logging.getLevelName(BUBBLE_NUMERIC_LOG_LEVEL))

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py

@@ -0,0 +1,204 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+import asyncio
+
+from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody
+
+from mitmproxy import http
+from mitmproxy.net.http import headers as nheaders
+from mitmproxy.proxy.protocol.request_capture import RequestCapture
+
+from bubble_api import bubble_get_flex_router, collect_response_headers, async_client, async_stream, \
+    HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, HEADER_CONTENT_TYPE
+
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
+
+bubble_log = logging.getLogger(__name__)
+
+FLEX_TIMEOUT = 20
+
+
+class FlexFlow(RequestCapture):
+    flex_host: None
+    mitm_flow: None
+    router: None
+    request_chunks: None
+    response_stream: None
+
+    def __init__(self, flex_host, mitm_flow, router):
+        super().__init__()
+        self.flex_host = flex_host
+        self.mitm_flow = mitm_flow
+        self.router = router
+        mitm_flow.request.stream = self
+        mitm_flow.response = http.HTTPResponse(http_version='HTTP/1.1',
+                                               status_code=523,
+                                               reason='FlexFlow Not Initialized',
+                                               headers={},
+                                               content=None)
+
+    def is_error(self):
+        return 'error_html' in self.router and self.router['error_html'] and len(self.router['error_html']) > 0
+
+    def capture(self, chunks):
+        self.request_chunks = chunks
+
+
+def process_no_flex(flex_flow):
+
+    flow = flex_flow.mitm_flow
+
+    response_headers = nheaders.Headers()
+    response_headers[HEADER_CONTENT_TYPE] = 'text/html'
+    response_headers[HEADER_CONTENT_LENGTH] = str(len(flex_flow.router['error_html']))
+
+    flow.response = http.HTTPResponse(http_version='HTTP/1.1',
+                                      status_code=200,
+                                      reason='OK',
+                                      headers=response_headers,
+                                      content=None)
+    error_html = flex_flow.router['error_html']
+    flex_flow.response_stream = lambda chunks: error_html
+    flow.response.stream = lambda chunks: error_html
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('process_no_flex: no router found, returning error_html')
+    return flex_flow
+
+
+def new_flex_flow(client_addr, flex_host, flow):
+    router = bubble_get_flex_router(client_addr, flex_host)
+    if router is None or 'auth' not in router:
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error('new_flex_flow: no flex router for host: '+flex_host)
+        return None
+
+    if bubble_log.isEnabledFor(INFO):
+        bubble_log.info('new_flex_flow: found router '+repr(router)+' for flex host: '+flex_host)
+    return FlexFlow(flex_host, flow, router)
+
+
+def process_flex(flex_flow):
+
+    if flex_flow.is_error():
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('process_flex: no router found, returning default flow')
+        return process_no_flex(flex_flow)
+    else:
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('process_flex: using router: '+repr(flex_flow.router))
+
+    flex_host = flex_flow.flex_host
+    flow = flex_flow.mitm_flow
+    router = flex_flow.router
+
+    # build the request URL
+    method = flow.request.method
+    scheme = flow.request.scheme
+    url = scheme + '://' + flex_host + flow.request.path
+
+    # copy request headers
+    # see: https://stackoverflow.com/questions/16789840/python-requests-cant-send-multiple-headers-with-same-key
+    request_headers = {}
+    for name in flow.request.headers:
+        if name in request_headers:
+            request_headers[name] = request_headers[name] + "," + flow.request.headers[name]
+        else:
+            request_headers[name] = flow.request.headers[name]
+
+    # setup proxies
+    proxy_url = router['proxyUrl']
+    proxies = {"http": proxy_url, "https": proxy_url}
+
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('process_flex: sending flex request for '+method+' '+url+' to '+proxy_url)
+
+    loop = asyncio.new_event_loop()
+    client = async_client(proxies=proxies, timeout=30)
+    try:
+        response = async_stream(client, 'process_flex', url,
+                                method=method,
+                                headers=request_headers,
+                                timeout=30,
+                                data=async_chunk_iter(flex_flow.request_chunks),
+                                loop=loop)
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('process_flex: response returned HTTP status '+str(response.status_code)+' for '+url)
+    except Exception as e:
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error('process_flex: error sending request to '+url+': '+repr(e))
+        return None
+
+    if response is None:
+        return None
+
+    # Status line
+    http_version = response.http_version
+
+    # Headers -- copy from requests dict to Headers multimap
+    # Remove Content-Length and Content-Encoding, we will rechunk the output
+    response_headers = collect_response_headers(response, [HEADER_CONTENT_LENGTH, HEADER_TRANSFER_ENCODING])
+
+    # Construct the real response
+    flow.response = http.HTTPResponse(http_version=http_version,
+                                      status_code=response.status_code,
+                                      reason=response.reason_phrase,
+                                      headers=response_headers,
+                                      content=None)
+
+    # If Content-Length header did not exist, or did exist and was > 0, then chunk the content
+    content_length = None
+    if HEADER_CONTENT_LENGTH in response.headers:
+        content_length = response.headers[HEADER_CONTENT_LENGTH]
+    if response.status_code // 100 != 2:
+        response_headers[HEADER_CONTENT_LENGTH] = '0'
+        flow.response.stream = lambda chunks: []
+
+    elif content_length is None or int(content_length) > 0:
+        response_headers[HEADER_TRANSFER_ENCODING] = 'chunked'
+        flow.response.stream = AsyncStreamBody(owner=client, loop=loop, chunks=response.aiter_raw(), finalize=cleanup_flex(url, loop, client, response))
+
+    else:
+        response_headers[HEADER_CONTENT_LENGTH] = '0'
+        flow.response.stream = lambda chunks: []
+
+    # Apply filters
+    if bubble_log.isEnabledFor(INFO):
+        bubble_log.info('process_flex: successfully requested url '+url+' from flex router, proceeding...')
+
+    flex_flow.response_stream = response
+    return flex_flow
+
+
+async def async_chunk_iter(chunks):
+    for chunk in chunks:
+        yield chunk
+
+
+def cleanup_flex(url, loop, client, response):
+    def cleanup():
+
+        errors = False
+
+        try:
+            loop.run_until_complete(response.aclose())
+        except Exception as e:
+            bubble_log.error('cleanup_flex: error closing response: '+repr(e))
+            errors = True
+
+        try:
+            loop.run_until_complete(client.aclose())
+        except Exception as e:
+            bubble_log.error('cleanup_flex: error: '+repr(e))
+            errors = True
+
+        if not errors:
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('cleanup_flex: successfully completed: '+url)
+        else:
+            if bubble_log.isEnabledFor(WARNING):
+                bubble_log.warning('cleanup_flex: successfully completed (but had errors closing): ' + url)
+
+    return cleanup

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py

@@ -0,0 +1,112 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Parts of this are borrowed from rawtcp.py in the mitmproxy project. The mitmproxy license is reprinted here:
+#
+# Copyright (c) 2013, Aldo Cortesi. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+import socket
+
+from OpenSSL import SSL
+
+from mitmproxy.exceptions import MitmproxyException
+from mitmproxy import tcp
+from mitmproxy import exceptions
+from mitmproxy.proxy.protocol import base
+from mitmproxy.connections import ServerConnection
+from mitmproxy.http import make_connect_request
+from mitmproxy.net.http.http1 import assemble_request
+from mitmproxy.net.tcp import ssl_read_select
+
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
+bubble_log = logging.getLogger(__name__)
+
+
+class BubbleFlexPassthruException(MitmproxyException):
+    pass
+
+
+class BubbleFlexPassthruLayer(base.Layer):
+    chunk_size = 4096
+    proxy_addr = None
+    host = None
+    port = None
+
+    def __init__(self, ctx, proxy_addr, host, port):
+        self.ignore = True
+        self.proxy_addr = proxy_addr
+        self.server_conn = ServerConnection(proxy_addr)
+        self.host = host
+        self.port = port
+        ctx.server_conn = self.server_conn
+        super().__init__(ctx)
+
+    def __call__(self):
+        self.connect()
+        client = self.client_conn.connection
+        server = self.server_conn.connection
+
+        buf = memoryview(bytearray(self.chunk_size))
+
+        # send CONNECT, expect 200 OK
+        connect_req = make_connect_request((self.host, self.port))
+        server.send(assemble_request(connect_req))
+        resp = server.recv(1024).decode()
+        if not resp.startswith('HTTP/1.1 200 OK'):
+            raise BubbleFlexPassthruException('CONNECT request error: '+resp)
+
+        conns = [client, server]
+
+        # https://github.com/openssl/openssl/issues/6234
+        for conn in conns:
+            if isinstance(conn, SSL.Connection) and hasattr(SSL._lib, "SSL_clear_mode"):
+                SSL._lib.SSL_clear_mode(conn._ssl, SSL._lib.SSL_MODE_AUTO_RETRY)
+
+        try:
+            while not self.channel.should_exit.is_set():
+                r = ssl_read_select(conns, 10)
+                for conn in r:
+                    dst = server if conn == client else client
+                    try:
+                        size = conn.recv_into(buf, self.chunk_size)
+                    except (SSL.WantReadError, SSL.WantWriteError):
+                        continue
+                    if not size:
+                        conns.remove(conn)
+                        # Shutdown connection to the other peer
+                        if isinstance(conn, SSL.Connection):
+                            # We can't half-close a connection, so we just close everything here.
+                            # Sockets will be cleaned up on a higher level.
+                            return
+                        else:
+                            dst.shutdown(socket.SHUT_WR)
+
+                        if len(conns) == 0:
+                            return
+                        continue
+
+                    tcp_message = tcp.TCPMessage(dst == server, buf[:size].tobytes())
+                    dst.sendall(tcp_message.content)
+
+        except (socket.error, exceptions.TcpException, SSL.Error) as e:
+            bubble_log.error('exception: '+repr(e))

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -1,24 +1,27 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
+import asyncio
 import json
 import re
-import requests
 import urllib
 import traceback
 from mitmproxy.net.http import Headers
-from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri
-from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, BUBBLE_URI_PREFIX, \
-    HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, \
-    CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \
-    HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header
+from bubble_config import bubble_port, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri
+from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_FLEX, \
+    status_reason, get_flow_ctx, add_flow_ctx, bubble_async, \
+    is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \
+    CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \
+    HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \
+    HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set
+from bubble_flex import process_flex
+
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
+bubble_log = logging.getLogger(__name__)
 
 BUFFER_SIZE = 4096
-HEADER_CONTENT_TYPE = 'Content-Type'
-HEADER_CONTENT_LENGTH = 'Content-Length'
-HEADER_CONTENT_ENCODING = 'Content-Encoding'
-HEADER_TRANSFER_ENCODING = 'Transfer-Encoding'
-HEADER_LOCATION = 'Location'
 CONTENT_TYPE_BINARY = 'application/octet-stream'
 STANDARD_FILTER_HEADERS = {HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY}
 
@@ -26,6 +29,7 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__'
 REDIS_FILTER_PASSTHRU_DURATION = 600
 
 DEBUG_STREAM_COUNTERS = {}
+MIN_FILTER_CHUNK_SIZE = 1024 * 32  # Filter data in 32KB chunks
 
 
 def add_csp_part(new_csp, part):
@@ -50,10 +54,12 @@ def ensure_bubble_script_csp(csp):
     return new_csp
 
 
-def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, content_type=None, content_length=None, csp=None):
+def filter_chunk(loop, flow, chunk, req_id, user_agent, last, content_encoding=None, content_type=None, content_length=None, csp=None):
+    name = 'filter_chunk'
     if debug_capture_fqdn:
         if debug_capture_fqdn in req_id:
-            bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn)
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn)
             f = open('/tmp/bubble_capture_'+req_id, mode='ab', buffering=0)
             f.write(chunk)
             f.close()
@@ -63,7 +69,8 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c
     redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + '~~~' + user_agent + ':' + flow.request.url
     do_pass = REDIS.get(redis_passthru_key)
     if do_pass:
-        bubble_log('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk')
         REDIS.touch(redis_passthru_key)
         return chunk
 
@@ -82,16 +89,21 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c
         else:
             url = url + '?last=true'
 
+    chunk_len = 0
+    if bubble_log.isEnabledFor(DEBUG):
+        if chunk is not None:
+            chunk_len = len(chunk)
     if csp:
-        # bubble_log('filter_chunk: url='+url+' (csp='+csp+')')
-        bubble_log('filter_chunk: url='+url+' (with csp) (last='+str(last)+')')
-        filter_headers = {
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('filter_chunk: url='+url+' (csp='+csp+') size='+str(chunk_len))
+        headers = {
             HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY,
             HEADER_CONTENT_SECURITY_POLICY: csp
         }
     else:
-        bubble_log('filter_chunk: url='+url+' (no csp) (last='+str(last)+')')
-        filter_headers = STANDARD_FILTER_HEADERS
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('filter_chunk: url='+url+' (no csp) size='+str(chunk_len))
+        headers = STANDARD_FILTER_HEADERS
 
     if debug_stream_fqdn and debug_stream_uri and debug_stream_fqdn in req_id and flow.request.path == debug_stream_uri:
         if req_id in DEBUG_STREAM_COUNTERS:
@@ -99,68 +111,91 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c
         else:
             count = 0
         DEBUG_STREAM_COUNTERS[req_id] = count
-        bubble_log('filter_chunk: debug_stream detected, capturing: '+debug_stream_fqdn)
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('filter_chunk: debug_stream detected, capturing: '+debug_stream_fqdn)
         f = open('/tmp/bubble_stream_'+req_id+'_chunk'+"{:04d}".format(count)+'.data', mode='wb', buffering=0)
         if chunk is not None:
             f.write(chunk)
         f.close()
         f = open('/tmp/bubble_stream_'+req_id+'_chunk'+"{:04d}".format(count)+'.headers.json', mode='w')
-        f.write(json.dumps(filter_headers))
+        f.write(json.dumps(headers))
         f.close()
         f = open('/tmp/bubble_stream_'+req_id+'_chunk'+"{:04d}".format(count)+'.url', mode='w')
         f.write(url)
         f.close()
 
-    response = requests.post(url, data=chunk, headers=filter_headers)
-    if not response.ok:
+    response = bubble_async(name, url, headers=headers, method='POST', data=chunk, loop=loop)
+    if not response.status_code == 200:
         err_message = 'filter_chunk: Error fetching ' + url + ', HTTP status ' + str(response.status_code)
-        bubble_log(err_message)
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error(err_message)
         return b''
 
     elif HEADER_FILTER_PASSTHRU in response.headers:
-        bubble_log('filter_chunk: server returned X-Bubble-Passthru, not filtering subsequent requests')
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('filter_chunk: server returned X-Bubble-Passthru, not filtering subsequent requests')
         redis_set(redis_passthru_key, 'passthru', ex=REDIS_FILTER_PASSTHRU_DURATION)
         return chunk
 
     return response.content
 
 
-def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, content_type, csp):
-    """
-    chunks is a generator that can be used to iterate over all chunks.
-    """
+def bubble_filter_chunks(flow, chunks, flex_flow, req_id, user_agent, content_encoding, content_type, csp):
+    loop = asyncio.new_event_loop()
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('bubble_filter_chunks: starting with content_type='+content_type)
     first = True
+    last = False
     content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH)
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('bubble_filter_chunks: found content_length='+str(content_length))
+    if flex_flow is not None:
+        # flex flows with errors are handled before we get here
+        chunks = flex_flow.response_stream.iter_content(8192)
     try:
+        buffer = b''
         for chunk in chunks:
+            buffer = buffer + chunk
+            if len(buffer) < MIN_FILTER_CHUNK_SIZE:
+                continue
+            chunk_len = len(buffer)
+            chunk = buffer
+            buffer = b''
             if content_length:
                 bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT)
-                chunk_len = len(chunk)
                 last = chunk_len + bytes_sent >= content_length
-                bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent))
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.debug('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent))
                 add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len)
             else:
                 last = False
             if first:
-                yield filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding, content_type, content_length, csp)
+                yield filter_chunk(loop, flow, chunk, req_id, user_agent, last, content_encoding, content_type, content_length, csp)
                 first = False
             else:
-                yield filter_chunk(flow, chunk, req_id, user_agent, last)
-        if not content_length:
-            yield filter_chunk(flow, None, req_id, user_agent, True)  # get the last bits of data
+                yield filter_chunk(loop, flow, chunk, req_id, user_agent, last)
+        # send whatever is left in the buffer
+        if len(buffer) > 0:
+            # bubble_log.debug('bubble_filter_chunks(end): sending remainder buffer of size '+str(len(buffer)))
+            if first:
+                yield filter_chunk(loop, flow, buffer, req_id, user_agent, last, content_encoding, content_type, content_length, csp)
+            else:
+                yield filter_chunk(loop, flow, buffer, req_id, user_agent, last)
+        if not content_length or not last:
+            # bubble_log.debug('bubble_filter_chunks(end): sending last empty chunk')
+            yield filter_chunk(loop, flow, None, req_id, user_agent, True)  # get the last bits of data
     except Exception as e:
-        bubble_log('bubble_filter_chunks: exception='+repr(e))
+        if bubble_log.isEnabledFor(ERROR):
+            bubble_log.error('bubble_filter_chunks: exception='+repr(e))
         traceback.print_exc()
         yield None
 
 
-def bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp):
-    return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, content_type, csp)
-
-
-def send_bubble_response(response):
-    for chunk in response.iter_content(8192):
-        yield chunk
+def bubble_modify(flow, flex_flow, req_id, user_agent, content_encoding, content_type, csp):
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('bubble_modify: modifying req_id='+req_id+' with content_type='+content_type)
+    return lambda chunks: bubble_filter_chunks(flow, chunks, flex_flow, req_id,
+                                               user_agent, content_encoding, content_type, csp)
 
 
 EMPTY_XML = [b'<?xml version="1.0" encoding="UTF-8"?><html></html>']
@@ -179,78 +214,82 @@ def abort_data(content_type):
 
 
 def responseheaders(flow):
+    flex_flow = get_flow_ctx(flow, CTX_BUBBLE_FLEX)
+    if flex_flow:
+        flex_flow = process_flex(flex_flow)
+    else:
+        flex_flow = None
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('responseheaders: flex_flow = '+repr(flex_flow))
+    bubble_filter_response(flow, flex_flow)
+
+
+def bubble_filter_response(flow, flex_flow):
+    # only filter once -- flex routing may have pre-filtered
+    if get_flow_ctx(flow, CTX_BUBBLE_FILTERED):
+        return
+    add_flow_ctx(flow, CTX_BUBBLE_FILTERED, True)
 
     path = flow.request.path
-    if path and path.startswith(BUBBLE_URI_PREFIX):
-        if path.startswith(HEALTH_CHECK_URI):
-            # bubble_log('responseheaders: special bubble health check request, responding with OK')
-            flow.response.headers = Headers()
-            flow.response.headers[HEADER_HEALTH_CHECK] = 'OK'
-            flow.response.headers[HEADER_CONTENT_LENGTH] = '3'
-            flow.response.status_code = 200
-            flow.response.stream = lambda chunks: [b'OK\n']
+    client_addr = flow.client_conn.address[0]
+    if is_bubble_special_path(path):
+        if is_bubble_health_check(path):
+            health_check_response(flow)
         else:
-            uri = 'http://127.0.0.1:' + bubble_port + '/' + path[len(BUBBLE_URI_PREFIX):]
-            bubble_log('responseheaders: sending special bubble request to '+uri)
-            headers = {
-                'Accept' : 'application/json',
-                'Content-Type': 'application/json'
-            }
-            response = None
-            if flow.request.method == 'GET':
-                response = requests.get(uri, headers=headers, stream=True)
-            elif flow.request.method == 'POST':
-                bubble_log('responseheaders: special bubble request: POST content is '+str(flow.request.content))
-                headers['Content-Length'] = str(len(flow.request.content))
-                response = requests.post(uri, data=flow.request.content, headers=headers)
-            else:
-                bubble_log('responseheaders: special bubble request: method '+flow.request.method+' not supported')
-            if response is not None:
-                bubble_log('responseheaders: special bubble request: response status = '+str(response.status_code))
-                flow.response.headers = Headers()
-                for key, value in response.headers.items():
-                    flow.response.headers[key] = value
-                flow.response.status_code = response.status_code
-                flow.response.stream = lambda chunks: send_bubble_response(response)
+            special_bubble_response(flow)
+
+    elif flex_flow and flex_flow.is_error():
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug('bubble_filter_response: flex_flow had error, returning error_html: ' + repr(flex_flow.response_stream))
+        flow.response.stream = flex_flow.response_stream
 
     else:
         abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT)
         if abort_code is not None:
             abort_location = get_flow_ctx(flow, CTX_BUBBLE_LOCATION)
             if abort_location is not None:
-                bubble_log('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path)
+                if bubble_log.isEnabledFor(INFO):
+                    bubble_log.info('bubble_filter_response: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path)
                 flow.response.headers = Headers()
                 flow.response.headers[HEADER_LOCATION] = abort_location
                 flow.response.status_code = abort_code
+                flow.response.reason = status_reason(abort_code)
                 flow.response.stream = lambda chunks: []
             else:
                 if HEADER_CONTENT_TYPE in flow.response.headers:
                     content_type = flow.response.headers[HEADER_CONTENT_TYPE]
                 else:
                     content_type = None
-                bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code)+', path was: '+path)
+                if bubble_log.isEnabledFor(INFO):
+                    bubble_log.info('bubble_filter_response: aborting request from '+client_addr+' with HTTP status '+str(abort_code)+', path was: '+path)
                 flow.response.headers = Headers()
                 flow.response.status_code = abort_code
+                flow.response.reason = status_reason(abort_code)
                 flow.response.stream = lambda chunks: abort_data(content_type)
 
         elif flow.response.status_code // 100 != 2:
-            bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('bubble_filter_response: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path)
+            flow.response.headers[HEADER_CONTENT_LENGTH] = '0'
             pass
 
         elif flow.response.headers is None or len(flow.response.headers) == 0:
-            bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('bubble_filter_response: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path)
             pass
 
         elif HEADER_CONTENT_LENGTH in flow.response.headers and flow.response.headers[HEADER_CONTENT_LENGTH] == "0":
-            bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path)
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('bubble_filter_response: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path)
             pass
 
         else:
             req_id = get_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID)
             matchers = get_flow_ctx(flow, CTX_BUBBLE_MATCHERS)
-            prefix = 'responseheaders(req_id='+str(req_id)+'): '
+            prefix = 'bubble_filter_response(req_id='+str(req_id)+'): '
             if req_id is not None and matchers is not None:
-                bubble_log(prefix+' matchers: '+repr(matchers))
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.debug(prefix+' matchers: '+repr(matchers))
                 if HEADER_USER_AGENT in flow.request.headers:
                     user_agent = flow.request.headers[HEADER_USER_AGENT]
                 else:
@@ -261,15 +300,17 @@ def responseheaders(flow):
                         any_content_type_matches = False
                         for m in matchers:
                             if 'contentTypeRegex' in m:
-                                typeRegex = m['contentTypeRegex']
-                                if typeRegex is None:
-                                    typeRegex = '^text/html.*'
-                                if re.match(typeRegex, content_type):
+                                type_regex = m['contentTypeRegex']
+                                if type_regex is None:
+                                    type_regex = '^text/html.*'
+                                if re.match(type_regex, content_type):
                                     any_content_type_matches = True
-                                    bubble_log(prefix+'found at least one matcher for content_type ('+content_type+'), filtering: '+path)
+                                    if bubble_log.isEnabledFor(DEBUG):
+                                        bubble_log.debug(prefix+'found at least one matcher for content_type ('+content_type+'), filtering: '+path)
                                     break
                         if not any_content_type_matches:
-                            bubble_log(prefix+'no matchers for content_type ('+content_type+'), passing thru: '+path)
+                            if bubble_log.isEnabledFor(DEBUG):
+                                bubble_log.debug(prefix+'no matchers for content_type ('+content_type+'), passing thru: '+path)
                             return
 
                         if HEADER_CONTENT_ENCODING in flow.response.headers:
@@ -284,8 +325,11 @@ def responseheaders(flow):
                             csp = None
 
                         content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None)
-                        # bubble_log(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
-                        flow.response.stream = bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp)
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
+
+                        flow.response.stream = bubble_modify(flow, flex_flow, req_id,
+                                                             user_agent, content_encoding, content_type, csp)
                         if content_length_value:
                             flow.response.headers['transfer-encoding'] = 'chunked'
                             # find server_conn to set fake_chunks on
@@ -295,10 +339,12 @@ def responseheaders(flow):
                                     if hasattr(ctx, 'ctx'):
                                         ctx = ctx.ctx
                                     else:
-                                        bubble_log(prefix+'error finding server_conn for path '+path+'. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx)))
+                                        if bubble_log.isEnabledFor(ERROR):
+                                            bubble_log.error(prefix+'error finding server_conn for path '+path+'. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx)))
                                         return
                                 if not hasattr(ctx, 'server_conn'):
-                                    bubble_log(prefix+'error finding server_conn for path '+path+'. ctx type='+str(type(ctx))+' vars='+str(vars(ctx)))
+                                    if bubble_log.isEnabledFor(ERROR):
+                                        bubble_log.error(prefix+'error finding server_conn for path '+path+'. ctx type='+str(type(ctx))+' vars='+str(vars(ctx)))
                                     return
                                 content_length = int(content_length_value)
                                 ctx.server_conn.rfile.fake_chunks = content_length
@@ -306,11 +352,14 @@ def responseheaders(flow):
                                 add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, 0)
 
                     else:
-                        bubble_log(prefix+'no matchers, passing thru: '+path)
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug(prefix+'no matchers, passing thru: '+path)
                         pass
                 else:
-                    bubble_log(prefix+'no '+HEADER_CONTENT_TYPE+' header, passing thru: '+path)
+                    if bubble_log.isEnabledFor(WARNING):
+                        bubble_log.warning(prefix+'no '+HEADER_CONTENT_TYPE+' header, passing thru: '+path)
                     pass
             else:
-                bubble_log(prefix+'no '+CTX_BUBBLE_MATCHERS+' in ctx, passing thru: '+path)
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.debug(prefix+'no '+CTX_BUBBLE_MATCHERS+' in ctx, passing thru: '+path)
                 pass

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py

@@ -0,0 +1,273 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Parts of this are borrowed from dns_spoofing.py in the mitmproxy project. The mitmproxy license is reprinted here:
+#
+# Copyright (c) 2013, Aldo Cortesi. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+
+import re
+import time
+import uuid
+from mitmproxy.net.http import headers as nheaders
+
+from bubble_api import bubble_matchers, bubble_activity_log, \
+    HEALTH_CHECK_URI, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \
+    CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \
+    is_bubble_special_path, special_bubble_response, is_bubble_health_check, \
+    is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain
+from bubble_config import bubble_host, bubble_host_alias
+from bubble_flex import new_flex_flow
+
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
+bubble_log = logging.getLogger(__name__)
+
+
+class Rerouter:
+    @staticmethod
+    def get_matchers(flow, host):
+        if host is None:
+            return None
+
+        is_health_check = is_bubble_health_check(flow.request.path)
+        if is_bubble_special_path(flow.request.path):
+            if not is_health_check:
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.debug("get_matchers: not filtering special bubble path: "+flow.request.path)
+            return None
+
+        client_addr = str(flow.client_conn.address[0])
+        server_addr = str(flow.server_conn.address[0])
+        try:
+            host = host.decode()
+        except (UnicodeDecodeError, AttributeError):
+            try:
+                host = str(host)
+            except Exception as e:
+                if bubble_log.isEnabledFor(WARNING):
+                    bubble_log.warning('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e))
+                return None
+
+        if host == bubble_host or host == bubble_host_alias:
+            if bubble_log.isEnabledFor(INFO):
+                bubble_log.info('get_matchers: request is for bubble itself ('+host+'), not matching')
+            return None
+
+        req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time())
+        if bubble_log.isEnabledFor(DEBUG):
+            bubble_log.debug("get_matchers: requesting match decision for req_id="+req_id)
+        resp = bubble_matchers(req_id, client_addr, server_addr, flow, host)
+
+        if not resp:
+            if bubble_log.isEnabledFor(WARNING):
+                bubble_log.warning('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host))
+            return None
+
+        matchers = []
+        if 'matchers' in resp and resp['matchers'] is not None:
+            for m in resp['matchers']:
+                if 'urlRegex' in m:
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex'])
+                else:
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping')
+                    continue
+                if re.match(m['urlRegex'], flow.request.path):
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('get_matchers: rule matched, adding rule: '+m['rule'])
+                    matchers.append(m)
+                else:
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule'])
+        else:
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('get_matchers: no matchers. response='+repr(resp))
+
+        decision = None
+        if 'decision' in resp:
+            decision = resp['decision']
+
+        matcher_response = {'decision': decision, 'matchers': matchers, 'request_id': req_id}
+        if bubble_log.isEnabledFor(INFO):
+            bubble_log.info("get_matchers: returning "+repr(matcher_response))
+        return matcher_response
+
+    def bubble_handle_request(self, flow):
+        client_addr = flow.client_conn.address[0]
+        server_addr = flow.server_conn.address[0]
+        is_http = False
+        if flow.client_conn.tls_established:
+            flow.request.scheme = "https"
+            sni = flow.client_conn.connection.get_servername()
+            port = 443
+        else:
+            flow.request.scheme = "http"
+            sni = None
+            port = 80
+            is_http = True
+
+        # check if https and sni is missing but we have a host header, fill in the sni
+        
+        host_header = flow.request.host_header
+        if host_header:
+            m = parse_host_header.match(host_header)
+            if m:
+                host_header = m.group("host").strip("[]")
+                if m.group("port"):
+                    port = int(m.group("port"))
+
+        # Determine if this request should be filtered
+        is_health_check = False
+        host = None
+        path = flow.request.path
+        if sni or host_header:
+            host = str(sni or host_header)
+            if host.startswith("b'"):
+                host = host[2:-1]
+            log_url = flow.request.scheme + '://' + host + path
+
+            # If https, we have already checked that the client/server are legal in bubble_conn_check.py
+            # If http, we validate client/server here
+            if is_http:
+                fqdns = [host]
+                if is_bubble_request(server_addr, fqdns):
+                    is_health_check = path.startswith(HEALTH_CHECK_URI)
+                    if not is_health_check:
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug('bubble_handle_request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path)
+                        add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
+                        add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path)
+                        return None
+
+                elif is_sage_request(server_addr, fqdns):
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('bubble_handle_request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr)
+                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
+                    add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path)
+                    return None
+
+                elif is_not_from_vpn(client_addr):
+                    # todo: add to fail2ban
+                    if bubble_log.isEnabledFor(WARNING):
+                        bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url)
+                    bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns)
+                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
+                    return None
+
+            if is_bubble_special_path(path):
+                add_flow_ctx(flow, CTX_BUBBLE_SPECIAL, True)
+            else:
+                matcher_response = self.get_matchers(flow, sni or host_header)
+                if matcher_response:
+                    has_decision = 'decision' in matcher_response and matcher_response['decision'] is not None
+                    if has_decision and matcher_response['decision'] == 'pass_thru':
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug('bubble_handle_request: passthru response returned, passing thru...')
+                        add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True)
+                        bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url)
+                        return host
+
+                    elif has_decision and matcher_response['decision'].startswith('abort_'):
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug('bubble_handle_request: found abort code: ' + str(matcher_response['decision']) + ', aborting')
+                        if matcher_response['decision'] == 'abort_ok':
+                            abort_code = 200
+                        elif matcher_response['decision'] == 'abort_not_found':
+                            abort_code = 404
+                        else:
+                            if bubble_log.isEnabledFor(DEBUG):
+                                bubble_log.debug('bubble_handle_request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found')
+                            abort_code = 404
+                        flow.request.headers = nheaders.Headers([])
+                        flow.request.content = b''
+                        add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code)
+                        bubble_activity_log(client_addr, server_addr, 'http_abort' + str(abort_code), log_url)
+                        return None
+
+                    elif has_decision and matcher_response['decision'] == 'no_match':
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug('bubble_handle_request: decision was no_match, passing thru...')
+                        bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url)
+                        return host
+
+                    elif ('matchers' in matcher_response
+                          and 'request_id' in matcher_response
+                          and len(matcher_response['matchers']) > 0):
+                        req_id = matcher_response['request_id']
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug("bubble_handle_request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
+                        add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers'])
+                        add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id)
+                        bubble_activity_log(client_addr, server_addr, 'http_match', log_url)
+                    else:
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug('bubble_handle_request: no rules returned, passing thru...')
+                        bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url)
+                else:
+                    if not is_health_check:
+                        if bubble_log.isEnabledFor(DEBUG):
+                            bubble_log.debug('bubble_handle_request: no matcher_response returned, passing thru...')
+                        # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url)
+
+        elif is_http and is_not_from_vpn(client_addr):
+            # todo: add to fail2ban
+            if bubble_log.isEnabledFor(WARNING):
+                bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr)
+            bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr])
+            add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
+            return None
+
+        else:
+            if bubble_log.isEnabledFor(WARNING):
+                bubble_log.warning('bubble_handle_request: no sni/host found, not applying rules to path: ' + path)
+            bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr])
+
+        flow.request.host_header = host_header
+        if host:
+            flow.request.host = host
+        else:
+            flow.request.host = host_header
+        flow.request.port = port
+        return host
+
+    def requestheaders(self, flow):
+        host = self.bubble_handle_request(flow)
+        path = flow.request.path
+        flow.request.capture_stream = True
+
+        if is_bubble_special_path(path):
+            # if bubble_log.isEnabledFor(DEBUG):
+            #     bubble_log.debug('request: is_bubble_special_path('+path+') returned true, sending special bubble response')
+            special_bubble_response(flow)
+
+        elif host is not None:
+            client_addr = flow.client_conn.address[0]
+            if is_flex_domain(client_addr, host):
+                flex_flow = new_flex_flow(client_addr, host, flow)
+                add_flow_ctx(flow, CTX_BUBBLE_FLEX, flex_flow)
+                if bubble_log.isEnabledFor(DEBUG):
+                    bubble_log.debug('request: is_flex_domain('+host+') returned true, setting ctx: '+CTX_BUBBLE_FLEX)
+
+
+addons = [Rerouter()]

bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py

@@ -1,191 +0,0 @@
-#
-# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
-#
-import re
-import time
-import uuid
-from mitmproxy.net.http import headers as nheaders
-
-from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, HEALTH_CHECK_URI, \
-    CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \
-    add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn
-from bubble_config import bubble_host, bubble_host_alias
-
-class Rerouter:
-    @staticmethod
-    def get_matchers(flow, host):
-        if host is None:
-            return None
-
-        is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI)
-        if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX):
-            if not is_health_check:
-                bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path)
-            return None
-
-        client_addr = str(flow.client_conn.address[0])
-        server_addr = str(flow.server_conn.address[0])
-        try:
-            host = host.decode()
-        except (UnicodeDecodeError, AttributeError):
-            try:
-                host = str(host)
-            except Exception as e:
-                bubble_log('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e))
-                return None
-
-        if host == bubble_host or host == bubble_host_alias:
-            bubble_log('get_matchers: request is for bubble itself ('+host+'), not matching')
-            return None
-
-        req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time())
-        bubble_log("get_matchers: requesting match decision for req_id="+req_id)
-        resp = bubble_matchers(req_id, client_addr, server_addr, flow, host)
-
-        if not resp:
-            bubble_log('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host))
-            return None
-
-        matchers = []
-        if 'matchers' in resp and resp['matchers'] is not None:
-            for m in resp['matchers']:
-                if 'urlRegex' in m:
-                    bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex'])
-                else:
-                    bubble_log('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping')
-                    continue
-                if re.match(m['urlRegex'], flow.request.path):
-                    bubble_log('get_matchers: rule matched, adding rule: '+m['rule'])
-                    matchers.append(m)
-                else:
-                    bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule'])
-        else:
-            bubble_log('get_matchers: no matchers. response='+repr(resp))
-
-        decision = None
-        if 'decision' in resp:
-            decision = resp['decision']
-
-        matcher_response = {'decision': decision, 'matchers': matchers, 'request_id': req_id}
-        bubble_log("get_matchers: returning "+repr(matcher_response))
-        return matcher_response
-
-    def request(self, flow):
-        client_addr = flow.client_conn.address[0]
-        server_addr = flow.server_conn.address[0]
-        is_http = False
-        if flow.client_conn.tls_established:
-            flow.request.scheme = "https"
-            sni = flow.client_conn.connection.get_servername()
-            port = 443
-        else:
-            flow.request.scheme = "http"
-            sni = None
-            port = 80
-            is_http = True
-
-        # check if https and sni is missing but we have a host header, fill in the sni
-        
-        host_header = flow.request.host_header
-        # bubble_log("dns_spoofing.request: host_header is "+repr(host_header))
-        if host_header:
-            m = parse_host_header.match(host_header)
-            if m:
-                host_header = m.group("host").strip("[]")
-                if m.group("port"):
-                    port = int(m.group("port"))
-
-        # Determine if this request should be filtered
-        is_health_check = False
-        if sni or host_header:
-            host = str(sni or host_header)
-            if host.startswith("b'"):
-                host = host[2:-1]
-            log_url = flow.request.scheme + '://' + host + flow.request.path
-
-            # If https, we have already checked that the client/server are legal in bubble_conn_check.py
-            # If http, we validate client/server here
-            if is_http:
-                fqdns = [host]
-                if is_bubble_request(server_addr, fqdns):
-                    is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI)
-                    if not is_health_check:
-                        bubble_log('dns_spoofing.request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns)+', path='+flow.request.path)
-                        add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
-                        add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path)
-                        return
-
-                elif is_sage_request(server_addr, fqdns):
-                    bubble_log('dns_spoofing.request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr)
-                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
-                    add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path)
-                    return
-
-                elif is_not_from_vpn(client_addr):
-                    # todo: add to fail2ban
-                    bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns))
-                    bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns)
-                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
-                    return
-
-            matcher_response = self.get_matchers(flow, sni or host_header)
-            if matcher_response:
-                if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru':
-                    bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...')
-                    add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True)
-                    bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url)
-                    return
-
-                elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'):
-                    bubble_log('dns_spoofing.request: found abort code: ' + str(matcher_response['decision']) + ', aborting')
-                    if matcher_response['decision'] == 'abort_ok':
-                        abort_code = 200
-                    elif matcher_response['decision'] == 'abort_not_found':
-                        abort_code = 404
-                    else:
-                        bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found')
-                        abort_code = 404
-                    flow.request.headers = nheaders.Headers([])
-                    flow.request.content = b''
-                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code)
-                    bubble_activity_log(client_addr, server_addr, 'http_abort' + str(abort_code), log_url)
-                    return
-
-                elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match':
-                    bubble_log('dns_spoofing.request: decision was no_match, passing thru...')
-                    bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url)
-                    return
-
-                elif ('matchers' in matcher_response
-                      and 'request_id' in matcher_response
-                      and len(matcher_response['matchers']) > 0):
-                    req_id = matcher_response['request_id']
-                    bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
-                    add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers'])
-                    add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id)
-                    bubble_activity_log(client_addr, server_addr, 'http_match', log_url)
-                else:
-                    bubble_log('dns_spoofing.request: no rules returned, passing thru...')
-                    bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url)
-            else:
-                if not is_health_check:
-                    bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...')
-                    # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url)
-
-        elif is_http and is_not_from_vpn(client_addr):
-            # todo: add to fail2ban
-            bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr)
-            bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr])
-            add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
-            return
-
-        else:
-            bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path)
-            bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr])
-
-        flow.request.host_header = host_header
-        flow.request.host = sni or host_header
-        flow.request.port = port
-
-
-addons = [Rerouter()]

bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh

@@ -14,7 +14,7 @@ fi
 
 cd /home/mitmproxy/mitmproxy && \
 ./dev.sh ${SETUP_VENV} && . ./venv/bin/activate && \
-mitmdump \
+BUBBLE_PORT=${PORT} mitmdump \
   --listen-host 0.0.0.0 \
   --listen-port ${PORT} \
   --showhost \
@@ -23,10 +23,10 @@ mitmdump \
   --set block_private=false \
   --set termlog_verbosity=warn \
   --set flow_detail=0 \
-  --set stream_large_bodies=5m \
+  --set stream_large_bodies=1 \
   --set keep_host_header \
   -s ./bubble_debug.py \
-  -s ./dns_spoofing.py \
   -s ./bubble_conn_check.py \
+  -s ./bubble_request.py \
   -s ./bubble_modify.py \
   --mode transparent

bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml

@@ -41,7 +41,7 @@
   get_url:
     url: https://github.com/getbubblenow/bubble-dist/raw/master/mitmproxy/mitmproxy.zip
     dest: /tmp/mitmproxy.zip
-    checksum: sha256:b288b55e0b25e453fbc08ec2ad130ea22033f200cab1319e7bc8d2332e538ec5
+    checksum: sha256:9883696cc304326d0d94f6d0a498721de9b8c77c833b30f4b1661646467172bf
 
 - name: Unzip mitmproxy.zip
   unarchive:
@@ -59,9 +59,11 @@
   with_items:
     - bubble_api.py
     - bubble_debug.py
-    - dns_spoofing.py
+    - bubble_request.py
     - bubble_conn_check.py
     - bubble_modify.py
+    - bubble_flex.py
+    - bubble_flex_passthru.py
     - run_mitm.sh
 
 - name: Install cert helper scripts
@@ -90,6 +92,13 @@
 - name: Install mitmproxy dependencies
   shell: su - mitmproxy -c "bash -c 'cd /home/mitmproxy/mitmproxy && ./dev.sh'"
 
+- name: Overwrite _client.py from httpx to fix bug with HTTP/2 redirects
+  file:
+    src: _client.py
+    dest: /home/mitmproxxy/mitmproxy/venv/lib/python3.8/site-packages/httpx/_client.py
+    owner: mitmproxy
+    group: mitmproxy
+
 - name: Install mitm_monitor
   copy:
     src: "mitm_monitor.sh"

bubble-server/src/test/java/bubble/test/filter/FlexRouterProximityComparatorTest.java

@@ -0,0 +1,101 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.test.filter;
+
+import bubble.cloud.geoLocation.GeoLocation;
+import bubble.model.device.DeviceStatus;
+import bubble.model.device.FlexRouter;
+import bubble.service.device.FlexRouterInfo;
+import bubble.service.device.FlexRouterProximityComparator;
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class FlexRouterProximityComparatorTest {
+
+    public static final GeoLocation GEO_NULL = new GeoLocation().setLat(null).setLon(null);
+    public static final GeoLocation GEO_NEW_YORK = new GeoLocation().setLat("40.661").setLon("-73.944");
+    public static final GeoLocation GEO_SINGAPORE = new GeoLocation().setLat("1.283333").setLon("103.833333");
+    public static final GeoLocation GEO_LONDON = new GeoLocation().setLat("51.507222").setLon("-0.1275");
+    public static final GeoLocation GEO_ATLANTA = new GeoLocation().setLat("33.755").setLon("-84.39");
+    public static final GeoLocation GEO_CHICAGO = new GeoLocation().setLat("41.881944").setLon("-87.627778");
+
+    private static FlexRouter router(int port) {
+        return new FlexRouter().setPort(port).setIp("127.0.0."+port);
+    }
+
+    private static int deviceIp = 1;
+    private static DeviceStatus device(GeoLocation geo) { return new DeviceStatus().setLocation(geo).setIp("127.2.2."+(deviceIp++)); }
+
+    public static final FlexRouterInfo ROUTER_NEW_YORK = new FlexRouterInfo(router(200), device(GEO_NEW_YORK));
+    public static final FlexRouterInfo ROUTER_SINGAPORE = new FlexRouterInfo(router(201), device(GEO_SINGAPORE));
+    public static final FlexRouterInfo ROUTER_LONDON = new FlexRouterInfo(router(202), device(GEO_LONDON));
+    public static final FlexRouterInfo ROUTER_ATLANTA = new FlexRouterInfo(router(203), device(GEO_ATLANTA));
+    public static final FlexRouterInfo ROUTER_CHICAGO = new FlexRouterInfo(router(204), device(GEO_CHICAGO));
+    public static final FlexRouterInfo ROUTER_NULL = new FlexRouterInfo(router(205), device(GEO_NULL));
+
+    private static final List<FlexRouterInfo> TEST_INFO = Arrays.asList(
+            ROUTER_NEW_YORK,
+            ROUTER_SINGAPORE,
+            ROUTER_LONDON,
+            ROUTER_ATLANTA,
+            ROUTER_CHICAGO,
+            ROUTER_NULL);
+
+    private static final List<FlexRouterInfo> EXPECTED_ATLANTA = Arrays.asList(
+            ROUTER_ATLANTA,
+            ROUTER_CHICAGO,
+            ROUTER_NEW_YORK,
+            ROUTER_LONDON,
+            ROUTER_SINGAPORE,
+            ROUTER_NULL);
+
+    private static final List<FlexRouterInfo> EXPECTED_LONDON = Arrays.asList(
+            ROUTER_LONDON,
+            ROUTER_NEW_YORK,
+            ROUTER_CHICAGO,
+            ROUTER_ATLANTA,
+            ROUTER_SINGAPORE,
+            ROUTER_NULL);
+
+    private static final List<FlexRouterInfo> EXPECTED_ATLANTA_PREFER_SINGAPORE = Arrays.asList(
+            ROUTER_SINGAPORE,
+            ROUTER_ATLANTA,
+            ROUTER_CHICAGO,
+            ROUTER_NEW_YORK,
+            ROUTER_LONDON,
+            ROUTER_NULL);
+
+    @Test public void testProximitySortAtlanta () throws Exception {
+        testProximitySort(GEO_ATLANTA, EXPECTED_ATLANTA);
+    }
+
+    @Test public void testProximitySortLondon () throws Exception {
+        testProximitySort(GEO_LONDON, EXPECTED_LONDON);
+    }
+
+    @Test public void testProximitySortFromAtlantaWithPreferredIpInSingapore () throws Exception {
+        testProximitySort(GEO_ATLANTA, EXPECTED_ATLANTA_PREFER_SINGAPORE, ROUTER_SINGAPORE.getVpnIp());
+    }
+
+    private void testProximitySort(GeoLocation geo, List<FlexRouterInfo> expected) throws Exception {
+        testProximitySort(geo, expected, "127.3.3.3");
+    }
+
+    private void testProximitySort(GeoLocation geo, List<FlexRouterInfo> expected, String preferredIp) throws Exception {
+        final List<FlexRouterInfo> test = new ArrayList<>(TEST_INFO);
+        Collections.shuffle(test);
+        final FlexRouterProximityComparator comparator = new FlexRouterProximityComparator(geo, preferredIp);
+        final Set<FlexRouterInfo> sorted = new TreeSet<>(comparator);
+        sorted.addAll(test);
+        final List<FlexRouterInfo> actual = new ArrayList<>(sorted);
+        assertEquals("wrong number of results", expected.size(), actual.size());
+        for (int i=0; i<expected.size(); i++) {
+            assertEquals("incorrect sort at index "+i, expected.get(i), actual.get(i));
+        }
+    }
+}

bubble-server/src/test/java/bubble/test/system/AuthTest.java

@@ -24,16 +24,16 @@ public class AuthTest extends ActivatedBubbleModelTestBase {
         accountDAO.update(rootUser.setHashedPassword(new HashedPassword(ROOT_PASSWORD)));
     }
 
-    @Test public void testBasicAuth () throws Exception { modelTest("auth/basic_auth"); }
-    @Test public void testAccountCrud () throws Exception { modelTest("auth/account_crud"); }
-    @Test public void testDeviceCrud () throws Exception { modelTest("auth/device_crud"); }
-    @Test public void testRegistration () throws Exception { modelTest("auth/account_registration"); }
-    @Test public void testForgotPassword () throws Exception { modelTest("auth/forgot_password"); }
-    @Test public void testChangePassword () throws Exception { modelTest("auth/change_password"); }
+    @Test public void testBasicAuth           () throws Exception { modelTest("auth/basic_auth"); }
+    @Test public void testAccountCrud         () throws Exception { modelTest("auth/account_crud"); }
+    @Test public void testDeviceCrud          () throws Exception { modelTest("auth/device_crud"); }
+    @Test public void testRegistration        () throws Exception { modelTest("auth/account_registration"); }
+    @Test public void testForgotPassword      () throws Exception { modelTest("auth/forgot_password"); }
+    @Test public void testChangePassword      () throws Exception { modelTest("auth/change_password"); }
     @Test public void testChangeAdminPassword () throws Exception { modelTest("auth/change_admin_password"); }
-    @Test public void testTotpAuth () throws Exception { modelTest("auth/totp_auth"); }
-    @Test public void testMultifactorAuth () throws Exception { modelTest("auth/multifactor_auth"); }
-    @Test public void testDownloadAccount () throws Exception { modelTest("auth/download_account"); }
-    @Test public void testNetworkAuth () throws Exception { modelTest("auth/network_auth"); }
+    @Test public void testTotpAuth            () throws Exception { modelTest("auth/totp_auth"); }
+    @Test public void testMultifactorAuth     () throws Exception { modelTest("auth/multifactor_auth"); }
+    @Test public void testDownloadAccount     () throws Exception { modelTest("auth/download_account"); }
+    @Test public void testNetworkAuth         () throws Exception { modelTest("auth/network_auth"); }
 
 }

pom.xml

@@ -76,6 +76,7 @@
                         <include>bubble.test.filter.ProxyTest</include>
                         <include>bubble.test.filter.TrafficAnalyticsTest</include>
                         <include>bubble.test.filter.BlockSummaryTest</include>
+                        <include>bubble.test.filter.FlexRouterProximityComparatorTest</include>
                         <include>bubble.test.system.BackupTest</include>
                         <include>bubble.test.system.NetworkTest</include>
                         <include>bubble.abp.spec.BlockListTest</include>

utils/abp-parser

@@ -1 +1 @@
-Subproject commit 23f140fc2f99df9a5712df4faf08e1068421a57b
+Subproject commit 072a11decff65461f12f47e5dae763b56a5a3247