push removed routes to flex routers. only track flex routes for first admin, they apply to all users

bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java

@@ -21,8 +21,7 @@ import java.util.stream.Collectors;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.json.JsonUtil.json;
-import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
-import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
+import static org.cobbzilla.wizard.resources.ResourceUtil.*;
 
 @Slf4j
 public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
@@ -63,12 +62,14 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
     }
 
     private Set<FlexFeed> loadManageFlexFeeds(Account account, BubbleApp app) {
+        if (!account.admin()) throw forbiddenEx();
         final TlsPassthruConfig config = getConfig(account, app);
         config.getFlexSet(); // ensure names are initialized
         return config.getFlexFeedSet();
     }
 
     private Set<FlexFqdn> loadManageFlexDomains(Account account, BubbleApp app) {
+        if (!account.admin()) throw forbiddenEx();
         final TlsPassthruConfig config = getConfig(account, app);
         return !config.hasFlexFqdnList() ? Collections.emptySet() :
                 Arrays.stream(config.getFlexFqdnList())
@@ -154,6 +155,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
     }
 
     private List<TlsPassthruFqdn> addFlexFqdn(Account account, BubbleApp app, JsonNode data) {
+        if (!account.admin()) throw forbiddenEx();
         final JsonNode fqdnNode = data.get(PARAM_FLEX_FQDN);
         if (fqdnNode == null || fqdnNode.textValue() == null || empty(fqdnNode.textValue().trim())) {
             throw invalidEx("err.flexFqdn.flexFqdnRequired");
@@ -177,6 +179,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
     }
 
     private Set<TlsPassthruFeed> addFlexFeed(Account account, BubbleApp app, Map<String, String> params, JsonNode data) {
+        if (!account.admin()) throw forbiddenEx();
         final JsonNode urlNode = data.get(PARAM_FLEX_FEED_URL);
         if (urlNode == null || urlNode.textValue() == null || empty(urlNode.textValue().trim())) {
             throw invalidEx("err.flexFeedUrl.feedUrlRequired");
@@ -233,6 +236,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
     }
 
     private List<TlsPassthruFqdn> removeFlexFqdn(Account account, BubbleApp app, String id) {
+        if (!account.admin()) throw forbiddenEx();
         final AppRule rule = loadRule(account, app);
         loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
         final TlsPassthruConfig config = getConfig(account, app);
@@ -245,6 +249,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
     }
 
     public Set<TlsPassthruFeed> removeFlexFeed(Account account, BubbleApp app, String id) {
+        if (!account.admin()) throw forbiddenEx();
         final AppRule rule = loadRule(account, app);
         loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
         final TlsPassthruConfig config = getConfig(account, app).removeFlexFeed(id);

bubble-server/src/main/java/bubble/dao/SessionDAO.java

@@ -4,13 +4,22 @@
  */
 package bubble.dao;
 
+import bubble.dao.account.AccountDAO;
 import bubble.model.account.Account;
 import org.cobbzilla.wizard.dao.AbstractSessionDAO;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
 @Repository
 public class SessionDAO extends AbstractSessionDAO<Account> {
 
+    @Autowired private AccountDAO accountDAO;
+
     @Override protected boolean canStartSession(Account account) { return !account.suspended(); }
 
+    @Override public String create(Account account) {
+        account.setFirstAdmin(account.getUuid().equals(accountDAO.getFirstAdmin().getUuid()));
+        return super.create(account);
+    }
+
 }

bubble-server/src/main/java/bubble/model/account/Account.java

@@ -141,6 +141,9 @@ public class Account extends IdentifiableBaseParentEntity implements TokenPrinci
     @Getter @Setter private Boolean admin = false;
     public boolean admin () { return bool(admin); }
 
+    // set in SessionDAO so UI can know if the user is first admin
+    @Transient @Getter @Setter private boolean firstAdmin = false;
+
     @ECIndex(unique=true, where="sage = true") @ECField(index=70)
     @Getter @Setter private Boolean sage = false;
     public boolean sage () { return bool(sage); }

bubble-server/src/main/java/bubble/model/device/DeviceStatus.java

@@ -34,6 +34,7 @@ public class DeviceStatus {
     @Getter @Setter private String bytesReceived;
     @Getter @Setter private String receivedUnits;
 
+    @Getter @Setter private Integer lastHandshakeDays;
     @Getter @Setter private Integer lastHandshakeHours;
     @Getter @Setter private Integer lastHandshakeMinutes;
     @Getter @Setter private Integer lastHandshakeSeconds;
@@ -101,6 +102,7 @@ public class DeviceStatus {
                         String unit = parts[i+1].trim();
                         if (unit.endsWith(",")) unit = unit.substring(0, unit.length()-1);
                         switch (unit) {
+                            case "day": case "days": setLastHandshakeDays(count); break;
                             case "hour": case "hours": setLastHandshakeHours(count); break;
                             case "minute": case "minutes": setLastHandshakeMinutes(count); break;
                             case "second": case "seconds": setLastHandshakeSeconds(count); break;

bubble-server/src/main/java/bubble/model/device/FlexRouterRemoveRoutes.java

@@ -0,0 +1,19 @@
+package bubble.model.device;
+
+import lombok.Getter;
+import lombok.experimental.Accessors;
+
+import java.util.Collection;
+
+@Accessors(chain=true)
+public class FlexRouterRemoveRoutes {
+
+    @Getter private final FlexRouterPing ping;
+    @Getter private final String[] routes;
+
+    public FlexRouterRemoveRoutes (FlexRouter router, Collection<String> routes) {
+        this.ping = router.pingObject();
+        this.routes = routes.toArray(String[]::new);
+    }
+
+}

bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java

@@ -9,6 +9,7 @@ import bubble.dao.device.FlexRouterDAO;
 import bubble.model.device.DeviceStatus;
 import bubble.model.device.FlexRouter;
 import bubble.model.device.FlexRouterPing;
+import bubble.model.device.FlexRouterRemoveRoutes;
 import bubble.service.cloud.GeoService;
 import lombok.AllArgsConstructor;
 import lombok.Cleanup;
@@ -17,6 +18,8 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.cobbzilla.util.collection.ExpirationEvictionPolicy;
+import org.cobbzilla.util.collection.ExpirationMap;
 import org.cobbzilla.util.collection.SingletonSet;
 import org.cobbzilla.util.daemon.AwaitResult;
 import org.cobbzilla.util.daemon.SimpleDaemon;
@@ -24,6 +27,7 @@ import org.cobbzilla.util.http.HttpRequestBean;
 import org.cobbzilla.util.http.HttpResponseBean;
 import org.cobbzilla.util.http.HttpUtil;
 import org.cobbzilla.util.io.FileUtil;
+import org.cobbzilla.util.string.StringUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
@@ -35,11 +39,14 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.HOME_DIR;
 import static bubble.model.device.FlexRouterPing.MAX_PING_AGE;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.function.Function.identity;
 import static org.cobbzilla.util.daemon.Await.awaitAll;
 import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
@@ -57,13 +64,18 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute
 
     private static final long PING_SLEEP_FACTOR = SECONDS.toMillis(2);
 
-    // HttpClient timeouts are in seconds
-    public static final int DEFAULT_PING_TIMEOUT = (int) SECONDS.toSeconds(MAX_PING_AGE/2);
+    public static final int DEFAULT_PING_TIMEOUT = (int) MAX_PING_AGE/2;
     public static final RequestConfig DEFAULT_PING_REQUEST_CONFIG = RequestConfig.custom()
             .setConnectTimeout(DEFAULT_PING_TIMEOUT)
             .setSocketTimeout(DEFAULT_PING_TIMEOUT)
             .setConnectionRequestTimeout(DEFAULT_PING_TIMEOUT).build();
 
+    public static final int DEFAULT_UPDATE_ROUTES_TIMEOUT = (int) SECONDS.toMillis(10);
+    public static final RequestConfig DEFAULT_UPDATE_ROUTES_REQUEST_CONFIG = RequestConfig.custom()
+            .setConnectTimeout(DEFAULT_UPDATE_ROUTES_TIMEOUT)
+            .setSocketTimeout(DEFAULT_UPDATE_ROUTES_TIMEOUT)
+            .setConnectionRequestTimeout(DEFAULT_UPDATE_ROUTES_TIMEOUT).build();
+
     // wait for ssh key to be written
     private static final long FIRST_TIME_WAIT = SECONDS.toMillis(10);
     private static final long INTERRUPT_WAIT = FIRST_TIME_WAIT/2;
@@ -71,14 +83,18 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute
     public static final long PING_ALL_TIMEOUT
             = (SECONDS.toMillis(1) * DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + FIRST_TIME_WAIT;
 
+    public static final long UPDATE_ROUTES_ALL_TIMEOUT = SECONDS.toMillis(30);
+
     // thread pool size
     public static final int DEFAULT_MAX_TUNNELS = 5;
 
-    private static CloseableHttpClient getHttpClient() {
+    private static CloseableHttpClient getHttpClient(RequestConfig requestConfig) {
         return HttpClientBuilder.create()
-                .setDefaultRequestConfig(DEFAULT_PING_REQUEST_CONFIG)
+                .setDefaultRequestConfig(requestConfig)
                 .build();
     }
+    private static CloseableHttpClient getPingHttpClient() { return getHttpClient(DEFAULT_PING_REQUEST_CONFIG); }
+    private static CloseableHttpClient getUpdateRoutesHttpClient() { return getHttpClient(DEFAULT_UPDATE_ROUTES_REQUEST_CONFIG); }
 
     public static final long DEFAULT_SLEEP_TIME = MINUTES.toMillis(2);
 
@@ -162,10 +178,52 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute
         }
     }
 
+    private final AtomicReference<Set<String>> mostRecentFlexDomains = new AtomicReference<>(null);
+    private final Map<String, String> recentlyRemovedFlexDomains = new ExpirationMap<>(MINUTES.toMillis(20), ExpirationEvictionPolicy.ctime_or_atime);
+
+    public void updateFlexRoutes(Set<String> flexDomains) {
+        synchronized (mostRecentFlexDomains) {
+            final Set<String> mostRecentDomains = mostRecentFlexDomains.get() == null ? Collections.emptySet() : mostRecentFlexDomains.get();
+            if (!mostRecentDomains.isEmpty()) {
+                // what should we remove now?
+                final Set<String> routesToRemove = new HashSet<>(mostRecentDomains);
+                routesToRemove.removeAll(flexDomains);
+
+                // add to recently removed, we will remove all of these in case a previous update was missed
+                recentlyRemovedFlexDomains.putAll(routesToRemove.stream().collect(Collectors.toMap(identity(), identity())));
+
+                // but exclude domains that are currently flex routed, we don't want to remove these
+                for (String current : flexDomains) recentlyRemovedFlexDomains.remove(current);
+
+                if (!recentlyRemovedFlexDomains.isEmpty()) {
+                    try {
+                        @Cleanup final CloseableHttpClient httpClient = getUpdateRoutesHttpClient();
+                        final List<FlexRouter> routers = flexRouterDAO.findEnabledAndRegistered();
+                        if (log.isDebugEnabled()) log.debug("updateFlexRoutes: updating "+routers.size()+" routers");
+                        final List<Future<?>> futures = new ArrayList<>();
+                        @Cleanup("shutdownNow") final ExecutorService exec = fixedPool(DEFAULT_MAX_TUNNELS, "StandardFlexRouterService.updateFlexRoutes");
+                        final Set<String> routes = recentlyRemovedFlexDomains.keySet();
+                        for (FlexRouter router : routers) {
+                            if (log.isDebugEnabled()) log.debug("updateFlexRoutes: starting job for router: " + router + " with routes to remove: "+StringUtil.toString(routes));
+                            futures.add(exec.submit(new FlexRemoveRoutesJob(router, routes, httpClient)));
+                        }
+                        final AwaitResult<Boolean> awaitResult = awaitAll(futures, UPDATE_ROUTES_ALL_TIMEOUT);
+                        if (log.isTraceEnabled()) log.trace("updateFlexRoutes: awaitResult=" + awaitResult);
+                    } catch (Exception e) {
+                        log.error("updateFlexRoutes: " + shortError(e));
+                    }
+                }
+            } else {
+                if (log.isDebugEnabled()) log.debug("updateFlexRoutes: no routes to remove");
+            }
+            mostRecentFlexDomains.set(flexDomains);
+        }
+    }
+
     @Override protected void process() {
         synchronized (interrupted) { interrupted.set(false); }
         try {
-            @Cleanup final CloseableHttpClient httpClient = getHttpClient();
+            @Cleanup final CloseableHttpClient httpClient = getPingHttpClient();
             final List<FlexRouter> routers = flexRouterDAO.findEnabledAndRegistered();
             if (log.isTraceEnabled()) log.trace("process: starting, will ping "+routers.size()+" routers");
             final List<Future<?>> futures = new ArrayList<>();
@@ -219,7 +277,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute
                 }
 
             } catch (Exception e) {
-                log.info(prefix+"error: "+shortError(e));
+                log.error(prefix+"error: "+shortError(e));
             }
             setStatus(router, FlexRouterStatus.unreachable);
         }
@@ -298,7 +356,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute
                 boolean update = false;
                 if (!active) {
                     if (pollFailures.computeIfAbsent(router.getUuid(), k -> new AtomicInteger(0)).incrementAndGet() > MAX_POLL_FAILURES) {
-                        log.warn("process: too many poll failures for router ("+router+"), marking unregistered");
+                        if (log.isWarnEnabled()) log.warn("process: too many poll failures for router ("+router+"), marking unregistered");
                         router.setRegistered(false);
                         update = true;
                     }
@@ -320,4 +378,31 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute
         }
     }
 
+    @AllArgsConstructor
+    private static class FlexRemoveRoutesJob implements Callable<Boolean> {
+        private final FlexRouter router;
+        private final Collection<String> routes;
+        private final HttpClient httpClient;
+
+        @Override public Boolean call() {
+            final String removeUrl = router.proxyBaseUri() + "/remove";
+            final HttpRequestBean request = new HttpRequestBean(POST, removeUrl);
+            final String prefix = "FlexRouterRemoveJob(" + router + ", "+ StringUtil.toString(routes)+"): ";
+            request.setEntity(json(new FlexRouterRemoveRoutes(router, routes)));
+            try {
+                if (log.isDebugEnabled()) log.debug(prefix+"sending JSON message to remove routes...");
+                final HttpResponseBean response = HttpUtil.getResponse(request, httpClient);
+                if (!response.isOk()) {
+                    log.error(prefix+"response not OK: "+response);
+                } else {
+                    if (log.isDebugEnabled()) log.debug(prefix+"routes removed from router");
+                    return true;
+                }
+            } catch (Exception e) {
+                log.error(prefix+"error: "+shortError(e));
+            }
+            return false;
+        }
+    }
+
 }

bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java

@@ -14,6 +14,7 @@ import bubble.model.device.Device;
 import bubble.rule.AppRuleDriver;
 import bubble.server.BubbleConfiguration;
 import bubble.service.device.DeviceService;
+import bubble.service.device.StandardFlexRouterService;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.SingletonList;
@@ -39,6 +40,7 @@ public class StandardAppPrimerService implements AppPrimerService {
     @Autowired private AppRuleDAO ruleDAO;
     @Autowired private RuleDriverDAO driverDAO;
     @Autowired private AppDataDAO dataDAO;
+    @Autowired private StandardFlexRouterService flexRouterService;
     @Autowired private RedisService redis;
     @Autowired private BubbleConfiguration configuration;
 
@@ -117,6 +119,13 @@ public class StandardAppPrimerService implements AppPrimerService {
             }
             if (accountDeviceIps.isEmpty()) return;
 
+            // flex domains can only be managed by the first admin
+            final Account firstAdmin = accountDAO.getFirstAdmin();
+            account.setFirstAdmin(account.getUuid().equals(firstAdmin.getUuid()));
+            boolean updateFlexRouters = false;
+            Set<String> flexDomains = null;
+            Set<String> flexExcludeDomains = null;
+
             final List<BubbleApp> appsToPrime = singleApp == null
                     ? appDAO.findByAccount(account.getUuid()).stream()
                     .filter(BubbleApp::canPrime)
@@ -145,8 +154,6 @@ public class StandardAppPrimerService implements AppPrimerService {
                         final Set<String> blockDomains = new HashSet<>();
                         final Set<String> whiteListDomains = new HashSet<>();
                         final Set<String> filterDomains = new HashSet<>();
-                        final Set<String> flexDomains = new HashSet<>();
-                        final Set<String> flexExcludeDomains = new HashSet<>();
                         for (AppMatcher matcher : matchers) {
                             final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device);
                             final Set<String> rejects = appRuleDriver.getPrimedRejectDomains();
@@ -173,17 +180,19 @@ public class StandardAppPrimerService implements AppPrimerService {
                             } else {
                                 filterDomains.addAll(filters);
                             }
-                            final Set<String> flexes = appRuleDriver.getPrimedFlexDomains();
-                            if (empty(flexes)) {
-                                log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                flexDomains.addAll(flexes);
-                            }
-                            final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
-                            if (empty(flexExcludes)) {
-                                log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                flexExcludeDomains.addAll(flexExcludes);
+                            if (account.isFirstAdmin() && flexDomains == null) {
+                                final Set<String> flexes = appRuleDriver.getPrimedFlexDomains();
+                                if (empty(flexes)) {
+                                    log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                                } else {
+                                    flexDomains = new HashSet<>(flexes);
+                                }
+                                final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
+                                if (empty(flexExcludes)) {
+                                    log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                                } else {
+                                    flexExcludeDomains = new HashSet<>(flexExcludes);
+                                }
                             }
                         }
                         if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) {
@@ -202,17 +211,23 @@ public class StandardAppPrimerService implements AppPrimerService {
                                 if (!empty(filterDomains)) {
                                     AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new));
                                 }
-                                if (!empty(flexDomains)) {
-                                    flexDomains.removeAll(flexExcludeDomains);
-                                    AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new));
-                                }
-                                if (!empty(flexExcludeDomains)) {
-                                    AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new));
+                                if (account.isFirstAdmin() && (!empty(flexDomains) || !empty(flexExcludeDomains))) {
+                                    updateFlexRouters = true;
+                                    if (!empty(flexDomains)) {
+                                        if (flexExcludeDomains != null) flexDomains.removeAll(flexExcludeDomains);
+                                        AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new));
+                                    }
+                                    if (!empty(flexExcludeDomains)) {
+                                        AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new));
+                                    }
                                 }
                             }
                         }
                     }
                 }
+                if (updateFlexRouters && !empty(flexDomains)) {
+                    flexRouterService.updateFlexRoutes(flexDomains);
+                }
             }
         } catch (Exception e) {
             die("_prime: "+shortError(e), e);

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json

@@ -48,6 +48,7 @@
       "name": "manageFlexDomains",
       "scope": "app",
       "root": "true",
+      "when": "account.firstAdmin === true",
       "fields": ["flexFqdn"],
       "actions": [
         {"name": "removeFlexFqdn", "index": 10},
@@ -61,6 +62,7 @@
       "name": "manageFlexFeeds",
       "scope": "app",
       "root": "true",
+      "when": "account.firstAdmin === true",
       "fields": ["flexFeedName", "flexFeedUrl"],
       "actions": [
         {"name": "removeFlexFeed", "index": 10},

bubble-web

@@ -1 +1 @@
-Subproject commit 37dd89bd78949a63c68a64596b4e1c105809a577
+Subproject commit 61a7f288e515a185f2055bbe1e513943d54ac66c