add flex config to TlsPassthru app, encrypt token column, fix update

bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java

@@ -2,13 +2,15 @@ package bubble.dao.device;
 
 import bubble.dao.account.AccountOwnedEntityDAO;
 import bubble.model.device.FlexRouter;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
 
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 import static org.hibernate.criterion.Restrictions.*;
 
-@Repository
+@Repository @Slf4j
 public class FlexRouterDAO extends AccountOwnedEntityDAO<FlexRouter> {
 
     @Override protected String getNameField() { return "ip"; }
@@ -16,7 +18,18 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO<FlexRouter> {
     public List<FlexRouter> findEnabledAndRegistered() {
         return list(criteria().add(and(
                 eq("enabled", true),
-                ne("port", 0),
+                gt("port", 1024),
+                le("port", 65535),
+                isNotNull("token"))));
+    }
+
+    public List<FlexRouter> findActive(long maxAge) {
+        return list(criteria().add(and(
+                eq("active", true),
+                eq("enabled", true),
+                gt("port", 1024),
+                le("port", 65535),
+                ge("lastSeen", now()-maxAge),
                 isNotNull("token"))));
     }
 

bubble-server/src/main/java/bubble/model/device/FlexRouter.java

@@ -15,6 +15,7 @@ import org.cobbzilla.wizard.model.IdentifiableBase;
 import org.cobbzilla.wizard.model.entityconfig.EntityFieldMode;
 import org.cobbzilla.wizard.model.entityconfig.EntityFieldType;
 import org.cobbzilla.wizard.model.entityconfig.annotations.*;
+import org.hibernate.annotations.Type;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -23,6 +24,8 @@ import javax.persistence.Transient;
 import static bubble.ApiConstants.EP_FLEX_ROUTERS;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING;
+import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
 
 @Entity
 @ECType(root=true) @ToString(of={"ip", "port"})
@@ -31,7 +34,7 @@ import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 @ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) })
 public class FlexRouter extends IdentifiableBase implements HasAccount {
 
-    public static final String[] UPDATE_FIELDS = { "enabled", "active", "proxy_port", "auth_token" };
+    public static final String[] UPDATE_FIELDS = { "enabled", "active", "proxy_port", "auth_token", "token" };
     public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip");
 
     public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); }
@@ -51,7 +54,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccount {
 
     // used for sending the port, we never send it back
     @Transient @Getter @Setter private Integer proxy_port;
-    public boolean hasProxyPort () { return proxy_port != null && proxy_port > 1024; }
+    public boolean hasProxyPort () { return proxy_port != null && proxy_port > 1024 && proxy_port < 65535; }
 
     public String id () { return getIp() + "/" + getUuid(); }
 
@@ -78,7 +81,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccount {
     @JsonIgnore @Transient public long getAge () { return lastSeen == null ? Long.MAX_VALUE : now() - lastSeen; }
 
     @ECSearchable(filter=true) @ECField(index=70)
-    @Column(length=100)
+    @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(100+ENC_PAD)+")")
     @JsonIgnore @Getter @Setter private String token;
     public boolean hasToken () { return !empty(token); }
 

bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java

@@ -5,22 +5,35 @@ import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.Accessors;
 
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
 
 @NoArgsConstructor @Accessors(chain=true)
 public class FlexRouterPing {
 
+    public static final long MAX_PING_AGE = SECONDS.toMillis(30);
+    public static final long MIN_PING_AGE = -1 * SECONDS.toMillis(5);
+
+    @Getter @Setter private long time;
     @Getter @Setter private String salt;
     @Getter @Setter private String hash;
 
     public FlexRouterPing (FlexRouter router) {
+        time = now();
         salt = randomAlphanumeric(50);
-        hash = sha256_hex(salt + ":" + router.getToken());
+        hash = sha256_hex(data(router));
     }
 
     public boolean validate(FlexRouter router) {
-        return sha256_hex(salt + ":" + router.getToken()).equals(hash);
+        if (empty(salt) || salt.length() < 50) return false;
+        final long age = now() - time;
+        if (age > MAX_PING_AGE || age < MIN_PING_AGE) return false;
+        return sha256_hex(data(router)).equals(hash);
     }
 
+    private String data(FlexRouter router) { return salt + ":" + time + ":" + router.getToken(); }
+
 }

bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java

@@ -155,7 +155,7 @@ public class AccountOwnedResource<E extends HasAccount, DAO extends AccountOwned
         }
         if (found != null) {
             if (!canUpdate(ctx, caller, found, request)) return ok(found);
-            setReferences(ctx, caller, request);
+            setReferences(ctx, req, caller, request);
             found.update(request);
             return ok(getDao().update(found));
         }

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java

@@ -34,6 +34,7 @@ import static org.cobbzilla.wizard.server.RestServerBase.reportError;
 public class TlsPassthruConfig {
 
     public static final long DEFAULT_TLS_FEED_REFRESH_INTERVAL = HOURS.toMillis(1);
+    public static final long DEFAULT_FLEX_FEED_REFRESH_INTERVAL = HOURS.toMillis(1);
     public static final String FEED_NAME_PREFIX = "# Name:";
 
     @Getter @Setter private String[] fqdnList;
@@ -70,13 +71,54 @@ public class TlsPassthruConfig {
                 .toArray(TlsPassthruFeed[]::new));
     }
 
-    private Map<String, Set<String>> recentFeedValues = new HashMap<>();
+    private final Map<String, Set<String>> recentFeedValues = new HashMap<>();
 
     @JsonIgnore public Set<TlsPassthruFeed> getFeedSet() {
         final TlsPassthruFeed[] feedList = getFeedList();
         return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
     }
 
+    @Getter @Setter private String[] flexFqdnList;
+    public boolean hasFlexFqdnList () { return !empty(flexFqdnList); }
+    public boolean hasFlexFqdn(String flexFqdn) { return hasFlexFqdnList() && ArrayUtils.indexOf(flexFqdnList, flexFqdn) != -1; }
+
+    public TlsPassthruConfig addFlexFqdn(String flexFqdn) {
+        return setFlexFqdnList(Arrays.stream(ArrayUtil.append(flexFqdnList, flexFqdn)).collect(Collectors.toSet()).toArray(String[]::new));
+    }
+
+    public TlsPassthruConfig removeFlexFqdn(String id) {
+        return !hasFlexFqdnList() ? this :
+                setFlexFqdnList(Arrays.stream(getFlexFqdnList())
+                        .filter(flexFqdn -> !flexFqdn.equalsIgnoreCase(id.trim()))
+                        .toArray(String[]::new));
+    }
+
+    @Getter @Setter private TlsPassthruFeed[] flexFeedList;
+    public boolean hasFlexFeedList () { return !empty(flexFeedList); }
+    public boolean hasFlexFeed (TlsPassthruFeed flexFeed) {
+        return hasFlexFeedList() && Arrays.stream(flexFeedList).anyMatch(f -> f.getFeedUrl().equals(flexFeed.getFeedUrl()));
+    }
+
+    public TlsPassthruConfig addFlexFeed(TlsPassthruFeed flexFeed) {
+        final Set<TlsPassthruFeed> flexFeeds = getFlexFeedSet();
+        if (empty(flexFeeds)) return setFlexFeedList(new TlsPassthruFeed[] {flexFeed});
+        flexFeeds.add(flexFeed);
+        return setFlexFeedList(flexFeeds.toArray(EMPTY_FEEDS));
+    }
+
+    public TlsPassthruConfig removeFlexFeed(String id) {
+        return setFlexFeedList(getFlexFeedSet().stream()
+                .filter(flexFeed -> !flexFeed.getId().equals(id))
+                .toArray(TlsPassthruFeed[]::new));
+    }
+
+    private final Map<String, Set<String>> recentFlexFeedValues = new HashMap<>();
+
+    @JsonIgnore public Set<TlsPassthruFeed> getFlexFeedSet() {
+        final TlsPassthruFeed[] flexFeedList = getFlexFeedList();
+        return !empty(flexFeedList) ? Arrays.stream(flexFeedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
+    }
+
     @ToString
     private static class TlsPassthruMatcher {
         @Getter @Setter private String fqdn;
@@ -103,13 +145,32 @@ public class TlsPassthruConfig {
     @JsonIgnore public Set<TlsPassthruMatcher> getPassthruSet() { return getPassthruSetRef().get(); }
 
     private Set<TlsPassthruMatcher> loadPassthruSet() {
+        final Set<TlsPassthruMatcher> set = loadFeeds(this.feedList, this.fqdnList, this.recentFeedValues);
+        if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", "));
+        return set;
+    }
+
+    @JsonIgnore @Getter(lazy=true) private final AutoRefreshingReference<Set<TlsPassthruMatcher>> flexSetRef = new AutoRefreshingReference<>() {
+        @Override public Set<TlsPassthruMatcher> refresh() { return loadFlexSet(); }
+        // todo: load refresh interval from config. implement a config view with an action to set it
+        @Override public long getTimeout() { return DEFAULT_FLEX_FEED_REFRESH_INTERVAL; }
+    };
+    @JsonIgnore public Set<TlsPassthruMatcher> getFlexSet() { return getFlexSetRef().get(); }
+
+    private Set<TlsPassthruMatcher> loadFlexSet() {
+        final Set<TlsPassthruMatcher> set = loadFeeds(this.flexFeedList, this.flexFqdnList, this.recentFlexFeedValues);
+        if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", "));
+        return set;
+    }
+
+    private Set<TlsPassthruMatcher> loadFeeds(TlsPassthruFeed[] feedList, String[] fqdnList, Map<String, Set<String>> recentValues) {
         final Set<TlsPassthruMatcher> set = new HashSet<>();
-        if (hasFqdnList()) {
-            for (String val : getFqdnList()) {
+        if (!empty(fqdnList)) {
+            for (String val : fqdnList) {
                 set.add(new TlsPassthruMatcher(val));
             }
         }
-        if (hasFeedList()) {
+        if (!empty(feedList)) {
             // put in a set to avoid duplicate URLs
             for (TlsPassthruFeed feed : new HashSet<>(Arrays.asList(feedList))) {
                 final TlsPassthruFeed loaded = loadFeed(feed.getFeedUrl());
@@ -118,13 +179,12 @@ public class TlsPassthruConfig {
                 if (!feed.hasFeedName() && loaded.hasFeedName()) feed.setFeedName(loaded.getFeedName());
 
                 // add to set if anything was found
-                if (loaded.hasFqdnList()) recentFeedValues.put(feed.getFeedUrl(), loaded.getFqdnList());
+                if (loaded.hasFqdnList()) recentValues.put(feed.getFeedUrl(), loaded.getFqdnList());
             }
         }
-        for (String val : recentFeedValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet())) {
+        for (String val : recentValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet())) {
             set.add(new TlsPassthruMatcher(val));
         }
-        if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", "));
         return set;
     }
 
@@ -162,4 +222,11 @@ public class TlsPassthruConfig {
         return false;
     }
 
+    public boolean isFlex(String fqdn) {
+        for (TlsPassthruMatcher match : getFlexSet()) {
+            if (match.matches(fqdn)) return true;
+        }
+        return false;
+    }
+
 }

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java

@@ -26,6 +26,10 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
             if (log.isDebugEnabled()) log.debug("checkConnection: returning passthru for fqdn/addr="+fqdn+"/"+addr);
             return ConnectionCheckResponse.passthru;
         }
+        if (passthruConfig.isFlex(fqdn)) {
+            if (log.isDebugEnabled()) log.debug("checkConnection: returning flex for fqdn/addr="+fqdn+"/"+addr);
+            return ConnectionCheckResponse.flex;
+        }
         if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr);
         return ConnectionCheckResponse.noop;
     }

bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java

@@ -10,7 +10,7 @@ import static bubble.ApiConstants.enumFromString;
 
 public enum ConnectionCheckResponse {
 
-    noop, passthru, block, error;
+    noop, passthru, flex, block, error;
 
     @JsonCreator public static ConnectionCheckResponse fromString (String v) { return enumFromString(ConnectionCheckResponse.class, v); }
 

bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql

@@ -8,7 +8,7 @@ CREATE TABLE flex_router (
     ip character varying(500) NOT NULL,
     last_seen bigint,
     port integer NOT NULL,
-    token character varying(100)
+    token character varying(200)
 );
 
 ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_pkey PRIMARY KEY (uuid);

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json

@@ -59,6 +59,10 @@
         "fqdnList": [],
         "feedList": [{
           "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/tls_passthru.txt"
+        }],
+        "flexFqdnList": [],
+        "flexFeedList": [{
+          "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/flex_routing.txt"
         }]
       }
     }],