configs for TlsPassthru app now working

bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java

@@ -7,14 +7,12 @@ package bubble.app.bblock;
 import bubble.abp.BlockDecision;
 import bubble.abp.BlockListSource;
 import bubble.abp.BlockSpec;
-import bubble.dao.app.AppRuleDAO;
-import bubble.dao.app.RuleDriverDAO;
 import bubble.model.account.Account;
 import bubble.model.app.AppMatcher;
 import bubble.model.app.AppRule;
 import bubble.model.app.BubbleApp;
 import bubble.model.app.RuleDriver;
-import bubble.model.app.config.AppConfigDriver;
+import bubble.model.app.config.AppConfigDriverBase;
 import bubble.model.device.Device;
 import bubble.rule.bblock.BubbleBlockConfig;
 import bubble.rule.bblock.BubbleBlockList;
@@ -31,7 +29,8 @@ import java.util.*;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.emptySet;
-import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 import static org.cobbzilla.util.http.HttpSchemes.SCHEME_HTTPS;
 import static org.cobbzilla.util.http.HttpSchemes.isHttpOrHttps;
 import static org.cobbzilla.util.http.URIUtil.getHost;
@@ -44,7 +43,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 
 @Slf4j
-public class BubbleBlockAppConfigDriver implements AppConfigDriver {
+public class BubbleBlockAppConfigDriver extends AppConfigDriverBase {
 
     public static final String VIEW_manageLists = "manageLists";
     public static final String VIEW_manageList = "manageList";
@@ -53,8 +52,6 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     public static final AppMatcher TEST_MATCHER = new AppMatcher();
     public static final Device TEST_DEVICE = new Device();
 
-    @Autowired private RuleDriverDAO driverDAO;
-    @Autowired private AppRuleDAO ruleDAO;
     @Autowired private BubbleConfiguration configuration;
 
     @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
@@ -103,7 +100,7 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     private List<BubbleBlockList> loadAllLists(Account account, BubbleApp app) {
 
         final AppRule rule = loadRule(account, app);
-        loadDriver(account, rule); // validate proper driver
+        loadDriver(account, rule, BubbleBlockRuleDriver.class); // validate proper driver
 
         final BubbleBlockConfig blockConfig = json(rule.getConfigJson(), BubbleBlockConfig.class);
         final List<BubbleBlockList> blockLists = new ArrayList<>();
@@ -116,28 +113,13 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     private BubbleUserAgentBlock[] loadUserAgentBlocks(Account account, BubbleApp app) {
 
         final AppRule rule = loadRule(account, app);
-        loadDriver(account, rule); // validate proper driver
+        loadDriver(account, rule, BubbleBlockRuleDriver.class); // validate proper driver
 
         final BubbleBlockConfig blockConfig = json(rule.getConfigJson(), BubbleBlockConfig.class);
         final BubbleUserAgentBlock[] blocks = blockConfig.getUserAgentBlocks();
         return empty(blocks) ? BubbleUserAgentBlock.NO_BLOCKS : blocks;
     }
 
-    private RuleDriver loadDriver(Account account, AppRule rule) {
-        final RuleDriver driver = driverDAO.findByAccountAndId(account.getUuid(), rule.getDriver());
-        if (driver == null || !driver.getDriverClass().equals(BubbleBlockRuleDriver.class.getName())) {
-            return die("expected BubbleBlockRuleDriver");
-        }
-        return driver;
-    }
-
-    private AppRule loadRule(Account account, BubbleApp app) {
-        final List<AppRule> rules = ruleDAO.findByAccountAndAppAndEnabled(account.getUuid(), app.getUuid());
-        if (rules.isEmpty()) return die("loadAllLists: no rule found");
-        if (rules.size() > 1) return die("loadAllLists: expected only one enabled rule");
-        return rules.get(0);
-    }
-
     public static final String ACTION_enableList = "enableList";
     public static final String ACTION_disableList = "disableList";
     public static final String ACTION_createList = "createList";
@@ -207,7 +189,7 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
 
         try {
             final AppRule rule = loadRule(account, app);
-            final RuleDriver ruleDriver = loadDriver(account, rule);
+            final RuleDriver ruleDriver = loadDriver(account, rule, BubbleBlockRuleDriver.class);
             final BubbleBlockRuleDriver unwiredDriver = (BubbleBlockRuleDriver) rule.initDriver(ruleDriver, TEST_MATCHER, account, TEST_DEVICE);
             final BubbleBlockRuleDriver driver = configuration.autowire(unwiredDriver);
             final BlockDecision decision = driver.getDecision(host, path, userAgent, primary);

bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java

@@ -4,30 +4,160 @@
  */
 package bubble.app.passthru;
 
+import bubble.dao.app.AppRuleDAO;
 import bubble.model.account.Account;
+import bubble.model.app.AppRule;
 import bubble.model.app.BubbleApp;
-import bubble.model.app.config.AppConfigDriver;
+import bubble.model.app.config.AppConfigDriverBase;
+import bubble.rule.passthru.TlsPassthruConfig;
+import bubble.rule.passthru.TlsPassthruFeed;
+import bubble.rule.passthru.TlsPassthruFqdn;
+import bubble.rule.passthru.TlsPassthruRuleDriver;
 import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.collection.ArrayUtil;
+import org.springframework.beans.factory.annotation.Autowired;
 
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
+import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 
 @Slf4j
-public class TlsPassthruAppConfigDriver implements AppConfigDriver {
+public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
+
+    public static final String VIEW_manageDomains = "manageDomains";
+    public static final String VIEW_manageFeeds = "manageFeeds";
+
+    @Autowired @Getter private AppRuleDAO ruleDAO;
 
     @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
-        // todo
-        return null;
+        switch (view) {
+            case VIEW_manageDomains:
+                return loadManageDomains(account, app);
+            case VIEW_manageFeeds:
+                return loadManageFeeds(account, app);
+        }
+        throw notFoundEx(view);
+    }
+
+    private Set<TlsPassthruFeed> loadManageFeeds(Account account, BubbleApp app) {
+        final TlsPassthruConfig config = getConfig(account, app);
+        config.getPassthruSet(); // ensure names are initialized
+        return config.getFeedSet();
+    }
+
+    private Set<TlsPassthruFqdn> loadManageDomains(Account account, BubbleApp app) {
+        final TlsPassthruConfig config = getConfig(account, app);
+        return !config.hasFqdnList() ? Collections.emptySet() :
+                Arrays.stream(config.getFqdnList())
+                        .map(TlsPassthruFqdn::new)
+                        .collect(Collectors.toCollection(TreeSet::new));
+    }
+
+    private TlsPassthruConfig getConfig(Account account, BubbleApp app) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        return json(rule.getConfigJson(), TlsPassthruConfig.class);
     }
 
+    public static final String ACTION_addFqdn = "addFqdn";
+    public static final String ACTION_removeFqdn = "removeFqdn";
+    public static final String ACTION_addFeed = "addFeed";
+    public static final String ACTION_removeFeed = "removeFeed";
+
+    public static final String PARAM_FQDN = "passthruFqdn";
+    public static final String PARAM_FEED_URL = "feedUrl";
+
     @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action, Map<String, String> params, JsonNode data) {
-        // todo
-        return null;
+        switch (action) {
+            case ACTION_addFqdn:
+                return addFqdn(account, app, data);
+            case ACTION_addFeed:
+                return addFeed(account, app, params, data);
+        }
+        log.debug("takeAppAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private List<TlsPassthruFqdn> addFqdn(Account account, BubbleApp app, JsonNode data) {
+        final JsonNode fqdnNode = data.get(PARAM_FQDN);
+        if (fqdnNode == null || fqdnNode.textValue() == null || empty(fqdnNode.textValue().trim())) {
+            throw invalidEx("err.addFqdn.passthruFqdnRequired");
+        }
+
+        final String fqdn = fqdnNode.textValue().trim().toLowerCase();
+
+        final TlsPassthruConfig config = getConfig(account, app)
+                .addFqdn(fqdn);
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        ruleDAO.update(rule.setConfigJson(json(config)));
+
+        return getFqdnList(config);
+    }
+
+    private List<TlsPassthruFqdn> getFqdnList(TlsPassthruConfig config) {
+        return Arrays.stream(config.getFqdnList())
+                .map(TlsPassthruFqdn::new)
+                .collect(Collectors.toList());
+    }
+
+    private Set<TlsPassthruFeed> addFeed(Account account, BubbleApp app, Map<String, String> params, JsonNode data) {
+        final JsonNode urlNode = data.get(PARAM_FEED_URL);
+        if (urlNode == null || urlNode.textValue() == null || empty(urlNode.textValue().trim())) {
+            throw invalidEx("err.addFeed.feedUrlRequired");
+        }
+
+        final String url = urlNode.textValue().trim().toLowerCase();
+
+        final TlsPassthruConfig config = getConfig(account, app);
+
+        final TlsPassthruFeed feed = config.loadFeed(url);
+        if (!feed.hasFqdnList()) throw invalidEx("err.addFeed.emptyFqdnList");
+        config.addFeed(feed);
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        ruleDAO.update(rule.setConfigJson(json(config)));
+
+        return config.getFeedSet();
     }
 
     @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id, Map<String, String> params, JsonNode data) {
-        // todo
-        return null;
+        switch (action) {
+            case ACTION_removeFqdn:
+                return removeFqdn(account, app, id);
+            case ACTION_removeFeed:
+                return removeFeed(account, app, id);
+        }
+        log.debug("takeItemAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private List<TlsPassthruFqdn> removeFqdn(Account account, BubbleApp app, String id) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        final TlsPassthruConfig config = getConfig(account, app);
+        log.debug("removeFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getFqdnList()));
+
+        final TlsPassthruConfig updated = config.removeFqdn(id);
+        log.debug("removeFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getFqdnList()));
+        ruleDAO.update(rule.setConfigJson(json(updated)));
+        return getFqdnList(updated);
+    }
+
+    public Set<TlsPassthruFeed> removeFeed(Account account, BubbleApp app, String id) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        final TlsPassthruConfig config = getConfig(account, app).removeFeed(id);
+        ruleDAO.update(rule.setConfigJson(json(config)));
+        return config.getFeedSet();
     }
 
 }

bubble-server/src/main/java/bubble/model/app/config/AppConfigDriver.java

@@ -4,12 +4,18 @@
  */
 package bubble.model.app.config;
 
+import bubble.dao.app.AppRuleDAO;
+import bubble.dao.app.RuleDriverDAO;
 import bubble.model.account.Account;
+import bubble.model.app.AppRule;
 import bubble.model.app.BubbleApp;
 import com.fasterxml.jackson.databind.JsonNode;
 
+import java.util.List;
 import java.util.Map;
 
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+
 public interface AppConfigDriver {
 
     String PARAM_ID = "id";
@@ -30,4 +36,17 @@ public interface AppConfigDriver {
                           String id,
                           Map<String, String> params,
                           JsonNode data);
+
+    AppRuleDAO getRuleDAO();
+    RuleDriverDAO getDriverDAO();
+
+    default AppRule loadRule(Account account, BubbleApp app) { return loadRule(account, app, getRuleDAO()); }
+
+    static AppRule loadRule(Account account, BubbleApp app, AppRuleDAO ruleDAO) {
+        final List<AppRule> rules = ruleDAO.findByAccountAndAppAndEnabled(account.getUuid(), app.getUuid());
+        if (rules.isEmpty()) return die("loadRule: no rule found");
+        if (rules.size() > 1) return die("loadRule: expected only one enabled rule, found "+rules.size());
+        return rules.get(0);
+    }
+
 }

bubble-server/src/main/java/bubble/model/app/config/AppConfigDriverBase.java

@@ -0,0 +1,31 @@
+package bubble.model.app.config;
+
+import bubble.dao.app.AppRuleDAO;
+import bubble.dao.app.RuleDriverDAO;
+import bubble.model.account.Account;
+import bubble.model.app.AppRule;
+import bubble.model.app.RuleDriver;
+import bubble.rule.AppRuleDriver;
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+
+public abstract class AppConfigDriverBase implements AppConfigDriver {
+
+    @Autowired @Getter protected RuleDriverDAO driverDAO;
+    @Autowired @Getter protected AppRuleDAO ruleDAO;
+
+    protected RuleDriver loadDriver(Account account, AppRule rule, Class<? extends AppRuleDriver> expectedDriverClass) {
+        return loadDriver(account, rule, getDriverDAO(), expectedDriverClass);
+    }
+
+    protected RuleDriver loadDriver(Account account, AppRule rule, RuleDriverDAO driverDAO, Class<? extends AppRuleDriver> expectedDriverClass) {
+        final RuleDriver driver = driverDAO.findByAccountAndId(account.getUuid(), rule.getDriver());
+        if (driver == null || !driver.getDriverClass().equals(expectedDriverClass.getName())) {
+            return die("expected BubbleBlockRuleDriver");
+        }
+        return driver;
+    }
+
+}

bubble-server/src/main/java/bubble/model/app/config/AppDataField.java

@@ -12,5 +12,6 @@ public class AppDataField extends EntityFieldConfig {
 
     @Getter @Setter private Boolean customFormat;
     @Getter @Setter private String when;
+    @Getter @Setter private Boolean truncate;
 
 }

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java

@@ -4,28 +4,124 @@
  */
 package bubble.rule.passthru;
 
+import com.amazonaws.util.IOUtils;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.cache.AutoRefreshingReference;
+import org.cobbzilla.util.collection.ArrayUtil;
+import org.cobbzilla.util.string.StringUtil;
 
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
+import java.io.InputStream;
+import java.util.*;
+import java.util.stream.Collectors;
 
+import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_FEEDS;
+import static java.util.concurrent.TimeUnit.HOURS;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
+import static org.cobbzilla.util.http.HttpUtil.getUrlInputStream;
+import static org.cobbzilla.wizard.server.RestServerBase.reportError;
 
+@Slf4j @Accessors(chain=true)
 public class TlsPassthruConfig {
 
-    @Getter @Setter private String[] passthruFqdn;
+    public static final long DEFAULT_TLS_FEED_REFRESH_INTERVAL = HOURS.toMillis(1);
+    public static final String FEED_NAME_PREFIX = "# Name:";
 
-    @JsonIgnore @Getter(lazy=true) private final Set<String> passthruSet = initPassthruSet();
+    @Getter @Setter private String[] fqdnList;
+    public boolean hasFqdnList () { return !empty(fqdnList); }
 
-    private Set<String> initPassthruSet() {
+    public TlsPassthruConfig addFqdn(String fqdn) {
+        return setFqdnList(Arrays.stream(ArrayUtil.append(fqdnList, fqdn)).collect(Collectors.toSet()).toArray(String[]::new));
+    }
+
+    public TlsPassthruConfig removeFqdn(String id) {
+        return !hasFqdnList() ? this :
+                setFqdnList(Arrays.stream(getFqdnList())
+                        .filter(fqdn -> !fqdn.equalsIgnoreCase(id.trim()))
+                        .toArray(String[]::new));
+    }
+
+    @Getter @Setter private TlsPassthruFeed[] feedList;
+    public boolean hasFeedList () { return !empty(feedList); }
+
+    public TlsPassthruConfig addFeed(TlsPassthruFeed feed) {
+        final Set<TlsPassthruFeed> feeds = getFeedSet();
+        if (empty(feeds)) return setFeedList(new TlsPassthruFeed[] {feed});
+        feeds.add(feed);
+        return setFeedList(feeds.toArray(EMPTY_FEEDS));
+    }
+
+    public TlsPassthruConfig removeFeed(String id) {
+        return setFeedList(getFeedSet().stream()
+                .filter(feed -> !feed.getId().equals(id))
+                .toArray(TlsPassthruFeed[]::new));
+    }
+
+    private Map<String, Set<String>> recentFeedValues = new HashMap<>();
+
+    @JsonIgnore public Set<TlsPassthruFeed> getFeedSet() {
+        final TlsPassthruFeed[] feedList = getFeedList();
+        return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet();
+    }
+
+    @JsonIgnore @Getter(lazy=true) private final AutoRefreshingReference<Set<String>> passthruSetRef = new AutoRefreshingReference<>() {
+        @Override public Set<String> refresh() { return loadPassthruSet(); }
+        // todo: load refresh interval from config. implement a config view with an action to set it
+        @Override public long getTimeout() { return DEFAULT_TLS_FEED_REFRESH_INTERVAL; }
+    };
+    @JsonIgnore public Set<String> getPassthruSet() { return getPassthruSetRef().get(); }
+
+    private Set<String> loadPassthruSet() {
         final Set<String> set = new HashSet<>();
-        if (!empty(passthruFqdn)) set.addAll(Arrays.asList(passthruFqdn));
+        if (hasFqdnList()) set.addAll(Arrays.asList(fqdnList));
+        if (hasFeedList()) {
+            // put in a set to avoid duplicate URLs
+            for (TlsPassthruFeed feed : new HashSet<>(Arrays.asList(feedList))) {
+                final TlsPassthruFeed loaded = loadFeed(feed.getFeedUrl());
+
+                // set name if found in special comment
+                if (!feed.hasFeedName() && loaded.hasFeedName()) feed.setFeedName(loaded.getFeedName());
+
+                // add to set if anything was found
+                if (loaded.hasFqdnList()) recentFeedValues.put(feed.getFeedUrl(), loaded.getFqdnList());
+            }
+        }
+        set.addAll(recentFeedValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()));
+        if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", "));
         return set;
     }
 
+    public TlsPassthruFeed loadFeed(String url) {
+        final TlsPassthruFeed loaded = new TlsPassthruFeed().setFeedUrl(url);
+        try (final InputStream in = getUrlInputStream(url)) {
+            final List<String> lines = StringUtil.split(IOUtils.toString(in), "\r\n");
+            final Set<String> fqdnList = new HashSet<>();
+            for (String line : lines) {
+                final String trimmed = line.trim();
+                if (trimmed.length() == 0) continue;
+                if (trimmed.startsWith("#")) {
+                    if (!loaded.hasFeedName() && trimmed.toLowerCase().startsWith(FEED_NAME_PREFIX.toLowerCase())) {
+                        final String name = trimmed.substring(FEED_NAME_PREFIX.length()).trim();
+                        if (log.isDebugEnabled()) log.debug("loadFeed("+url+"): setting name="+name+" from special comment: "+trimmed);
+                        loaded.setFeedName(name);
+                    } else {
+                        if (log.isDebugEnabled()) log.debug("loadFeed("+url+"): ignoring comment: "+trimmed);
+                    }
+                } else {
+                    fqdnList.add(trimmed);
+                }
+            }
+            loaded.setFqdnList(fqdnList);
+        } catch (Exception e) {
+            reportError("loadFeed("+url+"): "+shortError(e), e);
+        }
+        return loaded;
+    }
+
     public boolean isPassthru(String fqdn) { return getPassthruSet().contains(fqdn); }
 
 }

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruFeed.java

@@ -0,0 +1,40 @@
+package bubble.rule.passthru;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+import java.util.Set;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+
+@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of="feedUrl")
+public class TlsPassthruFeed implements Comparable<TlsPassthruFeed> {
+
+    public static final TlsPassthruFeed[] EMPTY_FEEDS = new TlsPassthruFeed[0];
+
+    public String getId() { return sha256_hex(getFeedUrl()); }
+    public void setId(String id) {} // noop
+
+    @Getter @Setter private String feedName;
+    public boolean hasFeedName() { return !empty(feedName); }
+
+    @Getter @Setter private String feedUrl;
+
+    @JsonIgnore @Getter @Setter private Set<String> fqdnList;
+    public boolean hasFqdnList () { return !empty(fqdnList); }
+
+    public TlsPassthruFeed(String url) { setFeedUrl(url); }
+
+    public TlsPassthruFeed(TlsPassthruFeed feed) { copy(this, feed); }
+
+    @Override public int compareTo(TlsPassthruFeed o) {
+        return getFeedUrl().toLowerCase().compareTo(o.getFeedUrl().toLowerCase());
+    }
+
+}

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruFqdn.java

@@ -0,0 +1,21 @@
+package bubble.rule.passthru;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class TlsPassthruFqdn implements Comparable<TlsPassthruFqdn> {
+
+    public String getId() { return passthruFqdn; }
+    public void setId(String id) {} // noop
+
+    @Getter @Setter private String passthruFqdn;
+
+    public TlsPassthruFqdn(String fqdn) { setPassthruFqdn(fqdn); }
+
+    @Override public int compareTo(TlsPassthruFqdn o) {
+        return getPassthruFqdn().toLowerCase().compareTo(o.getPassthruFqdn().toLowerCase());
+    }
+}

bubble-server/src/main/resources/logback.xml

@@ -52,6 +52,7 @@
 <!--    <logger name="bubble.dao.account.message.AccountMessageDAO" level="DEBUG" />-->
     <logger name="bubble.resources.message" level="INFO" />
     <logger name="bubble.app.analytics" level="DEBUG" />
+    <logger name="bubble.app.passthru" level="DEBUG" />
     <logger name="org.cobbzilla.util.io.regex.RegexFilterReader" level="WARN" />
     <logger name="org.cobbzilla.util.io.multi" level="INFO" />
     <logger name="bubble.rule.social.block" level="INFO" />

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -720,10 +720,6 @@ err.suspended.cannotSuspendSelf=You cannot suspend yourself
 err.tag.invalid=Tag is invalid
 err.tagsJson.length=Too many tags
 err.tagString.length=Too many tags
-err.testUrl.required=URL is required
-err.testUrl.loadingTestDriver=Error loading test driver
-err.testUrl.invalid=URL is invalid
-err.testUrl.invalidHostname=URL did not have a valid hostname
 err.tgzB64.invalid.noRolesDir=No roles directory found in tgz
 err.tgzB64.invalid.wrongNumberOfFiles=Wrong number of files in tgz base directory
 err.tgzB64.invalid.missingTasksMainYml=No tasks/main.yml file found for role in tgz
@@ -755,3 +751,14 @@ err.entity.manifest.required=No manifest.json file was found within the Model Ar
 # special values
 msg.nextBillDate.pending=Pending
 msg.nextBillDate.paymentsNotEnabled=Payments are not enabled on this system
+
+# bblock app errors
+err.testUrl.required=URL is required
+err.testUrl.loadingTestDriver=Error loading test driver
+err.testUrl.invalid=URL is invalid
+err.testUrl.invalidHostname=URL did not have a valid hostname
+
+# passthru app errors
+err.addFqdn.passthruFqdnRequired=Domain or Hostname field is required
+err.addFeed.feedUrlRequired=Feed URL is required
+err.addFeed.emptyFqdnList=Feed URL was not found or contained no data

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json

@@ -10,33 +10,36 @@
     "presentation": "app",
     "configDriver": "bubble.app.passthru.TlsPassthruAppConfigDriver",
     "configFields": [
-      {"name": "name"},
-      {"name": "description", "control": "textarea"},
-      {"name": "url", "type": "http_url"},
-      {"name": "tags"},
-      {"name": "tagString"},
-      {"name": "enabled", "type": "flag", "mode": "readOnly"},
-      {"name": "rule"},
-      {"name": "ruleType", "mode": "readOnly"},
-      {"name": "testUrl", "type": "http_url"},
-      {"name": "testUserAgent", "required": false},
-      {"name": "testUrlPrimary", "type": "flag"},
-      {"name": "urlRegex", "required": false},
-      {"name": "userAgentRegex"}
+      {"name": "passthruFqdn", "type": "hostname", "truncate": false},
+      {"name": "feedName", "truncate": false},
+      {"name": "feedUrl", "type": "http_url"}
     ],
     "configViews": [{
-      "name": "managePassthru",
+      "name": "manageDomains",
       "scope": "app",
       "root": "true",
-      "fields": ["fqdn"],
+      "fields": ["passthruFqdn"],
       "actions": [
         {"name": "removeFqdn", "index": 10},
         {
           "name": "addFqdn", "scope": "app", "index": 10,
-          "params": ["fqdn"],
+          "params": ["passthruFqdn"],
           "button": "addFqdn"
         }
       ]
+    }, {
+      "name": "manageFeeds",
+      "scope": "app",
+      "root": "true",
+      "fields": ["feedName", "feedUrl"],
+      "actions": [
+        {"name": "removeFeed", "index": 10},
+        {
+          "name": "addFeed", "scope": "app", "index": 10,
+          "params": ["feedUrl"],
+          "button": "addFeed"
+        }
+      ]
     }]
   },
   "children": {
@@ -52,7 +55,10 @@
       "driver": "TlsPassthruRuleDriver",
       "priority": -1000000,
       "config": {
-        "passthruFqdn": []
+        "fqdnList": [],
+        "feedList": [{
+          "feedUrl": "https://raw.githubusercontent.com/bubblev/bubble-filter-lists/master/tls_passthru.txt"
+        }]
       }
     }],
     "AppMessage": [{
@@ -63,12 +69,19 @@
         {"name": "summary", "value": "Network Bypass"},
         {"name": "description", "value": "Do not perform SSL interception for certificate-pinned domains"},
 
-        {"name": "config.view.managePassthru", "value": "Manage Bypass Domains"},
-        {"name": "config.field.fqdn", "value": "Hostname"},
-        {"name": "config.field.fqdn.description", "value": "Bypass traffic interception for this hostname"},
-        {"name": "config.action.addFqdn", "value": "Add New"},
+        {"name": "config.view.manageDomains", "value": "Manage Bypass Domains"},
+        {"name": "config.view.manageFeeds", "value": "Manage Bypass Domain Feeds"},
+        {"name": "config.field.passthruFqdn", "value": "Domain"},
+        {"name": "config.field.passthruFqdn.description", "value": "Bypass traffic interception for this hostname"},
+        {"name": "config.field.feedName", "value": "Name"},
+        {"name": "config.field.feedUrl", "value": "Bypass Domains List URL"},
+        {"name": "config.field.feedUrl.description", "value": "URL returning a list of bypass domains and/or hostnames, one per line"},
+        {"name": "config.action.addFqdn", "value": "Add New Bypass Domain"},
         {"name": "config.button.addFqdn", "value": "Add"},
-        {"name": "config.action.removeFqdn", "value": "Remove"}
+        {"name": "config.action.removeFqdn", "value": "Remove"},
+        {"name": "config.action.addFeed", "value": "Add New Bypass Domain Feed"},
+        {"name": "config.button.addFeed", "value": "Add"},
+        {"name": "config.action.removeFeed", "value": "Remove"}
       ]
     }]
   }

utils/abp-parser

@@ -1 +1 @@
-Subproject commit e50a6e004df9c3616a6936fbcc0f7ef299da378e
+Subproject commit 699a94f9df41f5d781b325576f79b86889d66cec

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 922b01b65c409a30950e76678d19daa17dd5b46f
+Subproject commit cd654568baeef1c11c9599d6029f1a930b5db2aa