filter list managment mvp

bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java

@@ -1,19 +1,24 @@
 package bubble.app.bblock;
 
+import bubble.abp.BlockDecision;
 import bubble.abp.BlockListSource;
 import bubble.abp.BlockSpec;
 import bubble.dao.app.AppRuleDAO;
 import bubble.dao.app.RuleDriverDAO;
 import bubble.model.account.Account;
+import bubble.model.app.AppMatcher;
 import bubble.model.app.AppRule;
 import bubble.model.app.BubbleApp;
 import bubble.model.app.RuleDriver;
 import bubble.model.app.config.AppConfigDriver;
+import bubble.model.device.Device;
 import bubble.rule.bblock.BubbleBlockConfig;
 import bubble.rule.bblock.BubbleBlockList;
 import bubble.rule.bblock.BubbleBlockRuleDriver;
+import bubble.server.BubbleConfiguration;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.string.ValidationRegexes;
 import org.cobbzilla.wizard.validation.ValidationResult;
 import org.springframework.beans.factory.annotation.Autowired;
 
@@ -25,6 +30,10 @@ import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.http.HttpSchemes.SCHEME_HTTPS;
+import static org.cobbzilla.util.http.HttpSchemes.isHttpOrHttps;
+import static org.cobbzilla.util.http.URIUtil.getHost;
+import static org.cobbzilla.util.http.URIUtil.getPath;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.string.ValidationRegexes.HTTPS_PATTERN;
 import static org.cobbzilla.util.string.ValidationRegexes.HTTP_PATTERN;
@@ -38,9 +47,12 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     public static final String VIEW_manageLists = "manageLists";
     public static final String VIEW_manageList = "manageList";
     public static final String VIEW_manageRules = "manageRules";
+    public static final AppMatcher TEST_MATCHER = new AppMatcher();
+    public static final Device TEST_DEVICE = new Device();
 
     @Autowired private RuleDriverDAO driverDAO;
     @Autowired private AppRuleDAO ruleDAO;
+    @Autowired private BubbleConfiguration configuration;
 
     @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
         final String id = params.get(PARAM_ID);
@@ -114,10 +126,12 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     public static final String ACTION_updateList = "updateList";
     public static final String ACTION_createRule = "createRule";
     public static final String ACTION_removeRule = "removeRule";
-    public static final String ACTION_test_url = "test_url";
+    public static final String ACTION_testUrl = "testUrl";
 
     public static final String PARAM_URL = "url";
     public static final String PARAM_RULE = "rule";
+    public static final String PARAM_TEST_URL = "testUrl";
+    public static final String PARAM_TEST_URL_PRIMARY = "testUrlPrimary";
 
     @Override public Object takeAppAction(Account account,
                                           BubbleApp app,
@@ -130,10 +144,47 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
                 return addList(account, app, data);
             case ACTION_createRule:
                 return addRule(account, app, params, data);
+            case ACTION_testUrl:
+                return testUrl(account, app, data);
         }
         throw notFoundEx(action);
     }
 
+    private BubbleBlockList testUrl(Account account, BubbleApp app, JsonNode data) {
+        final JsonNode testUrlNode = data.get(PARAM_TEST_URL);
+        if (testUrlNode == null || empty(testUrlNode.textValue())) throw invalidEx("err.testUrl.required");
+        String testUrl = testUrlNode.textValue();
+
+        final JsonNode testUrlPrimaryNode = data.get(PARAM_TEST_URL_PRIMARY);
+        final boolean primary = testUrlPrimaryNode == null || testUrlPrimaryNode.booleanValue();
+
+        if (!isHttpOrHttps(testUrl)) testUrl = SCHEME_HTTPS + testUrl;
+
+        final String host;
+        final String path;
+        try {
+            host = getHost(testUrl);
+            path = getPath(testUrl);
+        } catch (Exception e) {
+            throw invalidEx("err.testUrl.invalid", "Test URL was not valid", shortError(e));
+        }
+        if (empty(host) || !ValidationRegexes.HOST_PATTERN.matcher(host).matches()) {
+            throw invalidEx("err.testUrl.invalidHostname", "Test URL was not valid");
+        }
+
+        try {
+            final AppRule rule = loadRule(account, app);
+            final RuleDriver ruleDriver = loadDriver(account, rule);
+            final BubbleBlockRuleDriver unwiredDriver = (BubbleBlockRuleDriver) rule.initDriver(ruleDriver, TEST_MATCHER, account, TEST_DEVICE);
+            final BubbleBlockRuleDriver driver = configuration.autowire(unwiredDriver);
+            final BlockDecision decision = driver.getDecision(host, path, primary);
+            return getBuiltinList(account, app).setResponse(decision);
+
+        } catch (Exception e) {
+            throw invalidEx("err.testRule.loadingTestDriver", "Error loading test driver", shortError(e));
+        }
+    }
+
     private BubbleBlockList addRule(Account account, BubbleApp app, Map<String, String> params, JsonNode data) {
 
         final String id = params.get(PARAM_ID);
@@ -151,7 +202,8 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
             }
         }
         try {
-            BlockSpec.parse(line);
+            final List<BlockSpec> specs = BlockSpec.parse(line);
+            if (log.isDebugEnabled()) log.debug("addRule: parsed line ("+line+"): "+json(specs));
         } catch (Exception e) {
             log.warn("addRule: invalid line ("+line+"): "+shortError(e));
             throw invalidEx("err.rule.invalid", "Error parsing rule", e.getMessage());
@@ -233,11 +285,15 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     }
 
     private BubbleBlockList removeRule(Account account, BubbleApp app, String id) {
+        final BubbleBlockList builtin = getBuiltinList(account, app);
+        return updateList(builtin.removeRule(id));
+    }
+
+    private BubbleBlockList getBuiltinList(Account account, BubbleApp app) {
         final List<BubbleBlockList> customLists = loadAllLists(account, app).stream().filter(list -> !list.hasUrl()).collect(Collectors.toList());
         if (customLists.isEmpty()) throw invalidEx("err.removeRule.noCustomList");
         if (customLists.size() > 1) throw invalidEx("err.removeRule.multipleCustomLists");
-        final BubbleBlockList builtin = customLists.get(0);
-        return updateList(builtin.removeRule(id));
+        return customLists.get(0);
     }
 
     private ValidationResult validate(BubbleBlockList list, BubbleBlockList request, List<BubbleBlockList> allLists) {

bubble-server/src/main/java/bubble/model/app/config/AppConfigAction.java

@@ -12,6 +12,7 @@ public class AppConfigAction {
     @Getter @Setter private String when;
     @Getter @Setter private String view;
     @Getter @Setter private String successView;
+    @Getter @Setter private String successMessage;
     @Getter @Setter private Integer index = 0;
 
     @Getter @Setter private String[] params;

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockList.java

@@ -11,6 +11,7 @@ import lombok.experimental.Accessors;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ArrayUtil;
 
+import javax.persistence.Transient;
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -66,6 +67,8 @@ public class BubbleBlockList {
 
     @JsonIgnore @Getter @Setter private AppRule rule;
 
+    @Transient @Getter @Setter private Object response;  // non-standard config response (test URL) uses this
+
     public boolean hasEntry(String line) {
         return hasAdditionalEntries() && Arrays.asList(getAdditionalEntries()).contains(line);
     }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -75,13 +75,14 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
                 }
             }
             if (list.hasAdditionalEntries()) {
+                if (blockListSource == null) blockListSource = new BlockListSource(); // might be built-in source
                 try {
                     blockListSource.addEntries(list.getAdditionalEntries());
                 } catch (IOException e) {
                     log.error("init: error adding additional entries: "+shortError(e));
                 }
             }
-            blockList.merge(blockListSource.getBlockList());
+            if (blockListSource != null) blockList.merge(blockListSource.getBlockList());
         }
     }
 
@@ -95,7 +96,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         final String site = ruleHarness.getMatcher().getSite();
         final String fqdn = filter.getFqdn();
 
-        final BlockDecision decision = blockList.getDecision(filter.getFqdn(), filter.getUri());
+        final BlockDecision decision = getDecision(filter.getFqdn(), filter.getUri());
         switch (decision.getDecisionType()) {
             case block:
                 incrementCounters(account, device, app, site, fqdn);
@@ -107,6 +108,10 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         }
     }
 
+    public BlockDecision getDecision(String fqdn, String uri) { return blockList.getDecision(fqdn, uri, false); }
+
+    public BlockDecision getDecision(String fqdn, String uri, boolean primary) { return blockList.getDecision(fqdn, uri, primary); }
+
     public FilterMatchResponse getFilterMatchResponse(FilterMatchersRequest filter, BlockDecision decision) {
         switch (decision.getDecisionType()) {
             case block: return FilterMatchResponse.ABORT_NOT_FOUND;
@@ -144,7 +149,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         }
 
         // Now that we know the content type, re-check the BlockList
-        final BlockDecision decision = blockList.getDecision(request.getFqdn(), request.getUri(), contentType);
+        final BlockDecision decision = blockList.getDecision(request.getFqdn(), request.getUri(), contentType, true);
         switch (decision.getDecisionType()) {
             case block:
                 log.warn("doFilterRequest: preprocessed request was filtered, but ultimate decision was block, returning EMPTY_STREAM");

bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java

@@ -212,7 +212,7 @@ public class RuleEngineService {
         for (AppRuleHarness h : rules) {
             final RuleDriver ruleDriver = driverDAO.findByUuid(h.getRule().getDriver());
             if (ruleDriver == null) {
-                log.warn("get: driver not found: "+h.getRule().getDriver());
+                log.warn("initRules: driver not found: "+h.getRule().getDriver());
                 continue;
             }
             final AppRuleDriver unwiredDriver = h.getRule().initDriver(ruleDriver, h.getMatcher(), account, device);

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -614,6 +614,10 @@ err.suspended.cannotSuspendSelf=You cannot suspend yourself
 err.tag.invalid=Tag is invalid
 err.tagsJson.length=Too many tags
 err.tagString.length=Too many tags
+err.testUrl.required=URL is required
+err.testUrl.loadingTestDriver=Error loading test driver
+err.testUrl.invalid=URL is invalid
+err.testUrl.invalidHostname=URL did not have a valid hostname
 err.tgzB64.invalid.noRolesDir=No roles directory found in tgz
 err.tgzB64.invalid.wrongNumberOfFiles=Wrong number of files in tgz base directory
 err.tgzB64.invalid.missingTasksMainYml=No tasks/main.yml file found for role in tgz

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json

@@ -28,12 +28,13 @@
       {"name": "name"},
       {"name": "description", "control": "textarea"},
       {"name": "url", "type": "http_url"},
-      {"name": "testUrl", "type": "http_url"},
       {"name": "tags"},
       {"name": "tagString"},
       {"name": "enabled", "type": "flag", "mode": "readOnly"},
       {"name": "rule"},
-      {"name": "ruleType", "mode": "readOnly"}
+      {"name": "ruleType", "mode": "readOnly"},
+      {"name": "testUrl", "type": "http_url"},
+      {"name": "testUrlPrimary", "type": "flag"}
     ],
     "configViews": [{
       "name": "manageLists",
@@ -53,8 +54,9 @@
         },
         {
           "name": "testUrl", "scope": "app", "index": 20,
-          "params": ["testUrl"],
-          "button": "testUrl"
+          "params": ["testUrl", "testUrlPrimary"],
+          "button": "testUrl",
+          "successMessage": "response.decisionType"
         }
       ]
     }, {
@@ -81,8 +83,9 @@
         },
         {
           "name": "testUrl", "scope": "app", "index": 20,
-          "params": ["testUrl"],
-          "button": "testUrl"
+          "params": ["testUrl", "testUrlPrimary"],
+          "button": "testUrl",
+          "successMessage": "decisionType"
         }
       ]
     }]
@@ -159,6 +162,8 @@
         {"name": "config.field.ruleType", "value": "Rule Type"},
         {"name": "config.field.testUrl", "value": "Test URL"},
         {"name": "config.field.testUrl.description", "value": "URL to check against filters"},
+        {"name": "config.field.testUrlPrimary", "value": "Primary"},
+        {"name": "config.field.testUrlPrimary.description", "value": "A primary request will receive either an ALLOW or BLOCK decision from your Bubble. A non-primary request (for example a request for a webpage) may additionally receive a FILTER decision. This means the request will be permitted, but the response will be instrumented with Bubble filters to remove ads, malware and blocked elements."},
 
         {"name": "config.action.enableList", "value": "Enable"},
         {"name": "config.action.disableList", "value": "Disable"},
@@ -172,7 +177,13 @@
         {"name": "config.action.createRule", "value": "Add New Rule"},
         {"name": "config.button.createRule", "value": "Add"},
         {"name": "config.action.testUrl", "value": "Test URL"},
-        {"name": "config.button.testUrl", "value": "Test"}
+        {"name": "config.button.testUrl", "value": "Test"},
+        {"name": "config.response.block", "value": "Block"},
+        {"name": "config.response.block.description", "value": "Requests to this URL would be blocked by your Bubble"},
+        {"name": "config.response.allow", "value": "Allow"},
+        {"name": "config.response.allow.description", "value": "Requests to this URL would be allowed by your Bubble, and would not be filtered"},
+        {"name": "config.response.filter", "value": "Filter"},
+        {"name": "config.response.filter.description", "value": "Requests to this URL would be allowed by your Bubble, but would be filtered"}
       ]
     }]
   }

bubble-web

@@ -1 +1 @@
-Subproject commit 66eab3f6c7013af92ab0aa8e0bd24855edaf986a
+Subproject commit e25dbdfc8a4d6e617b17a4e5347d0cd143425576

utils/abp-parser

@@ -1 +1 @@
-Subproject commit 3922106b4227b8fa2e77eeea113e5cba2308c08b
+Subproject commit 5f96e9be2a720925347ba65c866a1cee2cdb4cd5