add support for configuring user-agent blocks

bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java

@@ -15,6 +15,7 @@ import bubble.model.device.Device;
 import bubble.rule.bblock.BubbleBlockConfig;
 import bubble.rule.bblock.BubbleBlockList;
 import bubble.rule.bblock.BubbleBlockRuleDriver;
+import bubble.rule.bblock.BubbleUserAgentBlock;
 import bubble.server.BubbleConfiguration;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.slf4j.Slf4j;
@@ -44,6 +45,7 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     public static final String VIEW_manageLists = "manageLists";
     public static final String VIEW_manageList = "manageList";
     public static final String VIEW_manageRules = "manageRules";
+    public static final String VIEW_manageUserAgents = "manageUserAgents";
     public static final AppMatcher TEST_MATCHER = new AppMatcher();
     public static final Device TEST_DEVICE = new Device();
 
@@ -68,6 +70,9 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
                     id = builtinList.getId();
                 }
                 return loadListEntries(account, app, id);
+
+            case VIEW_manageUserAgents:
+                return loadUserAgentBlocks(account, app);
         }
         throw notFoundEx(view);
     }
@@ -96,14 +101,24 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
         final AppRule rule = loadRule(account, app);
         loadDriver(account, rule); // validate proper driver
 
-        final List<BubbleBlockList> blockLists = new ArrayList<>();
         final BubbleBlockConfig blockConfig = json(rule.getConfigJson(), BubbleBlockConfig.class);
+        final List<BubbleBlockList> blockLists = new ArrayList<>();
         blockLists.addAll( Arrays.stream(blockConfig.getBlockLists())
                 .map(list -> list.setRule(rule))
                 .collect(Collectors.toList()) );
         return blockLists;
     }
 
+    private BubbleUserAgentBlock[] loadUserAgentBlocks(Account account, BubbleApp app) {
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule); // validate proper driver
+
+        final BubbleBlockConfig blockConfig = json(rule.getConfigJson(), BubbleBlockConfig.class);
+        final BubbleUserAgentBlock[] blocks = blockConfig.getUserAgentBlocks();
+        return empty(blocks) ? BubbleUserAgentBlock.NO_BLOCKS : blocks;
+    }
+
     private RuleDriver loadDriver(Account account, AppRule rule) {
         final RuleDriver driver = driverDAO.findByAccountAndId(account.getUuid(), rule.getDriver());
         if (driver == null || !driver.getDriverClass().equals(BubbleBlockRuleDriver.class.getName())) {
@@ -126,11 +141,15 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     public static final String ACTION_updateList = "updateList";
     public static final String ACTION_createRule = "createRule";
     public static final String ACTION_removeRule = "removeRule";
+    public static final String ACTION_createUserAgentBlock = "createUserAgentBlock";
+    public static final String ACTION_removeUserAgentBlock = "removeUserAgentBlock";
     public static final String ACTION_testUrl = "testUrl";
 
     public static final String PARAM_URL = "url";
     public static final String PARAM_RULE = "rule";
+    public static final String PARAM_USER_AGENT_REGEX = "userAgentRegex";
     public static final String PARAM_TEST_URL = "testUrl";
+    public static final String PARAM_TEST_USER_AGENT = "testUserAgent";
     public static final String PARAM_TEST_URL_PRIMARY = "testUrlPrimary";
 
     @Override public Object takeAppAction(Account account,
@@ -146,6 +165,8 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
                 return addRule(account, app, params, data);
             case ACTION_testUrl:
                 return testUrl(account, app, data);
+            case ACTION_createUserAgentBlock:
+                return createUserAgentBlock(account, app, params, data);
         }
         throw notFoundEx(action);
     }
@@ -160,6 +181,14 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
 
         if (!isHttpOrHttps(testUrl)) testUrl = SCHEME_HTTPS + testUrl;
 
+        final String userAgent;
+        final JsonNode userAgentNode = data.get(PARAM_TEST_USER_AGENT);
+        if (userAgentNode == null || empty(userAgentNode.textValue())) {
+            userAgent = "";
+        } else {
+            userAgent = userAgentNode.textValue();
+        }
+
         final String host;
         final String path;
         try {
@@ -177,7 +206,7 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
             final RuleDriver ruleDriver = loadDriver(account, rule);
             final BubbleBlockRuleDriver unwiredDriver = (BubbleBlockRuleDriver) rule.initDriver(ruleDriver, TEST_MATCHER, account, TEST_DEVICE);
             final BubbleBlockRuleDriver driver = configuration.autowire(unwiredDriver);
-            final BlockDecision decision = driver.getDecision(host, path, primary);
+            final BlockDecision decision = driver.getDecision(host, path, userAgent, primary);
             return getBuiltinList(account, app).setResponse(decision);
 
         } catch (Exception e) {
@@ -239,6 +268,38 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
         return list;
     }
 
+    private BubbleUserAgentBlock createUserAgentBlock(Account account, BubbleApp app, Map<String, String> params, JsonNode data) {
+
+        final JsonNode rule = data.get(PARAM_RULE);
+        final String urlRegex;
+        if (rule == null || rule.textValue() == null) {
+            urlRegex = ".*";
+        } else {
+            urlRegex = rule.textValue().trim();
+        }
+
+        final JsonNode userAgentRegexNode = data.get(PARAM_USER_AGENT_REGEX);
+        if (userAgentRegexNode == null) {
+            throw invalidEx("err.userAgentRegex.required");
+        }
+        final String userAgentRegex = userAgentRegexNode.textValue().trim();
+
+        final AppRule appRule = loadRule(account, app);
+        final BubbleBlockConfig blockConfig = json(appRule.getConfigJson(), BubbleBlockConfig.class);
+        final BubbleUserAgentBlock uaBlock = new BubbleUserAgentBlock()
+                .setUrlRegex(urlRegex)
+                .setUserAgentRegex(userAgentRegex);
+        ruleDAO.update(appRule.setConfigJson(json(blockConfig.addUserAgentBlock(uaBlock))));
+        return uaBlock;
+    }
+
+    private BubbleUserAgentBlock[] removeUserAgentBlock(Account account, BubbleApp app, String id) {
+        final AppRule appRule = loadRule(account, app);
+        final BubbleBlockConfig blockConfig = json(appRule.getConfigJson(), BubbleBlockConfig.class);
+        ruleDAO.update(appRule.setConfigJson(json(blockConfig.removeUserAgentBlock(id))));
+        return blockConfig.getUserAgentBlocks();
+    }
+
     @Override public Object takeItemAction(Account account,
                                            BubbleApp app,
                                            String view,
@@ -279,6 +340,9 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
 
             case ACTION_removeRule:
                 return removeRule(account, app, id);
+
+            case ACTION_removeUserAgentBlock:
+                return removeUserAgentBlock(account, app, id);
         }
 
         throw notFoundEx(action);

bubble-server/src/main/java/bubble/model/app/AppMatcher.java

@@ -108,31 +108,25 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H
     }
     public boolean matchesUserAgent (String value) { return getUserAgentPattern().matcher(value).find(); }
 
-    @ECSearchable(filter=true) @ECField(index=90)
-    @Size(max=4000, message="err.jsMatch.length")
-    @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(4000+ENC_PAD)+")")
-    @Getter @Setter private String jsMatch;
-    public boolean hasJsMatch() { return !empty(jsMatch); }
-
-    @ECSearchable @ECField(index=100)
+    @ECSearchable @ECField(index=90)
     @ECForeignKey(entity=AppRule.class)
     @Column(nullable=false, updatable=false, length=UUID_MAXLEN)
     @Getter @Setter private String rule;
 
-    @ECSearchable @ECField(index=110)
+    @ECSearchable @ECField(index=1100)
     @Column(nullable=false)
     @Getter @Setter private Boolean blocked = false;
     public boolean blocked() { return bool(blocked); }
 
-    @ECSearchable @ECField(index=120)
+    @ECSearchable @ECField(index=110)
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean template = false;
 
-    @ECSearchable @ECField(index=130)
+    @ECSearchable @ECField(index=120)
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean enabled = true;
 
-    @ECSearchable @ECField(index=140)
+    @ECSearchable @ECField(index=130)
     @Column(nullable=false)
     @Getter @Setter private Integer priority = 0;
 

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -187,7 +187,16 @@ public class FilterHttpResource {
             for (AppMatcher matcher : matchers) {
                 if (retainMatchers.containsKey(matcher.getUuid())) continue;
                 if (matcher.matchesUrl(uri)) {
-                    if (log.isDebugEnabled()) log.debug(prefix+"matcher "+matcher.getName()+" with pattern "+matcher.getUrlRegex()+" found match for uri: '"+uri+"'");
+                    if (matcher.hasUserAgentRegex()) {
+                        if (!matcher.matchesUserAgent(filterRequest.getUserAgent())) {
+                            if (log.isDebugEnabled()) log.debug(prefix+"matcher "+matcher.getName()+" with pattern "+matcher.getUrlRegex()+" found match for uri: '"+uri+"', but user-agent pattern "+matcher.getUserAgentRegex()+" does not match user-agent="+filterRequest.getUserAgent());
+                            continue;
+                        } else {
+                            if (log.isDebugEnabled()) log.debug(prefix + "matcher " + matcher.getName() + " with pattern " + matcher.getUrlRegex() + " found match for uri: '" + uri + "' and for user-agent pattern "+matcher.getUserAgentRegex()+" for user-agent="+filterRequest.getUserAgent());
+                        }
+                    } else {
+                        if (log.isDebugEnabled()) log.debug(prefix + "matcher " + matcher.getName() + " with pattern " + matcher.getUrlRegex() + " found match for uri: '" + uri + "'");
+                    }
                     final FilterMatchDecision matchResponse = ruleEngine.preprocess(filterRequest, req, request, caller, device, matcher);
                     switch (matchResponse) {
                         case abort_ok:        return FilterMatchersResponse.ABORT_OK;
@@ -350,7 +359,7 @@ public class FilterHttpResource {
                 return passthru(request);
             }
             if (!isContentTypeMatch(matchersResponse, contentType)) {
-                if (log.isInfoEnabled()) log.info(prefix+"none of the "+matchersResponse.getMatchers().size()+" matchers matched contentType="+contentType+", returning passthru");
+                if (log.isDebugEnabled()) log.debug(prefix+"none of the "+matchersResponse.getMatchers().size()+" matchers matched contentType="+contentType+", returning passthru");
                 return passthru(request);
             }
 
@@ -399,7 +408,7 @@ public class FilterHttpResource {
         final List<AppMatcher> matchers = matchersResponse.getMatchers();
         for (AppMatcher m : matchers) {
             if (log.isDebugEnabled()) log.debug(prefix+"checking contentType match, matcher.contentTypeRegex="+m.getContentTypeRegex()+", contentType="+ct);
-            if (m.getContentTypePattern().matcher(ct).matches()) {
+            if (m.matchesContentType(ct)) {
                 return true;
             }
         }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java

@@ -15,6 +15,8 @@ public class BubbleBlockConfig {
     @Getter @Setter private Boolean inPageBlocks;
     public boolean inPageBlocks() { return inPageBlocks != null && inPageBlocks; }
 
+    @Getter @Setter private BubbleUserAgentBlock[] userAgentBlocks;
+
     @Getter @Setter private BubbleBlockList[] blockLists;
 
     public BubbleBlockConfig updateList(BubbleBlockList list) {
@@ -49,4 +51,25 @@ public class BubbleBlockConfig {
         }
         return this;
     }
+
+    public BubbleBlockConfig addUserAgentBlock (BubbleUserAgentBlock uaBlock) {
+        if (userAgentBlocks != null) {
+            for (BubbleUserAgentBlock uab : userAgentBlocks) {
+                if (uab.equals(uaBlock)) return this;
+            }
+        }
+        userAgentBlocks = ArrayUtil.append(userAgentBlocks, uaBlock);
+        return this;
+    }
+
+    public BubbleBlockConfig removeUserAgentBlock (String id) {
+        if (userAgentBlocks == null) return this;
+        final List<BubbleUserAgentBlock> retained = new ArrayList<>(userAgentBlocks.length);
+        for (BubbleUserAgentBlock uab : userAgentBlocks) {
+            if (!uab.getId().equals(id)) retained.add(uab);
+        }
+        userAgentBlocks = retained.toArray(BubbleUserAgentBlock.NO_BLOCKS);
+        return this;
+    }
+
 }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -104,7 +104,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         final String fqdn = filter.getFqdn();
         final String prefix = "preprocess("+filter.getRequestId()+"): ";
 
-        final BlockDecision decision = getDecision(filter.getFqdn(), filter.getUri());
+        final BlockDecision decision = getDecision(filter.getFqdn(), filter.getUri(), filter.getUserAgent());
         switch (decision.getDecisionType()) {
             case block:
                 if (log.isInfoEnabled()) log.info(prefix+"decision is BLOCK");
@@ -137,9 +137,18 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         }
     }
 
-    public BlockDecision getDecision(String fqdn, String uri) { return blockList.getDecision(fqdn, uri, false); }
+    public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return blockList.getDecision(fqdn, uri, userAgent, false); }
 
-    public BlockDecision getDecision(String fqdn, String uri, boolean primary) { return blockList.getDecision(fqdn, uri, primary); }
+    public BlockDecision getDecision(String fqdn, String uri, String userAgent, boolean primary) {
+        if (!empty(userAgent) && !empty(bubbleBlockConfig.getUserAgentBlocks())) {
+            for (BubbleUserAgentBlock uaBlock : bubbleBlockConfig.getUserAgentBlocks()) {
+                if (uaBlock.hasUrlRegex() && uaBlock.urlMatches(uri)) {
+                    if (uaBlock.userAgentMatches(userAgent)) return BlockDecision.BLOCK;
+                }
+            }
+        }
+        return blockList.getDecision(fqdn, uri, primary);
+    }
 
     @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) {
 

bubble-server/src/main/java/bubble/rule/bblock/BubbleUserAgentBlock.java

@@ -0,0 +1,33 @@
+package bubble.rule.bblock;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+import java.util.regex.Pattern;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+
+@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of={"urlRegex", "userAgentRegex"})
+public class BubbleUserAgentBlock {
+
+    public static final BubbleUserAgentBlock[] NO_BLOCKS = new BubbleUserAgentBlock[0];
+
+    @Getter(lazy=true) private final String id = sha256_hex(getUrlRegex()+"\t"+getUserAgentRegex());
+
+    @Getter @Setter private String urlRegex;
+    public boolean hasUrlRegex () { return !empty(urlRegex); }
+
+    @JsonIgnore @Getter(lazy=true) private final Pattern urlPattern = Pattern.compile(getUrlRegex(), Pattern.CASE_INSENSITIVE);
+    public boolean urlMatches(String uri) { return getUrlPattern().matcher(uri).find(); }
+
+    @Getter @Setter private String userAgentRegex;
+
+    @JsonIgnore @Getter(lazy=true) private final Pattern uaPattern = Pattern.compile(getUserAgentRegex(), Pattern.CASE_INSENSITIVE);
+    public boolean userAgentMatches (String ua) { return getUaPattern().matcher(ua).find(); }
+
+}

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -669,6 +669,7 @@ err.type.invalid=Type is invalid
 err.type.required=Type is required
 err.url.length=URL is too long
 err.contentTypeRegex.length=Content-Type regex is too long
+err.userAgentRegex.required=User-Agent regex is required
 err.userAgentRegex.length=User-Agent regex is too long
 err.urlRegex.length=URL regex is too long
 err.urlRegex.required=URL regex is required

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json

@@ -34,7 +34,10 @@
       {"name": "rule"},
       {"name": "ruleType", "mode": "readOnly"},
       {"name": "testUrl", "type": "http_url"},
-      {"name": "testUrlPrimary", "type": "flag"}
+      {"name": "testUserAgent", "required": false},
+      {"name": "testUrlPrimary", "type": "flag"},
+      {"name": "urlRegex", "required": false},
+      {"name": "userAgentRegex"}
     ],
     "configViews": [{
       "name": "manageLists",
@@ -45,7 +48,7 @@
         {"name": "enableList", "when": "!item.enabled", "index": 10},
         {"name": "disableList", "when": "item.enabled", "index": 20},
         {"name": "manageList", "view": "manageList", "index": 30},
-        {"name": "manageRules", "view": "manageRules", "when": "item.url === ''", "index": 40},
+        {"name": "manageRules", "view": "manageRules", "when": "item.url === \\'\\'", "index": 40},
         {"name": "removeList", "index": 50, "when": "item.url !== ''"},
         {
           "name": "createList", "scope": "app", "view": "manageList", "index": 10,
@@ -83,7 +86,26 @@
         },
         {
           "name": "testUrl", "scope": "app", "index": 20,
-          "params": ["testUrl", "testUrlPrimary"],
+          "params": ["testUrl", "testUserAgent", "testUrlPrimary"],
+          "button": "testUrl",
+          "successMessage": "decisionType"
+        }
+      ]
+    }, {
+      "name": "manageUserAgents",
+      "scope": "app",
+      "root": "true",
+      "fields": ["userAgentRegex", "urlRegex"],
+      "actions": [
+        {"name": "removeUserAgentBlock", "index": 10},
+        {
+          "name": "createUserAgentBlock", "scope": "app", "index": 10,
+          "params": ["userAgentRegex", "urlRegex"],
+          "button": "createUserAgentBlock"
+        },
+        {
+          "name": "testUrl", "scope": "app", "index": 20,
+          "params": ["testUrl", "testUserAgent", "testUrlPrimary"],
           "button": "testUrl",
           "successMessage": "decisionType"
         }
@@ -150,6 +172,7 @@
         {"name": "config.view.manageLists", "value": "Manage Filter Lists"},
         {"name": "config.view.manageList", "value": "Manage Filter List"},
         {"name": "config.view.manageRules", "value": "Manage Filter Rules"},
+        {"name": "config.view.manageUserAgents", "value": "Manage User-Agents"},
 
         {"name": "config.field.name", "value": "Name"},
         {"name": "config.field.description", "value": "Description"},
@@ -164,8 +187,14 @@
         {"name": "config.field.ruleType", "value": "Rule Type"},
         {"name": "config.field.testUrl", "value": "Test URL"},
         {"name": "config.field.testUrl.description", "value": "URL to check against filters"},
+        {"name": "config.field.testUserAgent", "value": "Test User-Agent"},
+        {"name": "config.field.testUserAgent.description", "value": "User-Agent to check against filters"},
         {"name": "config.field.testUrlPrimary", "value": "Primary"},
         {"name": "config.field.testUrlPrimary.description", "value": "A primary request will receive either an ALLOW or BLOCK decision from your Bubble. A non-primary request (for example a request for a webpage) may additionally receive a FILTER decision. This means the request will be permitted, but the response will be instrumented with Bubble filters to remove ads, malware and blocked elements."},
+        {"name": "config.field.urlRegex", "value": "URL"},
+        {"name": "config.field.urlRegex.description", "value": "A regular expression to match against the URL"},
+        {"name": "config.field.userAgentRegex", "value": "User-Agent"},
+        {"name": "config.field.userAgentRegex.description", "value": "A regular expression to match against the User-Agent"},
 
         {"name": "config.action.enableList", "value": "Enable"},
         {"name": "config.action.disableList", "value": "Disable"},
@@ -175,9 +204,14 @@
         {"name": "config.button.createList", "value": "Add"},
         {"name": "config.action.updateList", "value": "Update"},
         {"name": "config.action.manageRules", "value": "Rules"},
+        {"name": "config.action.manageUserAgents", "value": "User-Agents"},
         {"name": "config.action.removeRule", "value": "Remove Rule"},
         {"name": "config.action.createRule", "value": "Add New Rule"},
         {"name": "config.button.createRule", "value": "Add"},
+        {"name": "config.action.createUserAgentBlock", "value": "Add New User-Agent Rule"},
+        {"name": "config.button.createUserAgentBlock", "value": "Add"},
+        {"name": "config.action.removeUserAgentBlock", "value": "Remove Rule"},
+        {"name": "config.button.removeUserAgentBlock", "value": "Remove"},
         {"name": "config.action.testUrl", "value": "Test URL"},
         {"name": "config.button.testUrl", "value": "Test"},
         {"name": "config.response.block", "value": "Block"},

bubble-web

@@ -1 +1 @@
-Subproject commit 5c61e5b7b443b1813a84d9cd4f0121af274e480b
+Subproject commit 228f86c5c0fede20b0901887823fceb7f9667ce8