add block tests, update block/filter interface

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -131,8 +131,7 @@ public class FilterHttpResource {
         final List<AppMatcher> matchers = matcherDAO.findByAccountAndFqdnAndEnabled(accountUuid, fqdn);
         if (log.isDebugEnabled()) log.debug("findMatchers: found "+matchers.size()+" candidate matchers");
         final List<AppMatcher> removeMatchers;
-        List<String> options = null;
-        List<String> selectors = null;
+        List<String> filters = null;
         if (matchers.isEmpty()) {
             removeMatchers = Collections.emptyList();
         } else {
@@ -148,13 +147,9 @@ public class FilterHttpResource {
                             removeMatchers.add(matcher);
                             break;
                         case match:
-                            if (matchResponse.hasOptions()) {
-                                if (options == null) options = new ArrayList<>();
-                                options.addAll(matchResponse.getOptions());
-                            }
-                            if (matchResponse.hasSelectors()) {
-                                if (selectors == null) selectors = new ArrayList<>();
-                                selectors.addAll(matchResponse.getSelectors());
+                            if (matchResponse.hasFilters()) {
+                                if (filters == null) filters = new ArrayList<>();
+                                filters.addAll(matchResponse.getFilters());
                             }
                             break;
                     }
@@ -168,8 +163,7 @@ public class FilterHttpResource {
         return response
                 .setMatchers(matchers)
                 .setDevice(device.getUuid())
-                .setOptions(options)
-                .setSelectors(selectors);
+                .setFilters(filters);
     }
 
     @POST @Path(EP_APPLY+"/{requestId}")

bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java

@@ -8,6 +8,8 @@ import lombok.experimental.Accessors;
 
 import java.util.List;
 
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+
 @NoArgsConstructor @Accessors(chain=true)
 public class FilterMatchResponse {
 
@@ -18,10 +20,7 @@ public class FilterMatchResponse {
 
     @Getter @Setter private FilterMatchDecision decision;
 
-    @Getter @Setter private List<String> options;
-    public boolean hasOptions () { return options != null && !options.isEmpty(); }
-
-    @Getter @Setter private List<String> selectors;
-    public boolean hasSelectors () { return selectors != null && !selectors.isEmpty(); }
+    @Getter @Setter private List<String> filters;
+    public boolean hasFilters() { return !empty(filters); }
 
 }

bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java

@@ -16,7 +16,6 @@ public class FilterMatchersResponse {
     @Getter @Setter private Integer abort;
     @Getter @Setter private String device;
     @Getter @Setter private List<AppMatcher> matchers;
-    @Getter @Setter private List<String> options;
-    @Getter @Setter private List<String> selectors;
+    @Getter @Setter private List<String> filters;
 
 }

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java

@@ -9,9 +9,9 @@ import lombok.experimental.Accessors;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
 @NoArgsConstructor @Accessors(chain=true)
 public class BlockDecision {
@@ -20,39 +20,24 @@ public class BlockDecision {
     public static final BlockDecision ALLOW = new BlockDecision().setDecisionType(BlockDecisionType.allow);
 
     @Getter @Setter BlockDecisionType decisionType = BlockDecisionType.allow;
-    @Getter @Setter List<String> selectors;
-    @Getter @Setter List<String> options;
+    @Getter @Setter List<BlockSpec> specs;
 
-    public BlockDecision add(BlockSpec block) {
-        if (block.hasSelector()) {
-            if (selectors == null) selectors = new ArrayList<>();
-            selectors.add(block.getSelector());
+    public BlockDecision add(BlockSpec spec) {
+        if (specs == null) specs = new ArrayList<>();
+        specs.add(spec);
+        if (decisionType != BlockDecisionType.block && (spec.hasTypeMatches() || spec.hasSelector())) {
+            decisionType = BlockDecisionType.filter;
         }
-        if (block.hasOptions()) {
-            if (options == null) options = new ArrayList<>();
-            options.addAll(block.getOptions());
-        }
-        if (!empty(selectors) || !empty(options)) decisionType = BlockDecisionType.filter;
         return this;
     }
 
-    public FilterMatchDecision getFilterMatchDecision() {
-        switch (decisionType) {
-            case block: return FilterMatchDecision.abort_not_found;
-            case allow: return FilterMatchDecision.no_match;
-            case filter: return FilterMatchDecision.match;
-        }
-        return die("getFilterMatchDecision: invalid decisionType: "+decisionType);
-    }
-
     public FilterMatchResponse getFilterMatchResponse() {
         switch (decisionType) {
             case block: return FilterMatchResponse.ABORT_NOT_FOUND;
             case allow: return FilterMatchResponse.NO_MATCH;
             case filter: return new FilterMatchResponse()
                     .setDecision(FilterMatchDecision.match)
-                    .setOptions(getOptions())
-                    .setSelectors(getSelectors());
+                    .setFilters(specs == null ? null : getSpecs().stream().map(BlockSpec::getLine).collect(Collectors.toList()));
         }
         return die("getFilterMatchResponse: invalid decisionType: "+decisionType);
     }

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java

@@ -1,30 +1,82 @@
 package bubble.rule.bblock.spec;
 
-import lombok.AllArgsConstructor;
 import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.http.HttpContentTypes;
 import org.cobbzilla.util.string.StringUtil;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.regex.Pattern;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.http.HttpContentTypes.contentType;
 
-@AllArgsConstructor
+@Slf4j
 public class BlockSpec {
 
+    public static final String OPT_DOMAIN_PREFIX = "domain=";
+    public static final String OPT_SCRIPT = "script";
+    public static final String OPT_IMAGE = "image";
+    public static final String OPT_STYLESHEET = "stylesheet";
+
+    @Getter private String line;
     @Getter private BlockSpecTarget target;
 
-    @Getter private List<String> options;
-    @Getter(lazy=true) private final Pattern domainPattern = Pattern.compile(target.getDomainRegex());
+    @Getter private List<String> domainExclusions;
+    @Getter private List<String> typeMatches;
+    public boolean hasTypeMatches () { return !empty(typeMatches); }
+
+    @Getter private List<String> typeExclusions;
+
+    public BlockSpec(String line, BlockSpecTarget target, List<String> options, String selector) {
+        this.line = line;
+        this.target = target;
+        this.selector = selector;
+        if (options != null) {
+            for (String opt : options) {
+                if (opt.startsWith(OPT_DOMAIN_PREFIX)) {
+                    processDomainOptions(opt.substring(OPT_DOMAIN_PREFIX.length()));
+
+                } else if (opt.startsWith("~")) {
+                    final String type = opt.substring(1);
+                    if (isTypeOption(type)) {
+                        if (typeExclusions == null) typeExclusions = new ArrayList<>();
+                        typeExclusions.add(type);
+                    } else {
+                        log.warn("unsupported option (ignoring): " + opt);
+                    }
+
+                } else {
+                    if (isTypeOption(opt)) {
+                        if (typeMatches == null) typeMatches = new ArrayList<>();
+                        typeMatches.add(opt);
+                    } else {
+                        log.warn("unsupported option (ignoring): "+opt);
+                    }
+                }
+            }
+        }
+    }
+
+    private void processDomainOptions(String option) {
+        final String[] parts = option.split("\\|");
+        for (String domainOption : parts) {
+            if (domainOption.startsWith("~")) {
+                if (domainExclusions == null) domainExclusions = new ArrayList<>();
+                domainExclusions.add(domainOption.substring(1));
+            } else {
+                log.warn("ignoring included domain: "+domainOption);
+            }
+        }
+    }
 
-    public boolean hasOptions () { return options != null && !options.isEmpty(); }
+    public boolean isTypeOption(String type) {
+        return type.equals(OPT_SCRIPT) || type.equals(OPT_IMAGE) || type.equals(OPT_STYLESHEET);
+    }
 
     @Getter private String selector;
     public boolean hasSelector() { return !empty(selector); }
 
-    public boolean isBlanket() { return !hasOptions() && !hasSelector(); }
-
     public static List<BlockSpec> parse(String line) {
 
         line = line.trim();
@@ -40,7 +92,7 @@ public class BlockSpec {
         if (optionStartPos == -1) {
             if (selectorStartPos == -1) {
                 // no options, no selector, entire line is the target
-                targets = BlockSpecTarget.parse(line);
+                targets = BlockSpecTarget.parseBareLine(line);
                 options = null;
                 selector = null;
             } else {
@@ -63,14 +115,64 @@ public class BlockSpec {
             }
         }
         final List<BlockSpec> specs = new ArrayList<>();
-        for (BlockSpecTarget target : targets) specs.add(new BlockSpec(target, options, selector));
+        for (BlockSpecTarget target : targets) specs.add(new BlockSpec(line, target, options, selector));
         return specs;
     }
 
     public boolean matches(String fqdn, String path) {
-        if (getDomainPattern().matcher(fqdn).find()) {
-            return true;
+        if (target.hasDomainRegex() && target.getDomainPattern().matcher(fqdn).find()) {
+            return checkDomainExclusionsAndType(fqdn, contentType(path));
+
+        } else if (target.hasRegex()) {
+            if (target.getRegexPattern().matcher(path).find()) {
+                return checkDomainExclusionsAndType(fqdn, contentType(path));
+            }
+            final String full = fqdn + path;
+            if (target.getRegexPattern().matcher(full).find()) {
+                return checkDomainExclusionsAndType(fqdn, contentType(path));
+            };
         }
         return false;
     }
+
+    public boolean checkDomainExclusionsAndType(String fqdn, String contentType) {
+        if (domainExclusions != null) {
+            for (String domain : domainExclusions) {
+                if (domain.equals(fqdn)) return false;
+            }
+        }
+        if (typeExclusions != null) {
+            for (String type : typeExclusions) {
+                switch (type) {
+                    case OPT_SCRIPT:
+                        if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return false;
+                        break;
+                    case OPT_IMAGE:
+                        if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return false;
+                        break;
+                    case OPT_STYLESHEET:
+                        if (contentType.equals(HttpContentTypes.TEXT_CSS)) return false;
+                        break;
+                }
+            }
+        }
+        if (typeMatches != null) {
+            for (String type : typeMatches) {
+                switch (type) {
+                    case OPT_SCRIPT:
+                        if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return true;
+                        break;
+                    case OPT_IMAGE:
+                        if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return true;
+                        break;
+                    case OPT_STYLESHEET:
+                        if (contentType.equals(HttpContentTypes.TEXT_CSS)) return true;
+                        break;
+                }
+            }
+            return false;
+        }
+        return true;
+    }
+
 }

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java

@@ -7,13 +7,22 @@ import lombok.experimental.Accessors;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.StringTokenizer;
 import java.util.regex.Pattern;
 
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.http.HttpSchemes.stripScheme;
+
 @NoArgsConstructor @Accessors(chain=true)
 public class BlockSpecTarget {
 
     @Getter @Setter private String domainRegex;
+    public boolean hasDomainRegex() { return !empty(domainRegex); }
+    @Getter(lazy=true) private final Pattern domainPattern = hasDomainRegex() ? Pattern.compile(getDomainRegex()) : null;
+
     @Getter @Setter private String regex;
+    public boolean hasRegex() { return !empty(regex); }
+    @Getter(lazy=true) private final Pattern regexPattern = hasRegex() ? Pattern.compile(getRegex()) : null;
 
     public static List<BlockSpecTarget> parse(String data) {
         final List<BlockSpecTarget> targets = new ArrayList<>();
@@ -23,18 +32,70 @@ public class BlockSpecTarget {
         return targets;
     }
 
+    public static List<BlockSpecTarget> parseBareLine(String data) {
+        if (data.contains("|") || data.contains("/") || data.contains("^")) return parse(data);
+
+        final List<BlockSpecTarget> targets = new ArrayList<>();
+        for (String part : data.split(",")) {
+            targets.add(new BlockSpecTarget().setDomainRegex(matchDomainOrAnySubdomains(part)));
+        }
+        return targets;
+    }
+
     private static BlockSpecTarget parseTarget(String data) {
         String domainRegex = null;
         String regex = null;
         if (data.startsWith("||")) {
             final int caretPos = data.indexOf("^");
             if (caretPos != -1) {
-                domainRegex = ".*?"+Pattern.quote(data.substring(2, caretPos))+"$";
+                // domain match
+                final String domain = data.substring(2, caretPos);
+                domainRegex = matchDomainOrAnySubdomains(domain);
+            } else {
+                final String domain = data.substring(2);
+                domainRegex = matchDomainOrAnySubdomains(domain);
+            }
+        } else if (data.startsWith("|") && data.endsWith("|")) {
+            // exact match
+            final String verbatimMatch = stripScheme(data.substring(1, data.length() - 1));
+            regex = "^" + Pattern.quote(verbatimMatch) + "$";
+
+        } else if (data.startsWith("/")) {
+            // path match, possibly regex
+            if (data.endsWith("/") && (
+                    data.contains("|") || data.contains("?")
+                            || (data.contains("(") && data.contains(")"))
+                            || (data.contains("{") && data.contains("}")))) {
+                regex = data.substring(1, data.length()-1);
+
+            } else if (data.contains("*")) {
+                regex = parseWildcardMatch(data);
             } else {
+                regex = "^" + Pattern.quote(data) + ".*";
+            }
 
+        } else {
+            if (data.contains("*")) {
+                regex = parseWildcardMatch(data);
+            } else {
+                regex = "^" + Pattern.quote(data) + ".*";
             }
         }
         return new BlockSpecTarget().setDomainRegex(domainRegex).setRegex(regex);
     }
 
+    private static String parseWildcardMatch(String data) {
+        final StringBuilder b = new StringBuilder("^");
+        final StringTokenizer st = new StringTokenizer(data, "*", true);
+        while (st.hasMoreTokens()) {
+            final String token = st.nextToken();
+            b.append(token.equals("*") ? ".*?" : token);
+        }
+        return b.append("$").toString();
+    }
+
+    private static String matchDomainOrAnySubdomains(String domain) {
+        return ".*?"+ Pattern.quote(domain)+"$";
+    }
+
 }

bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java

@@ -2,13 +2,156 @@ package bubble.rule.bblock.spec;
 
 import org.junit.Test;
 
+import java.util.Arrays;
+
 import static org.junit.Assert.assertEquals;
 
 public class BlockListTest {
 
-    @Test public void testBlanketBlock () throws Exception {
+    public static final String BLOCK = BlockDecisionType.block.name();
+    public static final String ALLOW = BlockDecisionType.allow.name();
+    public static final String FILTER = BlockDecisionType.filter.name();
+
+    public static final String[][] BLOCK_TESTS = {
+              // rule                  // fqdn                 // path          // expected decision
+
+            // bare hosts example (ala EasyList)
+            {"example.com",           "example.com",          "/some_path",      BLOCK},
+            {"example.com",           "foo.example.com",      "/some_path",      BLOCK},
+            {"example.com",           "example.org",          "/some_path",      ALLOW},
+
+            // block example.com and all subdomains
+            {"||example.com^",        "example.com",          "/some_path",      BLOCK},
+            {"||example.com^",        "foo.example.com",      "/some_path",      BLOCK},
+            {"||example.com^",        "example.org",          "/some_path",      ALLOW},
+
+            // block exact string
+            {"|example.com/|",        "example.com",          "/",               BLOCK},
+            {"|example.com/|",        "example.com",          "/some_path",      ALLOW},
+            {"|example.com/|",        "foo.example.com",      "/some_path",      ALLOW},
+
+            // block example.com, but not foo.example.com or bar.example.com
+            {"||example.com^$domain=~foo.example.com|~bar.example.com",
+                                      "example.com",          "/some_path",      BLOCK},
+            {"||example.com^$domain=~foo.example.com|~bar.example.com",
+                                      "foo.example.com",      "/some_path",      ALLOW},
+            {"||example.com^$domain=~foo.example.com|~bar.example.com",
+                                      "bar.example.com",      "/some_path",      ALLOW},
+            {"||example.com^$domain=~foo.example.com|~bar.example.com",
+                                      "baz.example.com",      "/some_path",      BLOCK},
+
+            // block images and scripts on example.com, but not foo.example.com or bar.example.com
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "example.com",            "/some_path",      ALLOW},
+
+            // test image blocking
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "example.com",            "/some_path.png",  BLOCK},
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "foo.example.com",        "/some_path.png",  ALLOW},
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "bar.example.com",        "/some_path.png",  ALLOW},
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "baz.example.com",         "/some_path.png", BLOCK},
+
+            // test script blocking
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "example.com",            "/some_path.js",   BLOCK},
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "foo.example.com",        "/some_path.js",   ALLOW},
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "bar.example.com",        "/some_path.js",   ALLOW},
+            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
+                                      "baz.example.com",         "/some_path.js",  BLOCK},
+
+            // test stylesheet blocking
+            {"||example.com^stylesheet,domain=~foo.example.com|~bar.example.com",
+                    "example.com",            "/some_path.css",  BLOCK},
+            {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
+                    "foo.example.com",        "/some_path.css",  ALLOW},
+            {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
+                    "bar.example.com",        "/some_path.css",  ALLOW},
+            {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
+                    "baz.example.com",         "/some_path.css", BLOCK},
+
+
+            // path matching
+            {"/foo",                  "example.com",          "/some_path",      ALLOW},
+            {"/foo",                  "example.com",          "/foo",            BLOCK},
+            {"/foo",                  "example.com",          "/foo/bar",        BLOCK},
+
+            // path matching with wildcard
+            {"/foo/*/img",            "example.com",          "/some_path",      ALLOW},
+            {"/foo/*/img",            "example.com",          "/foo",            ALLOW},
+            {"/foo/*/img",            "example.com",          "/foo/img",        ALLOW},
+            {"/foo/*/img",            "example.com",          "/foo/x/img",      BLOCK},
+            {"/foo/*/img",            "example.com",          "/foo/x/img.png",  ALLOW},
+            {"/foo/*/img",            "example.com",          "/foo/x/y/z//img", BLOCK},
+
+            // path matching with regex
+            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/x/y/z//img",      ALLOW},
+            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/apps/img.png",    BLOCK},
+            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/ads/img.png",     BLOCK},
+            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/bar/ads/img.png", ALLOW},
+
+            {"/(apps|ads)\\.example\\.(com|org)/",
+                                        "example.com",          "/ad.png",              ALLOW},
+            {"/(apps|ads)\\.example\\.(com|org)/",
+                                        "ads.example.com",      "/ad.png",              BLOCK},
+            {"/(apps|ads)\\.example\\.(com|org)/",
+                                        "apps.example.com",     "/ad.png",              BLOCK},
+            {"/(apps|ads)\\.example\\.(com|org)/",
+                                        "ads.example.org",      "/ad.png",              BLOCK},
+            {"/(apps|ads)\\.example\\.(com|org)/",
+                                        "apps.example.org",     "/ad.png",              BLOCK},
+            {"/(apps|ads)\\.example\\.(com|org)/",
+                                        "ads.example.net",      "/ad.png",              ALLOW},
+            {"/(apps|ads)\\.example\\.(com|org)/",
+                                        "apps.example.net",     "/ad.png",              ALLOW},
+
+            // selectors
+            {"example.com##.banner-ad", "example.com",          "/ad.png",              FILTER},
+
+            // putting it all together
+            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
+                    "example.com",          "/some_path",      FILTER},
+            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
+                    "baz.example.com",      "/some_path",      FILTER},
+            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
+                    "foo.example.com",      "/some_path",      ALLOW},
+            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
+                    "bar.example.com",      "/some_path",      ALLOW},
+    };
+
+    @Test public void testRules () throws Exception {
+        for (String[] test : BLOCK_TESTS) {
+            final BlockDecisionType expectedDecision = BlockDecisionType.fromString(test[3]);
+            final BlockList blockList = new BlockList();
+            blockList.addToBlacklist(BlockSpec.parse(test[0]));
+            assertEquals("testBlanketBlock: expected "+expectedDecision+" decision, test=" + Arrays.toString(test),
+                    expectedDecision,
+                    blockList.getDecision(test[1], test[2]).getDecisionType());
+        }
+    }
+
+    public String[] SELECTOR_SPECS = {
+            "||example.com##.banner-ad",
+            "||foo.example.com##.more-ads",
+    };
+
+    @Test public void testMultipleSelectorMatches () throws Exception {
         final BlockList blockList = new BlockList();
-        blockList.addToBlacklist(BlockSpec.parse("||fredfiber.no^"));
-        assertEquals("expected block", BlockDecisionType.block, blockList.getDecision("fredfiber.no", "/somepath").getDecisionType());
+        for (String line : SELECTOR_SPECS) {
+            blockList.addToBlacklist(BlockSpec.parse(line));
+        }
+        BlockDecision decision;
+
+        decision = blockList.getDecision("example.com", "/some_path");
+        assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType());
+        assertEquals("expected 1 filter specs", 1, decision.getSpecs().size());
+
+        decision = blockList.getDecision("foo.example.com", "/some_path");
+        assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType());
+        assertEquals("expected 2 filter specs", 2, decision.getSpecs().size());
     }
 }

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 006bcd1ff4390bb37ae35d50fa12a70de7e408c1
+Subproject commit a17850215a54359929b83a9c5b061fd608090444

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 78b8da16659be214013288328f932509ed4c9224
+Subproject commit 6a3824eee748ded81eb4caf065f444f565708b1c