add abp-parser submodule, allow filters to pass arbitrary metadata instead of just filters

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "bubble-web"]
 	path = bubble-web
 	url = git@git.bubblev.org:bubblev/bubble-web.git
+[submodule "utils/abp-parser"]
+	path = utils/abp-parser
+	url = git@git.bubblev.org:bubblev/abp-parser.git

bubble-server/pom.xml

@@ -41,6 +41,12 @@ For commercial use, please contact jonathan@kyuss.org
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>bubble</groupId>
+            <artifactId>abp-parser</artifactId>
+            <version>1.0.0-SNAPSHOT</version>
+        </dependency>
+
         <!-- RDBMS persistence -->
         <dependency>
             <groupId>org.hibernate</groupId>

bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java

@@ -7,6 +7,7 @@ import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.Accessors;
 import org.apache.commons.lang.ArrayUtils;
+import org.cobbzilla.util.collection.NameAndValue;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
@@ -16,7 +17,7 @@ public class FilterHttpRequest {
     @Getter @Setter private String id;
     @Getter @Setter private Device device;
     @Getter @Setter private Account account;
-    @Getter @Setter private String[] filters;
+    @Getter @Setter private NameAndValue[] meta;
     @Getter @Setter private String contentType;
 
     @Getter @Setter private String[] matchers;

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -15,6 +15,7 @@ import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ArrayUtil;
 import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.http.HttpContentEncodingType;
 import org.cobbzilla.util.http.HttpStatusCodes;
 import org.cobbzilla.wizard.cache.redis.RedisService;
@@ -87,7 +88,7 @@ public class FilterHttpResource {
     private Map<String, FilterMatchersResponse> matchersCache = new ExpirationMap<>(MINUTES.toMillis(5));
 
     private static final long REQUEST_FILTERS_TIMEOUT = MINUTES.toSeconds(1);
-    @Getter(lazy=true) private final RedisService filtersForRequest = redis.prefixNamespace(getClass().getSimpleName()+".filters");
+    @Getter(lazy=true) private final RedisService requestMeta = redis.prefixNamespace(getClass().getSimpleName()+".filters");
 
     @POST @Path(EP_MATCHERS)
     @Consumes(APPLICATION_JSON)
@@ -105,8 +106,8 @@ public class FilterHttpResource {
         final String cacheKey = remoteHost+":"+filterRequest.cacheKey();
         final FilterMatchersResponse response = matchersCache.computeIfAbsent(cacheKey, k -> findMatchers(filterRequest, req, request));
 
-        if (response.hasFilters()) {
-            getFiltersForRequest().set(filterRequest.getRequestId(), json(response.getFilters()), EX, REQUEST_FILTERS_TIMEOUT);
+        if (response.hasMeta()) {
+            getRequestMeta().set(filterRequest.getRequestId(), json(response.getMeta()), EX, REQUEST_FILTERS_TIMEOUT);
         }
 
         return ok(response);
@@ -138,7 +139,7 @@ public class FilterHttpResource {
         final List<AppMatcher> matchers = matcherDAO.findByAccountAndFqdnAndEnabled(accountUuid, fqdn);
         if (log.isDebugEnabled()) log.debug("findMatchers: found "+matchers.size()+" candidate matchers");
         final List<AppMatcher> removeMatchers;
-        List<String> filters = null;
+        NameAndValue[] meta = null;
         if (matchers.isEmpty()) {
             removeMatchers = Collections.emptyList();
         } else {
@@ -154,9 +155,9 @@ public class FilterHttpResource {
                             removeMatchers.add(matcher);
                             break;
                         case match:
-                            if (matchResponse.hasFilters()) {
-                                if (filters == null) filters = new ArrayList<>();
-                                filters.addAll(matchResponse.getFilters());
+                            if (matchResponse.hasMeta()) {
+                                if (meta == null) meta = NameAndValue.EMPTY_ARRAY;
+                                meta = ArrayUtil.concat(meta, matchResponse.getMeta());
                             }
                             break;
                     }
@@ -170,7 +171,7 @@ public class FilterHttpResource {
         return response
                 .setMatchers(matchers)
                 .setDevice(device.getUuid())
-                .setFilters(filters);
+                .setMeta(meta);
     }
 
     @POST @Path(EP_APPLY+"/{requestId}")
@@ -265,9 +266,9 @@ public class FilterHttpResource {
 
         // check for filters
         try {
-            final String filtersJson = getFiltersForRequest().get(requestId);
+            final String filtersJson = getRequestMeta().get(requestId);
             if (filtersJson != null) {
-                filterRequest.setFilters(json(filtersJson, String[].class));
+                filterRequest.setMeta(json(filtersJson, NameAndValue[].class));
             }
         } catch (Exception e) {
             log.error("filterHttp: error reading pageFilters: "+shortError(e));

bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java

@@ -5,8 +5,7 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.Accessors;
-
-import java.util.List;
+import org.cobbzilla.util.collection.NameAndValue;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
@@ -20,7 +19,7 @@ public class FilterMatchResponse {
 
     @Getter @Setter private FilterMatchDecision decision;
 
-    @Getter @Setter private List<String> filters;
-    public boolean hasFilters() { return !empty(filters); }
+    @Getter @Setter private NameAndValue[] meta;
+    public boolean hasMeta() { return !empty(meta); }
 
 }

bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java

@@ -5,6 +5,7 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.Accessors;
+import org.cobbzilla.util.collection.NameAndValue;
 
 import java.util.List;
 
@@ -18,7 +19,7 @@ public class FilterMatchersResponse {
     @Getter @Setter private Integer abort;
     @Getter @Setter private String device;
     @Getter @Setter private List<AppMatcher> matchers;
-    @Getter @Setter private List<String> filters;
-    public boolean hasFilters () { return !empty(filters); }
+    @Getter @Setter private NameAndValue[] meta;
+    public boolean hasMeta() { return !empty(meta); }
 
 }

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -9,6 +9,7 @@ import bubble.service.stream.AppRuleHarness;
 import bubble.resources.stream.FilterMatchersRequest;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jknack.handlebars.Handlebars;
+import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
@@ -49,12 +50,12 @@ public interface AppRuleDriver {
 
     default InputStream doFilterRequest(InputStream in) { return in; }
 
-    default InputStream filterResponse(String requestId, String contentType, String[] filters, InputStream in) {
-        if (hasNext()) return doFilterResponse(requestId, contentType, filters, getNext().filterResponse(requestId, contentType, filters, in));
-        return doFilterResponse(requestId, contentType, filters, in);
+    default InputStream filterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
+        if (hasNext()) return doFilterResponse(requestId, contentType, meta, getNext().filterResponse(requestId, contentType, meta, in));
+        return doFilterResponse(requestId, contentType, meta, in);
     }
 
-    default InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) { return in; }
+    default InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) { return in; }
 
     default String resolveResource(String res, Map<String, Object> ctx) {
         final String resource = locateResource(res);

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java

@@ -1,19 +1,22 @@
 package bubble.rule.bblock;
 
+import bubble.abp.spec.BlockDecision;
+import bubble.abp.spec.BlockList;
+import bubble.abp.spec.BlockListSource;
+import bubble.abp.spec.BlockSpec;
 import bubble.model.account.Account;
 import bubble.model.app.AppMatcher;
 import bubble.model.app.AppRule;
 import bubble.model.device.Device;
 import bubble.resources.stream.FilterMatchResponse;
 import bubble.resources.stream.FilterMatchersRequest;
+import bubble.rule.FilterMatchDecision;
 import bubble.rule.analytics.TrafficAnalytics;
-import bubble.rule.bblock.spec.BlockDecision;
-import bubble.rule.bblock.spec.BlockList;
-import bubble.rule.bblock.spec.BlockListSource;
 import bubble.service.stream.AppRuleHarness;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.ReaderInputStream;
+import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
 import org.cobbzilla.util.io.regex.RegexFilterReader;
 import org.cobbzilla.util.io.regex.RegexReplacementFilter;
@@ -23,12 +26,12 @@ import org.glassfish.jersey.server.ContainerRequest;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
@@ -38,6 +41,8 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath;
 @Slf4j
 public class BubbleBlock extends TrafficAnalytics {
 
+    private static final String META_BLOCK_FILTERS = "__bubble_block_filters";
+
     private BubbleBlockConfig bubbleBlockConfig;
 
     private BlockList blockList = new BlockList();
@@ -84,15 +89,34 @@ public class BubbleBlock extends TrafficAnalytics {
             case allow: default:
                 return FilterMatchResponse.NO_MATCH;
             case filter:
-                return decision.getFilterMatchResponse();
+                return getFilterMatchResponse(decision);
+        }
+    }
+
+    public FilterMatchResponse getFilterMatchResponse(BlockDecision decision) {
+        switch (decision.getDecisionType()) {
+            case block: return FilterMatchResponse.ABORT_NOT_FOUND;
+            case allow: return FilterMatchResponse.NO_MATCH;
+            case filter:
+                final List<BlockSpec> specs = decision.getSpecs();
+                if (empty(specs)) {
+                    log.warn("getFilterMatchResponse: decision was 'filter' but no specs were found, returning no_match");
+                    return FilterMatchResponse.NO_MATCH;
+                } else {
+                    return new FilterMatchResponse()
+                            .setDecision(FilterMatchDecision.match)
+                            .setMeta(new NameAndValue[]{new NameAndValue(META_BLOCK_FILTERS, json(specs))});
+                }
         }
+        return die("getFilterMatchResponse: invalid decisionType: "+decision.getDecisionType());
     }
 
-    @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) {
+    @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
 
-        if (empty(filters) || !isHtml(contentType)) return in;
+        final String blockSpecJson = NameAndValue.find(meta, META_BLOCK_FILTERS);
+        if (empty(blockSpecJson) || !isHtml(contentType)) return in;
 
-        final String replacement = "<head><script>" + getBubbleJs(requestId, filters) + "</script>";
+        final String replacement = "<head><script>" + getBubbleJs(requestId, blockSpecJson) + "</script>";
         final RegexReplacementFilter filter = new RegexReplacementFilter("<head>", replacement);
         final RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in), filter).setMaxMatches(1);
         return new ReaderInputStream(reader, UTF8cs);
@@ -100,15 +124,15 @@ public class BubbleBlock extends TrafficAnalytics {
 
     public static final Class<BubbleBlock> BB = BubbleBlock.class;
     public static final String BUBBLE_JS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+".js.hbs");
-    private static final String CTX_BUBBLE_FILTERS = "BUBBLE_FILTERS";
+    private static final String CTX_BUBBLE_BLOCK_SPEC_JSON = "BUBBLE_BLOCK_SPEC_JSON";
 
-    private String getBubbleJs(String requestId, String[] filters) {
+    private String getBubbleJs(String requestId, String blockSpecJson) {
         final Map<String, Object> ctx = new HashMap<>();
         ctx.put(CTX_JS_PREFIX, "__bubble_block_"+sha256_hex(requestId)+"_");
         ctx.put(CTX_BUBBLE_REQUEST_ID, requestId);
         ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase());
         ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId));
-        ctx.put(CTX_BUBBLE_FILTERS, filters);
+        ctx.put(CTX_BUBBLE_BLOCK_SPEC_JSON, blockSpecJson);
         return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx);
     }
 

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java

@@ -1,44 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import bubble.resources.stream.FilterMatchResponse;
-import bubble.rule.FilterMatchDecision;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-
-@NoArgsConstructor @Accessors(chain=true)
-public class BlockDecision {
-
-    public static final BlockDecision BLOCK = new BlockDecision().setDecisionType(BlockDecisionType.block);
-    public static final BlockDecision ALLOW = new BlockDecision().setDecisionType(BlockDecisionType.allow);
-
-    @Getter @Setter BlockDecisionType decisionType = BlockDecisionType.allow;
-    @Getter @Setter List<BlockSpec> specs;
-
-    public BlockDecision add(BlockSpec spec) {
-        if (specs == null) specs = new ArrayList<>();
-        specs.add(spec);
-        if (decisionType != BlockDecisionType.block && (spec.hasTypeMatches() || spec.hasSelector())) {
-            decisionType = BlockDecisionType.filter;
-        }
-        return this;
-    }
-
-    public FilterMatchResponse getFilterMatchResponse() {
-        switch (decisionType) {
-            case block: return FilterMatchResponse.ABORT_NOT_FOUND;
-            case allow: return FilterMatchResponse.NO_MATCH;
-            case filter: return new FilterMatchResponse()
-                    .setDecision(FilterMatchDecision.match)
-                    .setFilters(specs == null ? null : getSpecs().stream().map(BlockSpec::getLine).collect(Collectors.toList()));
-        }
-        return die("getFilterMatchResponse: invalid decisionType: "+decisionType);
-    }
-}

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java

@@ -1,13 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-
-import static bubble.ApiConstants.enumFromString;
-
-public enum BlockDecisionType {
-
-    block, allow, filter;
-
-    @JsonCreator public static BlockDecisionType fromString (String v) { return enumFromString(BlockDecisionType.class, v); }
-
-}

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java

@@ -1,55 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.experimental.Accessors;
-import lombok.extern.slf4j.Slf4j;
-
-import java.util.*;
-
-@NoArgsConstructor @Accessors(chain=true) @Slf4j
-public class BlockList {
-
-    @Getter private Set<BlockSpec> blacklist = new HashSet<>();
-    @Getter private Set<BlockSpec> whitelist = new HashSet<>();
-
-    public void addToWhitelist(BlockSpec spec) {
-        whitelist.add(spec);
-    }
-
-    public void addToWhitelist(List<BlockSpec> specs) {
-        for (BlockSpec spec : specs) addToWhitelist(spec);
-    }
-
-    public void addToBlacklist(BlockSpec spec) {
-        blacklist.add(spec);
-    }
-
-    public void addToBlacklist(List<BlockSpec> specs) {
-        for (BlockSpec spec : specs) addToBlacklist(spec);
-    }
-
-    public void merge(BlockList other) {
-        for (BlockSpec allow : other.getWhitelist()) {
-            addToWhitelist(allow);
-        }
-        for (BlockSpec block : other.getBlacklist()) {
-            addToBlacklist(block);
-        }
-    }
-
-    public BlockDecision getDecision(String fqdn, String path) {
-        for (BlockSpec allow : whitelist) {
-            if (allow.matches(fqdn, path)) return BlockDecision.ALLOW;
-        }
-        final BlockDecision decision = new BlockDecision();
-        for (BlockSpec block : blacklist) {
-            if (block.matches(fqdn, path)) {
-                if (!block.hasSelector()) return BlockDecision.BLOCK;
-                decision.add(block);
-            }
-        }
-        return decision;
-    }
-
-}

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java

@@ -1,55 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.http.HttpUtil;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.*;
-import static org.cobbzilla.util.daemon.ZillaRuntime.now;
-
-@NoArgsConstructor @Accessors(chain=true) @Slf4j
-public class BlockListSource {
-
-    @Getter @Setter private String url;
-    @Getter @Setter private String format;
-
-    @Getter @Setter private Long lastDownloaded;
-    public long age () { return lastDownloaded == null ? Long.MAX_VALUE : now() - lastDownloaded; }
-
-    @Getter @Setter private BlockList blockList = new BlockList();
-
-    public BlockListSource download() throws IOException {
-        try (BufferedReader r = new BufferedReader(new InputStreamReader(HttpUtil.get(url)))) {
-            String line;
-            boolean firstLine = true;
-            while ( (line = r.readLine()) != null ) {
-                if (empty(line)) continue;
-                line = line.trim();
-                if (firstLine && line.startsWith("[") && line.endsWith("]")) {
-                    format = line.substring(1, line.length()-1);
-                }
-                firstLine = false;
-                if (line.startsWith("!")) continue;
-                try {
-                    if (line.startsWith("@@")) {
-                        blockList.addToWhitelist(BlockSpec.parse(line));
-                    } else {
-                        blockList.addToBlacklist(BlockSpec.parse(line));
-                    }
-                } catch (Exception e) {
-                    log.warn("download("+url+"): error parsing line (skipping due to "+shortError(e)+"): " + line);
-                }
-            }
-        }
-        lastDownloaded = now();
-        return this;
-    }
-
-}

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java

@@ -1,178 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.http.HttpContentTypes;
-import org.cobbzilla.util.string.StringUtil;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.http.HttpContentTypes.contentType;
-
-@Slf4j
-public class BlockSpec {
-
-    public static final String OPT_DOMAIN_PREFIX = "domain=";
-    public static final String OPT_SCRIPT = "script";
-    public static final String OPT_IMAGE = "image";
-    public static final String OPT_STYLESHEET = "stylesheet";
-
-    @Getter private String line;
-    @Getter private BlockSpecTarget target;
-
-    @Getter private List<String> domainExclusions;
-    @Getter private List<String> typeMatches;
-    public boolean hasTypeMatches () { return !empty(typeMatches); }
-
-    @Getter private List<String> typeExclusions;
-
-    public BlockSpec(String line, BlockSpecTarget target, List<String> options, String selector) {
-        this.line = line;
-        this.target = target;
-        this.selector = selector;
-        if (options != null) {
-            for (String opt : options) {
-                if (opt.startsWith(OPT_DOMAIN_PREFIX)) {
-                    processDomainOptions(opt.substring(OPT_DOMAIN_PREFIX.length()));
-
-                } else if (opt.startsWith("~")) {
-                    final String type = opt.substring(1);
-                    if (isTypeOption(type)) {
-                        if (typeExclusions == null) typeExclusions = new ArrayList<>();
-                        typeExclusions.add(type);
-                    } else {
-                        log.warn("unsupported option (ignoring): " + opt);
-                    }
-
-                } else {
-                    if (isTypeOption(opt)) {
-                        if (typeMatches == null) typeMatches = new ArrayList<>();
-                        typeMatches.add(opt);
-                    } else {
-                        log.warn("unsupported option (ignoring): "+opt);
-                    }
-                }
-            }
-        }
-    }
-
-    private void processDomainOptions(String option) {
-        final String[] parts = option.split("\\|");
-        for (String domainOption : parts) {
-            if (domainOption.startsWith("~")) {
-                if (domainExclusions == null) domainExclusions = new ArrayList<>();
-                domainExclusions.add(domainOption.substring(1));
-            } else {
-                log.warn("ignoring included domain: "+domainOption);
-            }
-        }
-    }
-
-    public boolean isTypeOption(String type) {
-        return type.equals(OPT_SCRIPT) || type.equals(OPT_IMAGE) || type.equals(OPT_STYLESHEET);
-    }
-
-    @Getter private String selector;
-    public boolean hasSelector() { return !empty(selector); }
-
-    public static List<BlockSpec> parse(String line) {
-
-        line = line.trim();
-        int optionStartPos = line.indexOf('$');
-        int selectorStartPos = line.indexOf("##");
-
-        // sanity check that selectorStartPos > optionStartPos -- $ may occur AFTER ## if the selector contains a regex
-        if (selectorStartPos != -1 && optionStartPos > selectorStartPos) optionStartPos = -1;
-
-        final List<BlockSpecTarget> targets;
-        final List<String> options;
-        final String selector;
-        if (optionStartPos == -1) {
-            if (selectorStartPos == -1) {
-                // no options, no selector, entire line is the target
-                targets = BlockSpecTarget.parseBareLine(line);
-                options = null;
-                selector = null;
-            } else {
-                // no options, but selector present. split into target + selector
-                targets = BlockSpecTarget.parse(line.substring(0, selectorStartPos));
-                options = null;
-                selector = line.substring(selectorStartPos+1);
-            }
-        } else {
-            if (selectorStartPos == -1) {
-                // no selector, split into target + options
-                targets = BlockSpecTarget.parse(line.substring(0, optionStartPos));
-                options = StringUtil.splitAndTrim(line.substring(optionStartPos+1), ",");
-                selector = null;
-            } else {
-                // all 3 elements present
-                targets = BlockSpecTarget.parse(line.substring(0, optionStartPos));
-                options = StringUtil.splitAndTrim(line.substring(optionStartPos + 1, selectorStartPos), ",");
-                selector = line.substring(selectorStartPos+1);
-            }
-        }
-        final List<BlockSpec> specs = new ArrayList<>();
-        for (BlockSpecTarget target : targets) specs.add(new BlockSpec(line, target, options, selector));
-        return specs;
-    }
-
-    public boolean matches(String fqdn, String path) {
-        if (target.hasDomainRegex() && target.getDomainPattern().matcher(fqdn).find()) {
-            return checkDomainExclusionsAndType(fqdn, contentType(path));
-
-        } else if (target.hasRegex()) {
-            if (target.getRegexPattern().matcher(path).find()) {
-                return checkDomainExclusionsAndType(fqdn, contentType(path));
-            }
-            final String full = fqdn + path;
-            if (target.getRegexPattern().matcher(full).find()) {
-                return checkDomainExclusionsAndType(fqdn, contentType(path));
-            };
-        }
-        return false;
-    }
-
-    public boolean checkDomainExclusionsAndType(String fqdn, String contentType) {
-        if (domainExclusions != null) {
-            for (String domain : domainExclusions) {
-                if (domain.equals(fqdn)) return false;
-            }
-        }
-        if (typeExclusions != null) {
-            for (String type : typeExclusions) {
-                switch (type) {
-                    case OPT_SCRIPT:
-                        if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return false;
-                        break;
-                    case OPT_IMAGE:
-                        if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return false;
-                        break;
-                    case OPT_STYLESHEET:
-                        if (contentType.equals(HttpContentTypes.TEXT_CSS)) return false;
-                        break;
-                }
-            }
-        }
-        if (typeMatches != null) {
-            for (String type : typeMatches) {
-                switch (type) {
-                    case OPT_SCRIPT:
-                        if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return true;
-                        break;
-                    case OPT_IMAGE:
-                        if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return true;
-                        break;
-                    case OPT_STYLESHEET:
-                        if (contentType.equals(HttpContentTypes.TEXT_CSS)) return true;
-                        break;
-                }
-            }
-            return false;
-        }
-        return true;
-    }
-
-}

bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java

@@ -1,101 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.StringTokenizer;
-import java.util.regex.Pattern;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.http.HttpSchemes.stripScheme;
-
-@NoArgsConstructor @Accessors(chain=true)
-public class BlockSpecTarget {
-
-    @Getter @Setter private String domainRegex;
-    public boolean hasDomainRegex() { return !empty(domainRegex); }
-    @Getter(lazy=true) private final Pattern domainPattern = hasDomainRegex() ? Pattern.compile(getDomainRegex()) : null;
-
-    @Getter @Setter private String regex;
-    public boolean hasRegex() { return !empty(regex); }
-    @Getter(lazy=true) private final Pattern regexPattern = hasRegex() ? Pattern.compile(getRegex()) : null;
-
-    public static List<BlockSpecTarget> parse(String data) {
-        final List<BlockSpecTarget> targets = new ArrayList<>();
-        for (String part : data.split(",")) {
-            targets.add(parseTarget(part));
-        }
-        return targets;
-    }
-
-    public static List<BlockSpecTarget> parseBareLine(String data) {
-        if (data.contains("|") || data.contains("/") || data.contains("^")) return parse(data);
-
-        final List<BlockSpecTarget> targets = new ArrayList<>();
-        for (String part : data.split(",")) {
-            targets.add(new BlockSpecTarget().setDomainRegex(matchDomainOrAnySubdomains(part)));
-        }
-        return targets;
-    }
-
-    private static BlockSpecTarget parseTarget(String data) {
-        String domainRegex = null;
-        String regex = null;
-        if (data.startsWith("||")) {
-            final int caretPos = data.indexOf("^");
-            if (caretPos != -1) {
-                // domain match
-                final String domain = data.substring(2, caretPos);
-                domainRegex = matchDomainOrAnySubdomains(domain);
-            } else {
-                final String domain = data.substring(2);
-                domainRegex = matchDomainOrAnySubdomains(domain);
-            }
-        } else if (data.startsWith("|") && data.endsWith("|")) {
-            // exact match
-            final String verbatimMatch = stripScheme(data.substring(1, data.length() - 1));
-            regex = "^" + Pattern.quote(verbatimMatch) + "$";
-
-        } else if (data.startsWith("/")) {
-            // path match, possibly regex
-            if (data.endsWith("/") && (
-                    data.contains("|") || data.contains("?")
-                            || (data.contains("(") && data.contains(")"))
-                            || (data.contains("{") && data.contains("}")))) {
-                regex = data.substring(1, data.length()-1);
-
-            } else if (data.contains("*")) {
-                regex = parseWildcardMatch(data);
-            } else {
-                regex = "^" + Pattern.quote(data) + ".*";
-            }
-
-        } else {
-            if (data.contains("*")) {
-                regex = parseWildcardMatch(data);
-            } else {
-                regex = "^" + Pattern.quote(data) + ".*";
-            }
-        }
-        return new BlockSpecTarget().setDomainRegex(domainRegex).setRegex(regex);
-    }
-
-    private static String parseWildcardMatch(String data) {
-        final StringBuilder b = new StringBuilder("^");
-        final StringTokenizer st = new StringTokenizer(data, "*", true);
-        while (st.hasMoreTokens()) {
-            final String token = st.nextToken();
-            b.append(token.equals("*") ? ".*?" : token);
-        }
-        return b.append("$").toString();
-    }
-
-    private static String matchDomainOrAnySubdomains(String domain) {
-        return ".*?"+ Pattern.quote(domain)+"$";
-    }
-
-}

bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java

@@ -5,6 +5,7 @@ import bubble.rule.AbstractAppRuleDriver;
 import lombok.Getter;
 import org.apache.commons.io.input.ReaderInputStream;
 import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
 import org.cobbzilla.util.io.regex.RegexFilterReader;
 import org.cobbzilla.util.io.regex.RegexReplacementFilter;
@@ -27,7 +28,7 @@ public class JsUserBlocker extends AbstractAppRuleDriver {
 
     public static final String CTX_APPLY_BLOCKS_JS = "APPLY_BLOCKS_JS";
 
-    @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) {
+    @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
         if (!isHtml(contentType)) return in;
 
         final String replacement = "<head><script>" + getBubbleJs(requestId) + "</script>";

bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java

@@ -7,6 +7,7 @@ import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.ReaderInputStream;
+import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.io.regex.RegexFilterReader;
 import org.cobbzilla.util.io.regex.RegexInsertionFilter;
 import org.cobbzilla.util.io.regex.RegexStreamFilter;
@@ -47,7 +48,7 @@ public class UserBlocker extends AbstractAppRuleDriver {
 
     protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); }
 
-    @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) {
+    @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
         if (!isHtml(contentType)) return in;
 
         final UserBlockerStreamFilter filter = new UserBlockerStreamFilter(requestId, matcher, rule, configuration.getHttp().getBaseUri());

bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java

@@ -28,6 +28,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.cobbzilla.util.collection.ExpirationEvictionPolicy;
 import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.http.HttpClosingFilterInputStream;
 import org.cobbzilla.util.http.HttpContentEncodingType;
 import org.cobbzilla.util.http.HttpMethods;
@@ -130,7 +131,7 @@ public class RuleEngineService {
         // filter response. when stream is closed, close http client
         final Header contentTypeHeader = proxyResponse.getFirstHeader(CONTENT_TYPE);
         final String contentType = contentTypeHeader == null ? null : contentTypeHeader.getValue();
-        final InputStream responseEntity = firstRule.getDriver().filterResponse(filterRequest.getId(), contentType, filterRequest.getFilters(), new HttpClosingFilterInputStream(httpClient, proxyResponse));
+        final InputStream responseEntity = firstRule.getDriver().filterResponse(filterRequest.getId(), contentType, filterRequest.getMeta(), new HttpClosingFilterInputStream(httpClient, proxyResponse));
 
         // send response
         return sendResponse(responseEntity, proxyResponse);
@@ -157,7 +158,7 @@ public class RuleEngineService {
 
         // have we seen this request before?
         final ActiveStreamState state = activeProcessors.computeIfAbsent(filterRequest.getId(),
-                k -> new ActiveStreamState(k, contentEncoding, contentType, filterRequest.getFilters(),
+                k -> new ActiveStreamState(k, contentEncoding, contentType, filterRequest.getMeta(),
                         initRules(filterRequest.getAccount(), filterRequest.getDevice(), filterRequest.getMatchers())));
         final byte[] chunk = toBytes(request.getEntityStream(), contentLength);
         if (last) {
@@ -264,7 +265,7 @@ public class RuleEngineService {
         private String requestId;
         private HttpContentEncodingType encoding;
         private String contentType;
-        private String[] filters;
+        private NameAndValue[] meta;
         private MultiStream multiStream;
         private AppRuleHarness firstRule;
         private InputStream output = null;
@@ -274,12 +275,12 @@ public class RuleEngineService {
         public ActiveStreamState(String requestId,
                                  HttpContentEncodingType encoding,
                                  String contentType,
-                                 String[] filters,
+                                 NameAndValue[] meta,
                                  List<AppRuleHarness> rules) {
             this.requestId = requestId;
             this.encoding = encoding;
             this.contentType = contentType;
-            this.filters = filters;
+            this.meta = meta;
             this.firstRule = rules.get(0);
         }
 
@@ -288,7 +289,7 @@ public class RuleEngineService {
             totalBytesWritten += chunk.length;
             if (multiStream == null) {
                 multiStream = new MultiStream(new ByteArrayInputStream(chunk));
-                output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, filters, inputStream(multiStream)));
+                output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, meta, inputStream(multiStream)));
             } else {
                 multiStream.addStream(new ByteArrayInputStream(chunk));
             }
@@ -299,7 +300,7 @@ public class RuleEngineService {
             totalBytesWritten += chunk.length;
             if (multiStream == null) {
                 multiStream = new MultiStream(new ByteArrayInputStream(chunk), true);
-                output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, filters, inputStream(multiStream)));
+                output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, meta, inputStream(multiStream)));
             } else {
                 multiStream.addLastStream(new ByteArrayInputStream(chunk));
             }

bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java

@@ -1,157 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import org.junit.Test;
-
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-
-public class BlockListTest {
-
-    public static final String BLOCK = BlockDecisionType.block.name();
-    public static final String ALLOW = BlockDecisionType.allow.name();
-    public static final String FILTER = BlockDecisionType.filter.name();
-
-    public static final String[][] BLOCK_TESTS = {
-              // rule                  // fqdn                 // path          // expected decision
-
-            // bare hosts example (ala EasyList)
-            {"example.com",           "example.com",          "/some_path",      BLOCK},
-            {"example.com",           "foo.example.com",      "/some_path",      BLOCK},
-            {"example.com",           "example.org",          "/some_path",      ALLOW},
-
-            // block example.com and all subdomains
-            {"||example.com^",        "example.com",          "/some_path",      BLOCK},
-            {"||example.com^",        "foo.example.com",      "/some_path",      BLOCK},
-            {"||example.com^",        "example.org",          "/some_path",      ALLOW},
-
-            // block exact string
-            {"|example.com/|",        "example.com",          "/",               BLOCK},
-            {"|example.com/|",        "example.com",          "/some_path",      ALLOW},
-            {"|example.com/|",        "foo.example.com",      "/some_path",      ALLOW},
-
-            // block example.com, but not foo.example.com or bar.example.com
-            {"||example.com^$domain=~foo.example.com|~bar.example.com",
-                                      "example.com",          "/some_path",      BLOCK},
-            {"||example.com^$domain=~foo.example.com|~bar.example.com",
-                                      "foo.example.com",      "/some_path",      ALLOW},
-            {"||example.com^$domain=~foo.example.com|~bar.example.com",
-                                      "bar.example.com",      "/some_path",      ALLOW},
-            {"||example.com^$domain=~foo.example.com|~bar.example.com",
-                                      "baz.example.com",      "/some_path",      BLOCK},
-
-            // block images and scripts on example.com, but not foo.example.com or bar.example.com
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "example.com",            "/some_path",      ALLOW},
-
-            // test image blocking
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "example.com",            "/some_path.png",  BLOCK},
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "foo.example.com",        "/some_path.png",  ALLOW},
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "bar.example.com",        "/some_path.png",  ALLOW},
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "baz.example.com",         "/some_path.png", BLOCK},
-
-            // test script blocking
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "example.com",            "/some_path.js",   BLOCK},
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "foo.example.com",        "/some_path.js",   ALLOW},
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "bar.example.com",        "/some_path.js",   ALLOW},
-            {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
-                                      "baz.example.com",         "/some_path.js",  BLOCK},
-
-            // test stylesheet blocking
-            {"||example.com^stylesheet,domain=~foo.example.com|~bar.example.com",
-                    "example.com",            "/some_path.css",  BLOCK},
-            {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
-                    "foo.example.com",        "/some_path.css",  ALLOW},
-            {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
-                    "bar.example.com",        "/some_path.css",  ALLOW},
-            {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
-                    "baz.example.com",         "/some_path.css", BLOCK},
-
-
-            // path matching
-            {"/foo",                  "example.com",          "/some_path",      ALLOW},
-            {"/foo",                  "example.com",          "/foo",            BLOCK},
-            {"/foo",                  "example.com",          "/foo/bar",        BLOCK},
-
-            // path matching with wildcard
-            {"/foo/*/img",            "example.com",          "/some_path",      ALLOW},
-            {"/foo/*/img",            "example.com",          "/foo",            ALLOW},
-            {"/foo/*/img",            "example.com",          "/foo/img",        ALLOW},
-            {"/foo/*/img",            "example.com",          "/foo/x/img",      BLOCK},
-            {"/foo/*/img",            "example.com",          "/foo/x/img.png",  ALLOW},
-            {"/foo/*/img",            "example.com",          "/foo/x/y/z//img", BLOCK},
-
-            // path matching with regex
-            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/x/y/z//img",      ALLOW},
-            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/apps/img.png",    BLOCK},
-            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/ads/img.png",     BLOCK},
-            {"/foo/(apps|ads)/img.+/",  "example.com",          "/foo/bar/ads/img.png", ALLOW},
-
-            {"/(apps|ads)\\.example\\.(com|org)/",
-                                        "example.com",          "/ad.png",              ALLOW},
-            {"/(apps|ads)\\.example\\.(com|org)/",
-                                        "ads.example.com",      "/ad.png",              BLOCK},
-            {"/(apps|ads)\\.example\\.(com|org)/",
-                                        "apps.example.com",     "/ad.png",              BLOCK},
-            {"/(apps|ads)\\.example\\.(com|org)/",
-                                        "ads.example.org",      "/ad.png",              BLOCK},
-            {"/(apps|ads)\\.example\\.(com|org)/",
-                                        "apps.example.org",     "/ad.png",              BLOCK},
-            {"/(apps|ads)\\.example\\.(com|org)/",
-                                        "ads.example.net",      "/ad.png",              ALLOW},
-            {"/(apps|ads)\\.example\\.(com|org)/",
-                                        "apps.example.net",     "/ad.png",              ALLOW},
-
-            // selectors
-            {"example.com##.banner-ad", "example.com",          "/ad.png",              FILTER},
-
-            // putting it all together
-            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
-                    "example.com",          "/some_path",      FILTER},
-            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
-                    "baz.example.com",      "/some_path",      FILTER},
-            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
-                    "foo.example.com",      "/some_path",      ALLOW},
-            {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
-                    "bar.example.com",      "/some_path",      ALLOW},
-    };
-
-    @Test public void testRules () throws Exception {
-        for (String[] test : BLOCK_TESTS) {
-            final BlockDecisionType expectedDecision = BlockDecisionType.fromString(test[3]);
-            final BlockList blockList = new BlockList();
-            blockList.addToBlacklist(BlockSpec.parse(test[0]));
-            assertEquals("testBlanketBlock: expected "+expectedDecision+" decision, test=" + Arrays.toString(test),
-                    expectedDecision,
-                    blockList.getDecision(test[1], test[2]).getDecisionType());
-        }
-    }
-
-    public String[] SELECTOR_SPECS = {
-            "||example.com##.banner-ad",
-            "||foo.example.com##.more-ads",
-    };
-
-    @Test public void testMultipleSelectorMatches () throws Exception {
-        final BlockList blockList = new BlockList();
-        for (String line : SELECTOR_SPECS) {
-            blockList.addToBlacklist(BlockSpec.parse(line));
-        }
-        BlockDecision decision;
-
-        decision = blockList.getDecision("example.com", "/some_path");
-        assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType());
-        assertEquals("expected 1 filter specs", 1, decision.getSpecs().size());
-
-        decision = blockList.getDecision("foo.example.com", "/some_path");
-        assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType());
-        assertEquals("expected 2 filter specs", 2, decision.getSpecs().size());
-    }
-}

utils/abp-parser

@@ -0,0 +1 @@
+Subproject commit 801d5bbf6f4327a407f8af8e7d7788302f348639

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 44730182976085dd171cf7035fae5953a8841368
+Subproject commit cf3d35b3f6dbb2bd3a1b31abb2c1ded41aa15378