diff --git a/.gitmodules b/.gitmodules
index 68bec75c..97b67e71 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -16,3 +16,6 @@
[submodule "bubble-web"]
path = bubble-web
url = git@git.bubblev.org:bubblev/bubble-web.git
+[submodule "utils/abp-parser"]
+ path = utils/abp-parser
+ url = git@git.bubblev.org:bubblev/abp-parser.git
diff --git a/bubble-server/pom.xml b/bubble-server/pom.xml
index c0733023..2cb1fa3c 100644
--- a/bubble-server/pom.xml
+++ b/bubble-server/pom.xml
@@ -41,6 +41,12 @@ For commercial use, please contact jonathan@kyuss.org
+
+ bubble
+ abp-parser
+ 1.0.0-SNAPSHOT
+
+
org.hibernate
diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java
index d6853b9f..3dcdca23 100644
--- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java
+++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java
@@ -7,6 +7,7 @@ import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.apache.commons.lang.ArrayUtils;
+import org.cobbzilla.util.collection.NameAndValue;
import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
@@ -16,7 +17,7 @@ public class FilterHttpRequest {
@Getter @Setter private String id;
@Getter @Setter private Device device;
@Getter @Setter private Account account;
- @Getter @Setter private String[] filters;
+ @Getter @Setter private NameAndValue[] meta;
@Getter @Setter private String contentType;
@Getter @Setter private String[] matchers;
diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java
index e95c05e5..c07a9d76 100644
--- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java
+++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java
@@ -15,6 +15,7 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.cobbzilla.util.collection.ArrayUtil;
import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.NameAndValue;
import org.cobbzilla.util.http.HttpContentEncodingType;
import org.cobbzilla.util.http.HttpStatusCodes;
import org.cobbzilla.wizard.cache.redis.RedisService;
@@ -87,7 +88,7 @@ public class FilterHttpResource {
private Map matchersCache = new ExpirationMap<>(MINUTES.toMillis(5));
private static final long REQUEST_FILTERS_TIMEOUT = MINUTES.toSeconds(1);
- @Getter(lazy=true) private final RedisService filtersForRequest = redis.prefixNamespace(getClass().getSimpleName()+".filters");
+ @Getter(lazy=true) private final RedisService requestMeta = redis.prefixNamespace(getClass().getSimpleName()+".filters");
@POST @Path(EP_MATCHERS)
@Consumes(APPLICATION_JSON)
@@ -105,8 +106,8 @@ public class FilterHttpResource {
final String cacheKey = remoteHost+":"+filterRequest.cacheKey();
final FilterMatchersResponse response = matchersCache.computeIfAbsent(cacheKey, k -> findMatchers(filterRequest, req, request));
- if (response.hasFilters()) {
- getFiltersForRequest().set(filterRequest.getRequestId(), json(response.getFilters()), EX, REQUEST_FILTERS_TIMEOUT);
+ if (response.hasMeta()) {
+ getRequestMeta().set(filterRequest.getRequestId(), json(response.getMeta()), EX, REQUEST_FILTERS_TIMEOUT);
}
return ok(response);
@@ -138,7 +139,7 @@ public class FilterHttpResource {
final List matchers = matcherDAO.findByAccountAndFqdnAndEnabled(accountUuid, fqdn);
if (log.isDebugEnabled()) log.debug("findMatchers: found "+matchers.size()+" candidate matchers");
final List removeMatchers;
- List filters = null;
+ NameAndValue[] meta = null;
if (matchers.isEmpty()) {
removeMatchers = Collections.emptyList();
} else {
@@ -154,9 +155,9 @@ public class FilterHttpResource {
removeMatchers.add(matcher);
break;
case match:
- if (matchResponse.hasFilters()) {
- if (filters == null) filters = new ArrayList<>();
- filters.addAll(matchResponse.getFilters());
+ if (matchResponse.hasMeta()) {
+ if (meta == null) meta = NameAndValue.EMPTY_ARRAY;
+ meta = ArrayUtil.concat(meta, matchResponse.getMeta());
}
break;
}
@@ -170,7 +171,7 @@ public class FilterHttpResource {
return response
.setMatchers(matchers)
.setDevice(device.getUuid())
- .setFilters(filters);
+ .setMeta(meta);
}
@POST @Path(EP_APPLY+"/{requestId}")
@@ -265,9 +266,9 @@ public class FilterHttpResource {
// check for filters
try {
- final String filtersJson = getFiltersForRequest().get(requestId);
+ final String filtersJson = getRequestMeta().get(requestId);
if (filtersJson != null) {
- filterRequest.setFilters(json(filtersJson, String[].class));
+ filterRequest.setMeta(json(filtersJson, NameAndValue[].class));
}
} catch (Exception e) {
log.error("filterHttp: error reading pageFilters: "+shortError(e));
diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java
index 6780d4ae..51a043c9 100644
--- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java
+++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java
@@ -5,8 +5,7 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.Accessors;
-
-import java.util.List;
+import org.cobbzilla.util.collection.NameAndValue;
import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
@@ -20,7 +19,7 @@ public class FilterMatchResponse {
@Getter @Setter private FilterMatchDecision decision;
- @Getter @Setter private List filters;
- public boolean hasFilters() { return !empty(filters); }
+ @Getter @Setter private NameAndValue[] meta;
+ public boolean hasMeta() { return !empty(meta); }
}
diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java
index 4b0f56ef..de57549b 100644
--- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java
+++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java
@@ -5,6 +5,7 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.Accessors;
+import org.cobbzilla.util.collection.NameAndValue;
import java.util.List;
@@ -18,7 +19,7 @@ public class FilterMatchersResponse {
@Getter @Setter private Integer abort;
@Getter @Setter private String device;
@Getter @Setter private List matchers;
- @Getter @Setter private List filters;
- public boolean hasFilters () { return !empty(filters); }
+ @Getter @Setter private NameAndValue[] meta;
+ public boolean hasMeta() { return !empty(meta); }
}
diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java
index 321d8f3a..f5fe1c8c 100644
--- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java
+++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java
@@ -9,6 +9,7 @@ import bubble.service.stream.AppRuleHarness;
import bubble.resources.stream.FilterMatchersRequest;
import com.fasterxml.jackson.databind.JsonNode;
import com.github.jknack.handlebars.Handlebars;
+import org.cobbzilla.util.collection.NameAndValue;
import org.cobbzilla.util.handlebars.HandlebarsUtil;
import org.glassfish.grizzly.http.server.Request;
import org.glassfish.jersey.server.ContainerRequest;
@@ -49,12 +50,12 @@ public interface AppRuleDriver {
default InputStream doFilterRequest(InputStream in) { return in; }
- default InputStream filterResponse(String requestId, String contentType, String[] filters, InputStream in) {
- if (hasNext()) return doFilterResponse(requestId, contentType, filters, getNext().filterResponse(requestId, contentType, filters, in));
- return doFilterResponse(requestId, contentType, filters, in);
+ default InputStream filterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
+ if (hasNext()) return doFilterResponse(requestId, contentType, meta, getNext().filterResponse(requestId, contentType, meta, in));
+ return doFilterResponse(requestId, contentType, meta, in);
}
- default InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) { return in; }
+ default InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) { return in; }
default String resolveResource(String res, Map ctx) {
final String resource = locateResource(res);
diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java
index ef8a39af..a8f233d9 100644
--- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java
+++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java
@@ -1,19 +1,22 @@
package bubble.rule.bblock;
+import bubble.abp.spec.BlockDecision;
+import bubble.abp.spec.BlockList;
+import bubble.abp.spec.BlockListSource;
+import bubble.abp.spec.BlockSpec;
import bubble.model.account.Account;
import bubble.model.app.AppMatcher;
import bubble.model.app.AppRule;
import bubble.model.device.Device;
import bubble.resources.stream.FilterMatchResponse;
import bubble.resources.stream.FilterMatchersRequest;
+import bubble.rule.FilterMatchDecision;
import bubble.rule.analytics.TrafficAnalytics;
-import bubble.rule.bblock.spec.BlockDecision;
-import bubble.rule.bblock.spec.BlockList;
-import bubble.rule.bblock.spec.BlockListSource;
import bubble.service.stream.AppRuleHarness;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.input.ReaderInputStream;
+import org.cobbzilla.util.collection.NameAndValue;
import org.cobbzilla.util.handlebars.HandlebarsUtil;
import org.cobbzilla.util.io.regex.RegexFilterReader;
import org.cobbzilla.util.io.regex.RegexReplacementFilter;
@@ -23,12 +26,12 @@ import org.glassfish.jersey.server.ContainerRequest;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
import static org.cobbzilla.util.io.StreamUtil.stream2string;
import static org.cobbzilla.util.json.JsonUtil.json;
import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
@@ -38,6 +41,8 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath;
@Slf4j
public class BubbleBlock extends TrafficAnalytics {
+ private static final String META_BLOCK_FILTERS = "__bubble_block_filters";
+
private BubbleBlockConfig bubbleBlockConfig;
private BlockList blockList = new BlockList();
@@ -84,15 +89,34 @@ public class BubbleBlock extends TrafficAnalytics {
case allow: default:
return FilterMatchResponse.NO_MATCH;
case filter:
- return decision.getFilterMatchResponse();
+ return getFilterMatchResponse(decision);
+ }
+ }
+
+ public FilterMatchResponse getFilterMatchResponse(BlockDecision decision) {
+ switch (decision.getDecisionType()) {
+ case block: return FilterMatchResponse.ABORT_NOT_FOUND;
+ case allow: return FilterMatchResponse.NO_MATCH;
+ case filter:
+ final List specs = decision.getSpecs();
+ if (empty(specs)) {
+ log.warn("getFilterMatchResponse: decision was 'filter' but no specs were found, returning no_match");
+ return FilterMatchResponse.NO_MATCH;
+ } else {
+ return new FilterMatchResponse()
+ .setDecision(FilterMatchDecision.match)
+ .setMeta(new NameAndValue[]{new NameAndValue(META_BLOCK_FILTERS, json(specs))});
+ }
}
+ return die("getFilterMatchResponse: invalid decisionType: "+decision.getDecisionType());
}
- @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) {
+ @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
- if (empty(filters) || !isHtml(contentType)) return in;
+ final String blockSpecJson = NameAndValue.find(meta, META_BLOCK_FILTERS);
+ if (empty(blockSpecJson) || !isHtml(contentType)) return in;
- final String replacement = "";
+ final String replacement = "";
final RegexReplacementFilter filter = new RegexReplacementFilter("", replacement);
final RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in), filter).setMaxMatches(1);
return new ReaderInputStream(reader, UTF8cs);
@@ -100,15 +124,15 @@ public class BubbleBlock extends TrafficAnalytics {
public static final Class BB = BubbleBlock.class;
public static final String BUBBLE_JS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+".js.hbs");
- private static final String CTX_BUBBLE_FILTERS = "BUBBLE_FILTERS";
+ private static final String CTX_BUBBLE_BLOCK_SPEC_JSON = "BUBBLE_BLOCK_SPEC_JSON";
- private String getBubbleJs(String requestId, String[] filters) {
+ private String getBubbleJs(String requestId, String blockSpecJson) {
final Map ctx = new HashMap<>();
ctx.put(CTX_JS_PREFIX, "__bubble_block_"+sha256_hex(requestId)+"_");
ctx.put(CTX_BUBBLE_REQUEST_ID, requestId);
ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase());
ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId));
- ctx.put(CTX_BUBBLE_FILTERS, filters);
+ ctx.put(CTX_BUBBLE_BLOCK_SPEC_JSON, blockSpecJson);
return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx);
}
diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java
deleted file mode 100644
index cd269b24..00000000
--- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import bubble.resources.stream.FilterMatchResponse;
-import bubble.rule.FilterMatchDecision;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-
-@NoArgsConstructor @Accessors(chain=true)
-public class BlockDecision {
-
- public static final BlockDecision BLOCK = new BlockDecision().setDecisionType(BlockDecisionType.block);
- public static final BlockDecision ALLOW = new BlockDecision().setDecisionType(BlockDecisionType.allow);
-
- @Getter @Setter BlockDecisionType decisionType = BlockDecisionType.allow;
- @Getter @Setter List specs;
-
- public BlockDecision add(BlockSpec spec) {
- if (specs == null) specs = new ArrayList<>();
- specs.add(spec);
- if (decisionType != BlockDecisionType.block && (spec.hasTypeMatches() || spec.hasSelector())) {
- decisionType = BlockDecisionType.filter;
- }
- return this;
- }
-
- public FilterMatchResponse getFilterMatchResponse() {
- switch (decisionType) {
- case block: return FilterMatchResponse.ABORT_NOT_FOUND;
- case allow: return FilterMatchResponse.NO_MATCH;
- case filter: return new FilterMatchResponse()
- .setDecision(FilterMatchDecision.match)
- .setFilters(specs == null ? null : getSpecs().stream().map(BlockSpec::getLine).collect(Collectors.toList()));
- }
- return die("getFilterMatchResponse: invalid decisionType: "+decisionType);
- }
-}
diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java
deleted file mode 100644
index d0b7f642..00000000
--- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-
-import static bubble.ApiConstants.enumFromString;
-
-public enum BlockDecisionType {
-
- block, allow, filter;
-
- @JsonCreator public static BlockDecisionType fromString (String v) { return enumFromString(BlockDecisionType.class, v); }
-
-}
diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java
deleted file mode 100644
index 7ea23a36..00000000
--- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.experimental.Accessors;
-import lombok.extern.slf4j.Slf4j;
-
-import java.util.*;
-
-@NoArgsConstructor @Accessors(chain=true) @Slf4j
-public class BlockList {
-
- @Getter private Set blacklist = new HashSet<>();
- @Getter private Set whitelist = new HashSet<>();
-
- public void addToWhitelist(BlockSpec spec) {
- whitelist.add(spec);
- }
-
- public void addToWhitelist(List specs) {
- for (BlockSpec spec : specs) addToWhitelist(spec);
- }
-
- public void addToBlacklist(BlockSpec spec) {
- blacklist.add(spec);
- }
-
- public void addToBlacklist(List specs) {
- for (BlockSpec spec : specs) addToBlacklist(spec);
- }
-
- public void merge(BlockList other) {
- for (BlockSpec allow : other.getWhitelist()) {
- addToWhitelist(allow);
- }
- for (BlockSpec block : other.getBlacklist()) {
- addToBlacklist(block);
- }
- }
-
- public BlockDecision getDecision(String fqdn, String path) {
- for (BlockSpec allow : whitelist) {
- if (allow.matches(fqdn, path)) return BlockDecision.ALLOW;
- }
- final BlockDecision decision = new BlockDecision();
- for (BlockSpec block : blacklist) {
- if (block.matches(fqdn, path)) {
- if (!block.hasSelector()) return BlockDecision.BLOCK;
- decision.add(block);
- }
- }
- return decision;
- }
-
-}
diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java
deleted file mode 100644
index 8610e64c..00000000
--- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.http.HttpUtil;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.*;
-import static org.cobbzilla.util.daemon.ZillaRuntime.now;
-
-@NoArgsConstructor @Accessors(chain=true) @Slf4j
-public class BlockListSource {
-
- @Getter @Setter private String url;
- @Getter @Setter private String format;
-
- @Getter @Setter private Long lastDownloaded;
- public long age () { return lastDownloaded == null ? Long.MAX_VALUE : now() - lastDownloaded; }
-
- @Getter @Setter private BlockList blockList = new BlockList();
-
- public BlockListSource download() throws IOException {
- try (BufferedReader r = new BufferedReader(new InputStreamReader(HttpUtil.get(url)))) {
- String line;
- boolean firstLine = true;
- while ( (line = r.readLine()) != null ) {
- if (empty(line)) continue;
- line = line.trim();
- if (firstLine && line.startsWith("[") && line.endsWith("]")) {
- format = line.substring(1, line.length()-1);
- }
- firstLine = false;
- if (line.startsWith("!")) continue;
- try {
- if (line.startsWith("@@")) {
- blockList.addToWhitelist(BlockSpec.parse(line));
- } else {
- blockList.addToBlacklist(BlockSpec.parse(line));
- }
- } catch (Exception e) {
- log.warn("download("+url+"): error parsing line (skipping due to "+shortError(e)+"): " + line);
- }
- }
- }
- lastDownloaded = now();
- return this;
- }
-
-}
diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java
deleted file mode 100644
index 30b43971..00000000
--- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java
+++ /dev/null
@@ -1,178 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.http.HttpContentTypes;
-import org.cobbzilla.util.string.StringUtil;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.http.HttpContentTypes.contentType;
-
-@Slf4j
-public class BlockSpec {
-
- public static final String OPT_DOMAIN_PREFIX = "domain=";
- public static final String OPT_SCRIPT = "script";
- public static final String OPT_IMAGE = "image";
- public static final String OPT_STYLESHEET = "stylesheet";
-
- @Getter private String line;
- @Getter private BlockSpecTarget target;
-
- @Getter private List domainExclusions;
- @Getter private List typeMatches;
- public boolean hasTypeMatches () { return !empty(typeMatches); }
-
- @Getter private List typeExclusions;
-
- public BlockSpec(String line, BlockSpecTarget target, List options, String selector) {
- this.line = line;
- this.target = target;
- this.selector = selector;
- if (options != null) {
- for (String opt : options) {
- if (opt.startsWith(OPT_DOMAIN_PREFIX)) {
- processDomainOptions(opt.substring(OPT_DOMAIN_PREFIX.length()));
-
- } else if (opt.startsWith("~")) {
- final String type = opt.substring(1);
- if (isTypeOption(type)) {
- if (typeExclusions == null) typeExclusions = new ArrayList<>();
- typeExclusions.add(type);
- } else {
- log.warn("unsupported option (ignoring): " + opt);
- }
-
- } else {
- if (isTypeOption(opt)) {
- if (typeMatches == null) typeMatches = new ArrayList<>();
- typeMatches.add(opt);
- } else {
- log.warn("unsupported option (ignoring): "+opt);
- }
- }
- }
- }
- }
-
- private void processDomainOptions(String option) {
- final String[] parts = option.split("\\|");
- for (String domainOption : parts) {
- if (domainOption.startsWith("~")) {
- if (domainExclusions == null) domainExclusions = new ArrayList<>();
- domainExclusions.add(domainOption.substring(1));
- } else {
- log.warn("ignoring included domain: "+domainOption);
- }
- }
- }
-
- public boolean isTypeOption(String type) {
- return type.equals(OPT_SCRIPT) || type.equals(OPT_IMAGE) || type.equals(OPT_STYLESHEET);
- }
-
- @Getter private String selector;
- public boolean hasSelector() { return !empty(selector); }
-
- public static List parse(String line) {
-
- line = line.trim();
- int optionStartPos = line.indexOf('$');
- int selectorStartPos = line.indexOf("##");
-
- // sanity check that selectorStartPos > optionStartPos -- $ may occur AFTER ## if the selector contains a regex
- if (selectorStartPos != -1 && optionStartPos > selectorStartPos) optionStartPos = -1;
-
- final List targets;
- final List options;
- final String selector;
- if (optionStartPos == -1) {
- if (selectorStartPos == -1) {
- // no options, no selector, entire line is the target
- targets = BlockSpecTarget.parseBareLine(line);
- options = null;
- selector = null;
- } else {
- // no options, but selector present. split into target + selector
- targets = BlockSpecTarget.parse(line.substring(0, selectorStartPos));
- options = null;
- selector = line.substring(selectorStartPos+1);
- }
- } else {
- if (selectorStartPos == -1) {
- // no selector, split into target + options
- targets = BlockSpecTarget.parse(line.substring(0, optionStartPos));
- options = StringUtil.splitAndTrim(line.substring(optionStartPos+1), ",");
- selector = null;
- } else {
- // all 3 elements present
- targets = BlockSpecTarget.parse(line.substring(0, optionStartPos));
- options = StringUtil.splitAndTrim(line.substring(optionStartPos + 1, selectorStartPos), ",");
- selector = line.substring(selectorStartPos+1);
- }
- }
- final List specs = new ArrayList<>();
- for (BlockSpecTarget target : targets) specs.add(new BlockSpec(line, target, options, selector));
- return specs;
- }
-
- public boolean matches(String fqdn, String path) {
- if (target.hasDomainRegex() && target.getDomainPattern().matcher(fqdn).find()) {
- return checkDomainExclusionsAndType(fqdn, contentType(path));
-
- } else if (target.hasRegex()) {
- if (target.getRegexPattern().matcher(path).find()) {
- return checkDomainExclusionsAndType(fqdn, contentType(path));
- }
- final String full = fqdn + path;
- if (target.getRegexPattern().matcher(full).find()) {
- return checkDomainExclusionsAndType(fqdn, contentType(path));
- };
- }
- return false;
- }
-
- public boolean checkDomainExclusionsAndType(String fqdn, String contentType) {
- if (domainExclusions != null) {
- for (String domain : domainExclusions) {
- if (domain.equals(fqdn)) return false;
- }
- }
- if (typeExclusions != null) {
- for (String type : typeExclusions) {
- switch (type) {
- case OPT_SCRIPT:
- if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return false;
- break;
- case OPT_IMAGE:
- if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return false;
- break;
- case OPT_STYLESHEET:
- if (contentType.equals(HttpContentTypes.TEXT_CSS)) return false;
- break;
- }
- }
- }
- if (typeMatches != null) {
- for (String type : typeMatches) {
- switch (type) {
- case OPT_SCRIPT:
- if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return true;
- break;
- case OPT_IMAGE:
- if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return true;
- break;
- case OPT_STYLESHEET:
- if (contentType.equals(HttpContentTypes.TEXT_CSS)) return true;
- break;
- }
- }
- return false;
- }
- return true;
- }
-
-}
diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java
deleted file mode 100644
index 4a99b08e..00000000
--- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.experimental.Accessors;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.StringTokenizer;
-import java.util.regex.Pattern;
-
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.http.HttpSchemes.stripScheme;
-
-@NoArgsConstructor @Accessors(chain=true)
-public class BlockSpecTarget {
-
- @Getter @Setter private String domainRegex;
- public boolean hasDomainRegex() { return !empty(domainRegex); }
- @Getter(lazy=true) private final Pattern domainPattern = hasDomainRegex() ? Pattern.compile(getDomainRegex()) : null;
-
- @Getter @Setter private String regex;
- public boolean hasRegex() { return !empty(regex); }
- @Getter(lazy=true) private final Pattern regexPattern = hasRegex() ? Pattern.compile(getRegex()) : null;
-
- public static List parse(String data) {
- final List targets = new ArrayList<>();
- for (String part : data.split(",")) {
- targets.add(parseTarget(part));
- }
- return targets;
- }
-
- public static List parseBareLine(String data) {
- if (data.contains("|") || data.contains("/") || data.contains("^")) return parse(data);
-
- final List targets = new ArrayList<>();
- for (String part : data.split(",")) {
- targets.add(new BlockSpecTarget().setDomainRegex(matchDomainOrAnySubdomains(part)));
- }
- return targets;
- }
-
- private static BlockSpecTarget parseTarget(String data) {
- String domainRegex = null;
- String regex = null;
- if (data.startsWith("||")) {
- final int caretPos = data.indexOf("^");
- if (caretPos != -1) {
- // domain match
- final String domain = data.substring(2, caretPos);
- domainRegex = matchDomainOrAnySubdomains(domain);
- } else {
- final String domain = data.substring(2);
- domainRegex = matchDomainOrAnySubdomains(domain);
- }
- } else if (data.startsWith("|") && data.endsWith("|")) {
- // exact match
- final String verbatimMatch = stripScheme(data.substring(1, data.length() - 1));
- regex = "^" + Pattern.quote(verbatimMatch) + "$";
-
- } else if (data.startsWith("/")) {
- // path match, possibly regex
- if (data.endsWith("/") && (
- data.contains("|") || data.contains("?")
- || (data.contains("(") && data.contains(")"))
- || (data.contains("{") && data.contains("}")))) {
- regex = data.substring(1, data.length()-1);
-
- } else if (data.contains("*")) {
- regex = parseWildcardMatch(data);
- } else {
- regex = "^" + Pattern.quote(data) + ".*";
- }
-
- } else {
- if (data.contains("*")) {
- regex = parseWildcardMatch(data);
- } else {
- regex = "^" + Pattern.quote(data) + ".*";
- }
- }
- return new BlockSpecTarget().setDomainRegex(domainRegex).setRegex(regex);
- }
-
- private static String parseWildcardMatch(String data) {
- final StringBuilder b = new StringBuilder("^");
- final StringTokenizer st = new StringTokenizer(data, "*", true);
- while (st.hasMoreTokens()) {
- final String token = st.nextToken();
- b.append(token.equals("*") ? ".*?" : token);
- }
- return b.append("$").toString();
- }
-
- private static String matchDomainOrAnySubdomains(String domain) {
- return ".*?"+ Pattern.quote(domain)+"$";
- }
-
-}
diff --git a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java
index 9c3b1753..183fdc05 100644
--- a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java
+++ b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java
@@ -5,6 +5,7 @@ import bubble.rule.AbstractAppRuleDriver;
import lombok.Getter;
import org.apache.commons.io.input.ReaderInputStream;
import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.NameAndValue;
import org.cobbzilla.util.handlebars.HandlebarsUtil;
import org.cobbzilla.util.io.regex.RegexFilterReader;
import org.cobbzilla.util.io.regex.RegexReplacementFilter;
@@ -27,7 +28,7 @@ public class JsUserBlocker extends AbstractAppRuleDriver {
public static final String CTX_APPLY_BLOCKS_JS = "APPLY_BLOCKS_JS";
- @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) {
+ @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
if (!isHtml(contentType)) return in;
final String replacement = "";
diff --git a/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java b/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java
index ea20bfa3..f2ff8d0f 100644
--- a/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java
+++ b/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java
@@ -7,6 +7,7 @@ import com.github.jknack.handlebars.Handlebars;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.input.ReaderInputStream;
+import org.cobbzilla.util.collection.NameAndValue;
import org.cobbzilla.util.io.regex.RegexFilterReader;
import org.cobbzilla.util.io.regex.RegexInsertionFilter;
import org.cobbzilla.util.io.regex.RegexStreamFilter;
@@ -47,7 +48,7 @@ public class UserBlocker extends AbstractAppRuleDriver {
protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); }
- @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) {
+ @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
if (!isHtml(contentType)) return in;
final UserBlockerStreamFilter filter = new UserBlockerStreamFilter(requestId, matcher, rule, configuration.getHttp().getBaseUri());
diff --git a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java
index 0681d559..56a1bc6b 100644
--- a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java
+++ b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java
@@ -28,6 +28,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.cobbzilla.util.collection.ExpirationEvictionPolicy;
import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.NameAndValue;
import org.cobbzilla.util.http.HttpClosingFilterInputStream;
import org.cobbzilla.util.http.HttpContentEncodingType;
import org.cobbzilla.util.http.HttpMethods;
@@ -130,7 +131,7 @@ public class RuleEngineService {
// filter response. when stream is closed, close http client
final Header contentTypeHeader = proxyResponse.getFirstHeader(CONTENT_TYPE);
final String contentType = contentTypeHeader == null ? null : contentTypeHeader.getValue();
- final InputStream responseEntity = firstRule.getDriver().filterResponse(filterRequest.getId(), contentType, filterRequest.getFilters(), new HttpClosingFilterInputStream(httpClient, proxyResponse));
+ final InputStream responseEntity = firstRule.getDriver().filterResponse(filterRequest.getId(), contentType, filterRequest.getMeta(), new HttpClosingFilterInputStream(httpClient, proxyResponse));
// send response
return sendResponse(responseEntity, proxyResponse);
@@ -157,7 +158,7 @@ public class RuleEngineService {
// have we seen this request before?
final ActiveStreamState state = activeProcessors.computeIfAbsent(filterRequest.getId(),
- k -> new ActiveStreamState(k, contentEncoding, contentType, filterRequest.getFilters(),
+ k -> new ActiveStreamState(k, contentEncoding, contentType, filterRequest.getMeta(),
initRules(filterRequest.getAccount(), filterRequest.getDevice(), filterRequest.getMatchers())));
final byte[] chunk = toBytes(request.getEntityStream(), contentLength);
if (last) {
@@ -264,7 +265,7 @@ public class RuleEngineService {
private String requestId;
private HttpContentEncodingType encoding;
private String contentType;
- private String[] filters;
+ private NameAndValue[] meta;
private MultiStream multiStream;
private AppRuleHarness firstRule;
private InputStream output = null;
@@ -274,12 +275,12 @@ public class RuleEngineService {
public ActiveStreamState(String requestId,
HttpContentEncodingType encoding,
String contentType,
- String[] filters,
+ NameAndValue[] meta,
List rules) {
this.requestId = requestId;
this.encoding = encoding;
this.contentType = contentType;
- this.filters = filters;
+ this.meta = meta;
this.firstRule = rules.get(0);
}
@@ -288,7 +289,7 @@ public class RuleEngineService {
totalBytesWritten += chunk.length;
if (multiStream == null) {
multiStream = new MultiStream(new ByteArrayInputStream(chunk));
- output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, filters, inputStream(multiStream)));
+ output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, meta, inputStream(multiStream)));
} else {
multiStream.addStream(new ByteArrayInputStream(chunk));
}
@@ -299,7 +300,7 @@ public class RuleEngineService {
totalBytesWritten += chunk.length;
if (multiStream == null) {
multiStream = new MultiStream(new ByteArrayInputStream(chunk), true);
- output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, filters, inputStream(multiStream)));
+ output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, meta, inputStream(multiStream)));
} else {
multiStream.addLastStream(new ByteArrayInputStream(chunk));
}
diff --git a/bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java b/bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java
deleted file mode 100644
index d1bd1e11..00000000
--- a/bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-package bubble.rule.bblock.spec;
-
-import org.junit.Test;
-
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-
-public class BlockListTest {
-
- public static final String BLOCK = BlockDecisionType.block.name();
- public static final String ALLOW = BlockDecisionType.allow.name();
- public static final String FILTER = BlockDecisionType.filter.name();
-
- public static final String[][] BLOCK_TESTS = {
- // rule // fqdn // path // expected decision
-
- // bare hosts example (ala EasyList)
- {"example.com", "example.com", "/some_path", BLOCK},
- {"example.com", "foo.example.com", "/some_path", BLOCK},
- {"example.com", "example.org", "/some_path", ALLOW},
-
- // block example.com and all subdomains
- {"||example.com^", "example.com", "/some_path", BLOCK},
- {"||example.com^", "foo.example.com", "/some_path", BLOCK},
- {"||example.com^", "example.org", "/some_path", ALLOW},
-
- // block exact string
- {"|example.com/|", "example.com", "/", BLOCK},
- {"|example.com/|", "example.com", "/some_path", ALLOW},
- {"|example.com/|", "foo.example.com", "/some_path", ALLOW},
-
- // block example.com, but not foo.example.com or bar.example.com
- {"||example.com^$domain=~foo.example.com|~bar.example.com",
- "example.com", "/some_path", BLOCK},
- {"||example.com^$domain=~foo.example.com|~bar.example.com",
- "foo.example.com", "/some_path", ALLOW},
- {"||example.com^$domain=~foo.example.com|~bar.example.com",
- "bar.example.com", "/some_path", ALLOW},
- {"||example.com^$domain=~foo.example.com|~bar.example.com",
- "baz.example.com", "/some_path", BLOCK},
-
- // block images and scripts on example.com, but not foo.example.com or bar.example.com
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "example.com", "/some_path", ALLOW},
-
- // test image blocking
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "example.com", "/some_path.png", BLOCK},
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "foo.example.com", "/some_path.png", ALLOW},
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "bar.example.com", "/some_path.png", ALLOW},
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "baz.example.com", "/some_path.png", BLOCK},
-
- // test script blocking
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "example.com", "/some_path.js", BLOCK},
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "foo.example.com", "/some_path.js", ALLOW},
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "bar.example.com", "/some_path.js", ALLOW},
- {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com",
- "baz.example.com", "/some_path.js", BLOCK},
-
- // test stylesheet blocking
- {"||example.com^stylesheet,domain=~foo.example.com|~bar.example.com",
- "example.com", "/some_path.css", BLOCK},
- {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
- "foo.example.com", "/some_path.css", ALLOW},
- {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
- "bar.example.com", "/some_path.css", ALLOW},
- {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com",
- "baz.example.com", "/some_path.css", BLOCK},
-
-
- // path matching
- {"/foo", "example.com", "/some_path", ALLOW},
- {"/foo", "example.com", "/foo", BLOCK},
- {"/foo", "example.com", "/foo/bar", BLOCK},
-
- // path matching with wildcard
- {"/foo/*/img", "example.com", "/some_path", ALLOW},
- {"/foo/*/img", "example.com", "/foo", ALLOW},
- {"/foo/*/img", "example.com", "/foo/img", ALLOW},
- {"/foo/*/img", "example.com", "/foo/x/img", BLOCK},
- {"/foo/*/img", "example.com", "/foo/x/img.png", ALLOW},
- {"/foo/*/img", "example.com", "/foo/x/y/z//img", BLOCK},
-
- // path matching with regex
- {"/foo/(apps|ads)/img.+/", "example.com", "/foo/x/y/z//img", ALLOW},
- {"/foo/(apps|ads)/img.+/", "example.com", "/foo/apps/img.png", BLOCK},
- {"/foo/(apps|ads)/img.+/", "example.com", "/foo/ads/img.png", BLOCK},
- {"/foo/(apps|ads)/img.+/", "example.com", "/foo/bar/ads/img.png", ALLOW},
-
- {"/(apps|ads)\\.example\\.(com|org)/",
- "example.com", "/ad.png", ALLOW},
- {"/(apps|ads)\\.example\\.(com|org)/",
- "ads.example.com", "/ad.png", BLOCK},
- {"/(apps|ads)\\.example\\.(com|org)/",
- "apps.example.com", "/ad.png", BLOCK},
- {"/(apps|ads)\\.example\\.(com|org)/",
- "ads.example.org", "/ad.png", BLOCK},
- {"/(apps|ads)\\.example\\.(com|org)/",
- "apps.example.org", "/ad.png", BLOCK},
- {"/(apps|ads)\\.example\\.(com|org)/",
- "ads.example.net", "/ad.png", ALLOW},
- {"/(apps|ads)\\.example\\.(com|org)/",
- "apps.example.net", "/ad.png", ALLOW},
-
- // selectors
- {"example.com##.banner-ad", "example.com", "/ad.png", FILTER},
-
- // putting it all together
- {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
- "example.com", "/some_path", FILTER},
- {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
- "baz.example.com", "/some_path", FILTER},
- {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
- "foo.example.com", "/some_path", ALLOW},
- {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad",
- "bar.example.com", "/some_path", ALLOW},
- };
-
- @Test public void testRules () throws Exception {
- for (String[] test : BLOCK_TESTS) {
- final BlockDecisionType expectedDecision = BlockDecisionType.fromString(test[3]);
- final BlockList blockList = new BlockList();
- blockList.addToBlacklist(BlockSpec.parse(test[0]));
- assertEquals("testBlanketBlock: expected "+expectedDecision+" decision, test=" + Arrays.toString(test),
- expectedDecision,
- blockList.getDecision(test[1], test[2]).getDecisionType());
- }
- }
-
- public String[] SELECTOR_SPECS = {
- "||example.com##.banner-ad",
- "||foo.example.com##.more-ads",
- };
-
- @Test public void testMultipleSelectorMatches () throws Exception {
- final BlockList blockList = new BlockList();
- for (String line : SELECTOR_SPECS) {
- blockList.addToBlacklist(BlockSpec.parse(line));
- }
- BlockDecision decision;
-
- decision = blockList.getDecision("example.com", "/some_path");
- assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType());
- assertEquals("expected 1 filter specs", 1, decision.getSpecs().size());
-
- decision = blockList.getDecision("foo.example.com", "/some_path");
- assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType());
- assertEquals("expected 2 filter specs", 2, decision.getSpecs().size());
- }
-}
diff --git a/utils/abp-parser b/utils/abp-parser
new file mode 160000
index 00000000..801d5bbf
--- /dev/null
+++ b/utils/abp-parser
@@ -0,0 +1 @@
+Subproject commit 801d5bbf6f4327a407f8af8e7d7788302f348639
diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils
index 44730182..cf3d35b3 160000
--- a/utils/cobbzilla-utils
+++ b/utils/cobbzilla-utils
@@ -1 +1 @@
-Subproject commit 44730182976085dd171cf7035fae5953a8841368
+Subproject commit cf3d35b3f6dbb2bd3a1b31abb2c1ded41aa15378