From 354e3c37c3849aaeb4ad8643c91a1ee225663c0a Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 31 Jan 2020 12:04:49 -0500 Subject: [PATCH] add abp-parser submodule, allow filters to pass arbitrary metadata instead of just filters --- .gitmodules | 3 + bubble-server/pom.xml | 6 + .../resources/stream/FilterHttpRequest.java | 3 +- .../resources/stream/FilterHttpResource.java | 21 ++- .../resources/stream/FilterMatchResponse.java | 7 +- .../stream/FilterMatchersResponse.java | 5 +- .../main/java/bubble/rule/AppRuleDriver.java | 9 +- .../java/bubble/rule/bblock/BubbleBlock.java | 48 +++-- .../rule/bblock/spec/BlockDecision.java | 44 ----- .../rule/bblock/spec/BlockDecisionType.java | 13 -- .../bubble/rule/bblock/spec/BlockList.java | 55 ------ .../rule/bblock/spec/BlockListSource.java | 55 ------ .../bubble/rule/bblock/spec/BlockSpec.java | 178 ------------------ .../rule/bblock/spec/BlockSpecTarget.java | 101 ---------- .../rule/social/block/JsUserBlocker.java | 3 +- .../bubble/rule/social/block/UserBlocker.java | 3 +- .../service/stream/RuleEngineService.java | 15 +- .../rule/bblock/spec/BlockListTest.java | 157 --------------- utils/abp-parser | 1 + utils/cobbzilla-utils | 2 +- 20 files changed, 83 insertions(+), 646 deletions(-) delete mode 100644 bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java delete mode 100644 bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java delete mode 100644 bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java delete mode 100644 bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java delete mode 100644 bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java delete mode 100644 bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java delete mode 100644 bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java create mode 160000 utils/abp-parser diff --git a/.gitmodules b/.gitmodules index 68bec75c..97b67e71 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "bubble-web"] path = bubble-web url = git@git.bubblev.org:bubblev/bubble-web.git +[submodule "utils/abp-parser"] + path = utils/abp-parser + url = git@git.bubblev.org:bubblev/abp-parser.git diff --git a/bubble-server/pom.xml b/bubble-server/pom.xml index c0733023..2cb1fa3c 100644 --- a/bubble-server/pom.xml +++ b/bubble-server/pom.xml @@ -41,6 +41,12 @@ For commercial use, please contact jonathan@kyuss.org + + bubble + abp-parser + 1.0.0-SNAPSHOT + + org.hibernate diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java index d6853b9f..3dcdca23 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java @@ -7,6 +7,7 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; import org.apache.commons.lang.ArrayUtils; +import org.cobbzilla.util.collection.NameAndValue; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @@ -16,7 +17,7 @@ public class FilterHttpRequest { @Getter @Setter private String id; @Getter @Setter private Device device; @Getter @Setter private Account account; - @Getter @Setter private String[] filters; + @Getter @Setter private NameAndValue[] meta; @Getter @Setter private String contentType; @Getter @Setter private String[] matchers; diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index e95c05e5..c07a9d76 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -15,6 +15,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ArrayUtil; import org.cobbzilla.util.collection.ExpirationMap; +import org.cobbzilla.util.collection.NameAndValue; import org.cobbzilla.util.http.HttpContentEncodingType; import org.cobbzilla.util.http.HttpStatusCodes; import org.cobbzilla.wizard.cache.redis.RedisService; @@ -87,7 +88,7 @@ public class FilterHttpResource { private Map matchersCache = new ExpirationMap<>(MINUTES.toMillis(5)); private static final long REQUEST_FILTERS_TIMEOUT = MINUTES.toSeconds(1); - @Getter(lazy=true) private final RedisService filtersForRequest = redis.prefixNamespace(getClass().getSimpleName()+".filters"); + @Getter(lazy=true) private final RedisService requestMeta = redis.prefixNamespace(getClass().getSimpleName()+".filters"); @POST @Path(EP_MATCHERS) @Consumes(APPLICATION_JSON) @@ -105,8 +106,8 @@ public class FilterHttpResource { final String cacheKey = remoteHost+":"+filterRequest.cacheKey(); final FilterMatchersResponse response = matchersCache.computeIfAbsent(cacheKey, k -> findMatchers(filterRequest, req, request)); - if (response.hasFilters()) { - getFiltersForRequest().set(filterRequest.getRequestId(), json(response.getFilters()), EX, REQUEST_FILTERS_TIMEOUT); + if (response.hasMeta()) { + getRequestMeta().set(filterRequest.getRequestId(), json(response.getMeta()), EX, REQUEST_FILTERS_TIMEOUT); } return ok(response); @@ -138,7 +139,7 @@ public class FilterHttpResource { final List matchers = matcherDAO.findByAccountAndFqdnAndEnabled(accountUuid, fqdn); if (log.isDebugEnabled()) log.debug("findMatchers: found "+matchers.size()+" candidate matchers"); final List removeMatchers; - List filters = null; + NameAndValue[] meta = null; if (matchers.isEmpty()) { removeMatchers = Collections.emptyList(); } else { @@ -154,9 +155,9 @@ public class FilterHttpResource { removeMatchers.add(matcher); break; case match: - if (matchResponse.hasFilters()) { - if (filters == null) filters = new ArrayList<>(); - filters.addAll(matchResponse.getFilters()); + if (matchResponse.hasMeta()) { + if (meta == null) meta = NameAndValue.EMPTY_ARRAY; + meta = ArrayUtil.concat(meta, matchResponse.getMeta()); } break; } @@ -170,7 +171,7 @@ public class FilterHttpResource { return response .setMatchers(matchers) .setDevice(device.getUuid()) - .setFilters(filters); + .setMeta(meta); } @POST @Path(EP_APPLY+"/{requestId}") @@ -265,9 +266,9 @@ public class FilterHttpResource { // check for filters try { - final String filtersJson = getFiltersForRequest().get(requestId); + final String filtersJson = getRequestMeta().get(requestId); if (filtersJson != null) { - filterRequest.setFilters(json(filtersJson, String[].class)); + filterRequest.setMeta(json(filtersJson, NameAndValue[].class)); } } catch (Exception e) { log.error("filterHttp: error reading pageFilters: "+shortError(e)); diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java index 6780d4ae..51a043c9 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchResponse.java @@ -5,8 +5,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; - -import java.util.List; +import org.cobbzilla.util.collection.NameAndValue; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @@ -20,7 +19,7 @@ public class FilterMatchResponse { @Getter @Setter private FilterMatchDecision decision; - @Getter @Setter private List filters; - public boolean hasFilters() { return !empty(filters); } + @Getter @Setter private NameAndValue[] meta; + public boolean hasMeta() { return !empty(meta); } } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java index 4b0f56ef..de57549b 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java @@ -5,6 +5,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; +import org.cobbzilla.util.collection.NameAndValue; import java.util.List; @@ -18,7 +19,7 @@ public class FilterMatchersResponse { @Getter @Setter private Integer abort; @Getter @Setter private String device; @Getter @Setter private List matchers; - @Getter @Setter private List filters; - public boolean hasFilters () { return !empty(filters); } + @Getter @Setter private NameAndValue[] meta; + public boolean hasMeta() { return !empty(meta); } } diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 321d8f3a..f5fe1c8c 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -9,6 +9,7 @@ import bubble.service.stream.AppRuleHarness; import bubble.resources.stream.FilterMatchersRequest; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; +import org.cobbzilla.util.collection.NameAndValue; import org.cobbzilla.util.handlebars.HandlebarsUtil; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -49,12 +50,12 @@ public interface AppRuleDriver { default InputStream doFilterRequest(InputStream in) { return in; } - default InputStream filterResponse(String requestId, String contentType, String[] filters, InputStream in) { - if (hasNext()) return doFilterResponse(requestId, contentType, filters, getNext().filterResponse(requestId, contentType, filters, in)); - return doFilterResponse(requestId, contentType, filters, in); + default InputStream filterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) { + if (hasNext()) return doFilterResponse(requestId, contentType, meta, getNext().filterResponse(requestId, contentType, meta, in)); + return doFilterResponse(requestId, contentType, meta, in); } - default InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) { return in; } + default InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) { return in; } default String resolveResource(String res, Map ctx) { final String resource = locateResource(res); diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java index ef8a39af..a8f233d9 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java @@ -1,19 +1,22 @@ package bubble.rule.bblock; +import bubble.abp.spec.BlockDecision; +import bubble.abp.spec.BlockList; +import bubble.abp.spec.BlockListSource; +import bubble.abp.spec.BlockSpec; import bubble.model.account.Account; import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; import bubble.model.device.Device; import bubble.resources.stream.FilterMatchResponse; import bubble.resources.stream.FilterMatchersRequest; +import bubble.rule.FilterMatchDecision; import bubble.rule.analytics.TrafficAnalytics; -import bubble.rule.bblock.spec.BlockDecision; -import bubble.rule.bblock.spec.BlockList; -import bubble.rule.bblock.spec.BlockListSource; import bubble.service.stream.AppRuleHarness; import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.input.ReaderInputStream; +import org.cobbzilla.util.collection.NameAndValue; import org.cobbzilla.util.handlebars.HandlebarsUtil; import org.cobbzilla.util.io.regex.RegexFilterReader; import org.cobbzilla.util.io.regex.RegexReplacementFilter; @@ -23,12 +26,12 @@ import org.glassfish.jersey.server.ContainerRequest; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.security.ShaUtil.sha256_hex; @@ -38,6 +41,8 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath; @Slf4j public class BubbleBlock extends TrafficAnalytics { + private static final String META_BLOCK_FILTERS = "__bubble_block_filters"; + private BubbleBlockConfig bubbleBlockConfig; private BlockList blockList = new BlockList(); @@ -84,15 +89,34 @@ public class BubbleBlock extends TrafficAnalytics { case allow: default: return FilterMatchResponse.NO_MATCH; case filter: - return decision.getFilterMatchResponse(); + return getFilterMatchResponse(decision); + } + } + + public FilterMatchResponse getFilterMatchResponse(BlockDecision decision) { + switch (decision.getDecisionType()) { + case block: return FilterMatchResponse.ABORT_NOT_FOUND; + case allow: return FilterMatchResponse.NO_MATCH; + case filter: + final List specs = decision.getSpecs(); + if (empty(specs)) { + log.warn("getFilterMatchResponse: decision was 'filter' but no specs were found, returning no_match"); + return FilterMatchResponse.NO_MATCH; + } else { + return new FilterMatchResponse() + .setDecision(FilterMatchDecision.match) + .setMeta(new NameAndValue[]{new NameAndValue(META_BLOCK_FILTERS, json(specs))}); + } } + return die("getFilterMatchResponse: invalid decisionType: "+decision.getDecisionType()); } - @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) { + @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) { - if (empty(filters) || !isHtml(contentType)) return in; + final String blockSpecJson = NameAndValue.find(meta, META_BLOCK_FILTERS); + if (empty(blockSpecJson) || !isHtml(contentType)) return in; - final String replacement = ""; + final String replacement = ""; final RegexReplacementFilter filter = new RegexReplacementFilter("", replacement); final RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in), filter).setMaxMatches(1); return new ReaderInputStream(reader, UTF8cs); @@ -100,15 +124,15 @@ public class BubbleBlock extends TrafficAnalytics { public static final Class BB = BubbleBlock.class; public static final String BUBBLE_JS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+".js.hbs"); - private static final String CTX_BUBBLE_FILTERS = "BUBBLE_FILTERS"; + private static final String CTX_BUBBLE_BLOCK_SPEC_JSON = "BUBBLE_BLOCK_SPEC_JSON"; - private String getBubbleJs(String requestId, String[] filters) { + private String getBubbleJs(String requestId, String blockSpecJson) { final Map ctx = new HashMap<>(); ctx.put(CTX_JS_PREFIX, "__bubble_block_"+sha256_hex(requestId)+"_"); ctx.put(CTX_BUBBLE_REQUEST_ID, requestId); ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase()); ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId)); - ctx.put(CTX_BUBBLE_FILTERS, filters); + ctx.put(CTX_BUBBLE_BLOCK_SPEC_JSON, blockSpecJson); return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx); } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java deleted file mode 100644 index cd269b24..00000000 --- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecision.java +++ /dev/null @@ -1,44 +0,0 @@ -package bubble.rule.bblock.spec; - -import bubble.resources.stream.FilterMatchResponse; -import bubble.rule.FilterMatchDecision; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.Accessors; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import static org.cobbzilla.util.daemon.ZillaRuntime.die; - -@NoArgsConstructor @Accessors(chain=true) -public class BlockDecision { - - public static final BlockDecision BLOCK = new BlockDecision().setDecisionType(BlockDecisionType.block); - public static final BlockDecision ALLOW = new BlockDecision().setDecisionType(BlockDecisionType.allow); - - @Getter @Setter BlockDecisionType decisionType = BlockDecisionType.allow; - @Getter @Setter List specs; - - public BlockDecision add(BlockSpec spec) { - if (specs == null) specs = new ArrayList<>(); - specs.add(spec); - if (decisionType != BlockDecisionType.block && (spec.hasTypeMatches() || spec.hasSelector())) { - decisionType = BlockDecisionType.filter; - } - return this; - } - - public FilterMatchResponse getFilterMatchResponse() { - switch (decisionType) { - case block: return FilterMatchResponse.ABORT_NOT_FOUND; - case allow: return FilterMatchResponse.NO_MATCH; - case filter: return new FilterMatchResponse() - .setDecision(FilterMatchDecision.match) - .setFilters(specs == null ? null : getSpecs().stream().map(BlockSpec::getLine).collect(Collectors.toList())); - } - return die("getFilterMatchResponse: invalid decisionType: "+decisionType); - } -} diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java deleted file mode 100644 index d0b7f642..00000000 --- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockDecisionType.java +++ /dev/null @@ -1,13 +0,0 @@ -package bubble.rule.bblock.spec; - -import com.fasterxml.jackson.annotation.JsonCreator; - -import static bubble.ApiConstants.enumFromString; - -public enum BlockDecisionType { - - block, allow, filter; - - @JsonCreator public static BlockDecisionType fromString (String v) { return enumFromString(BlockDecisionType.class, v); } - -} diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java deleted file mode 100644 index 7ea23a36..00000000 --- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockList.java +++ /dev/null @@ -1,55 +0,0 @@ -package bubble.rule.bblock.spec; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.experimental.Accessors; -import lombok.extern.slf4j.Slf4j; - -import java.util.*; - -@NoArgsConstructor @Accessors(chain=true) @Slf4j -public class BlockList { - - @Getter private Set blacklist = new HashSet<>(); - @Getter private Set whitelist = new HashSet<>(); - - public void addToWhitelist(BlockSpec spec) { - whitelist.add(spec); - } - - public void addToWhitelist(List specs) { - for (BlockSpec spec : specs) addToWhitelist(spec); - } - - public void addToBlacklist(BlockSpec spec) { - blacklist.add(spec); - } - - public void addToBlacklist(List specs) { - for (BlockSpec spec : specs) addToBlacklist(spec); - } - - public void merge(BlockList other) { - for (BlockSpec allow : other.getWhitelist()) { - addToWhitelist(allow); - } - for (BlockSpec block : other.getBlacklist()) { - addToBlacklist(block); - } - } - - public BlockDecision getDecision(String fqdn, String path) { - for (BlockSpec allow : whitelist) { - if (allow.matches(fqdn, path)) return BlockDecision.ALLOW; - } - final BlockDecision decision = new BlockDecision(); - for (BlockSpec block : blacklist) { - if (block.matches(fqdn, path)) { - if (!block.hasSelector()) return BlockDecision.BLOCK; - decision.add(block); - } - } - return decision; - } - -} diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java deleted file mode 100644 index 8610e64c..00000000 --- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockListSource.java +++ /dev/null @@ -1,55 +0,0 @@ -package bubble.rule.bblock.spec; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.Accessors; -import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.http.HttpUtil; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; - -import static org.cobbzilla.util.daemon.ZillaRuntime.*; -import static org.cobbzilla.util.daemon.ZillaRuntime.now; - -@NoArgsConstructor @Accessors(chain=true) @Slf4j -public class BlockListSource { - - @Getter @Setter private String url; - @Getter @Setter private String format; - - @Getter @Setter private Long lastDownloaded; - public long age () { return lastDownloaded == null ? Long.MAX_VALUE : now() - lastDownloaded; } - - @Getter @Setter private BlockList blockList = new BlockList(); - - public BlockListSource download() throws IOException { - try (BufferedReader r = new BufferedReader(new InputStreamReader(HttpUtil.get(url)))) { - String line; - boolean firstLine = true; - while ( (line = r.readLine()) != null ) { - if (empty(line)) continue; - line = line.trim(); - if (firstLine && line.startsWith("[") && line.endsWith("]")) { - format = line.substring(1, line.length()-1); - } - firstLine = false; - if (line.startsWith("!")) continue; - try { - if (line.startsWith("@@")) { - blockList.addToWhitelist(BlockSpec.parse(line)); - } else { - blockList.addToBlacklist(BlockSpec.parse(line)); - } - } catch (Exception e) { - log.warn("download("+url+"): error parsing line (skipping due to "+shortError(e)+"): " + line); - } - } - } - lastDownloaded = now(); - return this; - } - -} diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java deleted file mode 100644 index 30b43971..00000000 --- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpec.java +++ /dev/null @@ -1,178 +0,0 @@ -package bubble.rule.bblock.spec; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.http.HttpContentTypes; -import org.cobbzilla.util.string.StringUtil; - -import java.util.ArrayList; -import java.util.List; - -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.http.HttpContentTypes.contentType; - -@Slf4j -public class BlockSpec { - - public static final String OPT_DOMAIN_PREFIX = "domain="; - public static final String OPT_SCRIPT = "script"; - public static final String OPT_IMAGE = "image"; - public static final String OPT_STYLESHEET = "stylesheet"; - - @Getter private String line; - @Getter private BlockSpecTarget target; - - @Getter private List domainExclusions; - @Getter private List typeMatches; - public boolean hasTypeMatches () { return !empty(typeMatches); } - - @Getter private List typeExclusions; - - public BlockSpec(String line, BlockSpecTarget target, List options, String selector) { - this.line = line; - this.target = target; - this.selector = selector; - if (options != null) { - for (String opt : options) { - if (opt.startsWith(OPT_DOMAIN_PREFIX)) { - processDomainOptions(opt.substring(OPT_DOMAIN_PREFIX.length())); - - } else if (opt.startsWith("~")) { - final String type = opt.substring(1); - if (isTypeOption(type)) { - if (typeExclusions == null) typeExclusions = new ArrayList<>(); - typeExclusions.add(type); - } else { - log.warn("unsupported option (ignoring): " + opt); - } - - } else { - if (isTypeOption(opt)) { - if (typeMatches == null) typeMatches = new ArrayList<>(); - typeMatches.add(opt); - } else { - log.warn("unsupported option (ignoring): "+opt); - } - } - } - } - } - - private void processDomainOptions(String option) { - final String[] parts = option.split("\\|"); - for (String domainOption : parts) { - if (domainOption.startsWith("~")) { - if (domainExclusions == null) domainExclusions = new ArrayList<>(); - domainExclusions.add(domainOption.substring(1)); - } else { - log.warn("ignoring included domain: "+domainOption); - } - } - } - - public boolean isTypeOption(String type) { - return type.equals(OPT_SCRIPT) || type.equals(OPT_IMAGE) || type.equals(OPT_STYLESHEET); - } - - @Getter private String selector; - public boolean hasSelector() { return !empty(selector); } - - public static List parse(String line) { - - line = line.trim(); - int optionStartPos = line.indexOf('$'); - int selectorStartPos = line.indexOf("##"); - - // sanity check that selectorStartPos > optionStartPos -- $ may occur AFTER ## if the selector contains a regex - if (selectorStartPos != -1 && optionStartPos > selectorStartPos) optionStartPos = -1; - - final List targets; - final List options; - final String selector; - if (optionStartPos == -1) { - if (selectorStartPos == -1) { - // no options, no selector, entire line is the target - targets = BlockSpecTarget.parseBareLine(line); - options = null; - selector = null; - } else { - // no options, but selector present. split into target + selector - targets = BlockSpecTarget.parse(line.substring(0, selectorStartPos)); - options = null; - selector = line.substring(selectorStartPos+1); - } - } else { - if (selectorStartPos == -1) { - // no selector, split into target + options - targets = BlockSpecTarget.parse(line.substring(0, optionStartPos)); - options = StringUtil.splitAndTrim(line.substring(optionStartPos+1), ","); - selector = null; - } else { - // all 3 elements present - targets = BlockSpecTarget.parse(line.substring(0, optionStartPos)); - options = StringUtil.splitAndTrim(line.substring(optionStartPos + 1, selectorStartPos), ","); - selector = line.substring(selectorStartPos+1); - } - } - final List specs = new ArrayList<>(); - for (BlockSpecTarget target : targets) specs.add(new BlockSpec(line, target, options, selector)); - return specs; - } - - public boolean matches(String fqdn, String path) { - if (target.hasDomainRegex() && target.getDomainPattern().matcher(fqdn).find()) { - return checkDomainExclusionsAndType(fqdn, contentType(path)); - - } else if (target.hasRegex()) { - if (target.getRegexPattern().matcher(path).find()) { - return checkDomainExclusionsAndType(fqdn, contentType(path)); - } - final String full = fqdn + path; - if (target.getRegexPattern().matcher(full).find()) { - return checkDomainExclusionsAndType(fqdn, contentType(path)); - }; - } - return false; - } - - public boolean checkDomainExclusionsAndType(String fqdn, String contentType) { - if (domainExclusions != null) { - for (String domain : domainExclusions) { - if (domain.equals(fqdn)) return false; - } - } - if (typeExclusions != null) { - for (String type : typeExclusions) { - switch (type) { - case OPT_SCRIPT: - if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return false; - break; - case OPT_IMAGE: - if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return false; - break; - case OPT_STYLESHEET: - if (contentType.equals(HttpContentTypes.TEXT_CSS)) return false; - break; - } - } - } - if (typeMatches != null) { - for (String type : typeMatches) { - switch (type) { - case OPT_SCRIPT: - if (contentType.equals(HttpContentTypes.APPLICATION_JAVASCRIPT)) return true; - break; - case OPT_IMAGE: - if (contentType.startsWith(HttpContentTypes.IMAGE_PREFIX)) return true; - break; - case OPT_STYLESHEET: - if (contentType.equals(HttpContentTypes.TEXT_CSS)) return true; - break; - } - } - return false; - } - return true; - } - -} diff --git a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java b/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java deleted file mode 100644 index 4a99b08e..00000000 --- a/bubble-server/src/main/java/bubble/rule/bblock/spec/BlockSpecTarget.java +++ /dev/null @@ -1,101 +0,0 @@ -package bubble.rule.bblock.spec; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.Accessors; - -import java.util.ArrayList; -import java.util.List; -import java.util.StringTokenizer; -import java.util.regex.Pattern; - -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.http.HttpSchemes.stripScheme; - -@NoArgsConstructor @Accessors(chain=true) -public class BlockSpecTarget { - - @Getter @Setter private String domainRegex; - public boolean hasDomainRegex() { return !empty(domainRegex); } - @Getter(lazy=true) private final Pattern domainPattern = hasDomainRegex() ? Pattern.compile(getDomainRegex()) : null; - - @Getter @Setter private String regex; - public boolean hasRegex() { return !empty(regex); } - @Getter(lazy=true) private final Pattern regexPattern = hasRegex() ? Pattern.compile(getRegex()) : null; - - public static List parse(String data) { - final List targets = new ArrayList<>(); - for (String part : data.split(",")) { - targets.add(parseTarget(part)); - } - return targets; - } - - public static List parseBareLine(String data) { - if (data.contains("|") || data.contains("/") || data.contains("^")) return parse(data); - - final List targets = new ArrayList<>(); - for (String part : data.split(",")) { - targets.add(new BlockSpecTarget().setDomainRegex(matchDomainOrAnySubdomains(part))); - } - return targets; - } - - private static BlockSpecTarget parseTarget(String data) { - String domainRegex = null; - String regex = null; - if (data.startsWith("||")) { - final int caretPos = data.indexOf("^"); - if (caretPos != -1) { - // domain match - final String domain = data.substring(2, caretPos); - domainRegex = matchDomainOrAnySubdomains(domain); - } else { - final String domain = data.substring(2); - domainRegex = matchDomainOrAnySubdomains(domain); - } - } else if (data.startsWith("|") && data.endsWith("|")) { - // exact match - final String verbatimMatch = stripScheme(data.substring(1, data.length() - 1)); - regex = "^" + Pattern.quote(verbatimMatch) + "$"; - - } else if (data.startsWith("/")) { - // path match, possibly regex - if (data.endsWith("/") && ( - data.contains("|") || data.contains("?") - || (data.contains("(") && data.contains(")")) - || (data.contains("{") && data.contains("}")))) { - regex = data.substring(1, data.length()-1); - - } else if (data.contains("*")) { - regex = parseWildcardMatch(data); - } else { - regex = "^" + Pattern.quote(data) + ".*"; - } - - } else { - if (data.contains("*")) { - regex = parseWildcardMatch(data); - } else { - regex = "^" + Pattern.quote(data) + ".*"; - } - } - return new BlockSpecTarget().setDomainRegex(domainRegex).setRegex(regex); - } - - private static String parseWildcardMatch(String data) { - final StringBuilder b = new StringBuilder("^"); - final StringTokenizer st = new StringTokenizer(data, "*", true); - while (st.hasMoreTokens()) { - final String token = st.nextToken(); - b.append(token.equals("*") ? ".*?" : token); - } - return b.append("$").toString(); - } - - private static String matchDomainOrAnySubdomains(String domain) { - return ".*?"+ Pattern.quote(domain)+"$"; - } - -} diff --git a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java index 9c3b1753..183fdc05 100644 --- a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java +++ b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlocker.java @@ -5,6 +5,7 @@ import bubble.rule.AbstractAppRuleDriver; import lombok.Getter; import org.apache.commons.io.input.ReaderInputStream; import org.cobbzilla.util.collection.ExpirationMap; +import org.cobbzilla.util.collection.NameAndValue; import org.cobbzilla.util.handlebars.HandlebarsUtil; import org.cobbzilla.util.io.regex.RegexFilterReader; import org.cobbzilla.util.io.regex.RegexReplacementFilter; @@ -27,7 +28,7 @@ public class JsUserBlocker extends AbstractAppRuleDriver { public static final String CTX_APPLY_BLOCKS_JS = "APPLY_BLOCKS_JS"; - @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) { + @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) { if (!isHtml(contentType)) return in; final String replacement = ""; diff --git a/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java b/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java index ea20bfa3..f2ff8d0f 100644 --- a/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java +++ b/bubble-server/src/main/java/bubble/rule/social/block/UserBlocker.java @@ -7,6 +7,7 @@ import com.github.jknack.handlebars.Handlebars; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.input.ReaderInputStream; +import org.cobbzilla.util.collection.NameAndValue; import org.cobbzilla.util.io.regex.RegexFilterReader; import org.cobbzilla.util.io.regex.RegexInsertionFilter; import org.cobbzilla.util.io.regex.RegexStreamFilter; @@ -47,7 +48,7 @@ public class UserBlocker extends AbstractAppRuleDriver { protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); } - @Override public InputStream doFilterResponse(String requestId, String contentType, String[] filters, InputStream in) { + @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) { if (!isHtml(contentType)) return in; final UserBlockerStreamFilter filter = new UserBlockerStreamFilter(requestId, matcher, rule, configuration.getHttp().getBaseUri()); diff --git a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java index 0681d559..56a1bc6b 100644 --- a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java @@ -28,6 +28,7 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.cobbzilla.util.collection.ExpirationEvictionPolicy; import org.cobbzilla.util.collection.ExpirationMap; +import org.cobbzilla.util.collection.NameAndValue; import org.cobbzilla.util.http.HttpClosingFilterInputStream; import org.cobbzilla.util.http.HttpContentEncodingType; import org.cobbzilla.util.http.HttpMethods; @@ -130,7 +131,7 @@ public class RuleEngineService { // filter response. when stream is closed, close http client final Header contentTypeHeader = proxyResponse.getFirstHeader(CONTENT_TYPE); final String contentType = contentTypeHeader == null ? null : contentTypeHeader.getValue(); - final InputStream responseEntity = firstRule.getDriver().filterResponse(filterRequest.getId(), contentType, filterRequest.getFilters(), new HttpClosingFilterInputStream(httpClient, proxyResponse)); + final InputStream responseEntity = firstRule.getDriver().filterResponse(filterRequest.getId(), contentType, filterRequest.getMeta(), new HttpClosingFilterInputStream(httpClient, proxyResponse)); // send response return sendResponse(responseEntity, proxyResponse); @@ -157,7 +158,7 @@ public class RuleEngineService { // have we seen this request before? final ActiveStreamState state = activeProcessors.computeIfAbsent(filterRequest.getId(), - k -> new ActiveStreamState(k, contentEncoding, contentType, filterRequest.getFilters(), + k -> new ActiveStreamState(k, contentEncoding, contentType, filterRequest.getMeta(), initRules(filterRequest.getAccount(), filterRequest.getDevice(), filterRequest.getMatchers()))); final byte[] chunk = toBytes(request.getEntityStream(), contentLength); if (last) { @@ -264,7 +265,7 @@ public class RuleEngineService { private String requestId; private HttpContentEncodingType encoding; private String contentType; - private String[] filters; + private NameAndValue[] meta; private MultiStream multiStream; private AppRuleHarness firstRule; private InputStream output = null; @@ -274,12 +275,12 @@ public class RuleEngineService { public ActiveStreamState(String requestId, HttpContentEncodingType encoding, String contentType, - String[] filters, + NameAndValue[] meta, List rules) { this.requestId = requestId; this.encoding = encoding; this.contentType = contentType; - this.filters = filters; + this.meta = meta; this.firstRule = rules.get(0); } @@ -288,7 +289,7 @@ public class RuleEngineService { totalBytesWritten += chunk.length; if (multiStream == null) { multiStream = new MultiStream(new ByteArrayInputStream(chunk)); - output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, filters, inputStream(multiStream))); + output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, meta, inputStream(multiStream))); } else { multiStream.addStream(new ByteArrayInputStream(chunk)); } @@ -299,7 +300,7 @@ public class RuleEngineService { totalBytesWritten += chunk.length; if (multiStream == null) { multiStream = new MultiStream(new ByteArrayInputStream(chunk), true); - output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, filters, inputStream(multiStream))); + output = outputStream(firstRule.getDriver().filterResponse(requestId, contentType, meta, inputStream(multiStream))); } else { multiStream.addLastStream(new ByteArrayInputStream(chunk)); } diff --git a/bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java b/bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java deleted file mode 100644 index d1bd1e11..00000000 --- a/bubble-server/src/test/java/bubble/rule/bblock/spec/BlockListTest.java +++ /dev/null @@ -1,157 +0,0 @@ -package bubble.rule.bblock.spec; - -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.assertEquals; - -public class BlockListTest { - - public static final String BLOCK = BlockDecisionType.block.name(); - public static final String ALLOW = BlockDecisionType.allow.name(); - public static final String FILTER = BlockDecisionType.filter.name(); - - public static final String[][] BLOCK_TESTS = { - // rule // fqdn // path // expected decision - - // bare hosts example (ala EasyList) - {"example.com", "example.com", "/some_path", BLOCK}, - {"example.com", "foo.example.com", "/some_path", BLOCK}, - {"example.com", "example.org", "/some_path", ALLOW}, - - // block example.com and all subdomains - {"||example.com^", "example.com", "/some_path", BLOCK}, - {"||example.com^", "foo.example.com", "/some_path", BLOCK}, - {"||example.com^", "example.org", "/some_path", ALLOW}, - - // block exact string - {"|example.com/|", "example.com", "/", BLOCK}, - {"|example.com/|", "example.com", "/some_path", ALLOW}, - {"|example.com/|", "foo.example.com", "/some_path", ALLOW}, - - // block example.com, but not foo.example.com or bar.example.com - {"||example.com^$domain=~foo.example.com|~bar.example.com", - "example.com", "/some_path", BLOCK}, - {"||example.com^$domain=~foo.example.com|~bar.example.com", - "foo.example.com", "/some_path", ALLOW}, - {"||example.com^$domain=~foo.example.com|~bar.example.com", - "bar.example.com", "/some_path", ALLOW}, - {"||example.com^$domain=~foo.example.com|~bar.example.com", - "baz.example.com", "/some_path", BLOCK}, - - // block images and scripts on example.com, but not foo.example.com or bar.example.com - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "example.com", "/some_path", ALLOW}, - - // test image blocking - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "example.com", "/some_path.png", BLOCK}, - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "foo.example.com", "/some_path.png", ALLOW}, - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "bar.example.com", "/some_path.png", ALLOW}, - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "baz.example.com", "/some_path.png", BLOCK}, - - // test script blocking - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "example.com", "/some_path.js", BLOCK}, - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "foo.example.com", "/some_path.js", ALLOW}, - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "bar.example.com", "/some_path.js", ALLOW}, - {"||example.com^$image,script,domain=~foo.example.com|~bar.example.com", - "baz.example.com", "/some_path.js", BLOCK}, - - // test stylesheet blocking - {"||example.com^stylesheet,domain=~foo.example.com|~bar.example.com", - "example.com", "/some_path.css", BLOCK}, - {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com", - "foo.example.com", "/some_path.css", ALLOW}, - {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com", - "bar.example.com", "/some_path.css", ALLOW}, - {"||example.com^$stylesheet,domain=~foo.example.com|~bar.example.com", - "baz.example.com", "/some_path.css", BLOCK}, - - - // path matching - {"/foo", "example.com", "/some_path", ALLOW}, - {"/foo", "example.com", "/foo", BLOCK}, - {"/foo", "example.com", "/foo/bar", BLOCK}, - - // path matching with wildcard - {"/foo/*/img", "example.com", "/some_path", ALLOW}, - {"/foo/*/img", "example.com", "/foo", ALLOW}, - {"/foo/*/img", "example.com", "/foo/img", ALLOW}, - {"/foo/*/img", "example.com", "/foo/x/img", BLOCK}, - {"/foo/*/img", "example.com", "/foo/x/img.png", ALLOW}, - {"/foo/*/img", "example.com", "/foo/x/y/z//img", BLOCK}, - - // path matching with regex - {"/foo/(apps|ads)/img.+/", "example.com", "/foo/x/y/z//img", ALLOW}, - {"/foo/(apps|ads)/img.+/", "example.com", "/foo/apps/img.png", BLOCK}, - {"/foo/(apps|ads)/img.+/", "example.com", "/foo/ads/img.png", BLOCK}, - {"/foo/(apps|ads)/img.+/", "example.com", "/foo/bar/ads/img.png", ALLOW}, - - {"/(apps|ads)\\.example\\.(com|org)/", - "example.com", "/ad.png", ALLOW}, - {"/(apps|ads)\\.example\\.(com|org)/", - "ads.example.com", "/ad.png", BLOCK}, - {"/(apps|ads)\\.example\\.(com|org)/", - "apps.example.com", "/ad.png", BLOCK}, - {"/(apps|ads)\\.example\\.(com|org)/", - "ads.example.org", "/ad.png", BLOCK}, - {"/(apps|ads)\\.example\\.(com|org)/", - "apps.example.org", "/ad.png", BLOCK}, - {"/(apps|ads)\\.example\\.(com|org)/", - "ads.example.net", "/ad.png", ALLOW}, - {"/(apps|ads)\\.example\\.(com|org)/", - "apps.example.net", "/ad.png", ALLOW}, - - // selectors - {"example.com##.banner-ad", "example.com", "/ad.png", FILTER}, - - // putting it all together - {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad", - "example.com", "/some_path", FILTER}, - {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad", - "baz.example.com", "/some_path", FILTER}, - {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad", - "foo.example.com", "/some_path", ALLOW}, - {"||example.com^$domain=~foo.example.com|~bar.example.com##.banner-ad", - "bar.example.com", "/some_path", ALLOW}, - }; - - @Test public void testRules () throws Exception { - for (String[] test : BLOCK_TESTS) { - final BlockDecisionType expectedDecision = BlockDecisionType.fromString(test[3]); - final BlockList blockList = new BlockList(); - blockList.addToBlacklist(BlockSpec.parse(test[0])); - assertEquals("testBlanketBlock: expected "+expectedDecision+" decision, test=" + Arrays.toString(test), - expectedDecision, - blockList.getDecision(test[1], test[2]).getDecisionType()); - } - } - - public String[] SELECTOR_SPECS = { - "||example.com##.banner-ad", - "||foo.example.com##.more-ads", - }; - - @Test public void testMultipleSelectorMatches () throws Exception { - final BlockList blockList = new BlockList(); - for (String line : SELECTOR_SPECS) { - blockList.addToBlacklist(BlockSpec.parse(line)); - } - BlockDecision decision; - - decision = blockList.getDecision("example.com", "/some_path"); - assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType()); - assertEquals("expected 1 filter specs", 1, decision.getSpecs().size()); - - decision = blockList.getDecision("foo.example.com", "/some_path"); - assertEquals("expected filter decision", BlockDecisionType.filter, decision.getDecisionType()); - assertEquals("expected 2 filter specs", 2, decision.getSpecs().size()); - } -} diff --git a/utils/abp-parser b/utils/abp-parser new file mode 160000 index 00000000..801d5bbf --- /dev/null +++ b/utils/abp-parser @@ -0,0 +1 @@ +Subproject commit 801d5bbf6f4327a407f8af8e7d7788302f348639 diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index 44730182..cf3d35b3 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit 44730182976085dd171cf7035fae5953a8841368 +Subproject commit cf3d35b3f6dbb2bd3a1b31abb2c1ded41aa15378