From a5ea3d24b151e9740ed56789f1c305d32ed0f377 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 7 Feb 2020 11:45:38 -0500 Subject: [PATCH] filter tweaks, update MR user blocker. add filter cache flush endpoint --- .../bubble/app/bblock/BlockListEntry.java | 6 +++- .../resources/stream/FilterHttpResource.java | 30 ++++++++++++----- .../stream/FilterMatchersRequest.java | 2 +- .../rule/bblock/BubbleBlockRuleDriver.java | 2 +- .../service/stream/RuleEngineService.java | 2 +- .../stream/StandardRuleEngineService.java | 7 +++- .../block/JsUserBlockerRuleDriver.js.hbs | 3 -- .../bubble/rule/social/block/site/MR.js.hbs | 33 +++++++++++-------- .../mr/bubbleApp_userBlock_mr_matchers.json | 9 ++++- 9 files changed, 63 insertions(+), 31 deletions(-) diff --git a/bubble-server/src/main/java/bubble/app/bblock/BlockListEntry.java b/bubble-server/src/main/java/bubble/app/bblock/BlockListEntry.java index bbfe5809..860b9a7e 100644 --- a/bubble-server/src/main/java/bubble/app/bblock/BlockListEntry.java +++ b/bubble-server/src/main/java/bubble/app/bblock/BlockListEntry.java @@ -28,7 +28,11 @@ public class BlockListEntry implements Comparable { @Override public int compareTo(BlockListEntry o) { if (rule == null) return o.rule == null ? 0 : -1; if (o.rule == null) return 1; - return safeFunctionName(rule).compareTo(safeFunctionName(o.rule)); + return normalizedString(rule).compareTo(normalizedString(o.rule)); + } + + public String normalizedString(String rule) { + return safeFunctionName(rule.replace("www.", "")); } } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index a6df4979..5ac1a01a 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -12,6 +12,7 @@ import bubble.service.stream.StandardRuleEngineService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ExpirationMap; +import org.cobbzilla.util.collection.MapBuilder; import org.cobbzilla.util.http.HttpContentEncodingType; import org.cobbzilla.wizard.cache.redis.RedisService; import org.glassfish.grizzly.http.server.Request; @@ -24,10 +25,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import static bubble.ApiConstants.*; @@ -232,6 +230,23 @@ public class FilterHttpResource { return matchers; } + @DELETE + @Produces(APPLICATION_JSON) + public Response flushCaches(@Context ContainerRequest request) { + final Account caller = userPrincipal(request); + if (!caller.admin()) return forbidden(); + + final RedisService matchersCache = getMatchersCache(); + final Collection keys = matchersCache.keys("*"); + for (String key : keys) matchersCache.del_withPrefix(key); + + final int ruleEngineCacheSize = ruleEngine.flushRuleCache(); + return ok(MapBuilder.build(new Object[][] { + {"matchersCache", keys.size()}, + {"ruleEngineCache", ruleEngineCacheSize} + })); + } + @POST @Path(EP_APPLY+"/{requestId}") @Consumes(MediaType.WILDCARD) @Produces(MediaType.WILDCARD) @@ -286,7 +301,7 @@ public class FilterHttpResource { return passthru(request); } - if (log.isDebugEnabled()) log.debug(prefix+"found FilterMatchersResponse: "+json(matchersResponse, COMPACT_MAPPER)); + if (log.isTraceEnabled()) log.trace(prefix+"found FilterMatchersResponse: "+json(matchersResponse, COMPACT_MAPPER)); if (matchersResponse.hasAbort()) { if (log.isWarnEnabled()) log.warn(prefix+"FilterMatchersResponse has abort code "+matchersResponse.httpStatus()+", MITM should have aborted. We are aborting now."); return status(matchersResponse.httpStatus()); @@ -326,7 +341,7 @@ public class FilterHttpResource { if (filterRequest == null) { if (log.isDebugEnabled()) log.debug(prefix+"filterRequest not found, initiating..."); if (empty(contentType)) { - if (log.isDebugEnabled()) log.debug(prefix+"filter request not found, and no contentType provided, returning passthru"); + if (log.isDebugEnabled()) log.debug(prefix+"filterRequest not found, and no contentType provided, returning passthru"); return passthru(request); } @@ -335,7 +350,7 @@ public class FilterHttpResource { if (log.isDebugEnabled()) log.debug(prefix+"device "+matchersResponse.getRequest().getDevice()+" not found, returning passthru"); return passthru(request); } else { - if (log.isDebugEnabled()) log.debug(prefix+"found device: "+device.id()+" ... "); + if (log.isTraceEnabled()) log.trace(prefix+"found device: "+device.id()+" ... "); } final Account caller = findCaller(device.getAccount()); if (caller == null) { @@ -352,7 +367,6 @@ public class FilterHttpResource { if (log.isDebugEnabled()) log.trace(prefix+"start filterRequest="+json(filterRequest, COMPACT_MAPPER)); getActiveRequestCache().set(requestId, json(filterRequest, COMPACT_MAPPER), EX, ACTIVE_REQUEST_TIMEOUT); } else { - if (log.isDebugEnabled()) log.debug(prefix+"filterRequest found, continuing..."); if (log.isTraceEnabled()) { if (isLast) { log.trace(prefix+"last filterRequest=" + json(filterRequest, COMPACT_MAPPER)); diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java index b198b048..76ccaf9d 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java @@ -29,6 +29,6 @@ public class FilterMatchersRequest { // FilterHttpResource.matchersCache cache would be useless, since every cache entry would be unique public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, remoteAddr); } - @JsonIgnore public String getUrl() { return fqdn + "/" + uri; } + @JsonIgnore public String getUrl() { return fqdn + uri; } } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index 8fd5bde4..770deb11 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -170,7 +170,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { return in; } - final String replacement = ""; + final String replacement = ""; final RegexReplacementFilter filter = new RegexReplacementFilter("", replacement); final RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in, UTF8cs), filter).setMaxMatches(1); if (log.isDebugEnabled()) { diff --git a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java index ba907ab8..1f9f14ef 100644 --- a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java @@ -2,6 +2,6 @@ package bubble.service.stream; public interface RuleEngineService { - default void flushRuleCache () {} + default int flushRuleCache () { return 0; } } diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java index c0e717d1..25286f5f 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java @@ -184,7 +184,12 @@ public class StandardRuleEngineService implements RuleEngineService { private ExpirationMap> ruleCache = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime); - public void flushRuleCache () { ruleCache.clear(); } + public int flushRuleCache () { + final int size = ruleCache.size(); + ruleCache.clear(); + log.info("flushRuleCache: flushed "+size+" ruleCache entries"); + return size; + } private List initRules(FilterHttpRequest filterRequest) { return initRules(filterRequest.getAccount(), filterRequest.getDevice(), filterRequest.getMatchers()); diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs index 6ce56031..875f92ba 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs @@ -1,6 +1,3 @@ -// todo: wrap window object in proxy. do not allow anyone else to change window.console -window.console = console; - let {{JS_PREFIX}}_blocked_users = null; let {{JS_PREFIX}}_doc_ready = false; const {{JS_PREFIX}}_request_id = '{{BUBBLE_REQUEST_ID}}'; diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/site/MR.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/site/MR.js.hbs index a7ccdb91..4575624b 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/site/MR.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/site/MR.js.hbs @@ -4,24 +4,29 @@ function {{JS_PREFIX}}_apply_blocks(blocked_users) { console.warn('No comments found, not filtering'); return; } - {{JS_PREFIX}}_consider_block(comments.querySelector('article.blog-comment'), blocked_users); + {{JS_PREFIX}}_consider_block(comments.querySelectorAll('article.blog-comment'), blocked_users); } function {{JS_PREFIX}}_consider_block(comments, blocked_users) { - for (let i=0; i 0) { + for (let i=0; iX]'; - userElement.parentNode.insertBefore(blockNode, userElement.nextSibling); + if (userElement.parentNode.querySelector('.bubble_block') === null) { + const blockNode = document.createElement('span'); + blockNode.className = "bubble_block"; + blockNode.innerHTML = ' [X]  '; + console.log('inserting block control...'); + userElement.parentNode.insertBefore(blockNode, userElement); + } + } } } } diff --git a/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json b/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json index 1716301f..abe5c964 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json +++ b/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json @@ -6,7 +6,14 @@ "site": "MarginalRevolution", "template": true, "fqdn": "marginalrevolution.com", - "urlRegex": "marginalrevolution//20\\d{2}/\\d{2}/\\w+", + "urlRegex": "marginalrevolution/20\\d{2}/\\d{2}/\\w+", + "rule": "mr_user_blocker" + }, { + "name": "MRTestMatcher", + "site": "MarginalRevolution", + "template": true, + "fqdn": "kyuss.org", + "urlRegex": "/20\\d{2}/\\d{2}/\\w+", "rule": "mr_user_blocker" }] }