filter tweaks, update MR user blocker. add filter cache flush endpoint

bubble-server/src/main/java/bubble/app/bblock/BlockListEntry.java

@@ -28,7 +28,11 @@ public class BlockListEntry implements Comparable<BlockListEntry> {
     @Override public int compareTo(BlockListEntry o) {
         if (rule == null) return o.rule == null ? 0 : -1;
         if (o.rule == null) return 1;
-        return safeFunctionName(rule).compareTo(safeFunctionName(o.rule));
+        return normalizedString(rule).compareTo(normalizedString(o.rule));
+    }
+
+    public String normalizedString(String rule) {
+        return safeFunctionName(rule.replace("www.", ""));
     }
 
 }

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -12,6 +12,7 @@ import bubble.service.stream.StandardRuleEngineService;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.MapBuilder;
 import org.cobbzilla.util.http.HttpContentEncodingType;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.glassfish.grizzly.http.server.Request;
@@ -24,10 +25,7 @@ import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.*;
@@ -232,6 +230,23 @@ public class FilterHttpResource {
         return matchers;
     }
 
+    @DELETE
+    @Produces(APPLICATION_JSON)
+    public Response flushCaches(@Context ContainerRequest request) {
+        final Account caller = userPrincipal(request);
+        if (!caller.admin()) return forbidden();
+
+        final RedisService matchersCache = getMatchersCache();
+        final Collection<String> keys = matchersCache.keys("*");
+        for (String key : keys) matchersCache.del_withPrefix(key);
+
+        final int ruleEngineCacheSize = ruleEngine.flushRuleCache();
+        return ok(MapBuilder.build(new Object[][] {
+                {"matchersCache", keys.size()},
+                {"ruleEngineCache", ruleEngineCacheSize}
+        }));
+    }
+
     @POST @Path(EP_APPLY+"/{requestId}")
     @Consumes(MediaType.WILDCARD)
     @Produces(MediaType.WILDCARD)
@@ -286,7 +301,7 @@ public class FilterHttpResource {
             return passthru(request);
 
         }
-        if (log.isDebugEnabled()) log.debug(prefix+"found FilterMatchersResponse: "+json(matchersResponse, COMPACT_MAPPER));
+        if (log.isTraceEnabled()) log.trace(prefix+"found FilterMatchersResponse: "+json(matchersResponse, COMPACT_MAPPER));
         if (matchersResponse.hasAbort()) {
             if (log.isWarnEnabled()) log.warn(prefix+"FilterMatchersResponse has abort code "+matchersResponse.httpStatus()+", MITM should have aborted. We are aborting now.");
             return status(matchersResponse.httpStatus());
@@ -326,7 +341,7 @@ public class FilterHttpResource {
         if (filterRequest == null) {
             if (log.isDebugEnabled()) log.debug(prefix+"filterRequest not found, initiating...");
             if (empty(contentType)) {
-                if (log.isDebugEnabled()) log.debug(prefix+"filter request not found, and no contentType provided, returning passthru");
+                if (log.isDebugEnabled()) log.debug(prefix+"filterRequest not found, and no contentType provided, returning passthru");
                 return passthru(request);
             }
 
@@ -335,7 +350,7 @@ public class FilterHttpResource {
                 if (log.isDebugEnabled()) log.debug(prefix+"device "+matchersResponse.getRequest().getDevice()+" not found, returning passthru");
                 return passthru(request);
             } else {
-                if (log.isDebugEnabled()) log.debug(prefix+"found device: "+device.id()+" ... ");
+                if (log.isTraceEnabled()) log.trace(prefix+"found device: "+device.id()+" ... ");
             }
             final Account caller = findCaller(device.getAccount());
             if (caller == null) {
@@ -352,7 +367,6 @@ public class FilterHttpResource {
             if (log.isDebugEnabled()) log.trace(prefix+"start filterRequest="+json(filterRequest, COMPACT_MAPPER));
             getActiveRequestCache().set(requestId, json(filterRequest, COMPACT_MAPPER), EX, ACTIVE_REQUEST_TIMEOUT);
         } else {
-            if (log.isDebugEnabled()) log.debug(prefix+"filterRequest found, continuing...");
             if (log.isTraceEnabled()) {
                 if (isLast) {
                     log.trace(prefix+"last filterRequest=" + json(filterRequest, COMPACT_MAPPER));

bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java

@@ -29,6 +29,6 @@ public class FilterMatchersRequest {
     // FilterHttpResource.matchersCache cache would be useless, since every cache entry would be unique
     public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, remoteAddr); }
 
-    @JsonIgnore public String getUrl() { return fqdn + "/" + uri; }
+    @JsonIgnore public String getUrl() { return fqdn + uri; }
 
 }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -170,7 +170,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
             return in;
         }
 
-        final String replacement = "<head><script>" + getBubbleJs(filterRequest.getId(), decision) + "</script>";
+        final String replacement = "<head><meta charset=\"UTF-8\"><script>" + getBubbleJs(filterRequest.getId(), decision) + "</script>";
         final RegexReplacementFilter filter = new RegexReplacementFilter("<head>", replacement);
         final RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in, UTF8cs), filter).setMaxMatches(1);
         if (log.isDebugEnabled()) {

bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java

@@ -2,6 +2,6 @@ package bubble.service.stream;
 
 public interface RuleEngineService {
 
-    default void flushRuleCache () {}
+    default int flushRuleCache () { return 0; }
 
 }

bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java

@@ -184,7 +184,12 @@ public class StandardRuleEngineService implements RuleEngineService {
 
     private ExpirationMap<String, List<AppRuleHarness>> ruleCache = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime);
 
-    public void flushRuleCache () { ruleCache.clear(); }
+    public int flushRuleCache () {
+        final int size = ruleCache.size();
+        ruleCache.clear();
+        log.info("flushRuleCache: flushed "+size+" ruleCache entries");
+        return size;
+    }
 
     private List<AppRuleHarness> initRules(FilterHttpRequest filterRequest) {
         return initRules(filterRequest.getAccount(), filterRequest.getDevice(), filterRequest.getMatchers());

bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs

@@ -1,6 +1,3 @@
-// todo: wrap window object in proxy. do not allow anyone else to change window.console
-window.console = console;
-
 let {{JS_PREFIX}}_blocked_users = null;
 let {{JS_PREFIX}}_doc_ready = false;
 const {{JS_PREFIX}}_request_id = '{{BUBBLE_REQUEST_ID}}';

bubble-server/src/main/resources/bubble/rule/social/block/site/MR.js.hbs

@@ -4,24 +4,29 @@ function {{JS_PREFIX}}_apply_blocks(blocked_users) {
         console.warn('No comments found, not filtering');
         return;
     }
-    {{JS_PREFIX}}_consider_block(comments.querySelector('article.blog-comment'), blocked_users);
+    {{JS_PREFIX}}_consider_block(comments.querySelectorAll('article.blog-comment'), blocked_users);
 }
 
 function {{JS_PREFIX}}_consider_block(comments, blocked_users) {
-    for (let i=0; i<comments.length; i++) {
-        const comment = comments[i];
-        const userElement = comment.querySelector('h3.comment-author');
-        if (userElement !== null) {
-            const author = userElement.innerText;
-            if (blocked_users !== null && blocked_users.includes(author)) {
-                comment.parentNode.removeChild(comment);
-                continue;
-            }
+    if (comments && comments.length && comments.length > 0) {
+        for (let i=0; i<comments.length; i++) {
+            const comment = comments[i];
+            const userElement = comment.querySelector('h3.comment-author');
+            if (userElement !== null) {
+                const author = userElement.innerText;
+                if (blocked_users !== null && blocked_users.includes(author)) {
+                    comment.parentNode.removeChild(comment);
+                    continue;
+                }
 
-            const blockNode = document.createElement('span');
-            blockNode.className = "bubble_block";
-            blockNode.innerHTML = ' [<b><a href="#" onclick="{{JS_PREFIX}}_block_user(\''+author+'\'); return false;">X</a></b>]';
-            userElement.parentNode.insertBefore(blockNode, userElement.nextSibling);
+                if (userElement.parentNode.querySelector('.bubble_block') === null) {
+                    const blockNode = document.createElement('span');
+                    blockNode.className = "bubble_block";
+                    blockNode.innerHTML = ' [<b><a href="#" onclick="_bubble_block_user(\'' + author.replace("'", "\\\'") + '\'); return false;">X</a></b>]&nbsp;&nbsp;';
+                    console.log('inserting block control...');
+                    userElement.parentNode.insertBefore(blockNode, userElement);
+                }
+            }
         }
     }
 }

bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json

@@ -6,7 +6,14 @@
       "site": "MarginalRevolution",
       "template": true,
       "fqdn": "marginalrevolution.com",
-      "urlRegex": "marginalrevolution//20\\d{2}/\\d{2}/\\w+",
+      "urlRegex": "marginalrevolution/20\\d{2}/\\d{2}/\\w+",
+      "rule": "mr_user_blocker"
+    }, {
+      "name": "MRTestMatcher",
+      "site": "MarginalRevolution",
+      "template": true,
+      "fqdn": "kyuss.org",
+      "urlRegex": "/20\\d{2}/\\d{2}/\\w+",
       "rule": "mr_user_blocker"
     }]
   }