WIP. debugging adblocker

bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java

@@ -51,11 +51,6 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver {
     protected String getDataId(String requestId) { return getDataId(requestId, matcher); }
     public static String getDataId(String requestId, AppMatcher matcher) { return requestId+"/"+matcher.getUuid(); }
 
-    public boolean isHtml (String contentType) {
-        final HttpContentTypeAndCharset type = new HttpContentTypeAndCharset(contentType);
-        return type.isContentType(TEXT_HTML);
-    }
-
     @Override public void init(JsonNode config,
                                JsonNode userConfig,
                                AppRule rule,

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -18,6 +18,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.ReaderInputStream;
 import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
+import org.cobbzilla.util.http.HttpContentTypes;
 import org.cobbzilla.util.io.regex.RegexFilterReader;
 import org.cobbzilla.util.io.regex.RegexReplacementFilter;
 import org.glassfish.grizzly.http.server.Request;
@@ -99,11 +100,14 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         final BlockDecision decision = getDecision(filter.getFqdn(), filter.getUri());
         switch (decision.getDecisionType()) {
             case block:
+                if (log.isDebugEnabled()) log.debug("preprocess: decision is BLOCK");
                 incrementCounters(account, device, app, site, fqdn);
                 return FilterMatchResponse.ABORT_NOT_FOUND;  // block this request
             case allow: default:
+                if (log.isDebugEnabled()) log.debug("preprocess: decision is ALLOW");
                 return FilterMatchResponse.NO_MATCH;
             case filter:
+                if (log.isDebugEnabled()) log.debug("preprocess: decision is FILTER");
                 return getFilterMatchResponse(filter, decision);
         }
     }
@@ -152,15 +156,15 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         final BlockDecision decision = blockList.getDecision(request.getFqdn(), request.getUri(), contentType, true);
         switch (decision.getDecisionType()) {
             case block:
-                log.warn("doFilterRequest: preprocessed request was filtered, but ultimate decision was block, returning EMPTY_STREAM");
+                log.warn("doFilterRequest: preprocessed request was filtered, but ultimate decision was block (contentType="+contentType+"), returning EMPTY_STREAM");
                 return EMPTY_STREAM;
             case allow:
-                log.warn("doFilterRequest: preprocessed request was filtered, but ultimate decision was allow, returning as-is");
+                log.warn("doFilterRequest: preprocessed request was filtered, but ultimate decision was allow (contentType="+contentType+"), returning as-is");
                 return in;
             case filter:
                 if (!decision.hasSpecs()) {
                     // should never happen
-                    log.warn("doFilterRequest: preprocessed request was filtered, but ultimate decision was filtered, but no filters provided, returning as-is");
+                    log.warn("doFilterRequest: preprocessed request was filtered, but ultimate decision was filtered (contentType="+contentType+"), but no filters provided, returning as-is");
                     return in;
                 }
                 break;
@@ -170,7 +174,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
                 return EMPTY_STREAM;
         }
 
-        if (!isHtml(contentType)) {
+        if (!HttpContentTypes.isHtml(contentType)) {
             log.warn("doFilterRequest: cannot filter non-html response ("+request.getUrl()+"), returning as-is: "+contentType);
             return in;
         }
@@ -178,6 +182,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         final String replacement = "<head><script>" + getBubbleJs(requestId, decision) + "</script>";
         final RegexReplacementFilter filter = new RegexReplacementFilter("<head>", replacement);
         final RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in), filter).setMaxMatches(1);
+        if (log.isDebugEnabled()) log.debug("doFilterResponse: filtering response for "+request.getUri()+" - replacement.length = "+replacement.length());
         return new ReaderInputStream(reader, UTF8cs);
     }
 
@@ -188,7 +193,6 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
     private static final String CTX_BUBBLE_WHITELIST = "BUBBLE_WHITELIST_JSON";
 
     private String getBubbleJs(String requestId, BlockDecision decision) {
-
         final Map<String, Object> ctx = new HashMap<>();
         ctx.put(CTX_JS_PREFIX, "__bubble_block_"+sha256_hex(requestId)+"_");
         ctx.put(CTX_BUBBLE_REQUEST_ID, requestId);

bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerRuleDriver.java

@@ -7,6 +7,7 @@ import org.apache.commons.io.input.ReaderInputStream;
 import org.cobbzilla.util.collection.ExpirationMap;
 import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
+import org.cobbzilla.util.http.HttpContentTypes;
 import org.cobbzilla.util.io.regex.RegexFilterReader;
 import org.cobbzilla.util.io.regex.RegexReplacementFilter;
 
@@ -29,7 +30,7 @@ public class JsUserBlockerRuleDriver extends AbstractAppRuleDriver {
     public static final String CTX_APPLY_BLOCKS_JS = "APPLY_BLOCKS_JS";
 
     @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
-        if (!isHtml(contentType)) return in;
+        if (!HttpContentTypes.isHtml(contentType)) return in;
 
         final String replacement = "<head><script>" + getBubbleJs(requestId) + "</script>";
         final RegexReplacementFilter filter = new RegexReplacementFilter("<head>", replacement);

bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java

@@ -8,6 +8,7 @@ import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.ReaderInputStream;
 import org.cobbzilla.util.collection.NameAndValue;
+import org.cobbzilla.util.http.HttpContentTypes;
 import org.cobbzilla.util.io.regex.RegexFilterReader;
 import org.cobbzilla.util.io.regex.RegexInsertionFilter;
 import org.cobbzilla.util.io.regex.RegexStreamFilter;
@@ -49,7 +50,7 @@ public class UserBlockerRuleDriver extends AbstractAppRuleDriver {
     protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); }
 
     @Override public InputStream doFilterResponse(String requestId, String contentType, NameAndValue[] meta, InputStream in) {
-        if (!isHtml(contentType)) return in;
+        if (!HttpContentTypes.isHtml(contentType)) return in;
 
         final UserBlockerStreamFilter filter = new UserBlockerStreamFilter(requestId, matcher, rule, configuration.getHttp().getBaseUri());
         filter.configure(getFullConfig());

bubble-server/src/main/resources/logback.xml

@@ -36,7 +36,7 @@
     <logger name="org.cobbzilla.wizard.dao.AbstractCRUDDAO" level="WARN" />
     <logger name="org.cobbzilla.wizard.server.listener.BrowserLauncherListener" level="INFO" />
     <logger name="bubble.service.notify.NotificationService" level="WARN" />
-    <logger name="bubble.service.dbfilter.EntityIterator" level="DEBUG" />
+    <logger name="bubble.rule.bblock.BubbleBlockRuleDriver" level="DEBUG" />
     <logger name="bubble.resources.stream" level="INFO" />
     <logger name="bubble.service.stream" level="INFO" />
     <logger name="bubble.resources.message" level="INFO" />

utils/abp-parser

@@ -1 +1 @@
-Subproject commit 1094d4976bd787aa92997e59b2dab9b993a26072
+Subproject commit 3237519654aa9bd13fad60b465bbabe7208252e2

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 12d7d0d1104ee1c2da52f2348a96b49232e0ed3b
+Subproject commit a45899b53152ef404cc32041ff7b9452b9b31b0a