improve analytics app, add filters to exclude from logging

bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsAppConfigDriver.java

@@ -0,0 +1,99 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://bubblev.com/bubble-license/
+ */
+package bubble.app.analytics;
+
+import bubble.model.account.Account;
+import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
+import bubble.model.app.config.AppConfigDriverBase;
+import bubble.rule.analytics.TrafficAnalyticsConfig;
+import bubble.rule.analytics.TrafficAnalyticsRuleDriver;
+import bubble.rule.passthru.TlsPassthruRuleDriver;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.collection.ArrayUtil;
+
+import java.util.Map;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
+import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
+
+@Slf4j
+public class TrafficAnalyticsAppConfigDriver extends AppConfigDriverBase {
+
+    public static final String VIEW_manageFilters = "manageFilters";
+
+    @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
+        switch (view) {
+            case VIEW_manageFilters:
+                return loadManageFilters(account, app);
+        }
+        log.debug("getView: view not found: "+view);
+        throw notFoundEx(view);
+    }
+
+    private Object loadManageFilters(Account account, BubbleApp app) {
+        final TrafficAnalyticsConfig config = getConfig(account, app);
+        return config.getPatterns();
+    }
+
+    private TrafficAnalyticsConfig getConfig(Account account, BubbleApp app) {
+        return getConfig(account, app, TrafficAnalyticsRuleDriver.class, TrafficAnalyticsConfig.class);
+    }
+
+    public static final String ACTION_addFilter = "addFilter";
+    public static final String ACTION_removeFilter = "removeFilter";
+
+    public static final String PARAM_FILTER = "analyticsFilter";
+
+    @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action, Map<String, String> params, JsonNode data) {
+        switch (action) {
+            case ACTION_addFilter:
+                return addFilter(account, app, data);
+        }
+        log.debug("takeAppAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private Object addFilter(Account account, BubbleApp app, JsonNode data) {
+        final JsonNode filterNode = data.get(PARAM_FILTER);
+        if (filterNode == null || filterNode.textValue() == null || empty(filterNode.textValue().trim())) {
+            throw invalidEx("err.addFilter.analyticsFilterRequired");
+        }
+
+        final String filter = filterNode.textValue().trim().toLowerCase();
+
+        final TrafficAnalyticsConfig config = getConfig(account, app)
+                .addFilter(filter);
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
+        ruleDAO.update(rule.setConfigJson(json(config)));
+
+        return config.getPatterns();
+    }
+
+    @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id, Map<String, String> params, JsonNode data) {
+        switch (action) {
+            case ACTION_removeFilter:
+                return removeFilter(account, app, id);
+        }
+        log.debug("takeAppAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private Object removeFilter(Account account, BubbleApp app, String id) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, TrafficAnalyticsRuleDriver.class); // validate proper driver
+        final TrafficAnalyticsConfig config = getConfig(account, app);
+
+        final TrafficAnalyticsConfig updated = config.removeFilter(id);
+        log.debug("removeFilter: updated.filterPatterns: "+ ArrayUtil.arrayToString(updated.getFilterPatterns()));
+        ruleDAO.update(rule.setConfigJson(json(updated)));
+        return config.getPatterns();
+    }
+}

bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java

@@ -60,9 +60,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase {
     }
 
     private TlsPassthruConfig getConfig(Account account, BubbleApp app) {
-        final AppRule rule = loadRule(account, app);
-        loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver
-        return json(rule.getConfigJson(), TlsPassthruConfig.class);
+        return getConfig(account, app, TlsPassthruRuleDriver.class, TlsPassthruConfig.class);
     }
 
     public static final String ACTION_addFqdn = "addFqdn";

bubble-server/src/main/java/bubble/model/app/config/AppConfigDriverBase.java

@@ -4,12 +4,14 @@ import bubble.dao.app.AppRuleDAO;
 import bubble.dao.app.RuleDriverDAO;
 import bubble.model.account.Account;
 import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
 import bubble.model.app.RuleDriver;
 import bubble.rule.AppRuleDriver;
 import lombok.Getter;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.json.JsonUtil.json;
 
 public abstract class AppConfigDriverBase implements AppConfigDriver {
 
@@ -28,4 +30,9 @@ public abstract class AppConfigDriverBase implements AppConfigDriver {
         return driver;
     }
 
+    protected <T> T getConfig (Account account, BubbleApp app, Class<? extends AppRuleDriver> driverClass, Class<T> configClass) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, driverClass); // validate proper driver
+        return json(rule.getConfigJson(), configClass);
+    }
 }

bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java

@@ -17,12 +17,11 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
 import lombok.Setter;
-import org.cobbzilla.util.http.HttpContentTypeAndCharset;
 import org.cobbzilla.util.system.Bytes;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 
-import static org.cobbzilla.util.http.HttpContentTypes.TEXT_HTML;
+import static org.cobbzilla.util.json.JsonUtil.json;
 
 public abstract class AbstractAppRuleDriver implements AppRuleDriver {
 
@@ -50,6 +49,10 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver {
     protected Account account;
     protected Device device;
 
+    public <C> Class<C> getConfigClass () { return null; }
+    protected Object ruleConfig;
+    public <C> C getRuleConfig () { return (C) ruleConfig; }
+
     public Handlebars getHandlebars () { return configuration.getHandlebars(); }
 
     protected String getDataId(String requestId) { return getDataId(requestId, matcher); }
@@ -67,6 +70,9 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver {
         this.rule = rule;
         this.account = account;
         this.device = device;
+        if (getConfigClass() != null) {
+            this.ruleConfig = json(json(config), getConfigClass());
+        }
     }
 
 }

bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsConfig.java

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://bubblev.com/bubble-license/
+ */
+package bubble.rule.analytics;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+
+@NoArgsConstructor @Accessors(chain=true) @ToString(of="filterPatterns")
+public class TrafficAnalyticsConfig {
+
+    @Getter @Setter private String[] filterPatterns;
+
+    @JsonIgnore public Set<TrafficAnalyticsFilterPattern> getPatterns () {
+        final Set<TrafficAnalyticsFilterPattern> set = new TreeSet<>();
+        if (!empty(filterPatterns)) {
+            set.addAll(Arrays.stream(filterPatterns)
+                    .map(TrafficAnalyticsFilterPattern::new)
+                    .collect(Collectors.toList()));
+        }
+        return set;
+    }
+
+    public TrafficAnalyticsConfig addFilter(String filter) {
+        final Set<TrafficAnalyticsFilterPattern> patterns = getPatterns();
+        patterns.add(new TrafficAnalyticsFilterPattern(filter));
+        return setFilterPatterns(patterns.stream()
+                .map(TrafficAnalyticsFilterPattern::getAnalyticsFilter)
+                .toArray(String[]::new));
+    }
+
+    public TrafficAnalyticsConfig removeFilter(String id) {
+        if (!empty(filterPatterns)) {
+            final Set<TrafficAnalyticsFilterPattern> patterns = getPatterns();
+            patterns.remove(new TrafficAnalyticsFilterPattern(id));
+            setFilterPatterns(patterns.stream()
+                    .map(TrafficAnalyticsFilterPattern::getAnalyticsFilter)
+                    .toArray(String[]::new));
+        }
+        return this;
+    }
+
+    @JsonIgnore @Getter(lazy=true) private final List<Pattern> regexes = initRegexes();
+    private List<Pattern> initRegexes() {
+        final List<Pattern> patterns = new ArrayList<>();
+        if (!empty(filterPatterns)) {
+            for (String pattern : filterPatterns) patterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE));
+        }
+        return patterns;
+    }
+
+    public boolean shouldSkip(String url) {
+        if (!empty(filterPatterns)) {
+            for (Pattern pattern : getRegexes()) {
+                if (pattern.matcher(url).find()) return true;
+            }
+        }
+        return false;
+    }
+
+}

bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsFilterPattern.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://bubblev.com/bubble-license/
+ */
+package bubble.rule.analytics;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import org.cobbzilla.util.string.StringUtil;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class TrafficAnalyticsFilterPattern implements Comparable<TrafficAnalyticsFilterPattern> {
+
+    public String getId() { return analyticsFilter; }
+    public void setId(String id) {} // noop
+
+    @Getter @Setter private String analyticsFilter;
+
+    @JsonIgnore public String getCanonicalName () { return empty(analyticsFilter) ? "" : StringUtil.safeFunctionName(analyticsFilter.toLowerCase()); }
+
+    public TrafficAnalyticsFilterPattern (String pattern) { this.analyticsFilter = pattern; }
+
+    @Override public int compareTo(TrafficAnalyticsFilterPattern o) { return getCanonicalName().compareTo(o.getCanonicalName()); }
+
+}

bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsRuleDriver.java

@@ -43,6 +43,8 @@ public class TrafficAnalyticsRuleDriver extends AbstractAppRuleDriver {
     private String initNetworkDomain() { return configuration.getThisNetwork() == null ? null : configuration.getThisNetwork().getNetworkDomain(); }
     @Getter(lazy=true) private final String networkDomainWithDotPrefix = "."+getNetworkDomain();
 
+    @Override public <C> Class<C> getConfigClass() { return (Class<C>) TrafficAnalyticsConfig.class; }
+
     @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness,
                                                     FilterMatchersRequest filter,
                                                     Account account,
@@ -58,6 +60,13 @@ public class TrafficAnalyticsRuleDriver extends AbstractAppRuleDriver {
             return FilterMatchDecision.no_match;
         }
 
+        final TrafficAnalyticsConfig config = getRuleConfig();
+        if (config != null && config.shouldSkip(filter.getUrl())) {
+            if (log.isDebugEnabled()) log.debug("preprocess: not logging request (matched filter): url="+filter.getUrl());
+            return FilterMatchDecision.no_match;
+        }
+
+        if (log.isDebugEnabled()) log.debug("preprocess: logging request (config="+config+"): url="+filter.getUrl());
         final TrafficRecord rec = new TrafficRecord(filter, account, device);
         recordRecentTraffic(rec);
         incrementCounters(account, device, app, site, fqdn);

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -49,19 +49,19 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath;
 @Slf4j
 public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
 
-    private BubbleBlockConfig bubbleBlockConfig;
-
     private BlockList blockList = new BlockList();
 
     private static Map<String, BlockListSource> blockListCache = new ConcurrentHashMap<>();
 
+    @Override public <C> Class<C> getConfigClass() { return (Class<C>) BubbleBlockConfig.class; }
+
     @Override public void init(JsonNode config, JsonNode userConfig, AppRule rule, AppMatcher matcher, Account account, Device device) {
         super.init(config, userConfig, rule, matcher, account, device);
-        bubbleBlockConfig = json(json(config), BubbleBlockConfig.class);
         refreshBlockLists();
     }
 
     public void refreshBlockLists() {
+        final BubbleBlockConfig bubbleBlockConfig = getRuleConfig();
         final BubbleBlockList[] blockLists = bubbleBlockConfig.getBlockLists();
         final Set<String> refreshed = new HashSet<>();
         for (BubbleBlockList list : blockLists) {
@@ -110,6 +110,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         final String fqdn = filter.getFqdn();
         final String prefix = "preprocess("+filter.getRequestId()+"): ";
 
+        final BubbleBlockConfig bubbleBlockConfig = getRuleConfig();
         final BlockDecision decision = getDecision(filter.getFqdn(), filter.getUri(), filter.getUserAgent());
         switch (decision.getDecisionType()) {
             case block:
@@ -166,6 +167,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
     public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return blockList.getDecision(fqdn, uri, userAgent, false); }
 
     public BlockDecision getDecision(String fqdn, String uri, String userAgent, boolean primary) {
+        final BubbleBlockConfig bubbleBlockConfig = getRuleConfig();
         if (!empty(userAgent) && !empty(bubbleBlockConfig.getUserAgentBlocks())) {
             for (BubbleUserAgentBlock uaBlock : bubbleBlockConfig.getUserAgentBlocks()) {
                 if (uaBlock.hasUrlRegex() && uaBlock.urlMatches(uri)) {

bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java

@@ -5,29 +5,19 @@
 package bubble.rule.passthru;
 
 import bubble.model.account.Account;
-import bubble.model.app.AppMatcher;
-import bubble.model.app.AppRule;
 import bubble.model.device.Device;
 import bubble.resources.stream.FilterMatchersRequest;
 import bubble.rule.AbstractAppRuleDriver;
 import bubble.rule.FilterMatchDecision;
 import bubble.service.stream.AppRuleHarness;
-import com.fasterxml.jackson.databind.JsonNode;
 import lombok.extern.slf4j.Slf4j;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
 
-import static org.cobbzilla.util.json.JsonUtil.json;
-
 @Slf4j
 public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
 
-    private TlsPassthruConfig passthruConfig;
-
-    @Override public void init(JsonNode config, JsonNode userConfig, AppRule rule, AppMatcher matcher, Account account, Device device) {
-        super.init(config, userConfig, rule, matcher, account, device);
-        passthruConfig = json(json(config), TlsPassthruConfig.class);
-    }
+    @Override public <C> Class<C> getConfigClass() { return (Class<C>) TlsPassthruConfig.class; }
 
     @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness,
                                                     FilterMatchersRequest filter,
@@ -35,6 +25,7 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver {
                                                     Device device,
                                                     Request req,
                                                     ContainerRequest request) {
+        final TlsPassthruConfig passthruConfig = getRuleConfig();
         final String fqdn = filter.getFqdn();
         if (passthruConfig.isPassthru(fqdn)) {
             if (log.isDebugEnabled()) log.debug("preprocess: returning pass_thru for fqdn="+fqdn);

bubble-server/src/main/resources/logback.xml

@@ -39,6 +39,7 @@
     <logger name="org.cobbzilla.wizard.server.listener.BrowserLauncherListener" level="INFO" />
     <logger name="bubble.service.notify.NotificationService" level="WARN" />
     <logger name="bubble.rule.bblock.BubbleBlockRuleDriver" level="INFO" />
+    <logger name="bubble.rule.analytics.TrafficAnalyticsRuleDriver" level="DEBUG" />
 <!--    <logger name="org.cobbzilla.util.io.multi.MultiStream" level="TRACE" />-->
 <!--    <logger name="bubble.filters.BubbleRateLimitFilter" level="TRACE" />-->
 <!--    <logger name="org.cobbzilla.wizard.filters.RateLimitFilter" level="TRACE" />-->

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -762,3 +762,6 @@ err.testUrl.invalidHostname=URL did not have a valid hostname
 err.addFqdn.passthruFqdnRequired=Domain or Hostname field is required
 err.addFeed.feedUrlRequired=Feed URL is required
 err.addFeed.emptyFqdnList=Feed URL was not found or contained no data
+
+# analytics app errors
+err.addFilter.analyticsFilterRequired=Filter pattern is required

bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json

@@ -41,7 +41,25 @@
       {"name": "last_24_hours"},
       {"name": "last_7_days"},
       {"name": "last_30_days"}
-    ]
+    ],
+    "configDriver": "bubble.app.analytics.TrafficAnalyticsAppConfigDriver",
+    "configFields": [
+      {"name": "analyticsFilter", "truncate": false}
+    ],
+    "configViews": [{
+      "name": "manageFilters",
+      "scope": "app",
+      "root": "true",
+      "fields": ["analyticsFilter"],
+      "actions": [
+        {"name": "removeFilter", "index": 10},
+        {
+          "name": "addFilter", "scope": "app", "index": 10,
+          "params": ["analyticsFilter"],
+          "button": "addFilter"
+        }
+      ]
+    }]
   },
   "children": {
     "AppSite": [{
@@ -54,7 +72,9 @@
       "name": "traffic_analytics",
       "template": true,
       "driver": "TrafficAnalyticsRuleDriver",
-      "config": {}
+      "config": {
+        "filterPatterns": ["\\.stripe\\.com"]
+      }
     }],
     "AppMessage": [{
       "locale": "en_US",
@@ -85,7 +105,14 @@
         {"name": "view.last_7_days", "value": "Last 7 Days"},
         {"name": "view.last_7_days.ctime.format", "value": "{{MMM}} {{d}}, {{YYYY}}"},
         {"name": "view.last_30_days", "value": "Last 30 Days"},
-        {"name": "view.last_30_days.ctime.format", "value": "{{MMM}} {{d}}, {{YYYY}}"}
+        {"name": "view.last_30_days.ctime.format", "value": "{{MMM}} {{d}}, {{YYYY}}"},
+
+        {"name": "config.view.manageFilters", "value": "Manage Filters"},
+        {"name": "config.field.analyticsFilter", "value": "Filter Pattern"},
+        {"name": "config.field.analyticsFilter.description", "value": "Skip logging for URLs that match this pattern"},
+        {"name": "config.action.addFilter", "value": "Add New Filter"},
+        {"name": "config.button.addFilter", "value": "Add"},
+        {"name": "config.action.removeFilter", "value": "Remove"}
       ]
     }]
   }

bubble-web

@@ -1 +1 @@
-Subproject commit 7cd7e0e140da2d048ad3addfac4bceb72ab5e89a
+Subproject commit 995c996c444bad30eca592b71ce8dd0daa3f5a05