diff --git a/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsAppConfigDriver.java new file mode 100644 index 00000000..2a181a9a --- /dev/null +++ b/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsAppConfigDriver.java @@ -0,0 +1,99 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://bubblev.com/bubble-license/ + */ +package bubble.app.analytics; + +import bubble.model.account.Account; +import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; +import bubble.model.app.config.AppConfigDriverBase; +import bubble.rule.analytics.TrafficAnalyticsConfig; +import bubble.rule.analytics.TrafficAnalyticsRuleDriver; +import bubble.rule.passthru.TlsPassthruRuleDriver; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.ArrayUtil; + +import java.util.Map; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; +import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; + +@Slf4j +public class TrafficAnalyticsAppConfigDriver extends AppConfigDriverBase { + + public static final String VIEW_manageFilters = "manageFilters"; + + @Override public Object getView(Account account, BubbleApp app, String view, Map params) { + switch (view) { + case VIEW_manageFilters: + return loadManageFilters(account, app); + } + log.debug("getView: view not found: "+view); + throw notFoundEx(view); + } + + private Object loadManageFilters(Account account, BubbleApp app) { + final TrafficAnalyticsConfig config = getConfig(account, app); + return config.getPatterns(); + } + + private TrafficAnalyticsConfig getConfig(Account account, BubbleApp app) { + return getConfig(account, app, TrafficAnalyticsRuleDriver.class, TrafficAnalyticsConfig.class); + } + + public static final String ACTION_addFilter = "addFilter"; + public static final String ACTION_removeFilter = "removeFilter"; + + public static final String PARAM_FILTER = "analyticsFilter"; + + @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action, Map params, JsonNode data) { + switch (action) { + case ACTION_addFilter: + return addFilter(account, app, data); + } + log.debug("takeAppAction: action not found: "+action); + throw notFoundEx(action); + } + + private Object addFilter(Account account, BubbleApp app, JsonNode data) { + final JsonNode filterNode = data.get(PARAM_FILTER); + if (filterNode == null || filterNode.textValue() == null || empty(filterNode.textValue().trim())) { + throw invalidEx("err.addFilter.analyticsFilterRequired"); + } + + final String filter = filterNode.textValue().trim().toLowerCase(); + + final TrafficAnalyticsConfig config = getConfig(account, app) + .addFilter(filter); + + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver + ruleDAO.update(rule.setConfigJson(json(config))); + + return config.getPatterns(); + } + + @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id, Map params, JsonNode data) { + switch (action) { + case ACTION_removeFilter: + return removeFilter(account, app, id); + } + log.debug("takeAppAction: action not found: "+action); + throw notFoundEx(action); + } + + private Object removeFilter(Account account, BubbleApp app, String id) { + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, TrafficAnalyticsRuleDriver.class); // validate proper driver + final TrafficAnalyticsConfig config = getConfig(account, app); + + final TrafficAnalyticsConfig updated = config.removeFilter(id); + log.debug("removeFilter: updated.filterPatterns: "+ ArrayUtil.arrayToString(updated.getFilterPatterns())); + ruleDAO.update(rule.setConfigJson(json(updated))); + return config.getPatterns(); + } +} diff --git a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java index 59500fe3..d190aec6 100644 --- a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java +++ b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java @@ -60,9 +60,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { } private TlsPassthruConfig getConfig(Account account, BubbleApp app) { - final AppRule rule = loadRule(account, app); - loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver - return json(rule.getConfigJson(), TlsPassthruConfig.class); + return getConfig(account, app, TlsPassthruRuleDriver.class, TlsPassthruConfig.class); } public static final String ACTION_addFqdn = "addFqdn"; diff --git a/bubble-server/src/main/java/bubble/model/app/config/AppConfigDriverBase.java b/bubble-server/src/main/java/bubble/model/app/config/AppConfigDriverBase.java index 5e8545be..fcb8bb44 100644 --- a/bubble-server/src/main/java/bubble/model/app/config/AppConfigDriverBase.java +++ b/bubble-server/src/main/java/bubble/model/app/config/AppConfigDriverBase.java @@ -4,12 +4,14 @@ import bubble.dao.app.AppRuleDAO; import bubble.dao.app.RuleDriverDAO; import bubble.model.account.Account; import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; import bubble.model.app.RuleDriver; import bubble.rule.AppRuleDriver; import lombok.Getter; import org.springframework.beans.factory.annotation.Autowired; import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.json.JsonUtil.json; public abstract class AppConfigDriverBase implements AppConfigDriver { @@ -28,4 +30,9 @@ public abstract class AppConfigDriverBase implements AppConfigDriver { return driver; } + protected T getConfig (Account account, BubbleApp app, Class driverClass, Class configClass) { + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, driverClass); // validate proper driver + return json(rule.getConfigJson(), configClass); + } } diff --git a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java index e01c603f..aa03a0b4 100644 --- a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java @@ -17,12 +17,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; import lombok.Getter; import lombok.Setter; -import org.cobbzilla.util.http.HttpContentTypeAndCharset; import org.cobbzilla.util.system.Bytes; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; -import static org.cobbzilla.util.http.HttpContentTypes.TEXT_HTML; +import static org.cobbzilla.util.json.JsonUtil.json; public abstract class AbstractAppRuleDriver implements AppRuleDriver { @@ -50,6 +49,10 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { protected Account account; protected Device device; + public Class getConfigClass () { return null; } + protected Object ruleConfig; + public C getRuleConfig () { return (C) ruleConfig; } + public Handlebars getHandlebars () { return configuration.getHandlebars(); } protected String getDataId(String requestId) { return getDataId(requestId, matcher); } @@ -67,6 +70,9 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { this.rule = rule; this.account = account; this.device = device; + if (getConfigClass() != null) { + this.ruleConfig = json(json(config), getConfigClass()); + } } } diff --git a/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsConfig.java b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsConfig.java new file mode 100644 index 00000000..d88e52f0 --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsConfig.java @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://bubblev.com/bubble-license/ + */ +package bubble.rule.analytics; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; + +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@NoArgsConstructor @Accessors(chain=true) @ToString(of="filterPatterns") +public class TrafficAnalyticsConfig { + + @Getter @Setter private String[] filterPatterns; + + @JsonIgnore public Set getPatterns () { + final Set set = new TreeSet<>(); + if (!empty(filterPatterns)) { + set.addAll(Arrays.stream(filterPatterns) + .map(TrafficAnalyticsFilterPattern::new) + .collect(Collectors.toList())); + } + return set; + } + + public TrafficAnalyticsConfig addFilter(String filter) { + final Set patterns = getPatterns(); + patterns.add(new TrafficAnalyticsFilterPattern(filter)); + return setFilterPatterns(patterns.stream() + .map(TrafficAnalyticsFilterPattern::getAnalyticsFilter) + .toArray(String[]::new)); + } + + public TrafficAnalyticsConfig removeFilter(String id) { + if (!empty(filterPatterns)) { + final Set patterns = getPatterns(); + patterns.remove(new TrafficAnalyticsFilterPattern(id)); + setFilterPatterns(patterns.stream() + .map(TrafficAnalyticsFilterPattern::getAnalyticsFilter) + .toArray(String[]::new)); + } + return this; + } + + @JsonIgnore @Getter(lazy=true) private final List regexes = initRegexes(); + private List initRegexes() { + final List patterns = new ArrayList<>(); + if (!empty(filterPatterns)) { + for (String pattern : filterPatterns) patterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE)); + } + return patterns; + } + + public boolean shouldSkip(String url) { + if (!empty(filterPatterns)) { + for (Pattern pattern : getRegexes()) { + if (pattern.matcher(url).find()) return true; + } + } + return false; + } + +} diff --git a/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsFilterPattern.java b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsFilterPattern.java new file mode 100644 index 00000000..bd618ac9 --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsFilterPattern.java @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://bubblev.com/bubble-license/ + */ +package bubble.rule.analytics; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.cobbzilla.util.string.StringUtil; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@NoArgsConstructor @Accessors(chain=true) +public class TrafficAnalyticsFilterPattern implements Comparable { + + public String getId() { return analyticsFilter; } + public void setId(String id) {} // noop + + @Getter @Setter private String analyticsFilter; + + @JsonIgnore public String getCanonicalName () { return empty(analyticsFilter) ? "" : StringUtil.safeFunctionName(analyticsFilter.toLowerCase()); } + + public TrafficAnalyticsFilterPattern (String pattern) { this.analyticsFilter = pattern; } + + @Override public int compareTo(TrafficAnalyticsFilterPattern o) { return getCanonicalName().compareTo(o.getCanonicalName()); } + +} diff --git a/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsRuleDriver.java b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsRuleDriver.java index 1eb36924..60e5b156 100644 --- a/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsRuleDriver.java @@ -43,6 +43,8 @@ public class TrafficAnalyticsRuleDriver extends AbstractAppRuleDriver { private String initNetworkDomain() { return configuration.getThisNetwork() == null ? null : configuration.getThisNetwork().getNetworkDomain(); } @Getter(lazy=true) private final String networkDomainWithDotPrefix = "."+getNetworkDomain(); + @Override public Class getConfigClass() { return (Class) TrafficAnalyticsConfig.class; } + @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness, FilterMatchersRequest filter, Account account, @@ -58,6 +60,13 @@ public class TrafficAnalyticsRuleDriver extends AbstractAppRuleDriver { return FilterMatchDecision.no_match; } + final TrafficAnalyticsConfig config = getRuleConfig(); + if (config != null && config.shouldSkip(filter.getUrl())) { + if (log.isDebugEnabled()) log.debug("preprocess: not logging request (matched filter): url="+filter.getUrl()); + return FilterMatchDecision.no_match; + } + + if (log.isDebugEnabled()) log.debug("preprocess: logging request (config="+config+"): url="+filter.getUrl()); final TrafficRecord rec = new TrafficRecord(filter, account, device); recordRecentTraffic(rec); incrementCounters(account, device, app, site, fqdn); diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index 8276327f..7653a340 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -49,19 +49,19 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath; @Slf4j public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { - private BubbleBlockConfig bubbleBlockConfig; - private BlockList blockList = new BlockList(); private static Map blockListCache = new ConcurrentHashMap<>(); + @Override public Class getConfigClass() { return (Class) BubbleBlockConfig.class; } + @Override public void init(JsonNode config, JsonNode userConfig, AppRule rule, AppMatcher matcher, Account account, Device device) { super.init(config, userConfig, rule, matcher, account, device); - bubbleBlockConfig = json(json(config), BubbleBlockConfig.class); refreshBlockLists(); } public void refreshBlockLists() { + final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); final BubbleBlockList[] blockLists = bubbleBlockConfig.getBlockLists(); final Set refreshed = new HashSet<>(); for (BubbleBlockList list : blockLists) { @@ -110,6 +110,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { final String fqdn = filter.getFqdn(); final String prefix = "preprocess("+filter.getRequestId()+"): "; + final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); final BlockDecision decision = getDecision(filter.getFqdn(), filter.getUri(), filter.getUserAgent()); switch (decision.getDecisionType()) { case block: @@ -166,6 +167,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return blockList.getDecision(fqdn, uri, userAgent, false); } public BlockDecision getDecision(String fqdn, String uri, String userAgent, boolean primary) { + final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); if (!empty(userAgent) && !empty(bubbleBlockConfig.getUserAgentBlocks())) { for (BubbleUserAgentBlock uaBlock : bubbleBlockConfig.getUserAgentBlocks()) { if (uaBlock.hasUrlRegex() && uaBlock.urlMatches(uri)) { diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index 726b9da2..f83968d2 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -5,29 +5,19 @@ package bubble.rule.passthru; import bubble.model.account.Account; -import bubble.model.app.AppMatcher; -import bubble.model.app.AppRule; import bubble.model.device.Device; import bubble.resources.stream.FilterMatchersRequest; import bubble.rule.AbstractAppRuleDriver; import bubble.rule.FilterMatchDecision; import bubble.service.stream.AppRuleHarness; -import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; -import static org.cobbzilla.util.json.JsonUtil.json; - @Slf4j public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { - private TlsPassthruConfig passthruConfig; - - @Override public void init(JsonNode config, JsonNode userConfig, AppRule rule, AppMatcher matcher, Account account, Device device) { - super.init(config, userConfig, rule, matcher, account, device); - passthruConfig = json(json(config), TlsPassthruConfig.class); - } + @Override public Class getConfigClass() { return (Class) TlsPassthruConfig.class; } @Override public FilterMatchDecision preprocess(AppRuleHarness ruleHarness, FilterMatchersRequest filter, @@ -35,6 +25,7 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { Device device, Request req, ContainerRequest request) { + final TlsPassthruConfig passthruConfig = getRuleConfig(); final String fqdn = filter.getFqdn(); if (passthruConfig.isPassthru(fqdn)) { if (log.isDebugEnabled()) log.debug("preprocess: returning pass_thru for fqdn="+fqdn); diff --git a/bubble-server/src/main/resources/logback.xml b/bubble-server/src/main/resources/logback.xml index 5e355e38..8b922efa 100644 --- a/bubble-server/src/main/resources/logback.xml +++ b/bubble-server/src/main/resources/logback.xml @@ -39,6 +39,7 @@ + diff --git a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties index a9d31c1e..70787515 100644 --- a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties +++ b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties @@ -762,3 +762,6 @@ err.testUrl.invalidHostname=URL did not have a valid hostname err.addFqdn.passthruFqdnRequired=Domain or Hostname field is required err.addFeed.feedUrlRequired=Feed URL is required err.addFeed.emptyFqdnList=Feed URL was not found or contained no data + +# analytics app errors +err.addFilter.analyticsFilterRequired=Filter pattern is required diff --git a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json index 6d1fea63..3281f7af 100644 --- a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json +++ b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json @@ -41,7 +41,25 @@ {"name": "last_24_hours"}, {"name": "last_7_days"}, {"name": "last_30_days"} - ] + ], + "configDriver": "bubble.app.analytics.TrafficAnalyticsAppConfigDriver", + "configFields": [ + {"name": "analyticsFilter", "truncate": false} + ], + "configViews": [{ + "name": "manageFilters", + "scope": "app", + "root": "true", + "fields": ["analyticsFilter"], + "actions": [ + {"name": "removeFilter", "index": 10}, + { + "name": "addFilter", "scope": "app", "index": 10, + "params": ["analyticsFilter"], + "button": "addFilter" + } + ] + }] }, "children": { "AppSite": [{ @@ -54,7 +72,9 @@ "name": "traffic_analytics", "template": true, "driver": "TrafficAnalyticsRuleDriver", - "config": {} + "config": { + "filterPatterns": ["\\.stripe\\.com"] + } }], "AppMessage": [{ "locale": "en_US", @@ -85,7 +105,14 @@ {"name": "view.last_7_days", "value": "Last 7 Days"}, {"name": "view.last_7_days.ctime.format", "value": "{{MMM}} {{d}}, {{YYYY}}"}, {"name": "view.last_30_days", "value": "Last 30 Days"}, - {"name": "view.last_30_days.ctime.format", "value": "{{MMM}} {{d}}, {{YYYY}}"} + {"name": "view.last_30_days.ctime.format", "value": "{{MMM}} {{d}}, {{YYYY}}"}, + + {"name": "config.view.manageFilters", "value": "Manage Filters"}, + {"name": "config.field.analyticsFilter", "value": "Filter Pattern"}, + {"name": "config.field.analyticsFilter.description", "value": "Skip logging for URLs that match this pattern"}, + {"name": "config.action.addFilter", "value": "Add New Filter"}, + {"name": "config.button.addFilter", "value": "Add"}, + {"name": "config.action.removeFilter", "value": "Remove"} ] }] } diff --git a/bubble-web b/bubble-web index 7cd7e0e1..995c996c 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 7cd7e0e140da2d048ad3addfac4bceb72ab5e89a +Subproject commit 995c996c444bad30eca592b71ce8dd0daa3f5a05