create rules from recent traffic

bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsAppDataDriver.java

@@ -1,7 +1,9 @@
 package bubble.app.analytics;
 
 import bubble.model.account.Account;
-import bubble.model.app.*;
+import bubble.model.app.AppData;
+import bubble.model.app.AppSite;
+import bubble.model.app.BubbleApp;
 import bubble.model.app.config.AppDataDriverBase;
 import bubble.model.app.config.AppDataView;
 import bubble.model.device.Device;
@@ -9,6 +11,7 @@ import bubble.rule.analytics.TrafficAnalyticsRuleDriver;
 import bubble.rule.analytics.TrafficRecord;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.network.NetworkUtil;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.cobbzilla.wizard.dao.SearchResults;
 import org.cobbzilla.wizard.model.search.SearchBoundComparison;
@@ -18,15 +21,20 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormatter;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 
-import static bubble.rule.analytics.TrafficAnalyticsRuleDriver.FQDN_SEP;
-import static bubble.rule.analytics.TrafficAnalyticsRuleDriver.RECENT_TRAFFIC_PREFIX;
+import static bubble.rule.analytics.TrafficAnalyticsRuleDriver.*;
 import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.HOURS;
+import static org.apache.commons.lang3.RandomUtils.nextInt;
 import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
+import static org.cobbzilla.util.http.HttpContentTypes.TYPICAL_WEB_TYPES;
+import static org.cobbzilla.util.http.HttpContentTypes.fileExt;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.string.StringUtil.PCT;
 import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_YYYY_MM_DD;
@@ -35,6 +43,7 @@ import static org.cobbzilla.wizard.model.search.SearchBoundComparison.Constants.
 import static org.cobbzilla.wizard.model.search.SearchField.OP_SEP;
 import static org.cobbzilla.wizard.model.search.SortOrder.ASC;
 import static org.cobbzilla.wizard.model.search.SortOrder.DESC;
+import static org.cobbzilla.wizard.util.TestNames.*;
 
 @Slf4j
 public class TrafficAnalyticsAppDataDriver extends AppDataDriverBase {
@@ -47,16 +56,23 @@ public class TrafficAnalyticsAppDataDriver extends AppDataDriverBase {
     public static final SearchSort SORT_TSTAMP_DESC = new SearchSort("meta1", DESC);
     public static final SearchSort SORT_FQDN_CASE_INSENSITIVE_ASC = new SearchSort("meta2", ASC, "lower");
 
+    public static final int MAX_RECENT_PAGE_SIZE = 50;
+
     @Getter(lazy=true) private final RedisService recentTraffic = redis.prefixNamespace(RECENT_TRAFFIC_PREFIX);
 
     @Override public SearchResults query(Account caller, Device device, BubbleApp app, AppSite site, AppDataView view, SearchQuery query) {
 
+        if (configuration.testMode()) recordTestTraffic(caller, device);
+
         if (view.getName().equals(VIEW_recent)) {
             final RedisService traffic = getRecentTraffic();
             final TreeSet<String> keys = new TreeSet<>(Collections.reverseOrder());
             keys.addAll(traffic.keys("*"));
             final List<TrafficRecord> records = new ArrayList<>();
             int i = 0;
+            if (query.getPageSize() > MAX_RECENT_PAGE_SIZE) {
+                query.setPageSize(MAX_RECENT_PAGE_SIZE);
+            }
             for (String key : keys) {
                 if (i < query.getPageOffset()) {
                     i++;
@@ -73,8 +89,12 @@ public class TrafficAnalyticsAppDataDriver extends AppDataDriverBase {
                         log.warn("query: error parsing TrafficRecord: "+shortError(e));
                     }
                 }
+                if (records.size() >= query.getPageSize()) {
+                    log.info("query: max page size reached "+query.getPageSize()+", breaking");
+                }
                 i++;
             }
+            log.info("query: VIEW_recent: returning "+records.size()+" / "+keys.size()+" recent traffic records");
             return new SearchResults(records, keys.size());
         }
 
@@ -92,6 +112,21 @@ public class TrafficAnalyticsAppDataDriver extends AppDataDriverBase {
         return processResults(searchService.search(false, caller, dataDAO, query));
     }
 
+    private void recordTestTraffic(Account caller, Device device) {
+        recordRecentTraffic(new TrafficRecord()
+                        .setRequestTime(now())
+                        .setIp(NetworkUtil.getFirstPublicIpv4())
+                        .setFqdn(safeNationality()+".example.com")
+                        .setUri("/traffic/"+safeColor()+"/"+ safeAnimal()
+                                + fileExt(TYPICAL_WEB_TYPES[nextInt(0, TYPICAL_WEB_TYPES.length)]))
+                        .setReferer("NONE")
+                        .setAccountUuid(caller.getUuid())
+                        .setAccountName(caller.getName())
+                        .setDeviceUuid(device.getUuid())
+                        .setDeviceName(device.getName()),
+                getRecentTraffic());
+    }
+
     private SearchResults processResults(SearchResults searchResults) {
         final List<TrafficAnalyticsData> data = new ArrayList<>();
         if (searchResults.hasResults()) {

bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java

@@ -55,7 +55,7 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
     @Autowired private BubbleConfiguration configuration;
 
     @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
-        final String id = params.get(PARAM_ID);
+        String id = params.get(PARAM_ID);
         switch (view) {
             case VIEW_manageLists:
                 return loadAllLists(account, app);
@@ -65,7 +65,11 @@ public class BubbleBlockAppConfigDriver implements AppConfigDriver {
                 return loadList(account, app, id);
 
             case VIEW_manageRules:
-                if (empty(id)) throw notFoundEx(id);
+                if (empty(id)) {
+                    final BubbleBlockList builtinList = getBuiltinList(account, app);
+                    if (builtinList == null) throw notFoundEx(id);
+                    id = builtinList.getId();
+                }
                 return loadListEntries(account, app, id);
         }
         throw notFoundEx(view);

bubble-server/src/main/java/bubble/model/app/BubbleApp.java

@@ -3,11 +3,13 @@ package bubble.model.app;
 import bubble.model.account.Account;
 import bubble.model.account.AccountTemplate;
 import bubble.model.app.config.AppDataConfig;
+import bubble.model.app.config.AppDataField;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.Accessors;
+import org.cobbzilla.util.collection.ArrayUtil;
 import org.cobbzilla.wizard.model.Identifiable;
 import org.cobbzilla.wizard.model.entityconfig.IdentifiableBaseParentEntity;
 import org.cobbzilla.wizard.model.entityconfig.annotations.*;
@@ -78,10 +80,19 @@ public class BubbleApp extends IdentifiableBaseParentEntity implements AccountTe
     @Column(length=100000, nullable=false) @ECField(index=50)
     @JsonIgnore @Getter @Setter private String dataConfigJson;
 
-    @Transient public AppDataConfig getDataConfig () { return dataConfigJson == null ? null : json(dataConfigJson, AppDataConfig.class); }
+    @Transient public AppDataConfig getDataConfig () { return dataConfigJson == null ? null : ensureDefaults(json(dataConfigJson, AppDataConfig.class)); }
     public BubbleApp setDataConfig (AppDataConfig adc) { return setDataConfigJson(adc == null ? null : json(adc, DB_JSON_MAPPER)); }
     public boolean hasDataConfig () { return getDataConfig() != null; }
 
+    private AppDataConfig ensureDefaults(AppDataConfig adc) {
+        for (AppDataField field : adc.getFields()) {
+            if (!adc.hasConfigField(field)) {
+                adc.setConfigFields(ArrayUtil.append(adc.getConfigFields(), field));
+            }
+        }
+        return adc;
+    }
+
     @ECSearchable @ECField(index=60)
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean template = false;

bubble-server/src/main/java/bubble/model/app/config/AppConfigAction.java

@@ -11,6 +11,7 @@ public class AppConfigAction {
     @Getter @Setter private AppConfigScope scope = AppConfigScope.item;
     @Getter @Setter private String when;
     @Getter @Setter private String view;
+    @Getter @Setter private String dataView;
     @Getter @Setter private String successView;
     @Getter @Setter private String successMessage;
     @Getter @Setter private Integer index = 0;

bubble-server/src/main/java/bubble/model/app/config/AppDataAction.java

@@ -7,5 +7,6 @@ public class AppDataAction {
 
     @Getter @Setter private String name;
     @Getter @Setter private String when;
+    @Getter @Setter private String route;
 
 }

bubble-server/src/main/java/bubble/model/app/config/AppDataConfig.java

@@ -4,6 +4,7 @@ import bubble.server.BubbleConfiguration;
 import lombok.Getter;
 import lombok.Setter;
 
+import java.util.Arrays;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -48,6 +49,10 @@ public class AppDataConfig {
     @Getter @Setter private AppDataField[] configFields;
     public boolean hasConfigFields () { return !empty(configFields); }
 
+    public boolean hasConfigField(AppDataField field) {
+        return hasConfigFields() && Arrays.stream(getConfigFields()).anyMatch(f -> f.getName().equals(field.getName()));
+    }
+
     @Getter @Setter private AppConfigView[] configViews;
     public boolean hasConfigViews () { return !empty(configViews); }
 

bubble-server/src/main/java/bubble/model/app/config/AppDataDriverBase.java

@@ -5,6 +5,7 @@ import bubble.model.account.Account;
 import bubble.model.app.AppSite;
 import bubble.model.app.BubbleApp;
 import bubble.model.device.Device;
+import bubble.server.BubbleConfiguration;
 import bubble.service.SearchService;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.cobbzilla.wizard.dao.SearchResults;
@@ -24,6 +25,7 @@ public abstract class AppDataDriverBase implements AppDataDriver {
     @Autowired protected AppDataDAO dataDAO;
     @Autowired protected SearchService searchService;
     @Autowired protected RedisService redis;
+    @Autowired protected BubbleConfiguration configuration;
 
     @Override public SearchResults query(Account caller, Device device, BubbleApp app, AppSite site, AppDataView view, SearchQuery query) {
         query.setBound("app", app.getUuid());

bubble-server/src/main/java/bubble/model/app/config/AppDataView.java

@@ -7,6 +7,7 @@ import org.cobbzilla.util.collection.HasPriority;
 public class AppDataView implements HasPriority {
 
     @Getter @Setter private AppDataPresentation presentation = AppDataPresentation.app;
+    @Getter @Setter private AppDataViewLayout layout = AppDataViewLayout.table;
     @Getter @Setter private String name;
     @Getter @Setter private Integer priority = 0;
 

bubble-server/src/main/java/bubble/model/app/config/AppDataViewLayout.java

@@ -0,0 +1,13 @@
+package bubble.model.app.config;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+import static bubble.ApiConstants.enumFromString;
+
+public enum AppDataViewLayout {
+
+    table, tiles;
+
+    @JsonCreator public static AppDataViewLayout fromString (String v) { return enumFromString(AppDataViewLayout.class, v); }
+
+}

bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalyticsRuleDriver.java

@@ -43,11 +43,18 @@ public class TrafficAnalyticsRuleDriver extends AbstractAppRuleDriver {
         final String site = ruleHarness.getMatcher().getSite();
         final String fqdn = filter.getFqdn();
 
-        getRecentTraffic().set(now()+"_"+randomAlphanumeric(10), json(new TrafficRecord(filter, account, device, req)), EX, RECENT_TRAFFIC_EXPIRATION);
+        final TrafficRecord rec = new TrafficRecord(filter, account, device, req);
+        recordRecentTraffic(rec);
         incrementCounters(account, device, app, site, fqdn);
         return FilterMatchResponse.NO_MATCH; // we are done, don't need to look at/modify stream
     }
 
+    public void recordRecentTraffic(TrafficRecord rec) { recordRecentTraffic(rec, getRecentTraffic()); }
+
+    public static void recordRecentTraffic(TrafficRecord rec, RedisService recentTraffic) {
+        recentTraffic.set(now() + "_" + randomAlphanumeric(10), json(rec), EX, RECENT_TRAFFIC_EXPIRATION);
+    }
+
     public void incrementCounters(Account account, Device device, String app, String site, String fqdn) {
         incr(account, device, app, site, fqdn, PREFIX_HOURLY, DATE_FORMAT_YYYY_MM_DD_HH.print(now()));
         incr(account, null, app, site, fqdn, PREFIX_HOURLY, DATE_FORMAT_YYYY_MM_DD_HH.print(now()));

bubble-server/src/main/java/bubble/rule/analytics/TrafficRecord.java

@@ -6,13 +6,14 @@ import bubble.resources.stream.FilterMatchersRequest;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import lombok.experimental.Accessors;
 import org.glassfish.grizzly.http.server.Request;
 
 import static bubble.ApiConstants.getRemoteHost;
 import static java.util.UUID.randomUUID;
 import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 
-@NoArgsConstructor
+@NoArgsConstructor @Accessors(chain=true)
 public class TrafficRecord {
 
     @Getter @Setter private String uuid = randomUUID().toString();

bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java

@@ -100,8 +100,9 @@ public class DeviceIdService {
             log.warn("findDeviceByIp("+ipAddr+") test mode and no admin devices, returning dummy device");
             return new Device().setAccount(adminUuid).setName("dummy");
         } else {
-            log.warn("findDeviceByIp("+ipAddr+") test mode, returning first admin device");
-            return adminDevices.get(0);
+            log.warn("findDeviceByIp("+ipAddr+") test mode, returning and possibly initializing first admin device");
+            final Device device = adminDevices.get(0);
+            return device.uninitialized() ? deviceDAO.update(device.initTotpKey()) : device;
         }
     }
 

bubble-server/src/main/java/bubble/service/message/AppMessageService.java

@@ -92,7 +92,6 @@ public class AppMessageService {
                         ensureFieldNameAndDescription(props, cfgKeyPrefix, field.getName());
                     }
                 }
-
                 if (cfg.hasConfigViews()) {
                     for (AppConfigView configView : cfg.getConfigViews()) {
                         final String viewKey = cfgKeyPrefix + MSG_SUFFIX_VIEW + configView.getName();
@@ -120,6 +119,11 @@ public class AppMessageService {
                         }
                     }
                 }
+
+                // anything from data fields not yet defined, copy as config field name/desc
+                for (AppDataField field : cfg.getFields()) {
+                    ensureFieldNameAndDescription(props, cfgKeyPrefix, field.getName());
+                }
             }
         }
         return props;

bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties

@@ -11,6 +11,8 @@ message_undefined=undefined
 # Display of percent values has localized variations
 label_percent={{percent}}%
 
+message_truncated_show_ellipsis={{msg}}...
+
 # Date/Calendar names
 label_date={{MMM}} {{d}}, {{YYYY}}
 label_date_short={{M}}/{{d}}/{{YYYY}}

bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json

@@ -24,8 +24,19 @@
       {"name": "device", "required": false, "index": 10, "when": "view !== \"recent\""},
       {"name": "meta2", "required": false, "operator": "like", "index": 20, "when": "view !== \"recent\""}
     ],
+    "actions": [
+      {
+        "name": "filterHost",
+        "when": "view === \"recent\"",
+        "route": "/app/BubbleBlock/config/manageRules/local?action=createRule&rule={{fqdn}}"
+      }, {
+        "name": "filterUrl",
+        "when": "view === \"recent\"",
+        "route": "/app/BubbleBlock/config/manageRules/local?action=createRule&rule={{ encodeURIComponent( fqdn + (uri.startsWith('/') ? uri : '/'+uri) ) }}"
+      }
+    ],
     "views": [
-      {"name": "recent"},
+      {"name": "recent", "layout": "tiles"},
       {"name": "last_24_hours"},
       {"name": "last_7_days"},
       {"name": "last_30_days"}
@@ -47,12 +58,13 @@
     "AppMessage": [{
       "locale": "en_US",
       "messages": [
-        {"name": "name", "value": "Stoolpidgeon"},
-        {"name": "description", "value": "Review recent internet traffic for your devices. Block stuff that looks off."},
+        {"name": "name", "value": "Castigator"},
+        {"name": "summary", "value": "Network Analytics and Filter Creator"},
+        {"name": "description", "value": "Review recent internet traffic for your devices. Block stuff that you don't like."},
         {"name": "field.ctime", "value": "When"},
         {"name": "field.requestTime", "value": "When"},
         {"name": "field.accountName", "value": "Account"},
-        {"name": "field.fqdn", "value": "URL"},
+        {"name": "field.fqdn", "value": "Host"},
         {"name": "field.device", "value": "Device"},
         {"name": "field.deviceName", "value": "Device"},
         {"name": "field.ip", "value": "From IP"},
@@ -62,6 +74,8 @@
         {"name": "field.data", "value": "Count"},
         {"name": "param.meta2", "value": "Site"},
         {"name": "param.device", "value": "Device"},
+        {"name": "action.filterHost", "value": "Block Host"},
+        {"name": "action.filterUrl", "value": "Block URL"},
         {"name": "view.recent", "value": "Recent Traffic"},
         {"name": "view.recent.requestTime.format", "value": "{{MM}} {{d}} @ {{h}}:{{m}}:{{s}} {{a}}"},
         {"name": "view.last_24_hours", "value": "Last 24 Hours"},

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json

@@ -46,7 +46,7 @@
         {"name": "disableList", "when": "item.enabled", "index": 20},
         {"name": "manageList", "view": "manageList", "index": 30},
         {"name": "manageRules", "view": "manageRules", "when": "item.url === ''", "index": 40},
-        {"name": "removeList", "index": 50, "when": "item.url !==''"},
+        {"name": "removeList", "index": 50, "when": "item.url !== ''"},
         {
           "name": "createList", "scope": "app", "view": "manageList", "index": 10,
           "params": ["url"],
@@ -131,7 +131,8 @@
     "AppMessage": [{
       "locale": "en_US",
       "messages": [
-        {"name": "name", "value": "BlockParty!"},
+        {"name": "name", "value": "Block Party!"},
+        {"name": "summary", "value": "Network Filter and Content Blocker"},
         {"name": "description", "value": "Block adware, malware, phishing/scam sites, and much more"},
         {"name": "field.ctime", "value": "When"},
         {"name": "field.fqdn", "value": "URL"},

bubble-server/src/main/resources/models/apps/user_block/bubbleApp_userBlock.json

@@ -22,7 +22,8 @@
     "AppMessage": [{
       "locale": "en_US",
       "messages": [
-        {"name": "name", "value": "ShadowBan"},
+        {"name": "name", "value": "Shadow Ban"},
+        {"name": "summary", "value": "User Blocker"},
         {"name": "description", "value": "Throw the garbage to the curb!"},
         {"name": "view.blocked_users", "value": "Manage Blocked Users"},
         {"name": "field.key", "value": "Username"},

bubble-web

@@ -1 +1 @@
-Subproject commit 47b7bc1d0500c5fc429acda124b0d16aa95d1f89
+Subproject commit c7a9253a2b7bfaf9af0cc78a9430acc5337416dd

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit b68eecfa79696b72b7242de27de530c4e0112c28
+Subproject commit e7c3727fc3e1405b3bba6b95de0271db23309e14

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 6a3824eee748ded81eb4caf065f444f565708b1c
+Subproject commit 0c7796f86e508d38f48a94270f52d3444ecf720c