traffic analysis needs to store fqdn and tstamp in meta columns

bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsApp.java

@@ -6,6 +6,9 @@ import bubble.model.device.Device;
 import org.cobbzilla.wizard.dao.SearchResults;
 import org.cobbzilla.wizard.model.search.SearchBoundComparison;
 import org.cobbzilla.wizard.model.search.SearchQuery;
+import org.cobbzilla.wizard.model.search.SearchSort;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -17,6 +20,7 @@ import static org.cobbzilla.util.string.StringUtil.PCT;
 import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_YYYY_MM_DD_HH;
 import static org.cobbzilla.wizard.model.search.SearchBoundComparison.Constants.ILIKE_SEP;
 import static org.cobbzilla.wizard.model.search.SearchField.OP_SEP;
+import static org.cobbzilla.wizard.model.search.SortOrder.ASC;
 
 public class TrafficAnalyticsApp extends AppDataDriverBase {
 
@@ -24,8 +28,19 @@ public class TrafficAnalyticsApp extends AppDataDriverBase {
     public static final String VIEW_last_7_days = "last_7_days";
     public static final String VIEW_last_30_days = "last_30_days";
 
+    public static final SearchSort SORT_TSTAMP_ASC = new SearchSort("meta1");
+    public static final SearchSort SORT_FQDN_CASE_INSENSITIVE_ASC = new SearchSort("meta2", ASC, "lower");
+
     @Override public SearchResults query(Account caller, Device device, BubbleApp app, AppSite site, AppDataView view, SearchQuery query) {
-        query = query.setBound("key", getBound(view));
+        final String deviceBound = device == null
+                ? SearchBoundComparison.is_null.name() + OP_SEP
+                : SearchBoundComparison.eq.name() + OP_SEP + device.getUuid();
+        query.setBound("device", deviceBound);
+        query.setBound("key", getKeyBound(view));
+        if (!query.hasSorts()) {
+            query.addSort(SORT_TSTAMP_ASC);
+            query.addSort(SORT_FQDN_CASE_INSENSITIVE_ASC);
+        }
         return processResults(searchService.search(false, caller, dataDAO, query));
     }
 
@@ -39,7 +54,7 @@ public class TrafficAnalyticsApp extends AppDataDriverBase {
         return searchResults.setResults(data);
     }
 
-    private String getBound(AppDataView view) {
+    private String getKeyBound(AppDataView view) {
         final StringBuilder b;
         final long now = now();
         final int limit;
@@ -53,7 +68,7 @@ public class TrafficAnalyticsApp extends AppDataDriverBase {
         b = new StringBuilder();
         for (int i = 0; i< limit; i++) {
             if (b.length() > 0) b.append(ILIKE_SEP);
-            b.append(PCT + FQDN_SEP).append(DATE_FORMAT_YYYY_MM_DD_HH.print(now - HOURS.toMillis(i))).append(PCT);
+            b.append(PCT + FQDN_SEP).append(DATE_FORMAT_YYYY_MM_DD_HH.print(new DateTime().withZone(DateTimeZone.UTC).plus(-1 * HOURS.toMillis(i)))).append(PCT);
         }
 
         return SearchBoundComparison.like_any.name() + OP_SEP + b.toString();

bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsData.java

@@ -5,20 +5,17 @@ import bubble.model.device.Device;
 import lombok.Getter;
 import lombok.Setter;
 
-import static bubble.rule.analytics.TrafficAnalytics.FQDN_SEP;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 
 public class TrafficAnalyticsData extends AppData {
 
     public TrafficAnalyticsData(AppData data) {
         copy(this, data);
-        final String[] parts = getKey().split(FQDN_SEP);
-        this.fqdn = parts[0];
-        this.timeInterval = parts[1];
+        this.fqdn = data.getMeta2();
         final Device device = data.getRelated().entity(Device.class);
         if (device != null) setDevice(device.getName());
     }
 
     @Getter @Setter private String fqdn;
-    @Getter @Setter private String timeInterval;
+
 }

bubble-server/src/main/java/bubble/model/app/AppData.java

@@ -27,6 +27,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING;
 import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
+import static org.cobbzilla.wizard.model.entityconfig.annotations.ECForeignKeySearchDepth.shallow;
 
 @ECType(root=true, pluralDisplayName="App Data")
 @ECTypeURIs(baseURI= EP_DATA, listFields={"app", "key", "data", "expiration"})
@@ -75,7 +76,7 @@ public class AppData extends IdentifiableBase implements AppTemplateEntity {
     @Column(nullable=false, updatable=false, length=UUID_MAXLEN)
     @Getter @Setter private String account;
 
-    @ECSearchable(fkDepth=ECForeignKeySearchDepth.shallow) @ECField(index=20)
+    @ECSearchable(fkDepth=shallow) @ECField(index=20)
     @ECForeignKey(entity=Device.class)
     @Column(updatable=false, length=UUID_MAXLEN)
     @Getter @Setter private String device;
@@ -137,6 +138,15 @@ public class AppData extends IdentifiableBase implements AppTemplateEntity {
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean enabled = true;
 
+    @ECSearchable @ECField(index=120)
+    @Column(length=1000) @Getter @Setter private String meta1;
+
+    @ECSearchable @ECField(index=130)
+    @Column(length=1000) @Getter @Setter private String meta2;
+
+    @ECSearchable @ECField(index=140)
+    @Column(length=1000) @Getter @Setter private String meta3;
+
     @JsonProperty @Override public long getCtime () { return super.getCtime(); }
     @JsonProperty @Override public long getMtime () { return super.getMtime(); }
 

bubble-server/src/main/java/bubble/model/app/AppDataConfig.java

@@ -19,7 +19,7 @@ public class AppDataConfig {
     @Getter @Setter private String driverClass;
     public boolean hasDriverClass () { return driverClass != null; }
 
-    @Getter @Setter private EntityFieldConfig[] fields;
+    @Getter @Setter private AppDataField[] fields;
     public boolean hasFields () { return !empty(fields); }
 
     @Getter @Setter private EntityFieldConfig[] params;

bubble-server/src/main/java/bubble/model/app/AppDataDriverBase.java

@@ -6,19 +6,25 @@ import bubble.model.device.Device;
 import bubble.service.SearchService;
 import org.cobbzilla.wizard.dao.SearchResults;
 import org.cobbzilla.wizard.model.search.SearchQuery;
+import org.cobbzilla.wizard.model.search.SearchSort;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import static org.cobbzilla.wizard.model.search.SortOrder.ASC;
+
 public abstract class AppDataDriverBase implements AppDataDriver {
 
+    public static final SearchSort CASE_INSENSITIVE_KEY_SORT_ASC = new SearchSort("key", ASC, "lower");
+    public static final SearchSort CTIME_SORT_ASC = new SearchSort("ctime");
+
+    public static final SearchSort DEFAULT_SORT = CASE_INSENSITIVE_KEY_SORT_ASC;
+
     @Autowired protected AppDataDAO dataDAO;
     @Autowired protected SearchService searchService;
 
     @Override public SearchResults query(Account caller, Device device, BubbleApp app, AppSite site, AppDataView view, SearchQuery query) {
         query.setBound("app", app.getUuid());
-        if (!query.getHasSortField()) {
-            query.setSortOrder(SearchQuery.SortOrder.ASC);
-            query.setSortField("lower(key)");
-        }
+        if (site != null) query.setBound("site", site.getUuid());
+        if (!query.hasSorts()) query.addSort(DEFAULT_SORT);
         return searchService.search(false, caller, dataDAO, query);
     }
 

bubble-server/src/main/java/bubble/model/app/AppDataField.java

@@ -0,0 +1,11 @@
+package bubble.model.app;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.cobbzilla.wizard.model.entityconfig.EntityFieldConfig;
+
+public class AppDataField extends EntityFieldConfig {
+
+    @Getter @Setter private Boolean customFormat;
+
+}

bubble-server/src/main/java/bubble/model/app/AppMessage.java

@@ -23,6 +23,7 @@ import static bubble.ApiConstants.EP_MESSAGES;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+import static org.cobbzilla.wizard.model.entityconfig.annotations.ECForeignKeySearchDepth.shallow;
 
 @ECType(root=true)
 @ECTypeURIs(baseURI=EP_MESSAGES, listFields={"app", "locale"})
@@ -45,7 +46,7 @@ public class AppMessage extends IdentifiableBase implements AccountTemplate, Has
     @Column(nullable=false, updatable=false, length=UUID_MAXLEN)
     @Getter @Setter private String account;
 
-    @ECSearchable(fkDepth=ECForeignKeySearchDepth.shallow) @ECField(index=20)
+    @ECSearchable(fkDepth=shallow) @ECField(index=20)
     @ECForeignKey(entity=BubbleApp.class)
     @Column(nullable=false, updatable=false, length=UUID_MAXLEN)
     @Getter @Setter private String app;

bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalytics.java

@@ -29,8 +29,8 @@ public class TrafficAnalytics extends AbstractAppRuleDriver {
         final String site = ruleHarness.getMatcher().getSite();
         final String fqdn = filter.getFqdn();
 
-        incr(account, device, app, site, fqdn + FQDN_SEP + DATE_FORMAT_YYYY_MM_DD_HH.print(now()));
-        incr(account, null, app, site, fqdn + FQDN_SEP + DATE_FORMAT_YYYY_MM_DD_HH.print(now()));
+        incr(account, device, app, site, fqdn, DATE_FORMAT_YYYY_MM_DD_HH.print(now()));
+        incr(account, null, app, site, fqdn, DATE_FORMAT_YYYY_MM_DD_HH.print(now()));
         return true;
     }
 
@@ -39,7 +39,8 @@ public class TrafficAnalytics extends AbstractAppRuleDriver {
     // is that we miss a few increments, hopefully not a huge deal in the big picture. The real bad case is
     // if the underlying db driver gets into an upset state because of the concurrent updates. We will cross
     // that bridge when we get to it.
-    private synchronized void incr(Account account, Device device, String app, String site, String key) {
+    private synchronized void incr(Account account, Device device, String app, String site, String fqdn, String tstamp) {
+        final String key = fqdn + FQDN_SEP + tstamp;
         final AppData found = appDataDAO.findByAppAndSiteAndKeyAndDevice(app, site, key, device == null ? null : device.getUuid());
         if (found == null) {
             appDataDAO.create(new AppData()
@@ -47,6 +48,8 @@ public class TrafficAnalytics extends AbstractAppRuleDriver {
                     .setSite(matcher.getSite())
                     .setMatcher(matcher.getUuid())
                     .setKey(key)
+                    .setMeta1(tstamp)
+                    .setMeta2(fqdn)
                     .setAccount(account.getUuid())
                     .setDevice(device == null ? null : device.getUuid())
                     .setExpiration(now() + ANALYTICS_EXPIRATION)

bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java

@@ -12,6 +12,7 @@ import bubble.server.BubbleConfiguration;
 import bubble.service.boot.SelfNodeService;
 import bubble.service.cloud.NetworkMonitorService;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.wizard.dao.AbstractDAO;
 import org.cobbzilla.wizard.server.RestServer;
 import org.cobbzilla.wizard.server.RestServerLifecycleListenerBase;
 
@@ -52,6 +53,13 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase<Bub
     @Override public void onStart(RestServer server) {
         final BubbleConfiguration c = (BubbleConfiguration) server.getConfiguration();
 
+        // ensure all search views can be created
+        if (!c.testMode()) {
+            c.getAllDAOs().stream()
+                    .filter(dao -> dao instanceof AbstractDAO)
+                    .forEach(dao -> ((AbstractDAO) dao).getSearchView());
+        }
+
         // ensure system configs can be loaded properly
         final Map<String, Object> configs = c.getPublicSystemConfigs();
         if (empty(configs)) die("onStart: no system configs found");  // should never happen

bubble-server/src/main/java/bubble/service/SearchService.java

@@ -4,7 +4,6 @@ import bubble.ApiConstants;
 import bubble.model.account.Account;
 import bubble.server.BubbleConfiguration;
 import bubble.service.cloud.GeoService;
-import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.ExpirationEvictionPolicy;
 import org.cobbzilla.util.collection.ExpirationMap;
@@ -12,6 +11,7 @@ import org.cobbzilla.wizard.dao.AbstractDAO;
 import org.cobbzilla.wizard.dao.DAO;
 import org.cobbzilla.wizard.dao.SearchResults;
 import org.cobbzilla.wizard.model.search.SearchQuery;
+import org.cobbzilla.wizard.model.search.SearchSort;
 import org.cobbzilla.wizard.model.search.SqlViewField;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -75,9 +75,7 @@ public class SearchService {
         q.setPageNumber(page != null ? page : 1);
         q.setPageSize(size != null ? Integer.min(size, MAX_SEARCH_PAGE) : Integer.min(q.getPageSize(), MAX_SEARCH_PAGE));
         if (sort != null) {
-            final SearchService.SortAndOrder s = new SearchService.SortAndOrder(sort);
-            q.setSortField(s.getSortField());
-            q.setSortOrder(s.getSortOrder());
+            q.addSort(new SearchSort(sort));
         }
         if (filter != null) q.setFilter(filter);
 
@@ -120,26 +118,4 @@ public class SearchService {
         return dao.search(q);
     }
 
-    public static class SortAndOrder {
-        @Getter private final String sortField;
-        @Getter private final SearchQuery.SortOrder sortOrder;
-        public SortAndOrder(String sort) {
-            if (sort.startsWith("+") || sort.startsWith(" ")) {
-                sortField = sort.substring(1).trim();
-                sortOrder = SearchQuery.SortOrder.ASC;
-            } else if (sort.startsWith("-")) {
-                sortField = sort.substring(1).trim();
-                sortOrder = SearchQuery.SortOrder.DESC;
-            } else if (sort.endsWith("+") || sort.endsWith(" ")) {
-                sortField = sort.substring(0, sort.length()-1).trim();
-                sortOrder = SearchQuery.SortOrder.ASC;
-            } else if (sort.endsWith("-")) {
-                sortField = sort.substring(0, sort.length()-1).trim();
-                sortOrder = SearchQuery.SortOrder.DESC;
-            } else {
-                sortField = sort.trim();
-                sortOrder = SearchQuery.SortOrder.ASC;
-            }
-        }
-    }
 }

bubble-server/src/main/resources/ansible/default_roles.json

@@ -24,16 +24,19 @@
     "priority": 300,
     "template": true,
     "config": [
-      {"name": "dns_port", "value": "[[configuration.dnsPort]]"},
       {"name": "node_uuid", "value": "[[node.uuid]]"},
       {"name": "network_uuid", "value": "[[node.network]]"},
       {"name": "admin_port", "value": "[[node.adminPort]]"},
       {"name": "ssl_port", "value": "[[node.sslPort]]"},
+      {"name": "public_base_uri", "value": "[[publicBaseUri]]"},
       {"name": "sage_node", "value": "[[sageNode]]"},
       {"name": "install_type", "value": "[[installType]]"},
+      {"name": "default_locale", "value": "[[network.locale]]"},
+      {"name": "time_zone", "value": "[[network.timezone]]"},
       {"name": "bubble_version", "value": "[[configuration.version]]"},
       {"name": "bubble_host", "value": "[[node.fqdn]]"},
-      {"name": "admin_user", "value": "[[node.user]]"},
+      {"name": "bubble_cname", "value": "[[network.networkDomain]]"},
+      {"name": "admin_user", "value": "[[node.ansibleUser]]"},
       {"name": "db_encoding", "value": "UTF-8"},
       {"name": "db_locale", "value": "en_US"},
       {"name": "db_user", "value": "bubble"},
@@ -43,9 +46,12 @@
       {"name": "is_fork", "value": "[[fork]]"},
       {"name": "restore_key", "value": "[[restoreKey]]"},
       {"name": "restore_timeout", "value": "[[restoreTimeoutSeconds]]"},
-      {"name": "test_mode", "value": "[[testMode]]"}
+      {"name": "test_mode", "value": "[[testMode]]"},
+      {"name": "errbit_url", "value": "[[#compare fork '==' true]][[configuration.errorApi.url]][[/compare]]"},
+      {"name": "errbit_key", "value": "[[#compare fork '==' true]][[configuration.errorApi.key]][[/compare]]"},
+      {"name": "errbit_env", "value": "[[#compare fork '==' true]][[node.fqdn]][[/compare]]"}
     ],
-    "optionalConfigNames": ["restore_key", "restore_timeout"],
+    "optionalConfigNames": ["restore_key", "restore_timeout", "errbit_url", "errbit_key", "errbit_env"],
     "tgzB64": ""
   },
   {

bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlocker.js.hbs

@@ -52,7 +52,6 @@ function {{JS_PREFIX}}_block_user (author) {
 {{{APPLY_BLOCKS_JS}}}
 
 {{JS_PREFIX}}_onReady(function() {
-    {{JS_PREFIX}}_blocked_users = [];
     const bubbleControlDiv = document.createElement('div');
     bubbleControlDiv.style.position = 'fixed';
     bubbleControlDiv.style.bottom = '0';

bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json

@@ -8,7 +8,7 @@
     "driverClass": "bubble.app.analytics.TrafficAnalyticsApp",
     "presentation": "app",
     "fields": [
-      {"name": "timeInterval"},
+      {"name": "ctime", "customFormat": true},
       {"name": "fqdn"},
       {"name": "device"},
       {"name": "data"}
@@ -41,15 +41,18 @@
       "messages": [
         {"name": "name", "value": "Traffic Analytics"},
         {"name": "description", "value": "Traffic analytics for your Bubble. Manage block lists."},
-        {"name": "field.timeInterval", "value": "When"},
+        {"name": "field.ctime", "value": "When"},
         {"name": "field.fqdn", "value": "URL"},
         {"name": "field.device", "value": "Device"},
         {"name": "field.data", "value": "Count"},
         {"name": "param.site", "value": "Site"},
         {"name": "param.device", "value": "Device"},
         {"name": "view.last_24_hours", "value": "Last 24 Hours"},
+        {"name": "view.last_24_hours.ctime.format", "value": "{{MM}} {{d}} @ {{h}}{{a}}"},
         {"name": "view.last_7_days", "value": "Last 7 Days"},
-        {"name": "view.last_30_days", "value": "Last 30 Days"}
+        {"name": "view.last_7_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"},
+        {"name": "view.last_30_days", "value": "Last 30 Days"},
+        {"name": "view.last_30_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"}
       ]
     }]
   }

bubble-web

@@ -1 +1 @@
-Subproject commit 37f50c4764bfa243aaa26b80a1772574e29679e2
+Subproject commit e69d6dfeab3dd979f52e81b43c4a30da0d6b6411

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 516376d0591bc89cdb02061604f6161223fc583b
+Subproject commit 415f75098dad61e3c5ff41d88d49d4fbbf7d0747