diff --git a/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsApp.java b/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsApp.java index f9101f94..2f98331c 100644 --- a/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsApp.java +++ b/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsApp.java @@ -6,6 +6,9 @@ import bubble.model.device.Device; import org.cobbzilla.wizard.dao.SearchResults; import org.cobbzilla.wizard.model.search.SearchBoundComparison; import org.cobbzilla.wizard.model.search.SearchQuery; +import org.cobbzilla.wizard.model.search.SearchSort; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; import java.util.ArrayList; import java.util.List; @@ -17,6 +20,7 @@ import static org.cobbzilla.util.string.StringUtil.PCT; import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_YYYY_MM_DD_HH; import static org.cobbzilla.wizard.model.search.SearchBoundComparison.Constants.ILIKE_SEP; import static org.cobbzilla.wizard.model.search.SearchField.OP_SEP; +import static org.cobbzilla.wizard.model.search.SortOrder.ASC; public class TrafficAnalyticsApp extends AppDataDriverBase { @@ -24,8 +28,19 @@ public class TrafficAnalyticsApp extends AppDataDriverBase { public static final String VIEW_last_7_days = "last_7_days"; public static final String VIEW_last_30_days = "last_30_days"; + public static final SearchSort SORT_TSTAMP_ASC = new SearchSort("meta1"); + public static final SearchSort SORT_FQDN_CASE_INSENSITIVE_ASC = new SearchSort("meta2", ASC, "lower"); + @Override public SearchResults query(Account caller, Device device, BubbleApp app, AppSite site, AppDataView view, SearchQuery query) { - query = query.setBound("key", getBound(view)); + final String deviceBound = device == null + ? SearchBoundComparison.is_null.name() + OP_SEP + : SearchBoundComparison.eq.name() + OP_SEP + device.getUuid(); + query.setBound("device", deviceBound); + query.setBound("key", getKeyBound(view)); + if (!query.hasSorts()) { + query.addSort(SORT_TSTAMP_ASC); + query.addSort(SORT_FQDN_CASE_INSENSITIVE_ASC); + } return processResults(searchService.search(false, caller, dataDAO, query)); } @@ -39,7 +54,7 @@ public class TrafficAnalyticsApp extends AppDataDriverBase { return searchResults.setResults(data); } - private String getBound(AppDataView view) { + private String getKeyBound(AppDataView view) { final StringBuilder b; final long now = now(); final int limit; @@ -53,7 +68,7 @@ public class TrafficAnalyticsApp extends AppDataDriverBase { b = new StringBuilder(); for (int i = 0; i< limit; i++) { if (b.length() > 0) b.append(ILIKE_SEP); - b.append(PCT + FQDN_SEP).append(DATE_FORMAT_YYYY_MM_DD_HH.print(now - HOURS.toMillis(i))).append(PCT); + b.append(PCT + FQDN_SEP).append(DATE_FORMAT_YYYY_MM_DD_HH.print(new DateTime().withZone(DateTimeZone.UTC).plus(-1 * HOURS.toMillis(i)))).append(PCT); } return SearchBoundComparison.like_any.name() + OP_SEP + b.toString(); diff --git a/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsData.java b/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsData.java index a95756e7..ce219e8e 100644 --- a/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsData.java +++ b/bubble-server/src/main/java/bubble/app/analytics/TrafficAnalyticsData.java @@ -5,20 +5,17 @@ import bubble.model.device.Device; import lombok.Getter; import lombok.Setter; -import static bubble.rule.analytics.TrafficAnalytics.FQDN_SEP; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; public class TrafficAnalyticsData extends AppData { public TrafficAnalyticsData(AppData data) { copy(this, data); - final String[] parts = getKey().split(FQDN_SEP); - this.fqdn = parts[0]; - this.timeInterval = parts[1]; + this.fqdn = data.getMeta2(); final Device device = data.getRelated().entity(Device.class); if (device != null) setDevice(device.getName()); } @Getter @Setter private String fqdn; - @Getter @Setter private String timeInterval; + } diff --git a/bubble-server/src/main/java/bubble/model/app/AppData.java b/bubble-server/src/main/java/bubble/model/app/AppData.java index 54f8743b..08ee6e29 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppData.java +++ b/bubble-server/src/main/java/bubble/model/app/AppData.java @@ -27,6 +27,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; +import static org.cobbzilla.wizard.model.entityconfig.annotations.ECForeignKeySearchDepth.shallow; @ECType(root=true, pluralDisplayName="App Data") @ECTypeURIs(baseURI= EP_DATA, listFields={"app", "key", "data", "expiration"}) @@ -75,7 +76,7 @@ public class AppData extends IdentifiableBase implements AppTemplateEntity { @Column(nullable=false, updatable=false, length=UUID_MAXLEN) @Getter @Setter private String account; - @ECSearchable(fkDepth=ECForeignKeySearchDepth.shallow) @ECField(index=20) + @ECSearchable(fkDepth=shallow) @ECField(index=20) @ECForeignKey(entity=Device.class) @Column(updatable=false, length=UUID_MAXLEN) @Getter @Setter private String device; @@ -137,6 +138,15 @@ public class AppData extends IdentifiableBase implements AppTemplateEntity { @ECIndex @Column(nullable=false) @Getter @Setter private Boolean enabled = true; + @ECSearchable @ECField(index=120) + @Column(length=1000) @Getter @Setter private String meta1; + + @ECSearchable @ECField(index=130) + @Column(length=1000) @Getter @Setter private String meta2; + + @ECSearchable @ECField(index=140) + @Column(length=1000) @Getter @Setter private String meta3; + @JsonProperty @Override public long getCtime () { return super.getCtime(); } @JsonProperty @Override public long getMtime () { return super.getMtime(); } diff --git a/bubble-server/src/main/java/bubble/model/app/AppDataConfig.java b/bubble-server/src/main/java/bubble/model/app/AppDataConfig.java index 0c86334f..c805d7f1 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppDataConfig.java +++ b/bubble-server/src/main/java/bubble/model/app/AppDataConfig.java @@ -19,7 +19,7 @@ public class AppDataConfig { @Getter @Setter private String driverClass; public boolean hasDriverClass () { return driverClass != null; } - @Getter @Setter private EntityFieldConfig[] fields; + @Getter @Setter private AppDataField[] fields; public boolean hasFields () { return !empty(fields); } @Getter @Setter private EntityFieldConfig[] params; diff --git a/bubble-server/src/main/java/bubble/model/app/AppDataDriverBase.java b/bubble-server/src/main/java/bubble/model/app/AppDataDriverBase.java index b45daa0f..30f9d6ab 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppDataDriverBase.java +++ b/bubble-server/src/main/java/bubble/model/app/AppDataDriverBase.java @@ -6,19 +6,25 @@ import bubble.model.device.Device; import bubble.service.SearchService; import org.cobbzilla.wizard.dao.SearchResults; import org.cobbzilla.wizard.model.search.SearchQuery; +import org.cobbzilla.wizard.model.search.SearchSort; import org.springframework.beans.factory.annotation.Autowired; +import static org.cobbzilla.wizard.model.search.SortOrder.ASC; + public abstract class AppDataDriverBase implements AppDataDriver { + public static final SearchSort CASE_INSENSITIVE_KEY_SORT_ASC = new SearchSort("key", ASC, "lower"); + public static final SearchSort CTIME_SORT_ASC = new SearchSort("ctime"); + + public static final SearchSort DEFAULT_SORT = CASE_INSENSITIVE_KEY_SORT_ASC; + @Autowired protected AppDataDAO dataDAO; @Autowired protected SearchService searchService; @Override public SearchResults query(Account caller, Device device, BubbleApp app, AppSite site, AppDataView view, SearchQuery query) { query.setBound("app", app.getUuid()); - if (!query.getHasSortField()) { - query.setSortOrder(SearchQuery.SortOrder.ASC); - query.setSortField("lower(key)"); - } + if (site != null) query.setBound("site", site.getUuid()); + if (!query.hasSorts()) query.addSort(DEFAULT_SORT); return searchService.search(false, caller, dataDAO, query); } diff --git a/bubble-server/src/main/java/bubble/model/app/AppDataField.java b/bubble-server/src/main/java/bubble/model/app/AppDataField.java new file mode 100644 index 00000000..cf395ce1 --- /dev/null +++ b/bubble-server/src/main/java/bubble/model/app/AppDataField.java @@ -0,0 +1,11 @@ +package bubble.model.app; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.wizard.model.entityconfig.EntityFieldConfig; + +public class AppDataField extends EntityFieldConfig { + + @Getter @Setter private Boolean customFormat; + +} diff --git a/bubble-server/src/main/java/bubble/model/app/AppMessage.java b/bubble-server/src/main/java/bubble/model/app/AppMessage.java index 69f2dd5b..35a33d2e 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppMessage.java +++ b/bubble-server/src/main/java/bubble/model/app/AppMessage.java @@ -23,6 +23,7 @@ import static bubble.ApiConstants.EP_MESSAGES; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; +import static org.cobbzilla.wizard.model.entityconfig.annotations.ECForeignKeySearchDepth.shallow; @ECType(root=true) @ECTypeURIs(baseURI=EP_MESSAGES, listFields={"app", "locale"}) @@ -45,7 +46,7 @@ public class AppMessage extends IdentifiableBase implements AccountTemplate, Has @Column(nullable=false, updatable=false, length=UUID_MAXLEN) @Getter @Setter private String account; - @ECSearchable(fkDepth=ECForeignKeySearchDepth.shallow) @ECField(index=20) + @ECSearchable(fkDepth=shallow) @ECField(index=20) @ECForeignKey(entity=BubbleApp.class) @Column(nullable=false, updatable=false, length=UUID_MAXLEN) @Getter @Setter private String app; diff --git a/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalytics.java b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalytics.java index 5eac51a2..f4231934 100644 --- a/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalytics.java +++ b/bubble-server/src/main/java/bubble/rule/analytics/TrafficAnalytics.java @@ -29,8 +29,8 @@ public class TrafficAnalytics extends AbstractAppRuleDriver { final String site = ruleHarness.getMatcher().getSite(); final String fqdn = filter.getFqdn(); - incr(account, device, app, site, fqdn + FQDN_SEP + DATE_FORMAT_YYYY_MM_DD_HH.print(now())); - incr(account, null, app, site, fqdn + FQDN_SEP + DATE_FORMAT_YYYY_MM_DD_HH.print(now())); + incr(account, device, app, site, fqdn, DATE_FORMAT_YYYY_MM_DD_HH.print(now())); + incr(account, null, app, site, fqdn, DATE_FORMAT_YYYY_MM_DD_HH.print(now())); return true; } @@ -39,7 +39,8 @@ public class TrafficAnalytics extends AbstractAppRuleDriver { // is that we miss a few increments, hopefully not a huge deal in the big picture. The real bad case is // if the underlying db driver gets into an upset state because of the concurrent updates. We will cross // that bridge when we get to it. - private synchronized void incr(Account account, Device device, String app, String site, String key) { + private synchronized void incr(Account account, Device device, String app, String site, String fqdn, String tstamp) { + final String key = fqdn + FQDN_SEP + tstamp; final AppData found = appDataDAO.findByAppAndSiteAndKeyAndDevice(app, site, key, device == null ? null : device.getUuid()); if (found == null) { appDataDAO.create(new AppData() @@ -47,6 +48,8 @@ public class TrafficAnalytics extends AbstractAppRuleDriver { .setSite(matcher.getSite()) .setMatcher(matcher.getUuid()) .setKey(key) + .setMeta1(tstamp) + .setMeta2(fqdn) .setAccount(account.getUuid()) .setDevice(device == null ? null : device.getUuid()) .setExpiration(now() + ANALYTICS_EXPIRATION) diff --git a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java index 60c8393e..fcca3fc7 100644 --- a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java @@ -12,6 +12,7 @@ import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.NetworkMonitorService; import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.wizard.dao.AbstractDAO; import org.cobbzilla.wizard.server.RestServer; import org.cobbzilla.wizard.server.RestServerLifecycleListenerBase; @@ -52,6 +53,13 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase dao instanceof AbstractDAO) + .forEach(dao -> ((AbstractDAO) dao).getSearchView()); + } + // ensure system configs can be loaded properly final Map configs = c.getPublicSystemConfigs(); if (empty(configs)) die("onStart: no system configs found"); // should never happen diff --git a/bubble-server/src/main/java/bubble/service/SearchService.java b/bubble-server/src/main/java/bubble/service/SearchService.java index f2e59486..78962b8c 100644 --- a/bubble-server/src/main/java/bubble/service/SearchService.java +++ b/bubble-server/src/main/java/bubble/service/SearchService.java @@ -4,7 +4,6 @@ import bubble.ApiConstants; import bubble.model.account.Account; import bubble.server.BubbleConfiguration; import bubble.service.cloud.GeoService; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ExpirationEvictionPolicy; import org.cobbzilla.util.collection.ExpirationMap; @@ -12,6 +11,7 @@ import org.cobbzilla.wizard.dao.AbstractDAO; import org.cobbzilla.wizard.dao.DAO; import org.cobbzilla.wizard.dao.SearchResults; import org.cobbzilla.wizard.model.search.SearchQuery; +import org.cobbzilla.wizard.model.search.SearchSort; import org.cobbzilla.wizard.model.search.SqlViewField; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -75,9 +75,7 @@ public class SearchService { q.setPageNumber(page != null ? page : 1); q.setPageSize(size != null ? Integer.min(size, MAX_SEARCH_PAGE) : Integer.min(q.getPageSize(), MAX_SEARCH_PAGE)); if (sort != null) { - final SearchService.SortAndOrder s = new SearchService.SortAndOrder(sort); - q.setSortField(s.getSortField()); - q.setSortOrder(s.getSortOrder()); + q.addSort(new SearchSort(sort)); } if (filter != null) q.setFilter(filter); @@ -120,26 +118,4 @@ public class SearchService { return dao.search(q); } - public static class SortAndOrder { - @Getter private final String sortField; - @Getter private final SearchQuery.SortOrder sortOrder; - public SortAndOrder(String sort) { - if (sort.startsWith("+") || sort.startsWith(" ")) { - sortField = sort.substring(1).trim(); - sortOrder = SearchQuery.SortOrder.ASC; - } else if (sort.startsWith("-")) { - sortField = sort.substring(1).trim(); - sortOrder = SearchQuery.SortOrder.DESC; - } else if (sort.endsWith("+") || sort.endsWith(" ")) { - sortField = sort.substring(0, sort.length()-1).trim(); - sortOrder = SearchQuery.SortOrder.ASC; - } else if (sort.endsWith("-")) { - sortField = sort.substring(0, sort.length()-1).trim(); - sortOrder = SearchQuery.SortOrder.DESC; - } else { - sortField = sort.trim(); - sortOrder = SearchQuery.SortOrder.ASC; - } - } - } } diff --git a/bubble-server/src/main/resources/ansible/default_roles.json b/bubble-server/src/main/resources/ansible/default_roles.json index 288ce140..c93cb6e1 100644 --- a/bubble-server/src/main/resources/ansible/default_roles.json +++ b/bubble-server/src/main/resources/ansible/default_roles.json @@ -24,16 +24,19 @@ "priority": 300, "template": true, "config": [ - {"name": "dns_port", "value": "[[configuration.dnsPort]]"}, {"name": "node_uuid", "value": "[[node.uuid]]"}, {"name": "network_uuid", "value": "[[node.network]]"}, {"name": "admin_port", "value": "[[node.adminPort]]"}, {"name": "ssl_port", "value": "[[node.sslPort]]"}, + {"name": "public_base_uri", "value": "[[publicBaseUri]]"}, {"name": "sage_node", "value": "[[sageNode]]"}, {"name": "install_type", "value": "[[installType]]"}, + {"name": "default_locale", "value": "[[network.locale]]"}, + {"name": "time_zone", "value": "[[network.timezone]]"}, {"name": "bubble_version", "value": "[[configuration.version]]"}, {"name": "bubble_host", "value": "[[node.fqdn]]"}, - {"name": "admin_user", "value": "[[node.user]]"}, + {"name": "bubble_cname", "value": "[[network.networkDomain]]"}, + {"name": "admin_user", "value": "[[node.ansibleUser]]"}, {"name": "db_encoding", "value": "UTF-8"}, {"name": "db_locale", "value": "en_US"}, {"name": "db_user", "value": "bubble"}, @@ -43,9 +46,12 @@ {"name": "is_fork", "value": "[[fork]]"}, {"name": "restore_key", "value": "[[restoreKey]]"}, {"name": "restore_timeout", "value": "[[restoreTimeoutSeconds]]"}, - {"name": "test_mode", "value": "[[testMode]]"} + {"name": "test_mode", "value": "[[testMode]]"}, + {"name": "errbit_url", "value": "[[#compare fork '==' true]][[configuration.errorApi.url]][[/compare]]"}, + {"name": "errbit_key", "value": "[[#compare fork '==' true]][[configuration.errorApi.key]][[/compare]]"}, + {"name": "errbit_env", "value": "[[#compare fork '==' true]][[node.fqdn]][[/compare]]"} ], - "optionalConfigNames": ["restore_key", "restore_timeout"], + "optionalConfigNames": ["restore_key", "restore_timeout", "errbit_url", "errbit_key", "errbit_env"], "tgzB64": "" }, { diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlocker.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlocker.js.hbs index 95d5b007..2dae9bc8 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlocker.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlocker.js.hbs @@ -52,7 +52,6 @@ function {{JS_PREFIX}}_block_user (author) { {{{APPLY_BLOCKS_JS}}} {{JS_PREFIX}}_onReady(function() { - {{JS_PREFIX}}_blocked_users = []; const bubbleControlDiv = document.createElement('div'); bubbleControlDiv.style.position = 'fixed'; bubbleControlDiv.style.bottom = '0'; diff --git a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json index 9b7843b6..72544dbb 100644 --- a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json +++ b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json @@ -8,7 +8,7 @@ "driverClass": "bubble.app.analytics.TrafficAnalyticsApp", "presentation": "app", "fields": [ - {"name": "timeInterval"}, + {"name": "ctime", "customFormat": true}, {"name": "fqdn"}, {"name": "device"}, {"name": "data"} @@ -41,15 +41,18 @@ "messages": [ {"name": "name", "value": "Traffic Analytics"}, {"name": "description", "value": "Traffic analytics for your Bubble. Manage block lists."}, - {"name": "field.timeInterval", "value": "When"}, + {"name": "field.ctime", "value": "When"}, {"name": "field.fqdn", "value": "URL"}, {"name": "field.device", "value": "Device"}, {"name": "field.data", "value": "Count"}, {"name": "param.site", "value": "Site"}, {"name": "param.device", "value": "Device"}, {"name": "view.last_24_hours", "value": "Last 24 Hours"}, + {"name": "view.last_24_hours.ctime.format", "value": "{{MM}} {{d}} @ {{h}}{{a}}"}, {"name": "view.last_7_days", "value": "Last 7 Days"}, - {"name": "view.last_30_days", "value": "Last 30 Days"} + {"name": "view.last_7_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"}, + {"name": "view.last_30_days", "value": "Last 30 Days"}, + {"name": "view.last_30_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"} ] }] } diff --git a/bubble-web b/bubble-web index 37f50c47..e69d6dfe 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 37f50c4764bfa243aaa26b80a1772574e29679e2 +Subproject commit e69d6dfeab3dd979f52e81b43c4a30da0d6b6411 diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 516376d0..415f7509 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 516376d0591bc89cdb02061604f6161223fc583b +Subproject commit 415f75098dad61e3c5ff41d88d49d4fbbf7d0747