start adding support for blocklist management, generic app config management

bubble-server/src/main/java/bubble/abp/BlockListType.java

@@ -0,0 +1,11 @@
+package bubble.abp;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+public enum BlockListType {
+
+    ads, malware, privacy, clickbait, gambling, phishing, nsfw, crypto, annoyances, custom;
+
+    @JsonCreator public BlockListType fromString (String v) { return valueOf(v.toLowerCase()); }
+
+}

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java

@@ -54,20 +54,28 @@ public class BubbleBlock extends TrafficAnalytics {
         super.init(config, userConfig, rule, matcher, account, device);
 
         bubbleBlockConfig = json(json(config), BubbleBlockConfig.class);
-        for (String listUrl : bubbleBlockConfig.getBlockLists()) {
-            BlockListSource blockListSource = blockListCache.get(listUrl);
-            if (blockListSource == null || blockListSource.age() > TimeUnit.DAYS.toMillis(5)) {
-                try {
-                    final BlockListSource newList = new BlockListSource()
-                            .setUrl(listUrl)
-                            .download();
-                    blockListCache.put(listUrl, newList);
-                    blockListSource = newList;
-                } catch (Exception e) {
-                    log.error("init: error downloading blocklist "+listUrl+": "+shortError(e));
-                    continue;
+        for (BubbleBlockList list : bubbleBlockConfig.getBlockLists()) {
+            if (!list.enabled()) continue;
+
+            BlockListSource blockListSource = blockListCache.get(list.getId());
+            if (list.hasUrl()) {
+                final String listUrl = list.getUrl();
+                if (blockListSource == null || blockListSource.age() > TimeUnit.DAYS.toMillis(5)) {
+                    try {
+                        final BlockListSource newList = new BlockListSource()
+                                .setUrl(listUrl)
+                                .download();
+                        blockListCache.put(list.getId(), newList);
+                        blockListSource = newList;
+                    } catch (Exception e) {
+                        log.error("init: error downloading blocklist " + listUrl + ": " + shortError(e));
+                        continue;
+                    }
                 }
             }
+            if (list.hasAdditionalEntries()) {
+                blockListSource.addEntries(list.getAdditionalEntries());
+            }
             blockList.merge(blockListSource.getBlockList());
         }
     }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java

@@ -7,6 +7,6 @@ import lombok.Setter;
 @NoArgsConstructor
 public class BubbleBlockConfig {
 
-    @Getter @Setter private String[] blockLists;
+    @Getter @Setter private BubbleBlockList[] blockLists;
 
 }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockList.java

@@ -0,0 +1,25 @@
+package bubble.rule.bblock;
+
+import bubble.abp.BlockListType;
+import lombok.Getter;
+import lombok.Setter;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+
+public class BubbleBlockList {
+
+    @Getter @Setter private String id;
+    @Getter @Setter private String name;
+    @Getter @Setter private String description;
+    @Getter @Setter private BlockListType[] tags;
+
+    @Getter @Setter private String url;
+    public boolean hasUrl () { return !empty(url); }
+
+    @Getter @Setter private String[] additionalEntries;
+    public boolean hasAdditionalEntries () { return !empty(additionalEntries); }
+
+    @Getter @Setter private Boolean enabled = true;
+    public boolean enabled() { return enabled != null && enabled; }
+
+}

bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java

@@ -186,6 +186,8 @@ public class RuleEngineService {
 
     private ExpirationMap<String, List<AppRuleHarness>> ruleCache = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime);
 
+    public void flushRuleCache () { ruleCache.clear(); }
+
     private List<AppRuleHarness> initRules(Account account, Device device, String[] matcherIds) {
         final String cacheKey = hashOf(account.getUuid(), device.getUuid(), matcherIds);
         return ruleCache.computeIfAbsent(cacheKey, k -> {

bubble-server/src/main/resources/models/apps/bblock/bubbleApp_bblock.json

@@ -22,7 +22,50 @@
       {"name": "last_24_hours"},
       {"name": "last_7_days"},
       {"name": "last_30_days"}
-    ]
+    ],
+    "configViews": [{
+      "name": "manage_blocklists",
+      "type": "list",
+      "columns": ["name", "description", "url", "enabled", "tags"],
+      "actions": [
+        {"name": "enable", "when": "!item.enabled"},
+        {"name": "disable", "when": "item.enabled"},
+        {"name": "manage_blocklist", "view": "manage_blocklist"},
+        {"name": "remove"},
+        {"name": "add", "type": "new", "view": "manage_blocklist"}
+      ]
+    }, {
+      "name": "manage_blocklist",
+      "type": "item",
+      "fields": [
+        {"name": "name"},
+        {"name": "description", "control": "textarea"},
+        {"name": "url", "type": "http_url"},
+        {"name": "tags"},
+        {"name": "enabled", "type": "flag"}
+      ],
+      "actions": [
+        {"name": "enable", "when": "!item.enabled"},
+        {"name": "disable", "when": "item.enabled"},
+        {"name": "remove", "when": "item.id !== null"},
+        {"name": "create", "when": "item.id === null"},
+        {"name": "update", "when": "item.id !== null"},
+        {"name": "manage_entries", "when": "item.id !== null"}
+      ]
+    }, {
+      "name": "manage_entries",
+      "type": "list",
+      "columns": ["block_rule"],
+      "actions": [
+        {"name": "remove"},
+        {"name": "add", "type": "new", "view": "add_entry"}
+      ]
+    }, {
+      "name": "add_entry",
+      "type": "item",
+      "fields": [ {"name": "block_rule"} ],
+      "actions": ["create"]
+    }]
   },
   "children": {
     "AppSite": [{
@@ -38,8 +81,27 @@
       "priority": -1000,
       "config": {
         "blockLists": [
-          "https://v.firebog.net/hosts/Easylist.txt",
-          "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareABP.txt"
+          {
+            "name": "EasyList",
+            "id": "easylist",
+            "url": "https://v.firebog.net/hosts/Easylist.txt",
+            "description": "EasyList is the primary filter list that removes most adverts from international web pages, including unwanted frames, images, and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists.",
+            "tags": ["ads"]
+          },
+          {
+            "name": "Dandelion Sprout's Anti-Malware List",
+            "id": "dandelion_sprouts_anti_malware_list",
+            "url": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareABP.txt",
+            "description": "Most anti-malware lists are pretty big and can cover a 5- or 6-digit amount of specific domains. But my list hereby claims to remove more than 25% of all known malware sites with just a 2-digit amount of entries. This is mostly done by blocking top-level domains that have become devastatingly abused by spammers, usually because they allowed for free and uncontrolled domain registrations. There's also additional categories that cover unusual malware and phishing domains that very few other lists seem to cover.",
+            "tags": ["malware", "phishing"]
+          },
+          {
+            "name": "Bubble Custom List",
+            "id": "local",
+            "url": "",
+            "description": "A place to maintain your own block rules for this Bubble.",
+            "tags": ["custom"]
+          }
         ]
       }
     }],
@@ -58,7 +120,14 @@
         {"name": "view.last_7_days", "value": "Last 7 Days"},
         {"name": "view.last_7_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"},
         {"name": "view.last_30_days", "value": "Last 30 Days"},
-        {"name": "view.last_30_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"}
+        {"name": "view.last_30_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"},
+
+        {"name": "list.easylist.name", "value": "EasyList"},
+        {"name": "list.easylist.description", "value": "EasyList is the primary filter list that removes most adverts from international web pages, including unwanted frames, images, and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists."},
+        {"name": "list.local.name", "value": "Bubble Custom List"},
+        {"name": "list.local.description", "value": "A place to maintain your own block rules for this Bubble."},
+        {"name": "list.dandelion_sprouts_anti_malware_list.name", "value": "Dandelion Sprout's Anti-Malware List"},
+        {"name": "list.dandelion_sprouts_anti_malware_list.description", "value": "Most anti-malware lists are pretty big and can cover a 5- or 6-digit amount of specific domains. But my list hereby claims to remove more than 25% of all known malware sites with just a 2-digit amount of entries. This is mostly done by blocking top-level domains that have become devastatingly abused by spammers, usually because they allowed for free and uncontrolled domain registrations. There's also additional categories that cover unusual malware and phishing domains that very few other lists seem to cover."}
       ]
     }]
   }

utils/abp-parser

@@ -1 +1 @@
-Subproject commit ab3185282fb6258c662fda4fd6d09a038a6560ea
+Subproject commit a374c29e86120ba20e6daf63cdf37ee2c8a3f0c2