diff --git a/bubble-server/src/main/java/bubble/abp/BlockListType.java b/bubble-server/src/main/java/bubble/abp/BlockListType.java new file mode 100644 index 00000000..50c6c6f3 --- /dev/null +++ b/bubble-server/src/main/java/bubble/abp/BlockListType.java @@ -0,0 +1,11 @@ +package bubble.abp; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum BlockListType { + + ads, malware, privacy, clickbait, gambling, phishing, nsfw, crypto, annoyances, custom; + + @JsonCreator public BlockListType fromString (String v) { return valueOf(v.toLowerCase()); } + +} diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java index 008e2a3a..f02375e1 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlock.java @@ -54,20 +54,28 @@ public class BubbleBlock extends TrafficAnalytics { super.init(config, userConfig, rule, matcher, account, device); bubbleBlockConfig = json(json(config), BubbleBlockConfig.class); - for (String listUrl : bubbleBlockConfig.getBlockLists()) { - BlockListSource blockListSource = blockListCache.get(listUrl); - if (blockListSource == null || blockListSource.age() > TimeUnit.DAYS.toMillis(5)) { - try { - final BlockListSource newList = new BlockListSource() - .setUrl(listUrl) - .download(); - blockListCache.put(listUrl, newList); - blockListSource = newList; - } catch (Exception e) { - log.error("init: error downloading blocklist "+listUrl+": "+shortError(e)); - continue; + for (BubbleBlockList list : bubbleBlockConfig.getBlockLists()) { + if (!list.enabled()) continue; + + BlockListSource blockListSource = blockListCache.get(list.getId()); + if (list.hasUrl()) { + final String listUrl = list.getUrl(); + if (blockListSource == null || blockListSource.age() > TimeUnit.DAYS.toMillis(5)) { + try { + final BlockListSource newList = new BlockListSource() + .setUrl(listUrl) + .download(); + blockListCache.put(list.getId(), newList); + blockListSource = newList; + } catch (Exception e) { + log.error("init: error downloading blocklist " + listUrl + ": " + shortError(e)); + continue; + } } } + if (list.hasAdditionalEntries()) { + blockListSource.addEntries(list.getAdditionalEntries()); + } blockList.merge(blockListSource.getBlockList()); } } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java index f50bd89a..1379a464 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java @@ -7,6 +7,6 @@ import lombok.Setter; @NoArgsConstructor public class BubbleBlockConfig { - @Getter @Setter private String[] blockLists; + @Getter @Setter private BubbleBlockList[] blockLists; } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockList.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockList.java new file mode 100644 index 00000000..21a2ffa1 --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockList.java @@ -0,0 +1,25 @@ +package bubble.rule.bblock; + +import bubble.abp.BlockListType; +import lombok.Getter; +import lombok.Setter; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class BubbleBlockList { + + @Getter @Setter private String id; + @Getter @Setter private String name; + @Getter @Setter private String description; + @Getter @Setter private BlockListType[] tags; + + @Getter @Setter private String url; + public boolean hasUrl () { return !empty(url); } + + @Getter @Setter private String[] additionalEntries; + public boolean hasAdditionalEntries () { return !empty(additionalEntries); } + + @Getter @Setter private Boolean enabled = true; + public boolean enabled() { return enabled != null && enabled; } + +} diff --git a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java index 56a1bc6b..626b844f 100644 --- a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java @@ -186,6 +186,8 @@ public class RuleEngineService { private ExpirationMap> ruleCache = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime); + public void flushRuleCache () { ruleCache.clear(); } + private List initRules(Account account, Device device, String[] matcherIds) { final String cacheKey = hashOf(account.getUuid(), device.getUuid(), matcherIds); return ruleCache.computeIfAbsent(cacheKey, k -> { diff --git a/bubble-server/src/main/resources/models/apps/bblock/bubbleApp_bblock.json b/bubble-server/src/main/resources/models/apps/bblock/bubbleApp_bblock.json index 199b97f1..f308412b 100644 --- a/bubble-server/src/main/resources/models/apps/bblock/bubbleApp_bblock.json +++ b/bubble-server/src/main/resources/models/apps/bblock/bubbleApp_bblock.json @@ -22,7 +22,50 @@ {"name": "last_24_hours"}, {"name": "last_7_days"}, {"name": "last_30_days"} - ] + ], + "configViews": [{ + "name": "manage_blocklists", + "type": "list", + "columns": ["name", "description", "url", "enabled", "tags"], + "actions": [ + {"name": "enable", "when": "!item.enabled"}, + {"name": "disable", "when": "item.enabled"}, + {"name": "manage_blocklist", "view": "manage_blocklist"}, + {"name": "remove"}, + {"name": "add", "type": "new", "view": "manage_blocklist"} + ] + }, { + "name": "manage_blocklist", + "type": "item", + "fields": [ + {"name": "name"}, + {"name": "description", "control": "textarea"}, + {"name": "url", "type": "http_url"}, + {"name": "tags"}, + {"name": "enabled", "type": "flag"} + ], + "actions": [ + {"name": "enable", "when": "!item.enabled"}, + {"name": "disable", "when": "item.enabled"}, + {"name": "remove", "when": "item.id !== null"}, + {"name": "create", "when": "item.id === null"}, + {"name": "update", "when": "item.id !== null"}, + {"name": "manage_entries", "when": "item.id !== null"} + ] + }, { + "name": "manage_entries", + "type": "list", + "columns": ["block_rule"], + "actions": [ + {"name": "remove"}, + {"name": "add", "type": "new", "view": "add_entry"} + ] + }, { + "name": "add_entry", + "type": "item", + "fields": [ {"name": "block_rule"} ], + "actions": ["create"] + }] }, "children": { "AppSite": [{ @@ -38,8 +81,27 @@ "priority": -1000, "config": { "blockLists": [ - "https://v.firebog.net/hosts/Easylist.txt", - "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareABP.txt" + { + "name": "EasyList", + "id": "easylist", + "url": "https://v.firebog.net/hosts/Easylist.txt", + "description": "EasyList is the primary filter list that removes most adverts from international web pages, including unwanted frames, images, and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists.", + "tags": ["ads"] + }, + { + "name": "Dandelion Sprout's Anti-Malware List", + "id": "dandelion_sprouts_anti_malware_list", + "url": "https://raw.githubusercontent.com/DandelionSprout/adfilt/master/Alternate%20versions%20Anti-Malware%20List/AntiMalwareABP.txt", + "description": "Most anti-malware lists are pretty big and can cover a 5- or 6-digit amount of specific domains. But my list hereby claims to remove more than 25% of all known malware sites with just a 2-digit amount of entries. This is mostly done by blocking top-level domains that have become devastatingly abused by spammers, usually because they allowed for free and uncontrolled domain registrations. There's also additional categories that cover unusual malware and phishing domains that very few other lists seem to cover.", + "tags": ["malware", "phishing"] + }, + { + "name": "Bubble Custom List", + "id": "local", + "url": "", + "description": "A place to maintain your own block rules for this Bubble.", + "tags": ["custom"] + } ] } }], @@ -58,7 +120,14 @@ {"name": "view.last_7_days", "value": "Last 7 Days"}, {"name": "view.last_7_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"}, {"name": "view.last_30_days", "value": "Last 30 Days"}, - {"name": "view.last_30_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"} + {"name": "view.last_30_days.ctime.format", "value": "{{MM}} {{d}}, {{YYYY}}"}, + + {"name": "list.easylist.name", "value": "EasyList"}, + {"name": "list.easylist.description", "value": "EasyList is the primary filter list that removes most adverts from international web pages, including unwanted frames, images, and objects. It is the most popular list used by many ad blockers and forms the basis of over a dozen combination and supplementary filter lists."}, + {"name": "list.local.name", "value": "Bubble Custom List"}, + {"name": "list.local.description", "value": "A place to maintain your own block rules for this Bubble."}, + {"name": "list.dandelion_sprouts_anti_malware_list.name", "value": "Dandelion Sprout's Anti-Malware List"}, + {"name": "list.dandelion_sprouts_anti_malware_list.description", "value": "Most anti-malware lists are pretty big and can cover a 5- or 6-digit amount of specific domains. But my list hereby claims to remove more than 25% of all known malware sites with just a 2-digit amount of entries. This is mostly done by blocking top-level domains that have become devastatingly abused by spammers, usually because they allowed for free and uncontrolled domain registrations. There's also additional categories that cover unusual malware and phishing domains that very few other lists seem to cover."} ] }] } diff --git a/utils/abp-parser b/utils/abp-parser index ab318528..a374c29e 160000 --- a/utils/abp-parser +++ b/utils/abp-parser @@ -1 +1 @@ -Subproject commit ab3185282fb6258c662fda4fd6d09a038a6560ea +Subproject commit a374c29e86120ba20e6daf63cdf37ee2c8a3f0c2