Merge branch 'master' into kris/request_protector_app

# Conflicts:
#	utils/cobbzilla-utils

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -253,6 +253,7 @@ public class ApiConstants {
     public static final String EP_UPGRADE = "/upgrade";
     public static final String EP_LOGS = "/logs";
     public static final String EP_FOLLOW = "/follow";
+    public static final String EP_FOLLOW_AND_APPLY_REGEX = "/followAndApplyRegex";
 
     public static final String DETECT_ENDPOINT = "/detect";
     public static final String EP_LOCALE = "/locale";

bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java

@@ -4,8 +4,11 @@
  */
 package bubble.dao.account;
 
+import bubble.dao.bill.AccountPlanDAO;
+import bubble.dao.cloud.BubbleNetworkDAO;
 import bubble.model.account.Account;
 import bubble.model.account.AccountSshKey;
+import bubble.model.bill.AccountPlan;
 import bubble.model.cloud.AnsibleInstallType;
 import bubble.model.cloud.BubbleNetwork;
 import bubble.server.BubbleConfiguration;
@@ -14,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
 import java.io.File;
+import java.util.List;
 
 import static bubble.ApiConstants.HOME_DIR;
 import static org.cobbzilla.util.io.FileUtil.touch;
@@ -25,6 +29,8 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 public class AccountSshKeyDAO extends AccountOwnedEntityDAO<AccountSshKey> {
 
     @Autowired private AccountDAO accountDAO;
+    @Autowired private AccountPlanDAO accountPlanDAO;
+    @Autowired private BubbleNetworkDAO networkDAO;
     @Autowired private BubbleConfiguration configuration;
 
     public AccountSshKey findByAccountAndHash(String accountUuid, String hash) {
@@ -90,6 +96,19 @@ public class AccountSshKeyDAO extends AccountOwnedEntityDAO<AccountSshKey> {
 
     @Override public void delete(String uuid) {
         final AccountSshKey key = findByUuid(uuid);
+
+        // remove from any AccountPlans that reference it
+        final List<AccountPlan> accountPlans = accountPlanDAO.findByAccountAndSshKey(key.getAccount(), key.getUuid());
+        for (AccountPlan plan : accountPlans) {
+            accountPlanDAO.update(plan.setSshKey(null));
+        }
+
+        // remove from any BubbleNetworks that reference it
+        final List<BubbleNetwork> bubbleNetworks = networkDAO.findByAccountAndSshKey(key.getAccount(), key.getUuid());
+        for (BubbleNetwork network : bubbleNetworks) {
+            networkDAO.update(network.setSshKey(null));
+        }
+
         super.delete(uuid);
         if (key.installSshKey()) refreshInstalledKeys();
     }

bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java

@@ -63,6 +63,10 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO<AccountPlan> {
 
     public AccountPlan findByNetwork(String networkUuid) { return findByUniqueField("network", networkUuid); }
 
+    public List<AccountPlan> findByAccountAndSshKey(String account, String keyUuid) {
+        return findByFields("account", account, "sshKey", keyUuid);
+    }
+
     public List<AccountPlan> findByAccountAndNotDeleted(String account) {
         return findByFields("account", account, "deleting", false, "deleted", null);
     }

bubble-server/src/main/java/bubble/dao/cloud/BubbleNetworkDAO.java

@@ -92,6 +92,10 @@ public class BubbleNetworkDAO extends AccountOwnedEntityDAO<BubbleNetwork> {
         return findByUniqueFields("name", name, "domain", domainUuid);
     }
 
+    public List<BubbleNetwork> findByAccountAndSshKey(String account, String keyUuid) {
+        return findByFields("account", account, "sshKey", keyUuid);
+    }
+
     @Override public void delete(String uuid) {
         final BubbleNetwork network = findByUuid(uuid);
         if (network == null) return;

bubble-server/src/main/java/bubble/model/app/AppMatcher.java

@@ -44,7 +44,8 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
 public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, HasPriority {
 
     public static final String[] VALUE_FIELDS = {
-            "fqdn", "urlRegex", "template", "enabled", "priority", "connCheck", "requestCheck", "requestModifier"
+            "fqdn", "urlRegex", "userAgentRegex", "template", "enabled", "priority",
+            "connCheck", "requestCheck", "requestModifier"
     };
     public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule");
 

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -34,13 +34,12 @@ import bubble.service.stream.StandardRuleEngineService;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.ArrayUtils;
 import org.cobbzilla.util.collection.ExpirationEvictionPolicy;
 import org.cobbzilla.util.collection.ExpirationMap;
 import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.http.HttpContentEncodingType;
-import org.cobbzilla.util.http.HttpUtil;
 import org.cobbzilla.util.network.NetworkUtil;
-import org.cobbzilla.util.string.StringUtil;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
@@ -64,15 +63,16 @@ import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIM
 import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY;
 import static java.util.Collections.emptyMap;
 import static java.util.concurrent.TimeUnit.*;
-import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
+import static org.apache.http.HttpHeaders.*;
 import static org.cobbzilla.util.collection.ArrayUtil.arrayToString;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
 import static org.cobbzilla.util.http.HttpContentTypes.TEXT_PLAIN;
 import static org.cobbzilla.util.http.HttpUtil.applyRegexToUrl;
-import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
-import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.http.HttpUtil.chaseRedirects;
+import static org.cobbzilla.util.json.JsonUtil.*;
 import static org.cobbzilla.util.network.NetworkUtil.isLocalIpv4;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
 import static org.cobbzilla.util.string.StringUtil.trimQuotes;
 import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 import static org.cobbzilla.wizard.model.NamedEntity.names;
@@ -446,8 +446,17 @@ public class FilterHttpResource {
     public Response flushCaches(@Context ContainerRequest request) {
         final Account caller = userPrincipal(request);
         if (!caller.admin()) return forbidden();
+
+        final int connCheckMatcherCacheSize = connCheckMatcherCache.size();
         connCheckMatcherCache.clear();
-        return ok(ruleEngine.flushCaches());
+
+        // disable redirect flushing for now -- it works well and it's a lot of work
+        // final Long redirectCacheSize = getRedirectCache().del_matching("*");
+
+        final Map<Object, Object> flushes = ruleEngine.flushCaches();
+        flushes.put("connCheckMatchersCache", connCheckMatcherCacheSize);
+        // flushes.put("redirectCache", redirectCacheSize == null ? 0 : redirectCacheSize);
+        return ok(flushes);
     }
 
     @DELETE @Path(EP_MATCHERS)
@@ -690,40 +699,68 @@ public class FilterHttpResource {
         return ok_empty();
     }
 
-    private final Map<String, String> redirectCache
-            = new ExpirationMap<>(1000, DAYS.toMillis(3), ExpirationEvictionPolicy.atime);
+    public static final String REDIS_PREFIX_REDIRECT_CACHE = "followLink_";
+    @Getter(lazy=true) private final RedisService redirectCache = redis.prefixNamespace(REDIS_PREFIX_REDIRECT_CACHE);
 
     @POST @Path(EP_FOLLOW+"/{requestId}")
+    @Consumes(APPLICATION_JSON)
     @Produces(TEXT_PLAIN)
     public Response followLink(@Context Request req,
                                @Context ContainerRequest ctx,
                                @PathParam("requestId") String requestId,
-                               JsonNode followSpec) {
+                               JsonNode urlNode) {
         final FilterSubContext filterCtx = new FilterSubContext(req, requestId);
+        final RedisService cache = getRedirectCache();
+        final String url = urlNode.textValue();
+        final String cacheKey = sha256_hex(url);
+        final String cachedValue = cache.get(cacheKey);
+        if (cachedValue != null) return ok(cachedValue);
+
+        final String result = chaseRedirects(url);
+        cache.set(cacheKey, result, EX, DAYS.toMillis(365));
+        return ok(result);
+    }
 
-        // is this a request to parse regexes from a URL?
-        if (followSpec.has("regex")) {
-            return ok(redirectCache.computeIfAbsent(json(followSpec), k -> {
-                final String url = followSpec.get("url").textValue();
-                final String regex = followSpec.get("regex").textValue();
-                final Integer group = followSpec.has("group") ? followSpec.get("group").asInt() : null;
-                final List<NameAndValue> headers = new ArrayList<>();
-                for (String name : req.getHeaderNames()) {
-                    final String value = req.getHeader(name);
-                    headers.add(new NameAndValue(name, value));
-                }
-                final List<String> matches = applyRegexToUrl(url, headers, regex, group);
-                return matches == null ? null : StringUtil.toString(matches, "\n");
-            }));
+    public static final String CLIENT_HEADER_PREFIX = "X-Bubble-Client-Header-";
 
-        } else if (followSpec.isTextual()) {
-            // just a regular follow -- chase redirects
-            return ok(redirectCache.computeIfAbsent(followSpec.textValue(), HttpUtil::chaseRedirects));
-        } else {
-            final String json = json(followSpec);
-            log.error("followLink: invalid json (expected String or {regex, url}): "+json);
-            return notFound(json);
+    public static final String[] EXCLUDED_CLIENT_HEADERS = {
+            ACCEPT.toLowerCase(),
+            CONTENT_TYPE.toLowerCase(), CONTENT_LENGTH.toLowerCase(),
+            CONTENT_ENCODING.toLowerCase(), TRANSFER_ENCODING.toLowerCase()
+    };
+
+    @POST @Path(EP_FOLLOW_AND_APPLY_REGEX+"/{requestId}")
+    @Consumes(APPLICATION_JSON)
+    @Produces(APPLICATION_JSON)
+    public Response followLinkThenApplyRegex(@Context Request req,
+                                             @Context ContainerRequest ctx,
+                                             @PathParam("requestId") String requestId,
+                                             FollowThenApplyRegex follow) {
+        final FilterSubContext filterCtx = new FilterSubContext(req, requestId);
+        final RedisService cache = getRedirectCache();
+        final String followJson = json(follow);
+        final String cacheKey = sha256_hex(followJson);
+        final String cachedValue = cache.get(cacheKey);
+        if (cachedValue != null) return ok(cachedValue);
+
+        // collect client headers
+        final List<NameAndValue> headers = new ArrayList<>();
+        for (String name : req.getHeaderNames()) {
+            if (name.toLowerCase().startsWith(CLIENT_HEADER_PREFIX.toLowerCase())) {
+                final String value = req.getHeader(name);
+                final String realName = name.substring(CLIENT_HEADER_PREFIX.length());
+                if (ArrayUtils.indexOf(EXCLUDED_CLIENT_HEADERS, realName.toLowerCase()) == -1) {
+                    headers.add(new NameAndValue(realName, value));
+                }
+            }
         }
+        headers.add(new NameAndValue(ACCEPT, "*/*"));
+        final List<Map<Integer, String>> matches
+                = applyRegexToUrl(follow.getUrl(), headers, follow.getRegex(), Arrays.asList(follow.getGroups()));
+        if (log.isWarnEnabled()) log.warn("followLink(" + follow.getUrl() + ") returning: " + json(matches));
+        final String result = matches == null ? EMPTY_JSON_ARRAY : json(matches);
+        cache.set(cacheKey, result, EX, DAYS.toMillis(365));
+        return ok(result);
     }
 
     @Path(EP_ASSETS+"/{requestId}/{appId}")

bubble-server/src/main/java/bubble/resources/stream/FollowThenApplyRegex.java

@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.resources.stream;
+
+import lombok.Getter;
+import lombok.Setter;
+
+public class FollowThenApplyRegex {
+
+    @Getter @Setter private String url;
+    @Getter @Setter private String regex;
+    @Getter @Setter private Integer[] groups;
+
+}

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -290,7 +290,9 @@ public class StandardNetworkService implements NetworkService {
             prepareLaunchFiles(nn, computeDriver, node, progressMeter, network, sageKey, account, plan, sageNode, automation, errors, sshKeyFile);
 
             // run ansible
-            final String sshArgs = "-o UserKnownHostsFile=/dev/null "
+            final String sshArgs
+                    = "-p 1202 "
+                    + "-o UserKnownHostsFile=/dev/null "
                     + "-o StrictHostKeyChecking=no "
                     + "-o PreferredAuthentications=publickey "
                     + "-i " + abs(sshKeyFile);

bubble-server/src/main/resources/META-INF/bubble/bubble.properties

@@ -1 +1 @@
-bubble.version=Adventure 1.1.3
+bubble.version=Adventure 1.2.1

bubble-server/src/main/resources/ansible/install_local.sh.hbs

@@ -7,6 +7,8 @@ LOG=/var/log/bubble/ansible.log
 # Stop unattended upgrades so that apt installs will work
 # unattended upgrades are re-enabled at the end of the ansible run
 systemctl stop unattended-upgrades
+UNATTENDED_UPGRADES_DISABLED=20auto-upgrades-disabled
+cp /usr/share/unattended-upgrades/${UNATTENDED_UPGRADES_DISABLED} /etc/apt/apt.conf.d/${UNATTENDED_UPGRADES_DISABLED}
 
 # Enable job control. Allows us to start creating dhparam in the background right now.
 {{#if isNode}}# For node, also allows us to install AlgoVPN in the background.{{/if}}
@@ -104,6 +106,7 @@ fi
 kill_bg_jobs
 
 # ansible should have already restarted unattended-upgrades, but just in case
-systemctl start unattended-upgrades
+rm -f /etc/apt/apt.conf.d/${UNATTENDED_UPGRADES_DISABLED}
+systemctl restart unattended-upgrades
 
 exit 0

bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml

@@ -3,7 +3,7 @@
 #
 # Insert additional firewall rules to allow required services to function
 # Insert them all on rule_num 5, and insert them in reverse order here:
-- name: Allow SSH
+- name: Allow SSH tarpit
   iptables:
     chain: INPUT
     protocol: tcp
@@ -11,6 +11,17 @@
     ctstate: NEW
     syn: match
     jump: ACCEPT
+    comment: Accept new SSH tarpit connections
+  become: yes
+
+- name: Allow SSH
+  iptables:
+    chain: INPUT
+    protocol: tcp
+    destination_port: 1202
+    ctstate: NEW
+    syn: match
+    jump: ACCEPT
     comment: Accept new SSH connections
   become: yes
 

bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml

@@ -18,7 +18,7 @@
 - name: Ensure mitmproxy user owns all mitmproxy files
   shell: chown -R mitmproxy /home/mitmproxy/mitmproxy
 
-- name: Install mitmproxy1 supervisor conf file
+- name: Install mitm8888 supervisor conf file
   template:
     src: supervisor_mitmproxy.conf.j2
     dest: /etc/supervisor/conf.d/mitm8888.conf
@@ -28,7 +28,7 @@
   vars:
     port: 8888
 
-- name: Install mitmproxy2 supervisor conf file
+- name: Install mitm9999 supervisor conf file
   template:
     src: supervisor_mitmproxy.conf.j2
     dest: /etc/supervisor/conf.d/mitm9999.conf

bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs

@@ -87,9 +87,12 @@ function {{JS_PREFIX}}_create_button(labelKey, labelDefault, onclick, labelForma
     return btn;
 }
 
-if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') {
+{{JS_PREFIX}}_follow_url = '/__bubble/api/filter/follow/{{BUBBLE_REQUEST_ID}}';
+{{JS_PREFIX}}_follow_and_apply_regex_url = '/__bubble/api/filter/followAndApplyRegex/{{BUBBLE_REQUEST_ID}}';
+
+{{JS_PREFIX}}_url_chasers = {};
 
-    let {{PAGE_PREFIX}}_url_chasers = {};
+if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') {
 
     {{PAGE_PREFIX}}_screenWidth = function () { return window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth };
 
@@ -232,10 +235,10 @@ if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') {
     });
 }
 
-function {{JS_PREFIX}}_chase_redirects (a, removeParams) {
+function {{JS_PREFIX}}_chase_redirects (a, removeParams, regex, groups, callback) {
     const initial_href = a.href;
-    if (initial_href in {{PAGE_PREFIX}}_url_chasers) {
-        a.href = {{PAGE_PREFIX}}_url_chasers[initial_href];
+    if (initial_href in {{JS_PREFIX}}_url_chasers) {
+        a.href = {{JS_PREFIX}}_url_chasers[initial_href];
         return;
     }
     if (a.className && a.className.indexOf('{{JS_PREFIX}}_followed') !== -1) return;
@@ -247,20 +250,37 @@ function {{JS_PREFIX}}_chase_redirects (a, removeParams) {
 
     a.rel = 'noopener noreferrer nofollow';
 
-    fetch('/__bubble/api/filter/follow/{{BUBBLE_REQUEST_ID}}', {method: 'POST', body: JSON.stringify(initial_href)})
-        .then(response => response.text())
+    let is_regex = (typeof regex !== 'undefined');
+    const follow_body = !is_regex ? initial_href :
+        {
+            'url': initial_href,
+            'regex': regex,
+            'groups': (typeof groups === 'undefined' || groups === null ? null : groups)
+        };
+    const request_opts = {
+        method: 'POST',
+        headers: {'Content-Type': 'application/json'},
+        body: JSON.stringify(follow_body)
+    }
+    const follow_url = is_regex ? {{JS_PREFIX}}_follow_and_apply_regex_url : {{JS_PREFIX}}_follow_url;
+    fetch(follow_url, request_opts)
+        .then(response => is_regex ? response.json() : response.text())
         .then(data => {
-            if (data && (data.startsWith('http://') || data.startsWith('https://'))) {
-                if (typeof removeParams === 'undefined' || removeParams === null || removeParams) {
-                    const qPos = data.indexOf('?');
-                    a.href = qPos === -1 ? data : data.substring(0, qPos);
+            if (is_regex) {
+                callback(data);
+            } else {
+                if (data && (data.startsWith('http://') || data.startsWith('https://'))) {
+                    if (typeof removeParams === 'undefined' || removeParams === null || removeParams) {
+                        const qPos = data.indexOf('?');
+                        a.href = qPos === -1 ? data : data.substring(0, qPos);
+                    } else {
+                        a.href = data;
+                    }
+                    {{JS_PREFIX}}_url_chasers[initial_href] = a.href;
+                    // console.log('chase_redirect: rewrote '+initial_href+' -> '+a.href);
                 } else {
-                    a.href = data;
+                    console.warn('chase_redirects: ' + a.href + ' returned non-URL response: ' + data);
                 }
-                {{PAGE_PREFIX}}_url_chasers[initial_href] = a.href;
-                // console.log('chase_redirect: rewrote '+initial_href+' -> '+a.href);
-            } else {
-                console.warn('chase_redirects: '+a.href+' returned non-URL response: '+data);
             }
         })
         .catch((error) => {

bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs

@@ -72,8 +72,11 @@ function {{JS_PREFIX}}_uuidv4() {
 
 const {{JS_PREFIX}}_create_block_img = function(size) {
     const img = document.createElement('img');
+    img.style.all = 'revert';
+    img.style.fontSize = 'x-small';
     img.src = {{JS_PREFIX}}_asset_img_url('icon');
-    img.width = typeof size !== 'undefined' ? size : 24;
+    img.width = typeof size !== 'undefined' && size !== null ? size : 24;
+    img.style.width = img.width+'px';
     return img;
 }
 

bubble-server/src/main/resources/bubble/rule/social/block/site/FB.js.hbs

@@ -32,7 +32,7 @@ Element.prototype.appendChild = function() {
                 const block = {{JS_PREFIX}}_should_block({{JS_PREFIX}}_blocked_users, node)
                 if (block) {
                     // log('>>> BLOCKING via appendChild: '+block);
-                    {{JS_PREFIX}}_appendChild.apply({{JS_PREFIX}}_jail, arguments);
+                    return {{JS_PREFIX}}_appendChild.apply({{JS_PREFIX}}_jail, arguments);
                 }
             }
         }
@@ -40,7 +40,7 @@ Element.prototype.appendChild = function() {
         console.log('>>> error inspecting: e='+e);
     }
     try {
-        {{JS_PREFIX}}_appendChild.apply(this, arguments);
+        return {{JS_PREFIX}}_appendChild.apply(this, arguments);
     } catch (e) {
         console.log('>>> error calling document.appendChild: arg[0].tagName = '+node.tagName+' e='+e);
     }

bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs

@@ -1,9 +1,32 @@
 {{JS_PREFIX}}_supports_keywords = true;
+{{JS_PREFIX}}_idle_interval = 5000;
 
 const {{JS_PREFIX}}_site_host = location.protocol + '//' + window.location.hostname + '/';
 
+function {{JS_PREFIX}}_mobile() {
+    const html = Array.from(document.getElementsByTagName('html'));
+    if (html.length !== 0) {
+        return html[0].className && html[0].className.indexOf(' mobile ') !== -1;
+    }
+    return false;
+}
+
+const {{JS_PREFIX}}_jail = document.createElement('div');
+{{JS_PREFIX}}_jail.style.display = 'none';
+
 function {{JS_PREFIX}}_apply_blocks(blocked_users) {
-    const articles = Array.from(document.getElementsByClassName('feed-shared-update-v2'));
+    const adBanner = Array.from(document.getElementsByTagName('iframe')).find(i => i.className && i.className === 'ad-banner');
+    if (typeof adBanner !== 'undefined') {
+        let adParent = adBanner.parentNode;
+        if (adParent != null) {
+            adParent.innerHTML = '';
+            adParent.style.display = 'none';
+        } else {
+        }
+    }
+    const articles = {{JS_PREFIX}}_mobile()
+        ? Array.from(document.getElementsByClassName('feed-item'))
+        : Array.from(document.getElementsByClassName('feed-shared-update-v2'));
     if (articles === null || articles.length === 0) {
         console.warn('No articles found, not filtering');
         return;
@@ -11,7 +34,21 @@ function {{JS_PREFIX}}_apply_blocks(blocked_users) {
     {{JS_PREFIX}}_consider_block(articles, blocked_users);
 }
 
-function {{JS_PREFIX}}_author_from_href(href) {
+function {{JS_PREFIX}}_is_valid_author_name(name) {
+    return !(name.startsWith('ACoAA') || name.length >= 38);
+}
+
+function {{JS_PREFIX}}_author_from_href(linkId, callback) {
+    if (typeof linkId === 'undefined' || linkId === null || linkId.length === 0) {
+        // console.log('author_from_href: invalid link ID: '+linkId);
+        return;
+    }
+    const link = document.getElementById(linkId);
+    if (link === null) {
+        // console.log('author_from_href: link with ID '+linkId+' not found');
+        return;
+    }
+    const href = link.href;
     if (typeof href === 'undefined' || href === null) return null;
     let h = href.startsWith({{JS_PREFIX}}_site_host) ? href.substring({{JS_PREFIX}}_site_host.length) : href;
     const qPos = h.indexOf('?');
@@ -19,14 +56,56 @@ function {{JS_PREFIX}}_author_from_href(href) {
         h = h.substring(0, qPos);
     }
     if (h.endsWith('/')) h = h.substring(0, h.length - 1);
-    if (!h.startsWith('in/') && !h.startsWith('company/')) {
-        return null;
+    let profile_type = null;
+
+    const mobile = {{JS_PREFIX}}_mobile();
+    if (mobile && h.startsWith('mwlite/')) {
+        h = h.substring('mwlite/'.length);
+    }
+    if (h.startsWith('in/')) {
+        profile_type = 'in/';
+    } else if (h.startsWith('company/')) {
+        profile_type = 'company/';
+    } else {
+        // console.log("author_from_href: skipping (not in/ or company/) href: "+href+', h='+h);
+        return;
     }
     const slashPos = h.indexOf('/');
-    const name = h.substring(slashPos);
-    if (name.length > 35 && name.indexOf('-') === -1 && name.indexOf('_') === -1) return null;
-    console.log("author_from_href: found "+name+' from '+href);
-    return name;
+    const name = h.substring(slashPos+1);
+    if ({{JS_PREFIX}}_is_valid_author_name(name)) {
+        // console.log("author_from_href: found " + name + ' from ' + href);
+        callback(linkId, name);
+    } else {
+        // only chase a link once
+        let linkClass = link.className;
+        const chaseClass = '{{JS_PREFIX}}_link_chased';
+        if (linkClass && linkClass.indexOf(chaseClass) !== -1) {
+            return;
+        } else {
+            link.className = link.className ? link.className + ' '+chaseClass : chaseClass;
+        }
+
+        {{JS_PREFIX}}_chase_redirects(link, true, '/voyager/api/identity/profiles/([^/]+)/privacySettings', [1], function (matches) {
+            if (typeof matches.length !== 'undefined') {
+                for (let i=0; i<matches.length; i++) {
+                    const match = matches[i];
+                    if (!('1' in match)) continue;
+                    const updated_name = matches[i]['1'];
+                    if ({{JS_PREFIX}}_is_valid_author_name(updated_name)) {
+                        const realLink = document.getElementById(linkId);
+                        if (realLink === null) {
+                            console.log('author_from_href: link with id '+linkId+' seems to have disappeared from the document');
+                            return;
+                        }
+                        link.href = {{JS_PREFIX}}_site_host + profile_type + updated_name;
+                        console.log('author_from_href: updated link.href from '+href+' to: '+link.href);
+                        callback(linkId, updated_name);
+                        return;
+                    }
+                }
+            }
+        });
+    }
 }
 
 function {{JS_PREFIX}}_remove_article_from_dom(article) {
@@ -57,45 +136,111 @@ function {{JS_PREFIX}}_create_block_control(article, authorName, articleLink) {
     blockSpan.appendChild(document.createTextNode('\u00A0\u00A0'));
     blockSpan.appendChild(blockLink);
     blockSpan.id = 'blockSpan_'+{{JS_PREFIX}}_uuidv4();
-    console.log('adding block control on '+authorName);
+    // console.log('adding block control on '+authorName);
     return blockSpan;
 }
 
+function {{JS_PREFIX}}_hash_url(url) { return btoa(url).replaceAll('=', ''); }
+
+function {{JS_PREFIX}}_is_ad(article) {
+    const mobile = {{JS_PREFIX}}_mobile();
+    return (mobile && article.getAttribute('data-is-sponsored') && article.getAttribute('data-is-sponsored') !== "false")
+    || (article.innerHTML.indexOf('<span>Promoted</span>') !== -1)
+    || (article.innerHTML.indexOf('<span dir="ltr">Promoted</span>') !== -1);
+}
+
+function {{JS_PREFIX}}_find_append_span(link) {
+    const mobile = {{JS_PREFIX}}_mobile();
+    if (!mobile) {
+        let authorSpans = Array.from(link.getElementsByClassName('feed-shared-actor__name'));
+        if (authorSpans.length > 0) {
+            return authorSpans[0];
+        } else {
+            return Array.from(link.getElementsByTagName('span'))
+                .find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type'));
+        }
+    } else {
+        const ltrSpan = Array.from(link.getElementsByTagName('span'))
+            .find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type'));
+        if (ltrSpan) return ltrSpan;
+        if (link.className && link.className.indexOf('profile-link') !== -1) {
+            return link.lastChild;
+        }
+    }
+}
+
 function {{JS_PREFIX}}_consider_block(articles, blocked_users) {
+    const mobile = {{JS_PREFIX}}_mobile();
     if (articles && articles.length && articles.length > 0) {
         for (let i=0; i<articles.length; i++) {
             const article = articles[i];
+            if ({{JS_PREFIX}}_is_ad(article)) {
+                {{JS_PREFIX}}_tally_author_block({{PAGE_PREFIX}}_msg_or_default({{JS_PREFIX}}_messages, 'web_advertOrOtherBlock', 'ad/other'));
+                {{JS_PREFIX}}_remove_article_from_dom(article);
+                continue;
+            }
             const firstEval = {{JS_PREFIX}}_mark_evaluated(article);
             if ({{JS_PREFIX}}_includes_block_keyword(article, firstEval)) {
                 {{JS_PREFIX}}_remove_article_from_dom(article);
                 continue;
             }
-            const articleLinks = Array.from(article.getElementsByTagName('a'));
+            const articleLinks = mobile
+                ? Array.from(article.getElementsByTagName('a')).filter(a => !a.hasAttribute('aria-hidden'))
+                : Array.from(article.getElementsByTagName('a'));
+            // console.log('consider_block: found '+articleLinks.length+' articleLinks');
             for (let j=0; j<articleLinks.length; j++) {
-                const articleLink = articleLinks[i];
-                console.log('consider_block: examining articleLink with href='+articleLink.href);
-                const author = {{JS_PREFIX}}_author_from_href(articleLink.href);
-                if (author === null) continue;
-                if (author in blocked_users) {
-                    {{JS_PREFIX}}_tally_author_block(author);
-                    if (!firstEval) {{JS_PREFIX}}_untally_allow();
-                    {{JS_PREFIX}}_remove_article_from_dom(article);
-
-                } else if (firstEval) {
-                    const authorSpans = Array.from(articleLink.getElementsByClassName('feed-shared-actor__name'));
-                    if (authorSpans.length === 0) {
-                        continue;
+                const articleLink = articleLinks[j];
+                if (typeof articleLink === 'undefined' || articleLink === null || typeof articleLink.href === 'undefined') {
+                    // console.log('consider_block: skipping invalid articleLink: '+JSON.stringify(articleLink));
+                    continue;
+                }
+                if (typeof articleLink.id === 'undefined' || articleLink.id === null || articleLink.id.length === 0) {
+                    articleLink.id = {{JS_PREFIX}}_uuidv4();
+                }
+                const href = articleLink.href;
+                if (href === null || href.trim().length === 0) continue;
+
+                let examinedClass = articleLink.className;
+                const seenClass = '{{JS_PREFIX}}_link_examined_'+{{JS_PREFIX}}_hash_url(href)+'_'+articleLink.id;
+                let seenBefore = false;
+                if (examinedClass && examinedClass.indexOf(seenClass) !== -1) {
+                    seenBefore = true;
+                } else {
+                    articleLink.className = articleLink.className ? articleLink.className + ' '+seenClass : seenClass;
+                }
+
+                {{JS_PREFIX}}_author_from_href(articleLink.id, function (linkId, author) {
+                    if (author === null) return;
+
+                    const realLink = document.getElementById(linkId);
+                    if (realLink === null) {
+                        // console.log('consider_block: link with id '+linkId+' seems to have disappeared from the document');
+                        return;
                     }
 
-                    let b = {{JS_PREFIX}}_create_block_control(article, author, articleLink);
-                    if (b !== null) {
-                        console.log('consider_block: inserting span='+b.id+' for article by '+author);
-                        authorSpans[0].parentNode.appendChild(b);
-                        {{JS_PREFIX}}_tally_allow();
-                    } else {
-                        console.log('consider_block: create_block_control returned null for author '+author)
+                    // console.log('consider_block: examining linkId with author='+author);
+                    if (author in blocked_users) {
+                        {{JS_PREFIX}}_tally_author_block(author);
+                        if (!firstEval) {{JS_PREFIX}}_untally_allow();
+                        {{JS_PREFIX}}_remove_article_from_dom(article);
+
+                    } else if (!seenBefore) {
+                        let appendToSpan = {{JS_PREFIX}}_find_append_span(realLink);
+                        if (!appendToSpan) {
+                            // console.log('consider_block: no span found to append to for author '+author);
+                            return;
+                        }
+
+                        let b = {{JS_PREFIX}}_create_block_control(article, author, realLink);
+                        if (b !== null) {
+                            // console.log('consider_block: inserting span='+b.id+' for article by '+author);
+                            appendToSpan.parentNode.appendChild(b);
+                            {{JS_PREFIX}}_tally_allow();
+                        } else {
+                            // console.log('consider_block: create_block_control returned null for author '+author)
+                        }
                     }
-                }
+                });
             }
         }
     }

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json

@@ -247,6 +247,12 @@
       "matcher": "BubbleBlockMatcher",
       "key": "hideStats_reddit.com",
       "data": "true"
+    }, {
+      "site": "All_Sites",
+      "template": true,
+      "matcher": "BubbleBlockMatcher",
+      "key": "hideStats_redfin.com",
+      "data": "true"
     }, {
       "site": "All_Sites",
       "template": true,
@@ -319,6 +325,12 @@
       "matcher": "BubbleBlockMatcher",
       "key": "hideStats_tripactions.com",
       "data": "true"
+    }, {
+      "site": "All_Sites",
+      "template": true,
+      "matcher": "BubbleBlockMatcher",
+      "key": "hideStats_trulia.com",
+      "data": "true"
     }, {
       "site": "All_Sites",
       "template": true,
@@ -373,6 +385,12 @@
       "matcher": "BubbleBlockMatcher",
       "key": "hideStats_zendesk.com",
       "data": "true"
+    }, {
+      "site": "All_Sites",
+      "template": true,
+      "matcher": "BubbleBlockMatcher",
+      "key": "hideStats_zillow.com",
+      "data": "true"
     }, {
       "site": "All_Sites",
       "template": true,

bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_data.json

@@ -1,12 +1,6 @@
 [{
   "name": "UserBlocker",
   "children": {
-    "AppData": [{
-      "site": "LinkedIn",
-      "template": true,
-      "matcher": "LIMatcher",
-      "key": "kw:_<span>Promoted</span>",
-      "data": "true"
-    }]
+    "AppData": []
   }
 }]

bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json

@@ -10,6 +10,16 @@
       "fqdn": "www.linkedin.com",
       "urlRegex": "/feed/",
       "rule": "li_user_blocker"
+    }, {
+      "name": "LIMobileMatcher",
+      "site": "LinkedIn",
+      "template": true,
+      "requestCheck": true,
+      "requestModifier": true,
+      "fqdn": "www.linkedin.com",
+      "urlRegex": "/",
+      "userAgentRegex": "Mobi|Android",
+      "rule": "li_user_blocker"
     }]
   }
 }]

bubble-server/src/main/resources/packer/node-roles.txt

@@ -1,5 +1,6 @@
 common
 firewall
+tarpit
 nginx
 algo
 mitmproxy

bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf

@@ -0,0 +1,14 @@
+Port 1202
+LoginGraceTime 10
+PasswordAuthentication no
+PermitEmptyPasswords no
+ChallengeResponseAuthentication no
+KerberosAuthentication no
+GSSAPIAuthentication no
+X11Forwarding no
+PermitUserEnvironment no
+HostKey /etc/ssh/ssh_host_ed25519_key
+HostKey /etc/ssh/ssh_host_rsa_key
+KexAlgorithms curve25519-sha256@libssh.org
+Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr
+MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com

bubble-server/src/main/resources/packer/roles/firewall/files/jail.local

@@ -0,0 +1,5 @@
+[sshd]
+mode    = aggressive
+port    = 1202
+logpath = %(sshd_log)s
+backend = %(sshd_backend)s

bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml

@@ -90,15 +90,34 @@
     dest: /usr/local/bin/bubble_peer_manager.py
     owner: root
     group: root
-    mode: 0555
+    mode: 0550
   when: fw_enable_admin
 
-- name: Install supervisor conf file for port manager
+- name: Install supervisor conf file for peer manager
   copy:
     src: supervisor_bubble_peer_manager.conf
     dest: /etc/supervisor/conf.d/bubble_peer_manager.conf
+    owner: root
+    group: root
+    mode: 0550
   when: fw_enable_admin
 
+- name: Install SSH hardening settings
+  copy:
+    src: bubble_sshd.conf
+    dest: /etc/ssh/sshd_config.d/bubble_sshd.conf
+    owner: root
+    group: root
+    mode: 0400
+
+- name: Install SSH fail2ban settings
+  copy:
+    src: jail.local
+    dest: /etc/fail2ban/jail.local
+    owner: root
+    group: root
+    mode: 0400
+
 - include: rules.yml
 
 - supervisorctl:

bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml

@@ -17,15 +17,18 @@
     comment: Allow related and established connections
   become: yes
 
-- name: Allow SSH
+- name: Allow SSH on ports 22 and 1202
   iptables:
     chain: INPUT
     protocol: tcp
-    destination_port: 22
+    destination_port: "{{ item }}"
     ctstate: NEW
     syn: match
     jump: ACCEPT
     comment: Accept new SSH connections
+  with_items:
+    - 22
+    - 1202
   become: yes
   when: fw_enable_ssh
 

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -64,6 +64,7 @@ LOCAL_IPS = []
 for local_ip in subprocess.check_output(['hostname', '-I']).split():
     LOCAL_IPS.append(local_ip.decode())
 
+TARPIT_PORT = 8080
 
 VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4)
 VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6)
@@ -469,6 +470,28 @@ def health_check_response(flow):
         flow.response.stream = lambda chunks: [b'OK\n']
 
 
+def tarpit_response(flow, host):
+    # if bubble_log.isEnabledFor(DEBUG):
+    #     bubble_log.debug('health_check_response: special bubble health check request, responding with OK')
+    response_headers = nheaders.Headers()
+    response_headers[HEADER_LOCATION] = 'http://'+host+':'+str(TARPIT_PORT)+'/admin/index.php'
+    if flow.response is None:
+        flow.response = http.HTTPResponse(http_version='HTTP/1.1',
+                                          status_code=301,
+                                          reason='Moved Permanently',
+                                          headers=response_headers,
+                                          content=b'')
+    else:
+        flow.response.headers = nheaders.Headers()
+        flow.response.headers = response_headers
+        flow.response.status_code = 301
+        flow.response.reason = 'Moved Permanently'
+
+
+def include_request_headers(path):
+    return '/followAndApplyRegex' in path
+
+
 def special_bubble_response(flow):
     name = 'special_bubble_response'
     path = flow.request.path
@@ -478,7 +501,7 @@ def special_bubble_response(flow):
 
     uri = make_bubble_special_path(path)
     if bubble_log.isEnabledFor(DEBUG):
-        bubble_log.debug('special_bubble_response: sending special bubble request to '+uri)
+        bubble_log.debug('special_bubble_response: sending special bubble '+flow.request.method+' to '+uri)
     headers = {
         'Accept': 'application/json',
         'Content-Type': 'application/json'
@@ -489,12 +512,22 @@ def special_bubble_response(flow):
         response = async_stream(client, name, uri, headers=headers, loop=loop)
 
     elif flow.request.method == 'POST':
-        loop = asyncio.new_event_loop()
-        client = async_client(timeout=30)
+        if include_request_headers(flow.request.path):
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('special_bubble_request: including client headers: '+repr(flow.request.headers))
+            # add client request headers
+            for name, value in flow.request.headers.items():
+                headers['X-Bubble-Client-Header-'+name] = value
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug('special_bubble_request: NOW headers='+repr(headers))
+
         data = None
         if flow.request.content and flow.request.content:
             headers[HEADER_CONTENT_LENGTH] = str(len(flow.request.content))
             data = flow.request.content
+
+        loop = asyncio.new_event_loop()
+        client = async_client(timeout=30)
         response = async_stream(client, name, uri, headers=headers, method='POST', data=data, loop=loop)
 
     else:

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py

@@ -32,7 +32,7 @@ from mitmproxy.net.http import headers as nheaders
 from bubble_api import bubble_matchers, bubble_activity_log, \
     CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \
     CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \
-    is_bubble_special_path, is_bubble_health_check, health_check_response, \
+    is_bubble_special_path, is_bubble_health_check, health_check_response, tarpit_response,\
     is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain
 from bubble_config import bubble_host, bubble_host_alias
 from bubble_flex import new_flex_flow
@@ -168,11 +168,10 @@ class Rerouter:
                     return None
 
                 elif is_not_from_vpn(client_addr):
-                    # todo: add to fail2ban
                     if bubble_log.isEnabledFor(WARNING):
-                        bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url+' host='+host)
-                    bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns)
-                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
+                        bubble_log.warning('bubble_handle_request: sending to tarpit: non-VPN client='+client_addr+', url='+log_url+' host='+host)
+                    bubble_activity_log(client_addr, server_addr, 'http_tarpit_non_vpn', fqdns)
+                    tarpit_response(flow, host)
                     return None
 
             if is_bubble_special_path(path):
@@ -230,11 +229,10 @@ class Rerouter:
                     # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url)
 
         elif is_http and is_not_from_vpn(client_addr):
-            # todo: add to fail2ban
             if bubble_log.isEnabledFor(WARNING):
-                bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr)
-            bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr])
-            add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
+                bubble_log.warning('bubble_handle_request: sending to tarpit: non-VPN client='+client_addr)
+            bubble_activity_log(client_addr, server_addr, 'http_tarpit_non_vpn', [server_addr])
+            tarpit_response(flow, host)
             return None
 
         else:

bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_http_tarpit.py

@@ -0,0 +1,89 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Adapted from: https://nullprogram.com/blog/2019/03/22/
+#
+import asyncio
+import random
+import os
+import sys
+
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
+from pathlib import Path
+
+TARPIT_LOG = '/var/log/bubble/http_tarpit.log'
+TARPIT_LOG_LEVEL_FILE = '/home/tarpit/http_tarpit_log_level.txt'
+TARPIT_LOG_LEVEL_ENV_VAR = 'HTTP_TARPIT_LOG_LEVEL'
+DEFAULT_TARPIT_LOG_LEVEL = 'INFO'
+TARPIT_LOG_LEVEL = None
+
+TARPIT_PORT_FILE = '/home/tarpit/http_tarpit_port.txt'
+TARPIT_PORT_ENV_VAR = 'HTTP_TARPIT_PORT'
+DEFAULT_TARPIT_PORT = '8080'
+
+tarpit_log = logging.getLogger(__name__)
+
+try:
+    TARPIT_LOG_LEVEL = Path(TARPIT_LOG_LEVEL_FILE).read_text().strip()
+except IOError:
+    print('error reading log level from '+TARPIT_LOG_LEVEL_FILE+', checking env var '+TARPIT_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True)
+    TARPIT_LOG_LEVEL = os.getenv(TARPIT_LOG_LEVEL_ENV_VAR, DEFAULT_TARPIT_LOG_LEVEL)
+
+TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, TARPIT_LOG_LEVEL.upper(), None)
+if not isinstance(TARPIT_NUMERIC_LOG_LEVEL, int):
+    print('Invalid log level: ' + TARPIT_LOG_LEVEL + ' - using default '+DEFAULT_TARPIT_LOG_LEVEL, file=sys.stderr, flush=True)
+    TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_TARPIT_LOG_LEVEL.upper(), None)
+
+try:
+    with open(TARPIT_LOG, 'w+') as f:
+        logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=TARPIT_LOG, level=TARPIT_NUMERIC_LOG_LEVEL)
+except IOError:
+    logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', stream=sys.stdout, level=TARPIT_NUMERIC_LOG_LEVEL)
+
+tarpit_log = logging.getLogger(__name__)
+
+if tarpit_log.isEnabledFor(INFO):
+    tarpit_log.info('tarpit initialized, default log level = '+logging.getLevelName(TARPIT_NUMERIC_LOG_LEVEL))
+
+TARPIT_PORT = 8080
+try:
+    TARPIT_PORT = int(Path(TARPIT_PORT_FILE).read_text().strip())
+except IOError:
+    print('error reading port from '+TARPIT_PORT_FILE+', checking env var '+TARPIT_PORT_ENV_VAR, file=sys.stderr, flush=True)
+    TARPIT_PORT = int(os.getenv(TARPIT_PORT_ENV_VAR, DEFAULT_TARPIT_PORT))
+
+TRAP_COUNT = 0
+
+
+async def handler(_reader, writer):
+    global TRAP_COUNT
+    TRAP_COUNT = TRAP_COUNT + 1
+    peer_addr = writer.get_extra_info('socket').getpeername()[0]
+    if tarpit_log.isEnabledFor(INFO):
+        tarpit_log.info('trapped '+peer_addr+' - trap count: ' + str(TRAP_COUNT))
+    writer.write(b'HTTP/1.1 200 OK\r\n')
+    try:
+        while True:
+            header = random.randint(0, 2**32)
+            value = random.randint(0, 2**32)
+            await asyncio.sleep(3 + (header % 4))
+            writer.write(b'X-WOPR-%x: %x\r\n' % (header, value))
+            await writer.drain()
+    except ConnectionResetError:
+        TRAP_COUNT = TRAP_COUNT - 1
+        if tarpit_log.isEnabledFor(INFO):
+            tarpit_log.info('dropped '+peer_addr+' - trap count: ' + str(TRAP_COUNT))
+        pass
+
+
+async def main():
+    if tarpit_log.isEnabledFor(INFO):
+        tarpit_log.info('starting HTTP tarpit on port '+str(TARPIT_PORT))
+    server = await asyncio.start_server(handler, '0.0.0.0', TARPIT_PORT)
+    async with server:
+        await server.serve_forever()
+
+asyncio.run(main())

bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_ssh_tarpit.py

@@ -0,0 +1,87 @@
+#!/usr/bin/python3
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Adapted from: https://nullprogram.com/blog/2019/03/22/
+#
+import asyncio
+import random
+import os
+import sys
+
+import logging
+from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL
+
+from pathlib import Path
+
+TARPIT_LOG = '/var/log/bubble/ssh_tarpit.log'
+TARPIT_LOG_LEVEL_FILE = '/home/tarpit/ssh_tarpit_log_level.txt'
+TARPIT_LOG_LEVEL_ENV_VAR = 'SSH_TARPIT_LOG_LEVEL'
+DEFAULT_TARPIT_LOG_LEVEL = 'INFO'
+TARPIT_LOG_LEVEL = None
+
+TARPIT_PORT_FILE = '/home/tarpit/ssh_tarpit_port.txt'
+TARPIT_PORT_ENV_VAR = 'SSH_TARPIT_PORT'
+DEFAULT_TARPIT_PORT = '22'
+
+tarpit_log = logging.getLogger(__name__)
+
+try:
+    TARPIT_LOG_LEVEL = Path(TARPIT_LOG_LEVEL_FILE).read_text().strip()
+except IOError:
+    print('error reading log level from '+TARPIT_LOG_LEVEL_FILE+', checking env var '+TARPIT_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True)
+    TARPIT_LOG_LEVEL = os.getenv(TARPIT_LOG_LEVEL_ENV_VAR, DEFAULT_TARPIT_LOG_LEVEL)
+
+TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, TARPIT_LOG_LEVEL.upper(), None)
+if not isinstance(TARPIT_NUMERIC_LOG_LEVEL, int):
+    print('Invalid log level: ' + TARPIT_LOG_LEVEL + ' - using default '+DEFAULT_TARPIT_LOG_LEVEL, file=sys.stderr, flush=True)
+    TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_TARPIT_LOG_LEVEL.upper(), None)
+
+try:
+    with open(TARPIT_LOG, 'w+') as f:
+        logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=TARPIT_LOG, level=TARPIT_NUMERIC_LOG_LEVEL)
+except IOError:
+    logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', stream=sys.stdout, level=TARPIT_NUMERIC_LOG_LEVEL)
+
+tarpit_log = logging.getLogger(__name__)
+
+if tarpit_log.isEnabledFor(INFO):
+    tarpit_log.info('tarpit initialized, default log level = '+logging.getLevelName(TARPIT_NUMERIC_LOG_LEVEL))
+
+TARPIT_PORT = 8080
+try:
+    TARPIT_PORT = int(Path(TARPIT_PORT_FILE).read_text().strip())
+except IOError:
+    print('error reading port from '+TARPIT_PORT_FILE+', checking env var '+TARPIT_PORT_ENV_VAR, file=sys.stderr, flush=True)
+    TARPIT_PORT = int(os.getenv(TARPIT_PORT_ENV_VAR, DEFAULT_TARPIT_PORT))
+
+TRAP_COUNT = 0
+
+
+async def handler(_reader, writer):
+    global TRAP_COUNT
+    TRAP_COUNT = TRAP_COUNT + 1
+    peer_addr = writer.get_extra_info('socket').getpeername()[0]
+    if tarpit_log.isEnabledFor(INFO):
+        tarpit_log.info('trapped '+peer_addr+' - trap count: ' + str(TRAP_COUNT))
+    try:
+        while True:
+            val = random.randint(0, 2 ** 32)
+            await asyncio.sleep(6 + (val % 4))
+            writer.write(b'%x\r\n' % val)
+            await writer.drain()
+    except ConnectionResetError:
+        TRAP_COUNT = TRAP_COUNT - 1
+        if tarpit_log.isEnabledFor(INFO):
+            tarpit_log.info('dropped '+peer_addr+' - trap count: ' + str(TRAP_COUNT))
+        pass
+
+
+async def main():
+    if tarpit_log.isEnabledFor(INFO):
+        tarpit_log.info('starting SSH tarpit on port '+str(TARPIT_PORT))
+    server = await asyncio.start_server(handler, '0.0.0.0', TARPIT_PORT)
+    async with server:
+        await server.serve_forever()
+
+asyncio.run(main())

bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_http_tarpit.conf

@@ -0,0 +1,6 @@
+
+[program:http_tarpit]
+stdout_logfile = /dev/null
+stderr_logfile = /dev/null
+command=sudo -u tarpit /home/tarpit/bubble_http_tarpit.py
+stopsignal=QUIT

bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_ssh_tarpit.conf

@@ -0,0 +1,6 @@
+
+[program:ssh_tarpit]
+stdout_logfile = /dev/null
+stderr_logfile = /dev/null
+command=/home/tarpit/bubble_ssh_tarpit.py
+stopsignal=QUIT

bubble-server/src/main/resources/packer/roles/tarpit/tasks/main.yml

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+- name: Create tarpit user
+  user:
+    name: tarpit
+    comment: tarpit user
+    shell: /bin/false
+    system: yes
+    home: /home/tarpit
+    groups: bubble-log
+
+- name: Copy bubble_ssh_tarpit script
+  copy:
+    src: bubble_ssh_tarpit.py
+    dest: /home/tarpit/bubble_ssh_tarpit.py
+    owner: tarpit
+    group: tarpit
+    mode: 0500
+
+- name: Copy bubble_http_tarpit script
+  copy:
+    src: bubble_http_tarpit.py
+    dest: /home/tarpit/bubble_http_tarpit.py
+    owner: tarpit
+    group: tarpit
+    mode: 0500
+
+- name: Install ssh tarpit supervisor conf file
+  copy:
+    src: supervisor_ssh_tarpit.conf
+    dest: /etc/supervisor/conf.d/ssh_tarpit.conf
+    owner: root
+    group: root
+    mode: 0400
+
+- name: Install http tarpit supervisor conf file
+  copy:
+    src: supervisor_http_tarpit.conf
+    dest: /etc/supervisor/conf.d/http_tarpit.conf
+    owner: root
+    group: root
+    mode: 0400
+
+- name: Allow HTTP tarpit port
+  iptables:
+    chain: INPUT
+    protocol: tcp
+    destination_port: 8080
+    ctstate: NEW
+    syn: match
+    jump: ACCEPT
+    comment: Accept new connections on HTTP tarpit port
+  become: yes

bubble-web

@@ -1 +1 @@
-Subproject commit 8845990e046eb27dc71da14d9348c84fed6bd7c1
+Subproject commit 66d46695a64ff58934560c4b35aa43a0ab32fbe2

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit cad625431e357e94647a1d99da2efc171740d8e3
+Subproject commit dfafe62c7eb3413cf1210e40e551094458f4d9d0