Merge branch 'master' into kris/additional_spare_devices

# Conflicts:
#	bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java

bubble-server/src/main/java/bubble/dao/account/AccountDAO.java

@@ -452,4 +452,7 @@ public class AccountDAO extends AbstractCRUDDAO<Account> implements SqlViewSearc
     @NonNull public List<Account> findDeleted() {
         return list(criteria().add(isNotNull("deleted")));
     }
+
+    @NonNull public List<Account> findNotDeleted() { return findByField("deleted", null); }
+
 }

bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java

@@ -46,18 +46,15 @@ public class AccountSshKeyDAO extends AccountOwnedEntityDAO<AccountSshKey> {
         if (key.hasExpiration() && key.expired()) throw invalidEx("err.expiration.cannotCreateSshKeyAlreadyExpired");
 
         final Account owner = accountDAO.findByUuid(key.getAccount());
-        if (key.installSshKey()) {
-            if (owner.admin()) {
-                // admin keys are always installed on a node
-                // never allow installation of a key on sage. must be manually set in the database.
-                final BubbleNetwork thisNetwork = configuration.getThisNetwork();
-                if (thisNetwork != null && thisNetwork.getInstallType() == AnsibleInstallType.node) {
-                    key.setInstallSshKey(true);
-                }
-            } else {
-                // never install key for non-admin
-                key.setInstallSshKey(false);
-            }
+        final BubbleNetwork thisNetwork = configuration.getThisNetwork();
+        if (thisNetwork == null || thisNetwork.getInstallType() != AnsibleInstallType.sage) {
+            // never allow installation of a key on sage. must be manually set in the database.
+            key.setInstallSshKey(false);
+
+        } else {
+            // admin keys are always installed on a node
+            // never install key for non-admin
+            key.setInstallSshKey(owner.admin() && thisNetwork.getInstallType() == AnsibleInstallType.node);
         }
 
         final String hash = sha256_hex(key.getSshPublicKey());

bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java

@@ -9,6 +9,7 @@ import bubble.model.account.Account;
 import bubble.model.app.AppMatcher;
 import bubble.model.app.AppRule;
 import bubble.model.app.BubbleApp;
+import bubble.service.stream.AppPrimerService;
 import bubble.service.stream.RuleEngineService;
 import lombok.extern.slf4j.Slf4j;
 import org.hibernate.criterion.Order;
@@ -17,6 +18,7 @@ import org.springframework.stereotype.Repository;
 
 import java.util.List;
 
+import static bubble.model.app.AppMatcher.WILDCARD_FQDN;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.wizard.model.Identifiable.MTIME;
 import static org.hibernate.criterion.Restrictions.*;
@@ -28,6 +30,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
     @Autowired private BubbleAppDAO appDAO;
     @Autowired private AppRuleDAO ruleDAO;
     @Autowired private RuleEngineService ruleEngineService;
+    @Autowired private AppPrimerService primerService;
 
     @Override public Order getDefaultSortOrder() { return PRIORITY_ASC; }
 
@@ -39,7 +42,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
                         eq("passthru", false),
                         or(
                                 eq("fqdn", fqdn),
-                                eq("fqdn", "*")
+                                eq("fqdn", WILDCARD_FQDN)
                         ))
         ).addOrder(PRIORITY_ASC));
     }
@@ -88,6 +91,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> {
             }
         }
         ruleEngineService.flushCaches();
+        primerService.prime(app);
         return super.postUpdate(matcher, context);
     }
 

bubble-server/src/main/java/bubble/model/app/AppMatcher.java

@@ -48,6 +48,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H
     public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "passthru");
 
     public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE);
+    public static final String WILDCARD_FQDN = "*";
+    public static final String WILDCARD_URL = ".*";
 
     public AppMatcher(AppMatcher other) {
         copy(this, other, CREATE_FIELDS);
@@ -80,12 +82,14 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H
     @Size(max=1024, message="err.fqdn.length")
     @ECIndex @Column(nullable=false, length=1024)
     @Getter @Setter private String fqdn;
+    @JsonIgnore @Transient public boolean isWildcardFqdn () { return fqdn != null && fqdn.equals(WILDCARD_FQDN); }
 
     @ECSearchable(filter=true) @ECField(index=60)
     @HasValue(message="err.urlRegex.required")
     @Size(max=1024, message="err.urlRegex.length")
     @Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(1024+ENC_PAD)+") NOT NULL")
     @Getter @Setter private String urlRegex;
+    public boolean hasUrlRegex() { return !empty(urlRegex) && !urlRegex.equals(WILDCARD_URL); }
 
     @Transient @JsonIgnore public Pattern getUrlPattern() { return Pattern.compile(getUrlRegex()); }
     public boolean matchesUrl (String value) { return getUrlPattern().matcher(value).find(); }

bubble-server/src/main/java/bubble/model/app/BubbleApp.java

@@ -21,12 +21,11 @@ import org.cobbzilla.wizard.model.entityconfig.annotations.*;
 import org.cobbzilla.wizard.validation.HasValue;
 import org.hibernate.annotations.Type;
 
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.Transient;
+import javax.persistence.*;
 import javax.validation.constraints.Size;
 
 import static bubble.ApiConstants.*;
+import static org.cobbzilla.util.daemon.ZillaRuntime.bool;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
@@ -84,7 +83,12 @@ public class BubbleApp extends IdentifiableBaseParentEntity implements AccountTe
     @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(10000+ENC_PAD)+")")
     @Getter @Setter private String description;
 
-    @Column(length=100000, nullable=false) @ECField(index=50)
+    @ECSearchable @ECField(index=50)
+    @Column(nullable=false, updatable=false)
+    @Getter @Setter private Boolean canPrime;
+    public boolean canPrime () { return bool(canPrime); }
+
+    @Column(length=100000, nullable=false) @ECField(index=60)
     @JsonIgnore @Getter @Setter private String dataConfigJson;
 
     @Transient public AppDataConfig getDataConfig () { return dataConfigJson == null ? null : ensureDefaults(json(dataConfigJson, AppDataConfig.class)); }
@@ -106,22 +110,22 @@ public class BubbleApp extends IdentifiableBaseParentEntity implements AccountTe
     // to a new node. This App will become a template/root BubbleApp for a new node, if it
     // is owned by a user and applicable to the BubblePlan (via BubblePlanApp)
     // For system apps, this can be null
-    @ECField(index=60)
+    @ECField(index=70)
     @Column(length=UUID_MAXLEN, updatable=false)
     @Getter @Setter private String templateApp;
 
-    @ECSearchable @ECField(index=70)
+    @ECSearchable @ECField(index=80)
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean template = false;
 
-    @ECSearchable @ECField(index=80)
+    @ECSearchable @ECField(index=90)
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean enabled = true;
 
-    @ECSearchable @ECField(index=90) @Column(nullable=false)
+    @ECSearchable @ECField(index=100) @Column(nullable=false)
     @ECIndex @Getter @Setter private Integer priority;
 
-    @ECSearchable @ECField(index=100)
+    @ECSearchable @ECField(index=110)
     @ECIndex @Getter @Setter private Boolean needsUpdate = false;
 
 }

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -37,10 +37,7 @@ import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.*;
@@ -167,12 +164,15 @@ public class FilterHttpResource {
             retained.add(matcher);
         }
 
-        final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), passthruRequest.getFqdn());
-        if (passthru) {
-            if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+passthruRequest.getFqdn()+"/"+passthruRequest.getAddr());
-            return ok();
+        final String[] fqdns = passthruRequest.getFqdns();
+        for (String fqdn : fqdns) {
+            final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), fqdn);
+            if (passthru) {
+                if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+fqdn+"/"+passthruRequest.getAddr());
+                return ok();
+            }
         }
-        if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdn/addr="+passthruRequest.getFqdn()+"/"+passthruRequest.getAddr());
+        if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdns/addr="+Arrays.toString(fqdns)+"/"+passthruRequest.getAddr());
         return notFound();
     }
 

bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java

@@ -14,8 +14,8 @@ public class FilterPassthruRequest {
     @Getter @Setter private String addr;
     public boolean hasAddr() { return !empty(addr); }
 
-    @Getter @Setter private String fqdn;
-    public boolean hasFqdn() { return !empty(fqdn); }
+    @Getter @Setter private String[] fqdns;
+    public boolean hasFqdns() { return !empty(fqdns); }
 
     @Getter @Setter private String remoteAddr;
     public boolean hasRemoteAddr() { return !empty(remoteAddr); }

bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java

@@ -13,6 +13,7 @@ import bubble.model.app.AppMatcher;
 import bubble.model.app.AppRule;
 import bubble.model.device.Device;
 import bubble.server.BubbleConfiguration;
+import bubble.service.stream.AppPrimerService;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
@@ -39,6 +40,7 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver {
     @Autowired protected RedisService redis;
     @Autowired protected BubbleNetworkDAO networkDAO;
     @Autowired protected DeviceDAO deviceDAO;
+    @Autowired protected AppPrimerService appPrimerService;
 
     @Getter @Setter private AppRuleDriver next;
 

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -14,13 +14,19 @@ import bubble.service.stream.AppRuleHarness;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.github.jknack.handlebars.Handlebars;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
+import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.util.Map;
+import java.util.Set;
 
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 import static org.cobbzilla.util.io.StreamUtil.stream2bytes;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
@@ -28,8 +34,38 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath;
 
 public interface AppRuleDriver {
 
+    Logger log = LoggerFactory.getLogger(AppRuleDriver.class);
+
     InputStream EMPTY_STREAM = new ByteArrayInputStream(new byte[0]);
 
+    // also used in dnscrypt-proxy/plugin_reverse_resolve_cache.go
+    String REDIS_BLOCK_LISTS = "blockLists";
+    String REDIS_FILTER_LISTS = "filterLists";
+    String REDIS_LIST_SUFFIX = "~UNION";
+
+    default Set<String> getPrimedBlockDomains () { return null; }
+    default Set<String> getPrimedFilterDomains () { return null; }
+
+    static void defineRedisBlockSet(RedisService redis, String ip, String list, String[] fullyBlockedDomains) {
+        defineRedisSet(redis, ip, REDIS_BLOCK_LISTS, list, fullyBlockedDomains);
+    }
+
+    static void defineRedisFilterSet(RedisService redis, String ip, String list, String[] filterDomains) {
+        defineRedisSet(redis, ip, REDIS_FILTER_LISTS, list, filterDomains);
+    }
+
+    static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) {
+        final String listOfListsForIp = listOfListsName + "~" + ip;
+        final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX;
+        final String ipList = listOfListsForIp + "~" + listName;
+        final String tempList = ipList + "~"+now()+randomAlphanumeric(5);
+        redis.sadd_plaintext(tempList, domains);
+        redis.rename(tempList, ipList);
+        redis.sadd_plaintext(listOfListsForIp, ipList);
+        final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp));
+        log.info("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count);
+    }
+
     AppRuleDriver getNext();
     void setNext(AppRuleDriver next);
     default boolean hasNext() { return getNext() != null; }

bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java

@@ -33,6 +33,7 @@ import java.net.URI;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
@@ -40,19 +41,30 @@ import static org.cobbzilla.util.http.HttpContentTypes.isHtml;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
 import static org.cobbzilla.util.json.JsonUtil.json;
-import static org.cobbzilla.util.string.StringUtil.UTF8cs;
-import static org.cobbzilla.util.string.StringUtil.getPackagePath;
+import static org.cobbzilla.util.string.StringUtil.*;
 
 @Slf4j
 public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
 
-    private BlockList blockList = new BlockList();
+    private final AtomicReference<BlockList> blockList = new AtomicReference<>(new BlockList());
+    private BlockList getBlockList() { return blockList.get(); }
 
-    private static Map<String, BlockListSource> blockListCache = new ConcurrentHashMap<>();
+    private final AtomicReference<Set<String>> fullyBlockedDomains = new AtomicReference<>(Collections.emptySet());
+    @Override public Set<String> getPrimedBlockDomains() { return fullyBlockedDomains.get(); }
+
+    private final AtomicReference<Set<String>> partiallyBlockedDomains = new AtomicReference<>(Collections.emptySet());
+    @Override public Set<String> getPrimedFilterDomains() { return partiallyBlockedDomains.get(); }
+
+    private final static Map<String, BlockListSource> blockListCache = new ConcurrentHashMap<>();
 
     @Override public <C> Class<C> getConfigClass() { return (Class<C>) BubbleBlockConfig.class; }
 
-    @Override public void init(JsonNode config, JsonNode userConfig, AppRule rule, AppMatcher matcher, Account account, Device device) {
+    @Override public void init(JsonNode config,
+                               JsonNode userConfig,
+                               AppRule rule,
+                               AppMatcher matcher,
+                               Account account,
+                               Device device) {
         super.init(config, userConfig, rule, matcher, account, device);
         refreshBlockLists();
     }
@@ -61,6 +73,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         final BubbleBlockConfig bubbleBlockConfig = getRuleConfig();
         final BubbleBlockList[] blockLists = bubbleBlockConfig.getBlockLists();
         final Set<String> refreshed = new HashSet<>();
+        final BlockList newBlockList = new BlockList();
         for (BubbleBlockList list : blockLists) {
             if (!list.enabled()) continue;
 
@@ -91,8 +104,22 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
                     log.error("init: error adding additional entries: "+shortError(e));
                 }
             }
-            if (blockListSource != null) blockList.merge(blockListSource.getBlockList());
+            if (blockListSource != null) newBlockList.merge(blockListSource.getBlockList());
+        }
+        blockList.set(newBlockList);
+
+        boolean doPrime = false;
+        if (!newBlockList.getFullyBlockedDomains().equals(fullyBlockedDomains.get())) {
+            fullyBlockedDomains.set(newBlockList.getFullyBlockedDomains());
+            doPrime = true;
+        }
+        if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) {
+            partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains());
+            doPrime = true;
         }
+
+        log.info("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size());
+        log.info("refreshBlockLists: partiallyBlockedDomains="+partiallyBlockedDomains.get().size());
         log.info("refreshBlockLists: refreshed "+refreshed.size()+" block lists: "+StringUtil.toString(refreshed));
     }
 
@@ -178,7 +205,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         }
     }
 
-    public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return blockList.getDecision(fqdn, uri, userAgent, false); }
+    public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return getBlockList().getDecision(fqdn, uri, userAgent, false); }
 
     public BlockDecision getDecision(String fqdn, String uri, String userAgent, boolean primary) {
         final BubbleBlockConfig bubbleBlockConfig = getRuleConfig();
@@ -189,7 +216,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
                 }
             }
         }
-        return blockList.getDecision(fqdn, uri, primary);
+        return getBlockList().getDecision(fqdn, uri, primary);
     }
 
     @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) {
@@ -199,7 +226,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
 
         // Now that we know the content type, re-check the BlockList
         final String contentType = filterRequest.getContentType();
-        final BlockDecision decision = blockList.getDecision(request.getFqdn(), request.getUri(), contentType, true);
+        final BlockDecision decision = getBlockList().getDecision(request.getFqdn(), request.getUri(), contentType, true);
         if (log.isDebugEnabled()) log.debug(prefix+"preprocess decision was "+decision+", but now we know contentType="+contentType);
         switch (decision.getDecisionType()) {
             case block:
@@ -255,8 +282,8 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver {
         ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase());
         ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId));
         ctx.put(CTX_BUBBLE_SELECTORS, json(decision.getSelectors(), COMPACT_MAPPER));
-        ctx.put(CTX_BUBBLE_WHITELIST, json(blockList.getWhitelistDomains(), COMPACT_MAPPER));
-        ctx.put(CTX_BUBBLE_BLACKLIST, json(blockList.getBlacklistDomains(), COMPACT_MAPPER));
+        ctx.put(CTX_BUBBLE_WHITELIST, json(getBlockList().getWhitelistDomains(), COMPACT_MAPPER));
+        ctx.put(CTX_BUBBLE_BLACKLIST, json(getBlockList().getBlacklistDomains(), COMPACT_MAPPER));
         return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx);
     }
 

bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java

@@ -12,6 +12,7 @@ import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.input.ReaderInputStream;
+import org.cobbzilla.util.collection.SingletonSet;
 import org.cobbzilla.util.io.regex.RegexFilterReader;
 import org.cobbzilla.util.io.regex.RegexInsertionFilter;
 import org.cobbzilla.util.io.regex.RegexStreamFilter;
@@ -19,6 +20,7 @@ import org.cobbzilla.util.io.regex.RegexStreamFilter;
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.http.HttpContentTypes.isHtml;
@@ -51,6 +53,10 @@ public class UserBlockerRuleDriver extends AbstractAppRuleDriver {
         return json(json, JsonNode.class);
     }
 
+    @Override public Set<String> getPrimedFilterDomains() {
+        return matcher.isWildcardFqdn() ? null : new SingletonSet<>(matcher.getFqdn());
+    }
+
     protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); }
 
     @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) {

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -88,6 +88,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration
     public static final String TAG_PROMO_CODE_POLICY = "promoCodePolicy";
     public static final String TAG_REQUIRE_SEND_METRICS = "requireSendMetrics";
     public static final String TAG_SUPPORT = "support";
+    public static final String TAG_CERT_VALIDATION_HOST = "certValidationHost"; // must match bubble_passthru.py
 
     public static final String DEFAULT_LOCAL_STORAGE_DIR = HOME_DIR + "/.bubble_local_storage";
 
@@ -137,6 +138,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration
     public boolean hasSageNode () { return getSageNode() != null; }
 
     @Getter @Setter private String letsencryptEmail;
+    @Getter @Setter private String certValidationHost;
+
     @Setter private String localStorageDir = DEFAULT_LOCAL_STORAGE_DIR;
     public String getLocalStorageDir () { return empty(localStorageDir) ? DEFAULT_LOCAL_STORAGE_DIR : localStorageDir; }
 
@@ -297,7 +300,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration
                         {TAG_LOCKED, accountDAO.locked()},
                         {TAG_LAUNCH_LOCK, isSageLauncher() || thisNetwork == null ? null : thisNetwork.launchLock()},
                         {TAG_SSL_PORT, getDefaultSslPort()},
-                        {TAG_SUPPORT, getSupport()}
+                        {TAG_SUPPORT, getSupport()},
+                        {TAG_CERT_VALIDATION_HOST, getCertValidationHost()}
                 }));
             }
             return publicSystemConfigs.get();

bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java

@@ -7,18 +7,23 @@ package bubble.server.listener;
 import bubble.dao.account.AccountDAO;
 import bubble.dao.cloud.CloudServiceDAO;
 import bubble.model.account.Account;
+import bubble.model.cloud.AnsibleInstallType;
+import bubble.model.cloud.BubbleNetwork;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.CloudService;
 import bubble.server.BubbleConfiguration;
 import bubble.service.boot.SelfNodeService;
 import bubble.service.cloud.NetworkMonitorService;
+import bubble.service.stream.AppPrimerService;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.cobbzilla.wizard.server.RestServer;
 import org.cobbzilla.wizard.server.RestServerLifecycleListenerBase;
 
 import java.io.File;
 import java.util.Map;
 
+import static bubble.server.BubbleConfiguration.TAG_CERT_VALIDATION_HOST;
 import static bubble.service.boot.StandardSelfNodeService.SELF_NODE_JSON;
 import static bubble.service.boot.StandardSelfNodeService.THIS_NODE_FILE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
@@ -101,6 +106,18 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase<Bub
             }
         }
 
+        // ensure default devices exist, cert_validation_host is set, and apps are primed
+        if (thisNode != null) {
+            final BubbleNetwork thisNetwork = c.getThisNetwork();
+            if (thisNetwork != null && thisNetwork.getInstallType() == AnsibleInstallType.node) {
+                final String certValidationHost = c.getCertValidationHost();
+                if (!empty(certValidationHost)) {
+                    c.getBean(RedisService.class).set(TAG_CERT_VALIDATION_HOST, certValidationHost);
+                }
+                c.getBean(AppPrimerService.class).primeApps();
+            }
+        }
+
         log.info("onStart: completed");
     }
 

bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java

@@ -81,6 +81,7 @@ public class AnsiblePrepService {
         }
         ctx.put("sslPort", network.getSslPort());
         ctx.put("publicBaseUri", network.getPublicUri());
+        ctx.put("cert_validation_host", configuration.getCertValidationHost());
         ctx.put("support", configuration.getSupport());
         ctx.put("appLinks", configuration.getAppLinks());
 

bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java

@@ -42,7 +42,7 @@ public class DeviceIdService {
     @Autowired private BubbleConfiguration configuration;
     @Autowired private AccountDAO accountDAO;
 
-    private Map<String, Device> deviceCache = new ExpirationMap<>(MINUTES.toMillis(10));
+    private final Map<String, Device> deviceCache = new ExpirationMap<>(MINUTES.toMillis(10));
 
     public Device findDeviceByIp (String ipAddr) {
 

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -74,8 +74,7 @@ import static bubble.service.boot.StandardSelfNodeService.*;
 import static bubble.service.cloud.NodeProgressMeter.getProgressMeterKey;
 import static bubble.service.cloud.NodeProgressMeter.getProgressMeterPrefix;
 import static bubble.service.cloud.NodeProgressMeterConstants.*;
-import static java.util.concurrent.TimeUnit.MINUTES;
-import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.concurrent.TimeUnit.*;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
 import static org.cobbzilla.util.daemon.Await.awaitAll;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;

bubble-server/src/main/java/bubble/service/stream/AppPrimerService.java

@@ -0,0 +1,14 @@
+package bubble.service.stream;
+
+import bubble.model.account.Account;
+import bubble.model.app.BubbleApp;
+
+public interface AppPrimerService {
+
+    void primeApps();
+
+    void prime(BubbleApp app);
+
+    void prime(Account account, String app);
+
+}

bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java

@@ -0,0 +1,165 @@
+package bubble.service.stream;
+
+import bubble.dao.account.AccountDAO;
+import bubble.dao.app.AppMatcherDAO;
+import bubble.dao.app.AppRuleDAO;
+import bubble.dao.app.BubbleAppDAO;
+import bubble.dao.app.RuleDriverDAO;
+import bubble.dao.device.DeviceDAO;
+import bubble.model.account.Account;
+import bubble.model.app.AppMatcher;
+import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
+import bubble.model.app.RuleDriver;
+import bubble.model.cloud.AnsibleInstallType;
+import bubble.model.cloud.BubbleNetwork;
+import bubble.model.device.Device;
+import bubble.rule.AppRuleDriver;
+import bubble.server.BubbleConfiguration;
+import bubble.service.cloud.DeviceIdService;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.collection.SingletonList;
+import org.cobbzilla.wizard.cache.redis.RedisService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.stream.Collectors;
+
+import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+
+@Service @Slf4j
+public class StandardAppPrimerService implements AppPrimerService {
+
+    @Autowired private AccountDAO accountDAO;
+    @Autowired private DeviceDAO deviceDAO;
+    @Autowired private DeviceIdService deviceIdService;
+    @Autowired private BubbleAppDAO appDAO;
+    @Autowired private AppMatcherDAO matcherDAO;
+    @Autowired private AppRuleDAO ruleDAO;
+    @Autowired private RuleDriverDAO driverDAO;
+    @Autowired private RedisService redis;
+    @Autowired private BubbleConfiguration configuration;
+
+    @Getter(lazy=true) private final boolean primingEnabled = initPrimingEnabled();
+    private boolean initPrimingEnabled() {
+        if (configuration == null) return die("initPrimingEnabled: configuration was null");
+        final BubbleNetwork thisNetwork = configuration.getThisNetwork();
+        if (thisNetwork == null) {
+            log.info("primeApps: thisNetwork is null, not priming");
+            return false;
+        }
+        if (thisNetwork.getInstallType() != AnsibleInstallType.node) {
+            log.info("primeApps: thisNetwork is not a node, not priming");
+            return false;
+        }
+        return true;
+    }
+
+    public void primeApps() {
+        if (!isPrimingEnabled()) {
+            log.info("primeApps: not enabled");
+            return;
+        }
+        for (Account account : accountDAO.findNotDeleted()) {
+            try {
+                prime(account);
+            } catch (Exception e) {
+                log.error("primeApps("+account.getName()+"): "+shortError(e), e);
+            }
+        }
+    }
+
+    public void prime(Account account) { prime(account, (BubbleApp) null); }
+
+    public void prime(BubbleApp app) {
+        final Account account = accountDAO.findByUuid(app.getAccount());
+        if (account == null) {
+            log.warn("prime("+app.getName()+"): account not found: "+app.getAccount());
+            return;
+        }
+        prime(account, app.getUuid());
+    }
+
+    public synchronized void prime(Account account, String singleAppUuid) {
+        final BubbleApp singleApp = appDAO.findByAccountAndId(account.getUuid(), singleAppUuid);
+        if (singleApp == null) {
+            log.warn("prime("+account.getName()+", "+singleAppUuid+"): app not found: "+singleAppUuid);
+            return;
+        }
+        prime(account, singleApp);
+    }
+
+    @Getter(lazy=true) private final ExecutorService primerThread = fixedPool(1);
+
+    private void prime(Account account, BubbleApp singleApp) {
+        if (!isPrimingEnabled()) {
+            log.info("prime: not enabled");
+            return;
+        }
+        getPrimerThread().submit(() -> _prime(account, singleApp));
+    }
+
+    private synchronized void _prime(Account account, BubbleApp singleApp) {
+        try {
+            final Map<String, List<String>> accountDeviceIps = new HashMap<>();
+            final List<Device> devices = deviceDAO.findByAccount(account.getUuid());
+            for (Device device : devices) {
+                accountDeviceIps.put(device.getUuid(), deviceIdService.findIpsByDevice(device.getUuid()));
+            }
+            if (accountDeviceIps.isEmpty()) return;
+
+            final List<BubbleApp> appsToPrime = singleApp == null
+                    ? appDAO.findByAccount(account.getUuid()).stream()
+                    .filter(BubbleApp::canPrime)
+                    .collect(Collectors.toList())
+                    : new SingletonList<>(singleApp);
+            for (BubbleApp app : appsToPrime) {
+                final List<AppRule> rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
+                final List<AppMatcher> matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
+                for (AppRule rule : rules) {
+                    final RuleDriver driver = driverDAO.findByUuid(rule.getDriver());
+                    if (driver == null) {
+                        log.warn("_prime: driver not found for app/rule " + app.getName() + "/" + rule.getName() + ": " + rule.getDriver());
+                        continue;
+                    }
+                    for (Device device : devices) {
+                        final Set<String> blockDomains = new HashSet<>();
+                        final Set<String> filterDomains = new HashSet<>();
+                        for (AppMatcher matcher : matchers) {
+                            final AppRuleDriver appRuleDriver = rule.initDriver(driver, matcher, account, device);
+                            final Set<String> blocks = appRuleDriver.getPrimedBlockDomains();
+                            if (empty(blocks)) {
+                                log.info("_prime: no blockDomains for app/rule/matcher: " + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                            } else {
+                                blockDomains.addAll(blocks);
+                            }
+                            final Set<String> filters = appRuleDriver.getPrimedFilterDomains();
+                            if (empty(filters)) {
+                                log.info("_prime: no filterDomains for app/rule/matcher: " + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                            } else {
+                                filterDomains.addAll(filters);
+                            }
+                        }
+                        if (!empty(blockDomains) || !empty(filterDomains)) {
+                            for (String ip : accountDeviceIps.get(device.getUuid())) {
+                                if (!empty(blockDomains)) {
+                                    AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new));
+                                }
+                                if (!empty(filterDomains)) {
+                                    AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new));
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            die("_prime: "+shortError(e), e);
+        }
+    }
+
+}

bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java

@@ -173,7 +173,8 @@ public class StandardRuleEngineService implements RuleEngineService {
         return sendResponse(state.getResponseStream(last));
     }
 
-    private ExpirationMap<String, List<AppRuleHarness>> ruleCache = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime);
+    private final ExpirationMap<String, List<AppRuleHarness>> ruleCache
+            = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime);
 
     public Map<Object, Object> flushCaches() {
         final int ruleEngineCacheSize = ruleCache.size();

bubble-server/src/main/java/bubble/service_dbfilter/DbFilterAppPrimerService.java

@@ -0,0 +1,19 @@
+package bubble.service_dbfilter;
+
+import bubble.model.account.Account;
+import bubble.model.app.BubbleApp;
+import bubble.service.stream.AppPrimerService;
+import org.springframework.stereotype.Service;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported;
+
+@Service
+public class DbFilterAppPrimerService implements AppPrimerService {
+
+    @Override public void primeApps() { notSupported("primeApps"); }
+
+    @Override public void prime(Account account, String app) { notSupported("prime"); }
+
+    @Override public void prime(BubbleApp app) { notSupported("prime"); }
+
+}

bubble-server/src/main/resources/META-INF/bubble/bubble.properties

@@ -1 +1 @@
-bubble.version=0.11.3
+bubble.version=0.12.0

bubble-server/src/main/resources/ansible/install_local.sh.hbs

@@ -22,20 +22,17 @@ if [[ "$(whoami)" != "{{node.user}}" ]] ; then
 fi
 
 ANSIBLE_DIR="${ANSIBLE_HOME}/ansible"
-ID_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa"
-PUB_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa.pub"
 AUTH_KEYS="${ANSIBLE_HOME}/.ssh/authorized_keys"
 
 if [[ ! -d "${ANSIBLE_DIR}" ]] ; then
   die "Ansible dir not found or not a directory: ${ANSIBLE_DIR}"
 fi
 
-if [[ ! -f "${ID_FILE}" ]] ; then
-  ssh-keygen -t rsa -q -N '' -f ${ID_FILE} || die "Error generating RSA key"
-fi
-
 # lockout the node that started us
-cat "${PUB_FILE}" > "${AUTH_KEYS}" || die "Error updating ${AUTH_KEYS} file"
+cat /dev/null > "${AUTH_KEYS}" || die "Error truncating ${AUTH_KEYS} file"
+
+# ensure proper permissions on authorized_keys file
+chmod 600 "${AUTH_KEYS}" || die "Error setting permissions on ${AUTH_KEYS} file"
 
 # add admin ssh key, if one was given
 ADMIN_PUB_KEY="${ANSIBLE_DIR}/roles/bubble/files/admin_ssh_key.pub"
@@ -43,8 +40,6 @@ if [[ -f "${ADMIN_PUB_KEY}" ]] ; then
   cat "${ADMIN_PUB_KEY}" >> "${AUTH_KEYS}"
 fi
 
-SSH_OPTIONS="--ssh-extra-args '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=publickey -i ${ID_FILE}'"
-
 SKIP_TAGS=""
 if [[ -n "{{restoreKey}}" ]] ; then
   SKIP_TAGS="--skip-tags algo_related"
@@ -55,5 +50,5 @@ cd "${ANSIBLE_DIR}" && \
   virtualenv -p python3 ./venv && \
   . ./venv/bin/activate && \
   pip3 install ansible && \
-  bash -c "ansible-playbook ${SSH_OPTIONS} ${SKIP_TAGS} --inventory ./hosts ./playbook.yml 2>&1 | tee ${LOG}" \
+  ansible-playbook ${SKIP_TAGS} --inventory ./hosts ./playbook.yml \
 || die "Error running ansible. journalctl -xe = $(journalctl -xe | tail -n 50)"

bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml

@@ -15,12 +15,8 @@
   when: restore_key is not defined
 
 # Don't start monitors when in restore mode, bubble_restore_monitor.sh will start it after algo is installed
-- name: Restart algo monitors
-  shell: bash -c "supervisorctl reload && sleep 5s && supervisorctl restart algo_refresh_users_monitor && supervisorctl restart wg_monitor_connections"
-  when: restore_key is not defined
-
 - name: Stop algo monitors (in restore mode)
-  shell: bash -c "supervisorctl reload && sleep 5s && supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections"
+  shell: bash -c "supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections"
   when: restore_key is defined
 
 # Add bubble rules

bubble-server/src/main/resources/ansible/roles/algo/templates/install_algo.sh.j2

@@ -43,4 +43,10 @@ CONFIGS_BACKUP=/home/bubble/.BUBBLE_ALGO_CONFIGS.tgz
 cd ${ALGO_BASE} && tar czf ${CONFIGS_BACKUP} configs && chgrp bubble ${CONFIGS_BACKUP} && chmod 660 ${CONFIGS_BACKUP} || die "Error backing up algo configs"
 cd /home/bubble && tar xzf ${CONFIGS_BACKUP} && chgrp -R bubble configs && chown -R bubble configs && chmod 500 configs || die "Error unpacking algo configs to bubble home"
 
+# Restart algo_refresh_users_monitor and wg_monitor_connections
+supervisorctl reload && sleep 5s && supervisorctl restart algo_refresh_users_monitor && supervisorctl restart wg_monitor_connections
+
+# Restart dnscrypt-proxy
+service dnscrypt-proxy restart
+
 touch "${ALGO_BASE}/.install_marker"

bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json

@@ -30,6 +30,8 @@
     {"name": "error_key", "value": "[[error_key]]"},
     {"name": "error_env", "value": "[[error_env]]"},
 
+    {"name": "cert_validation_host", "value": "[[cert_validation_host]]"},
+
     {"name": "support_email", "value": "[[support.email]]"},
     {"name": "support_site", "value": "[[support.site]]"},
 

bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2

@@ -12,6 +12,7 @@ export ERRBIT_URL={{ error_url | default('') }}
 export ERRBIT_KEY={{ error_key | default('') }}
 export ERRBIT_ENV={{ error_env | default('') }}
 
+export CERT_VALIDATION_HOST={{ cert_validation_host }}
 export SUPPORT_EMAIL={{ support_email }}
 export SUPPORT_SITE={{ support_site }}
 

bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml

@@ -1,8 +1,8 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
-- name: Snapshot ansible roles
-  shell: snapshot_ansible.sh
+- name: Snapshot ansible roles in the background
+  command: /usr/local/bin/snapshot_ansible.sh
 
 - name: Touch first-time setup file
   shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n install > /home/bubble/first_time_marker ; fi"

bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml

@@ -20,9 +20,6 @@
 
 - import_tasks: route.yml
 
-- name: Restart dnscrypt-proxy
-  shell: service dnscrypt-proxy restart
-
 - name: Install supervisor conf file
   copy:
     src: supervisor_mitmproxy.conf

bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml

@@ -16,7 +16,6 @@
   template: src=stronger_dhparams.conf dest=/etc/nginx/conf.d/stronger_dhparams.conf
 
 - include: site.yml
-- meta: flush_handlers # nginx has to be restarted right now if it has to
 
 - name: Init certbot
   shell: init_certbot.sh {{ letsencrypt_email }} {{ server_name }} {{ server_alias }}

bubble-server/src/main/resources/bubble-config.yml

@@ -88,6 +88,8 @@ rateLimits:
   - { limit: 50000, interval: 1h, block: 24h }
   - { limit: 100000, interval: 6h, block: 96h }
 
+certValidationHost: {{CERT_VALIDATION_HOST}}
+
 support:
   email: {{SUPPORT_EMAIL}}
   site: {{SUPPORT_SITE}}

bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties

@@ -20,6 +20,12 @@ support_email_link=Send us an email
 support_not_available=Sorry, no support options are available
 link_support=Support
 
+# Cert checker
+title_check_certificate=Bubble Certificate Verification
+check_cert_null=Verifying certification installation...
+check_cert_true=Your Bubble Certificate is properly installed
+check_cert_false=Your Bubble Certificate is not properly installed
+
 # Legal page links
 title_legal_topics=Legal Stuff
 legal_topics=terms,privacy,source,license,3rdParty_licenses

bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json

@@ -5,6 +5,7 @@
   "template": true,
   "enabled": true,
   "priority": 100,
+  "canPrime": false,
   "dataConfig": {
     "dataDriver": "bubble.app.analytics.TrafficAnalyticsAppDataDriver",
     "presentation": "app",

bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json

@@ -5,6 +5,7 @@
   "template": true,
   "enabled": true,
   "priority": 200,
+  "canPrime": true,
   "dataConfig": {
     "dataDriver": "bubble.app.bblock.BubbleBlockAppDataDriver",
     "presentation": "app",
@@ -16,8 +17,7 @@
       {"name": "data"}
     ],
     "params": [
-      {"name": "device", "required": false, "index": 10},
-      {"name": "meta2", "required": false, "operator": "like", "index": 20}
+      {"name": "device", "required": false, "index": 10}
     ],
     "views": [
       {"name": "last_24_hours"},

bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json

@@ -5,6 +5,7 @@
   "template": true,
   "enabled": true,
   "priority": 1000000,
+  "canPrime": false,
   "dataConfig": {
     "dataDriver": "bubble.app.passthru.TlsPassthruAppDataDriver",
     "presentation": "none",

bubble-server/src/main/resources/models/apps/user_block/bubbleApp_userBlock.json

@@ -5,6 +5,7 @@
   "template": true,
   "enabled": true,
   "priority": 300,
+  "canPrime": true,
   "dataConfig": {
     "dataDriver": "bubble.app.social.block.UserBlockerAppDataDriver",
     "presentation": "site",

bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml

@@ -13,7 +13,7 @@
   get_url:
     url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip
     dest: /tmp/algo.zip
-    checksum: sha256:f9709ced6cbaf92473c975355050915657da50dfc0eaf655f178643953a9cd42
+    checksum: sha256:357e613833385626e88564c97f0b5726f49686c33a774be9a7766bd1a1249915
 
 - name: Unzip algo master.zip
   unarchive:

bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh

@@ -114,8 +114,6 @@ if [[ ! -f ${CONFIGS_BACKUP} ]] ; then
 else
   ANSIBLE_HOME="/root"
   ANSIBLE_DIR="${ANSIBLE_HOME}/ansible"
-  ID_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa"
-  SSH_OPTIONS="--ssh-extra-args '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=publickey -i ${ID_FILE}'"
 
   ALGO_BASE=${ANSIBLE_DIR}/roles/algo/algo
   if [[ ! -d ${ALGO_BASE} ]] ; then
@@ -126,7 +124,7 @@ else
   cd "${ANSIBLE_DIR}" && \
     . ./venv/bin/activate && \
     bash -c \
-      "ansible-playbook ${SSH_OPTIONS} --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \
+      "ansible-playbook --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \
   || die "Error running ansible in post-restore. journalctl -xe = $(journalctl -xe | tail -n 50)"
 fi
 

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -49,14 +49,14 @@ def bubble_activity_log(client_addr, server_addr, event, data):
         'client_addr': client_addr,
         'server_addr': server_addr,
         'event': event,
-        'data': data
+        'data': str(data)
     })
     bubble_log('bubble_activity_log: setting '+key+' = '+value)
     redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION)
     pass
 
 
-def bubble_passthru(remote_addr, addr, fqdn):
+def bubble_passthru(remote_addr, addr, fqdns):
     headers = {
         'X-Forwarded-For': remote_addr,
         'Accept' : 'application/json',
@@ -65,7 +65,7 @@ def bubble_passthru(remote_addr, addr, fqdn):
     try:
         data = {
             'addr': str(addr),
-            'fqdn': str(fqdn),
+            'fqdns': fqdns,
             'remoteAddr': remote_addr
         }
         response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data)

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py

@@ -29,18 +29,69 @@ from mitmproxy.exceptions import TlsProtocolException
 from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set
 import redis
 import json
+import subprocess
 
 REDIS_DNS_PREFIX = 'bubble_dns_'
 REDIS_PASSTHRU_PREFIX = 'bubble_passthru_'
+REDIS_CLIENT_CERT_STATUS_PREFIX = 'bubble_cert_status_'
 REDIS_PASSTHRU_DURATION = 60 * 60  # 1 hour timeout on passthru
 
 REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
 
+cert_validation_host = None
+local_ips = None
+
+
+def get_ip_cert_status(client_addr):
+    status = REDIS.get(REDIS_CLIENT_CERT_STATUS_PREFIX+client_addr)
+    if status is None:
+        return None
+    enabled = status.decode() == "True"
+    return enabled
+
+
+def set_ip_cert_status(client_addr, enabled):
+    REDIS.set(REDIS_CLIENT_CERT_STATUS_PREFIX+client_addr, str(enabled))
+    bubble_log('set_ip_cert_status: set '+client_addr+' = '+str(enabled))
+
+
+def get_local_ips():
+    global local_ips
+    if local_ips is None:
+        local_ips = []
+        for ip in subprocess.check_output(['hostname', '-I']).split():
+            local_ips.append(ip.decode())
+    return local_ips
+
+
+def get_cert_validation_host():
+    global cert_validation_host
+    if cert_validation_host is None:
+        cert_validation_host = REDIS.get('certValidationHost')
+        if cert_validation_host is not None:
+            cert_validation_host = cert_validation_host.decode()
+    #     bubble_log('get_cert_validation_host: initialized to '+cert_validation_host)
+    # bubble_log('get_cert_validation_host: returning '+cert_validation_host)
+    return cert_validation_host
+
 
 def passthru_cache_prefix(client_addr, server_addr):
     return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr
 
 
+def fqdns_for_addr(addr):
+    prefix = REDIS_DNS_PREFIX + addr
+    keys = REDIS.keys(prefix + '_*')
+    if keys is None or len(keys) == 0:
+        bubble_log('fqdns_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr')
+        return ''
+    fqdns = []
+    for k in keys:
+        fqdn = k.decode()[len(prefix)+1:]
+        fqdns.append(fqdn)
+    return fqdns
+
+
 class TlsFeedback(TlsLayer):
     """
     Monkey-patch _establish_tls_with_client to get feedback if TLS could be established
@@ -49,51 +100,61 @@ class TlsFeedback(TlsLayer):
     def _establish_tls_with_client(self):
         client_address = self.client_conn.address[0]
         server_address = self.server_conn.address[0]
+        fqdns = fqdns_for_addr(server_address)
         try:
             super(TlsFeedback, self)._establish_tls_with_client()
+            if fqdns and get_cert_validation_host() in fqdns:
+                # bubble_log('_establish_tls_with_client: TLS success for '+repr(server_address)+', enabling SSL interception for client '+client_address)
+                set_ip_cert_status(client_address, True)
+
         except TlsProtocolException as e:
-            bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru')
             cache_key = passthru_cache_prefix(client_address, server_address)
-            fqdn = fqdn_for_addr(server_address)
-            redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION)
+            bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key)
+            if fqdns and get_cert_validation_host() in fqdns:
+                set_ip_cert_status(client_address, False)
+            else:
+                redis_set(cache_key, json.dumps({'fqdns': fqdns, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION)
             raise e
 
 
-def fqdn_for_addr(addr):
-    fqdn = REDIS.get(REDIS_DNS_PREFIX + addr)
-    if fqdn is None or len(fqdn) == 0:
-        bubble_log('fqdn_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr')
-        fqdn = b''
-    return fqdn.decode()
+def check_bubble_passthru(client_addr, addr, fqdns):
+    cert_status = get_ip_cert_status(client_addr)
 
+    if cert_status is not None and not cert_status:
+        bubble_log('check_bubble_passthru: returning True because cert_status for '+client_addr+' was False')
+        return {'fqdns': fqdns, 'addr': addr, 'passthru': True}
+    else:
+        bubble_log('check_bubble_passthru: request is NOT for cert_validation_host: '+cert_validation_host+", it is for one of fqdn="+repr(fqdns)+", checking bubble_passthru...")
 
-def check_bubble_passthru(remote_addr, addr, fqdn):
-    passthru = bubble_passthru(remote_addr, addr, fqdn)
-    if passthru is None:
-        return None
-    if passthru:
-        bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True')
-        return {'fqdn': fqdn, 'addr': addr, 'passthru': True}
-    bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False')
-    return {'fqdn': fqdn, 'addr': addr, 'passthru': False}
+    passthru = bubble_passthru(client_addr, addr, fqdns)
+    if passthru is None or passthru:
+        bubble_log('check_bubble_passthru: bubble_passthru returned '+repr(passthru)+' for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning True')
+        return {'fqdns': fqdns, 'addr': addr, 'passthru': True}
+    bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning False')
+    return {'fqdns': fqdns, 'addr': addr, 'passthru': False}
+
+
+def should_passthru(client_addr, addr, fqdns):
+    # always passthru for local ips
+    if addr in get_local_ips():
+        # bubble_log('should_passthru: local ip is always passthru: '+addr)
+        return {'fqdns': fqdns, 'addr': addr, 'passthru': True}
+    else:
+        # bubble_log('should_passthru: addr ('+addr+') is not a local ip: '+repr(get_local_ips()))
+        pass
 
+    cache_key = passthru_cache_prefix(client_addr, addr)
+    prefix = 'should_passthru: ip='+repr(addr)+' (fqdns='+repr(fqdns)+') cache_key='+cache_key+': '
 
-def should_passthru(remote_addr, addr):
-    prefix = 'should_passthru: '+repr(addr)+' '
-    bubble_log(prefix+'starting...')
-    cache_key = passthru_cache_prefix(remote_addr, addr)
     passthru_json = REDIS.get(cache_key)
     if passthru_json is None or len(passthru_json) == 0:
-        bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...')
-        fqdn = fqdn_for_addr(addr)
-        if fqdn is None or len(fqdn) == 0:
-            fqdn = 'NONE'
-        passthru = check_bubble_passthru(remote_addr, addr, fqdn)
-        bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...")
-        if passthru is not None:
-            redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION)
+        bubble_log(prefix+'not in redis or empty, calling check_bubble_passthru against fqdns='+repr(fqdns))
+        passthru = check_bubble_passthru(client_addr, addr, fqdns)
+        bubble_log(prefix+'check_bubble_passthru('+repr(fqdns)+') returned '+repr(passthru)+", storing in redis...")
+        redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION)
+
     else:
-        bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis')
+        bubble_log(prefix+'found passthru_json='+str(passthru_json)+', touching key in redis')
         passthru = json.loads(passthru_json)
         REDIS.touch(cache_key)
     bubble_log(prefix+'returning '+repr(passthru))
@@ -104,13 +165,23 @@ def next_layer(next_layer):
     if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
         client_address = next_layer.client_conn.address[0]
         server_address = next_layer.server_conn.address[0]
-        passthru = should_passthru(client_address, server_address)
-        if passthru is None or passthru['passthru']:
-            bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address))
-            if passthru is not None and 'fqdn' in passthru:
-                bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn'])
-            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
-            next_layer.reply.send(next_layer_replacement)
-        else:
-            bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn'])
+
+        fqdns = fqdns_for_addr(server_address)
+        validation_host = get_cert_validation_host()
+        if fqdns and validation_host in fqdns:
+            bubble_log('next_layer: never passing thru (always getting feedback) for cert_validation_host='+validation_host)
             next_layer.__class__ = TlsFeedback
+
+        else:
+            bubble_log('next_layer: checking should_passthru for client_address='+client_address)
+            passthru = should_passthru(client_address, server_address, fqdns)
+            if passthru is None or passthru['passthru']:
+                # bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address))
+                if passthru is not None and 'fqdns' in passthru:
+                    bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdns'])
+                next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
+                next_layer.reply.send(next_layer_replacement)
+            else:
+                # bubble_log('next_layer: NO PASSTHRU (getting feedback) for client_address='+client_address+', server_address='+server_address)
+                bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdns'])
+                next_layer.__class__ = TlsFeedback

bubble-server/src/test/resources/models/tests/promo/account_credit.json

@@ -19,16 +19,16 @@
 
   {
     "before": "sleep 5s",
-    "comment": "root: check email inbox for verification message",
+    "comment": "root: check email inbox for welcome message",
     "request": {
       "session": "rootSession",
-      "uri": "debug/inbox/email/account_credit_user@example.com?type=request&action=verify&target=account"
+      "uri": "debug/inbox/email/account_credit_user@example.com?type=notice&action=welcome&target=account"
     },
     "response": {
       "store": "emailInbox",
       "check": [
-        {"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"},
-        {"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"},
+        {"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"},
+        {"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"},
         {"condition": "'{{json.[0].ctx.message.target}}' == 'account'"}
       ]
     }

bubble-server/src/test/resources/models/tests/promo/multi_promo.json

@@ -47,16 +47,16 @@
   },
 
   {
-    "comment": "root: check email inbox for verification message for referring user",
+    "comment": "root: check email inbox for welcome message for referring user",
     "request": {
       "session": "rootSession",
-      "uri": "debug/inbox/email/test_user_referring_multi@example.com?type=request&action=verify&target=account"
+      "uri": "debug/inbox/email/test_user_referring_multi@example.com?type=notice&action=welcome&target=account"
     },
     "response": {
       "store": "emailInbox",
       "check": [
-        {"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"},
-        {"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"},
+        {"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"},
+        {"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"},
         {"condition": "'{{json.[0].ctx.message.target}}' == 'account'"}
       ]
     }

bubble-server/src/test/resources/models/tests/promo/referral_month_free.json

@@ -42,16 +42,16 @@
   },
 
   {
-    "comment": "root: check email inbox for verification message for referring user",
+    "comment": "root: check email inbox for welcome message for referring user",
     "request": {
       "session": "rootSession",
-      "uri": "debug/inbox/email/test_user_referring_free@example.com?type=request&action=verify&target=account"
+      "uri": "debug/inbox/email/test_user_referring_free@example.com?type=notice&action=welcome&target=account"
     },
     "response": {
       "store": "emailInbox",
       "check": [
-        {"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"},
-        {"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"},
+        {"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"},
+        {"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"},
         {"condition": "'{{json.[0].ctx.message.target}}' == 'account'"}
       ]
     }

utils/abp-parser

@@ -1 +1 @@
-Subproject commit e25605ea6a578609b47d2f7c9da9dc87debbb8bb
+Subproject commit df343fc4b3e1f123b3caa025a493861edf46b81a

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 17d93ddf2d3436228504ec98a52008efa665d339
+Subproject commit d450510d6f1be5b328b919ae73d8c1450f008d91