diff --git a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java index 2d574c5d..7aad6710 100644 --- a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java +++ b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java @@ -452,4 +452,7 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc @NonNull public List findDeleted() { return list(criteria().add(isNotNull("deleted"))); } + + @NonNull public List findNotDeleted() { return findByField("deleted", null); } + } diff --git a/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java b/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java index 1bdf0aca..d0b92684 100644 --- a/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java +++ b/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java @@ -46,18 +46,15 @@ public class AccountSshKeyDAO extends AccountOwnedEntityDAO { if (key.hasExpiration() && key.expired()) throw invalidEx("err.expiration.cannotCreateSshKeyAlreadyExpired"); final Account owner = accountDAO.findByUuid(key.getAccount()); - if (key.installSshKey()) { - if (owner.admin()) { - // admin keys are always installed on a node - // never allow installation of a key on sage. must be manually set in the database. - final BubbleNetwork thisNetwork = configuration.getThisNetwork(); - if (thisNetwork != null && thisNetwork.getInstallType() == AnsibleInstallType.node) { - key.setInstallSshKey(true); - } - } else { - // never install key for non-admin - key.setInstallSshKey(false); - } + final BubbleNetwork thisNetwork = configuration.getThisNetwork(); + if (thisNetwork == null || thisNetwork.getInstallType() != AnsibleInstallType.sage) { + // never allow installation of a key on sage. must be manually set in the database. + key.setInstallSshKey(false); + + } else { + // admin keys are always installed on a node + // never install key for non-admin + key.setInstallSshKey(owner.admin() && thisNetwork.getInstallType() == AnsibleInstallType.node); } final String hash = sha256_hex(key.getSshPublicKey()); diff --git a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java index 37eafa79..f29ac991 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java @@ -9,6 +9,7 @@ import bubble.model.account.Account; import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; import bubble.model.app.BubbleApp; +import bubble.service.stream.AppPrimerService; import bubble.service.stream.RuleEngineService; import lombok.extern.slf4j.Slf4j; import org.hibernate.criterion.Order; @@ -17,6 +18,7 @@ import org.springframework.stereotype.Repository; import java.util.List; +import static bubble.model.app.AppMatcher.WILDCARD_FQDN; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.wizard.model.Identifiable.MTIME; import static org.hibernate.criterion.Restrictions.*; @@ -28,6 +30,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { @Autowired private BubbleAppDAO appDAO; @Autowired private AppRuleDAO ruleDAO; @Autowired private RuleEngineService ruleEngineService; + @Autowired private AppPrimerService primerService; @Override public Order getDefaultSortOrder() { return PRIORITY_ASC; } @@ -39,7 +42,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { eq("passthru", false), or( eq("fqdn", fqdn), - eq("fqdn", "*") + eq("fqdn", WILDCARD_FQDN) )) ).addOrder(PRIORITY_ASC)); } @@ -88,6 +91,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { } } ruleEngineService.flushCaches(); + primerService.prime(app); return super.postUpdate(matcher, context); } diff --git a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java index 88f9b107..f319784b 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java +++ b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java @@ -48,6 +48,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "passthru"); public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE); + public static final String WILDCARD_FQDN = "*"; + public static final String WILDCARD_URL = ".*"; public AppMatcher(AppMatcher other) { copy(this, other, CREATE_FIELDS); @@ -80,12 +82,14 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H @Size(max=1024, message="err.fqdn.length") @ECIndex @Column(nullable=false, length=1024) @Getter @Setter private String fqdn; + @JsonIgnore @Transient public boolean isWildcardFqdn () { return fqdn != null && fqdn.equals(WILDCARD_FQDN); } @ECSearchable(filter=true) @ECField(index=60) @HasValue(message="err.urlRegex.required") @Size(max=1024, message="err.urlRegex.length") @Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(1024+ENC_PAD)+") NOT NULL") @Getter @Setter private String urlRegex; + public boolean hasUrlRegex() { return !empty(urlRegex) && !urlRegex.equals(WILDCARD_URL); } @Transient @JsonIgnore public Pattern getUrlPattern() { return Pattern.compile(getUrlRegex()); } public boolean matchesUrl (String value) { return getUrlPattern().matcher(value).find(); } diff --git a/bubble-server/src/main/java/bubble/model/app/BubbleApp.java b/bubble-server/src/main/java/bubble/model/app/BubbleApp.java index 44d4af86..986cbfef 100644 --- a/bubble-server/src/main/java/bubble/model/app/BubbleApp.java +++ b/bubble-server/src/main/java/bubble/model/app/BubbleApp.java @@ -21,12 +21,11 @@ import org.cobbzilla.wizard.model.entityconfig.annotations.*; import org.cobbzilla.wizard.validation.HasValue; import org.hibernate.annotations.Type; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Transient; +import javax.persistence.*; import javax.validation.constraints.Size; import static bubble.ApiConstants.*; +import static org.cobbzilla.util.daemon.ZillaRuntime.bool; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @@ -84,7 +83,12 @@ public class BubbleApp extends IdentifiableBaseParentEntity implements AccountTe @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(10000+ENC_PAD)+")") @Getter @Setter private String description; - @Column(length=100000, nullable=false) @ECField(index=50) + @ECSearchable @ECField(index=50) + @Column(nullable=false, updatable=false) + @Getter @Setter private Boolean canPrime; + public boolean canPrime () { return bool(canPrime); } + + @Column(length=100000, nullable=false) @ECField(index=60) @JsonIgnore @Getter @Setter private String dataConfigJson; @Transient public AppDataConfig getDataConfig () { return dataConfigJson == null ? null : ensureDefaults(json(dataConfigJson, AppDataConfig.class)); } @@ -106,22 +110,22 @@ public class BubbleApp extends IdentifiableBaseParentEntity implements AccountTe // to a new node. This App will become a template/root BubbleApp for a new node, if it // is owned by a user and applicable to the BubblePlan (via BubblePlanApp) // For system apps, this can be null - @ECField(index=60) + @ECField(index=70) @Column(length=UUID_MAXLEN, updatable=false) @Getter @Setter private String templateApp; - @ECSearchable @ECField(index=70) + @ECSearchable @ECField(index=80) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean template = false; - @ECSearchable @ECField(index=80) + @ECSearchable @ECField(index=90) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean enabled = true; - @ECSearchable @ECField(index=90) @Column(nullable=false) + @ECSearchable @ECField(index=100) @Column(nullable=false) @ECIndex @Getter @Setter private Integer priority; - @ECSearchable @ECField(index=100) + @ECSearchable @ECField(index=110) @ECIndex @Getter @Setter private Boolean needsUpdate = false; } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index d18dfab9..2abbaba0 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -37,10 +37,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import static bubble.ApiConstants.*; @@ -167,12 +164,15 @@ public class FilterHttpResource { retained.add(matcher); } - final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), passthruRequest.getFqdn()); - if (passthru) { - if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+passthruRequest.getFqdn()+"/"+passthruRequest.getAddr()); - return ok(); + final String[] fqdns = passthruRequest.getFqdns(); + for (String fqdn : fqdns) { + final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), fqdn); + if (passthru) { + if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+fqdn+"/"+passthruRequest.getAddr()); + return ok(); + } } - if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdn/addr="+passthruRequest.getFqdn()+"/"+passthruRequest.getAddr()); + if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdns/addr="+Arrays.toString(fqdns)+"/"+passthruRequest.getAddr()); return notFound(); } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java index 2d4828a8..8684ca35 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterPassthruRequest.java @@ -14,8 +14,8 @@ public class FilterPassthruRequest { @Getter @Setter private String addr; public boolean hasAddr() { return !empty(addr); } - @Getter @Setter private String fqdn; - public boolean hasFqdn() { return !empty(fqdn); } + @Getter @Setter private String[] fqdns; + public boolean hasFqdns() { return !empty(fqdns); } @Getter @Setter private String remoteAddr; public boolean hasRemoteAddr() { return !empty(remoteAddr); } diff --git a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java index 6c1a0d05..f20e7452 100644 --- a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java @@ -13,6 +13,7 @@ import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; import bubble.model.device.Device; import bubble.server.BubbleConfiguration; +import bubble.service.stream.AppPrimerService; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; import lombok.Getter; @@ -39,6 +40,7 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { @Autowired protected RedisService redis; @Autowired protected BubbleNetworkDAO networkDAO; @Autowired protected DeviceDAO deviceDAO; + @Autowired protected AppPrimerService appPrimerService; @Getter @Setter private AppRuleDriver next; diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 64cc9b16..163d0d6b 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -14,13 +14,19 @@ import bubble.service.stream.AppRuleHarness; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; import org.cobbzilla.util.handlebars.HandlebarsUtil; +import org.cobbzilla.wizard.cache.redis.RedisService; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.Map; +import java.util.Set; +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.cobbzilla.util.io.StreamUtil.stream2bytes; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.security.ShaUtil.sha256_hex; @@ -28,8 +34,38 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath; public interface AppRuleDriver { + Logger log = LoggerFactory.getLogger(AppRuleDriver.class); + InputStream EMPTY_STREAM = new ByteArrayInputStream(new byte[0]); + // also used in dnscrypt-proxy/plugin_reverse_resolve_cache.go + String REDIS_BLOCK_LISTS = "blockLists"; + String REDIS_FILTER_LISTS = "filterLists"; + String REDIS_LIST_SUFFIX = "~UNION"; + + default Set getPrimedBlockDomains () { return null; } + default Set getPrimedFilterDomains () { return null; } + + static void defineRedisBlockSet(RedisService redis, String ip, String list, String[] fullyBlockedDomains) { + defineRedisSet(redis, ip, REDIS_BLOCK_LISTS, list, fullyBlockedDomains); + } + + static void defineRedisFilterSet(RedisService redis, String ip, String list, String[] filterDomains) { + defineRedisSet(redis, ip, REDIS_FILTER_LISTS, list, filterDomains); + } + + static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { + final String listOfListsForIp = listOfListsName + "~" + ip; + final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX; + final String ipList = listOfListsForIp + "~" + listName; + final String tempList = ipList + "~"+now()+randomAlphanumeric(5); + redis.sadd_plaintext(tempList, domains); + redis.rename(tempList, ipList); + redis.sadd_plaintext(listOfListsForIp, ipList); + final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp)); + log.info("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count); + } + AppRuleDriver getNext(); void setNext(AppRuleDriver next); default boolean hasNext() { return getNext() != null; } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index 847422b1..ea483b20 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -33,6 +33,7 @@ import java.net.URI; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; @@ -40,19 +41,30 @@ import static org.cobbzilla.util.http.HttpContentTypes.isHtml; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; -import static org.cobbzilla.util.string.StringUtil.UTF8cs; -import static org.cobbzilla.util.string.StringUtil.getPackagePath; +import static org.cobbzilla.util.string.StringUtil.*; @Slf4j public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { - private BlockList blockList = new BlockList(); + private final AtomicReference blockList = new AtomicReference<>(new BlockList()); + private BlockList getBlockList() { return blockList.get(); } - private static Map blockListCache = new ConcurrentHashMap<>(); + private final AtomicReference> fullyBlockedDomains = new AtomicReference<>(Collections.emptySet()); + @Override public Set getPrimedBlockDomains() { return fullyBlockedDomains.get(); } + + private final AtomicReference> partiallyBlockedDomains = new AtomicReference<>(Collections.emptySet()); + @Override public Set getPrimedFilterDomains() { return partiallyBlockedDomains.get(); } + + private final static Map blockListCache = new ConcurrentHashMap<>(); @Override public Class getConfigClass() { return (Class) BubbleBlockConfig.class; } - @Override public void init(JsonNode config, JsonNode userConfig, AppRule rule, AppMatcher matcher, Account account, Device device) { + @Override public void init(JsonNode config, + JsonNode userConfig, + AppRule rule, + AppMatcher matcher, + Account account, + Device device) { super.init(config, userConfig, rule, matcher, account, device); refreshBlockLists(); } @@ -61,6 +73,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); final BubbleBlockList[] blockLists = bubbleBlockConfig.getBlockLists(); final Set refreshed = new HashSet<>(); + final BlockList newBlockList = new BlockList(); for (BubbleBlockList list : blockLists) { if (!list.enabled()) continue; @@ -91,8 +104,22 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { log.error("init: error adding additional entries: "+shortError(e)); } } - if (blockListSource != null) blockList.merge(blockListSource.getBlockList()); + if (blockListSource != null) newBlockList.merge(blockListSource.getBlockList()); + } + blockList.set(newBlockList); + + boolean doPrime = false; + if (!newBlockList.getFullyBlockedDomains().equals(fullyBlockedDomains.get())) { + fullyBlockedDomains.set(newBlockList.getFullyBlockedDomains()); + doPrime = true; + } + if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) { + partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains()); + doPrime = true; } + + log.info("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size()); + log.info("refreshBlockLists: partiallyBlockedDomains="+partiallyBlockedDomains.get().size()); log.info("refreshBlockLists: refreshed "+refreshed.size()+" block lists: "+StringUtil.toString(refreshed)); } @@ -178,7 +205,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { } } - public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return blockList.getDecision(fqdn, uri, userAgent, false); } + public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return getBlockList().getDecision(fqdn, uri, userAgent, false); } public BlockDecision getDecision(String fqdn, String uri, String userAgent, boolean primary) { final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); @@ -189,7 +216,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { } } } - return blockList.getDecision(fqdn, uri, primary); + return getBlockList().getDecision(fqdn, uri, primary); } @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { @@ -199,7 +226,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { // Now that we know the content type, re-check the BlockList final String contentType = filterRequest.getContentType(); - final BlockDecision decision = blockList.getDecision(request.getFqdn(), request.getUri(), contentType, true); + final BlockDecision decision = getBlockList().getDecision(request.getFqdn(), request.getUri(), contentType, true); if (log.isDebugEnabled()) log.debug(prefix+"preprocess decision was "+decision+", but now we know contentType="+contentType); switch (decision.getDecisionType()) { case block: @@ -255,8 +282,8 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase()); ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId)); ctx.put(CTX_BUBBLE_SELECTORS, json(decision.getSelectors(), COMPACT_MAPPER)); - ctx.put(CTX_BUBBLE_WHITELIST, json(blockList.getWhitelistDomains(), COMPACT_MAPPER)); - ctx.put(CTX_BUBBLE_BLACKLIST, json(blockList.getBlacklistDomains(), COMPACT_MAPPER)); + ctx.put(CTX_BUBBLE_WHITELIST, json(getBlockList().getWhitelistDomains(), COMPACT_MAPPER)); + ctx.put(CTX_BUBBLE_BLACKLIST, json(getBlockList().getBlacklistDomains(), COMPACT_MAPPER)); return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx); } diff --git a/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java b/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java index 24f690dc..c758b64f 100644 --- a/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java @@ -12,6 +12,7 @@ import com.github.jknack.handlebars.Handlebars; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.input.ReaderInputStream; +import org.cobbzilla.util.collection.SingletonSet; import org.cobbzilla.util.io.regex.RegexFilterReader; import org.cobbzilla.util.io.regex.RegexInsertionFilter; import org.cobbzilla.util.io.regex.RegexStreamFilter; @@ -19,6 +20,7 @@ import org.cobbzilla.util.io.regex.RegexStreamFilter; import java.io.InputStream; import java.util.HashMap; import java.util.Map; +import java.util.Set; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.http.HttpContentTypes.isHtml; @@ -51,6 +53,10 @@ public class UserBlockerRuleDriver extends AbstractAppRuleDriver { return json(json, JsonNode.class); } + @Override public Set getPrimedFilterDomains() { + return matcher.isWildcardFqdn() ? null : new SingletonSet<>(matcher.getFqdn()); + } + protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); } @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { diff --git a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java index 6b61493f..10ece81e 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java +++ b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java @@ -88,6 +88,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration public static final String TAG_PROMO_CODE_POLICY = "promoCodePolicy"; public static final String TAG_REQUIRE_SEND_METRICS = "requireSendMetrics"; public static final String TAG_SUPPORT = "support"; + public static final String TAG_CERT_VALIDATION_HOST = "certValidationHost"; // must match bubble_passthru.py public static final String DEFAULT_LOCAL_STORAGE_DIR = HOME_DIR + "/.bubble_local_storage"; @@ -137,6 +138,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration public boolean hasSageNode () { return getSageNode() != null; } @Getter @Setter private String letsencryptEmail; + @Getter @Setter private String certValidationHost; + @Setter private String localStorageDir = DEFAULT_LOCAL_STORAGE_DIR; public String getLocalStorageDir () { return empty(localStorageDir) ? DEFAULT_LOCAL_STORAGE_DIR : localStorageDir; } @@ -297,7 +300,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration {TAG_LOCKED, accountDAO.locked()}, {TAG_LAUNCH_LOCK, isSageLauncher() || thisNetwork == null ? null : thisNetwork.launchLock()}, {TAG_SSL_PORT, getDefaultSslPort()}, - {TAG_SUPPORT, getSupport()} + {TAG_SUPPORT, getSupport()}, + {TAG_CERT_VALIDATION_HOST, getCertValidationHost()} })); } return publicSystemConfigs.get(); diff --git a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java index 6c9fa825..ecc4144c 100644 --- a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java @@ -7,18 +7,23 @@ package bubble.server.listener; import bubble.dao.account.AccountDAO; import bubble.dao.cloud.CloudServiceDAO; import bubble.model.account.Account; +import bubble.model.cloud.AnsibleInstallType; +import bubble.model.cloud.BubbleNetwork; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.NetworkMonitorService; +import bubble.service.stream.AppPrimerService; import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.wizard.cache.redis.RedisService; import org.cobbzilla.wizard.server.RestServer; import org.cobbzilla.wizard.server.RestServerLifecycleListenerBase; import java.io.File; import java.util.Map; +import static bubble.server.BubbleConfiguration.TAG_CERT_VALIDATION_HOST; import static bubble.service.boot.StandardSelfNodeService.SELF_NODE_JSON; import static bubble.service.boot.StandardSelfNodeService.THIS_NODE_FILE; import static org.cobbzilla.util.daemon.ZillaRuntime.*; @@ -101,6 +106,18 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase deviceCache = new ExpirationMap<>(MINUTES.toMillis(10)); + private final Map deviceCache = new ExpirationMap<>(MINUTES.toMillis(10)); public Device findDeviceByIp (String ipAddr) { diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index f44e4e4f..2f756d34 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -74,8 +74,7 @@ import static bubble.service.boot.StandardSelfNodeService.*; import static bubble.service.cloud.NodeProgressMeter.getProgressMeterKey; import static bubble.service.cloud.NodeProgressMeter.getProgressMeterPrefix; import static bubble.service.cloud.NodeProgressMeterConstants.*; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.concurrent.TimeUnit.*; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.cobbzilla.util.daemon.Await.awaitAll; import static org.cobbzilla.util.daemon.ZillaRuntime.*; diff --git a/bubble-server/src/main/java/bubble/service/stream/AppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/AppPrimerService.java new file mode 100644 index 00000000..59c25e25 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/stream/AppPrimerService.java @@ -0,0 +1,14 @@ +package bubble.service.stream; + +import bubble.model.account.Account; +import bubble.model.app.BubbleApp; + +public interface AppPrimerService { + + void primeApps(); + + void prime(BubbleApp app); + + void prime(Account account, String app); + +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java new file mode 100644 index 00000000..dfb0b189 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -0,0 +1,165 @@ +package bubble.service.stream; + +import bubble.dao.account.AccountDAO; +import bubble.dao.app.AppMatcherDAO; +import bubble.dao.app.AppRuleDAO; +import bubble.dao.app.BubbleAppDAO; +import bubble.dao.app.RuleDriverDAO; +import bubble.dao.device.DeviceDAO; +import bubble.model.account.Account; +import bubble.model.app.AppMatcher; +import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; +import bubble.model.app.RuleDriver; +import bubble.model.cloud.AnsibleInstallType; +import bubble.model.cloud.BubbleNetwork; +import bubble.model.device.Device; +import bubble.rule.AppRuleDriver; +import bubble.server.BubbleConfiguration; +import bubble.service.cloud.DeviceIdService; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.SingletonList; +import org.cobbzilla.wizard.cache.redis.RedisService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; + +import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; + +@Service @Slf4j +public class StandardAppPrimerService implements AppPrimerService { + + @Autowired private AccountDAO accountDAO; + @Autowired private DeviceDAO deviceDAO; + @Autowired private DeviceIdService deviceIdService; + @Autowired private BubbleAppDAO appDAO; + @Autowired private AppMatcherDAO matcherDAO; + @Autowired private AppRuleDAO ruleDAO; + @Autowired private RuleDriverDAO driverDAO; + @Autowired private RedisService redis; + @Autowired private BubbleConfiguration configuration; + + @Getter(lazy=true) private final boolean primingEnabled = initPrimingEnabled(); + private boolean initPrimingEnabled() { + if (configuration == null) return die("initPrimingEnabled: configuration was null"); + final BubbleNetwork thisNetwork = configuration.getThisNetwork(); + if (thisNetwork == null) { + log.info("primeApps: thisNetwork is null, not priming"); + return false; + } + if (thisNetwork.getInstallType() != AnsibleInstallType.node) { + log.info("primeApps: thisNetwork is not a node, not priming"); + return false; + } + return true; + } + + public void primeApps() { + if (!isPrimingEnabled()) { + log.info("primeApps: not enabled"); + return; + } + for (Account account : accountDAO.findNotDeleted()) { + try { + prime(account); + } catch (Exception e) { + log.error("primeApps("+account.getName()+"): "+shortError(e), e); + } + } + } + + public void prime(Account account) { prime(account, (BubbleApp) null); } + + public void prime(BubbleApp app) { + final Account account = accountDAO.findByUuid(app.getAccount()); + if (account == null) { + log.warn("prime("+app.getName()+"): account not found: "+app.getAccount()); + return; + } + prime(account, app.getUuid()); + } + + public synchronized void prime(Account account, String singleAppUuid) { + final BubbleApp singleApp = appDAO.findByAccountAndId(account.getUuid(), singleAppUuid); + if (singleApp == null) { + log.warn("prime("+account.getName()+", "+singleAppUuid+"): app not found: "+singleAppUuid); + return; + } + prime(account, singleApp); + } + + @Getter(lazy=true) private final ExecutorService primerThread = fixedPool(1); + + private void prime(Account account, BubbleApp singleApp) { + if (!isPrimingEnabled()) { + log.info("prime: not enabled"); + return; + } + getPrimerThread().submit(() -> _prime(account, singleApp)); + } + + private synchronized void _prime(Account account, BubbleApp singleApp) { + try { + final Map> accountDeviceIps = new HashMap<>(); + final List devices = deviceDAO.findByAccount(account.getUuid()); + for (Device device : devices) { + accountDeviceIps.put(device.getUuid(), deviceIdService.findIpsByDevice(device.getUuid())); + } + if (accountDeviceIps.isEmpty()) return; + + final List appsToPrime = singleApp == null + ? appDAO.findByAccount(account.getUuid()).stream() + .filter(BubbleApp::canPrime) + .collect(Collectors.toList()) + : new SingletonList<>(singleApp); + for (BubbleApp app : appsToPrime) { + final List rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); + final List matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); + for (AppRule rule : rules) { + final RuleDriver driver = driverDAO.findByUuid(rule.getDriver()); + if (driver == null) { + log.warn("_prime: driver not found for app/rule " + app.getName() + "/" + rule.getName() + ": " + rule.getDriver()); + continue; + } + for (Device device : devices) { + final Set blockDomains = new HashSet<>(); + final Set filterDomains = new HashSet<>(); + for (AppMatcher matcher : matchers) { + final AppRuleDriver appRuleDriver = rule.initDriver(driver, matcher, account, device); + final Set blocks = appRuleDriver.getPrimedBlockDomains(); + if (empty(blocks)) { + log.info("_prime: no blockDomains for app/rule/matcher: " + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + blockDomains.addAll(blocks); + } + final Set filters = appRuleDriver.getPrimedFilterDomains(); + if (empty(filters)) { + log.info("_prime: no filterDomains for app/rule/matcher: " + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + filterDomains.addAll(filters); + } + } + if (!empty(blockDomains) || !empty(filterDomains)) { + for (String ip : accountDeviceIps.get(device.getUuid())) { + if (!empty(blockDomains)) { + AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new)); + } + if (!empty(filterDomains)) { + AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new)); + } + } + } + } + } + } + } catch (Exception e) { + die("_prime: "+shortError(e), e); + } + } + +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java index 3e7462d4..94db1fed 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java @@ -173,7 +173,8 @@ public class StandardRuleEngineService implements RuleEngineService { return sendResponse(state.getResponseStream(last)); } - private ExpirationMap> ruleCache = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime); + private final ExpirationMap> ruleCache + = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime); public Map flushCaches() { final int ruleEngineCacheSize = ruleCache.size(); diff --git a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterAppPrimerService.java b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterAppPrimerService.java new file mode 100644 index 00000000..20c3eaf2 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterAppPrimerService.java @@ -0,0 +1,19 @@ +package bubble.service_dbfilter; + +import bubble.model.account.Account; +import bubble.model.app.BubbleApp; +import bubble.service.stream.AppPrimerService; +import org.springframework.stereotype.Service; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; + +@Service +public class DbFilterAppPrimerService implements AppPrimerService { + + @Override public void primeApps() { notSupported("primeApps"); } + + @Override public void prime(Account account, String app) { notSupported("prime"); } + + @Override public void prime(BubbleApp app) { notSupported("prime"); } + +} diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index 0c69ea9f..5229864e 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=0.11.3 +bubble.version=0.12.0 diff --git a/bubble-server/src/main/resources/ansible/install_local.sh.hbs b/bubble-server/src/main/resources/ansible/install_local.sh.hbs index 47a3eb7a..99b264f3 100644 --- a/bubble-server/src/main/resources/ansible/install_local.sh.hbs +++ b/bubble-server/src/main/resources/ansible/install_local.sh.hbs @@ -22,20 +22,17 @@ if [[ "$(whoami)" != "{{node.user}}" ]] ; then fi ANSIBLE_DIR="${ANSIBLE_HOME}/ansible" -ID_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa" -PUB_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa.pub" AUTH_KEYS="${ANSIBLE_HOME}/.ssh/authorized_keys" if [[ ! -d "${ANSIBLE_DIR}" ]] ; then die "Ansible dir not found or not a directory: ${ANSIBLE_DIR}" fi -if [[ ! -f "${ID_FILE}" ]] ; then - ssh-keygen -t rsa -q -N '' -f ${ID_FILE} || die "Error generating RSA key" -fi - # lockout the node that started us -cat "${PUB_FILE}" > "${AUTH_KEYS}" || die "Error updating ${AUTH_KEYS} file" +cat /dev/null > "${AUTH_KEYS}" || die "Error truncating ${AUTH_KEYS} file" + +# ensure proper permissions on authorized_keys file +chmod 600 "${AUTH_KEYS}" || die "Error setting permissions on ${AUTH_KEYS} file" # add admin ssh key, if one was given ADMIN_PUB_KEY="${ANSIBLE_DIR}/roles/bubble/files/admin_ssh_key.pub" @@ -43,8 +40,6 @@ if [[ -f "${ADMIN_PUB_KEY}" ]] ; then cat "${ADMIN_PUB_KEY}" >> "${AUTH_KEYS}" fi -SSH_OPTIONS="--ssh-extra-args '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=publickey -i ${ID_FILE}'" - SKIP_TAGS="" if [[ -n "{{restoreKey}}" ]] ; then SKIP_TAGS="--skip-tags algo_related" @@ -55,5 +50,5 @@ cd "${ANSIBLE_DIR}" && \ virtualenv -p python3 ./venv && \ . ./venv/bin/activate && \ pip3 install ansible && \ - bash -c "ansible-playbook ${SSH_OPTIONS} ${SKIP_TAGS} --inventory ./hosts ./playbook.yml 2>&1 | tee ${LOG}" \ + ansible-playbook ${SKIP_TAGS} --inventory ./hosts ./playbook.yml \ || die "Error running ansible. journalctl -xe = $(journalctl -xe | tail -n 50)" diff --git a/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml index 846701ba..b1c9be5c 100644 --- a/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml @@ -15,12 +15,8 @@ when: restore_key is not defined # Don't start monitors when in restore mode, bubble_restore_monitor.sh will start it after algo is installed -- name: Restart algo monitors - shell: bash -c "supervisorctl reload && sleep 5s && supervisorctl restart algo_refresh_users_monitor && supervisorctl restart wg_monitor_connections" - when: restore_key is not defined - - name: Stop algo monitors (in restore mode) - shell: bash -c "supervisorctl reload && sleep 5s && supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections" + shell: bash -c "supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections" when: restore_key is defined # Add bubble rules diff --git a/bubble-server/src/main/resources/ansible/roles/algo/templates/install_algo.sh.j2 b/bubble-server/src/main/resources/ansible/roles/algo/templates/install_algo.sh.j2 index 25590bbc..65d2f2c6 100644 --- a/bubble-server/src/main/resources/ansible/roles/algo/templates/install_algo.sh.j2 +++ b/bubble-server/src/main/resources/ansible/roles/algo/templates/install_algo.sh.j2 @@ -43,4 +43,10 @@ CONFIGS_BACKUP=/home/bubble/.BUBBLE_ALGO_CONFIGS.tgz cd ${ALGO_BASE} && tar czf ${CONFIGS_BACKUP} configs && chgrp bubble ${CONFIGS_BACKUP} && chmod 660 ${CONFIGS_BACKUP} || die "Error backing up algo configs" cd /home/bubble && tar xzf ${CONFIGS_BACKUP} && chgrp -R bubble configs && chown -R bubble configs && chmod 500 configs || die "Error unpacking algo configs to bubble home" +# Restart algo_refresh_users_monitor and wg_monitor_connections +supervisorctl reload && sleep 5s && supervisorctl restart algo_refresh_users_monitor && supervisorctl restart wg_monitor_connections + +# Restart dnscrypt-proxy +service dnscrypt-proxy restart + touch "${ALGO_BASE}/.install_marker" diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json index e6eeaaba..97fd670f 100644 --- a/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json +++ b/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_role.json @@ -30,6 +30,8 @@ {"name": "error_key", "value": "[[error_key]]"}, {"name": "error_env", "value": "[[error_env]]"}, + {"name": "cert_validation_host", "value": "[[cert_validation_host]]"}, + {"name": "support_email", "value": "[[support.email]]"}, {"name": "support_site", "value": "[[support.site]]"}, diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2 b/bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2 index c20748ca..dca39c20 100644 --- a/bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2 +++ b/bubble-server/src/main/resources/ansible/roles/bubble/templates/bubble.env.j2 @@ -12,6 +12,7 @@ export ERRBIT_URL={{ error_url | default('') }} export ERRBIT_KEY={{ error_key | default('') }} export ERRBIT_ENV={{ error_env | default('') }} +export CERT_VALIDATION_HOST={{ cert_validation_host }} export SUPPORT_EMAIL={{ support_email }} export SUPPORT_SITE={{ support_site }} diff --git a/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml index a7262036..c0663421 100644 --- a/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml @@ -1,8 +1,8 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # -- name: Snapshot ansible roles - shell: snapshot_ansible.sh +- name: Snapshot ansible roles in the background + command: /usr/local/bin/snapshot_ansible.sh - name: Touch first-time setup file shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n install > /home/bubble/first_time_marker ; fi" diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml index c0fd85a7..9579da9f 100644 --- a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml @@ -20,9 +20,6 @@ - import_tasks: route.yml -- name: Restart dnscrypt-proxy - shell: service dnscrypt-proxy restart - - name: Install supervisor conf file copy: src: supervisor_mitmproxy.conf diff --git a/bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml index 2681681c..65f0b203 100644 --- a/bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/nginx/tasks/main.yml @@ -16,7 +16,6 @@ template: src=stronger_dhparams.conf dest=/etc/nginx/conf.d/stronger_dhparams.conf - include: site.yml -- meta: flush_handlers # nginx has to be restarted right now if it has to - name: Init certbot shell: init_certbot.sh {{ letsencrypt_email }} {{ server_name }} {{ server_alias }} diff --git a/bubble-server/src/main/resources/bubble-config.yml b/bubble-server/src/main/resources/bubble-config.yml index 4f35fdcb..42ac43dc 100644 --- a/bubble-server/src/main/resources/bubble-config.yml +++ b/bubble-server/src/main/resources/bubble-config.yml @@ -88,6 +88,8 @@ rateLimits: - { limit: 50000, interval: 1h, block: 24h } - { limit: 100000, interval: 6h, block: 96h } +certValidationHost: {{CERT_VALIDATION_HOST}} + support: email: {{SUPPORT_EMAIL}} site: {{SUPPORT_SITE}} diff --git a/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties b/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties index 736ee05f..24a530ab 100644 --- a/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties +++ b/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties @@ -20,6 +20,12 @@ support_email_link=Send us an email support_not_available=Sorry, no support options are available link_support=Support +# Cert checker +title_check_certificate=Bubble Certificate Verification +check_cert_null=Verifying certification installation... +check_cert_true=Your Bubble Certificate is properly installed +check_cert_false=Your Bubble Certificate is not properly installed + # Legal page links title_legal_topics=Legal Stuff legal_topics=terms,privacy,source,license,3rdParty_licenses diff --git a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json index c79f9457..6826b23e 100644 --- a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json +++ b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json @@ -5,6 +5,7 @@ "template": true, "enabled": true, "priority": 100, + "canPrime": false, "dataConfig": { "dataDriver": "bubble.app.analytics.TrafficAnalyticsAppDataDriver", "presentation": "app", diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json index ab4053fc..9f69be13 100644 --- a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json +++ b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json @@ -5,6 +5,7 @@ "template": true, "enabled": true, "priority": 200, + "canPrime": true, "dataConfig": { "dataDriver": "bubble.app.bblock.BubbleBlockAppDataDriver", "presentation": "app", @@ -16,8 +17,7 @@ {"name": "data"} ], "params": [ - {"name": "device", "required": false, "index": 10}, - {"name": "meta2", "required": false, "operator": "like", "index": 20} + {"name": "device", "required": false, "index": 10} ], "views": [ {"name": "last_24_hours"}, diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json index f86377a8..295e72b3 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json @@ -5,6 +5,7 @@ "template": true, "enabled": true, "priority": 1000000, + "canPrime": false, "dataConfig": { "dataDriver": "bubble.app.passthru.TlsPassthruAppDataDriver", "presentation": "none", diff --git a/bubble-server/src/main/resources/models/apps/user_block/bubbleApp_userBlock.json b/bubble-server/src/main/resources/models/apps/user_block/bubbleApp_userBlock.json index 2241a352..1bd2e38b 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/bubbleApp_userBlock.json +++ b/bubble-server/src/main/resources/models/apps/user_block/bubbleApp_userBlock.json @@ -5,6 +5,7 @@ "template": true, "enabled": true, "priority": 300, + "canPrime": true, "dataConfig": { "dataDriver": "bubble.app.social.block.UserBlockerAppDataDriver", "presentation": "site", diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index b3952643..0ed08917 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:f9709ced6cbaf92473c975355050915657da50dfc0eaf655f178643953a9cd42 + checksum: sha256:357e613833385626e88564c97f0b5726f49686c33a774be9a7766bd1a1249915 - name: Unzip algo master.zip unarchive: diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh index 2c4a5d31..499567be 100755 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh @@ -114,8 +114,6 @@ if [[ ! -f ${CONFIGS_BACKUP} ]] ; then else ANSIBLE_HOME="/root" ANSIBLE_DIR="${ANSIBLE_HOME}/ansible" - ID_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa" - SSH_OPTIONS="--ssh-extra-args '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=publickey -i ${ID_FILE}'" ALGO_BASE=${ANSIBLE_DIR}/roles/algo/algo if [[ ! -d ${ALGO_BASE} ]] ; then @@ -126,7 +124,7 @@ else cd "${ANSIBLE_DIR}" && \ . ./venv/bin/activate && \ bash -c \ - "ansible-playbook ${SSH_OPTIONS} --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \ + "ansible-playbook --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \ || die "Error running ansible in post-restore. journalctl -xe = $(journalctl -xe | tail -n 50)" fi diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 86110382..47ef2d44 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -49,14 +49,14 @@ def bubble_activity_log(client_addr, server_addr, event, data): 'client_addr': client_addr, 'server_addr': server_addr, 'event': event, - 'data': data + 'data': str(data) }) bubble_log('bubble_activity_log: setting '+key+' = '+value) redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION) pass -def bubble_passthru(remote_addr, addr, fqdn): +def bubble_passthru(remote_addr, addr, fqdns): headers = { 'X-Forwarded-For': remote_addr, 'Accept' : 'application/json', @@ -65,7 +65,7 @@ def bubble_passthru(remote_addr, addr, fqdn): try: data = { 'addr': str(addr), - 'fqdn': str(fqdn), + 'fqdns': fqdns, 'remoteAddr': remote_addr } response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py index 614a5245..8848d8fc 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_passthru.py @@ -29,18 +29,69 @@ from mitmproxy.exceptions import TlsProtocolException from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set import redis import json +import subprocess REDIS_DNS_PREFIX = 'bubble_dns_' REDIS_PASSTHRU_PREFIX = 'bubble_passthru_' +REDIS_CLIENT_CERT_STATUS_PREFIX = 'bubble_cert_status_' REDIS_PASSTHRU_DURATION = 60 * 60 # 1 hour timeout on passthru REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) +cert_validation_host = None +local_ips = None + + +def get_ip_cert_status(client_addr): + status = REDIS.get(REDIS_CLIENT_CERT_STATUS_PREFIX+client_addr) + if status is None: + return None + enabled = status.decode() == "True" + return enabled + + +def set_ip_cert_status(client_addr, enabled): + REDIS.set(REDIS_CLIENT_CERT_STATUS_PREFIX+client_addr, str(enabled)) + bubble_log('set_ip_cert_status: set '+client_addr+' = '+str(enabled)) + + +def get_local_ips(): + global local_ips + if local_ips is None: + local_ips = [] + for ip in subprocess.check_output(['hostname', '-I']).split(): + local_ips.append(ip.decode()) + return local_ips + + +def get_cert_validation_host(): + global cert_validation_host + if cert_validation_host is None: + cert_validation_host = REDIS.get('certValidationHost') + if cert_validation_host is not None: + cert_validation_host = cert_validation_host.decode() + # bubble_log('get_cert_validation_host: initialized to '+cert_validation_host) + # bubble_log('get_cert_validation_host: returning '+cert_validation_host) + return cert_validation_host + def passthru_cache_prefix(client_addr, server_addr): return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr +def fqdns_for_addr(addr): + prefix = REDIS_DNS_PREFIX + addr + keys = REDIS.keys(prefix + '_*') + if keys is None or len(keys) == 0: + bubble_log('fqdns_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr') + return '' + fqdns = [] + for k in keys: + fqdn = k.decode()[len(prefix)+1:] + fqdns.append(fqdn) + return fqdns + + class TlsFeedback(TlsLayer): """ Monkey-patch _establish_tls_with_client to get feedback if TLS could be established @@ -49,51 +100,61 @@ class TlsFeedback(TlsLayer): def _establish_tls_with_client(self): client_address = self.client_conn.address[0] server_address = self.server_conn.address[0] + fqdns = fqdns_for_addr(server_address) try: super(TlsFeedback, self)._establish_tls_with_client() + if fqdns and get_cert_validation_host() in fqdns: + # bubble_log('_establish_tls_with_client: TLS success for '+repr(server_address)+', enabling SSL interception for client '+client_address) + set_ip_cert_status(client_address, True) + except TlsProtocolException as e: - bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru') cache_key = passthru_cache_prefix(client_address, server_address) - fqdn = fqdn_for_addr(server_address) - redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key) + if fqdns and get_cert_validation_host() in fqdns: + set_ip_cert_status(client_address, False) + else: + redis_set(cache_key, json.dumps({'fqdns': fqdns, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) raise e -def fqdn_for_addr(addr): - fqdn = REDIS.get(REDIS_DNS_PREFIX + addr) - if fqdn is None or len(fqdn) == 0: - bubble_log('fqdn_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr') - fqdn = b'' - return fqdn.decode() +def check_bubble_passthru(client_addr, addr, fqdns): + cert_status = get_ip_cert_status(client_addr) + if cert_status is not None and not cert_status: + bubble_log('check_bubble_passthru: returning True because cert_status for '+client_addr+' was False') + return {'fqdns': fqdns, 'addr': addr, 'passthru': True} + else: + bubble_log('check_bubble_passthru: request is NOT for cert_validation_host: '+cert_validation_host+", it is for one of fqdn="+repr(fqdns)+", checking bubble_passthru...") -def check_bubble_passthru(remote_addr, addr, fqdn): - passthru = bubble_passthru(remote_addr, addr, fqdn) - if passthru is None: - return None - if passthru: - bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True') - return {'fqdn': fqdn, 'addr': addr, 'passthru': True} - bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False') - return {'fqdn': fqdn, 'addr': addr, 'passthru': False} + passthru = bubble_passthru(client_addr, addr, fqdns) + if passthru is None or passthru: + bubble_log('check_bubble_passthru: bubble_passthru returned '+repr(passthru)+' for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning True') + return {'fqdns': fqdns, 'addr': addr, 'passthru': True} + bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning False') + return {'fqdns': fqdns, 'addr': addr, 'passthru': False} + + +def should_passthru(client_addr, addr, fqdns): + # always passthru for local ips + if addr in get_local_ips(): + # bubble_log('should_passthru: local ip is always passthru: '+addr) + return {'fqdns': fqdns, 'addr': addr, 'passthru': True} + else: + # bubble_log('should_passthru: addr ('+addr+') is not a local ip: '+repr(get_local_ips())) + pass + cache_key = passthru_cache_prefix(client_addr, addr) + prefix = 'should_passthru: ip='+repr(addr)+' (fqdns='+repr(fqdns)+') cache_key='+cache_key+': ' -def should_passthru(remote_addr, addr): - prefix = 'should_passthru: '+repr(addr)+' ' - bubble_log(prefix+'starting...') - cache_key = passthru_cache_prefix(remote_addr, addr) passthru_json = REDIS.get(cache_key) if passthru_json is None or len(passthru_json) == 0: - bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...') - fqdn = fqdn_for_addr(addr) - if fqdn is None or len(fqdn) == 0: - fqdn = 'NONE' - passthru = check_bubble_passthru(remote_addr, addr, fqdn) - bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...") - if passthru is not None: - redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) + bubble_log(prefix+'not in redis or empty, calling check_bubble_passthru against fqdns='+repr(fqdns)) + passthru = check_bubble_passthru(client_addr, addr, fqdns) + bubble_log(prefix+'check_bubble_passthru('+repr(fqdns)+') returned '+repr(passthru)+", storing in redis...") + redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) + else: - bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis') + bubble_log(prefix+'found passthru_json='+str(passthru_json)+', touching key in redis') passthru = json.loads(passthru_json) REDIS.touch(cache_key) bubble_log(prefix+'returning '+repr(passthru)) @@ -104,13 +165,23 @@ def next_layer(next_layer): if isinstance(next_layer, TlsLayer) and next_layer._client_tls: client_address = next_layer.client_conn.address[0] server_address = next_layer.server_conn.address[0] - passthru = should_passthru(client_address, server_address) - if passthru is None or passthru['passthru']: - bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address)) - if passthru is not None and 'fqdn' in passthru: - bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn']) - next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) - next_layer.reply.send(next_layer_replacement) - else: - bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn']) + + fqdns = fqdns_for_addr(server_address) + validation_host = get_cert_validation_host() + if fqdns and validation_host in fqdns: + bubble_log('next_layer: never passing thru (always getting feedback) for cert_validation_host='+validation_host) next_layer.__class__ = TlsFeedback + + else: + bubble_log('next_layer: checking should_passthru for client_address='+client_address) + passthru = should_passthru(client_address, server_address, fqdns) + if passthru is None or passthru['passthru']: + # bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address)) + if passthru is not None and 'fqdns' in passthru: + bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdns']) + next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) + next_layer.reply.send(next_layer_replacement) + else: + # bubble_log('next_layer: NO PASSTHRU (getting feedback) for client_address='+client_address+', server_address='+server_address) + bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdns']) + next_layer.__class__ = TlsFeedback diff --git a/bubble-server/src/test/resources/models/tests/promo/account_credit.json b/bubble-server/src/test/resources/models/tests/promo/account_credit.json index d8f08147..f6bb8f80 100644 --- a/bubble-server/src/test/resources/models/tests/promo/account_credit.json +++ b/bubble-server/src/test/resources/models/tests/promo/account_credit.json @@ -19,16 +19,16 @@ { "before": "sleep 5s", - "comment": "root: check email inbox for verification message", + "comment": "root: check email inbox for welcome message", "request": { "session": "rootSession", - "uri": "debug/inbox/email/account_credit_user@example.com?type=request&action=verify&target=account" + "uri": "debug/inbox/email/account_credit_user@example.com?type=notice&action=welcome&target=account" }, "response": { "store": "emailInbox", "check": [ - {"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"}, - {"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"}, + {"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"}, + {"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"}, {"condition": "'{{json.[0].ctx.message.target}}' == 'account'"} ] } diff --git a/bubble-server/src/test/resources/models/tests/promo/multi_promo.json b/bubble-server/src/test/resources/models/tests/promo/multi_promo.json index 787a8d5e..3000ab95 100644 --- a/bubble-server/src/test/resources/models/tests/promo/multi_promo.json +++ b/bubble-server/src/test/resources/models/tests/promo/multi_promo.json @@ -47,16 +47,16 @@ }, { - "comment": "root: check email inbox for verification message for referring user", + "comment": "root: check email inbox for welcome message for referring user", "request": { "session": "rootSession", - "uri": "debug/inbox/email/test_user_referring_multi@example.com?type=request&action=verify&target=account" + "uri": "debug/inbox/email/test_user_referring_multi@example.com?type=notice&action=welcome&target=account" }, "response": { "store": "emailInbox", "check": [ - {"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"}, - {"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"}, + {"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"}, + {"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"}, {"condition": "'{{json.[0].ctx.message.target}}' == 'account'"} ] } diff --git a/bubble-server/src/test/resources/models/tests/promo/referral_month_free.json b/bubble-server/src/test/resources/models/tests/promo/referral_month_free.json index 5deb1609..1c22dd3f 100644 --- a/bubble-server/src/test/resources/models/tests/promo/referral_month_free.json +++ b/bubble-server/src/test/resources/models/tests/promo/referral_month_free.json @@ -42,16 +42,16 @@ }, { - "comment": "root: check email inbox for verification message for referring user", + "comment": "root: check email inbox for welcome message for referring user", "request": { "session": "rootSession", - "uri": "debug/inbox/email/test_user_referring_free@example.com?type=request&action=verify&target=account" + "uri": "debug/inbox/email/test_user_referring_free@example.com?type=notice&action=welcome&target=account" }, "response": { "store": "emailInbox", "check": [ - {"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"}, - {"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"}, + {"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"}, + {"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"}, {"condition": "'{{json.[0].ctx.message.target}}' == 'account'"} ] } diff --git a/utils/abp-parser b/utils/abp-parser index e25605ea..df343fc4 160000 --- a/utils/abp-parser +++ b/utils/abp-parser @@ -1 +1 @@ -Subproject commit e25605ea6a578609b47d2f7c9da9dc87debbb8bb +Subproject commit df343fc4b3e1f123b3caa025a493861edf46b81a diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 17d93ddf..d450510d 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 17d93ddf2d3436228504ec98a52008efa665d339 +Subproject commit d450510d6f1be5b328b919ae73d8c1450f008d91