# Conflicts: # bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.javapull/22/head
@@ -452,4 +452,7 @@ public class AccountDAO extends AbstractCRUDDAO<Account> implements SqlViewSearc | |||
@NonNull public List<Account> findDeleted() { | |||
return list(criteria().add(isNotNull("deleted"))); | |||
} | |||
@NonNull public List<Account> findNotDeleted() { return findByField("deleted", null); } | |||
} |
@@ -46,18 +46,15 @@ public class AccountSshKeyDAO extends AccountOwnedEntityDAO<AccountSshKey> { | |||
if (key.hasExpiration() && key.expired()) throw invalidEx("err.expiration.cannotCreateSshKeyAlreadyExpired"); | |||
final Account owner = accountDAO.findByUuid(key.getAccount()); | |||
if (key.installSshKey()) { | |||
if (owner.admin()) { | |||
// admin keys are always installed on a node | |||
// never allow installation of a key on sage. must be manually set in the database. | |||
final BubbleNetwork thisNetwork = configuration.getThisNetwork(); | |||
if (thisNetwork != null && thisNetwork.getInstallType() == AnsibleInstallType.node) { | |||
key.setInstallSshKey(true); | |||
} | |||
} else { | |||
// never install key for non-admin | |||
key.setInstallSshKey(false); | |||
} | |||
final BubbleNetwork thisNetwork = configuration.getThisNetwork(); | |||
if (thisNetwork == null || thisNetwork.getInstallType() != AnsibleInstallType.sage) { | |||
// never allow installation of a key on sage. must be manually set in the database. | |||
key.setInstallSshKey(false); | |||
} else { | |||
// admin keys are always installed on a node | |||
// never install key for non-admin | |||
key.setInstallSshKey(owner.admin() && thisNetwork.getInstallType() == AnsibleInstallType.node); | |||
} | |||
final String hash = sha256_hex(key.getSshPublicKey()); | |||
@@ -9,6 +9,7 @@ import bubble.model.account.Account; | |||
import bubble.model.app.AppMatcher; | |||
import bubble.model.app.AppRule; | |||
import bubble.model.app.BubbleApp; | |||
import bubble.service.stream.AppPrimerService; | |||
import bubble.service.stream.RuleEngineService; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.hibernate.criterion.Order; | |||
@@ -17,6 +18,7 @@ import org.springframework.stereotype.Repository; | |||
import java.util.List; | |||
import static bubble.model.app.AppMatcher.WILDCARD_FQDN; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.die; | |||
import static org.cobbzilla.wizard.model.Identifiable.MTIME; | |||
import static org.hibernate.criterion.Restrictions.*; | |||
@@ -28,6 +30,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> { | |||
@Autowired private BubbleAppDAO appDAO; | |||
@Autowired private AppRuleDAO ruleDAO; | |||
@Autowired private RuleEngineService ruleEngineService; | |||
@Autowired private AppPrimerService primerService; | |||
@Override public Order getDefaultSortOrder() { return PRIORITY_ASC; } | |||
@@ -39,7 +42,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> { | |||
eq("passthru", false), | |||
or( | |||
eq("fqdn", fqdn), | |||
eq("fqdn", "*") | |||
eq("fqdn", WILDCARD_FQDN) | |||
)) | |||
).addOrder(PRIORITY_ASC)); | |||
} | |||
@@ -88,6 +91,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO<AppMatcher> { | |||
} | |||
} | |||
ruleEngineService.flushCaches(); | |||
primerService.prime(app); | |||
return super.postUpdate(matcher, context); | |||
} | |||
@@ -48,6 +48,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H | |||
public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule", "passthru"); | |||
public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE); | |||
public static final String WILDCARD_FQDN = "*"; | |||
public static final String WILDCARD_URL = ".*"; | |||
public AppMatcher(AppMatcher other) { | |||
copy(this, other, CREATE_FIELDS); | |||
@@ -80,12 +82,14 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H | |||
@Size(max=1024, message="err.fqdn.length") | |||
@ECIndex @Column(nullable=false, length=1024) | |||
@Getter @Setter private String fqdn; | |||
@JsonIgnore @Transient public boolean isWildcardFqdn () { return fqdn != null && fqdn.equals(WILDCARD_FQDN); } | |||
@ECSearchable(filter=true) @ECField(index=60) | |||
@HasValue(message="err.urlRegex.required") | |||
@Size(max=1024, message="err.urlRegex.length") | |||
@Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(1024+ENC_PAD)+") NOT NULL") | |||
@Getter @Setter private String urlRegex; | |||
public boolean hasUrlRegex() { return !empty(urlRegex) && !urlRegex.equals(WILDCARD_URL); } | |||
@Transient @JsonIgnore public Pattern getUrlPattern() { return Pattern.compile(getUrlRegex()); } | |||
public boolean matchesUrl (String value) { return getUrlPattern().matcher(value).find(); } | |||
@@ -21,12 +21,11 @@ import org.cobbzilla.wizard.model.entityconfig.annotations.*; | |||
import org.cobbzilla.wizard.validation.HasValue; | |||
import org.hibernate.annotations.Type; | |||
import javax.persistence.Column; | |||
import javax.persistence.Entity; | |||
import javax.persistence.Transient; | |||
import javax.persistence.*; | |||
import javax.validation.constraints.Size; | |||
import static bubble.ApiConstants.*; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.bool; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.empty; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
import static org.cobbzilla.util.reflect.ReflectionUtil.copy; | |||
@@ -84,7 +83,12 @@ public class BubbleApp extends IdentifiableBaseParentEntity implements AccountTe | |||
@Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(10000+ENC_PAD)+")") | |||
@Getter @Setter private String description; | |||
@Column(length=100000, nullable=false) @ECField(index=50) | |||
@ECSearchable @ECField(index=50) | |||
@Column(nullable=false, updatable=false) | |||
@Getter @Setter private Boolean canPrime; | |||
public boolean canPrime () { return bool(canPrime); } | |||
@Column(length=100000, nullable=false) @ECField(index=60) | |||
@JsonIgnore @Getter @Setter private String dataConfigJson; | |||
@Transient public AppDataConfig getDataConfig () { return dataConfigJson == null ? null : ensureDefaults(json(dataConfigJson, AppDataConfig.class)); } | |||
@@ -106,22 +110,22 @@ public class BubbleApp extends IdentifiableBaseParentEntity implements AccountTe | |||
// to a new node. This App will become a template/root BubbleApp for a new node, if it | |||
// is owned by a user and applicable to the BubblePlan (via BubblePlanApp) | |||
// For system apps, this can be null | |||
@ECField(index=60) | |||
@ECField(index=70) | |||
@Column(length=UUID_MAXLEN, updatable=false) | |||
@Getter @Setter private String templateApp; | |||
@ECSearchable @ECField(index=70) | |||
@ECSearchable @ECField(index=80) | |||
@ECIndex @Column(nullable=false) | |||
@Getter @Setter private Boolean template = false; | |||
@ECSearchable @ECField(index=80) | |||
@ECSearchable @ECField(index=90) | |||
@ECIndex @Column(nullable=false) | |||
@Getter @Setter private Boolean enabled = true; | |||
@ECSearchable @ECField(index=90) @Column(nullable=false) | |||
@ECSearchable @ECField(index=100) @Column(nullable=false) | |||
@ECIndex @Getter @Setter private Integer priority; | |||
@ECSearchable @ECField(index=100) | |||
@ECSearchable @ECField(index=110) | |||
@ECIndex @Getter @Setter private Boolean needsUpdate = false; | |||
} |
@@ -37,10 +37,7 @@ import javax.ws.rs.core.Context; | |||
import javax.ws.rs.core.MediaType; | |||
import javax.ws.rs.core.Response; | |||
import java.io.IOException; | |||
import java.util.ArrayList; | |||
import java.util.HashMap; | |||
import java.util.List; | |||
import java.util.Map; | |||
import java.util.*; | |||
import java.util.stream.Collectors; | |||
import static bubble.ApiConstants.*; | |||
@@ -167,12 +164,15 @@ public class FilterHttpResource { | |||
retained.add(matcher); | |||
} | |||
final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), passthruRequest.getFqdn()); | |||
if (passthru) { | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+passthruRequest.getFqdn()+"/"+passthruRequest.getAddr()); | |||
return ok(); | |||
final String[] fqdns = passthruRequest.getFqdns(); | |||
for (String fqdn : fqdns) { | |||
final boolean passthru = ruleEngine.isTlsPassthru(account, device, retained, passthruRequest.getAddr(), fqdn); | |||
if (passthru) { | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning true for fqdn/addr="+fqdn+"/"+passthruRequest.getAddr()); | |||
return ok(); | |||
} | |||
} | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdn/addr="+passthruRequest.getFqdn()+"/"+passthruRequest.getAddr()); | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning false for fqdns/addr="+Arrays.toString(fqdns)+"/"+passthruRequest.getAddr()); | |||
return notFound(); | |||
} | |||
@@ -14,8 +14,8 @@ public class FilterPassthruRequest { | |||
@Getter @Setter private String addr; | |||
public boolean hasAddr() { return !empty(addr); } | |||
@Getter @Setter private String fqdn; | |||
public boolean hasFqdn() { return !empty(fqdn); } | |||
@Getter @Setter private String[] fqdns; | |||
public boolean hasFqdns() { return !empty(fqdns); } | |||
@Getter @Setter private String remoteAddr; | |||
public boolean hasRemoteAddr() { return !empty(remoteAddr); } | |||
@@ -13,6 +13,7 @@ import bubble.model.app.AppMatcher; | |||
import bubble.model.app.AppRule; | |||
import bubble.model.device.Device; | |||
import bubble.server.BubbleConfiguration; | |||
import bubble.service.stream.AppPrimerService; | |||
import com.fasterxml.jackson.databind.JsonNode; | |||
import com.github.jknack.handlebars.Handlebars; | |||
import lombok.Getter; | |||
@@ -39,6 +40,7 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { | |||
@Autowired protected RedisService redis; | |||
@Autowired protected BubbleNetworkDAO networkDAO; | |||
@Autowired protected DeviceDAO deviceDAO; | |||
@Autowired protected AppPrimerService appPrimerService; | |||
@Getter @Setter private AppRuleDriver next; | |||
@@ -14,13 +14,19 @@ import bubble.service.stream.AppRuleHarness; | |||
import com.fasterxml.jackson.databind.JsonNode; | |||
import com.github.jknack.handlebars.Handlebars; | |||
import org.cobbzilla.util.handlebars.HandlebarsUtil; | |||
import org.cobbzilla.wizard.cache.redis.RedisService; | |||
import org.glassfish.grizzly.http.server.Request; | |||
import org.glassfish.jersey.server.ContainerRequest; | |||
import org.slf4j.Logger; | |||
import org.slf4j.LoggerFactory; | |||
import java.io.ByteArrayInputStream; | |||
import java.io.InputStream; | |||
import java.util.Map; | |||
import java.util.Set; | |||
import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.now; | |||
import static org.cobbzilla.util.io.StreamUtil.stream2bytes; | |||
import static org.cobbzilla.util.io.StreamUtil.stream2string; | |||
import static org.cobbzilla.util.security.ShaUtil.sha256_hex; | |||
@@ -28,8 +34,38 @@ import static org.cobbzilla.util.string.StringUtil.getPackagePath; | |||
public interface AppRuleDriver { | |||
Logger log = LoggerFactory.getLogger(AppRuleDriver.class); | |||
InputStream EMPTY_STREAM = new ByteArrayInputStream(new byte[0]); | |||
// also used in dnscrypt-proxy/plugin_reverse_resolve_cache.go | |||
String REDIS_BLOCK_LISTS = "blockLists"; | |||
String REDIS_FILTER_LISTS = "filterLists"; | |||
String REDIS_LIST_SUFFIX = "~UNION"; | |||
default Set<String> getPrimedBlockDomains () { return null; } | |||
default Set<String> getPrimedFilterDomains () { return null; } | |||
static void defineRedisBlockSet(RedisService redis, String ip, String list, String[] fullyBlockedDomains) { | |||
defineRedisSet(redis, ip, REDIS_BLOCK_LISTS, list, fullyBlockedDomains); | |||
} | |||
static void defineRedisFilterSet(RedisService redis, String ip, String list, String[] filterDomains) { | |||
defineRedisSet(redis, ip, REDIS_FILTER_LISTS, list, filterDomains); | |||
} | |||
static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { | |||
final String listOfListsForIp = listOfListsName + "~" + ip; | |||
final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX; | |||
final String ipList = listOfListsForIp + "~" + listName; | |||
final String tempList = ipList + "~"+now()+randomAlphanumeric(5); | |||
redis.sadd_plaintext(tempList, domains); | |||
redis.rename(tempList, ipList); | |||
redis.sadd_plaintext(listOfListsForIp, ipList); | |||
final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp)); | |||
log.info("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count); | |||
} | |||
AppRuleDriver getNext(); | |||
void setNext(AppRuleDriver next); | |||
default boolean hasNext() { return getNext() != null; } | |||
@@ -33,6 +33,7 @@ import java.net.URI; | |||
import java.util.*; | |||
import java.util.concurrent.ConcurrentHashMap; | |||
import java.util.concurrent.TimeUnit; | |||
import java.util.concurrent.atomic.AtomicReference; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.empty; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; | |||
@@ -40,19 +41,30 @@ import static org.cobbzilla.util.http.HttpContentTypes.isHtml; | |||
import static org.cobbzilla.util.io.StreamUtil.stream2string; | |||
import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
import static org.cobbzilla.util.string.StringUtil.UTF8cs; | |||
import static org.cobbzilla.util.string.StringUtil.getPackagePath; | |||
import static org.cobbzilla.util.string.StringUtil.*; | |||
@Slf4j | |||
public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { | |||
private BlockList blockList = new BlockList(); | |||
private final AtomicReference<BlockList> blockList = new AtomicReference<>(new BlockList()); | |||
private BlockList getBlockList() { return blockList.get(); } | |||
private static Map<String, BlockListSource> blockListCache = new ConcurrentHashMap<>(); | |||
private final AtomicReference<Set<String>> fullyBlockedDomains = new AtomicReference<>(Collections.emptySet()); | |||
@Override public Set<String> getPrimedBlockDomains() { return fullyBlockedDomains.get(); } | |||
private final AtomicReference<Set<String>> partiallyBlockedDomains = new AtomicReference<>(Collections.emptySet()); | |||
@Override public Set<String> getPrimedFilterDomains() { return partiallyBlockedDomains.get(); } | |||
private final static Map<String, BlockListSource> blockListCache = new ConcurrentHashMap<>(); | |||
@Override public <C> Class<C> getConfigClass() { return (Class<C>) BubbleBlockConfig.class; } | |||
@Override public void init(JsonNode config, JsonNode userConfig, AppRule rule, AppMatcher matcher, Account account, Device device) { | |||
@Override public void init(JsonNode config, | |||
JsonNode userConfig, | |||
AppRule rule, | |||
AppMatcher matcher, | |||
Account account, | |||
Device device) { | |||
super.init(config, userConfig, rule, matcher, account, device); | |||
refreshBlockLists(); | |||
} | |||
@@ -61,6 +73,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { | |||
final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); | |||
final BubbleBlockList[] blockLists = bubbleBlockConfig.getBlockLists(); | |||
final Set<String> refreshed = new HashSet<>(); | |||
final BlockList newBlockList = new BlockList(); | |||
for (BubbleBlockList list : blockLists) { | |||
if (!list.enabled()) continue; | |||
@@ -91,8 +104,22 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { | |||
log.error("init: error adding additional entries: "+shortError(e)); | |||
} | |||
} | |||
if (blockListSource != null) blockList.merge(blockListSource.getBlockList()); | |||
if (blockListSource != null) newBlockList.merge(blockListSource.getBlockList()); | |||
} | |||
blockList.set(newBlockList); | |||
boolean doPrime = false; | |||
if (!newBlockList.getFullyBlockedDomains().equals(fullyBlockedDomains.get())) { | |||
fullyBlockedDomains.set(newBlockList.getFullyBlockedDomains()); | |||
doPrime = true; | |||
} | |||
if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) { | |||
partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains()); | |||
doPrime = true; | |||
} | |||
log.info("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size()); | |||
log.info("refreshBlockLists: partiallyBlockedDomains="+partiallyBlockedDomains.get().size()); | |||
log.info("refreshBlockLists: refreshed "+refreshed.size()+" block lists: "+StringUtil.toString(refreshed)); | |||
} | |||
@@ -178,7 +205,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { | |||
} | |||
} | |||
public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return blockList.getDecision(fqdn, uri, userAgent, false); } | |||
public BlockDecision getDecision(String fqdn, String uri, String userAgent) { return getBlockList().getDecision(fqdn, uri, userAgent, false); } | |||
public BlockDecision getDecision(String fqdn, String uri, String userAgent, boolean primary) { | |||
final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); | |||
@@ -189,7 +216,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { | |||
} | |||
} | |||
} | |||
return blockList.getDecision(fqdn, uri, primary); | |||
return getBlockList().getDecision(fqdn, uri, primary); | |||
} | |||
@Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { | |||
@@ -199,7 +226,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { | |||
// Now that we know the content type, re-check the BlockList | |||
final String contentType = filterRequest.getContentType(); | |||
final BlockDecision decision = blockList.getDecision(request.getFqdn(), request.getUri(), contentType, true); | |||
final BlockDecision decision = getBlockList().getDecision(request.getFqdn(), request.getUri(), contentType, true); | |||
if (log.isDebugEnabled()) log.debug(prefix+"preprocess decision was "+decision+", but now we know contentType="+contentType); | |||
switch (decision.getDecisionType()) { | |||
case block: | |||
@@ -255,8 +282,8 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { | |||
ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase()); | |||
ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId)); | |||
ctx.put(CTX_BUBBLE_SELECTORS, json(decision.getSelectors(), COMPACT_MAPPER)); | |||
ctx.put(CTX_BUBBLE_WHITELIST, json(blockList.getWhitelistDomains(), COMPACT_MAPPER)); | |||
ctx.put(CTX_BUBBLE_BLACKLIST, json(blockList.getBlacklistDomains(), COMPACT_MAPPER)); | |||
ctx.put(CTX_BUBBLE_WHITELIST, json(getBlockList().getWhitelistDomains(), COMPACT_MAPPER)); | |||
ctx.put(CTX_BUBBLE_BLACKLIST, json(getBlockList().getBlacklistDomains(), COMPACT_MAPPER)); | |||
return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx); | |||
} | |||
@@ -12,6 +12,7 @@ import com.github.jknack.handlebars.Handlebars; | |||
import lombok.Getter; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.apache.commons.io.input.ReaderInputStream; | |||
import org.cobbzilla.util.collection.SingletonSet; | |||
import org.cobbzilla.util.io.regex.RegexFilterReader; | |||
import org.cobbzilla.util.io.regex.RegexInsertionFilter; | |||
import org.cobbzilla.util.io.regex.RegexStreamFilter; | |||
@@ -19,6 +20,7 @@ import org.cobbzilla.util.io.regex.RegexStreamFilter; | |||
import java.io.InputStream; | |||
import java.util.HashMap; | |||
import java.util.Map; | |||
import java.util.Set; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.die; | |||
import static org.cobbzilla.util.http.HttpContentTypes.isHtml; | |||
@@ -51,6 +53,10 @@ public class UserBlockerRuleDriver extends AbstractAppRuleDriver { | |||
return json(json, JsonNode.class); | |||
} | |||
@Override public Set<String> getPrimedFilterDomains() { | |||
return matcher.isWildcardFqdn() ? null : new SingletonSet<>(matcher.getFqdn()); | |||
} | |||
protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); } | |||
@Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { | |||
@@ -88,6 +88,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration | |||
public static final String TAG_PROMO_CODE_POLICY = "promoCodePolicy"; | |||
public static final String TAG_REQUIRE_SEND_METRICS = "requireSendMetrics"; | |||
public static final String TAG_SUPPORT = "support"; | |||
public static final String TAG_CERT_VALIDATION_HOST = "certValidationHost"; // must match bubble_passthru.py | |||
public static final String DEFAULT_LOCAL_STORAGE_DIR = HOME_DIR + "/.bubble_local_storage"; | |||
@@ -137,6 +138,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration | |||
public boolean hasSageNode () { return getSageNode() != null; } | |||
@Getter @Setter private String letsencryptEmail; | |||
@Getter @Setter private String certValidationHost; | |||
@Setter private String localStorageDir = DEFAULT_LOCAL_STORAGE_DIR; | |||
public String getLocalStorageDir () { return empty(localStorageDir) ? DEFAULT_LOCAL_STORAGE_DIR : localStorageDir; } | |||
@@ -297,7 +300,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration | |||
{TAG_LOCKED, accountDAO.locked()}, | |||
{TAG_LAUNCH_LOCK, isSageLauncher() || thisNetwork == null ? null : thisNetwork.launchLock()}, | |||
{TAG_SSL_PORT, getDefaultSslPort()}, | |||
{TAG_SUPPORT, getSupport()} | |||
{TAG_SUPPORT, getSupport()}, | |||
{TAG_CERT_VALIDATION_HOST, getCertValidationHost()} | |||
})); | |||
} | |||
return publicSystemConfigs.get(); | |||
@@ -7,18 +7,23 @@ package bubble.server.listener; | |||
import bubble.dao.account.AccountDAO; | |||
import bubble.dao.cloud.CloudServiceDAO; | |||
import bubble.model.account.Account; | |||
import bubble.model.cloud.AnsibleInstallType; | |||
import bubble.model.cloud.BubbleNetwork; | |||
import bubble.model.cloud.BubbleNode; | |||
import bubble.model.cloud.CloudService; | |||
import bubble.server.BubbleConfiguration; | |||
import bubble.service.boot.SelfNodeService; | |||
import bubble.service.cloud.NetworkMonitorService; | |||
import bubble.service.stream.AppPrimerService; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.cobbzilla.wizard.cache.redis.RedisService; | |||
import org.cobbzilla.wizard.server.RestServer; | |||
import org.cobbzilla.wizard.server.RestServerLifecycleListenerBase; | |||
import java.io.File; | |||
import java.util.Map; | |||
import static bubble.server.BubbleConfiguration.TAG_CERT_VALIDATION_HOST; | |||
import static bubble.service.boot.StandardSelfNodeService.SELF_NODE_JSON; | |||
import static bubble.service.boot.StandardSelfNodeService.THIS_NODE_FILE; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.*; | |||
@@ -101,6 +106,18 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase<Bub | |||
} | |||
} | |||
// ensure default devices exist, cert_validation_host is set, and apps are primed | |||
if (thisNode != null) { | |||
final BubbleNetwork thisNetwork = c.getThisNetwork(); | |||
if (thisNetwork != null && thisNetwork.getInstallType() == AnsibleInstallType.node) { | |||
final String certValidationHost = c.getCertValidationHost(); | |||
if (!empty(certValidationHost)) { | |||
c.getBean(RedisService.class).set(TAG_CERT_VALIDATION_HOST, certValidationHost); | |||
} | |||
c.getBean(AppPrimerService.class).primeApps(); | |||
} | |||
} | |||
log.info("onStart: completed"); | |||
} | |||
@@ -81,6 +81,7 @@ public class AnsiblePrepService { | |||
} | |||
ctx.put("sslPort", network.getSslPort()); | |||
ctx.put("publicBaseUri", network.getPublicUri()); | |||
ctx.put("cert_validation_host", configuration.getCertValidationHost()); | |||
ctx.put("support", configuration.getSupport()); | |||
ctx.put("appLinks", configuration.getAppLinks()); | |||
@@ -42,7 +42,7 @@ public class DeviceIdService { | |||
@Autowired private BubbleConfiguration configuration; | |||
@Autowired private AccountDAO accountDAO; | |||
private Map<String, Device> deviceCache = new ExpirationMap<>(MINUTES.toMillis(10)); | |||
private final Map<String, Device> deviceCache = new ExpirationMap<>(MINUTES.toMillis(10)); | |||
public Device findDeviceByIp (String ipAddr) { | |||
@@ -74,8 +74,7 @@ import static bubble.service.boot.StandardSelfNodeService.*; | |||
import static bubble.service.cloud.NodeProgressMeter.getProgressMeterKey; | |||
import static bubble.service.cloud.NodeProgressMeter.getProgressMeterPrefix; | |||
import static bubble.service.cloud.NodeProgressMeterConstants.*; | |||
import static java.util.concurrent.TimeUnit.MINUTES; | |||
import static java.util.concurrent.TimeUnit.SECONDS; | |||
import static java.util.concurrent.TimeUnit.*; | |||
import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; | |||
import static org.cobbzilla.util.daemon.Await.awaitAll; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.*; | |||
@@ -0,0 +1,14 @@ | |||
package bubble.service.stream; | |||
import bubble.model.account.Account; | |||
import bubble.model.app.BubbleApp; | |||
public interface AppPrimerService { | |||
void primeApps(); | |||
void prime(BubbleApp app); | |||
void prime(Account account, String app); | |||
} |
@@ -0,0 +1,165 @@ | |||
package bubble.service.stream; | |||
import bubble.dao.account.AccountDAO; | |||
import bubble.dao.app.AppMatcherDAO; | |||
import bubble.dao.app.AppRuleDAO; | |||
import bubble.dao.app.BubbleAppDAO; | |||
import bubble.dao.app.RuleDriverDAO; | |||
import bubble.dao.device.DeviceDAO; | |||
import bubble.model.account.Account; | |||
import bubble.model.app.AppMatcher; | |||
import bubble.model.app.AppRule; | |||
import bubble.model.app.BubbleApp; | |||
import bubble.model.app.RuleDriver; | |||
import bubble.model.cloud.AnsibleInstallType; | |||
import bubble.model.cloud.BubbleNetwork; | |||
import bubble.model.device.Device; | |||
import bubble.rule.AppRuleDriver; | |||
import bubble.server.BubbleConfiguration; | |||
import bubble.service.cloud.DeviceIdService; | |||
import lombok.Getter; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.cobbzilla.util.collection.SingletonList; | |||
import org.cobbzilla.wizard.cache.redis.RedisService; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import java.util.*; | |||
import java.util.concurrent.ExecutorService; | |||
import java.util.stream.Collectors; | |||
import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.*; | |||
@Service @Slf4j | |||
public class StandardAppPrimerService implements AppPrimerService { | |||
@Autowired private AccountDAO accountDAO; | |||
@Autowired private DeviceDAO deviceDAO; | |||
@Autowired private DeviceIdService deviceIdService; | |||
@Autowired private BubbleAppDAO appDAO; | |||
@Autowired private AppMatcherDAO matcherDAO; | |||
@Autowired private AppRuleDAO ruleDAO; | |||
@Autowired private RuleDriverDAO driverDAO; | |||
@Autowired private RedisService redis; | |||
@Autowired private BubbleConfiguration configuration; | |||
@Getter(lazy=true) private final boolean primingEnabled = initPrimingEnabled(); | |||
private boolean initPrimingEnabled() { | |||
if (configuration == null) return die("initPrimingEnabled: configuration was null"); | |||
final BubbleNetwork thisNetwork = configuration.getThisNetwork(); | |||
if (thisNetwork == null) { | |||
log.info("primeApps: thisNetwork is null, not priming"); | |||
return false; | |||
} | |||
if (thisNetwork.getInstallType() != AnsibleInstallType.node) { | |||
log.info("primeApps: thisNetwork is not a node, not priming"); | |||
return false; | |||
} | |||
return true; | |||
} | |||
public void primeApps() { | |||
if (!isPrimingEnabled()) { | |||
log.info("primeApps: not enabled"); | |||
return; | |||
} | |||
for (Account account : accountDAO.findNotDeleted()) { | |||
try { | |||
prime(account); | |||
} catch (Exception e) { | |||
log.error("primeApps("+account.getName()+"): "+shortError(e), e); | |||
} | |||
} | |||
} | |||
public void prime(Account account) { prime(account, (BubbleApp) null); } | |||
public void prime(BubbleApp app) { | |||
final Account account = accountDAO.findByUuid(app.getAccount()); | |||
if (account == null) { | |||
log.warn("prime("+app.getName()+"): account not found: "+app.getAccount()); | |||
return; | |||
} | |||
prime(account, app.getUuid()); | |||
} | |||
public synchronized void prime(Account account, String singleAppUuid) { | |||
final BubbleApp singleApp = appDAO.findByAccountAndId(account.getUuid(), singleAppUuid); | |||
if (singleApp == null) { | |||
log.warn("prime("+account.getName()+", "+singleAppUuid+"): app not found: "+singleAppUuid); | |||
return; | |||
} | |||
prime(account, singleApp); | |||
} | |||
@Getter(lazy=true) private final ExecutorService primerThread = fixedPool(1); | |||
private void prime(Account account, BubbleApp singleApp) { | |||
if (!isPrimingEnabled()) { | |||
log.info("prime: not enabled"); | |||
return; | |||
} | |||
getPrimerThread().submit(() -> _prime(account, singleApp)); | |||
} | |||
private synchronized void _prime(Account account, BubbleApp singleApp) { | |||
try { | |||
final Map<String, List<String>> accountDeviceIps = new HashMap<>(); | |||
final List<Device> devices = deviceDAO.findByAccount(account.getUuid()); | |||
for (Device device : devices) { | |||
accountDeviceIps.put(device.getUuid(), deviceIdService.findIpsByDevice(device.getUuid())); | |||
} | |||
if (accountDeviceIps.isEmpty()) return; | |||
final List<BubbleApp> appsToPrime = singleApp == null | |||
? appDAO.findByAccount(account.getUuid()).stream() | |||
.filter(BubbleApp::canPrime) | |||
.collect(Collectors.toList()) | |||
: new SingletonList<>(singleApp); | |||
for (BubbleApp app : appsToPrime) { | |||
final List<AppRule> rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); | |||
final List<AppMatcher> matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); | |||
for (AppRule rule : rules) { | |||
final RuleDriver driver = driverDAO.findByUuid(rule.getDriver()); | |||
if (driver == null) { | |||
log.warn("_prime: driver not found for app/rule " + app.getName() + "/" + rule.getName() + ": " + rule.getDriver()); | |||
continue; | |||
} | |||
for (Device device : devices) { | |||
final Set<String> blockDomains = new HashSet<>(); | |||
final Set<String> filterDomains = new HashSet<>(); | |||
for (AppMatcher matcher : matchers) { | |||
final AppRuleDriver appRuleDriver = rule.initDriver(driver, matcher, account, device); | |||
final Set<String> blocks = appRuleDriver.getPrimedBlockDomains(); | |||
if (empty(blocks)) { | |||
log.info("_prime: no blockDomains for app/rule/matcher: " + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); | |||
} else { | |||
blockDomains.addAll(blocks); | |||
} | |||
final Set<String> filters = appRuleDriver.getPrimedFilterDomains(); | |||
if (empty(filters)) { | |||
log.info("_prime: no filterDomains for app/rule/matcher: " + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); | |||
} else { | |||
filterDomains.addAll(filters); | |||
} | |||
} | |||
if (!empty(blockDomains) || !empty(filterDomains)) { | |||
for (String ip : accountDeviceIps.get(device.getUuid())) { | |||
if (!empty(blockDomains)) { | |||
AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new)); | |||
} | |||
if (!empty(filterDomains)) { | |||
AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new)); | |||
} | |||
} | |||
} | |||
} | |||
} | |||
} | |||
} catch (Exception e) { | |||
die("_prime: "+shortError(e), e); | |||
} | |||
} | |||
} |
@@ -173,7 +173,8 @@ public class StandardRuleEngineService implements RuleEngineService { | |||
return sendResponse(state.getResponseStream(last)); | |||
} | |||
private ExpirationMap<String, List<AppRuleHarness>> ruleCache = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime); | |||
private final ExpirationMap<String, List<AppRuleHarness>> ruleCache | |||
= new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime); | |||
public Map<Object, Object> flushCaches() { | |||
final int ruleEngineCacheSize = ruleCache.size(); | |||
@@ -0,0 +1,19 @@ | |||
package bubble.service_dbfilter; | |||
import bubble.model.account.Account; | |||
import bubble.model.app.BubbleApp; | |||
import bubble.service.stream.AppPrimerService; | |||
import org.springframework.stereotype.Service; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; | |||
@Service | |||
public class DbFilterAppPrimerService implements AppPrimerService { | |||
@Override public void primeApps() { notSupported("primeApps"); } | |||
@Override public void prime(Account account, String app) { notSupported("prime"); } | |||
@Override public void prime(BubbleApp app) { notSupported("prime"); } | |||
} |
@@ -1 +1 @@ | |||
bubble.version=0.11.3 | |||
bubble.version=0.12.0 |
@@ -22,20 +22,17 @@ if [[ "$(whoami)" != "{{node.user}}" ]] ; then | |||
fi | |||
ANSIBLE_DIR="${ANSIBLE_HOME}/ansible" | |||
ID_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa" | |||
PUB_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa.pub" | |||
AUTH_KEYS="${ANSIBLE_HOME}/.ssh/authorized_keys" | |||
if [[ ! -d "${ANSIBLE_DIR}" ]] ; then | |||
die "Ansible dir not found or not a directory: ${ANSIBLE_DIR}" | |||
fi | |||
if [[ ! -f "${ID_FILE}" ]] ; then | |||
ssh-keygen -t rsa -q -N '' -f ${ID_FILE} || die "Error generating RSA key" | |||
fi | |||
# lockout the node that started us | |||
cat "${PUB_FILE}" > "${AUTH_KEYS}" || die "Error updating ${AUTH_KEYS} file" | |||
cat /dev/null > "${AUTH_KEYS}" || die "Error truncating ${AUTH_KEYS} file" | |||
# ensure proper permissions on authorized_keys file | |||
chmod 600 "${AUTH_KEYS}" || die "Error setting permissions on ${AUTH_KEYS} file" | |||
# add admin ssh key, if one was given | |||
ADMIN_PUB_KEY="${ANSIBLE_DIR}/roles/bubble/files/admin_ssh_key.pub" | |||
@@ -43,8 +40,6 @@ if [[ -f "${ADMIN_PUB_KEY}" ]] ; then | |||
cat "${ADMIN_PUB_KEY}" >> "${AUTH_KEYS}" | |||
fi | |||
SSH_OPTIONS="--ssh-extra-args '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=publickey -i ${ID_FILE}'" | |||
SKIP_TAGS="" | |||
if [[ -n "{{restoreKey}}" ]] ; then | |||
SKIP_TAGS="--skip-tags algo_related" | |||
@@ -55,5 +50,5 @@ cd "${ANSIBLE_DIR}" && \ | |||
virtualenv -p python3 ./venv && \ | |||
. ./venv/bin/activate && \ | |||
pip3 install ansible && \ | |||
bash -c "ansible-playbook ${SSH_OPTIONS} ${SKIP_TAGS} --inventory ./hosts ./playbook.yml 2>&1 | tee ${LOG}" \ | |||
ansible-playbook ${SKIP_TAGS} --inventory ./hosts ./playbook.yml \ | |||
|| die "Error running ansible. journalctl -xe = $(journalctl -xe | tail -n 50)" |
@@ -15,12 +15,8 @@ | |||
when: restore_key is not defined | |||
# Don't start monitors when in restore mode, bubble_restore_monitor.sh will start it after algo is installed | |||
- name: Restart algo monitors | |||
shell: bash -c "supervisorctl reload && sleep 5s && supervisorctl restart algo_refresh_users_monitor && supervisorctl restart wg_monitor_connections" | |||
when: restore_key is not defined | |||
- name: Stop algo monitors (in restore mode) | |||
shell: bash -c "supervisorctl reload && sleep 5s && supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections" | |||
shell: bash -c "supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections" | |||
when: restore_key is defined | |||
# Add bubble rules | |||
@@ -43,4 +43,10 @@ CONFIGS_BACKUP=/home/bubble/.BUBBLE_ALGO_CONFIGS.tgz | |||
cd ${ALGO_BASE} && tar czf ${CONFIGS_BACKUP} configs && chgrp bubble ${CONFIGS_BACKUP} && chmod 660 ${CONFIGS_BACKUP} || die "Error backing up algo configs" | |||
cd /home/bubble && tar xzf ${CONFIGS_BACKUP} && chgrp -R bubble configs && chown -R bubble configs && chmod 500 configs || die "Error unpacking algo configs to bubble home" | |||
# Restart algo_refresh_users_monitor and wg_monitor_connections | |||
supervisorctl reload && sleep 5s && supervisorctl restart algo_refresh_users_monitor && supervisorctl restart wg_monitor_connections | |||
# Restart dnscrypt-proxy | |||
service dnscrypt-proxy restart | |||
touch "${ALGO_BASE}/.install_marker" |
@@ -30,6 +30,8 @@ | |||
{"name": "error_key", "value": "[[error_key]]"}, | |||
{"name": "error_env", "value": "[[error_env]]"}, | |||
{"name": "cert_validation_host", "value": "[[cert_validation_host]]"}, | |||
{"name": "support_email", "value": "[[support.email]]"}, | |||
{"name": "support_site", "value": "[[support.site]]"}, | |||
@@ -12,6 +12,7 @@ export ERRBIT_URL={{ error_url | default('') }} | |||
export ERRBIT_KEY={{ error_key | default('') }} | |||
export ERRBIT_ENV={{ error_env | default('') }} | |||
export CERT_VALIDATION_HOST={{ cert_validation_host }} | |||
export SUPPORT_EMAIL={{ support_email }} | |||
export SUPPORT_SITE={{ support_site }} | |||
@@ -1,8 +1,8 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
- name: Snapshot ansible roles | |||
shell: snapshot_ansible.sh | |||
- name: Snapshot ansible roles in the background | |||
command: /usr/local/bin/snapshot_ansible.sh | |||
- name: Touch first-time setup file | |||
shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n install > /home/bubble/first_time_marker ; fi" | |||
@@ -20,9 +20,6 @@ | |||
- import_tasks: route.yml | |||
- name: Restart dnscrypt-proxy | |||
shell: service dnscrypt-proxy restart | |||
- name: Install supervisor conf file | |||
copy: | |||
src: supervisor_mitmproxy.conf | |||
@@ -16,7 +16,6 @@ | |||
template: src=stronger_dhparams.conf dest=/etc/nginx/conf.d/stronger_dhparams.conf | |||
- include: site.yml | |||
- meta: flush_handlers # nginx has to be restarted right now if it has to | |||
- name: Init certbot | |||
shell: init_certbot.sh {{ letsencrypt_email }} {{ server_name }} {{ server_alias }} | |||
@@ -88,6 +88,8 @@ rateLimits: | |||
- { limit: 50000, interval: 1h, block: 24h } | |||
- { limit: 100000, interval: 6h, block: 96h } | |||
certValidationHost: {{CERT_VALIDATION_HOST}} | |||
support: | |||
email: {{SUPPORT_EMAIL}} | |||
site: {{SUPPORT_SITE}} | |||
@@ -20,6 +20,12 @@ support_email_link=Send us an email | |||
support_not_available=Sorry, no support options are available | |||
link_support=Support | |||
# Cert checker | |||
title_check_certificate=Bubble Certificate Verification | |||
check_cert_null=Verifying certification installation... | |||
check_cert_true=Your Bubble Certificate is properly installed | |||
check_cert_false=Your Bubble Certificate is not properly installed | |||
# Legal page links | |||
title_legal_topics=Legal Stuff | |||
legal_topics=terms,privacy,source,license,3rdParty_licenses | |||
@@ -5,6 +5,7 @@ | |||
"template": true, | |||
"enabled": true, | |||
"priority": 100, | |||
"canPrime": false, | |||
"dataConfig": { | |||
"dataDriver": "bubble.app.analytics.TrafficAnalyticsAppDataDriver", | |||
"presentation": "app", | |||
@@ -5,6 +5,7 @@ | |||
"template": true, | |||
"enabled": true, | |||
"priority": 200, | |||
"canPrime": true, | |||
"dataConfig": { | |||
"dataDriver": "bubble.app.bblock.BubbleBlockAppDataDriver", | |||
"presentation": "app", | |||
@@ -16,8 +17,7 @@ | |||
{"name": "data"} | |||
], | |||
"params": [ | |||
{"name": "device", "required": false, "index": 10}, | |||
{"name": "meta2", "required": false, "operator": "like", "index": 20} | |||
{"name": "device", "required": false, "index": 10} | |||
], | |||
"views": [ | |||
{"name": "last_24_hours"}, | |||
@@ -5,6 +5,7 @@ | |||
"template": true, | |||
"enabled": true, | |||
"priority": 1000000, | |||
"canPrime": false, | |||
"dataConfig": { | |||
"dataDriver": "bubble.app.passthru.TlsPassthruAppDataDriver", | |||
"presentation": "none", | |||
@@ -5,6 +5,7 @@ | |||
"template": true, | |||
"enabled": true, | |||
"priority": 300, | |||
"canPrime": true, | |||
"dataConfig": { | |||
"dataDriver": "bubble.app.social.block.UserBlockerAppDataDriver", | |||
"presentation": "site", | |||
@@ -13,7 +13,7 @@ | |||
get_url: | |||
url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip | |||
dest: /tmp/algo.zip | |||
checksum: sha256:f9709ced6cbaf92473c975355050915657da50dfc0eaf655f178643953a9cd42 | |||
checksum: sha256:357e613833385626e88564c97f0b5726f49686c33a774be9a7766bd1a1249915 | |||
- name: Unzip algo master.zip | |||
unarchive: | |||
@@ -114,8 +114,6 @@ if [[ ! -f ${CONFIGS_BACKUP} ]] ; then | |||
else | |||
ANSIBLE_HOME="/root" | |||
ANSIBLE_DIR="${ANSIBLE_HOME}/ansible" | |||
ID_FILE="${ANSIBLE_HOME}/.ssh/bubble_rsa" | |||
SSH_OPTIONS="--ssh-extra-args '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o PreferredAuthentications=publickey -i ${ID_FILE}'" | |||
ALGO_BASE=${ANSIBLE_DIR}/roles/algo/algo | |||
if [[ ! -d ${ALGO_BASE} ]] ; then | |||
@@ -126,7 +124,7 @@ else | |||
cd "${ANSIBLE_DIR}" && \ | |||
. ./venv/bin/activate && \ | |||
bash -c \ | |||
"ansible-playbook ${SSH_OPTIONS} --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \ | |||
"ansible-playbook --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \ | |||
|| die "Error running ansible in post-restore. journalctl -xe = $(journalctl -xe | tail -n 50)" | |||
fi | |||
@@ -49,14 +49,14 @@ def bubble_activity_log(client_addr, server_addr, event, data): | |||
'client_addr': client_addr, | |||
'server_addr': server_addr, | |||
'event': event, | |||
'data': data | |||
'data': str(data) | |||
}) | |||
bubble_log('bubble_activity_log: setting '+key+' = '+value) | |||
redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION) | |||
pass | |||
def bubble_passthru(remote_addr, addr, fqdn): | |||
def bubble_passthru(remote_addr, addr, fqdns): | |||
headers = { | |||
'X-Forwarded-For': remote_addr, | |||
'Accept' : 'application/json', | |||
@@ -65,7 +65,7 @@ def bubble_passthru(remote_addr, addr, fqdn): | |||
try: | |||
data = { | |||
'addr': str(addr), | |||
'fqdn': str(fqdn), | |||
'fqdns': fqdns, | |||
'remoteAddr': remote_addr | |||
} | |||
response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data) | |||
@@ -29,18 +29,69 @@ from mitmproxy.exceptions import TlsProtocolException | |||
from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set | |||
import redis | |||
import json | |||
import subprocess | |||
REDIS_DNS_PREFIX = 'bubble_dns_' | |||
REDIS_PASSTHRU_PREFIX = 'bubble_passthru_' | |||
REDIS_CLIENT_CERT_STATUS_PREFIX = 'bubble_cert_status_' | |||
REDIS_PASSTHRU_DURATION = 60 * 60 # 1 hour timeout on passthru | |||
REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) | |||
cert_validation_host = None | |||
local_ips = None | |||
def get_ip_cert_status(client_addr): | |||
status = REDIS.get(REDIS_CLIENT_CERT_STATUS_PREFIX+client_addr) | |||
if status is None: | |||
return None | |||
enabled = status.decode() == "True" | |||
return enabled | |||
def set_ip_cert_status(client_addr, enabled): | |||
REDIS.set(REDIS_CLIENT_CERT_STATUS_PREFIX+client_addr, str(enabled)) | |||
bubble_log('set_ip_cert_status: set '+client_addr+' = '+str(enabled)) | |||
def get_local_ips(): | |||
global local_ips | |||
if local_ips is None: | |||
local_ips = [] | |||
for ip in subprocess.check_output(['hostname', '-I']).split(): | |||
local_ips.append(ip.decode()) | |||
return local_ips | |||
def get_cert_validation_host(): | |||
global cert_validation_host | |||
if cert_validation_host is None: | |||
cert_validation_host = REDIS.get('certValidationHost') | |||
if cert_validation_host is not None: | |||
cert_validation_host = cert_validation_host.decode() | |||
# bubble_log('get_cert_validation_host: initialized to '+cert_validation_host) | |||
# bubble_log('get_cert_validation_host: returning '+cert_validation_host) | |||
return cert_validation_host | |||
def passthru_cache_prefix(client_addr, server_addr): | |||
return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr | |||
def fqdns_for_addr(addr): | |||
prefix = REDIS_DNS_PREFIX + addr | |||
keys = REDIS.keys(prefix + '_*') | |||
if keys is None or len(keys) == 0: | |||
bubble_log('fqdns_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr') | |||
return '' | |||
fqdns = [] | |||
for k in keys: | |||
fqdn = k.decode()[len(prefix)+1:] | |||
fqdns.append(fqdn) | |||
return fqdns | |||
class TlsFeedback(TlsLayer): | |||
""" | |||
Monkey-patch _establish_tls_with_client to get feedback if TLS could be established | |||
@@ -49,51 +100,61 @@ class TlsFeedback(TlsLayer): | |||
def _establish_tls_with_client(self): | |||
client_address = self.client_conn.address[0] | |||
server_address = self.server_conn.address[0] | |||
fqdns = fqdns_for_addr(server_address) | |||
try: | |||
super(TlsFeedback, self)._establish_tls_with_client() | |||
if fqdns and get_cert_validation_host() in fqdns: | |||
# bubble_log('_establish_tls_with_client: TLS success for '+repr(server_address)+', enabling SSL interception for client '+client_address) | |||
set_ip_cert_status(client_address, True) | |||
except TlsProtocolException as e: | |||
bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru') | |||
cache_key = passthru_cache_prefix(client_address, server_address) | |||
fqdn = fqdn_for_addr(server_address) | |||
redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) | |||
bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key) | |||
if fqdns and get_cert_validation_host() in fqdns: | |||
set_ip_cert_status(client_address, False) | |||
else: | |||
redis_set(cache_key, json.dumps({'fqdns': fqdns, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) | |||
raise e | |||
def fqdn_for_addr(addr): | |||
fqdn = REDIS.get(REDIS_DNS_PREFIX + addr) | |||
if fqdn is None or len(fqdn) == 0: | |||
bubble_log('fqdn_for_addr: no FQDN found for addr '+repr(addr)+', checking raw addr') | |||
fqdn = b'' | |||
return fqdn.decode() | |||
def check_bubble_passthru(client_addr, addr, fqdns): | |||
cert_status = get_ip_cert_status(client_addr) | |||
if cert_status is not None and not cert_status: | |||
bubble_log('check_bubble_passthru: returning True because cert_status for '+client_addr+' was False') | |||
return {'fqdns': fqdns, 'addr': addr, 'passthru': True} | |||
else: | |||
bubble_log('check_bubble_passthru: request is NOT for cert_validation_host: '+cert_validation_host+", it is for one of fqdn="+repr(fqdns)+", checking bubble_passthru...") | |||
def check_bubble_passthru(remote_addr, addr, fqdn): | |||
passthru = bubble_passthru(remote_addr, addr, fqdn) | |||
if passthru is None: | |||
return None | |||
if passthru: | |||
bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True') | |||
return {'fqdn': fqdn, 'addr': addr, 'passthru': True} | |||
bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False') | |||
return {'fqdn': fqdn, 'addr': addr, 'passthru': False} | |||
passthru = bubble_passthru(client_addr, addr, fqdns) | |||
if passthru is None or passthru: | |||
bubble_log('check_bubble_passthru: bubble_passthru returned '+repr(passthru)+' for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning True') | |||
return {'fqdns': fqdns, 'addr': addr, 'passthru': True} | |||
bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdns)+'/'+repr(addr)+', returning False') | |||
return {'fqdns': fqdns, 'addr': addr, 'passthru': False} | |||
def should_passthru(client_addr, addr, fqdns): | |||
# always passthru for local ips | |||
if addr in get_local_ips(): | |||
# bubble_log('should_passthru: local ip is always passthru: '+addr) | |||
return {'fqdns': fqdns, 'addr': addr, 'passthru': True} | |||
else: | |||
# bubble_log('should_passthru: addr ('+addr+') is not a local ip: '+repr(get_local_ips())) | |||
pass | |||
cache_key = passthru_cache_prefix(client_addr, addr) | |||
prefix = 'should_passthru: ip='+repr(addr)+' (fqdns='+repr(fqdns)+') cache_key='+cache_key+': ' | |||
def should_passthru(remote_addr, addr): | |||
prefix = 'should_passthru: '+repr(addr)+' ' | |||
bubble_log(prefix+'starting...') | |||
cache_key = passthru_cache_prefix(remote_addr, addr) | |||
passthru_json = REDIS.get(cache_key) | |||
if passthru_json is None or len(passthru_json) == 0: | |||
bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...') | |||
fqdn = fqdn_for_addr(addr) | |||
if fqdn is None or len(fqdn) == 0: | |||
fqdn = 'NONE' | |||
passthru = check_bubble_passthru(remote_addr, addr, fqdn) | |||
bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...") | |||
if passthru is not None: | |||
redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) | |||
bubble_log(prefix+'not in redis or empty, calling check_bubble_passthru against fqdns='+repr(fqdns)) | |||
passthru = check_bubble_passthru(client_addr, addr, fqdns) | |||
bubble_log(prefix+'check_bubble_passthru('+repr(fqdns)+') returned '+repr(passthru)+", storing in redis...") | |||
redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) | |||
else: | |||
bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis') | |||
bubble_log(prefix+'found passthru_json='+str(passthru_json)+', touching key in redis') | |||
passthru = json.loads(passthru_json) | |||
REDIS.touch(cache_key) | |||
bubble_log(prefix+'returning '+repr(passthru)) | |||
@@ -104,13 +165,23 @@ def next_layer(next_layer): | |||
if isinstance(next_layer, TlsLayer) and next_layer._client_tls: | |||
client_address = next_layer.client_conn.address[0] | |||
server_address = next_layer.server_conn.address[0] | |||
passthru = should_passthru(client_address, server_address) | |||
if passthru is None or passthru['passthru']: | |||
bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address)) | |||
if passthru is not None and 'fqdn' in passthru: | |||
bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn']) | |||
next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) | |||
next_layer.reply.send(next_layer_replacement) | |||
else: | |||
bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn']) | |||
fqdns = fqdns_for_addr(server_address) | |||
validation_host = get_cert_validation_host() | |||
if fqdns and validation_host in fqdns: | |||
bubble_log('next_layer: never passing thru (always getting feedback) for cert_validation_host='+validation_host) | |||
next_layer.__class__ = TlsFeedback | |||
else: | |||
bubble_log('next_layer: checking should_passthru for client_address='+client_address) | |||
passthru = should_passthru(client_address, server_address, fqdns) | |||
if passthru is None or passthru['passthru']: | |||
# bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address)) | |||
if passthru is not None and 'fqdns' in passthru: | |||
bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdns']) | |||
next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) | |||
next_layer.reply.send(next_layer_replacement) | |||
else: | |||
# bubble_log('next_layer: NO PASSTHRU (getting feedback) for client_address='+client_address+', server_address='+server_address) | |||
bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdns']) | |||
next_layer.__class__ = TlsFeedback |
@@ -19,16 +19,16 @@ | |||
{ | |||
"before": "sleep 5s", | |||
"comment": "root: check email inbox for verification message", | |||
"comment": "root: check email inbox for welcome message", | |||
"request": { | |||
"session": "rootSession", | |||
"uri": "debug/inbox/email/account_credit_user@example.com?type=request&action=verify&target=account" | |||
"uri": "debug/inbox/email/account_credit_user@example.com?type=notice&action=welcome&target=account" | |||
}, | |||
"response": { | |||
"store": "emailInbox", | |||
"check": [ | |||
{"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"}, | |||
{"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"}, | |||
{"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"}, | |||
{"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"}, | |||
{"condition": "'{{json.[0].ctx.message.target}}' == 'account'"} | |||
] | |||
} | |||
@@ -47,16 +47,16 @@ | |||
}, | |||
{ | |||
"comment": "root: check email inbox for verification message for referring user", | |||
"comment": "root: check email inbox for welcome message for referring user", | |||
"request": { | |||
"session": "rootSession", | |||
"uri": "debug/inbox/email/test_user_referring_multi@example.com?type=request&action=verify&target=account" | |||
"uri": "debug/inbox/email/test_user_referring_multi@example.com?type=notice&action=welcome&target=account" | |||
}, | |||
"response": { | |||
"store": "emailInbox", | |||
"check": [ | |||
{"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"}, | |||
{"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"}, | |||
{"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"}, | |||
{"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"}, | |||
{"condition": "'{{json.[0].ctx.message.target}}' == 'account'"} | |||
] | |||
} | |||
@@ -42,16 +42,16 @@ | |||
}, | |||
{ | |||
"comment": "root: check email inbox for verification message for referring user", | |||
"comment": "root: check email inbox for welcome message for referring user", | |||
"request": { | |||
"session": "rootSession", | |||
"uri": "debug/inbox/email/test_user_referring_free@example.com?type=request&action=verify&target=account" | |||
"uri": "debug/inbox/email/test_user_referring_free@example.com?type=notice&action=welcome&target=account" | |||
}, | |||
"response": { | |||
"store": "emailInbox", | |||
"check": [ | |||
{"condition": "'{{json.[0].ctx.message.messageType}}' == 'request'"}, | |||
{"condition": "'{{json.[0].ctx.message.action}}' == 'verify'"}, | |||
{"condition": "'{{json.[0].ctx.message.messageType}}' == 'notice'"}, | |||
{"condition": "'{{json.[0].ctx.message.action}}' == 'welcome'"}, | |||
{"condition": "'{{json.[0].ctx.message.target}}' == 'account'"} | |||
] | |||
} | |||
@@ -1 +1 @@ | |||
Subproject commit e25605ea6a578609b47d2f7c9da9dc87debbb8bb | |||
Subproject commit df343fc4b3e1f123b3caa025a493861edf46b81a |
@@ -1 +1 @@ | |||
Subproject commit 17d93ddf2d3436228504ec98a52008efa665d339 | |||
Subproject commit d450510d6f1be5b328b919ae73d8c1450f008d91 |