diff --git a/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java new file mode 100644 index 00000000..1da8f3c4 --- /dev/null +++ b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java @@ -0,0 +1,115 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.app.request; + +import bubble.model.account.Account; +import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; +import bubble.model.app.config.AppConfigDriverBase; +import bubble.rule.request.HeaderReplacement; +import bubble.rule.request.RequestProtectorConfig; +import bubble.rule.request.RequestProtectorRuleDriver; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Set; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; +import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; + +@Slf4j +public class RequestProtectorAppConfigDriver extends AppConfigDriverBase { + + public static final String VIEW_manageHeaderReplacements = "manageHeaderReplacements"; + + @Override public Object getView(Account account, BubbleApp app, String view, Map params) { + switch (view) { + case VIEW_manageHeaderReplacements: + return loadManageCookiesReplacements(account, app); + } + throw notFoundEx(view); + } + + private Set loadManageCookiesReplacements(Account account, BubbleApp app) { + final RequestProtectorConfig config = getConfig(account, app); + return config.getHeaderReplacements(); + } + + private RequestProtectorConfig getConfig(Account account, BubbleApp app) { + return getConfig(account, app, RequestProtectorRuleDriver.class, RequestProtectorConfig.class); + } + + public static final String ACTION_addHeaderReplacement = "addHeaderReplacement"; + public static final String ACTION_removeHeaderReplacement = "removeHeaderReplacement"; + + public static final String PARAM_REGEX = "regex"; + public static final String PARAM_REPLACEMENT = "replacement"; + + @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action, + Map params, JsonNode data) { + switch (action) { + case ACTION_addHeaderReplacement: + return addHeaderReplacement(account, app, data); + } + if (log.isWarnEnabled()) log.warn("takeAppAction: action not found: "+action); + throw notFoundEx(action); + } + + private Set addHeaderReplacement(Account account, BubbleApp app, JsonNode data) { + final JsonNode regexNode = data.get(PARAM_REGEX); + if (regexNode == null || regexNode.textValue() == null) { + throw invalidEx("err.requestProtector.headerRegexRequired"); + } + final String regex = regexNode.textValue().trim(); + if (empty(regex)) throw invalidEx("err.requestProtector.headerRegexRequired"); + + final JsonNode replacementNode = data.get(PARAM_REPLACEMENT); + final String replacement = (replacementNode == null || replacementNode.textValue() == null) + ? "" : replacementNode.textValue().trim(); + + final RequestProtectorConfig config = getConfig(account, app).addHeaderReplacement(regex, replacement); + + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver + if (log.isDebugEnabled()) { + log.debug("addHeaderReplacement: updating rule: " + rule.getName() + ", adding regex: " + regex); + } + ruleDAO.update(rule.setConfigJson(json(config))); + + return config.getHeaderReplacements(); + } + + @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id, + Map params, JsonNode data) { + switch (action) { + case ACTION_removeHeaderReplacement: + return removeHeaderReplacement(account, app, id); + } + if (log.isWarnEnabled()) log.warn("takeItemAction: action not found: "+action); + throw notFoundEx(action); + } + + private Set removeHeaderReplacement(Account account, BubbleApp app, String regex) { + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver + final RequestProtectorConfig config = getConfig(account, app); + if (log.isDebugEnabled()) { + log.debug("removeHeaderReplacement: removing regex: " + regex + " from config.cookiesReplacements: " + + config.getHeaderReplacements().toString()); + } + + final RequestProtectorConfig updated = config.removeHeaderReplacement(regex); + if (log.isDebugEnabled()) { + log.debug("removeHeaderReplacement: updated.cookiesReplacements: " + + updated.getHeaderReplacements().toString()); + } + ruleDAO.update(rule.setConfigJson(json(updated))); + + return updated.getHeaderReplacements(); + } +} diff --git a/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java new file mode 100644 index 00000000..d072acda --- /dev/null +++ b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java @@ -0,0 +1,9 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.app.request; + +import bubble.model.app.config.AppDataDriverBase; + +public class RequestProtectorAppDataDriver extends AppDataDriverBase {} diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 7b951950..f52a6828 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -47,6 +47,7 @@ public interface AppRuleDriver { String REDIS_FILTER_LISTS = "filterLists"; String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy and dnscrypt-proxy for flex routing String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists"; // used in mitmproxy and dnscrypt-proxy for flex routing + String REDIS_RESPONSE_HEADER_MODIFIER_LISTS = "responseHeaderModifierLists"; // used in mitmproxy String REDIS_LIST_SUFFIX = "~UNION"; default Set getPrimedRejectDomains () { return null; } @@ -55,6 +56,7 @@ public interface AppRuleDriver { default Set getPrimedFilterDomains () { return null; } default Set getPrimedFlexDomains () { return null; } default Set getPrimedFlexExcludeDomains () { return null; } + default Set getPrimedResponseHeaderModifiers () { return null; } static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) { defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains); @@ -80,12 +82,22 @@ public interface AppRuleDriver { defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains); } - static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { + static void defineRedisResponseHeaderModifiersSet(RedisService redis, String ip, String list, + String[] modifiers) { + defineRedisSet(redis, ip, REDIS_RESPONSE_HEADER_MODIFIER_LISTS, list, modifiers); + } + + /** + * `settings` parameter may be list of domains or any other list of strings - i.e. list of JSONs with specific setup + * for the prime option of the driver. + */ + static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, + String[] settings) { final String listOfListsForIp = listOfListsName + "~" + ip; final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX; final String ipList = listOfListsForIp + "~" + listName; final String tempList = ipList + "~"+now()+randomAlphanumeric(5); - redis.sadd_plaintext(tempList, domains); + redis.sadd_plaintext(tempList, settings); redis.rename(tempList, ipList); redis.sadd_plaintext(listOfListsForIp, ipList); final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp)); diff --git a/bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java b/bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java new file mode 100644 index 00000000..ba7befb4 --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule.request; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.Setter; +import lombok.experimental.Accessors; + +@NoArgsConstructor @Accessors(chain=true) +public class HeaderReplacement implements Comparable { + + public String getId() { return regex; } + public void setId(String id) {} // noop + + @Getter @Setter private String regex; + @Getter @Setter private String replacement; + + public HeaderReplacement(@NonNull final String regex, @NonNull final String replacement) { + this.regex = regex; + this.replacement = replacement; + } + + @Override public int compareTo(@NonNull final HeaderReplacement o) { + return getRegex().compareTo(o.getRegex().toLowerCase()); + } +} diff --git a/bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java new file mode 100644 index 00000000..ae2555ab --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule.request; + +import lombok.Getter; +import lombok.NonNull; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; + +import java.util.Set; +import java.util.TreeSet; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@Slf4j @Accessors(chain=true) +public class RequestProtectorConfig { + + @Getter @Setter private Set headerReplacements = new TreeSet<>(); + public boolean hasHeaderReplacements() { return !empty(headerReplacements); } + + @NonNull public RequestProtectorConfig addHeaderReplacement(@NonNull final String regex, + @NonNull final String replacement) { + headerReplacements.add(new HeaderReplacement(regex, replacement)); + return this; + } + + @NonNull public RequestProtectorConfig removeHeaderReplacement(@NonNull final String regex) { + if (hasHeaderReplacements()) headerReplacements.removeIf(r -> r.getRegex().equals(regex)); + return this; + } +} diff --git a/bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java new file mode 100644 index 00000000..860585ee --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule.request; + +import bubble.model.account.Account; +import bubble.model.app.AppMatcher; +import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; +import bubble.model.device.Device; +import bubble.rule.AbstractAppRuleDriver; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.json.JsonUtil; + +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequestProtectorRuleDriver extends AbstractAppRuleDriver { + + @Override public Class getConfigClass() { return (Class) RequestProtectorConfig.class; } + + @Override public Set getPrimedResponseHeaderModifiers() { + final RequestProtectorConfig config = getRuleConfig(); + return config.getHeaderReplacements().stream().map(JsonUtil::json).collect(Collectors.toSet()); + } + + @Override public void init(JsonNode config, JsonNode userConfig, BubbleApp app, AppRule rule, AppMatcher matcher, + Account account, Device device) { + super.init(config, userConfig, app, rule, matcher, account, device); + + // refresh list + final RequestProtectorConfig ruleConfig = getRuleConfig(); + ruleConfig.getHeaderReplacements(); + } +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index bd94ffa3..23d813d7 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -16,14 +16,19 @@ import bubble.server.BubbleConfiguration; import bubble.service.device.DeviceService; import bubble.service.device.StandardFlexRouterService; import lombok.Getter; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.collection.SingletonList; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.*; +import javax.annotation.Nullable; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.function.Function; import java.util.stream.Collectors; import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; @@ -110,130 +115,185 @@ public class StandardAppPrimerService implements AppPrimerService { getPrimerThread().submit(() -> _prime(account, singleApp)); } - private synchronized void _prime(Account account, BubbleApp singleApp) { + private synchronized void _prime(@NonNull final Account account, @Nullable final BubbleApp singleApp) { try { - final Map> accountDeviceIps = new HashMap<>(); final List devices = deviceDAO.findByAccount(account.getUuid()); - for (Device device : devices) { - accountDeviceIps.put(device.getUuid(), deviceService.findIpsByDevice(device.getUuid())); - } - if (accountDeviceIps.isEmpty()) return; + if (devices.isEmpty()) return; + + final Map> accountDeviceIps = + devices.stream() + .map(Device::getUuid) + .collect(Collectors.toMap(Function.identity(), deviceService::findIpsByDevice)); // flex domains can only be managed by the first admin final Account firstAdmin = accountDAO.getFirstAdmin(); account.setFirstAdmin(account.getUuid().equals(firstAdmin.getUuid())); - boolean updateFlexRouters = false; - Set flexDomains = null; - Set flexExcludeDomains = null; - - final List appsToPrime = singleApp == null - ? appDAO.findByAccount(account.getUuid()).stream() - .filter(BubbleApp::canPrime) - .collect(Collectors.toList()) - : new SingletonList<>(singleApp); - for (BubbleApp app : appsToPrime) { - log.info("_prime: priming app: "+app.getUuid()+"/"+app.getName()); - final List rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); - final List matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); - for (AppRule rule : rules) { - final RuleDriver driver = driverDAO.findByUuid(rule.getDriver()); - if (driver == null) { - log.warn("_prime: driver not found for app/rule " + app.getName() + "/" + rule.getName() + ": " + rule.getDriver()); - continue; + + if (singleApp != null) { + _primeApp(account, accountDeviceIps, devices, singleApp); + } else { + appDAO.findByAccount(account.getUuid()) + .stream() + .filter(BubbleApp::canPrime) + .forEach(app -> _primeApp(account, accountDeviceIps, devices, app)); + } + } catch (Exception e) { + die("_prime: " + shortError(e), e); + } finally { + log.info("_prime: completed"); + } + } + + private void _primeApp(@NonNull final Account account, @NonNull final Map> accountDeviceIps, + @NonNull final List devices, @NonNull final BubbleApp app) { + log.info("_primeApp: " + app.getUuid() + "/" + app.getName()); + + final List rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); + final List matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); + + boolean updateFlexRouters = false; + Set flexDomains = null; + for (AppRule rule : rules) { + final RuleDriver driver = driverDAO.findByUuid(rule.getDriver()); + if (driver == null) { + log.warn("_primeApp: driver not found for app/rule " + + app.getName() + "/" + rule.getName() + ": " + rule.getDriver()); + continue; + } + + // handle AppData callback registration with a basic driver + final AppRuleDriver cbDriver = driver.getDriver(); + if (cbDriver instanceof HasAppDataCallback) { + log.debug("_primeApp: AppRuleDriver (" + cbDriver.getClass().getSimpleName() + + ") implements HasAppDataCallback, registering: " + app.getUuid() + "/" + app.getName()); + final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver; + dataCallback.prime(account, app, configuration); + dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration)); + } + + for (final Device device : devices) { + final Set rejectDomains = new HashSet<>(); + final Set blockDomains = new HashSet<>(); + final Set whiteListDomains = new HashSet<>(); + final Set filterDomains = new HashSet<>(); + final Set flexExcludeDomains = new HashSet<>(); + final Set requestHeaderModifiers = new HashSet<>(); + + boolean areAllSetsEmpty = true; + for (AppMatcher matcher : matchers) { + final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); + + final Set rejects = appRuleDriver.getPrimedRejectDomains(); + if (empty(rejects)) { + log.debug("_primeApp: no rejectDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + rejectDomains.addAll(rejects); + areAllSetsEmpty = empty(rejects); } - // handle AppData callback registration with a basic driver - final AppRuleDriver cbDriver = driver.getDriver(); - if (cbDriver instanceof HasAppDataCallback) { - log.debug("_prime: AppRuleDriver ("+cbDriver.getClass().getSimpleName()+") implements HasAppDataCallback, registering: "+app.getUuid()+"/"+app.getName()); - final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver; - dataCallback.prime(account, app, configuration); - dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration)); + + final Set blocks = appRuleDriver.getPrimedBlockDomains(); + if (empty(blocks)) { + log.debug("_primeApp: no blockDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + blockDomains.addAll(blocks); + areAllSetsEmpty = areAllSetsEmpty && empty(blocks); + } + + final Set whiteList = appRuleDriver.getPrimedWhiteListDomains(); + if (empty(whiteList)) { + log.debug("_primeApp: no whiteListDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + whiteListDomains.addAll(whiteList); + areAllSetsEmpty = areAllSetsEmpty && empty(whiteList); } - for (Device device : devices) { - final Set rejectDomains = new HashSet<>(); - final Set blockDomains = new HashSet<>(); - final Set whiteListDomains = new HashSet<>(); - final Set filterDomains = new HashSet<>(); - for (AppMatcher matcher : matchers) { - final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); - final Set rejects = appRuleDriver.getPrimedRejectDomains(); - if (empty(rejects)) { - log.debug("_prime: no rejectDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - rejectDomains.addAll(rejects); - } - final Set blocks = appRuleDriver.getPrimedBlockDomains(); - if (empty(blocks)) { - log.debug("_prime: no blockDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - blockDomains.addAll(blocks); - } - final Set whiteList = appRuleDriver.getPrimedWhiteListDomains(); - if (empty(whiteList)) { - log.debug("_prime: no whiteListDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - whiteListDomains.addAll(whiteList); - } - final Set filters = appRuleDriver.getPrimedFilterDomains(); - if (empty(filters)) { - log.debug("_prime: no filterDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - filterDomains.addAll(filters); - } - if (account.isFirstAdmin() && flexDomains == null) { - final Set flexes = appRuleDriver.getPrimedFlexDomains(); - if (empty(flexes)) { - log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - flexDomains = new HashSet<>(flexes); - } - final Set flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains(); - if (empty(flexExcludes)) { - log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - flexExcludeDomains = new HashSet<>(flexExcludes); - } - } + + final Set filters = appRuleDriver.getPrimedFilterDomains(); + if (empty(filters)) { + log.debug("_primeApp: no filterDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + filterDomains.addAll(filters); + areAllSetsEmpty = areAllSetsEmpty && empty(filters); + } + + final Set modifiers = appRuleDriver.getPrimedResponseHeaderModifiers(); + if (empty(modifiers)) { + log.debug("_primeApp: no responseHeaderModifiers for device/app/rule/matcher: " + + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + + matcher.getName()); + } else { + requestHeaderModifiers.addAll(modifiers); + areAllSetsEmpty = areAllSetsEmpty && empty(modifiers); + } + + if (account.isFirstAdmin() && flexDomains == null) { + final Set flexes = appRuleDriver.getPrimedFlexDomains(); + if (empty(flexes)) { + log.debug("_primeApp: no flexDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + flexDomains = new HashSet<>(flexes); + areAllSetsEmpty = areAllSetsEmpty && empty(flexes); } - if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) { - for (String ip : accountDeviceIps.get(device.getUuid())) { - if (!empty(rejectDomains)) { - rejectDomains.removeAll(whiteListDomains); - AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new)); - } - if (!empty(blockDomains)) { - blockDomains.removeAll(whiteListDomains); - AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new)); - } - if (!empty(whiteListDomains)) { - AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), whiteListDomains.toArray(String[]::new)); - } - if (!empty(filterDomains)) { - AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new)); - } - if (account.isFirstAdmin() && (!empty(flexDomains) || !empty(flexExcludeDomains))) { - updateFlexRouters = true; - if (!empty(flexDomains)) { - if (flexExcludeDomains != null) flexDomains.removeAll(flexExcludeDomains); - AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new)); - } - if (!empty(flexExcludeDomains)) { - AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new)); - } - } - } + + final Set flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains(); + if (empty(flexExcludes)) { + log.debug("_primeApp: no flexExcludeDomains for device/app/rule/matcher: " + + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + + matcher.getName()); + } else { + flexExcludeDomains.addAll(flexExcludes); + areAllSetsEmpty = areAllSetsEmpty && empty(flexExcludes); } } } - if (updateFlexRouters && !empty(flexDomains)) { - flexRouterService.updateFlexRoutes(flexDomains); + + if (areAllSetsEmpty) continue; + + for (final String ip : accountDeviceIps.get(device.getUuid())) { + if (!empty(rejectDomains)) { + rejectDomains.removeAll(whiteListDomains); + AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), + rejectDomains.toArray(String[]::new)); + } + if (!empty(blockDomains)) { + blockDomains.removeAll(whiteListDomains); + AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), + blockDomains.toArray(String[]::new)); + } + if (!empty(whiteListDomains)) { + AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), + whiteListDomains.toArray(String[]::new)); + } + if (!empty(filterDomains)) { + AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), + filterDomains.toArray(String[]::new)); + } + if (!empty(requestHeaderModifiers)) { + AppRuleDriver.defineRedisResponseHeaderModifiersSet( + redis, ip, app.getName() + ":" + app.getUuid(), + requestHeaderModifiers.toArray(String[]::new)); + } + if (account.isFirstAdmin()) { + if (!empty(flexDomains)) { + if (!empty(flexExcludeDomains)) flexDomains.removeAll(flexExcludeDomains); + AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), + flexDomains.toArray(String[]::new)); + updateFlexRouters = true; + } + if (!empty(flexExcludeDomains)) { + AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), + flexExcludeDomains.toArray(String[]::new)); + } + } } } - } catch (Exception e) { - die("_prime: "+shortError(e), e); - } finally { - log.info("_primeApps: completed"); } + + if (updateFlexRouters) flexRouterService.updateFlexRoutes(flexDomains); } } diff --git a/bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json new file mode 100644 index 00000000..5001ce5d --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json @@ -0,0 +1,81 @@ +[{ + "name": "RequestProtector", + "description": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response", + "url": "https://getbubblenow.com/apps/request", + "template": true, + "enabled": true, + "priority": 1000, + "canPrime": true, + "dataConfig": { + "dataDriver": "bubble.app.request.RequestProtectorAppDataDriver", + "presentation": "app", + "configDriver": "bubble.app.request.RequestProtectorAppConfigDriver", + "configFields": [ + {"name": "regex", "truncate": false}, + {"name": "replacement", "truncate": false} + ], + "configViews": [{ + "name": "manageHeaderReplacements", + "scope": "app", + "root": "true", + "fields": [ "regex", "replacement" ], + "actions": [ + {"name": "removeHeaderReplacement", "index": 10}, + { + "name": "addHeaderReplacement", "scope": "app", "index": 10, + "params": [ "regex", "replacement" ], + "button": "addHeaderReplacement" + } + ] + }] + }, + "children": { + "AppSite": [{ + "name": "All_Sites", + "url": "*", + "description": "All websites", + "template": true + }], + "AppRule": [{ + "name": "request", + "template": true, + "driver": "RequestProtectorRuleDriver", + "priority": -1000, + "config": { + "headerReplacements": [{ + "regex": "^(?i:Set-Cookie):(.*;)?\\s*(?i:Domain)=(((([\\*\\.].*)|(.*\\.))\\s*(;.*)?)|((?!([^;,]*\\.)?{{fqdn}}\\s*(;|$)).*))$", + "replacement": "" + }] + } + }], + "AppMessage": [{ + "locale": "en_US", + "messages": [ + { "name": "name", "value": "RequestProtector" }, + { "name": "icon", "value": "classpath:models/apps/request/request-icon.svg" }, + { "name": "summary", "value": "Request Protector" }, + { + "name": "description", + "value": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response" + }, + + { "name": "config.view.manageHeaderReplacements", "value": "Manage Header Replacements" }, + { "name": "config.field.regex", "value": "RegEx" }, + { + "name": "config.field.regex.description", + "value": "Regular expression compared with full header's line string value" + }, + { "name": "config.field.replacement", "value": "Replacement" }, + { + "name": "config.field.replacement.description", + "value": "May use reference from regex as in python's re.Pattern.sub method. If set to empty string, found header will be fully removed from response" + }, + { "name": "config.button.addHeaderReplacement", "value": "Add" }, + { "name": "config.action.addHeaderReplacement", "value": "Add New Header Replacement" }, + { "name": "config.action.removeHeaderReplacement", "value": "Remove" }, + + { "name": "err.requestProtector.headerRegexRequired", "value": "RegEx field is required" } + ] + }] + } +}] diff --git a/bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json new file mode 100644 index 00000000..787f10f6 --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json @@ -0,0 +1,15 @@ +[{ + "name": "RequestProtector", + "children": { + "AppMatcher": [{ + "name": "RequestProtectorMatcher", + "template": true, + "connCheck": true, + "site": "All_Sites", + "fqdn": "*", + "urlRegex": ".*", + "rule": "request", + "priority": -1000000 + }] + } +}] diff --git a/bubble-server/src/main/resources/models/apps/request/request-icon.svg b/bubble-server/src/main/resources/models/apps/request/request-icon.svg new file mode 100644 index 00000000..5c29e3a5 --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/request/request-icon.svg @@ -0,0 +1,89 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + diff --git a/bubble-server/src/main/resources/models/defaults/bubblePlan.json b/bubble-server/src/main/resources/models/defaults/bubblePlan.json index 188c5fc9..fb5e32e9 100644 --- a/bubble-server/src/main/resources/models/defaults/bubblePlan.json +++ b/bubble-server/src/main/resources/models/defaults/bubblePlan.json @@ -14,7 +14,8 @@ "children": { "BubblePlanApp": [ {"app": "BubbleBlock"}, - {"app": "TlsPassthru"} + {"app": "TlsPassthru"}, + {"app": "RequestProtector"} ] } }, @@ -36,7 +37,8 @@ {"app": "TrafficAnalytics"}, {"app": "BubbleBlock"}, {"app": "UserBlocker"}, - {"app": "TlsPassthru"} + {"app": "TlsPassthru"}, + {"app": "RequestProtector"} ] } }, @@ -58,7 +60,8 @@ {"app": "TrafficAnalytics"}, {"app": "BubbleBlock"}, {"app": "UserBlocker"}, - {"app": "TlsPassthru"} + {"app": "TlsPassthru"}, + {"app": "RequestProtector"} ] } } diff --git a/bubble-server/src/main/resources/models/defaults/ruleDriver.json b/bubble-server/src/main/resources/models/defaults/ruleDriver.json index 113019f0..d9a1346b 100644 --- a/bubble-server/src/main/resources/models/defaults/ruleDriver.json +++ b/bubble-server/src/main/resources/models/defaults/ruleDriver.json @@ -28,5 +28,11 @@ "driverClass": "bubble.rule.bblock.BubbleBlockRuleDriver", "template": true, "userConfig": { "fields": [] } + }, + { + "name": "RequestProtectorRuleDriver", + "driverClass": "bubble.rule.request.RequestProtectorRuleDriver", + "template": true, + "userConfig": { "fields": [] } } -] \ No newline at end of file +] diff --git a/bubble-server/src/main/resources/models/manifest-app-request.json b/bubble-server/src/main/resources/models/manifest-app-request.json new file mode 100644 index 00000000..d7d41a06 --- /dev/null +++ b/bubble-server/src/main/resources/models/manifest-app-request.json @@ -0,0 +1,4 @@ +[ + "apps/request/bubbleApp_request", + "apps/request/bubbleApp_request_matchers" +] diff --git a/bubble-server/src/main/resources/models/manifest-defaults.json b/bubble-server/src/main/resources/models/manifest-defaults.json index c5d15598..26c9d2bf 100644 --- a/bubble-server/src/main/resources/models/manifest-defaults.json +++ b/bubble-server/src/main/resources/models/manifest-defaults.json @@ -5,5 +5,6 @@ "manifest-app-user-block", "manifest-app-bubble-block", "manifest-app-passthru", + "manifest-app-request", "defaults/bubblePlan" -] \ No newline at end of file +] diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index a42661df..df82e87b 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -25,6 +25,7 @@ from bubble_config import bubble_port, debug_capture_fqdn, \ from mitmproxy import http from mitmproxy.net.http import headers as nheaders from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody +from mitmproxy.utils import strutils bubble_log = logging.getLogger(__name__) @@ -449,6 +450,131 @@ def original_flex_ip(client_addr, fqdns): return None +def update_host_and_port(flow): + if flow.request: + if flow.client_conn.tls_established: + flow.request.scheme = "https" + sni = flow.client_conn.connection.get_servername() + port = 443 + else: + flow.request.scheme = "http" + sni = None + port = 80 + + host_header = flow.request.host_header + if host_header: + m = parse_host_header.match(host_header) + if m: + host_header = m.group("host").strip("[]") + if m.group("port"): + port = int(m.group("port")) + + host = None + if sni or host_header: + host = str(sni or host_header) + if host.startswith("b'"): + host = host[2:-1] + + flow.request.host_header = host_header + if host: + flow.request.host = host + else: + flow.request.host = host_header + flow.request.port = port + + return flow + + +def _replace_in_headers(headers: nheaders.Headers, modifiers_dict: dict) -> int: + """ + Taken from original mitmproxy's Header class implementation with some changes. + + Replaces a regular expression pattern with repl in each "name: value" + header line. + + Returns: + The number of replacements made. + """ + repl_count = 0 + fields = [] + + for name, value in headers.fields: + + line = name + b": " + value + inner_repl_count = 0 + for pattern, replacement in modifiers_dict.items(): + line, n = pattern.subn(replacement, line) + inner_repl_count += n + if len(line) == 0: + # No need to go though other patterns for this line + break + + if len(line) == 0: + # Skip (remove) this header line in this case + break + + if inner_repl_count > 0: + # only in case when there were some replacements: + try: + name, value = line.split(b": ", 1) + except ValueError: + # We get a ValueError if the replacement removed the ": " + # There's not much we can do about this, so we just keep the header as-is. + pass + else: + repl_count += inner_repl_count + + fields.append((name, value)) + + headers.fields = tuple(fields) + return repl_count + + +def response_header_modify(flow) -> int: + if flow.response is None: + return None + + flow = update_host_and_port(flow) + ctx = {'fqdn': flow.request.host} + return _header_modify(flow.client_conn.address[0], ctx, flow.response.headers) + + +def _header_modify(client_addr: str, ctx: dict, headers: nheaders.Headers) -> int: + modifiers_set = 'responseHeaderModifierLists~' + client_addr + '~UNION' + modifiers = REDIS.smembers(modifiers_set) + + repl_count = 0 + if modifiers: + modifiers_dict = {} + for modifier in modifiers: + regex, replacement = _extract_modifier_config(modifier, ctx) + modifiers_dict[regex] = replacement + repl_count += _replace_in_headers(headers, modifiers_dict) + + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_header_modify: replacing headers - replacements count: ' + repl_count) + + return repl_count + + +def _extract_modifier_config(modifier: bytes, ctx: dict) -> tuple: + modifier_obj = json.loads(modifier) + + regex = _replace_modifier_values(modifier_obj['regex'], ctx) + replacement = _replace_modifier_values(modifier_obj['replacement'], ctx) + + regex = re.compile(strutils.escaped_str_to_bytes(regex)) + replacement = strutils.escaped_str_to_bytes(replacement) + + return regex, replacement + + +def _replace_modifier_values(s: str, ctx: dict) -> str: + # no loop over ctx currently to speed up as there's just 1 variable inside + s = s.replace('{{fqdn}}', re.escape(ctx['fqdn'])) + return s + + def health_check_response(flow): # if bubble_log.isEnabledFor(DEBUG): # bubble_log.debug('health_check_response: special bubble health check request, responding with OK') diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 44e5fc8a..49a28907 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -16,7 +16,7 @@ from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATIO is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \ HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \ - HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set + HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, response_header_modify from bubble_flex import process_flex import logging @@ -311,6 +311,7 @@ def responseheaders(flow): else: flex_flow = None bubble_filter_response(flow, flex_flow) + response_header_modify(flow) def bubble_filter_response(flow, flex_flow): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 658f2745..d8447401 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -33,7 +33,7 @@ from bubble_api import bubble_matchers, bubble_activity_log, \ CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ is_bubble_special_path, is_bubble_health_check, health_check_response, tarpit_response,\ - is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain + is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, update_host_and_port from bubble_config import bubble_host, bubble_host_alias from bubble_flex import new_flex_flow @@ -114,34 +114,20 @@ class Rerouter: def bubble_handle_request(self, flow): client_addr = flow.client_conn.address[0] server_addr = flow.server_conn.address[0] - is_http = False + flow = update_host_and_port(flow) + if flow.client_conn.tls_established: - flow.request.scheme = "https" sni = flow.client_conn.connection.get_servername() - port = 443 + is_http = False else: - flow.request.scheme = "http" sni = None - port = 80 is_http = True - # check if https and sni is missing but we have a host header, fill in the sni - - host_header = flow.request.host_header - if host_header: - m = parse_host_header.match(host_header) - if m: - host_header = m.group("host").strip("[]") - if m.group("port"): - port = int(m.group("port")) - # Determine if this request should be filtered - host = None + host_header = flow.request.host_header + host = flow.request.host path = flow.request.path if sni or host_header: - host = str(sni or host_header) - if host.startswith("b'"): - host = host[2:-1] log_url = flow.request.scheme + '://' + host + path # If https, we have already checked that the client/server are legal in bubble_conn_check.py @@ -240,12 +226,6 @@ class Rerouter: bubble_log.warning('bubble_handle_request: no sni/host found, not applying rules to path: ' + path) bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr]) - flow.request.host_header = host_header - if host: - flow.request.host = host - else: - flow.request.host = host_header - flow.request.port = port return host def requestheaders(self, flow):