From 237e90d1263efcb0a9037e2520cffe2980c1addd Mon Sep 17 00:00:00 2001 From: Kristijan Mitrovic Date: Fri, 25 Sep 2020 12:10:21 +0000 Subject: [PATCH] Add request protector app with cross-domain cookies filtering (#58) Add RequestProtector app to cheapest plan Update comment with typo Merge branch 'master' into kris/request_protector_app Merge branch 'master' into kris/request_protector_app Merge branch 'kris/request_protector_app' of git.bubblev.org:bubblev/bubble into kris/request_protector_app Extract method for updating requests host and port Fix header replacement Replace all cross-domain cookies with empty Add fqdn variable support in header replacements Merge branch 'master' into kris/request_protector_app Merge branch 'master' into kris/request_protector_app Merge branch 'master' into kris/request_protector_app Merge branch 'master' into kris/request_protector_app Add add header replacement button label Try to add initial header replacement for cross-domain cookies Merge branch 'master' into kris/request_protector_app Merge branch 'master' into kris/request_protector_app use special header replacement to skip emptied headers Set RequestProtector replacement optional again Fix replacement reference in RequestProtector app Update flex domains with empty set if needed Add back request heades modifiers as prime app Extract and refactor _primeApp method Merge branch 'master' into kris/request_protector_app # Conflicts - WIP: # bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java Use HeaderReplacement's id field in JSONs Make RequestProtector's replacement field required Add new app to some plans Set new app to have `app` presentation Add RuleDriver and AppMatcher for the new app Add request protector app Remove not used filter Merge branch 'master' into kris/request_protector_app Add full support for response header modification Merge branch 'master' into kris/request_protector_app # Conflicts: # utils/cobbzilla-utils Add RequestProtector app Co-authored-by: jonathan Co-authored-by: Kristijan Mitrovic Reviewed-on: https://git.bubblev.org/bubblev/bubble/pulls/58 --- .../RequestProtectorAppConfigDriver.java | 115 +++++++ .../RequestProtectorAppDataDriver.java | 9 + .../main/java/bubble/rule/AppRuleDriver.java | 16 +- .../rule/request/HeaderReplacement.java | 30 ++ .../rule/request/RequestProtectorConfig.java | 34 +++ .../request/RequestProtectorRuleDriver.java | 38 +++ .../stream/StandardAppPrimerService.java | 282 +++++++++++------- .../apps/request/bubbleApp_request.json | 81 +++++ .../request/bubbleApp_request_matchers.json | 15 + .../models/apps/request/request-icon.svg | 89 ++++++ .../resources/models/defaults/bubblePlan.json | 9 +- .../resources/models/defaults/ruleDriver.json | 8 +- .../models/manifest-app-request.json | 4 + .../resources/models/manifest-defaults.json | 3 +- .../roles/mitmproxy/files/bubble_api.py | 126 ++++++++ .../roles/mitmproxy/files/bubble_modify.py | 3 +- .../roles/mitmproxy/files/bubble_request.py | 32 +- 17 files changed, 749 insertions(+), 145 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java create mode 100644 bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java create mode 100644 bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java create mode 100644 bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java create mode 100644 bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java create mode 100644 bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json create mode 100644 bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json create mode 100644 bubble-server/src/main/resources/models/apps/request/request-icon.svg create mode 100644 bubble-server/src/main/resources/models/manifest-app-request.json diff --git a/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java new file mode 100644 index 00000000..1da8f3c4 --- /dev/null +++ b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java @@ -0,0 +1,115 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.app.request; + +import bubble.model.account.Account; +import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; +import bubble.model.app.config.AppConfigDriverBase; +import bubble.rule.request.HeaderReplacement; +import bubble.rule.request.RequestProtectorConfig; +import bubble.rule.request.RequestProtectorRuleDriver; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Set; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; +import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; + +@Slf4j +public class RequestProtectorAppConfigDriver extends AppConfigDriverBase { + + public static final String VIEW_manageHeaderReplacements = "manageHeaderReplacements"; + + @Override public Object getView(Account account, BubbleApp app, String view, Map params) { + switch (view) { + case VIEW_manageHeaderReplacements: + return loadManageCookiesReplacements(account, app); + } + throw notFoundEx(view); + } + + private Set loadManageCookiesReplacements(Account account, BubbleApp app) { + final RequestProtectorConfig config = getConfig(account, app); + return config.getHeaderReplacements(); + } + + private RequestProtectorConfig getConfig(Account account, BubbleApp app) { + return getConfig(account, app, RequestProtectorRuleDriver.class, RequestProtectorConfig.class); + } + + public static final String ACTION_addHeaderReplacement = "addHeaderReplacement"; + public static final String ACTION_removeHeaderReplacement = "removeHeaderReplacement"; + + public static final String PARAM_REGEX = "regex"; + public static final String PARAM_REPLACEMENT = "replacement"; + + @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action, + Map params, JsonNode data) { + switch (action) { + case ACTION_addHeaderReplacement: + return addHeaderReplacement(account, app, data); + } + if (log.isWarnEnabled()) log.warn("takeAppAction: action not found: "+action); + throw notFoundEx(action); + } + + private Set addHeaderReplacement(Account account, BubbleApp app, JsonNode data) { + final JsonNode regexNode = data.get(PARAM_REGEX); + if (regexNode == null || regexNode.textValue() == null) { + throw invalidEx("err.requestProtector.headerRegexRequired"); + } + final String regex = regexNode.textValue().trim(); + if (empty(regex)) throw invalidEx("err.requestProtector.headerRegexRequired"); + + final JsonNode replacementNode = data.get(PARAM_REPLACEMENT); + final String replacement = (replacementNode == null || replacementNode.textValue() == null) + ? "" : replacementNode.textValue().trim(); + + final RequestProtectorConfig config = getConfig(account, app).addHeaderReplacement(regex, replacement); + + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver + if (log.isDebugEnabled()) { + log.debug("addHeaderReplacement: updating rule: " + rule.getName() + ", adding regex: " + regex); + } + ruleDAO.update(rule.setConfigJson(json(config))); + + return config.getHeaderReplacements(); + } + + @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id, + Map params, JsonNode data) { + switch (action) { + case ACTION_removeHeaderReplacement: + return removeHeaderReplacement(account, app, id); + } + if (log.isWarnEnabled()) log.warn("takeItemAction: action not found: "+action); + throw notFoundEx(action); + } + + private Set removeHeaderReplacement(Account account, BubbleApp app, String regex) { + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver + final RequestProtectorConfig config = getConfig(account, app); + if (log.isDebugEnabled()) { + log.debug("removeHeaderReplacement: removing regex: " + regex + " from config.cookiesReplacements: " + + config.getHeaderReplacements().toString()); + } + + final RequestProtectorConfig updated = config.removeHeaderReplacement(regex); + if (log.isDebugEnabled()) { + log.debug("removeHeaderReplacement: updated.cookiesReplacements: " + + updated.getHeaderReplacements().toString()); + } + ruleDAO.update(rule.setConfigJson(json(updated))); + + return updated.getHeaderReplacements(); + } +} diff --git a/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java new file mode 100644 index 00000000..d072acda --- /dev/null +++ b/bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java @@ -0,0 +1,9 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.app.request; + +import bubble.model.app.config.AppDataDriverBase; + +public class RequestProtectorAppDataDriver extends AppDataDriverBase {} diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 7b951950..f52a6828 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -47,6 +47,7 @@ public interface AppRuleDriver { String REDIS_FILTER_LISTS = "filterLists"; String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy and dnscrypt-proxy for flex routing String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists"; // used in mitmproxy and dnscrypt-proxy for flex routing + String REDIS_RESPONSE_HEADER_MODIFIER_LISTS = "responseHeaderModifierLists"; // used in mitmproxy String REDIS_LIST_SUFFIX = "~UNION"; default Set getPrimedRejectDomains () { return null; } @@ -55,6 +56,7 @@ public interface AppRuleDriver { default Set getPrimedFilterDomains () { return null; } default Set getPrimedFlexDomains () { return null; } default Set getPrimedFlexExcludeDomains () { return null; } + default Set getPrimedResponseHeaderModifiers () { return null; } static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) { defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains); @@ -80,12 +82,22 @@ public interface AppRuleDriver { defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains); } - static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { + static void defineRedisResponseHeaderModifiersSet(RedisService redis, String ip, String list, + String[] modifiers) { + defineRedisSet(redis, ip, REDIS_RESPONSE_HEADER_MODIFIER_LISTS, list, modifiers); + } + + /** + * `settings` parameter may be list of domains or any other list of strings - i.e. list of JSONs with specific setup + * for the prime option of the driver. + */ + static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, + String[] settings) { final String listOfListsForIp = listOfListsName + "~" + ip; final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX; final String ipList = listOfListsForIp + "~" + listName; final String tempList = ipList + "~"+now()+randomAlphanumeric(5); - redis.sadd_plaintext(tempList, domains); + redis.sadd_plaintext(tempList, settings); redis.rename(tempList, ipList); redis.sadd_plaintext(listOfListsForIp, ipList); final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp)); diff --git a/bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java b/bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java new file mode 100644 index 00000000..ba7befb4 --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java @@ -0,0 +1,30 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule.request; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import lombok.Setter; +import lombok.experimental.Accessors; + +@NoArgsConstructor @Accessors(chain=true) +public class HeaderReplacement implements Comparable { + + public String getId() { return regex; } + public void setId(String id) {} // noop + + @Getter @Setter private String regex; + @Getter @Setter private String replacement; + + public HeaderReplacement(@NonNull final String regex, @NonNull final String replacement) { + this.regex = regex; + this.replacement = replacement; + } + + @Override public int compareTo(@NonNull final HeaderReplacement o) { + return getRegex().compareTo(o.getRegex().toLowerCase()); + } +} diff --git a/bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java new file mode 100644 index 00000000..ae2555ab --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java @@ -0,0 +1,34 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule.request; + +import lombok.Getter; +import lombok.NonNull; +import lombok.Setter; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; + +import java.util.Set; +import java.util.TreeSet; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@Slf4j @Accessors(chain=true) +public class RequestProtectorConfig { + + @Getter @Setter private Set headerReplacements = new TreeSet<>(); + public boolean hasHeaderReplacements() { return !empty(headerReplacements); } + + @NonNull public RequestProtectorConfig addHeaderReplacement(@NonNull final String regex, + @NonNull final String replacement) { + headerReplacements.add(new HeaderReplacement(regex, replacement)); + return this; + } + + @NonNull public RequestProtectorConfig removeHeaderReplacement(@NonNull final String regex) { + if (hasHeaderReplacements()) headerReplacements.removeIf(r -> r.getRegex().equals(regex)); + return this; + } +} diff --git a/bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java new file mode 100644 index 00000000..860585ee --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule.request; + +import bubble.model.account.Account; +import bubble.model.app.AppMatcher; +import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; +import bubble.model.device.Device; +import bubble.rule.AbstractAppRuleDriver; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.json.JsonUtil; + +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +public class RequestProtectorRuleDriver extends AbstractAppRuleDriver { + + @Override public Class getConfigClass() { return (Class) RequestProtectorConfig.class; } + + @Override public Set getPrimedResponseHeaderModifiers() { + final RequestProtectorConfig config = getRuleConfig(); + return config.getHeaderReplacements().stream().map(JsonUtil::json).collect(Collectors.toSet()); + } + + @Override public void init(JsonNode config, JsonNode userConfig, BubbleApp app, AppRule rule, AppMatcher matcher, + Account account, Device device) { + super.init(config, userConfig, app, rule, matcher, account, device); + + // refresh list + final RequestProtectorConfig ruleConfig = getRuleConfig(); + ruleConfig.getHeaderReplacements(); + } +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index bd94ffa3..23d813d7 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -16,14 +16,19 @@ import bubble.server.BubbleConfiguration; import bubble.service.device.DeviceService; import bubble.service.device.StandardFlexRouterService; import lombok.Getter; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.collection.SingletonList; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.*; +import javax.annotation.Nullable; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.function.Function; import java.util.stream.Collectors; import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; @@ -110,130 +115,185 @@ public class StandardAppPrimerService implements AppPrimerService { getPrimerThread().submit(() -> _prime(account, singleApp)); } - private synchronized void _prime(Account account, BubbleApp singleApp) { + private synchronized void _prime(@NonNull final Account account, @Nullable final BubbleApp singleApp) { try { - final Map> accountDeviceIps = new HashMap<>(); final List devices = deviceDAO.findByAccount(account.getUuid()); - for (Device device : devices) { - accountDeviceIps.put(device.getUuid(), deviceService.findIpsByDevice(device.getUuid())); - } - if (accountDeviceIps.isEmpty()) return; + if (devices.isEmpty()) return; + + final Map> accountDeviceIps = + devices.stream() + .map(Device::getUuid) + .collect(Collectors.toMap(Function.identity(), deviceService::findIpsByDevice)); // flex domains can only be managed by the first admin final Account firstAdmin = accountDAO.getFirstAdmin(); account.setFirstAdmin(account.getUuid().equals(firstAdmin.getUuid())); - boolean updateFlexRouters = false; - Set flexDomains = null; - Set flexExcludeDomains = null; - - final List appsToPrime = singleApp == null - ? appDAO.findByAccount(account.getUuid()).stream() - .filter(BubbleApp::canPrime) - .collect(Collectors.toList()) - : new SingletonList<>(singleApp); - for (BubbleApp app : appsToPrime) { - log.info("_prime: priming app: "+app.getUuid()+"/"+app.getName()); - final List rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); - final List matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); - for (AppRule rule : rules) { - final RuleDriver driver = driverDAO.findByUuid(rule.getDriver()); - if (driver == null) { - log.warn("_prime: driver not found for app/rule " + app.getName() + "/" + rule.getName() + ": " + rule.getDriver()); - continue; + + if (singleApp != null) { + _primeApp(account, accountDeviceIps, devices, singleApp); + } else { + appDAO.findByAccount(account.getUuid()) + .stream() + .filter(BubbleApp::canPrime) + .forEach(app -> _primeApp(account, accountDeviceIps, devices, app)); + } + } catch (Exception e) { + die("_prime: " + shortError(e), e); + } finally { + log.info("_prime: completed"); + } + } + + private void _primeApp(@NonNull final Account account, @NonNull final Map> accountDeviceIps, + @NonNull final List devices, @NonNull final BubbleApp app) { + log.info("_primeApp: " + app.getUuid() + "/" + app.getName()); + + final List rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); + final List matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid()); + + boolean updateFlexRouters = false; + Set flexDomains = null; + for (AppRule rule : rules) { + final RuleDriver driver = driverDAO.findByUuid(rule.getDriver()); + if (driver == null) { + log.warn("_primeApp: driver not found for app/rule " + + app.getName() + "/" + rule.getName() + ": " + rule.getDriver()); + continue; + } + + // handle AppData callback registration with a basic driver + final AppRuleDriver cbDriver = driver.getDriver(); + if (cbDriver instanceof HasAppDataCallback) { + log.debug("_primeApp: AppRuleDriver (" + cbDriver.getClass().getSimpleName() + + ") implements HasAppDataCallback, registering: " + app.getUuid() + "/" + app.getName()); + final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver; + dataCallback.prime(account, app, configuration); + dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration)); + } + + for (final Device device : devices) { + final Set rejectDomains = new HashSet<>(); + final Set blockDomains = new HashSet<>(); + final Set whiteListDomains = new HashSet<>(); + final Set filterDomains = new HashSet<>(); + final Set flexExcludeDomains = new HashSet<>(); + final Set requestHeaderModifiers = new HashSet<>(); + + boolean areAllSetsEmpty = true; + for (AppMatcher matcher : matchers) { + final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); + + final Set rejects = appRuleDriver.getPrimedRejectDomains(); + if (empty(rejects)) { + log.debug("_primeApp: no rejectDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + rejectDomains.addAll(rejects); + areAllSetsEmpty = empty(rejects); } - // handle AppData callback registration with a basic driver - final AppRuleDriver cbDriver = driver.getDriver(); - if (cbDriver instanceof HasAppDataCallback) { - log.debug("_prime: AppRuleDriver ("+cbDriver.getClass().getSimpleName()+") implements HasAppDataCallback, registering: "+app.getUuid()+"/"+app.getName()); - final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver; - dataCallback.prime(account, app, configuration); - dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration)); + + final Set blocks = appRuleDriver.getPrimedBlockDomains(); + if (empty(blocks)) { + log.debug("_primeApp: no blockDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + blockDomains.addAll(blocks); + areAllSetsEmpty = areAllSetsEmpty && empty(blocks); + } + + final Set whiteList = appRuleDriver.getPrimedWhiteListDomains(); + if (empty(whiteList)) { + log.debug("_primeApp: no whiteListDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + whiteListDomains.addAll(whiteList); + areAllSetsEmpty = areAllSetsEmpty && empty(whiteList); } - for (Device device : devices) { - final Set rejectDomains = new HashSet<>(); - final Set blockDomains = new HashSet<>(); - final Set whiteListDomains = new HashSet<>(); - final Set filterDomains = new HashSet<>(); - for (AppMatcher matcher : matchers) { - final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); - final Set rejects = appRuleDriver.getPrimedRejectDomains(); - if (empty(rejects)) { - log.debug("_prime: no rejectDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - rejectDomains.addAll(rejects); - } - final Set blocks = appRuleDriver.getPrimedBlockDomains(); - if (empty(blocks)) { - log.debug("_prime: no blockDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - blockDomains.addAll(blocks); - } - final Set whiteList = appRuleDriver.getPrimedWhiteListDomains(); - if (empty(whiteList)) { - log.debug("_prime: no whiteListDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - whiteListDomains.addAll(whiteList); - } - final Set filters = appRuleDriver.getPrimedFilterDomains(); - if (empty(filters)) { - log.debug("_prime: no filterDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - filterDomains.addAll(filters); - } - if (account.isFirstAdmin() && flexDomains == null) { - final Set flexes = appRuleDriver.getPrimedFlexDomains(); - if (empty(flexes)) { - log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - flexDomains = new HashSet<>(flexes); - } - final Set flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains(); - if (empty(flexExcludes)) { - log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); - } else { - flexExcludeDomains = new HashSet<>(flexExcludes); - } - } + + final Set filters = appRuleDriver.getPrimedFilterDomains(); + if (empty(filters)) { + log.debug("_primeApp: no filterDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + filterDomains.addAll(filters); + areAllSetsEmpty = areAllSetsEmpty && empty(filters); + } + + final Set modifiers = appRuleDriver.getPrimedResponseHeaderModifiers(); + if (empty(modifiers)) { + log.debug("_primeApp: no responseHeaderModifiers for device/app/rule/matcher: " + + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + + matcher.getName()); + } else { + requestHeaderModifiers.addAll(modifiers); + areAllSetsEmpty = areAllSetsEmpty && empty(modifiers); + } + + if (account.isFirstAdmin() && flexDomains == null) { + final Set flexes = appRuleDriver.getPrimedFlexDomains(); + if (empty(flexes)) { + log.debug("_primeApp: no flexDomains for device/app/rule/matcher: " + device.getName() + + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + flexDomains = new HashSet<>(flexes); + areAllSetsEmpty = areAllSetsEmpty && empty(flexes); } - if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) { - for (String ip : accountDeviceIps.get(device.getUuid())) { - if (!empty(rejectDomains)) { - rejectDomains.removeAll(whiteListDomains); - AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new)); - } - if (!empty(blockDomains)) { - blockDomains.removeAll(whiteListDomains); - AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new)); - } - if (!empty(whiteListDomains)) { - AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), whiteListDomains.toArray(String[]::new)); - } - if (!empty(filterDomains)) { - AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new)); - } - if (account.isFirstAdmin() && (!empty(flexDomains) || !empty(flexExcludeDomains))) { - updateFlexRouters = true; - if (!empty(flexDomains)) { - if (flexExcludeDomains != null) flexDomains.removeAll(flexExcludeDomains); - AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new)); - } - if (!empty(flexExcludeDomains)) { - AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new)); - } - } - } + + final Set flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains(); + if (empty(flexExcludes)) { + log.debug("_primeApp: no flexExcludeDomains for device/app/rule/matcher: " + + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + + matcher.getName()); + } else { + flexExcludeDomains.addAll(flexExcludes); + areAllSetsEmpty = areAllSetsEmpty && empty(flexExcludes); } } } - if (updateFlexRouters && !empty(flexDomains)) { - flexRouterService.updateFlexRoutes(flexDomains); + + if (areAllSetsEmpty) continue; + + for (final String ip : accountDeviceIps.get(device.getUuid())) { + if (!empty(rejectDomains)) { + rejectDomains.removeAll(whiteListDomains); + AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), + rejectDomains.toArray(String[]::new)); + } + if (!empty(blockDomains)) { + blockDomains.removeAll(whiteListDomains); + AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), + blockDomains.toArray(String[]::new)); + } + if (!empty(whiteListDomains)) { + AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), + whiteListDomains.toArray(String[]::new)); + } + if (!empty(filterDomains)) { + AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), + filterDomains.toArray(String[]::new)); + } + if (!empty(requestHeaderModifiers)) { + AppRuleDriver.defineRedisResponseHeaderModifiersSet( + redis, ip, app.getName() + ":" + app.getUuid(), + requestHeaderModifiers.toArray(String[]::new)); + } + if (account.isFirstAdmin()) { + if (!empty(flexDomains)) { + if (!empty(flexExcludeDomains)) flexDomains.removeAll(flexExcludeDomains); + AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), + flexDomains.toArray(String[]::new)); + updateFlexRouters = true; + } + if (!empty(flexExcludeDomains)) { + AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), + flexExcludeDomains.toArray(String[]::new)); + } + } } } - } catch (Exception e) { - die("_prime: "+shortError(e), e); - } finally { - log.info("_primeApps: completed"); } + + if (updateFlexRouters) flexRouterService.updateFlexRoutes(flexDomains); } } diff --git a/bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json new file mode 100644 index 00000000..5001ce5d --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json @@ -0,0 +1,81 @@ +[{ + "name": "RequestProtector", + "description": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response", + "url": "https://getbubblenow.com/apps/request", + "template": true, + "enabled": true, + "priority": 1000, + "canPrime": true, + "dataConfig": { + "dataDriver": "bubble.app.request.RequestProtectorAppDataDriver", + "presentation": "app", + "configDriver": "bubble.app.request.RequestProtectorAppConfigDriver", + "configFields": [ + {"name": "regex", "truncate": false}, + {"name": "replacement", "truncate": false} + ], + "configViews": [{ + "name": "manageHeaderReplacements", + "scope": "app", + "root": "true", + "fields": [ "regex", "replacement" ], + "actions": [ + {"name": "removeHeaderReplacement", "index": 10}, + { + "name": "addHeaderReplacement", "scope": "app", "index": 10, + "params": [ "regex", "replacement" ], + "button": "addHeaderReplacement" + } + ] + }] + }, + "children": { + "AppSite": [{ + "name": "All_Sites", + "url": "*", + "description": "All websites", + "template": true + }], + "AppRule": [{ + "name": "request", + "template": true, + "driver": "RequestProtectorRuleDriver", + "priority": -1000, + "config": { + "headerReplacements": [{ + "regex": "^(?i:Set-Cookie):(.*;)?\\s*(?i:Domain)=(((([\\*\\.].*)|(.*\\.))\\s*(;.*)?)|((?!([^;,]*\\.)?{{fqdn}}\\s*(;|$)).*))$", + "replacement": "" + }] + } + }], + "AppMessage": [{ + "locale": "en_US", + "messages": [ + { "name": "name", "value": "RequestProtector" }, + { "name": "icon", "value": "classpath:models/apps/request/request-icon.svg" }, + { "name": "summary", "value": "Request Protector" }, + { + "name": "description", + "value": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response" + }, + + { "name": "config.view.manageHeaderReplacements", "value": "Manage Header Replacements" }, + { "name": "config.field.regex", "value": "RegEx" }, + { + "name": "config.field.regex.description", + "value": "Regular expression compared with full header's line string value" + }, + { "name": "config.field.replacement", "value": "Replacement" }, + { + "name": "config.field.replacement.description", + "value": "May use reference from regex as in python's re.Pattern.sub method. If set to empty string, found header will be fully removed from response" + }, + { "name": "config.button.addHeaderReplacement", "value": "Add" }, + { "name": "config.action.addHeaderReplacement", "value": "Add New Header Replacement" }, + { "name": "config.action.removeHeaderReplacement", "value": "Remove" }, + + { "name": "err.requestProtector.headerRegexRequired", "value": "RegEx field is required" } + ] + }] + } +}] diff --git a/bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json new file mode 100644 index 00000000..787f10f6 --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json @@ -0,0 +1,15 @@ +[{ + "name": "RequestProtector", + "children": { + "AppMatcher": [{ + "name": "RequestProtectorMatcher", + "template": true, + "connCheck": true, + "site": "All_Sites", + "fqdn": "*", + "urlRegex": ".*", + "rule": "request", + "priority": -1000000 + }] + } +}] diff --git a/bubble-server/src/main/resources/models/apps/request/request-icon.svg b/bubble-server/src/main/resources/models/apps/request/request-icon.svg new file mode 100644 index 00000000..5c29e3a5 --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/request/request-icon.svg @@ -0,0 +1,89 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + diff --git a/bubble-server/src/main/resources/models/defaults/bubblePlan.json b/bubble-server/src/main/resources/models/defaults/bubblePlan.json index 188c5fc9..fb5e32e9 100644 --- a/bubble-server/src/main/resources/models/defaults/bubblePlan.json +++ b/bubble-server/src/main/resources/models/defaults/bubblePlan.json @@ -14,7 +14,8 @@ "children": { "BubblePlanApp": [ {"app": "BubbleBlock"}, - {"app": "TlsPassthru"} + {"app": "TlsPassthru"}, + {"app": "RequestProtector"} ] } }, @@ -36,7 +37,8 @@ {"app": "TrafficAnalytics"}, {"app": "BubbleBlock"}, {"app": "UserBlocker"}, - {"app": "TlsPassthru"} + {"app": "TlsPassthru"}, + {"app": "RequestProtector"} ] } }, @@ -58,7 +60,8 @@ {"app": "TrafficAnalytics"}, {"app": "BubbleBlock"}, {"app": "UserBlocker"}, - {"app": "TlsPassthru"} + {"app": "TlsPassthru"}, + {"app": "RequestProtector"} ] } } diff --git a/bubble-server/src/main/resources/models/defaults/ruleDriver.json b/bubble-server/src/main/resources/models/defaults/ruleDriver.json index 113019f0..d9a1346b 100644 --- a/bubble-server/src/main/resources/models/defaults/ruleDriver.json +++ b/bubble-server/src/main/resources/models/defaults/ruleDriver.json @@ -28,5 +28,11 @@ "driverClass": "bubble.rule.bblock.BubbleBlockRuleDriver", "template": true, "userConfig": { "fields": [] } + }, + { + "name": "RequestProtectorRuleDriver", + "driverClass": "bubble.rule.request.RequestProtectorRuleDriver", + "template": true, + "userConfig": { "fields": [] } } -] \ No newline at end of file +] diff --git a/bubble-server/src/main/resources/models/manifest-app-request.json b/bubble-server/src/main/resources/models/manifest-app-request.json new file mode 100644 index 00000000..d7d41a06 --- /dev/null +++ b/bubble-server/src/main/resources/models/manifest-app-request.json @@ -0,0 +1,4 @@ +[ + "apps/request/bubbleApp_request", + "apps/request/bubbleApp_request_matchers" +] diff --git a/bubble-server/src/main/resources/models/manifest-defaults.json b/bubble-server/src/main/resources/models/manifest-defaults.json index c5d15598..26c9d2bf 100644 --- a/bubble-server/src/main/resources/models/manifest-defaults.json +++ b/bubble-server/src/main/resources/models/manifest-defaults.json @@ -5,5 +5,6 @@ "manifest-app-user-block", "manifest-app-bubble-block", "manifest-app-passthru", + "manifest-app-request", "defaults/bubblePlan" -] \ No newline at end of file +] diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index a42661df..df82e87b 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -25,6 +25,7 @@ from bubble_config import bubble_port, debug_capture_fqdn, \ from mitmproxy import http from mitmproxy.net.http import headers as nheaders from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody +from mitmproxy.utils import strutils bubble_log = logging.getLogger(__name__) @@ -449,6 +450,131 @@ def original_flex_ip(client_addr, fqdns): return None +def update_host_and_port(flow): + if flow.request: + if flow.client_conn.tls_established: + flow.request.scheme = "https" + sni = flow.client_conn.connection.get_servername() + port = 443 + else: + flow.request.scheme = "http" + sni = None + port = 80 + + host_header = flow.request.host_header + if host_header: + m = parse_host_header.match(host_header) + if m: + host_header = m.group("host").strip("[]") + if m.group("port"): + port = int(m.group("port")) + + host = None + if sni or host_header: + host = str(sni or host_header) + if host.startswith("b'"): + host = host[2:-1] + + flow.request.host_header = host_header + if host: + flow.request.host = host + else: + flow.request.host = host_header + flow.request.port = port + + return flow + + +def _replace_in_headers(headers: nheaders.Headers, modifiers_dict: dict) -> int: + """ + Taken from original mitmproxy's Header class implementation with some changes. + + Replaces a regular expression pattern with repl in each "name: value" + header line. + + Returns: + The number of replacements made. + """ + repl_count = 0 + fields = [] + + for name, value in headers.fields: + + line = name + b": " + value + inner_repl_count = 0 + for pattern, replacement in modifiers_dict.items(): + line, n = pattern.subn(replacement, line) + inner_repl_count += n + if len(line) == 0: + # No need to go though other patterns for this line + break + + if len(line) == 0: + # Skip (remove) this header line in this case + break + + if inner_repl_count > 0: + # only in case when there were some replacements: + try: + name, value = line.split(b": ", 1) + except ValueError: + # We get a ValueError if the replacement removed the ": " + # There's not much we can do about this, so we just keep the header as-is. + pass + else: + repl_count += inner_repl_count + + fields.append((name, value)) + + headers.fields = tuple(fields) + return repl_count + + +def response_header_modify(flow) -> int: + if flow.response is None: + return None + + flow = update_host_and_port(flow) + ctx = {'fqdn': flow.request.host} + return _header_modify(flow.client_conn.address[0], ctx, flow.response.headers) + + +def _header_modify(client_addr: str, ctx: dict, headers: nheaders.Headers) -> int: + modifiers_set = 'responseHeaderModifierLists~' + client_addr + '~UNION' + modifiers = REDIS.smembers(modifiers_set) + + repl_count = 0 + if modifiers: + modifiers_dict = {} + for modifier in modifiers: + regex, replacement = _extract_modifier_config(modifier, ctx) + modifiers_dict[regex] = replacement + repl_count += _replace_in_headers(headers, modifiers_dict) + + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_header_modify: replacing headers - replacements count: ' + repl_count) + + return repl_count + + +def _extract_modifier_config(modifier: bytes, ctx: dict) -> tuple: + modifier_obj = json.loads(modifier) + + regex = _replace_modifier_values(modifier_obj['regex'], ctx) + replacement = _replace_modifier_values(modifier_obj['replacement'], ctx) + + regex = re.compile(strutils.escaped_str_to_bytes(regex)) + replacement = strutils.escaped_str_to_bytes(replacement) + + return regex, replacement + + +def _replace_modifier_values(s: str, ctx: dict) -> str: + # no loop over ctx currently to speed up as there's just 1 variable inside + s = s.replace('{{fqdn}}', re.escape(ctx['fqdn'])) + return s + + def health_check_response(flow): # if bubble_log.isEnabledFor(DEBUG): # bubble_log.debug('health_check_response: special bubble health check request, responding with OK') diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 44e5fc8a..49a28907 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -16,7 +16,7 @@ from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATIO is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \ HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \ - HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set + HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, response_header_modify from bubble_flex import process_flex import logging @@ -311,6 +311,7 @@ def responseheaders(flow): else: flex_flow = None bubble_filter_response(flow, flex_flow) + response_header_modify(flow) def bubble_filter_response(flow, flex_flow): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 658f2745..d8447401 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -33,7 +33,7 @@ from bubble_api import bubble_matchers, bubble_activity_log, \ CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ is_bubble_special_path, is_bubble_health_check, health_check_response, tarpit_response,\ - is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain + is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, update_host_and_port from bubble_config import bubble_host, bubble_host_alias from bubble_flex import new_flex_flow @@ -114,34 +114,20 @@ class Rerouter: def bubble_handle_request(self, flow): client_addr = flow.client_conn.address[0] server_addr = flow.server_conn.address[0] - is_http = False + flow = update_host_and_port(flow) + if flow.client_conn.tls_established: - flow.request.scheme = "https" sni = flow.client_conn.connection.get_servername() - port = 443 + is_http = False else: - flow.request.scheme = "http" sni = None - port = 80 is_http = True - # check if https and sni is missing but we have a host header, fill in the sni - - host_header = flow.request.host_header - if host_header: - m = parse_host_header.match(host_header) - if m: - host_header = m.group("host").strip("[]") - if m.group("port"): - port = int(m.group("port")) - # Determine if this request should be filtered - host = None + host_header = flow.request.host_header + host = flow.request.host path = flow.request.path if sni or host_header: - host = str(sni or host_header) - if host.startswith("b'"): - host = host[2:-1] log_url = flow.request.scheme + '://' + host + path # If https, we have already checked that the client/server are legal in bubble_conn_check.py @@ -240,12 +226,6 @@ class Rerouter: bubble_log.warning('bubble_handle_request: no sni/host found, not applying rules to path: ' + path) bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr]) - flow.request.host_header = host_header - if host: - flow.request.host = host - else: - flow.request.host = host_header - flow.request.port = port return host def requestheaders(self, flow):