bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java

@@ -0,0 +1,115 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.app.request;
+
+import bubble.model.account.Account;
+import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
+import bubble.model.app.config.AppConfigDriverBase;
+import bubble.rule.request.HeaderReplacement;
+import bubble.rule.request.RequestProtectorConfig;
+import bubble.rule.request.RequestProtectorRuleDriver;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+import java.util.Set;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
+import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
+
+@Slf4j
+public class RequestProtectorAppConfigDriver extends AppConfigDriverBase {
+
+    public static final String VIEW_manageHeaderReplacements = "manageHeaderReplacements";
+
+    @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
+        switch (view) {
+            case VIEW_manageHeaderReplacements:
+                return loadManageCookiesReplacements(account, app);
+        }
+        throw notFoundEx(view);
+    }
+
+    private Set<HeaderReplacement> loadManageCookiesReplacements(Account account, BubbleApp app) {
+        final RequestProtectorConfig config = getConfig(account, app);
+        return config.getHeaderReplacements();
+    }
+
+    private RequestProtectorConfig getConfig(Account account, BubbleApp app) {
+        return getConfig(account, app, RequestProtectorRuleDriver.class, RequestProtectorConfig.class);
+    }
+
+    public static final String ACTION_addHeaderReplacement = "addHeaderReplacement";
+    public static final String ACTION_removeHeaderReplacement = "removeHeaderReplacement";
+
+    public static final String PARAM_REGEX = "regex";
+    public static final String PARAM_REPLACEMENT = "replacement";
+
+    @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action,
+                                          Map<String, String> params, JsonNode data) {
+        switch (action) {
+            case ACTION_addHeaderReplacement:
+                return addHeaderReplacement(account, app, data);
+        }
+        if (log.isWarnEnabled()) log.warn("takeAppAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private Set<HeaderReplacement> addHeaderReplacement(Account account, BubbleApp app, JsonNode data) {
+        final JsonNode regexNode = data.get(PARAM_REGEX);
+        if (regexNode == null || regexNode.textValue() == null) {
+            throw invalidEx("err.requestProtector.headerRegexRequired");
+        }
+        final String regex = regexNode.textValue().trim();
+        if (empty(regex)) throw invalidEx("err.requestProtector.headerRegexRequired");
+
+        final JsonNode replacementNode = data.get(PARAM_REPLACEMENT);
+        final String replacement = (replacementNode == null || replacementNode.textValue() == null)
+                                   ? "" : replacementNode.textValue().trim();
+
+        final RequestProtectorConfig config = getConfig(account, app).addHeaderReplacement(regex, replacement);
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver
+        if (log.isDebugEnabled()) {
+            log.debug("addHeaderReplacement: updating rule: " + rule.getName() + ", adding regex: " + regex);
+        }
+        ruleDAO.update(rule.setConfigJson(json(config)));
+
+        return config.getHeaderReplacements();
+    }
+
+    @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id,
+                                           Map<String, String> params, JsonNode data) {
+        switch (action) {
+            case ACTION_removeHeaderReplacement:
+                return removeHeaderReplacement(account, app, id);
+        }
+        if (log.isWarnEnabled()) log.warn("takeItemAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private Set<HeaderReplacement> removeHeaderReplacement(Account account, BubbleApp app, String regex) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver
+        final RequestProtectorConfig config = getConfig(account, app);
+        if (log.isDebugEnabled()) {
+            log.debug("removeHeaderReplacement: removing regex: " + regex + " from config.cookiesReplacements: "
+                      + config.getHeaderReplacements().toString());
+        }
+
+        final RequestProtectorConfig updated = config.removeHeaderReplacement(regex);
+        if (log.isDebugEnabled()) {
+            log.debug("removeHeaderReplacement: updated.cookiesReplacements: "
+                      + updated.getHeaderReplacements().toString());
+        }
+        ruleDAO.update(rule.setConfigJson(json(updated)));
+
+        return updated.getHeaderReplacements();
+    }
+}

bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java

@@ -0,0 +1,9 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.app.request;
+
+import bubble.model.app.config.AppDataDriverBase;
+
+public class RequestProtectorAppDataDriver extends AppDataDriverBase {}

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -47,6 +47,7 @@ public interface AppRuleDriver {
     String REDIS_FILTER_LISTS = "filterLists";
     String REDIS_FLEX_LISTS = "flexLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
     String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
+    String REDIS_RESPONSE_HEADER_MODIFIER_LISTS = "responseHeaderModifierLists";  // used in mitmproxy
     String REDIS_LIST_SUFFIX = "~UNION";
 
     default Set<String> getPrimedRejectDomains () { return null; }
@@ -55,6 +56,7 @@ public interface AppRuleDriver {
     default Set<String> getPrimedFilterDomains () { return null; }
     default Set<String> getPrimedFlexDomains () { return null; }
     default Set<String> getPrimedFlexExcludeDomains () { return null; }
+    default Set<String> getPrimedResponseHeaderModifiers () { return null; }
 
     static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) {
         defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains);
@@ -80,12 +82,22 @@ public interface AppRuleDriver {
         defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains);
     }
 
-    static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) {
+    static void defineRedisResponseHeaderModifiersSet(RedisService redis, String ip, String list,
+                                                      String[] modifiers) {
+        defineRedisSet(redis, ip, REDIS_RESPONSE_HEADER_MODIFIER_LISTS, list, modifiers);
+    }
+
+    /**
+     * `settings` parameter may be list of domains or any other list of strings - i.e. list of JSONs with specific setup
+     * for the prime option of the driver.
+     */
+    static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName,
+                               String[] settings) {
         final String listOfListsForIp = listOfListsName + "~" + ip;
         final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX;
         final String ipList = listOfListsForIp + "~" + listName;
         final String tempList = ipList + "~"+now()+randomAlphanumeric(5);
-        redis.sadd_plaintext(tempList, domains);
+        redis.sadd_plaintext(tempList, settings);
         redis.rename(tempList, ipList);
         redis.sadd_plaintext(listOfListsForIp, ipList);
         final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp));

bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.request;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class HeaderReplacement implements Comparable<HeaderReplacement> {
+
+    public String getId() { return regex; }
+    public void setId(String id) {} // noop
+
+    @Getter @Setter private String regex;
+    @Getter @Setter private String replacement;
+
+    public HeaderReplacement(@NonNull final String regex, @NonNull final String replacement) {
+        this.regex = regex;
+        this.replacement = replacement;
+    }
+
+    @Override public int compareTo(@NonNull final HeaderReplacement o) {
+        return getRegex().compareTo(o.getRegex().toLowerCase());
+    }
+}

bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.request;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+
+@Slf4j @Accessors(chain=true)
+public class RequestProtectorConfig {
+
+    @Getter @Setter private Set<HeaderReplacement> headerReplacements = new TreeSet<>();
+    public boolean hasHeaderReplacements() { return !empty(headerReplacements); }
+
+    @NonNull public RequestProtectorConfig addHeaderReplacement(@NonNull final String regex,
+                                                                @NonNull final String replacement) {
+        headerReplacements.add(new HeaderReplacement(regex, replacement));
+        return this;
+    }
+
+    @NonNull public RequestProtectorConfig removeHeaderReplacement(@NonNull final String regex) {
+        if (hasHeaderReplacements()) headerReplacements.removeIf(r -> r.getRegex().equals(regex));
+        return this;
+    }
+}

bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.request;
+
+import bubble.model.account.Account;
+import bubble.model.app.AppMatcher;
+import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
+import bubble.model.device.Device;
+import bubble.rule.AbstractAppRuleDriver;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.json.JsonUtil;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class RequestProtectorRuleDriver extends AbstractAppRuleDriver {
+
+    @Override public <C> Class<C> getConfigClass() { return (Class<C>) RequestProtectorConfig.class; }
+
+    @Override public Set<String> getPrimedResponseHeaderModifiers() {
+        final RequestProtectorConfig config = getRuleConfig();
+        return config.getHeaderReplacements().stream().map(JsonUtil::json).collect(Collectors.toSet());
+    }
+
+    @Override public void init(JsonNode config, JsonNode userConfig, BubbleApp app, AppRule rule, AppMatcher matcher,
+                               Account account, Device device) {
+        super.init(config, userConfig, app, rule, matcher, account, device);
+
+        // refresh list
+        final RequestProtectorConfig ruleConfig = getRuleConfig();
+        ruleConfig.getHeaderReplacements();
+    }
+}

bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java

@@ -16,14 +16,19 @@ import bubble.server.BubbleConfiguration;
 import bubble.service.device.DeviceService;
 import bubble.service.device.StandardFlexRouterService;
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.collection.SingletonList;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.*;
+import javax.annotation.Nullable;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool;
@@ -110,130 +115,185 @@ public class StandardAppPrimerService implements AppPrimerService {
         getPrimerThread().submit(() -> _prime(account, singleApp));
     }
 
-    private synchronized void _prime(Account account, BubbleApp singleApp) {
+    private synchronized void _prime(@NonNull final Account account, @Nullable final BubbleApp singleApp) {
         try {
-            final Map<String, List<String>> accountDeviceIps = new HashMap<>();
             final List<Device> devices = deviceDAO.findByAccount(account.getUuid());
-            for (Device device : devices) {
-                accountDeviceIps.put(device.getUuid(), deviceService.findIpsByDevice(device.getUuid()));
-            }
-            if (accountDeviceIps.isEmpty()) return;
+            if (devices.isEmpty()) return;
+
+            final Map<String, List<String>> accountDeviceIps =
+                    devices.stream()
+                           .map(Device::getUuid)
+                           .collect(Collectors.toMap(Function.identity(), deviceService::findIpsByDevice));
 
             // flex domains can only be managed by the first admin
             final Account firstAdmin = accountDAO.getFirstAdmin();
             account.setFirstAdmin(account.getUuid().equals(firstAdmin.getUuid()));
-            boolean updateFlexRouters = false;
-            Set<String> flexDomains = null;
-            Set<String> flexExcludeDomains = null;
-
-            final List<BubbleApp> appsToPrime = singleApp == null
-                    ? appDAO.findByAccount(account.getUuid()).stream()
-                    .filter(BubbleApp::canPrime)
-                    .collect(Collectors.toList())
-                    : new SingletonList<>(singleApp);
-            for (BubbleApp app : appsToPrime) {
-                log.info("_prime: priming app: "+app.getUuid()+"/"+app.getName());
-                final List<AppRule> rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
-                final List<AppMatcher> matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
-                for (AppRule rule : rules) {
-                    final RuleDriver driver = driverDAO.findByUuid(rule.getDriver());
-                    if (driver == null) {
-                        log.warn("_prime: driver not found for app/rule " + app.getName() + "/" + rule.getName() + ": " + rule.getDriver());
-                        continue;
+
+            if (singleApp != null) {
+                _primeApp(account, accountDeviceIps, devices, singleApp);
+            } else {
+                appDAO.findByAccount(account.getUuid())
+                      .stream()
+                      .filter(BubbleApp::canPrime)
+                      .forEach(app -> _primeApp(account, accountDeviceIps, devices, app));
+            }
+        } catch (Exception e) {
+            die("_prime: " + shortError(e), e);
+        } finally {
+            log.info("_prime: completed");
+        }
+    }
+
+    private void _primeApp(@NonNull final Account account, @NonNull final Map<String, List<String>> accountDeviceIps,
+                           @NonNull final List<Device> devices, @NonNull final BubbleApp app) {
+        log.info("_primeApp: " + app.getUuid() + "/" + app.getName());
+
+        final List<AppRule> rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
+        final List<AppMatcher> matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
+
+        boolean updateFlexRouters = false;
+        Set<String> flexDomains = null;
+        for (AppRule rule : rules) {
+            final RuleDriver driver = driverDAO.findByUuid(rule.getDriver());
+            if (driver == null) {
+                log.warn("_primeApp: driver not found for app/rule "
+                         + app.getName() + "/" + rule.getName() + ": " + rule.getDriver());
+                continue;
+            }
+
+            // handle AppData callback registration with a basic driver
+            final AppRuleDriver cbDriver = driver.getDriver();
+            if (cbDriver instanceof HasAppDataCallback) {
+                log.debug("_primeApp: AppRuleDriver (" + cbDriver.getClass().getSimpleName()
+                          + ") implements HasAppDataCallback, registering: " + app.getUuid() + "/" + app.getName());
+                final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver;
+                dataCallback.prime(account, app, configuration);
+                dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration));
+            }
+
+            for (final Device device : devices) {
+                final Set<String> rejectDomains = new HashSet<>();
+                final Set<String> blockDomains = new HashSet<>();
+                final Set<String> whiteListDomains = new HashSet<>();
+                final Set<String> filterDomains = new HashSet<>();
+                final Set<String> flexExcludeDomains = new HashSet<>();
+                final Set<String> requestHeaderModifiers = new HashSet<>();
+
+                boolean areAllSetsEmpty = true;
+                for (AppMatcher matcher : matchers) {
+                    final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device);
+
+                    final Set<String> rejects = appRuleDriver.getPrimedRejectDomains();
+                    if (empty(rejects)) {
+                        log.debug("_primeApp: no rejectDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        rejectDomains.addAll(rejects);
+                        areAllSetsEmpty = empty(rejects);
                     }
-                    // handle AppData callback registration with a basic driver
-                    final AppRuleDriver cbDriver = driver.getDriver();
-                    if (cbDriver instanceof HasAppDataCallback) {
-                        log.debug("_prime: AppRuleDriver ("+cbDriver.getClass().getSimpleName()+") implements HasAppDataCallback, registering: "+app.getUuid()+"/"+app.getName());
-                        final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver;
-                        dataCallback.prime(account, app, configuration);
-                        dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration));
+
+                    final Set<String> blocks = appRuleDriver.getPrimedBlockDomains();
+                    if (empty(blocks)) {
+                        log.debug("_primeApp: no blockDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        blockDomains.addAll(blocks);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(blocks);
+                    }
+
+                    final Set<String> whiteList = appRuleDriver.getPrimedWhiteListDomains();
+                    if (empty(whiteList)) {
+                        log.debug("_primeApp: no whiteListDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        whiteListDomains.addAll(whiteList);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(whiteList);
                     }
-                    for (Device device : devices) {
-                        final Set<String> rejectDomains = new HashSet<>();
-                        final Set<String> blockDomains = new HashSet<>();
-                        final Set<String> whiteListDomains = new HashSet<>();
-                        final Set<String> filterDomains = new HashSet<>();
-                        for (AppMatcher matcher : matchers) {
-                            final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device);
-                            final Set<String> rejects = appRuleDriver.getPrimedRejectDomains();
-                            if (empty(rejects)) {
-                                log.debug("_prime: no rejectDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                rejectDomains.addAll(rejects);
-                            }
-                            final Set<String> blocks = appRuleDriver.getPrimedBlockDomains();
-                            if (empty(blocks)) {
-                                log.debug("_prime: no blockDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                blockDomains.addAll(blocks);
-                            }
-                            final Set<String> whiteList = appRuleDriver.getPrimedWhiteListDomains();
-                            if (empty(whiteList)) {
-                                log.debug("_prime: no whiteListDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                whiteListDomains.addAll(whiteList);
-                            }
-                            final Set<String> filters = appRuleDriver.getPrimedFilterDomains();
-                            if (empty(filters)) {
-                                log.debug("_prime: no filterDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                filterDomains.addAll(filters);
-                            }
-                            if (account.isFirstAdmin() && flexDomains == null) {
-                                final Set<String> flexes = appRuleDriver.getPrimedFlexDomains();
-                                if (empty(flexes)) {
-                                    log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                                } else {
-                                    flexDomains = new HashSet<>(flexes);
-                                }
-                                final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
-                                if (empty(flexExcludes)) {
-                                    log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                                } else {
-                                    flexExcludeDomains = new HashSet<>(flexExcludes);
-                                }
-                            }
+
+                    final Set<String> filters = appRuleDriver.getPrimedFilterDomains();
+                    if (empty(filters)) {
+                        log.debug("_primeApp: no filterDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        filterDomains.addAll(filters);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(filters);
+                    }
+
+                    final Set<String> modifiers = appRuleDriver.getPrimedResponseHeaderModifiers();
+                    if (empty(modifiers)) {
+                        log.debug("_primeApp: no responseHeaderModifiers for device/app/rule/matcher: "
+                                  + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/"
+                                  + matcher.getName());
+                    } else {
+                        requestHeaderModifiers.addAll(modifiers);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(modifiers);
+                    }
+
+                    if (account.isFirstAdmin() && flexDomains == null) {
+                        final Set<String> flexes = appRuleDriver.getPrimedFlexDomains();
+                        if (empty(flexes)) {
+                            log.debug("_primeApp: no flexDomains for device/app/rule/matcher: " + device.getName()
+                                      + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                        } else {
+                            flexDomains = new HashSet<>(flexes);
+                            areAllSetsEmpty = areAllSetsEmpty && empty(flexes);
                         }
-                        if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) {
-                            for (String ip : accountDeviceIps.get(device.getUuid())) {
-                                if (!empty(rejectDomains)) {
-                                    rejectDomains.removeAll(whiteListDomains);
-                                    AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new));
-                                }
-                                if (!empty(blockDomains)) {
-                                    blockDomains.removeAll(whiteListDomains);
-                                    AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new));
-                                }
-                                if (!empty(whiteListDomains)) {
-                                    AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), whiteListDomains.toArray(String[]::new));
-                                }
-                                if (!empty(filterDomains)) {
-                                    AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new));
-                                }
-                                if (account.isFirstAdmin() && (!empty(flexDomains) || !empty(flexExcludeDomains))) {
-                                    updateFlexRouters = true;
-                                    if (!empty(flexDomains)) {
-                                        if (flexExcludeDomains != null) flexDomains.removeAll(flexExcludeDomains);
-                                        AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new));
-                                    }
-                                    if (!empty(flexExcludeDomains)) {
-                                        AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new));
-                                    }
-                                }
-                            }
+
+                        final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
+                        if (empty(flexExcludes)) {
+                            log.debug("_primeApp: no flexExcludeDomains for device/app/rule/matcher: "
+                                      + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/"
+                                      + matcher.getName());
+                        } else {
+                            flexExcludeDomains.addAll(flexExcludes);
+                            areAllSetsEmpty = areAllSetsEmpty && empty(flexExcludes);
                         }
                     }
                 }
-                if (updateFlexRouters && !empty(flexDomains)) {
-                    flexRouterService.updateFlexRoutes(flexDomains);
+
+                if (areAllSetsEmpty) continue;
+
+                for (final String ip : accountDeviceIps.get(device.getUuid())) {
+                    if (!empty(rejectDomains)) {
+                        rejectDomains.removeAll(whiteListDomains);
+                        AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                           rejectDomains.toArray(String[]::new));
+                    }
+                    if (!empty(blockDomains)) {
+                        blockDomains.removeAll(whiteListDomains);
+                        AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                          blockDomains.toArray(String[]::new));
+                    }
+                    if (!empty(whiteListDomains)) {
+                        AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                              whiteListDomains.toArray(String[]::new));
+                    }
+                    if (!empty(filterDomains)) {
+                        AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                           filterDomains.toArray(String[]::new));
+                    }
+                    if (!empty(requestHeaderModifiers)) {
+                        AppRuleDriver.defineRedisResponseHeaderModifiersSet(
+                                redis, ip, app.getName() + ":" + app.getUuid(),
+                                requestHeaderModifiers.toArray(String[]::new));
+                    }
+                    if (account.isFirstAdmin()) {
+                        if (!empty(flexDomains)) {
+                            if (!empty(flexExcludeDomains)) flexDomains.removeAll(flexExcludeDomains);
+                            AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                             flexDomains.toArray(String[]::new));
+                            updateFlexRouters = true;
+                        }
+                        if (!empty(flexExcludeDomains)) {
+                            AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                                    flexExcludeDomains.toArray(String[]::new));
+                        }
+                    }
                 }
             }
-        } catch (Exception e) {
-            die("_prime: "+shortError(e), e);
-        } finally {
-            log.info("_primeApps: completed");
         }
+
+        if (updateFlexRouters) flexRouterService.updateFlexRoutes(flexDomains);
     }
 
 }

bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json

@@ -0,0 +1,81 @@
+[{
+  "name": "RequestProtector",
+  "description": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response",
+  "url": "https://getbubblenow.com/apps/request",
+  "template": true,
+  "enabled": true,
+  "priority": 1000,
+  "canPrime": true,
+  "dataConfig": {
+    "dataDriver": "bubble.app.request.RequestProtectorAppDataDriver",
+    "presentation": "app",
+    "configDriver": "bubble.app.request.RequestProtectorAppConfigDriver",
+    "configFields": [
+      {"name": "regex", "truncate": false},
+      {"name": "replacement", "truncate": false}
+    ],
+    "configViews": [{
+      "name": "manageHeaderReplacements",
+      "scope": "app",
+      "root": "true",
+      "fields": [ "regex", "replacement" ],
+      "actions": [
+        {"name": "removeHeaderReplacement", "index": 10},
+        {
+          "name": "addHeaderReplacement", "scope": "app", "index": 10,
+          "params": [ "regex", "replacement" ],
+          "button": "addHeaderReplacement"
+        }
+      ]
+    }]
+  },
+  "children": {
+    "AppSite": [{
+      "name": "All_Sites",
+      "url": "*",
+      "description": "All websites",
+      "template": true
+    }],
+    "AppRule": [{
+      "name": "request",
+      "template": true,
+      "driver": "RequestProtectorRuleDriver",
+      "priority": -1000,
+      "config": {
+        "headerReplacements": [{
+          "regex": "^(?i:Set-Cookie):(.*;)?\\s*(?i:Domain)=(((([\\*\\.].*)|(.*\\.))\\s*(;.*)?)|((?!([^;,]*\\.)?{{fqdn}}\\s*(;|$)).*))$",
+          "replacement": ""
+        }]
+      }
+    }],
+    "AppMessage": [{
+      "locale": "en_US",
+      "messages": [
+        { "name": "name", "value": "RequestProtector" },
+        { "name": "icon", "value": "classpath:models/apps/request/request-icon.svg" },
+        { "name": "summary", "value": "Request Protector" },
+        {
+          "name": "description",
+          "value": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response"
+        },
+
+        { "name": "config.view.manageHeaderReplacements", "value": "Manage Header Replacements" },
+        { "name": "config.field.regex", "value": "RegEx" },
+        {
+          "name": "config.field.regex.description",
+          "value": "Regular expression compared with full header's line string value"
+        },
+        { "name": "config.field.replacement", "value": "Replacement" },
+        {
+          "name": "config.field.replacement.description",
+          "value": "May use reference from regex as in python's re.Pattern.sub method. If set to empty string, found header will be fully removed from response"
+        },
+        { "name": "config.button.addHeaderReplacement", "value": "Add" },
+        { "name": "config.action.addHeaderReplacement", "value": "Add New Header Replacement" },
+        { "name": "config.action.removeHeaderReplacement", "value": "Remove" },
+
+        { "name": "err.requestProtector.headerRegexRequired", "value": "RegEx field is required" }
+      ]
+    }]
+  }
+}]

bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json

@@ -0,0 +1,15 @@
+[{
+  "name": "RequestProtector",
+  "children": {
+    "AppMatcher": [{
+      "name": "RequestProtectorMatcher",
+      "template": true,
+      "connCheck": true,
+      "site": "All_Sites",
+      "fqdn": "*",
+      "urlRegex": ".*",
+      "rule": "request",
+      "priority": -1000000
+    }]
+  }
+}]

bubble-server/src/main/resources/models/apps/request/request-icon.svg

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   width="80"
+   height="80"
+   id="svg10029"
+   sodipodi:docname="Hazard light icon.svg"
+   inkscape:version="0.92.4 (5da689c313, 2019-01-14)">
+  <metadata
+     id="metadata9">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1680"
+     inkscape:window-height="998"
+     id="namedview7"
+     showgrid="true"
+     showguides="true"
+     inkscape:guide-bbox="true"
+     inkscape:snap-page="true"
+     inkscape:zoom="2.085965"
+     inkscape:cx="-18.750976"
+     inkscape:cy="1.4877119"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g10556">
+    <sodipodi:guide
+       position="-143,40"
+       orientation="0,1"
+       id="guide819"
+       inkscape:locked="false" />
+    <inkscape:grid
+       type="xygrid"
+       id="grid823" />
+    <sodipodi:guide
+       position="40,55"
+       orientation="1,0"
+       id="guide4529"
+       inkscape:locked="false" />
+  </sodipodi:namedview>
+  <defs
+     id="defs10032" />
+  <g
+     transform="translate(-19.885022,2.1230082)"
+     id="g10556">
+    <path
+       sodipodi:type="star"
+       style="fill:none;fill-opacity:1;stroke:#ff0000;stroke-width:10;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none"
+       id="path4531"
+       sodipodi:sides="3"
+       sodipodi:cx="59.885021"
+       sodipodi:cy="47.876992"
+       sodipodi:r1="40"
+       sodipodi:r2="20"
+       sodipodi:arg1="-1.5707963"
+       sodipodi:arg2="-0.52359878"
+       inkscape:flatsided="false"
+       inkscape:rounded="0"
+       inkscape:randomized="0"
+       d="m 59.885022,7.8769913 17.320507,29.9999997 17.320508,30.000001 -34.641016,-10e-7 -34.641016,-10e-7 17.320508,-29.999999 z"
+       inkscape:transform-center-y="-10" />
+  </g>
+</svg>

bubble-server/src/main/resources/models/defaults/bubblePlan.json

@@ -14,7 +14,8 @@
     "children": {
       "BubblePlanApp": [
         {"app": "BubbleBlock"},
-        {"app": "TlsPassthru"}
+        {"app": "TlsPassthru"},
+        {"app": "RequestProtector"}
       ]
     }
   },
@@ -36,7 +37,8 @@
         {"app": "TrafficAnalytics"},
         {"app": "BubbleBlock"},
         {"app": "UserBlocker"},
-        {"app": "TlsPassthru"}
+        {"app": "TlsPassthru"},
+        {"app": "RequestProtector"}
       ]
     }
   },
@@ -58,7 +60,8 @@
         {"app": "TrafficAnalytics"},
         {"app": "BubbleBlock"},
         {"app": "UserBlocker"},
-        {"app": "TlsPassthru"}
+        {"app": "TlsPassthru"},
+        {"app": "RequestProtector"}
       ]
     }
   }

bubble-server/src/main/resources/models/defaults/ruleDriver.json

@@ -28,5 +28,11 @@
     "driverClass": "bubble.rule.bblock.BubbleBlockRuleDriver",
     "template": true,
     "userConfig": { "fields": [] }
+  },
+  {
+    "name": "RequestProtectorRuleDriver",
+    "driverClass": "bubble.rule.request.RequestProtectorRuleDriver",
+    "template": true,
+    "userConfig": { "fields": [] }
   }
-]
+]

bubble-server/src/main/resources/models/manifest-app-request.json

@@ -0,0 +1,4 @@
+[
+  "apps/request/bubbleApp_request",
+  "apps/request/bubbleApp_request_matchers"
+]

bubble-server/src/main/resources/models/manifest-defaults.json

@@ -5,5 +5,6 @@
   "manifest-app-user-block",
   "manifest-app-bubble-block",
   "manifest-app-passthru",
+  "manifest-app-request",
   "defaults/bubblePlan"
-]
+]

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -25,6 +25,7 @@ from bubble_config import bubble_port, debug_capture_fqdn, \
 from mitmproxy import http
 from mitmproxy.net.http import headers as nheaders
 from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody
+from mitmproxy.utils import strutils
 
 bubble_log = logging.getLogger(__name__)
 
@@ -449,6 +450,131 @@ def original_flex_ip(client_addr, fqdns):
     return None
 
 
+def update_host_and_port(flow):
+    if flow.request:
+        if flow.client_conn.tls_established:
+            flow.request.scheme = "https"
+            sni = flow.client_conn.connection.get_servername()
+            port = 443
+        else:
+            flow.request.scheme = "http"
+            sni = None
+            port = 80
+
+        host_header = flow.request.host_header
+        if host_header:
+            m = parse_host_header.match(host_header)
+            if m:
+                host_header = m.group("host").strip("[]")
+                if m.group("port"):
+                    port = int(m.group("port"))
+
+        host = None
+        if sni or host_header:
+            host = str(sni or host_header)
+            if host.startswith("b'"):
+                host = host[2:-1]
+
+        flow.request.host_header = host_header
+        if host:
+            flow.request.host = host
+        else:
+            flow.request.host = host_header
+        flow.request.port = port
+
+    return flow
+
+
+def _replace_in_headers(headers: nheaders.Headers, modifiers_dict: dict) -> int:
+    """
+    Taken from original mitmproxy's Header class implementation with some changes.
+
+    Replaces a regular expression pattern with repl in each "name: value"
+    header line.
+
+    Returns:
+        The number of replacements made.
+    """
+    repl_count = 0
+    fields = []
+
+    for name, value in headers.fields:
+
+        line = name + b": " + value
+        inner_repl_count = 0
+        for pattern, replacement in modifiers_dict.items():
+            line, n = pattern.subn(replacement, line)
+            inner_repl_count += n
+            if len(line) == 0:
+                # No need to go though other patterns for this line
+                break
+
+        if len(line) == 0:
+            # Skip (remove) this header line in this case
+            break
+
+        if inner_repl_count > 0:
+            # only in case when there were some replacements:
+            try:
+                name, value = line.split(b": ", 1)
+            except ValueError:
+                # We get a ValueError if the replacement removed the ": "
+                # There's not much we can do about this, so we just keep the header as-is.
+                pass
+            else:
+                repl_count += inner_repl_count
+
+        fields.append((name, value))
+
+    headers.fields = tuple(fields)
+    return repl_count
+
+
+def response_header_modify(flow) -> int:
+    if flow.response is None:
+        return None
+
+    flow = update_host_and_port(flow)
+    ctx = {'fqdn': flow.request.host}
+    return _header_modify(flow.client_conn.address[0], ctx, flow.response.headers)
+
+
+def _header_modify(client_addr: str, ctx: dict, headers: nheaders.Headers) -> int:
+    modifiers_set = 'responseHeaderModifierLists~' + client_addr + '~UNION'
+    modifiers = REDIS.smembers(modifiers_set)
+
+    repl_count = 0
+    if modifiers:
+        modifiers_dict = {}
+        for modifier in modifiers:
+            regex, replacement = _extract_modifier_config(modifier, ctx)
+            modifiers_dict[regex] = replacement
+        repl_count += _replace_in_headers(headers, modifiers_dict)
+
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('_header_modify: replacing headers - replacements count: ' + repl_count)
+
+    return repl_count
+
+
+def _extract_modifier_config(modifier: bytes, ctx: dict) -> tuple:
+    modifier_obj = json.loads(modifier)
+
+    regex = _replace_modifier_values(modifier_obj['regex'], ctx)
+    replacement = _replace_modifier_values(modifier_obj['replacement'], ctx)
+
+    regex = re.compile(strutils.escaped_str_to_bytes(regex))
+    replacement = strutils.escaped_str_to_bytes(replacement)
+
+    return regex, replacement
+
+
+def _replace_modifier_values(s: str, ctx: dict) -> str:
+    # no loop over ctx currently to speed up as there's just 1 variable inside
+    s = s.replace('{{fqdn}}', re.escape(ctx['fqdn']))
+    return s
+
+
 def health_check_response(flow):
     # if bubble_log.isEnabledFor(DEBUG):
     #     bubble_log.debug('health_check_response: special bubble health check request, responding with OK')

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -16,7 +16,7 @@ from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATIO
     is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \
     CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \
     HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \
-    HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set
+    HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, response_header_modify
 from bubble_flex import process_flex
 
 import logging
@@ -311,6 +311,7 @@ def responseheaders(flow):
     else:
         flex_flow = None
     bubble_filter_response(flow, flex_flow)
+    response_header_modify(flow)
 
 
 def bubble_filter_response(flow, flex_flow):

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py

@@ -33,7 +33,7 @@ from bubble_api import bubble_matchers, bubble_activity_log, \
     CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \
     CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \
     is_bubble_special_path, is_bubble_health_check, health_check_response, tarpit_response,\
-    is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain
+    is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, update_host_and_port
 from bubble_config import bubble_host, bubble_host_alias
 from bubble_flex import new_flex_flow
 
@@ -114,34 +114,20 @@ class Rerouter:
     def bubble_handle_request(self, flow):
         client_addr = flow.client_conn.address[0]
         server_addr = flow.server_conn.address[0]
-        is_http = False
+        flow = update_host_and_port(flow)
+
         if flow.client_conn.tls_established:
-            flow.request.scheme = "https"
             sni = flow.client_conn.connection.get_servername()
-            port = 443
+            is_http = False
         else:
-            flow.request.scheme = "http"
             sni = None
-            port = 80
             is_http = True
 
-        # check if https and sni is missing but we have a host header, fill in the sni
-        
-        host_header = flow.request.host_header
-        if host_header:
-            m = parse_host_header.match(host_header)
-            if m:
-                host_header = m.group("host").strip("[]")
-                if m.group("port"):
-                    port = int(m.group("port"))
-
         # Determine if this request should be filtered
-        host = None
+        host_header = flow.request.host_header
+        host = flow.request.host
         path = flow.request.path
         if sni or host_header:
-            host = str(sni or host_header)
-            if host.startswith("b'"):
-                host = host[2:-1]
             log_url = flow.request.scheme + '://' + host + path
 
             # If https, we have already checked that the client/server are legal in bubble_conn_check.py
@@ -240,12 +226,6 @@ class Rerouter:
                 bubble_log.warning('bubble_handle_request: no sni/host found, not applying rules to path: ' + path)
             bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr])
 
-        flow.request.host_header = host_header
-        if host:
-            flow.request.host = host
-        else:
-            flow.request.host = host_header
-        flow.request.port = port
         return host
 
     def requestheaders(self, flow):