Add request protector app with cross-domain cookies filtering (#58)

Add RequestProtector app to cheapest plan

Update comment with typo

Merge branch 'master' into kris/request_protector_app

Merge branch 'master' into kris/request_protector_app

Merge branch 'kris/request_protector_app' of git.bubblev.org:bubblev/bubble into kris/request_protector_app

Extract method for updating requests host and port

Fix header replacement

Replace all cross-domain cookies with empty

Add fqdn variable support in header replacements

Merge branch 'master' into kris/request_protector_app

Merge branch 'master' into kris/request_protector_app

Merge branch 'master' into kris/request_protector_app

Merge branch 'master' into kris/request_protector_app

Add add header replacement button label

Try to add initial header replacement for cross-domain cookies

Merge branch 'master' into kris/request_protector_app

Merge branch 'master' into kris/request_protector_app

use special header replacement to skip emptied headers

Set RequestProtector replacement optional again

Fix replacement reference in RequestProtector app

Update flex domains with empty set if needed

Add back request heades modifiers as prime app

Extract and refactor _primeApp method

Merge branch 'master' into kris/request_protector_app

# Conflicts - WIP:
#	bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java

Use HeaderReplacement's id field in JSONs

Make RequestProtector's replacement field required

Add new app to some plans

Set new app to have `app` presentation

Add RuleDriver and AppMatcher for the new app

Add request protector app

Remove not used filter

Merge branch 'master' into kris/request_protector_app

Add full support for response header modification

Merge branch 'master' into kris/request_protector_app

# Conflicts:
#	utils/cobbzilla-utils

Add RequestProtector app

Co-authored-by: jonathan <jonathan@noreply.git.bubblev.org>
Co-authored-by: Kristijan Mitrovic <kmitrovic@itekako.com>
Reviewed-on: #58

bubble-server/src/main/java/bubble/app/request/RequestProtectorAppConfigDriver.java

@@ -0,0 +1,115 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.app.request;
+
+import bubble.model.account.Account;
+import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
+import bubble.model.app.config.AppConfigDriverBase;
+import bubble.rule.request.HeaderReplacement;
+import bubble.rule.request.RequestProtectorConfig;
+import bubble.rule.request.RequestProtectorRuleDriver;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Map;
+import java.util.Set;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
+import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
+
+@Slf4j
+public class RequestProtectorAppConfigDriver extends AppConfigDriverBase {
+
+    public static final String VIEW_manageHeaderReplacements = "manageHeaderReplacements";
+
+    @Override public Object getView(Account account, BubbleApp app, String view, Map<String, String> params) {
+        switch (view) {
+            case VIEW_manageHeaderReplacements:
+                return loadManageCookiesReplacements(account, app);
+        }
+        throw notFoundEx(view);
+    }
+
+    private Set<HeaderReplacement> loadManageCookiesReplacements(Account account, BubbleApp app) {
+        final RequestProtectorConfig config = getConfig(account, app);
+        return config.getHeaderReplacements();
+    }
+
+    private RequestProtectorConfig getConfig(Account account, BubbleApp app) {
+        return getConfig(account, app, RequestProtectorRuleDriver.class, RequestProtectorConfig.class);
+    }
+
+    public static final String ACTION_addHeaderReplacement = "addHeaderReplacement";
+    public static final String ACTION_removeHeaderReplacement = "removeHeaderReplacement";
+
+    public static final String PARAM_REGEX = "regex";
+    public static final String PARAM_REPLACEMENT = "replacement";
+
+    @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action,
+                                          Map<String, String> params, JsonNode data) {
+        switch (action) {
+            case ACTION_addHeaderReplacement:
+                return addHeaderReplacement(account, app, data);
+        }
+        if (log.isWarnEnabled()) log.warn("takeAppAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private Set<HeaderReplacement> addHeaderReplacement(Account account, BubbleApp app, JsonNode data) {
+        final JsonNode regexNode = data.get(PARAM_REGEX);
+        if (regexNode == null || regexNode.textValue() == null) {
+            throw invalidEx("err.requestProtector.headerRegexRequired");
+        }
+        final String regex = regexNode.textValue().trim();
+        if (empty(regex)) throw invalidEx("err.requestProtector.headerRegexRequired");
+
+        final JsonNode replacementNode = data.get(PARAM_REPLACEMENT);
+        final String replacement = (replacementNode == null || replacementNode.textValue() == null)
+                                   ? "" : replacementNode.textValue().trim();
+
+        final RequestProtectorConfig config = getConfig(account, app).addHeaderReplacement(regex, replacement);
+
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver
+        if (log.isDebugEnabled()) {
+            log.debug("addHeaderReplacement: updating rule: " + rule.getName() + ", adding regex: " + regex);
+        }
+        ruleDAO.update(rule.setConfigJson(json(config)));
+
+        return config.getHeaderReplacements();
+    }
+
+    @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id,
+                                           Map<String, String> params, JsonNode data) {
+        switch (action) {
+            case ACTION_removeHeaderReplacement:
+                return removeHeaderReplacement(account, app, id);
+        }
+        if (log.isWarnEnabled()) log.warn("takeItemAction: action not found: "+action);
+        throw notFoundEx(action);
+    }
+
+    private Set<HeaderReplacement> removeHeaderReplacement(Account account, BubbleApp app, String regex) {
+        final AppRule rule = loadRule(account, app);
+        loadDriver(account, rule, RequestProtectorRuleDriver.class); // validate proper driver
+        final RequestProtectorConfig config = getConfig(account, app);
+        if (log.isDebugEnabled()) {
+            log.debug("removeHeaderReplacement: removing regex: " + regex + " from config.cookiesReplacements: "
+                      + config.getHeaderReplacements().toString());
+        }
+
+        final RequestProtectorConfig updated = config.removeHeaderReplacement(regex);
+        if (log.isDebugEnabled()) {
+            log.debug("removeHeaderReplacement: updated.cookiesReplacements: "
+                      + updated.getHeaderReplacements().toString());
+        }
+        ruleDAO.update(rule.setConfigJson(json(updated)));
+
+        return updated.getHeaderReplacements();
+    }
+}

bubble-server/src/main/java/bubble/app/request/RequestProtectorAppDataDriver.java

@@ -0,0 +1,9 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.app.request;
+
+import bubble.model.app.config.AppDataDriverBase;
+
+public class RequestProtectorAppDataDriver extends AppDataDriverBase {}

bubble-server/src/main/java/bubble/rule/AppRuleDriver.java

@@ -47,6 +47,7 @@ public interface AppRuleDriver {
     String REDIS_FILTER_LISTS = "filterLists";
     String REDIS_FLEX_LISTS = "flexLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
     String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists";  // used in mitmproxy and dnscrypt-proxy for flex routing
+    String REDIS_RESPONSE_HEADER_MODIFIER_LISTS = "responseHeaderModifierLists";  // used in mitmproxy
     String REDIS_LIST_SUFFIX = "~UNION";
 
     default Set<String> getPrimedRejectDomains () { return null; }
@@ -55,6 +56,7 @@ public interface AppRuleDriver {
     default Set<String> getPrimedFilterDomains () { return null; }
     default Set<String> getPrimedFlexDomains () { return null; }
     default Set<String> getPrimedFlexExcludeDomains () { return null; }
+    default Set<String> getPrimedResponseHeaderModifiers () { return null; }
 
     static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) {
         defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains);
@@ -80,12 +82,22 @@ public interface AppRuleDriver {
         defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains);
     }
 
-    static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) {
+    static void defineRedisResponseHeaderModifiersSet(RedisService redis, String ip, String list,
+                                                      String[] modifiers) {
+        defineRedisSet(redis, ip, REDIS_RESPONSE_HEADER_MODIFIER_LISTS, list, modifiers);
+    }
+
+    /**
+     * `settings` parameter may be list of domains or any other list of strings - i.e. list of JSONs with specific setup
+     * for the prime option of the driver.
+     */
+    static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName,
+                               String[] settings) {
         final String listOfListsForIp = listOfListsName + "~" + ip;
         final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX;
         final String ipList = listOfListsForIp + "~" + listName;
         final String tempList = ipList + "~"+now()+randomAlphanumeric(5);
-        redis.sadd_plaintext(tempList, domains);
+        redis.sadd_plaintext(tempList, settings);
         redis.rename(tempList, ipList);
         redis.sadd_plaintext(listOfListsForIp, ipList);
         final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp));

bubble-server/src/main/java/bubble/rule/request/HeaderReplacement.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.request;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class HeaderReplacement implements Comparable<HeaderReplacement> {
+
+    public String getId() { return regex; }
+    public void setId(String id) {} // noop
+
+    @Getter @Setter private String regex;
+    @Getter @Setter private String replacement;
+
+    public HeaderReplacement(@NonNull final String regex, @NonNull final String replacement) {
+        this.regex = regex;
+        this.replacement = replacement;
+    }
+
+    @Override public int compareTo(@NonNull final HeaderReplacement o) {
+        return getRegex().compareTo(o.getRegex().toLowerCase());
+    }
+}

bubble-server/src/main/java/bubble/rule/request/RequestProtectorConfig.java

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.request;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+
+@Slf4j @Accessors(chain=true)
+public class RequestProtectorConfig {
+
+    @Getter @Setter private Set<HeaderReplacement> headerReplacements = new TreeSet<>();
+    public boolean hasHeaderReplacements() { return !empty(headerReplacements); }
+
+    @NonNull public RequestProtectorConfig addHeaderReplacement(@NonNull final String regex,
+                                                                @NonNull final String replacement) {
+        headerReplacements.add(new HeaderReplacement(regex, replacement));
+        return this;
+    }
+
+    @NonNull public RequestProtectorConfig removeHeaderReplacement(@NonNull final String regex) {
+        if (hasHeaderReplacements()) headerReplacements.removeIf(r -> r.getRegex().equals(regex));
+        return this;
+    }
+}

bubble-server/src/main/java/bubble/rule/request/RequestProtectorRuleDriver.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.rule.request;
+
+import bubble.model.account.Account;
+import bubble.model.app.AppMatcher;
+import bubble.model.app.AppRule;
+import bubble.model.app.BubbleApp;
+import bubble.model.device.Device;
+import bubble.rule.AbstractAppRuleDriver;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.json.JsonUtil;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class RequestProtectorRuleDriver extends AbstractAppRuleDriver {
+
+    @Override public <C> Class<C> getConfigClass() { return (Class<C>) RequestProtectorConfig.class; }
+
+    @Override public Set<String> getPrimedResponseHeaderModifiers() {
+        final RequestProtectorConfig config = getRuleConfig();
+        return config.getHeaderReplacements().stream().map(JsonUtil::json).collect(Collectors.toSet());
+    }
+
+    @Override public void init(JsonNode config, JsonNode userConfig, BubbleApp app, AppRule rule, AppMatcher matcher,
+                               Account account, Device device) {
+        super.init(config, userConfig, app, rule, matcher, account, device);
+
+        // refresh list
+        final RequestProtectorConfig ruleConfig = getRuleConfig();
+        ruleConfig.getHeaderReplacements();
+    }
+}

bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java

@@ -16,14 +16,19 @@ import bubble.server.BubbleConfiguration;
 import bubble.service.device.DeviceService;
 import bubble.service.device.StandardFlexRouterService;
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.collection.SingletonList;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.util.*;
+import javax.annotation.Nullable;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool;
@@ -110,130 +115,185 @@ public class StandardAppPrimerService implements AppPrimerService {
         getPrimerThread().submit(() -> _prime(account, singleApp));
     }
 
-    private synchronized void _prime(Account account, BubbleApp singleApp) {
+    private synchronized void _prime(@NonNull final Account account, @Nullable final BubbleApp singleApp) {
         try {
-            final Map<String, List<String>> accountDeviceIps = new HashMap<>();
             final List<Device> devices = deviceDAO.findByAccount(account.getUuid());
-            for (Device device : devices) {
-                accountDeviceIps.put(device.getUuid(), deviceService.findIpsByDevice(device.getUuid()));
-            }
-            if (accountDeviceIps.isEmpty()) return;
+            if (devices.isEmpty()) return;
+
+            final Map<String, List<String>> accountDeviceIps =
+                    devices.stream()
+                           .map(Device::getUuid)
+                           .collect(Collectors.toMap(Function.identity(), deviceService::findIpsByDevice));
 
             // flex domains can only be managed by the first admin
             final Account firstAdmin = accountDAO.getFirstAdmin();
             account.setFirstAdmin(account.getUuid().equals(firstAdmin.getUuid()));
-            boolean updateFlexRouters = false;
-            Set<String> flexDomains = null;
-            Set<String> flexExcludeDomains = null;
-
-            final List<BubbleApp> appsToPrime = singleApp == null
-                    ? appDAO.findByAccount(account.getUuid()).stream()
-                    .filter(BubbleApp::canPrime)
-                    .collect(Collectors.toList())
-                    : new SingletonList<>(singleApp);
-            for (BubbleApp app : appsToPrime) {
-                log.info("_prime: priming app: "+app.getUuid()+"/"+app.getName());
-                final List<AppRule> rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
-                final List<AppMatcher> matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
-                for (AppRule rule : rules) {
-                    final RuleDriver driver = driverDAO.findByUuid(rule.getDriver());
-                    if (driver == null) {
-                        log.warn("_prime: driver not found for app/rule " + app.getName() + "/" + rule.getName() + ": " + rule.getDriver());
-                        continue;
+
+            if (singleApp != null) {
+                _primeApp(account, accountDeviceIps, devices, singleApp);
+            } else {
+                appDAO.findByAccount(account.getUuid())
+                      .stream()
+                      .filter(BubbleApp::canPrime)
+                      .forEach(app -> _primeApp(account, accountDeviceIps, devices, app));
+            }
+        } catch (Exception e) {
+            die("_prime: " + shortError(e), e);
+        } finally {
+            log.info("_prime: completed");
+        }
+    }
+
+    private void _primeApp(@NonNull final Account account, @NonNull final Map<String, List<String>> accountDeviceIps,
+                           @NonNull final List<Device> devices, @NonNull final BubbleApp app) {
+        log.info("_primeApp: " + app.getUuid() + "/" + app.getName());
+
+        final List<AppRule> rules = ruleDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
+        final List<AppMatcher> matchers = matcherDAO.findByAccountAndApp(account.getUuid(), app.getUuid());
+
+        boolean updateFlexRouters = false;
+        Set<String> flexDomains = null;
+        for (AppRule rule : rules) {
+            final RuleDriver driver = driverDAO.findByUuid(rule.getDriver());
+            if (driver == null) {
+                log.warn("_primeApp: driver not found for app/rule "
+                         + app.getName() + "/" + rule.getName() + ": " + rule.getDriver());
+                continue;
+            }
+
+            // handle AppData callback registration with a basic driver
+            final AppRuleDriver cbDriver = driver.getDriver();
+            if (cbDriver instanceof HasAppDataCallback) {
+                log.debug("_primeApp: AppRuleDriver (" + cbDriver.getClass().getSimpleName()
+                          + ") implements HasAppDataCallback, registering: " + app.getUuid() + "/" + app.getName());
+                final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver;
+                dataCallback.prime(account, app, configuration);
+                dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration));
+            }
+
+            for (final Device device : devices) {
+                final Set<String> rejectDomains = new HashSet<>();
+                final Set<String> blockDomains = new HashSet<>();
+                final Set<String> whiteListDomains = new HashSet<>();
+                final Set<String> filterDomains = new HashSet<>();
+                final Set<String> flexExcludeDomains = new HashSet<>();
+                final Set<String> requestHeaderModifiers = new HashSet<>();
+
+                boolean areAllSetsEmpty = true;
+                for (AppMatcher matcher : matchers) {
+                    final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device);
+
+                    final Set<String> rejects = appRuleDriver.getPrimedRejectDomains();
+                    if (empty(rejects)) {
+                        log.debug("_primeApp: no rejectDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        rejectDomains.addAll(rejects);
+                        areAllSetsEmpty = empty(rejects);
                     }
-                    // handle AppData callback registration with a basic driver
-                    final AppRuleDriver cbDriver = driver.getDriver();
-                    if (cbDriver instanceof HasAppDataCallback) {
-                        log.debug("_prime: AppRuleDriver ("+cbDriver.getClass().getSimpleName()+") implements HasAppDataCallback, registering: "+app.getUuid()+"/"+app.getName());
-                        final HasAppDataCallback dataCallback = (HasAppDataCallback) cbDriver;
-                        dataCallback.prime(account, app, configuration);
-                        dataDAO.registerCallback(app.getUuid(), dataCallback.createCallback(account, app, configuration));
+
+                    final Set<String> blocks = appRuleDriver.getPrimedBlockDomains();
+                    if (empty(blocks)) {
+                        log.debug("_primeApp: no blockDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        blockDomains.addAll(blocks);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(blocks);
+                    }
+
+                    final Set<String> whiteList = appRuleDriver.getPrimedWhiteListDomains();
+                    if (empty(whiteList)) {
+                        log.debug("_primeApp: no whiteListDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        whiteListDomains.addAll(whiteList);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(whiteList);
                     }
-                    for (Device device : devices) {
-                        final Set<String> rejectDomains = new HashSet<>();
-                        final Set<String> blockDomains = new HashSet<>();
-                        final Set<String> whiteListDomains = new HashSet<>();
-                        final Set<String> filterDomains = new HashSet<>();
-                        for (AppMatcher matcher : matchers) {
-                            final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device);
-                            final Set<String> rejects = appRuleDriver.getPrimedRejectDomains();
-                            if (empty(rejects)) {
-                                log.debug("_prime: no rejectDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                rejectDomains.addAll(rejects);
-                            }
-                            final Set<String> blocks = appRuleDriver.getPrimedBlockDomains();
-                            if (empty(blocks)) {
-                                log.debug("_prime: no blockDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                blockDomains.addAll(blocks);
-                            }
-                            final Set<String> whiteList = appRuleDriver.getPrimedWhiteListDomains();
-                            if (empty(whiteList)) {
-                                log.debug("_prime: no whiteListDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                whiteListDomains.addAll(whiteList);
-                            }
-                            final Set<String> filters = appRuleDriver.getPrimedFilterDomains();
-                            if (empty(filters)) {
-                                log.debug("_prime: no filterDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                            } else {
-                                filterDomains.addAll(filters);
-                            }
-                            if (account.isFirstAdmin() && flexDomains == null) {
-                                final Set<String> flexes = appRuleDriver.getPrimedFlexDomains();
-                                if (empty(flexes)) {
-                                    log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                                } else {
-                                    flexDomains = new HashSet<>(flexes);
-                                }
-                                final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
-                                if (empty(flexExcludes)) {
-                                    log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
-                                } else {
-                                    flexExcludeDomains = new HashSet<>(flexExcludes);
-                                }
-                            }
+
+                    final Set<String> filters = appRuleDriver.getPrimedFilterDomains();
+                    if (empty(filters)) {
+                        log.debug("_primeApp: no filterDomains for device/app/rule/matcher: " + device.getName()
+                                  + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                    } else {
+                        filterDomains.addAll(filters);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(filters);
+                    }
+
+                    final Set<String> modifiers = appRuleDriver.getPrimedResponseHeaderModifiers();
+                    if (empty(modifiers)) {
+                        log.debug("_primeApp: no responseHeaderModifiers for device/app/rule/matcher: "
+                                  + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/"
+                                  + matcher.getName());
+                    } else {
+                        requestHeaderModifiers.addAll(modifiers);
+                        areAllSetsEmpty = areAllSetsEmpty && empty(modifiers);
+                    }
+
+                    if (account.isFirstAdmin() && flexDomains == null) {
+                        final Set<String> flexes = appRuleDriver.getPrimedFlexDomains();
+                        if (empty(flexes)) {
+                            log.debug("_primeApp: no flexDomains for device/app/rule/matcher: " + device.getName()
+                                      + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName());
+                        } else {
+                            flexDomains = new HashSet<>(flexes);
+                            areAllSetsEmpty = areAllSetsEmpty && empty(flexes);
                         }
-                        if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) {
-                            for (String ip : accountDeviceIps.get(device.getUuid())) {
-                                if (!empty(rejectDomains)) {
-                                    rejectDomains.removeAll(whiteListDomains);
-                                    AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new));
-                                }
-                                if (!empty(blockDomains)) {
-                                    blockDomains.removeAll(whiteListDomains);
-                                    AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new));
-                                }
-                                if (!empty(whiteListDomains)) {
-                                    AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), whiteListDomains.toArray(String[]::new));
-                                }
-                                if (!empty(filterDomains)) {
-                                    AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new));
-                                }
-                                if (account.isFirstAdmin() && (!empty(flexDomains) || !empty(flexExcludeDomains))) {
-                                    updateFlexRouters = true;
-                                    if (!empty(flexDomains)) {
-                                        if (flexExcludeDomains != null) flexDomains.removeAll(flexExcludeDomains);
-                                        AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new));
-                                    }
-                                    if (!empty(flexExcludeDomains)) {
-                                        AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new));
-                                    }
-                                }
-                            }
+
+                        final Set<String> flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains();
+                        if (empty(flexExcludes)) {
+                            log.debug("_primeApp: no flexExcludeDomains for device/app/rule/matcher: "
+                                      + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/"
+                                      + matcher.getName());
+                        } else {
+                            flexExcludeDomains.addAll(flexExcludes);
+                            areAllSetsEmpty = areAllSetsEmpty && empty(flexExcludes);
                         }
                     }
                 }
-                if (updateFlexRouters && !empty(flexDomains)) {
-                    flexRouterService.updateFlexRoutes(flexDomains);
+
+                if (areAllSetsEmpty) continue;
+
+                for (final String ip : accountDeviceIps.get(device.getUuid())) {
+                    if (!empty(rejectDomains)) {
+                        rejectDomains.removeAll(whiteListDomains);
+                        AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                           rejectDomains.toArray(String[]::new));
+                    }
+                    if (!empty(blockDomains)) {
+                        blockDomains.removeAll(whiteListDomains);
+                        AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                          blockDomains.toArray(String[]::new));
+                    }
+                    if (!empty(whiteListDomains)) {
+                        AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                              whiteListDomains.toArray(String[]::new));
+                    }
+                    if (!empty(filterDomains)) {
+                        AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                           filterDomains.toArray(String[]::new));
+                    }
+                    if (!empty(requestHeaderModifiers)) {
+                        AppRuleDriver.defineRedisResponseHeaderModifiersSet(
+                                redis, ip, app.getName() + ":" + app.getUuid(),
+                                requestHeaderModifiers.toArray(String[]::new));
+                    }
+                    if (account.isFirstAdmin()) {
+                        if (!empty(flexDomains)) {
+                            if (!empty(flexExcludeDomains)) flexDomains.removeAll(flexExcludeDomains);
+                            AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                             flexDomains.toArray(String[]::new));
+                            updateFlexRouters = true;
+                        }
+                        if (!empty(flexExcludeDomains)) {
+                            AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(),
+                                                                    flexExcludeDomains.toArray(String[]::new));
+                        }
+                    }
                 }
             }
-        } catch (Exception e) {
-            die("_prime: "+shortError(e), e);
-        } finally {
-            log.info("_primeApps: completed");
         }
+
+        if (updateFlexRouters) flexRouterService.updateFlexRoutes(flexDomains);
     }
 
 }

bubble-server/src/main/resources/models/apps/request/bubbleApp_request.json

@@ -0,0 +1,81 @@
+[{
+  "name": "RequestProtector",
+  "description": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response",
+  "url": "https://getbubblenow.com/apps/request",
+  "template": true,
+  "enabled": true,
+  "priority": 1000,
+  "canPrime": true,
+  "dataConfig": {
+    "dataDriver": "bubble.app.request.RequestProtectorAppDataDriver",
+    "presentation": "app",
+    "configDriver": "bubble.app.request.RequestProtectorAppConfigDriver",
+    "configFields": [
+      {"name": "regex", "truncate": false},
+      {"name": "replacement", "truncate": false}
+    ],
+    "configViews": [{
+      "name": "manageHeaderReplacements",
+      "scope": "app",
+      "root": "true",
+      "fields": [ "regex", "replacement" ],
+      "actions": [
+        {"name": "removeHeaderReplacement", "index": 10},
+        {
+          "name": "addHeaderReplacement", "scope": "app", "index": 10,
+          "params": [ "regex", "replacement" ],
+          "button": "addHeaderReplacement"
+        }
+      ]
+    }]
+  },
+  "children": {
+    "AppSite": [{
+      "name": "All_Sites",
+      "url": "*",
+      "description": "All websites",
+      "template": true
+    }],
+    "AppRule": [{
+      "name": "request",
+      "template": true,
+      "driver": "RequestProtectorRuleDriver",
+      "priority": -1000,
+      "config": {
+        "headerReplacements": [{
+          "regex": "^(?i:Set-Cookie):(.*;)?\\s*(?i:Domain)=(((([\\*\\.].*)|(.*\\.))\\s*(;.*)?)|((?!([^;,]*\\.)?{{fqdn}}\\s*(;|$)).*))$",
+          "replacement": ""
+        }]
+      }
+    }],
+    "AppMessage": [{
+      "locale": "en_US",
+      "messages": [
+        { "name": "name", "value": "RequestProtector" },
+        { "name": "icon", "value": "classpath:models/apps/request/request-icon.svg" },
+        { "name": "summary", "value": "Request Protector" },
+        {
+          "name": "description",
+          "value": "Change or remove parts of request/response - i.e. remove cross-domain cookies from response"
+        },
+
+        { "name": "config.view.manageHeaderReplacements", "value": "Manage Header Replacements" },
+        { "name": "config.field.regex", "value": "RegEx" },
+        {
+          "name": "config.field.regex.description",
+          "value": "Regular expression compared with full header's line string value"
+        },
+        { "name": "config.field.replacement", "value": "Replacement" },
+        {
+          "name": "config.field.replacement.description",
+          "value": "May use reference from regex as in python's re.Pattern.sub method. If set to empty string, found header will be fully removed from response"
+        },
+        { "name": "config.button.addHeaderReplacement", "value": "Add" },
+        { "name": "config.action.addHeaderReplacement", "value": "Add New Header Replacement" },
+        { "name": "config.action.removeHeaderReplacement", "value": "Remove" },
+
+        { "name": "err.requestProtector.headerRegexRequired", "value": "RegEx field is required" }
+      ]
+    }]
+  }
+}]

bubble-server/src/main/resources/models/apps/request/bubbleApp_request_matchers.json

@@ -0,0 +1,15 @@
+[{
+  "name": "RequestProtector",
+  "children": {
+    "AppMatcher": [{
+      "name": "RequestProtectorMatcher",
+      "template": true,
+      "connCheck": true,
+      "site": "All_Sites",
+      "fqdn": "*",
+      "urlRegex": ".*",
+      "rule": "request",
+      "priority": -1000000
+    }]
+  }
+}]

bubble-server/src/main/resources/models/apps/request/request-icon.svg

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   version="1.1"
+   width="80"
+   height="80"
+   id="svg10029"
+   sodipodi:docname="Hazard light icon.svg"
+   inkscape:version="0.92.4 (5da689c313, 2019-01-14)">
+  <metadata
+     id="metadata9">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1680"
+     inkscape:window-height="998"
+     id="namedview7"
+     showgrid="true"
+     showguides="true"
+     inkscape:guide-bbox="true"
+     inkscape:snap-page="true"
+     inkscape:zoom="2.085965"
+     inkscape:cx="-18.750976"
+     inkscape:cy="1.4877119"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g10556">
+    <sodipodi:guide
+       position="-143,40"
+       orientation="0,1"
+       id="guide819"
+       inkscape:locked="false" />
+    <inkscape:grid
+       type="xygrid"
+       id="grid823" />
+    <sodipodi:guide
+       position="40,55"
+       orientation="1,0"
+       id="guide4529"
+       inkscape:locked="false" />
+  </sodipodi:namedview>
+  <defs
+     id="defs10032" />
+  <g
+     transform="translate(-19.885022,2.1230082)"
+     id="g10556">
+    <path
+       sodipodi:type="star"
+       style="fill:none;fill-opacity:1;stroke:#ff0000;stroke-width:10;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none"
+       id="path4531"
+       sodipodi:sides="3"
+       sodipodi:cx="59.885021"
+       sodipodi:cy="47.876992"
+       sodipodi:r1="40"
+       sodipodi:r2="20"
+       sodipodi:arg1="-1.5707963"
+       sodipodi:arg2="-0.52359878"
+       inkscape:flatsided="false"
+       inkscape:rounded="0"
+       inkscape:randomized="0"
+       d="m 59.885022,7.8769913 17.320507,29.9999997 17.320508,30.000001 -34.641016,-10e-7 -34.641016,-10e-7 17.320508,-29.999999 z"
+       inkscape:transform-center-y="-10" />
+  </g>
+</svg>

bubble-server/src/main/resources/models/defaults/bubblePlan.json

@@ -14,7 +14,8 @@
     "children": {
       "BubblePlanApp": [
         {"app": "BubbleBlock"},
-        {"app": "TlsPassthru"}
+        {"app": "TlsPassthru"},
+        {"app": "RequestProtector"}
       ]
     }
   },
@@ -36,7 +37,8 @@
         {"app": "TrafficAnalytics"},
         {"app": "BubbleBlock"},
         {"app": "UserBlocker"},
-        {"app": "TlsPassthru"}
+        {"app": "TlsPassthru"},
+        {"app": "RequestProtector"}
       ]
     }
   },
@@ -58,7 +60,8 @@
         {"app": "TrafficAnalytics"},
         {"app": "BubbleBlock"},
         {"app": "UserBlocker"},
-        {"app": "TlsPassthru"}
+        {"app": "TlsPassthru"},
+        {"app": "RequestProtector"}
       ]
     }
   }

bubble-server/src/main/resources/models/defaults/ruleDriver.json

@@ -28,5 +28,11 @@
     "driverClass": "bubble.rule.bblock.BubbleBlockRuleDriver",
     "template": true,
     "userConfig": { "fields": [] }
+  },
+  {
+    "name": "RequestProtectorRuleDriver",
+    "driverClass": "bubble.rule.request.RequestProtectorRuleDriver",
+    "template": true,
+    "userConfig": { "fields": [] }
   }
-]
+]

bubble-server/src/main/resources/models/manifest-app-request.json

@@ -0,0 +1,4 @@
+[
+  "apps/request/bubbleApp_request",
+  "apps/request/bubbleApp_request_matchers"
+]

bubble-server/src/main/resources/models/manifest-defaults.json

@@ -5,5 +5,6 @@
   "manifest-app-user-block",
   "manifest-app-bubble-block",
   "manifest-app-passthru",
+  "manifest-app-request",
   "defaults/bubblePlan"
-]
+]

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -25,6 +25,7 @@ from bubble_config import bubble_port, debug_capture_fqdn, \
 from mitmproxy import http
 from mitmproxy.net.http import headers as nheaders
 from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody
+from mitmproxy.utils import strutils
 
 bubble_log = logging.getLogger(__name__)
 
@@ -449,6 +450,131 @@ def original_flex_ip(client_addr, fqdns):
     return None
 
 
+def update_host_and_port(flow):
+    if flow.request:
+        if flow.client_conn.tls_established:
+            flow.request.scheme = "https"
+            sni = flow.client_conn.connection.get_servername()
+            port = 443
+        else:
+            flow.request.scheme = "http"
+            sni = None
+            port = 80
+
+        host_header = flow.request.host_header
+        if host_header:
+            m = parse_host_header.match(host_header)
+            if m:
+                host_header = m.group("host").strip("[]")
+                if m.group("port"):
+                    port = int(m.group("port"))
+
+        host = None
+        if sni or host_header:
+            host = str(sni or host_header)
+            if host.startswith("b'"):
+                host = host[2:-1]
+
+        flow.request.host_header = host_header
+        if host:
+            flow.request.host = host
+        else:
+            flow.request.host = host_header
+        flow.request.port = port
+
+    return flow
+
+
+def _replace_in_headers(headers: nheaders.Headers, modifiers_dict: dict) -> int:
+    """
+    Taken from original mitmproxy's Header class implementation with some changes.
+
+    Replaces a regular expression pattern with repl in each "name: value"
+    header line.
+
+    Returns:
+        The number of replacements made.
+    """
+    repl_count = 0
+    fields = []
+
+    for name, value in headers.fields:
+
+        line = name + b": " + value
+        inner_repl_count = 0
+        for pattern, replacement in modifiers_dict.items():
+            line, n = pattern.subn(replacement, line)
+            inner_repl_count += n
+            if len(line) == 0:
+                # No need to go though other patterns for this line
+                break
+
+        if len(line) == 0:
+            # Skip (remove) this header line in this case
+            break
+
+        if inner_repl_count > 0:
+            # only in case when there were some replacements:
+            try:
+                name, value = line.split(b": ", 1)
+            except ValueError:
+                # We get a ValueError if the replacement removed the ": "
+                # There's not much we can do about this, so we just keep the header as-is.
+                pass
+            else:
+                repl_count += inner_repl_count
+
+        fields.append((name, value))
+
+    headers.fields = tuple(fields)
+    return repl_count
+
+
+def response_header_modify(flow) -> int:
+    if flow.response is None:
+        return None
+
+    flow = update_host_and_port(flow)
+    ctx = {'fqdn': flow.request.host}
+    return _header_modify(flow.client_conn.address[0], ctx, flow.response.headers)
+
+
+def _header_modify(client_addr: str, ctx: dict, headers: nheaders.Headers) -> int:
+    modifiers_set = 'responseHeaderModifierLists~' + client_addr + '~UNION'
+    modifiers = REDIS.smembers(modifiers_set)
+
+    repl_count = 0
+    if modifiers:
+        modifiers_dict = {}
+        for modifier in modifiers:
+            regex, replacement = _extract_modifier_config(modifier, ctx)
+            modifiers_dict[regex] = replacement
+        repl_count += _replace_in_headers(headers, modifiers_dict)
+
+    if bubble_log.isEnabledFor(DEBUG):
+        bubble_log.debug('_header_modify: replacing headers - replacements count: ' + repl_count)
+
+    return repl_count
+
+
+def _extract_modifier_config(modifier: bytes, ctx: dict) -> tuple:
+    modifier_obj = json.loads(modifier)
+
+    regex = _replace_modifier_values(modifier_obj['regex'], ctx)
+    replacement = _replace_modifier_values(modifier_obj['replacement'], ctx)
+
+    regex = re.compile(strutils.escaped_str_to_bytes(regex))
+    replacement = strutils.escaped_str_to_bytes(replacement)
+
+    return regex, replacement
+
+
+def _replace_modifier_values(s: str, ctx: dict) -> str:
+    # no loop over ctx currently to speed up as there's just 1 variable inside
+    s = s.replace('{{fqdn}}', re.escape(ctx['fqdn']))
+    return s
+
+
 def health_check_response(flow):
     # if bubble_log.isEnabledFor(DEBUG):
     #     bubble_log.debug('health_check_response: special bubble health check request, responding with OK')

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -16,7 +16,7 @@ from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATIO
     is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \
     CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \
     HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \
-    HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set
+    HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, response_header_modify
 from bubble_flex import process_flex
 
 import logging
@@ -311,6 +311,7 @@ def responseheaders(flow):
     else:
         flex_flow = None
     bubble_filter_response(flow, flex_flow)
+    response_header_modify(flow)
 
 
 def bubble_filter_response(flow, flex_flow):

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py

@@ -33,7 +33,7 @@ from bubble_api import bubble_matchers, bubble_activity_log, \
     CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \
     CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \
     is_bubble_special_path, is_bubble_health_check, health_check_response, tarpit_response,\
-    is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain
+    is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, update_host_and_port
 from bubble_config import bubble_host, bubble_host_alias
 from bubble_flex import new_flex_flow
 
@@ -114,34 +114,20 @@ class Rerouter:
     def bubble_handle_request(self, flow):
         client_addr = flow.client_conn.address[0]
         server_addr = flow.server_conn.address[0]
-        is_http = False
+        flow = update_host_and_port(flow)
+
         if flow.client_conn.tls_established:
-            flow.request.scheme = "https"
             sni = flow.client_conn.connection.get_servername()
-            port = 443
+            is_http = False
         else:
-            flow.request.scheme = "http"
             sni = None
-            port = 80
             is_http = True
 
-        # check if https and sni is missing but we have a host header, fill in the sni
-        
-        host_header = flow.request.host_header
-        if host_header:
-            m = parse_host_header.match(host_header)
-            if m:
-                host_header = m.group("host").strip("[]")
-                if m.group("port"):
-                    port = int(m.group("port"))
-
         # Determine if this request should be filtered
-        host = None
+        host_header = flow.request.host_header
+        host = flow.request.host
         path = flow.request.path
         if sni or host_header:
-            host = str(sni or host_header)
-            if host.startswith("b'"):
-                host = host[2:-1]
             log_url = flow.request.scheme + '://' + host + path
 
             # If https, we have already checked that the client/server are legal in bubble_conn_check.py
@@ -240,12 +226,6 @@ class Rerouter:
                 bubble_log.warning('bubble_handle_request: no sni/host found, not applying rules to path: ' + path)
             bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr])
 
-        flow.request.host_header = host_header
-        if host:
-            flow.request.host = host
-        else:
-            flow.request.host = host_header
-        flow.request.port = port
         return host
 
     def requestheaders(self, flow):