improve support for pinned apps - detecting tls block will not switch to passthru, so webapp version of app still works

bin/rkeys

@@ -3,6 +3,6 @@
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
 KEY_MATCH="${1}"
-for k in $(echo 'keys *'"""${KEY_MATCH}"""'*' | redis-cli )  ; do
+for k in $(echo 'keys *'"""${KEY_MATCH}"""'*' | redis-cli)  ; do
   echo "$k => $(echo "get $k" | redis-cli)"
 done

bin/rmembers

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+KEY_MATCH="${1}"
+SEP="${2:- }"
+for k in $(echo 'keys *'"""${KEY_MATCH}"""'*' | redis-cli)  ; do
+  echo "$k => $(echo "smembers $k" | redis-cli | tr '\n' ''"${SEP}"'')"
+done

bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java

@@ -5,6 +5,7 @@
 package bubble.dao.app;
 
 import bubble.model.app.AppSite;
+import bubble.service.cloud.DeviceIdService;
 import bubble.service.stream.RuleEngineService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
@@ -13,15 +14,18 @@ import org.springframework.stereotype.Repository;
 public class AppSiteDAO extends AppTemplateEntityDAO<AppSite> {
 
     @Autowired private RuleEngineService ruleEngineService;
+    @Autowired private DeviceIdService deviceService;
 
     @Override public AppSite postCreate(AppSite site, Object context) {
         // todo: update entities based on this template if account has updates enabled
+        if (site.hasMaxSecurityHosts()) deviceService.initDeviceSecurityLevels();
         return super.postCreate(site, context);
     }
 
-    @Override public AppSite postUpdate(AppSite entity, Object context) {
+    @Override public AppSite postUpdate(AppSite site, Object context) {
         ruleEngineService.flushCaches();
-        return super.postUpdate(entity, context);
+        if (site.hasMaxSecurityHosts()) deviceService.initDeviceSecurityLevels();
+        return super.postUpdate(site, context);
     }
 
     @Override public void delete(String uuid) {

bubble-server/src/main/java/bubble/model/app/AppSite.java

@@ -5,6 +5,7 @@
 package bubble.model.app;
 
 import bubble.model.account.Account;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -17,8 +18,12 @@ import org.cobbzilla.wizard.model.entityconfig.annotations.*;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.Transient;
 
 import static bubble.ApiConstants.EP_SITES;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
+import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 
 @ECType(root=true)
@@ -36,7 +41,9 @@ import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 })
 public class AppSite extends IdentifiableBase implements AppTemplateEntity {
 
-    public static final String[] VALUE_FIELDS = {"template", "enabled", "description", "url"};
+    public static final String[] VALUE_FIELDS = {
+            "template", "enabled", "description", "url", "maxSecurityHosts", "enableMaxSecurityHosts"
+    };
     public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "app");
 
     public AppSite (AppSite other) { copy(this, other, CREATE_FIELDS); }
@@ -77,4 +84,17 @@ public class AppSite extends IdentifiableBase implements AppTemplateEntity {
     @Column(nullable=false, length=1024)
     @Getter @Setter private String url;
 
+    // Json array of hosts that should always get maximum security
+    @ECField(index=80)
+    @Column(length=5000)
+    @JsonIgnore @Getter @Setter private String maxSecurityHostsJson;
+
+    @Transient public String[] getMaxSecurityHosts () { return empty(maxSecurityHostsJson) ? null : json(maxSecurityHostsJson, String[].class); }
+    public AppSite setMaxSecurityHosts(String[] hosts) { return setMaxSecurityHostsJson(empty(hosts) ? null : json(hosts, COMPACT_MAPPER)); }
+    public boolean hasMaxSecurityHosts () { return !empty(getMaxSecurityHosts()); }
+
+    @ECField(index=90)
+    @Getter @Setter private Boolean enableMaxSecurityHosts;
+    public boolean enableMaxSecurityHosts() { return enableMaxSecurityHosts == null ? true : enableMaxSecurityHosts; }
+
 }

bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java

@@ -5,7 +5,9 @@
 package bubble.service.cloud;
 
 import bubble.dao.account.AccountDAO;
+import bubble.dao.app.AppSiteDAO;
 import bubble.dao.device.DeviceDAO;
+import bubble.model.app.AppSite;
 import bubble.model.device.Device;
 import bubble.model.device.DeviceStatus;
 import bubble.server.BubbleConfiguration;
@@ -44,11 +46,13 @@ public class StandardDeviceIdService implements DeviceIdService {
 
     // used in dnscrypt-proxy and mitmproxy to check device security level
     public static final String REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = "bubble_device_security_level_";
+    public static final String REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = "bubble_device_site_max_security_level_";
 
     @Autowired private DeviceDAO deviceDAO;
     @Autowired private AccountDAO accountDAO;
     @Autowired private RedisService redis;
     @Autowired private GeoService geoService;
+    @Autowired private AppSiteDAO siteDAO;
     @Autowired private BubbleConfiguration configuration;
 
     private final Map<String, Device> deviceCache = new ExpirationMap<>(MINUTES.toMillis(10));
@@ -123,6 +127,17 @@ public class StandardDeviceIdService implements DeviceIdService {
         if (configuration.testMode()) return;
         for (String ip : findIpsByDevice(device.getUuid())) {
             redis.set_plaintext(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+ip, device.getSecurityLevel().name());
+
+            for (AppSite site : siteDAO.findByAccount(device.getAccount())) {
+                if (site.hasMaxSecurityHosts()) {
+                    final String siteKey = REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX + ip;
+                    if (site.enableMaxSecurityHosts()) {
+                        redis.sadd_plaintext(siteKey, site.getMaxSecurityHosts());
+                    } else {
+                        redis.srem(siteKey, site.getMaxSecurityHosts());
+                    }
+                }
+            }
         }
     }
 

bubble-server/src/main/resources/ansible/bubble_scripts.txt

@@ -17,4 +17,5 @@ list_bubble_databases
 cleanup_bubble_databases
 install_packer.sh
 rkeys
+rmembers
 rdelkeys

bubble-server/src/main/resources/db/migration/V2020072301__add_appsite_max_security_hosts.sql

@@ -0,0 +1,4 @@
+ALTER TABLE app_site ADD COLUMN max_security_hosts_json VARCHAR(5000);
+ALTER TABLE app_site ADD COLUMN enable_max_security_hosts BOOLEAN;
+UPDATE app_site SET max_security_hosts_json = '["twitter.com","*.twitter.com","*.twimg.com","t.co"]' WHERE name = 'Twitter';
+UPDATE app_site SET enable_max_security_hosts = true WHERE name = 'Twitter';

bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter.json

@@ -5,7 +5,9 @@
       "name": "Twitter",
       "url": "https://twitter.com",
       "description": "what’s happening in the world and what people are talking about right now.",
-      "template": true
+      "template": true,
+      "maxSecurityHosts": ["twitter.com", "*.twitter.com", "*.twimg.com", "t.co"],
+      "enableMaxSecurityHosts": true
     }],
     "AppRule": [{
       "name": "twitter_user_blocker",

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -59,9 +59,11 @@ def bubble_activity_log(client_addr, server_addr, event, data):
 
 
 def bubble_conn_check(remote_addr, addr, fqdns, security_level):
-    if debug_capture_fqdn and fqdns and debug_capture_fqdn in fqdns:
-        bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+debug_capture_fqdn)
-        return 'noop'
+    if debug_capture_fqdn and fqdns:
+        for f in debug_capture_fqdn:
+            if f in fqdns:
+                bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f)
+                return 'noop'
 
     headers = {
         'X-Forwarded-For': remote_addr,
@@ -83,7 +85,7 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level):
     except Exception as e:
         bubble_log('bubble_conn_check API call failed: '+repr(e))
         traceback.print_exc()
-        if security_level is not None and security_level == 'maximum':
+        if security_level is not None and security_level['level'] == 'maximum':
             return False
         return None
 
@@ -100,8 +102,8 @@ DEBUG_MATCHER = {
 }
 
 def bubble_matchers(req_id, remote_addr, flow, host):
-    if debug_capture_fqdn and host and debug_capture_fqdn == host:
-        bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+debug_capture_fqdn)
+    if debug_capture_fqdn and host and host in debug_capture_fqdn:
+        bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host)
         return DEBUG_MATCHER
 
     headers = {

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py

@@ -40,6 +40,7 @@ REDIS_DNS_PREFIX = 'bubble_dns_'
 REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_'
 REDIS_CHECK_DURATION = 60 * 60  # 1 hour timeout
 REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_'  # defined in StandardDeviceIdService
+REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = 'bubble_device_site_max_security_level_'  # defined in StandardDeviceIdService
 
 FORCE_PASSTHRU = {'passthru': True}
 FORCE_BLOCK = {'block': True}
@@ -56,11 +57,24 @@ VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4)
 VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6)
 
 
-def get_device_security_level(client_addr):
+def get_device_security_level(client_addr, fqdns):
     level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr)
     if level is None:
-        return SEC_MAX
-    return level.decode()
+        return {'level': SEC_MAX}
+    level = level.decode()
+    if level == SEC_STD:
+        bubble_log('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns))
+        if fqdns:
+            max_required_fqdns = REDIS.smembers(REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX+client_addr)
+            if max_required_fqdns is not None:
+                bubble_log('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns))
+                for max_required in max_required_fqdns:
+                    max_required = max_required.decode()
+                    for fqdn in fqdns:
+                        if max_required == fqdn or (max_required.startswith('*.') and fqdn.endswith(max_required[1:])):
+                            bubble_log('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required)
+                            return {'level': SEC_MAX, 'pinned': True}
+    return {'level': level}
 
 
 def get_local_ips():
@@ -115,7 +129,7 @@ class TlsFeedback(TlsLayer):
     def _establish_tls_with_client(self):
         client_address = self.client_conn.address[0]
         server_address = self.server_conn.address[0]
-        security_level = get_device_security_level(client_address)
+        security_level = self.security_level
         try:
             super(TlsFeedback, self)._establish_tls_with_client()
 
@@ -128,15 +142,19 @@ class TlsFeedback(TlsLayer):
             elif self.fqdns is not None and len(self.fqdns) > 0:
                 for fqdn in self.fqdns:
                     cache_key = conn_check_cache_prefix(client_address, fqdn)
-                    if security_level == SEC_MAX:
-                        redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
-                        bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
+                    if security_level['level'] == SEC_MAX:
+                        if 'pinned' in security_level and security_level['pinned']:
+                            redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': False, 'reason': 'tls_failure_pinned'}), ex=REDIS_CHECK_DURATION)
+                            bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
+                        else:
+                            redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
+                            bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
                     else:
                         redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
                         bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e))
             else:
                 cache_key = conn_check_cache_prefix(client_address, server_address)
-                if security_level == SEC_MAX:
+                if security_level['level'] == SEC_MAX:
                     redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION)
                     bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e))
                 else:
@@ -148,7 +166,7 @@ class TlsFeedback(TlsLayer):
 def check_bubble_connection(client_addr, server_addr, fqdns, security_level):
     check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level)
     if check_response is None or check_response == 'error':
-        if security_level == SEC_MAX:
+        if security_level['level'] == SEC_MAX:
             bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block')
             return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'}
         else:
@@ -205,10 +223,11 @@ def next_layer(next_layer):
             bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns))
         next_layer.fqdns = fqdns
         no_fqdns = fqdns is None or len(fqdns) == 0
-        security_level = get_device_security_level(client_addr)
+        security_level = get_device_security_level(client_addr, fqdns)
+        next_layer.security_level = security_level
         check = None
         if server_addr in get_local_ips():
-            bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
+            bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_PASSTHRU
 
         elif is_not_from_vpn(client_addr):
@@ -217,27 +236,27 @@ def next_layer(next_layer):
             next_layer.__class__ = TlsBlock
 
         elif is_sage_request(server_addr, fqdns):
-            bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr)
+            bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_PASSTHRU
 
-        elif security_level == SEC_OFF or security_level == SEC_BASIC:
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr)
+        elif security_level['level'] == SEC_OFF or security_level['level'] == SEC_BASIC:
+            bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_PASSTHRU
 
         elif fqdns is not None and len(fqdns) == 1 and cert_validation_host == fqdns[0]:
             bubble_log('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr)
             return
 
-        elif security_level == SEC_STD and no_fqdns:
-            bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
+        elif security_level['level'] == SEC_STD and no_fqdns:
+            bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_PASSTHRU
 
-        elif security_level == SEC_MAX and no_fqdns:
-            bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr)
+        elif security_level['level'] == SEC_MAX and no_fqdns:
+            bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr)
             check = FORCE_BLOCK
 
         else:
-            bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+security_level)
+            bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level))
             check = check_connection(client_addr, server_addr, fqdns, security_level)
 
         if check is None or ('passthru' in check and check['passthru']):

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -38,16 +38,12 @@ def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type=
         if host:
             if host.startswith("b'"):
                 host = host[2:-1]
-            if host == debug_capture_fqdn:
-                bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn)
+            if host in debug_capture_fqdn:
+                bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+host)
                 f = open('/tmp/bubble_capture_'+req_id, mode='ab', buffering=0)
                 f.write(chunk)
                 f.close()
                 return chunk
-            else:
-                bubble_log('filter_chunk: debug_capture_fqdn detected but host='+repr(host)+', NOT capturing: '+debug_capture_fqdn)
-        else:
-            bubble_log('filter_chunk: debug_capture_fqdn detected but no host could be detected, NOT capturing: '+debug_capture_fqdn)
 
     # should we just passthru?
     redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + ':' + flow.request.url
@@ -74,7 +70,8 @@ def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type=
             url = url + '?last=true'
 
     if csp:
-        bubble_log('filter_chunk: url='+url+' (csp='+csp+')')
+        # bubble_log('filter_chunk: url='+url+' (csp='+csp+')')
+        bubble_log('filter_chunk: url='+url+' (with csp)')
         filter_headers = {
             HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY,
             HEADER_CONTENT_SECURITY_POLICY: csp