From 59174357fcc360a83f976780fa2a9a4cfbcdb9ed Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 23 Jul 2020 02:57:55 -0400 Subject: [PATCH] improve support for pinned apps - detecting tls block will not switch to passthru, so webapp version of app still works --- bin/rkeys | 2 +- bin/rmembers | 9 +++ .../main/java/bubble/dao/app/AppSiteDAO.java | 8 ++- .../main/java/bubble/model/app/AppSite.java | 22 ++++++- .../cloud/StandardDeviceIdService.java | 15 +++++ .../main/resources/ansible/bubble_scripts.txt | 1 + ...072301__add_appsite_max_security_hosts.sql | 4 ++ .../twitter/bubbleApp_userBlock_twitter.json | 4 +- .../roles/mitmproxy/files/bubble_api.py | 14 +++-- .../mitmproxy/files/bubble_conn_check.py | 57 ++++++++++++------- .../roles/mitmproxy/files/bubble_modify.py | 11 ++-- 11 files changed, 110 insertions(+), 37 deletions(-) create mode 100755 bin/rmembers create mode 100644 bubble-server/src/main/resources/db/migration/V2020072301__add_appsite_max_security_hosts.sql diff --git a/bin/rkeys b/bin/rkeys index c2bc51df..2b8c6998 100755 --- a/bin/rkeys +++ b/bin/rkeys @@ -3,6 +3,6 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # KEY_MATCH="${1}" -for k in $(echo 'keys *'"""${KEY_MATCH}"""'*' | redis-cli ) ; do +for k in $(echo 'keys *'"""${KEY_MATCH}"""'*' | redis-cli) ; do echo "$k => $(echo "get $k" | redis-cli)" done diff --git a/bin/rmembers b/bin/rmembers new file mode 100755 index 00000000..57d4109f --- /dev/null +++ b/bin/rmembers @@ -0,0 +1,9 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +KEY_MATCH="${1}" +SEP="${2:- }" +for k in $(echo 'keys *'"""${KEY_MATCH}"""'*' | redis-cli) ; do + echo "$k => $(echo "smembers $k" | redis-cli | tr '\n' ''"${SEP}"'')" +done diff --git a/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java index cffbb615..bdc5992f 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java @@ -5,6 +5,7 @@ package bubble.dao.app; import bubble.model.app.AppSite; +import bubble.service.cloud.DeviceIdService; import bubble.service.stream.RuleEngineService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -13,15 +14,18 @@ import org.springframework.stereotype.Repository; public class AppSiteDAO extends AppTemplateEntityDAO { @Autowired private RuleEngineService ruleEngineService; + @Autowired private DeviceIdService deviceService; @Override public AppSite postCreate(AppSite site, Object context) { // todo: update entities based on this template if account has updates enabled + if (site.hasMaxSecurityHosts()) deviceService.initDeviceSecurityLevels(); return super.postCreate(site, context); } - @Override public AppSite postUpdate(AppSite entity, Object context) { + @Override public AppSite postUpdate(AppSite site, Object context) { ruleEngineService.flushCaches(); - return super.postUpdate(entity, context); + if (site.hasMaxSecurityHosts()) deviceService.initDeviceSecurityLevels(); + return super.postUpdate(site, context); } @Override public void delete(String uuid) { diff --git a/bubble-server/src/main/java/bubble/model/app/AppSite.java b/bubble-server/src/main/java/bubble/model/app/AppSite.java index 0ee88634..2044cec4 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppSite.java +++ b/bubble-server/src/main/java/bubble/model/app/AppSite.java @@ -5,6 +5,7 @@ package bubble.model.app; import bubble.model.account.Account; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -17,8 +18,12 @@ import org.cobbzilla.wizard.model.entityconfig.annotations.*; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.Transient; import static bubble.ApiConstants.EP_SITES; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; +import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @ECType(root=true) @@ -36,7 +41,9 @@ import static org.cobbzilla.util.reflect.ReflectionUtil.copy; }) public class AppSite extends IdentifiableBase implements AppTemplateEntity { - public static final String[] VALUE_FIELDS = {"template", "enabled", "description", "url"}; + public static final String[] VALUE_FIELDS = { + "template", "enabled", "description", "url", "maxSecurityHosts", "enableMaxSecurityHosts" + }; public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "app"); public AppSite (AppSite other) { copy(this, other, CREATE_FIELDS); } @@ -77,4 +84,17 @@ public class AppSite extends IdentifiableBase implements AppTemplateEntity { @Column(nullable=false, length=1024) @Getter @Setter private String url; + // Json array of hosts that should always get maximum security + @ECField(index=80) + @Column(length=5000) + @JsonIgnore @Getter @Setter private String maxSecurityHostsJson; + + @Transient public String[] getMaxSecurityHosts () { return empty(maxSecurityHostsJson) ? null : json(maxSecurityHostsJson, String[].class); } + public AppSite setMaxSecurityHosts(String[] hosts) { return setMaxSecurityHostsJson(empty(hosts) ? null : json(hosts, COMPACT_MAPPER)); } + public boolean hasMaxSecurityHosts () { return !empty(getMaxSecurityHosts()); } + + @ECField(index=90) + @Getter @Setter private Boolean enableMaxSecurityHosts; + public boolean enableMaxSecurityHosts() { return enableMaxSecurityHosts == null ? true : enableMaxSecurityHosts; } + } diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java index dcfcd8d0..702d4dd7 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java @@ -5,7 +5,9 @@ package bubble.service.cloud; import bubble.dao.account.AccountDAO; +import bubble.dao.app.AppSiteDAO; import bubble.dao.device.DeviceDAO; +import bubble.model.app.AppSite; import bubble.model.device.Device; import bubble.model.device.DeviceStatus; import bubble.server.BubbleConfiguration; @@ -44,11 +46,13 @@ public class StandardDeviceIdService implements DeviceIdService { // used in dnscrypt-proxy and mitmproxy to check device security level public static final String REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = "bubble_device_security_level_"; + public static final String REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = "bubble_device_site_max_security_level_"; @Autowired private DeviceDAO deviceDAO; @Autowired private AccountDAO accountDAO; @Autowired private RedisService redis; @Autowired private GeoService geoService; + @Autowired private AppSiteDAO siteDAO; @Autowired private BubbleConfiguration configuration; private final Map deviceCache = new ExpirationMap<>(MINUTES.toMillis(10)); @@ -123,6 +127,17 @@ public class StandardDeviceIdService implements DeviceIdService { if (configuration.testMode()) return; for (String ip : findIpsByDevice(device.getUuid())) { redis.set_plaintext(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+ip, device.getSecurityLevel().name()); + + for (AppSite site : siteDAO.findByAccount(device.getAccount())) { + if (site.hasMaxSecurityHosts()) { + final String siteKey = REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX + ip; + if (site.enableMaxSecurityHosts()) { + redis.sadd_plaintext(siteKey, site.getMaxSecurityHosts()); + } else { + redis.srem(siteKey, site.getMaxSecurityHosts()); + } + } + } } } diff --git a/bubble-server/src/main/resources/ansible/bubble_scripts.txt b/bubble-server/src/main/resources/ansible/bubble_scripts.txt index 1703ecc7..2e4a0af2 100644 --- a/bubble-server/src/main/resources/ansible/bubble_scripts.txt +++ b/bubble-server/src/main/resources/ansible/bubble_scripts.txt @@ -17,4 +17,5 @@ list_bubble_databases cleanup_bubble_databases install_packer.sh rkeys +rmembers rdelkeys \ No newline at end of file diff --git a/bubble-server/src/main/resources/db/migration/V2020072301__add_appsite_max_security_hosts.sql b/bubble-server/src/main/resources/db/migration/V2020072301__add_appsite_max_security_hosts.sql new file mode 100644 index 00000000..333db8a6 --- /dev/null +++ b/bubble-server/src/main/resources/db/migration/V2020072301__add_appsite_max_security_hosts.sql @@ -0,0 +1,4 @@ +ALTER TABLE app_site ADD COLUMN max_security_hosts_json VARCHAR(5000); +ALTER TABLE app_site ADD COLUMN enable_max_security_hosts BOOLEAN; +UPDATE app_site SET max_security_hosts_json = '["twitter.com","*.twitter.com","*.twimg.com","t.co"]' WHERE name = 'Twitter'; +UPDATE app_site SET enable_max_security_hosts = true WHERE name = 'Twitter'; diff --git a/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter.json b/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter.json index dad23970..6ef96145 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter.json +++ b/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter.json @@ -5,7 +5,9 @@ "name": "Twitter", "url": "https://twitter.com", "description": "what’s happening in the world and what people are talking about right now.", - "template": true + "template": true, + "maxSecurityHosts": ["twitter.com", "*.twitter.com", "*.twimg.com", "t.co"], + "enableMaxSecurityHosts": true }], "AppRule": [{ "name": "twitter_user_blocker", diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index f9334c10..1ad96c23 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -59,9 +59,11 @@ def bubble_activity_log(client_addr, server_addr, event, data): def bubble_conn_check(remote_addr, addr, fqdns, security_level): - if debug_capture_fqdn and fqdns and debug_capture_fqdn in fqdns: - bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+debug_capture_fqdn) - return 'noop' + if debug_capture_fqdn and fqdns: + for f in debug_capture_fqdn: + if f in fqdns: + bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f) + return 'noop' headers = { 'X-Forwarded-For': remote_addr, @@ -83,7 +85,7 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level): except Exception as e: bubble_log('bubble_conn_check API call failed: '+repr(e)) traceback.print_exc() - if security_level is not None and security_level == 'maximum': + if security_level is not None and security_level['level'] == 'maximum': return False return None @@ -100,8 +102,8 @@ DEBUG_MATCHER = { } def bubble_matchers(req_id, remote_addr, flow, host): - if debug_capture_fqdn and host and debug_capture_fqdn == host: - bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+debug_capture_fqdn) + if debug_capture_fqdn and host and host in debug_capture_fqdn: + bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host) return DEBUG_MATCHER headers = { diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index adbe2d51..e2629f62 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -40,6 +40,7 @@ REDIS_DNS_PREFIX = 'bubble_dns_' REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService +REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = 'bubble_device_site_max_security_level_' # defined in StandardDeviceIdService FORCE_PASSTHRU = {'passthru': True} FORCE_BLOCK = {'block': True} @@ -56,11 +57,24 @@ VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4) VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6) -def get_device_security_level(client_addr): +def get_device_security_level(client_addr, fqdns): level = REDIS.get(REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX+client_addr) if level is None: - return SEC_MAX - return level.decode() + return {'level': SEC_MAX} + level = level.decode() + if level == SEC_STD: + bubble_log('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns)) + if fqdns: + max_required_fqdns = REDIS.smembers(REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX+client_addr) + if max_required_fqdns is not None: + bubble_log('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns)) + for max_required in max_required_fqdns: + max_required = max_required.decode() + for fqdn in fqdns: + if max_required == fqdn or (max_required.startswith('*.') and fqdn.endswith(max_required[1:])): + bubble_log('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required) + return {'level': SEC_MAX, 'pinned': True} + return {'level': level} def get_local_ips(): @@ -115,7 +129,7 @@ class TlsFeedback(TlsLayer): def _establish_tls_with_client(self): client_address = self.client_conn.address[0] server_address = self.server_conn.address[0] - security_level = get_device_security_level(client_address) + security_level = self.security_level try: super(TlsFeedback, self)._establish_tls_with_client() @@ -128,15 +142,19 @@ class TlsFeedback(TlsLayer): elif self.fqdns is not None and len(self.fqdns) > 0: for fqdn in self.fqdns: cache_key = conn_check_cache_prefix(client_address, fqdn) - if security_level == SEC_MAX: - redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) + if security_level['level'] == SEC_MAX: + if 'pinned' in security_level and security_level['pinned']: + redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': False, 'reason': 'tls_failure_pinned'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) + else: + redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) else: redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) else: cache_key = conn_check_cache_prefix(client_address, server_address) - if security_level == SEC_MAX: + if security_level['level'] == SEC_MAX: redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e)) else: @@ -148,7 +166,7 @@ class TlsFeedback(TlsLayer): def check_bubble_connection(client_addr, server_addr, fqdns, security_level): check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level) if check_response is None or check_response == 'error': - if security_level == SEC_MAX: + if security_level['level'] == SEC_MAX: bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'} else: @@ -205,10 +223,11 @@ def next_layer(next_layer): bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns)) next_layer.fqdns = fqdns no_fqdns = fqdns is None or len(fqdns) == 0 - security_level = get_device_security_level(client_addr) + security_level = get_device_security_level(client_addr, fqdns) + next_layer.security_level = security_level check = None if server_addr in get_local_ips(): - bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) + bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU elif is_not_from_vpn(client_addr): @@ -217,27 +236,27 @@ def next_layer(next_layer): next_layer.__class__ = TlsBlock elif is_sage_request(server_addr, fqdns): - bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+security_level+' for client='+client_addr) + bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU - elif security_level == SEC_OFF or security_level == SEC_BASIC: - bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+security_level+' for client='+client_addr) + elif security_level['level'] == SEC_OFF or security_level['level'] == SEC_BASIC: + bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU elif fqdns is not None and len(fqdns) == 1 and cert_validation_host == fqdns[0]: bubble_log('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr) return - elif security_level == SEC_STD and no_fqdns: - bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) + elif security_level['level'] == SEC_STD and no_fqdns: + bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU - elif security_level == SEC_MAX and no_fqdns: - bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+security_level+' for client='+client_addr) + elif security_level['level'] == SEC_MAX and no_fqdns: + bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_BLOCK else: - bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+security_level) + bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level)) check = check_connection(client_addr, server_addr, fqdns, security_level) if check is None or ('passthru' in check and check['passthru']): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index de3c1d64..f8c6944d 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -38,16 +38,12 @@ def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type= if host: if host.startswith("b'"): host = host[2:-1] - if host == debug_capture_fqdn: - bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn) + if host in debug_capture_fqdn: + bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+host) f = open('/tmp/bubble_capture_'+req_id, mode='ab', buffering=0) f.write(chunk) f.close() return chunk - else: - bubble_log('filter_chunk: debug_capture_fqdn detected but host='+repr(host)+', NOT capturing: '+debug_capture_fqdn) - else: - bubble_log('filter_chunk: debug_capture_fqdn detected but no host could be detected, NOT capturing: '+debug_capture_fqdn) # should we just passthru? redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + ':' + flow.request.url @@ -74,7 +70,8 @@ def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type= url = url + '?last=true' if csp: - bubble_log('filter_chunk: url='+url+' (csp='+csp+')') + # bubble_log('filter_chunk: url='+url+' (csp='+csp+')') + bubble_log('filter_chunk: url='+url+' (with csp)') filter_headers = { HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY, HEADER_CONTENT_SECURITY_POLICY: csp