diff --git a/bubble-server/pom.xml b/bubble-server/pom.xml index 2e9891b4..f711b6a4 100644 --- a/bubble-server/pom.xml +++ b/bubble-server/pom.xml @@ -327,7 +327,7 @@ - + org.codehaus.mojo exec-maven-plugin diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index 7946548d..64bbfea1 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -44,8 +44,8 @@ public class ApiConstants { public static final String DEFAULT_LOCALE = "en_US"; - public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "bubble_finalizer"}; - public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "bubble_finalizer"}; + public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "finalizer"}; + public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "finalizer"}; public static final String ANSIBLE_DIR = "ansible"; public static final List BUBBLE_SCRIPTS = splitAndTrim(stream2string(ANSIBLE_DIR + "/bubble_scripts.txt"), "\n") diff --git a/bubble-server/src/main/java/bubble/service/backup/RestoreService.java b/bubble-server/src/main/java/bubble/service/backup/RestoreService.java index baabebc9..1dac81dd 100644 --- a/bubble-server/src/main/java/bubble/service/backup/RestoreService.java +++ b/bubble-server/src/main/java/bubble/service/backup/RestoreService.java @@ -47,7 +47,7 @@ public class RestoreService { // this is how long bubble_restore_monitor.sh will allow a restore after it starts // we add some time because, in the ansible setup, the script starts (in role bubble) before the - // API is started (in role bubble_finalizer) + // API is started (in role finalizer) public static final long RESTORE_MONITOR_SCRIPT_TIMEOUT_SECONDS = RESTORE_WINDOW_SECONDS + MINUTES.toSeconds(5); private static final long RESTORE_LOCK_TIMEOUT = MINUTES.toMillis(31); diff --git a/bubble-server/src/main/java/bubble/service/boot/ActivationService.java b/bubble-server/src/main/java/bubble/service/boot/ActivationService.java index 7ab79e47..8586fdde 100644 --- a/bubble-server/src/main/java/bubble/service/boot/ActivationService.java +++ b/bubble-server/src/main/java/bubble/service/boot/ActivationService.java @@ -30,11 +30,9 @@ import org.cobbzilla.wizard.validation.ValidationResult; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.io.File; import java.util.*; import static bubble.ApiConstants.ROOT_NETWORK_UUID; -import static bubble.cloud.storage.StorageServiceDriver.STORAGE_PREFIX; import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE; import static bubble.model.cloud.BubbleFootprint.DEFAULT_FOOTPRINT; import static bubble.model.cloud.BubbleFootprint.DEFAULT_FOOTPRINT_OBJECT; @@ -42,12 +40,10 @@ import static bubble.model.cloud.BubbleNetwork.TAG_ALLOW_REGISTRATION; import static bubble.model.cloud.BubbleNetwork.TAG_PARENT_ACCOUNT; import static java.util.concurrent.TimeUnit.SECONDS; import static org.cobbzilla.util.daemon.ZillaRuntime.*; -import static org.cobbzilla.util.io.FileUtil.toStringOrDie; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.network.NetworkUtil.getFirstPublicIpv4; import static org.cobbzilla.util.network.NetworkUtil.getLocalhostIpv4; -import static org.cobbzilla.util.system.CommandShell.execScript; import static org.cobbzilla.util.system.Sleep.sleep; import static org.cobbzilla.wizard.model.entityconfig.ModelSetup.scrubSpecial; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @@ -55,8 +51,6 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @Service @Slf4j public class ActivationService { - public static final String DEFAULT_ROLES = "ansible/default_roles.json"; - public static final long ACTIVATION_TIMEOUT = SECONDS.toMillis(10); @Autowired private AccountSshKeyDAO sshKeyDAO; @@ -233,17 +227,6 @@ public class ActivationService { return node; } - public String loadDefaultRoles() { - if (configuration.testMode()) { - final File roleFile = new File("target/classes/"+DEFAULT_ROLES); - final String rolesJson = toStringOrDie(roleFile); - if (rolesJson == null || !rolesJson.contains(STORAGE_PREFIX)) execScript("../bin/prep_bubble_jar"); - return toStringOrDie(roleFile); - } else { - return stream2string(DEFAULT_ROLES); - } - } - public BubbleNetwork createRootNetwork(BubbleNetwork network) { network.setUuid(ROOT_NETWORK_UUID); return networkDAO.create(network); diff --git a/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java b/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java index 18319db6..ab8a0caf 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java @@ -119,7 +119,9 @@ public class AnsiblePrepService { for (String roleName : installRoles) { final TempDir roleTemp = copyClasspathDirectory("ansible/roles/"+roleName); final File roleDir = new File(rolesDir, roleName); - if (!roleTemp.renameTo(roleDir)) return die("prepAnsible: error renaming role dir "+abs(roleTemp)+" -> "+abs(roleDir)); + if (!roleTemp.renameTo(roleDir)) { + return die("prepAnsible: error renaming role dir "+abs(roleTemp)+" -> "+abs(roleDir)); + } final File bubbleRoleJson = new File(abs(roleDir)+"/files/bubble_role.json"); if (bubbleRoleJson.exists()) { final File varsDir = mkdirOrDie(new File(abs(roleDir)+"/vars")); diff --git a/bubble-server/src/main/resources/ansible/default_roles.json b/bubble-server/src/main/resources/ansible/default_roles.json deleted file mode 100644 index 312c8c3e..00000000 --- a/bubble-server/src/main/resources/ansible/default_roles.json +++ /dev/null @@ -1,108 +0,0 @@ -[ - { - "name": "common-0.0.1", - "priority": 100, - "template": true, - "config": [ - {"name": "hostname", "value": "[[node.fqdn]]"} - ], - "tgzB64": "" - }, - { - "name": "firewall-0.0.1", - "priority": 200, - "template": true, - "config": [ - {"name": "ssl_port", "value": "[[node.sslPort]]"}, - {"name": "admin_port", "value": "[[node.adminPort]]"}, - {"name": "dns_port", "value": "[[configuration.dnsPort]]"} - ], - "tgzB64": "" - }, - { - "name": "bubble-0.0.1", - "priority": 300, - "template": true, - "config": [ - {"name": "node_uuid", "value": "[[node.uuid]]"}, - {"name": "network_uuid", "value": "[[node.network]]"}, - {"name": "admin_port", "value": "[[node.adminPort]]"}, - {"name": "ssl_port", "value": "[[node.sslPort]]"}, - {"name": "public_base_uri", "value": "[[publicBaseUri]]"}, - {"name": "sage_node", "value": "[[sageNode]]"}, - {"name": "promo_code_policy", "value": "[[#compare fork '==' true]][[configuration.promoCodePolicy]][[else]]disabled[[/compare]]"}, - {"name": "install_type", "value": "[[installType]]"}, - {"name": "default_locale", "value": "[[network.locale]]"}, - {"name": "time_zone", "value": "[[network.timezone]]"}, - {"name": "bubble_version", "value": "[[configuration.version]]"}, - {"name": "bubble_host", "value": "[[node.fqdn]]"}, - {"name": "bubble_cname", "value": "[[network.networkDomain]]"}, - {"name": "admin_user", "value": "[[node.ansibleUser]]"}, - {"name": "db_encoding", "value": "UTF-8"}, - {"name": "db_locale", "value": "en_US"}, - {"name": "db_user", "value": "bubble"}, - {"name": "db_name", "value": "bubble"}, - {"name": "db_key", "value": "[[dbEncryptionKey]]"}, - {"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"}, - {"name": "is_fork", "value": "[[fork]]"}, - {"name": "restore_key", "value": "[[restoreKey]]"}, - {"name": "restore_timeout", "value": "[[restoreTimeoutSeconds]]"}, - {"name": "test_mode", "value": "[[testMode]]"}, - {"name": "error_url", "value": "[[error_url]]"}, - {"name": "error_key", "value": "[[error_key]]"}, - {"name": "error_env", "value": "[[error_env]]"} - ], - "optionalConfigNames": ["restore_key", "restore_timeout", "error_url", "error_key", "error_env"], - "tgzB64": "" - }, - { - "name": "algo-0.0.1", - "priority": 400, - "template": true, - "install": "node", - "config": [ - {"name": "server_name", "value": "[[node.fqdn]]"}, - {"name": "endpoint", "value": "[[node.ip4]]"}, - {"name": "dns_port", "value": "[[configuration.dnsPort]]"}, - {"name": "ssl_port", "value": "[[node.sslPort]]"} - ], - "tgzB64": "" - }, - { - "name": "nginx-0.0.1", - "priority": 500, - "template": true, - "config": [ - {"name": "server_name", "value": "[[node.fqdn]]"}, - {"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"}, - {"name": "ssl_port", "value": "[[node.sslPort]]"}, - {"name": "admin_port", "value": "[[node.adminPort]]"} - ], - "tgzB64": "" - }, - { - "name": "mitmproxy-0.0.1", - "priority": 600, - "template": true, - "install": "node", - "config": [ - {"name": "admin_port", "value": "[[node.adminPort]]"}, - {"name": "bubble_network", "value": "[[node.network]]"}, - {"name": "mitm_port", "value": "[[configuration.defaultMitmProxyPort]]"} - ], - "tgzB64": "" - }, - { - "name": "bubble_finalizer-0.0.1", - "priority": 9999999, - "template": true, - "config": [ - {"name": "server_alias", "value": "[[network.networkDomain]]"}, - {"name": "restore_key", "value": "[[restoreKey]]"}, - {"name": "install_type", "value": "[[installType]]"}, - {"name": "bubble_java_opts", "value": "-XX:MaxRAM=[[expr nodeSize.memoryMB '//' '2.625']]m"} - ], - "optionalConfigNames": ["restore_key"], - "tgzB64": "" - } -] \ No newline at end of file diff --git a/bubble-server/src/main/resources/ansible/roles/algo/files/master.zip b/bubble-server/src/main/resources/ansible/roles/algo/files/master.zip deleted file mode 100644 index 7410ee1a..00000000 Binary files a/bubble-server/src/main/resources/ansible/roles/algo/files/master.zip and /dev/null differ diff --git a/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json similarity index 91% rename from bubble-server/src/main/resources/ansible/roles/bubble_finalizer/files/bubble_role.json rename to bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json index 7870da24..6a6d519a 100644 --- a/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/files/bubble_role.json +++ b/bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json @@ -1,5 +1,5 @@ { - "name": "bubble_finalizer", + "name": "finalizer", "config": [ {"name": "server_alias", "value": "[[network.networkDomain]]"}, {"name": "restore_key", "value": "[[restoreKey]]"}, diff --git a/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml similarity index 100% rename from bubble-server/src/main/resources/ansible/roles/bubble_finalizer/tasks/main.yml rename to bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml diff --git a/bubble-server/src/main/resources/ansible/roles/bubble_finalizer/templates/supervisor_bubble.conf.j2 b/bubble-server/src/main/resources/ansible/roles/finalizer/templates/supervisor_bubble.conf.j2 similarity index 100% rename from bubble-server/src/main/resources/ansible/roles/bubble_finalizer/templates/supervisor_bubble.conf.j2 rename to bubble-server/src/main/resources/ansible/roles/finalizer/templates/supervisor_bubble.conf.j2 diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_api.py new file mode 100644 index 00000000..9e49748d --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_api.py @@ -0,0 +1,129 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +import requests +import traceback +import sys +import os +import time +import uuid +import datetime +import redis +import json +from bubble_config import bubble_network, bubble_port + +# Write python PID to file so that mitmdump_monitor.sh can check for excessive memory usage and restart if needed +MITMDUMP_PID_FILE_PATH = '/home/mitmproxy/mitmdump.pid' +MITMDUMP_PID_FILE = open(MITMDUMP_PID_FILE_PATH, "w") +MITMDUMP_PID_FILE.write("%d" % os.getpid()) +MITMDUMP_PID_FILE.close() + +HEADER_USER_AGENT = 'User-Agent' +HEADER_REFERER = 'Referer' + +CTX_BUBBLE_MATCHERS='X-Bubble-Matchers' +CTX_BUBBLE_ABORT='X-Bubble-Abort' +CTX_BUBBLE_PASSTHRU='X-Bubble-Passthru' +CTX_BUBBLE_REQUEST_ID='X-Bubble-RequestId' +CTX_CONTENT_LENGTH='X-Bubble-Content-Length' +CTX_CONTENT_LENGTH_SENT='X-Bubble-Content-Length-Sent' +BUBBLE_URI_PREFIX='/__bubble/' + +REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) +BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_' +BUBBLE_ACTIVITY_LOG_EXPIRATION = 600 + +def redis_set(name, value, ex): + REDIS.set(name, value, nx=True, ex=ex) + REDIS.set(name, value, xx=True, ex=ex) + + +def bubble_log(message): + print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr) + + +def bubble_activity_log(client_addr, server_addr, event, data): + key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4()) + value = json.dumps({ + 'source': 'mitmproxy', + 'client_addr': client_addr, + 'server_addr': server_addr, + 'event': event, + 'data': data + }) + bubble_log('bubble_activity_log: setting '+key+' = '+value) + redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION) + pass + + +def bubble_passthru(remote_addr, addr, fqdn): + headers = { + 'X-Forwarded-For': remote_addr, + 'Accept' : 'application/json', + 'Content-Type': 'application/json' + } + try: + data = { + 'addr': str(addr), + 'fqdn': str(fqdn), + 'remoteAddr': remote_addr + } + response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data) + return response.ok + except Exception as e: + bubble_log('bubble_passthru API call failed: '+repr(e)) + traceback.print_exc() + return False + + +def bubble_matchers(req_id, remote_addr, flow, host): + headers = { + 'X-Forwarded-For': remote_addr, + 'Accept' : 'application/json', + 'Content-Type': 'application/json' + } + if HEADER_USER_AGENT not in flow.request.headers: + bubble_log('bubble_matchers: no User-Agent header, setting to UNKNOWN') + user_agent = 'UNKNOWN' + else: + user_agent = flow.request.headers[HEADER_USER_AGENT] + + if HEADER_REFERER not in flow.request.headers: + bubble_log('bubble_matchers: no Referer header, setting to NONE') + referer = 'NONE' + else: + try: + referer = flow.request.headers[HEADER_REFERER].encode().decode() + except Exception as e: + bubble_log('bubble_matchers: error parsing Referer header: '+repr(e)) + referer = 'NONE' + + try: + data = { + 'requestId': req_id, + 'fqdn': host, + 'uri': flow.request.path, + 'userAgent': user_agent, + 'referer': referer, + 'remoteAddr': remote_addr + } + response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data) + if response.ok: + return response.json() + bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) + except Exception as e: + bubble_log('bubble_matchers API call failed: '+repr(e)) + traceback.print_exc() + return None + +def add_flow_ctx(flow, name, value): + if not hasattr(flow, 'bubble_ctx'): + flow.bubble_ctx = {} + flow.bubble_ctx[name] = value + +def get_flow_ctx(flow, name): + if not hasattr(flow, 'bubble_ctx'): + return None + if not name in flow.bubble_ctx: + return None + return flow.bubble_ctx[name] diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_modify.py new file mode 100644 index 00000000..b4f818bd --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_modify.py @@ -0,0 +1,178 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +import re +import requests +import urllib +import traceback +from mitmproxy.net.http import Headers +from bubble_config import bubble_port, bubble_host_alias +from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \ + CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx + +BUFFER_SIZE = 4096 +HEADER_CONTENT_TYPE = 'Content-Type' +HEADER_CONTENT_LENGTH = 'Content-Length' +HEADER_CONTENT_ENCODING = 'Content-Encoding' +HEADER_TRANSFER_ENCODING = 'Transfer-Encoding' +BINARY_DATA_HEADER = {HEADER_CONTENT_TYPE: 'application/octet-stream'} + + +def filter_chunk(chunk, req_id, last, content_encoding=None, content_type=None, content_length=None): + url = 'http://127.0.0.1:' + bubble_port + '/api/filter/apply/' + req_id + params_added = False + if chunk and content_type: + params_added = True + url = (url + + '?type=' + urllib.parse.quote_plus(content_type)) + if content_encoding: + url = url + '&encoding=' + urllib.parse.quote_plus(content_encoding) + if content_length: + url = url + '&length=' + str(content_length) + if last: + if params_added: + url = url + '&last=true' + else: + url = url + '?last=true' + + bubble_log('filter_chunk: url='+url) + + response = requests.post(url, data=chunk, headers=BINARY_DATA_HEADER) + if not response.ok: + err_message = 'filter_chunk: Error fetching ' + url + ', HTTP status ' + str(response.status_code) + bubble_log(err_message) + return b'' + + return response.content + + +def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type): + """ + chunks is a generator that can be used to iterate over all chunks. + """ + first = True + content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) + try: + for chunk in chunks: + if content_length: + bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT) + chunk_len = len(chunk) + last = chunk_len + bytes_sent >= content_length + bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent)) + add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len) + else: + last = False + if first: + yield filter_chunk(chunk, req_id, last, content_encoding, content_type, content_length) + first = False + else: + yield filter_chunk(chunk, req_id, last) + if not content_length: + yield filter_chunk(None, req_id, True) # get the last bits of data + except Exception as e: + bubble_log('bubble_filter_chunks: exception='+repr(e)) + traceback.print_exc() + yield None + + +def bubble_modify(flow, req_id, content_encoding, content_type): + return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type) + + +def send_bubble_response(response): + for chunk in response.iter_content(8192): + yield chunk + + +def responseheaders(flow): + + if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX): + uri = 'http://127.0.0.1:' + bubble_port + '/' + flow.request.path[len(BUBBLE_URI_PREFIX):] + bubble_log('responseheaders: sending special bubble request to '+uri) + headers = { + 'Accept' : 'application/json', + 'Content-Type': 'application/json' + } + response = None + if flow.request.method == 'GET': + response = requests.get(uri, headers=headers, stream=True) + elif flow.request.method == 'POST': + bubble_log('responseheaders: special bubble request: POST content is '+str(flow.request.content)) + headers['Content-Length'] = str(len(flow.request.content)) + response = requests.post(uri, data=flow.request.content, headers=headers) + else: + bubble_log('responseheaders: special bubble request: method '+flow.request.method+' not supported') + if response is not None: + bubble_log('responseheaders: special bubble request: response status = '+str(response.status_code)) + flow.response.headers = Headers() + for key, value in response.headers.items(): + flow.response.headers[key] = value + flow.response.status_code = response.status_code + flow.response.stream = lambda chunks: send_bubble_response(response) + + else: + abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT) + if abort_code is not None: + bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code)) + flow.response.headers = Headers() + flow.response.status_code = abort_code + flow.response.stream = lambda chunks: [] + + else: + req_id = get_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID) + matchers = get_flow_ctx(flow, CTX_BUBBLE_MATCHERS) + if req_id is not None and matchers is not None: + bubble_log('responseheaders: req_id='+req_id+' with matchers: '+repr(matchers)) + if HEADER_CONTENT_TYPE in flow.response.headers: + content_type = flow.response.headers[HEADER_CONTENT_TYPE] + if matchers: + any_content_type_matches = False + for m in matchers: + if 'contentTypeRegex' in m: + typeRegex = m['contentTypeRegex'] + if typeRegex is None: + typeRegex = '^text/html.*' + if re.match(typeRegex, content_type): + any_content_type_matches = True + bubble_log('responseheaders: req_id='+req_id+' found at least one matcher for content_type ('+content_type+'), filtering') + break + if not any_content_type_matches: + bubble_log('responseheaders: req_id='+req_id+' no matchers for content_type ('+content_type+'), passing thru') + return + + if HEADER_CONTENT_ENCODING in flow.response.headers: + content_encoding = flow.response.headers[HEADER_CONTENT_ENCODING] + else: + content_encoding = None + + content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) + bubble_log('responseheaders: req_id='+req_id+' content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) + flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type) + if content_length_value: + flow.response.headers['transfer-encoding'] = 'chunked' + # find server_conn to set fake_chunks on + if flow.live and flow.live.ctx: + ctx = flow.live.ctx + while not hasattr(ctx, 'server_conn'): + if hasattr(ctx, 'ctx'): + ctx = ctx.ctx + else: + bubble_log('responseheaders: error finding server_conn. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx))) + return + if not hasattr(ctx, 'server_conn'): + bubble_log('responseheaders: error finding server_conn. ctx type='+str(type(ctx))+' vars='+str(vars(ctx))) + return + content_length = int(content_length_value) + ctx.server_conn.rfile.fake_chunks = content_length + add_flow_ctx(flow, CTX_CONTENT_LENGTH, content_length) + add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, 0) + + else: + bubble_log('responseheaders: no matchers, passing thru') + pass + else: + bubble_log('responseheaders: no '+HEADER_CONTENT_TYPE +' header, passing thru') + pass + else: + bubble_log('responseheaders: no '+CTX_BUBBLE_MATCHERS +' in ctx, passing thru') + pass diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_passthru.py b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_passthru.py new file mode 100644 index 00000000..adf8600f --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_passthru.py @@ -0,0 +1,112 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here: +# +# Copyright (c) 2013, Aldo Cortesi. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer +from mitmproxy.exceptions import TlsProtocolException + +from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set +import redis +import json + +REDIS_DNS_PREFIX = 'bubble_dns_' +REDIS_PASSTHRU_PREFIX = 'bubble_passthru_' +REDIS_PASSTHRU_DURATION = 60 * 60 # 1 hour timeout on passthru + +REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) + + +def passthru_cache_prefix(client_addr, server_addr): + return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr + + +class TlsFeedback(TlsLayer): + """ + Monkey-patch _establish_tls_with_client to get feedback if TLS could be established + successfully on the client connection (which may fail due to cert pinning). + """ + def _establish_tls_with_client(self): + client_address = self.client_conn.address[0] + server_address = self.server_conn.address[0] + try: + super(TlsFeedback, self)._establish_tls_with_client() + except TlsProtocolException as e: + bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru') + cache_key = passthru_cache_prefix(client_address, server_address) + fqdn = fqdn_for_addr(server_address) + redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) + raise e + + +def fqdn_for_addr(addr): + fqdn = REDIS.get(REDIS_DNS_PREFIX + addr) + if fqdn is None or len(fqdn) == 0: + bubble_log('check_bubble_passthru: no FQDN found for addr '+repr(addr)+', checking raw addr') + fqdn = b'' + return fqdn.decode() + + +def check_bubble_passthru(remote_addr, addr, fqdn): + if bubble_passthru(remote_addr, addr, fqdn): + bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True') + return {'fqdn': fqdn, 'addr': addr, 'passthru': True} + bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False') + return {'fqdn': fqdn, 'addr': addr, 'passthru': False} + + +def should_passthru(remote_addr, addr): + prefix = 'should_passthru: '+repr(addr)+' ' + bubble_log(prefix+'starting...') + cache_key = passthru_cache_prefix(remote_addr, addr) + passthru_json = REDIS.get(cache_key) + if passthru_json is None or len(passthru_json) == 0: + bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...') + fqdn = fqdn_for_addr(addr) + if fqdn is None or len(fqdn) == 0: + bubble_log(prefix+' no fqdn found for addr '+addr+', returning (uncached) passthru = False') + return {'fqdn': None, 'addr': addr, 'passthru': False} + passthru = check_bubble_passthru(remote_addr, addr, fqdn) + bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...") + redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) + else: + bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis') + passthru = json.loads(passthru_json) + REDIS.touch(cache_key) + bubble_log(prefix+'returning '+repr(passthru)) + return passthru + + +def next_layer(next_layer): + if isinstance(next_layer, TlsLayer) and next_layer._client_tls: + client_address = next_layer.client_conn.address[0] + server_address = next_layer.server_conn.address[0] + passthru = should_passthru(client_address, server_address) + if passthru['passthru']: + bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address)) + bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn']) + next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) + next_layer.reply.send(next_layer_replacement) + else: + bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn']) + next_layer.__class__ = TlsFeedback diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_role.json b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_role.json new file mode 100644 index 00000000..edd9e8b8 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_role.json @@ -0,0 +1,15 @@ +{ + "name": "mitmproxy-0.0.1", + "priority": 600, + "template": true, + "install": "node", + "config": [ + {"name": "admin_port", "value": "[[node.adminPort]]"}, + {"name": "bubble_network", "value": "[[node.network]]"}, + {"name": "mitm_port", "value": "[[configuration.defaultMitmProxyPort]]"}, + {"name": "server_name", "value": "[[node.fqdn]]"}, + {"name": "server_alias", "value": "[[network.networkDomain]]"}, + {"name": "ssl_port", "value": "[[node.sslPort]]"} + ], + "tgzB64": "" +} \ No newline at end of file diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/dns_spoofing.py new file mode 100644 index 00000000..74990d86 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/dns_spoofing.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +import re +import time +import uuid +from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx +from bubble_config import bubble_host, bubble_host_alias + +# This regex extracts splits the host header into host and port. +# Handles the edge case of IPv6 addresses containing colons. +# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 +parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") + + +class Rerouter: + @staticmethod + def get_matchers(flow, host): + if host is None: + return None + + if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX): + bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path) + return None + + remote_addr = str(flow.client_conn.address[0]) + try: + host = host.decode() + except (UnicodeDecodeError, AttributeError): + try: + host = str(host) + except Exception as e: + bubble_log('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e)) + return None + + if host == bubble_host or host == bubble_host_alias: + bubble_log('get_matchers: request is for bubble itself ('+host+'), not matching') + return None + + req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time()) + bubble_log("get_matchers: requesting match decision for req_id="+req_id) + resp = bubble_matchers(req_id, remote_addr, flow, host) + + if not resp: + bubble_log('get_matchers: no response for remote_addr/host: '+remote_addr+'/'+str(host)) + return None + + matchers = [] + if 'matchers' in resp and resp['matchers'] is not None: + for m in resp['matchers']: + if 'urlRegex' in m: + bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex']) + else: + bubble_log('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping') + continue + if re.match(m['urlRegex'], flow.request.path): + bubble_log('get_matchers: rule matched, adding rule: '+m['rule']) + matchers.append(m) + else: + bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule']) + else: + bubble_log('get_matchers: no matchers. response='+repr(resp)) + + decision = None + if 'decision' in resp: + decision = resp['decision'] + + matcher_response = { 'decision': decision, 'matchers': matchers, 'request_id': req_id } + bubble_log("get_matchers: returning "+repr(matcher_response)) + return matcher_response + + def request(self, flow): + client_address = flow.client_conn.address[0] + server_address = flow.server_conn.address[0] + if flow.client_conn.tls_established: + flow.request.scheme = "https" + sni = flow.client_conn.connection.get_servername() + port = 443 + else: + flow.request.scheme = "http" + sni = None + port = 80 + + host_header = flow.request.host_header + # bubble_log("dns_spoofing.request: host_header is "+repr(host_header)) + if host_header: + m = parse_host_header.match(host_header) + if m: + host_header = m.group("host").strip("[]") + if m.group("port"): + port = int(m.group("port")) + + # Determine if this request should be filtered + if sni or host_header: + host = str(sni or host_header) + if host.startswith("b'"): + host = host[2:-1] + log_url = flow.request.scheme + '://' + host + flow.request.path + matcher_response = self.get_matchers(flow, sni or host_header) + if matcher_response: + if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru': + bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...') + add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) + bubble_activity_log(client_address, server_address, 'http_passthru', log_url) + return + + elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'): + bubble_log('dns_spoofing.request: found abort code: ' + str(matcher_response['decision']) + ', aborting') + if matcher_response['decision'] == 'abort_ok': + abort_code = 200 + elif matcher_response['decision'] == 'abort_not_found': + abort_code = 404 + else: + bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') + abort_code = 404 + add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code) + bubble_activity_log(client_address, server_address, 'http_abort' + str(abort_code), log_url) + return + + elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match': + bubble_log('dns_spoofing.request: decision was no_match, passing thru...') + bubble_activity_log(client_address, server_address, 'http_no_match', log_url) + return + + elif ('matchers' in matcher_response + and 'request_id' in matcher_response + and len(matcher_response['matchers']) > 0): + req_id = matcher_response['request_id'] + bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) + add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) + add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) + bubble_activity_log(client_address, server_address, 'http_match', log_url) + else: + bubble_log('dns_spoofing.request: no rules returned, passing thru...') + bubble_activity_log(client_address, server_address, 'http_no_rules', log_url) + else: + bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...') + # bubble_activity_log(client_address, server_address, 'http_no_matcher_response', log_url) + else: + bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path) + bubble_activity_log(client_address, server_address, 'http_no_sni_or_host', 'n/a') + + flow.request.host_header = host_header + flow.request.host = sni or host_header + flow.request.port = port + + +addons = [Rerouter()] diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/install_cert.sh b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/install_cert.sh new file mode 100644 index 00000000..a9e87cce --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/install_cert.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +CERT="${1:?no cert provided}" +TIMEOUT=${2:-0} + +function die { + echo 1>&2 "${1}" + exit 1 +} + +START=$(date +%s) +while [[ ! -f "${CERT}" ]] ; do + ELAPSED=$(expr $(date +%s) - ${START}) + if [[ ${ELAPSED} -gt ${TIMEOUT} ]] ; then + break + fi + echo "Cert file does not exist, sleeping then rechecking: ${CERT}" + sleep 5s +done + +if [[ ! -f "${CERT}" ]] ; then + die "Cert file does not exist: ${CERT}" +fi + +if [[ "${CERT}" == *.pem || "${CERT}" == *.p12 ]] ; then + openssl x509 -in "${CERT}" -inform PEM -out "${CERT}.crt" || die "Error converting certificate" + CERT="${CERT}.crt" +fi + +mkdir -p /usr/local/share/ca-certificates || die "Error ensuring CA certs directory exists" +cp "${CERT}" /usr/local/share/ca-certificates || die "Error installing certificate" +update-ca-certificates || die "Error updating CA certificates" diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/mitmdump_monitor.sh b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/mitmdump_monitor.sh new file mode 100644 index 00000000..5e2157a0 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/mitmdump_monitor.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +LOG=/tmp/bubble.mitmdump_monitor.log + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "$(date): ${1}" >> ${LOG} +} + +BUBBLE_MITM_MARKER=/home/bubble/.mitmdump_monitor +ROOT_KEY_MARKER=/usr/share/bubble/mitmdump_monitor +MITMDUMP_PID_FILE=/home/mitmproxy/mitmdump.pid +MAX_MITM_PCT_MEM=18 + +# Start with MITM proxy turned off +if [[ ! -f ${BUBBLE_MITM_MARKER} ]] ; then + echo -n off > ${BUBBLE_MITM_MARKER} && chown bubble ${BUBBLE_MITM_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}" +fi +if [[ ! -f ${ROOT_KEY_MARKER} ]] ; then + sleep 1s + mkdir -p "$(dirname ${ROOT_KEY_MARKER})" && chmod 755 "$(dirname ${ROOT_KEY_MARKER})" || log "Error creating or setting permissions on ${ROOT_KEY_MARKER}" + echo -n on > ${ROOT_KEY_MARKER} && touch ${ROOT_KEY_MARKER} && chmod 644 ${ROOT_KEY_MARKER} || log "Error writing 'on' to ${ROOT_KEY_MARKER}" +fi + +function ensureMitmOn { + log "Flushing PREROUTING before enabling MITM services" + iptables -F PREROUTING -t nat || log "Error flushing port forwarding when enabling MITM services" + log "Enabling MITM port forwarding on TCP port 80 -> 8888" + iptables -I PREROUTING 1 -t nat -p tcp --dport 80 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 80 -> 8888" + log "Enabling MITM port forwarding on TCP port 443 -> 8888" + iptables -I PREROUTING 1 -t nat -p tcp --dport 443 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 443 -> 8888" + echo -n on > ${ROOT_KEY_MARKER} +} + +function ensureMitmOff { + log "Flushing PREROUTING to disable MITM services" + iptables -F PREROUTING -t nat || log "Error flushing port forwarding when disabling MITM services" + echo -n off > ${ROOT_KEY_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}" +} + +log "Watching marker file ${BUBBLE_MITM_MARKER} ..." +sleep 2s && touch ${BUBBLE_MITM_MARKER} || log "Error touching ${BUBBLE_MITM_MARKER}" # first time through, always check and set on/off state +while : ; do + if [[ $(stat -c %Y ${BUBBLE_MITM_MARKER}) -gt $(stat -c %Y ${ROOT_KEY_MARKER}) ]] ; then + if [[ ! -z "$(cmp -b ${ROOT_KEY_MARKER} ${BUBBLE_MITM_MARKER})" ]] ; then + if [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "on" ]] ; then + ensureMitmOn + elif [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "off" ]] ; then + ensureMitmOff + else + log "Error: marker file ${BUBBLE_MITM_MARKER} contained invalid value: $(cat ${BUBBLE_MITM_MARKER} | head -c 5)" + fi + fi + fi + + # Check process memory usage, restart mitmdump if memory goes above max % allowed + if [[ -f ${MITMDUMP_PID_FILE} && -s ${MITMDUMP_PID_FILE} ]] ; then + MITM_PID="$(cat ${MITMDUMP_PID_FILE})" + PCT_MEM="$(ps q ${MITM_PID} -o %mem --no-headers | tr -d [[:space:]] | cut -f1 -d. | sed 's/[^0-9]*//g')" + # log "Info: mitmdump pid ${MITM_PID} using ${PCT_MEM}% of memory" + if [[ ! -z "${PCT_MEM}" ]] ; then + if [[ ${PCT_MEM} -ge ${MAX_MITM_PCT_MEM} ]] ; then + log "Warn: mitmdump: pid=$(cat ${MITMDUMP_PID_FILE}) memory used > max, restarting: ${PCT_MEM}% >= ${MAX_MITM_PCT_MEM}%" + supervisorctl restart mitmdump + fi + else + log "Error: could not determine mitmdump % memory, maybe PID file ${MITMDUMP_PID_FILE} is out of date? pid found was ${MITM_PID}" + fi + else + log "Error: mitmdump PID file ${MITMDUMP_PID_FILE} not found or empty" + fi + sleep 5s +done diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/reuse_bubble_mitm_certs.sh b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/reuse_bubble_mitm_certs.sh new file mode 100644 index 00000000..e1ea2d76 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/reuse_bubble_mitm_certs.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +function die { + echo 1>&2 "${1}" + exit 1 +} + +CERTS_BACKUP=/home/bubble/mitm_certs +if [[ ! -d ${CERTS_BACKUP} ]] ; then + echo "No mitm_certs backup found, skipping restore" + exit 0 +fi + +MITM_CERTS=/home/mitmproxy/.mitmproxy +if [[ -d ${MITM_CERTS} ]] ; then + echo "Removing obsolete mitm certs: ${MITM_CERTS}" + rm -rf ${MITM_CERTS} || die "Error removing obsolete mitm certs" + if [[ -d ${MITM_CERTS} ]] ; then + die "Error removing obsolete mitm certs: dir still exists: ${MITM_CERTS}" + fi +fi + +mkdir -p ${MITM_CERTS} || die "Error creating mitm certs dir: ${MITM_CERTS}" +chmod 750 ${MITM_CERTS} || die "Error setting permissions on mitm certs dir: ${MITM_CERTS}" +cp -R ${CERTS_BACKUP}/* ${MITM_CERTS}/ || die "Error restoring mitm certs" +chown -R mitmproxy ${MITM_CERTS} || die "Error changing ownership of ${MITM_CERTS}" +chgrp -R root ${MITM_CERTS} || die "Error changing group ownership of ${MITM_CERTS}" +chmod 440 ${MITM_CERTS}/* || die "Error setting permissions on mitm certs files" diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/run_mitmdump.sh b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/run_mitmdump.sh new file mode 100644 index 00000000..537e9880 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/run_mitmdump.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +MITM_PORT=${1:?no port provided} +cd /home/mitmproxy/mitmproxy && \ +./dev.sh && . ./venv/bin/activate && \ +mitmdump \ + --listen-host 0.0.0.0 \ + --listen-port ${MITM_PORT} \ + --showhost \ + --no-http2 \ + --set block_global=true \ + --set block_private=false \ + --set termlog_verbosity=debug \ + --set flow_detail=3 \ + --set stream_large_bodies=5m \ + --set keep_host_header \ + -s ./dns_spoofing.py \ + -s ./bubble_passthru.py \ + -s ./bubble_modify.py \ + --mode transparent diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/set_cert_name.sh b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/set_cert_name.sh new file mode 100755 index 00000000..78d696a8 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/set_cert_name.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +MITM_DIR=${1:?no mitm dir specified} +CERT_NAME=${2:?no cert name specified} + +if [[ ! -d "${MITM_DIR}" ]] ; then + echo "mitm dir does not exist or is not a directory: ${MITM_DIR}" + exit 1 +fi + +OPTIONS_FILE="${MITM_DIR}/mitmproxy/options.py" +if [[ ! -f "${OPTIONS_FILE}" ]] ; then + echo "options.py not found in mitm dir: ${MITM_DIR}" + exit 1 +fi + +if [[ $(cat "${OPTIONS_FILE}" | egrep '^CONF_BASENAME =' | grep "${CERT_NAME}" | wc -l | tr -d ' ') -eq 0 ]] ; then + temp="$(mktemp /tmp/options.py.XXXXXXX)" + cat "${OPTIONS_FILE}" | sed -e 's/^CONF_BASENAME\s*=.*/CONF_BASENAME = "'"${CERT_NAME}"'"/' > "${temp}" + mv "${temp}" "${OPTIONS_FILE}" +fi diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitmdump_monitor.conf b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitmdump_monitor.conf new file mode 100644 index 00000000..60e0b54c --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitmdump_monitor.conf @@ -0,0 +1,5 @@ + +[program:mitmdump_monitor] +stdout_logfile = /dev/null +stderr_logfile = /dev/null +command=/usr/local/sbin/mitmdump_monitor.sh diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml new file mode 100644 index 00000000..4e3ef096 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml @@ -0,0 +1,115 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Install python3, pip, virtualenv and required dependencies + apt: + name: [ 'python3-pip', 'python3-venv', 'libc6-dev', 'libpython3-dev', 'g++', 'libffi-dev' ] + state: present + update_cache: yes + +- name: Install supervisor conf file + template: + src: supervisor_mitmproxy.conf.j2 + dest: /etc/supervisor/conf.d/mitmproxy.conf + owner: root + group: root + mode: 0400 + +- name: Create mitmproxy user + user: + name: mitmproxy + comment: mitmdump user + shell: /bin/bash + system: yes + home: /home/mitmproxy + +- name: Creates mitmproxy dir + file: + path: /home/mitmproxy/mitmproxy + owner: mitmproxy + group: mitmproxy + mode: 0755 + state: directory + +- name: Unzip mitmproxy.zip + unarchive: + src: mitmproxy.zip + dest: /home/mitmproxy/mitmproxy + +- name: Copy mitmdump files + copy: + src: "{{ item }}" + dest: "/home/mitmproxy/mitmproxy/{{ item }}" + owner: mitmproxy + group: mitmproxy + mode: 0500 + with_items: + - bubble_api.py + - dns_spoofing.py + - bubble_passthru.py + - bubble_modify.py + - run_mitmdump.sh + +- name: Install cert helper scripts + copy: + src: "{{ item }}" + dest: "/usr/local/bin/{{ item }}" + owner: root + group: root + mode: 0500 + with_items: + - install_cert.sh + - set_cert_name.sh + - reuse_bubble_mitm_certs.sh + +- name: Set the cert name + shell: set_cert_name.sh /home/mitmproxy/mitmproxy {{ server_alias }} + +- name: Set ownership of mitmproxy files + shell: chown -R mitmproxy /home/mitmproxy/mitmproxy + +- name: Reuse bubble mitm certs if available + shell: reuse_bubble_mitm_certs.sh + +- name: Copy bubble_config.py to /home/mitmproxy/mitmproxy + template: + src: bubble_config.py.j2 + dest: /home/mitmproxy/mitmproxy/bubble_config.py + owner: mitmproxy + group: mitmproxy + mode: 0500 + +- name: Fix missing symlink for libstdc++ + file: + src: /usr/lib/x86_64-linux-gnu/libstdc++.so.6 + dest: /usr/lib/x86_64-linux-gnu/libstdc++.so + owner: root + group: root + state: link + +- name: Restart dnscrypt-proxy + shell: service dnscrypt-proxy restart + +- name: restart supervisord + service: + name: supervisor + enabled: yes + state: restarted + +- import_tasks: route.yml + +- name: Install mitmdump_monitor + copy: + src: "mitmdump_monitor.sh" + dest: "/usr/local/sbin/mitmdump_monitor.sh" + owner: root + group: root + mode: 0500 + +- name: Install mitmdump_monitor supervisor conf file + copy: + src: supervisor_mitmdump_monitor.conf + dest: /etc/supervisor/conf.d/mitmdump_monitor.conf + +- name: Ensure mitmdump_monitor is started + shell: supervisorctl restart mitmdump_monitor diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/route.yml b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/route.yml new file mode 100644 index 00000000..fa917b73 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/route.yml @@ -0,0 +1,58 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- sysctl: + name: net.ipv4.ip_forward + value: 1 + sysctl_set: yes +- sysctl: + name: net.ipv6.conf.all.forwarding + value: 1 + sysctl_set: yes +- sysctl: + name: net.ipv4.conf.all.send_redirects + value: 0 + sysctl_set: yes + +- name: "Allow MITM private port" + iptables: + chain: INPUT + action: insert + rule_num: 10 + protocol: tcp + destination_port: "{{ mitm_port }}" + ctstate: NEW + syn: match + jump: ACCEPT + comment: Accept new local TCP DNS connections on private port + become: yes + +- name: Route port 80 through mitmproxy + iptables: + table: nat + chain: PREROUTING + action: insert + rule_num: 1 + protocol: tcp + destination_port: 80 + jump: REDIRECT + to_ports: "{{ mitm_port }}" + +- name: Route port 443 through mitmproxy + iptables: + table: nat + chain: PREROUTING + action: insert + rule_num: 2 + protocol: tcp + destination_port: 443 + jump: REDIRECT + to_ports: "{{ mitm_port }}" + +- name: save iptables rules + shell: iptables-save > /etc/iptables/rules.v4 + become: yes + +- name: save iptables v6 rules + shell: ip6tables-save > /etc/iptables/rules.v6 + become: yes diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2 b/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2 new file mode 100644 index 00000000..aee0d31e --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2 @@ -0,0 +1,5 @@ +bubble_network = '{{ bubble_network }}' +bubble_port = '{{ admin_port }}'; +bubble_host = '{{ server_name }}' +bubble_host_alias = '{{ server_alias }}' +bubble_ssl_port = '{{ ssl_port }}' diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/supervisor_mitmproxy.conf.j2 b/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/supervisor_mitmproxy.conf.j2 new file mode 100644 index 00000000..a6c915d5 --- /dev/null +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/supervisor_mitmproxy.conf.j2 @@ -0,0 +1,7 @@ + +[program:mitmdump] +stdout_logfile = /home/mitmproxy/mitmdump-out.log +stderr_logfile = /home/mitmproxy/mitmdump-err.log +command=sudo -H -u mitmproxy bash -c "/home/mitmproxy/mitmproxy/run_mitmdump.sh {{ mitm_port }}" +stopasgroup=true +stopsignal=QUIT diff --git a/bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json b/bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json index a2677627..e97d37ab 100644 --- a/bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json +++ b/bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json @@ -17,8 +17,8 @@ { "percent": 76,"messageKey":"role_nginx", "pattern":"TASK \\[nginx : [\\w\\s]+] \\*{5,}" }, { "percent": 81,"messageKey":"role_nginx_certbot", "pattern":"TASK \\[nginx : Init certbot] \\*{5,}" }, { "percent": 91,"messageKey":"role_mitmproxy", "pattern":"TASK \\[mitmproxy : [\\w\\s]+] \\*{5,}" }, -{ "percent": 94,"messageKey":"role_bubble_finalizer", "pattern":"TASK \\[bubble_finalizer : [\\w\\s]+] \\*{5,}" }, -{ "percent": 98,"messageKey":"role_bubble_finalizer_touch", "pattern":"TASK \\[bubble_finalizer : Touch first-time setup file] \\*{5,}" }, -{ "percent": 99,"messageKey":"role_bubble_finalizer_start", "pattern":"TASK \\[bubble_finalizer : Ensure bubble is started] \\*{5,}" }, +{ "percent": 94,"messageKey":"role_finalizer", "pattern":"TASK \\[finalizer : [\\w\\s]+] \\*{5,}" }, +{ "percent": 98,"messageKey":"role_finalizer_touch", "pattern":"TASK \\[finalizer : Touch first-time setup file] \\*{5,}" }, +{ "percent": 99,"messageKey":"role_finalizer_start", "pattern":"TASK \\[finalizer : Ensure bubble is started] \\*{5,}" }, {"percent": 100,"messageKey":"install_complete", "pattern":"PLAY RECAP \\*{5,}" } ] diff --git a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties index d6c0d643..fa982cf0 100644 --- a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties +++ b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties @@ -353,9 +353,9 @@ meter_tick_role_bubble_algo=Setting up VPN meter_tick_role_nginx=Setting up web server meter_tick_role_nginx_certbot=Installing SSL certificates meter_tick_role_mitmproxy=Setting up MITM server -meter_tick_role_bubble_finalizer=Finalizing Bubble installation -meter_tick_role_bubble_finalizer_touch=Turning on "first-time" setting to allow you to unlock your Bubble -meter_tick_role_bubble_finalizer_start=Starting Bubble API services +meter_tick_role_finalizer=Finalizing Bubble installation +meter_tick_role_finalizer_touch=Turning on "first-time" setting to allow you to unlock your Bubble +meter_tick_role_finalizer_start=Starting Bubble API services meter_tick_install_complete=Bubble installation completed # Launch progress meter: success marker diff --git a/bubble-server/src/main/resources/packer/node-roles.txt b/bubble-server/src/main/resources/packer/node-roles.txt index 53d3f934..1fbe509d 100644 --- a/bubble-server/src/main/resources/packer/node-roles.txt +++ b/bubble-server/src/main/resources/packer/node-roles.txt @@ -4,4 +4,4 @@ nginx algo mitmproxy bubble -bubble_finalizer \ No newline at end of file +finalizer \ No newline at end of file diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/bubble-nodemanager b/bubble-server/src/main/resources/packer/roles/finalizer/files/bubble-nodemanager similarity index 100% rename from bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/bubble-nodemanager rename to bubble-server/src/main/resources/packer/roles/finalizer/files/bubble-nodemanager diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/copy_certs_to_bubble.sh b/bubble-server/src/main/resources/packer/roles/finalizer/files/copy_certs_to_bubble.sh similarity index 100% rename from bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/copy_certs_to_bubble.sh rename to bubble-server/src/main/resources/packer/roles/finalizer/files/copy_certs_to_bubble.sh diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/supervisor_bubble_nodemanager.conf b/bubble-server/src/main/resources/packer/roles/finalizer/files/supervisor_bubble_nodemanager.conf similarity index 100% rename from bubble-server/src/main/resources/packer/roles/bubble_finalizer/files/supervisor_bubble_nodemanager.conf rename to bubble-server/src/main/resources/packer/roles/finalizer/files/supervisor_bubble_nodemanager.conf diff --git a/bubble-server/src/main/resources/packer/roles/bubble_finalizer/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/finalizer/tasks/main.yml similarity index 100% rename from bubble-server/src/main/resources/packer/roles/bubble_finalizer/tasks/main.yml rename to bubble-server/src/main/resources/packer/roles/finalizer/tasks/main.yml diff --git a/bubble-server/src/main/resources/packer/sage-roles.txt b/bubble-server/src/main/resources/packer/sage-roles.txt index 97011583..1b29005e 100644 --- a/bubble-server/src/main/resources/packer/sage-roles.txt +++ b/bubble-server/src/main/resources/packer/sage-roles.txt @@ -2,4 +2,4 @@ common firewall nginx bubble -bubble_finalizer \ No newline at end of file +finalizer \ No newline at end of file