rename bubble_finalizer to just finalizer, remove default_roles

bubble-server/pom.xml

@@ -327,7 +327,7 @@
                 </executions>
             </plugin>
 
-            <!-- update ansible/default_roles.json, copy scripts into jar -->
+            <!-- copy scripts and web ui into jar -->
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>exec-maven-plugin</artifactId>

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -44,8 +44,8 @@ public class ApiConstants {
 
     public static final String DEFAULT_LOCALE = "en_US";
 
-    public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "bubble_finalizer"};
-    public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "bubble_finalizer"};
+    public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "finalizer"};
+    public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "finalizer"};
 
     public static final String ANSIBLE_DIR = "ansible";
     public static final List<String> BUBBLE_SCRIPTS = splitAndTrim(stream2string(ANSIBLE_DIR + "/bubble_scripts.txt"), "\n")

bubble-server/src/main/java/bubble/service/backup/RestoreService.java

@@ -47,7 +47,7 @@ public class RestoreService {
 
     // this is how long bubble_restore_monitor.sh will allow a restore after it starts
     // we add some time because, in the ansible setup, the script starts (in role bubble) before the
-    // API is started (in role bubble_finalizer)
+    // API is started (in role finalizer)
     public static final long RESTORE_MONITOR_SCRIPT_TIMEOUT_SECONDS = RESTORE_WINDOW_SECONDS + MINUTES.toSeconds(5);
 
     private static final long RESTORE_LOCK_TIMEOUT = MINUTES.toMillis(31);

bubble-server/src/main/java/bubble/service/boot/ActivationService.java

@@ -30,11 +30,9 @@ import org.cobbzilla.wizard.validation.ValidationResult;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.io.File;
 import java.util.*;
 
 import static bubble.ApiConstants.ROOT_NETWORK_UUID;
-import static bubble.cloud.storage.StorageServiceDriver.STORAGE_PREFIX;
 import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE;
 import static bubble.model.cloud.BubbleFootprint.DEFAULT_FOOTPRINT;
 import static bubble.model.cloud.BubbleFootprint.DEFAULT_FOOTPRINT_OBJECT;
@@ -42,12 +40,10 @@ import static bubble.model.cloud.BubbleNetwork.TAG_ALLOW_REGISTRATION;
 import static bubble.model.cloud.BubbleNetwork.TAG_PARENT_ACCOUNT;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
-import static org.cobbzilla.util.io.FileUtil.toStringOrDie;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.network.NetworkUtil.getFirstPublicIpv4;
 import static org.cobbzilla.util.network.NetworkUtil.getLocalhostIpv4;
-import static org.cobbzilla.util.system.CommandShell.execScript;
 import static org.cobbzilla.util.system.Sleep.sleep;
 import static org.cobbzilla.wizard.model.entityconfig.ModelSetup.scrubSpecial;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
@@ -55,8 +51,6 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 @Service @Slf4j
 public class ActivationService {
 
-    public static final String DEFAULT_ROLES = "ansible/default_roles.json";
-
     public static final long ACTIVATION_TIMEOUT = SECONDS.toMillis(10);
 
     @Autowired private AccountSshKeyDAO sshKeyDAO;
@@ -233,17 +227,6 @@ public class ActivationService {
         return node;
     }
 
-    public String loadDefaultRoles() {
-        if (configuration.testMode()) {
-            final File roleFile = new File("target/classes/"+DEFAULT_ROLES);
-            final String rolesJson = toStringOrDie(roleFile);
-            if (rolesJson == null || !rolesJson.contains(STORAGE_PREFIX)) execScript("../bin/prep_bubble_jar");
-            return toStringOrDie(roleFile);
-        } else {
-            return stream2string(DEFAULT_ROLES);
-        }
-    }
-
     public BubbleNetwork createRootNetwork(BubbleNetwork network) {
         network.setUuid(ROOT_NETWORK_UUID);
         return networkDAO.create(network);

bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java

@@ -119,7 +119,9 @@ public class AnsiblePrepService {
         for (String roleName : installRoles) {
             final TempDir roleTemp = copyClasspathDirectory("ansible/roles/"+roleName);
             final File roleDir = new File(rolesDir, roleName);
-            if (!roleTemp.renameTo(roleDir)) return die("prepAnsible: error renaming role dir "+abs(roleTemp)+" -> "+abs(roleDir));
+            if (!roleTemp.renameTo(roleDir)) {
+                return die("prepAnsible: error renaming role dir "+abs(roleTemp)+" -> "+abs(roleDir));
+            }
             final File bubbleRoleJson = new File(abs(roleDir)+"/files/bubble_role.json");
             if (bubbleRoleJson.exists()) {
                 final File varsDir = mkdirOrDie(new File(abs(roleDir)+"/vars"));

bubble-server/src/main/resources/ansible/default_roles.json

@@ -1,108 +0,0 @@
-[
-  {
-    "name": "common-0.0.1",
-    "priority": 100,
-    "template": true,
-    "config": [
-      {"name": "hostname", "value": "[[node.fqdn]]"}
-    ],
-    "tgzB64": ""
-  },
-  {
-    "name": "firewall-0.0.1",
-    "priority": 200,
-    "template": true,
-    "config": [
-      {"name": "ssl_port", "value": "[[node.sslPort]]"},
-      {"name": "admin_port", "value": "[[node.adminPort]]"},
-      {"name": "dns_port", "value": "[[configuration.dnsPort]]"}
-    ],
-    "tgzB64": ""
-  },
-  {
-    "name": "bubble-0.0.1",
-    "priority": 300,
-    "template": true,
-    "config": [
-      {"name": "node_uuid", "value": "[[node.uuid]]"},
-      {"name": "network_uuid", "value": "[[node.network]]"},
-      {"name": "admin_port", "value": "[[node.adminPort]]"},
-      {"name": "ssl_port", "value": "[[node.sslPort]]"},
-      {"name": "public_base_uri", "value": "[[publicBaseUri]]"},
-      {"name": "sage_node", "value": "[[sageNode]]"},
-      {"name": "promo_code_policy", "value": "[[#compare fork '==' true]][[configuration.promoCodePolicy]][[else]]disabled[[/compare]]"},
-      {"name": "install_type", "value": "[[installType]]"},
-      {"name": "default_locale", "value": "[[network.locale]]"},
-      {"name": "time_zone", "value": "[[network.timezone]]"},
-      {"name": "bubble_version", "value": "[[configuration.version]]"},
-      {"name": "bubble_host", "value": "[[node.fqdn]]"},
-      {"name": "bubble_cname", "value": "[[network.networkDomain]]"},
-      {"name": "admin_user", "value": "[[node.ansibleUser]]"},
-      {"name": "db_encoding", "value": "UTF-8"},
-      {"name": "db_locale", "value": "en_US"},
-      {"name": "db_user", "value": "bubble"},
-      {"name": "db_name", "value": "bubble"},
-      {"name": "db_key", "value": "[[dbEncryptionKey]]"},
-      {"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"},
-      {"name": "is_fork", "value": "[[fork]]"},
-      {"name": "restore_key", "value": "[[restoreKey]]"},
-      {"name": "restore_timeout", "value": "[[restoreTimeoutSeconds]]"},
-      {"name": "test_mode", "value": "[[testMode]]"},
-      {"name": "error_url", "value": "[[error_url]]"},
-      {"name": "error_key", "value": "[[error_key]]"},
-      {"name": "error_env", "value": "[[error_env]]"}
-    ],
-    "optionalConfigNames": ["restore_key", "restore_timeout", "error_url", "error_key", "error_env"],
-    "tgzB64": ""
-  },
-  {
-    "name": "algo-0.0.1",
-    "priority": 400,
-    "template": true,
-    "install": "node",
-    "config": [
-      {"name": "server_name", "value": "[[node.fqdn]]"},
-      {"name": "endpoint", "value": "[[node.ip4]]"},
-      {"name": "dns_port", "value": "[[configuration.dnsPort]]"},
-      {"name": "ssl_port", "value": "[[node.sslPort]]"}
-    ],
-    "tgzB64": ""
-  },
-  {
-    "name": "nginx-0.0.1",
-    "priority": 500,
-    "template": true,
-    "config": [
-      {"name": "server_name", "value": "[[node.fqdn]]"},
-      {"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"},
-      {"name": "ssl_port", "value": "[[node.sslPort]]"},
-      {"name": "admin_port", "value": "[[node.adminPort]]"}
-    ],
-    "tgzB64": ""
-  },
-  {
-    "name": "mitmproxy-0.0.1",
-    "priority": 600,
-    "template": true,
-    "install": "node",
-    "config": [
-      {"name": "admin_port", "value": "[[node.adminPort]]"},
-      {"name": "bubble_network", "value": "[[node.network]]"},
-      {"name": "mitm_port", "value": "[[configuration.defaultMitmProxyPort]]"}
-    ],
-    "tgzB64": ""
-  },
-  {
-    "name": "bubble_finalizer-0.0.1",
-    "priority": 9999999,
-    "template": true,
-    "config": [
-      {"name": "server_alias", "value": "[[network.networkDomain]]"},
-      {"name": "restore_key", "value": "[[restoreKey]]"},
-      {"name": "install_type", "value": "[[installType]]"},
-      {"name": "bubble_java_opts", "value": "-XX:MaxRAM=[[expr nodeSize.memoryMB '//' '2.625']]m"}
-    ],
-    "optionalConfigNames": ["restore_key"],
-    "tgzB64": ""
-  }
-]

bubble-server/src/main/resources/ansible/roles/bubble_finalizer/files/bubble_role.json → bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json

@@ -1,5 +1,5 @@
 {
-  "name": "bubble_finalizer",
+  "name": "finalizer",
   "config": [
     {"name": "server_alias", "value": "[[network.networkDomain]]"},
     {"name": "restore_key", "value": "[[restoreKey]]"},

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_api.py

@@ -0,0 +1,129 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+import requests
+import traceback
+import sys
+import os
+import time
+import uuid
+import datetime
+import redis
+import json
+from bubble_config import bubble_network, bubble_port
+
+# Write python PID to file so that mitmdump_monitor.sh can check for excessive memory usage and restart if needed
+MITMDUMP_PID_FILE_PATH = '/home/mitmproxy/mitmdump.pid'
+MITMDUMP_PID_FILE = open(MITMDUMP_PID_FILE_PATH, "w")
+MITMDUMP_PID_FILE.write("%d" % os.getpid())
+MITMDUMP_PID_FILE.close()
+
+HEADER_USER_AGENT = 'User-Agent'
+HEADER_REFERER = 'Referer'
+
+CTX_BUBBLE_MATCHERS='X-Bubble-Matchers'
+CTX_BUBBLE_ABORT='X-Bubble-Abort'
+CTX_BUBBLE_PASSTHRU='X-Bubble-Passthru'
+CTX_BUBBLE_REQUEST_ID='X-Bubble-RequestId'
+CTX_CONTENT_LENGTH='X-Bubble-Content-Length'
+CTX_CONTENT_LENGTH_SENT='X-Bubble-Content-Length-Sent'
+BUBBLE_URI_PREFIX='/__bubble/'
+
+REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
+BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_'
+BUBBLE_ACTIVITY_LOG_EXPIRATION = 600
+
+def redis_set(name, value, ex):
+    REDIS.set(name, value, nx=True, ex=ex)
+    REDIS.set(name, value, xx=True, ex=ex)
+
+
+def bubble_log(message):
+    print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr)
+
+
+def bubble_activity_log(client_addr, server_addr, event, data):
+    key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4())
+    value = json.dumps({
+        'source': 'mitmproxy',
+        'client_addr': client_addr,
+        'server_addr': server_addr,
+        'event': event,
+        'data': data
+    })
+    bubble_log('bubble_activity_log: setting '+key+' = '+value)
+    redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION)
+    pass
+
+
+def bubble_passthru(remote_addr, addr, fqdn):
+    headers = {
+        'X-Forwarded-For': remote_addr,
+        'Accept' : 'application/json',
+        'Content-Type': 'application/json'
+    }
+    try:
+        data = {
+            'addr': str(addr),
+            'fqdn': str(fqdn),
+            'remoteAddr': remote_addr
+        }
+        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data)
+        return response.ok
+    except Exception as e:
+        bubble_log('bubble_passthru API call failed: '+repr(e))
+        traceback.print_exc()
+    return False
+
+
+def bubble_matchers(req_id, remote_addr, flow, host):
+    headers = {
+        'X-Forwarded-For': remote_addr,
+        'Accept' : 'application/json',
+        'Content-Type': 'application/json'
+    }
+    if HEADER_USER_AGENT not in flow.request.headers:
+        bubble_log('bubble_matchers: no User-Agent header, setting to UNKNOWN')
+        user_agent = 'UNKNOWN'
+    else:
+        user_agent = flow.request.headers[HEADER_USER_AGENT]
+
+    if HEADER_REFERER not in flow.request.headers:
+        bubble_log('bubble_matchers: no Referer header, setting to NONE')
+        referer = 'NONE'
+    else:
+        try:
+            referer = flow.request.headers[HEADER_REFERER].encode().decode()
+        except Exception as e:
+            bubble_log('bubble_matchers: error parsing Referer header: '+repr(e))
+            referer = 'NONE'
+
+    try:
+        data = {
+            'requestId': req_id,
+            'fqdn': host,
+            'uri': flow.request.path,
+            'userAgent': user_agent,
+            'referer': referer,
+            'remoteAddr': remote_addr
+        }
+        response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data)
+        if response.ok:
+            return response.json()
+        bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text))
+    except Exception as e:
+        bubble_log('bubble_matchers API call failed: '+repr(e))
+        traceback.print_exc()
+    return None
+
+def add_flow_ctx(flow, name, value):
+    if not hasattr(flow, 'bubble_ctx'):
+        flow.bubble_ctx = {}
+    flow.bubble_ctx[name] = value
+
+def get_flow_ctx(flow, name):
+    if not hasattr(flow, 'bubble_ctx'):
+        return None
+    if not name in flow.bubble_ctx:
+        return None
+    return flow.bubble_ctx[name]

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_modify.py

@@ -0,0 +1,178 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+import re
+import requests
+import urllib
+import traceback
+from mitmproxy.net.http import Headers
+from bubble_config import bubble_port, bubble_host_alias
+from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \
+    CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx
+
+BUFFER_SIZE = 4096
+HEADER_CONTENT_TYPE = 'Content-Type'
+HEADER_CONTENT_LENGTH = 'Content-Length'
+HEADER_CONTENT_ENCODING = 'Content-Encoding'
+HEADER_TRANSFER_ENCODING = 'Transfer-Encoding'
+BINARY_DATA_HEADER = {HEADER_CONTENT_TYPE: 'application/octet-stream'}
+
+
+def filter_chunk(chunk, req_id, last, content_encoding=None, content_type=None, content_length=None):
+    url = 'http://127.0.0.1:' + bubble_port + '/api/filter/apply/' + req_id
+    params_added = False
+    if chunk and content_type:
+        params_added = True
+        url = (url
+               + '?type=' + urllib.parse.quote_plus(content_type))
+        if content_encoding:
+            url = url + '&encoding=' + urllib.parse.quote_plus(content_encoding)
+        if content_length:
+            url = url + '&length=' + str(content_length)
+    if last:
+        if params_added:
+            url = url + '&last=true'
+        else:
+            url = url + '?last=true'
+
+    bubble_log('filter_chunk: url='+url)
+
+    response = requests.post(url, data=chunk, headers=BINARY_DATA_HEADER)
+    if not response.ok:
+        err_message = 'filter_chunk: Error fetching ' + url + ', HTTP status ' + str(response.status_code)
+        bubble_log(err_message)
+        return b''
+
+    return response.content
+
+
+def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type):
+    """
+    chunks is a generator that can be used to iterate over all chunks.
+    """
+    first = True
+    content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH)
+    try:
+        for chunk in chunks:
+            if content_length:
+                bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT)
+                chunk_len = len(chunk)
+                last = chunk_len + bytes_sent >= content_length
+                bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent))
+                add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len)
+            else:
+                last = False
+            if first:
+                yield filter_chunk(chunk, req_id, last, content_encoding, content_type, content_length)
+                first = False
+            else:
+                yield filter_chunk(chunk, req_id, last)
+        if not content_length:
+            yield filter_chunk(None, req_id, True)  # get the last bits of data
+    except Exception as e:
+        bubble_log('bubble_filter_chunks: exception='+repr(e))
+        traceback.print_exc()
+        yield None
+
+
+def bubble_modify(flow, req_id, content_encoding, content_type):
+    return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type)
+
+
+def send_bubble_response(response):
+    for chunk in response.iter_content(8192):
+        yield chunk
+
+
+def responseheaders(flow):
+
+    if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX):
+        uri = 'http://127.0.0.1:' + bubble_port + '/' + flow.request.path[len(BUBBLE_URI_PREFIX):]
+        bubble_log('responseheaders: sending special bubble request to '+uri)
+        headers = {
+            'Accept' : 'application/json',
+            'Content-Type': 'application/json'
+        }
+        response = None
+        if flow.request.method == 'GET':
+            response = requests.get(uri, headers=headers, stream=True)
+        elif flow.request.method == 'POST':
+            bubble_log('responseheaders: special bubble request: POST content is '+str(flow.request.content))
+            headers['Content-Length'] = str(len(flow.request.content))
+            response = requests.post(uri, data=flow.request.content, headers=headers)
+        else:
+            bubble_log('responseheaders: special bubble request: method '+flow.request.method+' not supported')
+        if response is not None:
+            bubble_log('responseheaders: special bubble request: response status = '+str(response.status_code))
+            flow.response.headers = Headers()
+            for key, value in response.headers.items():
+                flow.response.headers[key] = value
+            flow.response.status_code = response.status_code
+            flow.response.stream = lambda chunks: send_bubble_response(response)
+
+    else:
+        abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT)
+        if abort_code is not None:
+            bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code))
+            flow.response.headers = Headers()
+            flow.response.status_code = abort_code
+            flow.response.stream = lambda chunks: []
+
+        else:
+            req_id = get_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID)
+            matchers = get_flow_ctx(flow, CTX_BUBBLE_MATCHERS)
+            if req_id is not None and matchers is not None:
+                bubble_log('responseheaders: req_id='+req_id+' with matchers: '+repr(matchers))
+                if HEADER_CONTENT_TYPE in flow.response.headers:
+                    content_type = flow.response.headers[HEADER_CONTENT_TYPE]
+                    if matchers:
+                        any_content_type_matches = False
+                        for m in matchers:
+                            if 'contentTypeRegex' in m:
+                                typeRegex = m['contentTypeRegex']
+                                if typeRegex is None:
+                                    typeRegex = '^text/html.*'
+                                if re.match(typeRegex, content_type):
+                                    any_content_type_matches = True
+                                    bubble_log('responseheaders: req_id='+req_id+' found at least one matcher for content_type ('+content_type+'), filtering')
+                                    break
+                        if not any_content_type_matches:
+                            bubble_log('responseheaders: req_id='+req_id+' no matchers for content_type ('+content_type+'), passing thru')
+                            return
+
+                        if HEADER_CONTENT_ENCODING in flow.response.headers:
+                            content_encoding = flow.response.headers[HEADER_CONTENT_ENCODING]
+                        else:
+                            content_encoding = None
+
+                        content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None)
+                        bubble_log('responseheaders: req_id='+req_id+' content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
+                        flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type)
+                        if content_length_value:
+                            flow.response.headers['transfer-encoding'] = 'chunked'
+                            # find server_conn to set fake_chunks on
+                            if flow.live and flow.live.ctx:
+                                ctx = flow.live.ctx
+                                while not hasattr(ctx, 'server_conn'):
+                                    if hasattr(ctx, 'ctx'):
+                                        ctx = ctx.ctx
+                                    else:
+                                        bubble_log('responseheaders: error finding server_conn. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx)))
+                                        return
+                                if not hasattr(ctx, 'server_conn'):
+                                    bubble_log('responseheaders: error finding server_conn. ctx type='+str(type(ctx))+' vars='+str(vars(ctx)))
+                                    return
+                                content_length = int(content_length_value)
+                                ctx.server_conn.rfile.fake_chunks = content_length
+                                add_flow_ctx(flow, CTX_CONTENT_LENGTH, content_length)
+                                add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, 0)
+
+                    else:
+                        bubble_log('responseheaders: no matchers, passing thru')
+                        pass
+                else:
+                    bubble_log('responseheaders: no '+HEADER_CONTENT_TYPE +' header, passing thru')
+                    pass
+            else:
+                bubble_log('responseheaders: no '+CTX_BUBBLE_MATCHERS +' in ctx, passing thru')
+                pass

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_passthru.py

@@ -0,0 +1,112 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here:
+#
+# Copyright (c) 2013, Aldo Cortesi. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer
+from mitmproxy.exceptions import TlsProtocolException
+
+from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set
+import redis
+import json
+
+REDIS_DNS_PREFIX = 'bubble_dns_'
+REDIS_PASSTHRU_PREFIX = 'bubble_passthru_'
+REDIS_PASSTHRU_DURATION = 60 * 60  # 1 hour timeout on passthru
+
+REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0)
+
+
+def passthru_cache_prefix(client_addr, server_addr):
+    return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr
+
+
+class TlsFeedback(TlsLayer):
+    """
+    Monkey-patch _establish_tls_with_client to get feedback if TLS could be established
+    successfully on the client connection (which may fail due to cert pinning).
+    """
+    def _establish_tls_with_client(self):
+        client_address = self.client_conn.address[0]
+        server_address = self.server_conn.address[0]
+        try:
+            super(TlsFeedback, self)._establish_tls_with_client()
+        except TlsProtocolException as e:
+            bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru')
+            cache_key = passthru_cache_prefix(client_address, server_address)
+            fqdn = fqdn_for_addr(server_address)
+            redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION)
+            raise e
+
+
+def fqdn_for_addr(addr):
+    fqdn = REDIS.get(REDIS_DNS_PREFIX + addr)
+    if fqdn is None or len(fqdn) == 0:
+        bubble_log('check_bubble_passthru: no FQDN found for addr '+repr(addr)+', checking raw addr')
+        fqdn = b''
+    return fqdn.decode()
+
+
+def check_bubble_passthru(remote_addr, addr, fqdn):
+    if bubble_passthru(remote_addr, addr, fqdn):
+        bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True')
+        return {'fqdn': fqdn, 'addr': addr, 'passthru': True}
+    bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False')
+    return {'fqdn': fqdn, 'addr': addr, 'passthru': False}
+
+
+def should_passthru(remote_addr, addr):
+    prefix = 'should_passthru: '+repr(addr)+' '
+    bubble_log(prefix+'starting...')
+    cache_key = passthru_cache_prefix(remote_addr, addr)
+    passthru_json = REDIS.get(cache_key)
+    if passthru_json is None or len(passthru_json) == 0:
+        bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...')
+        fqdn = fqdn_for_addr(addr)
+        if fqdn is None or len(fqdn) == 0:
+            bubble_log(prefix+' no fqdn found for addr '+addr+', returning (uncached) passthru = False')
+            return {'fqdn': None, 'addr': addr, 'passthru': False}
+        passthru = check_bubble_passthru(remote_addr, addr, fqdn)
+        bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...")
+        redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION)
+    else:
+        bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis')
+        passthru = json.loads(passthru_json)
+        REDIS.touch(cache_key)
+    bubble_log(prefix+'returning '+repr(passthru))
+    return passthru
+
+
+def next_layer(next_layer):
+    if isinstance(next_layer, TlsLayer) and next_layer._client_tls:
+        client_address = next_layer.client_conn.address[0]
+        server_address = next_layer.server_conn.address[0]
+        passthru = should_passthru(client_address, server_address)
+        if passthru['passthru']:
+            bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address))
+            bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn'])
+            next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True)
+            next_layer.reply.send(next_layer_replacement)
+        else:
+            bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn'])
+            next_layer.__class__ = TlsFeedback

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/bubble_role.json

@@ -0,0 +1,15 @@
+{
+  "name": "mitmproxy-0.0.1",
+  "priority": 600,
+  "template": true,
+  "install": "node",
+  "config": [
+    {"name": "admin_port", "value": "[[node.adminPort]]"},
+    {"name": "bubble_network", "value": "[[node.network]]"},
+    {"name": "mitm_port", "value": "[[configuration.defaultMitmProxyPort]]"},
+    {"name": "server_name", "value": "[[node.fqdn]]"},
+    {"name": "server_alias", "value": "[[network.networkDomain]]"},
+    {"name": "ssl_port", "value": "[[node.sslPort]]"}
+  ],
+  "tgzB64": ""
+}

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/dns_spoofing.py

@@ -0,0 +1,148 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+import re
+import time
+import uuid
+from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx
+from bubble_config import bubble_host, bubble_host_alias
+
+# This regex extracts splits the host header into host and port.
+# Handles the edge case of IPv6 addresses containing colons.
+# https://bugzilla.mozilla.org/show_bug.cgi?id=45891
+parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$")
+
+
+class Rerouter:
+    @staticmethod
+    def get_matchers(flow, host):
+        if host is None:
+            return None
+
+        if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX):
+            bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path)
+            return None
+
+        remote_addr = str(flow.client_conn.address[0])
+        try:
+            host = host.decode()
+        except (UnicodeDecodeError, AttributeError):
+            try:
+                host = str(host)
+            except Exception as e:
+                bubble_log('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e))
+                return None
+
+        if host == bubble_host or host == bubble_host_alias:
+            bubble_log('get_matchers: request is for bubble itself ('+host+'), not matching')
+            return None
+
+        req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time())
+        bubble_log("get_matchers: requesting match decision for req_id="+req_id)
+        resp = bubble_matchers(req_id, remote_addr, flow, host)
+
+        if not resp:
+            bubble_log('get_matchers: no response for remote_addr/host: '+remote_addr+'/'+str(host))
+            return None
+
+        matchers = []
+        if 'matchers' in resp and resp['matchers'] is not None:
+            for m in resp['matchers']:
+                if 'urlRegex' in m:
+                    bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex'])
+                else:
+                    bubble_log('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping')
+                    continue
+                if re.match(m['urlRegex'], flow.request.path):
+                    bubble_log('get_matchers: rule matched, adding rule: '+m['rule'])
+                    matchers.append(m)
+                else:
+                    bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule'])
+        else:
+            bubble_log('get_matchers: no matchers. response='+repr(resp))
+
+        decision = None
+        if 'decision' in resp:
+            decision = resp['decision']
+
+        matcher_response = { 'decision': decision, 'matchers': matchers, 'request_id': req_id }
+        bubble_log("get_matchers: returning "+repr(matcher_response))
+        return matcher_response
+
+    def request(self, flow):
+        client_address = flow.client_conn.address[0]
+        server_address = flow.server_conn.address[0]
+        if flow.client_conn.tls_established:
+            flow.request.scheme = "https"
+            sni = flow.client_conn.connection.get_servername()
+            port = 443
+        else:
+            flow.request.scheme = "http"
+            sni = None
+            port = 80
+
+        host_header = flow.request.host_header
+        # bubble_log("dns_spoofing.request: host_header is "+repr(host_header))
+        if host_header:
+            m = parse_host_header.match(host_header)
+            if m:
+                host_header = m.group("host").strip("[]")
+                if m.group("port"):
+                    port = int(m.group("port"))
+
+        # Determine if this request should be filtered
+        if sni or host_header:
+            host = str(sni or host_header)
+            if host.startswith("b'"):
+                host = host[2:-1]
+            log_url = flow.request.scheme + '://' + host + flow.request.path
+            matcher_response = self.get_matchers(flow, sni or host_header)
+            if matcher_response:
+                if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru':
+                    bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...')
+                    add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True)
+                    bubble_activity_log(client_address, server_address, 'http_passthru', log_url)
+                    return
+
+                elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'):
+                    bubble_log('dns_spoofing.request: found abort code: ' + str(matcher_response['decision']) + ', aborting')
+                    if matcher_response['decision'] == 'abort_ok':
+                        abort_code = 200
+                    elif matcher_response['decision'] == 'abort_not_found':
+                        abort_code = 404
+                    else:
+                        bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found')
+                        abort_code = 404
+                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code)
+                    bubble_activity_log(client_address, server_address, 'http_abort' + str(abort_code), log_url)
+                    return
+
+                elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match':
+                    bubble_log('dns_spoofing.request: decision was no_match, passing thru...')
+                    bubble_activity_log(client_address, server_address, 'http_no_match', log_url)
+                    return
+
+                elif ('matchers' in matcher_response
+                      and 'request_id' in matcher_response
+                      and len(matcher_response['matchers']) > 0):
+                    req_id = matcher_response['request_id']
+                    bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
+                    add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers'])
+                    add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id)
+                    bubble_activity_log(client_address, server_address, 'http_match', log_url)
+                else:
+                    bubble_log('dns_spoofing.request: no rules returned, passing thru...')
+                    bubble_activity_log(client_address, server_address, 'http_no_rules', log_url)
+            else:
+                bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...')
+                # bubble_activity_log(client_address, server_address, 'http_no_matcher_response', log_url)
+        else:
+            bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path)
+            bubble_activity_log(client_address, server_address, 'http_no_sni_or_host', 'n/a')
+
+        flow.request.host_header = host_header
+        flow.request.host = sni or host_header
+        flow.request.port = port
+
+
+addons = [Rerouter()]

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/install_cert.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+CERT="${1:?no cert provided}"
+TIMEOUT=${2:-0}
+
+function die {
+  echo 1>&2 "${1}"
+  exit 1
+}
+
+START=$(date +%s)
+while [[ ! -f "${CERT}" ]] ; do
+  ELAPSED=$(expr $(date +%s) - ${START})
+  if [[ ${ELAPSED} -gt ${TIMEOUT} ]] ; then
+    break
+  fi
+  echo "Cert file does not exist, sleeping then rechecking: ${CERT}"
+  sleep 5s
+done
+
+if [[ ! -f "${CERT}" ]] ; then
+  die "Cert file does not exist: ${CERT}"
+fi
+
+if [[ "${CERT}" == *.pem || "${CERT}" == *.p12 ]] ; then
+  openssl x509 -in "${CERT}" -inform PEM -out "${CERT}.crt" || die "Error converting certificate"
+  CERT="${CERT}.crt"
+fi
+
+mkdir -p /usr/local/share/ca-certificates || die "Error ensuring CA certs directory exists"
+cp "${CERT}" /usr/local/share/ca-certificates || die "Error installing certificate"
+update-ca-certificates || die "Error updating CA certificates"

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/mitmdump_monitor.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+LOG=/tmp/bubble.mitmdump_monitor.log
+
+function die {
+  echo 1>&2 "${1}"
+  log "${1}"
+  exit 1
+}
+
+function log {
+  echo "$(date): ${1}" >> ${LOG}
+}
+
+BUBBLE_MITM_MARKER=/home/bubble/.mitmdump_monitor
+ROOT_KEY_MARKER=/usr/share/bubble/mitmdump_monitor
+MITMDUMP_PID_FILE=/home/mitmproxy/mitmdump.pid
+MAX_MITM_PCT_MEM=18
+
+# Start with MITM proxy turned off
+if [[ ! -f ${BUBBLE_MITM_MARKER} ]] ; then
+  echo -n off > ${BUBBLE_MITM_MARKER} && chown bubble ${BUBBLE_MITM_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}"
+fi
+if [[ ! -f ${ROOT_KEY_MARKER} ]] ; then
+  sleep 1s
+  mkdir -p "$(dirname ${ROOT_KEY_MARKER})" && chmod 755 "$(dirname ${ROOT_KEY_MARKER})" || log "Error creating or setting permissions on ${ROOT_KEY_MARKER}"
+  echo -n on > ${ROOT_KEY_MARKER} && touch ${ROOT_KEY_MARKER} && chmod 644 ${ROOT_KEY_MARKER} || log "Error writing 'on' to ${ROOT_KEY_MARKER}"
+fi
+
+function ensureMitmOn {
+  log "Flushing PREROUTING before enabling MITM services"
+  iptables -F PREROUTING  -t nat || log "Error flushing port forwarding when enabling MITM services"
+  log "Enabling MITM port forwarding on TCP port 80 -> 8888"
+  iptables -I PREROUTING 1 -t nat -p tcp --dport 80 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 80 -> 8888"
+  log "Enabling MITM port forwarding on TCP port 443 -> 8888"
+  iptables -I PREROUTING 1 -t nat -p tcp --dport 443 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 443 -> 8888"
+  echo -n on > ${ROOT_KEY_MARKER}
+}
+
+function ensureMitmOff {
+  log "Flushing PREROUTING to disable MITM services"
+  iptables -F PREROUTING  -t nat || log "Error flushing port forwarding when disabling MITM services"
+  echo -n off > ${ROOT_KEY_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}"
+}
+
+log "Watching marker file ${BUBBLE_MITM_MARKER} ..."
+sleep 2s && touch ${BUBBLE_MITM_MARKER} || log "Error touching ${BUBBLE_MITM_MARKER}" # first time through, always check and set on/off state
+while : ; do
+  if [[ $(stat -c %Y ${BUBBLE_MITM_MARKER}) -gt $(stat -c %Y ${ROOT_KEY_MARKER}) ]] ; then
+    if [[ ! -z "$(cmp -b ${ROOT_KEY_MARKER} ${BUBBLE_MITM_MARKER})" ]] ; then
+      if [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "on" ]] ; then
+        ensureMitmOn
+      elif [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "off" ]] ; then
+        ensureMitmOff
+      else
+        log "Error: marker file ${BUBBLE_MITM_MARKER} contained invalid value: $(cat ${BUBBLE_MITM_MARKER} | head -c 5)"
+      fi
+    fi
+  fi
+
+  # Check process memory usage, restart mitmdump if memory goes above max % allowed
+  if [[ -f ${MITMDUMP_PID_FILE} && -s ${MITMDUMP_PID_FILE} ]] ; then
+    MITM_PID="$(cat ${MITMDUMP_PID_FILE})"
+    PCT_MEM="$(ps q ${MITM_PID} -o %mem --no-headers | tr -d [[:space:]] | cut -f1 -d. | sed 's/[^0-9]*//g')"
+    # log "Info: mitmdump pid ${MITM_PID} using ${PCT_MEM}% of memory"
+    if [[ ! -z "${PCT_MEM}" ]] ; then
+      if [[ ${PCT_MEM} -ge ${MAX_MITM_PCT_MEM} ]] ; then
+        log "Warn: mitmdump: pid=$(cat ${MITMDUMP_PID_FILE}) memory used > max, restarting: ${PCT_MEM}% >= ${MAX_MITM_PCT_MEM}%"
+        supervisorctl restart mitmdump
+      fi
+    else
+      log "Error: could not determine mitmdump % memory, maybe PID file ${MITMDUMP_PID_FILE} is out of date? pid found was ${MITM_PID}"
+    fi
+  else
+    log "Error: mitmdump PID file ${MITMDUMP_PID_FILE} not found or empty"
+  fi
+  sleep 5s
+done

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/reuse_bubble_mitm_certs.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+function die {
+  echo 1>&2 "${1}"
+  exit 1
+}
+
+CERTS_BACKUP=/home/bubble/mitm_certs
+if [[ ! -d ${CERTS_BACKUP} ]] ; then
+  echo "No mitm_certs backup found, skipping restore"
+  exit 0
+fi
+
+MITM_CERTS=/home/mitmproxy/.mitmproxy
+if [[ -d ${MITM_CERTS} ]] ; then
+  echo "Removing obsolete mitm certs: ${MITM_CERTS}"
+  rm -rf ${MITM_CERTS} || die "Error removing obsolete mitm certs"
+  if [[ -d ${MITM_CERTS} ]] ; then
+    die "Error removing obsolete mitm certs: dir still exists: ${MITM_CERTS}"
+  fi
+fi
+
+mkdir -p ${MITM_CERTS} || die "Error creating mitm certs dir: ${MITM_CERTS}"
+chmod 750 ${MITM_CERTS} || die "Error setting permissions on mitm certs dir: ${MITM_CERTS}"
+cp -R ${CERTS_BACKUP}/* ${MITM_CERTS}/ || die "Error restoring mitm certs"
+chown -R mitmproxy ${MITM_CERTS} || die "Error changing ownership of ${MITM_CERTS}"
+chgrp -R root ${MITM_CERTS} || die "Error changing group ownership of ${MITM_CERTS}"
+chmod 440 ${MITM_CERTS}/* || die "Error setting permissions on mitm certs files"

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/run_mitmdump.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+MITM_PORT=${1:?no port provided}
+cd /home/mitmproxy/mitmproxy && \
+./dev.sh && . ./venv/bin/activate && \
+mitmdump \
+  --listen-host 0.0.0.0 \
+  --listen-port ${MITM_PORT} \
+  --showhost \
+  --no-http2 \
+  --set block_global=true \
+  --set block_private=false \
+  --set termlog_verbosity=debug \
+  --set flow_detail=3 \
+  --set stream_large_bodies=5m \
+  --set keep_host_header \
+  -s ./dns_spoofing.py \
+  -s ./bubble_passthru.py \
+  -s ./bubble_modify.py \
+  --mode transparent

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/set_cert_name.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+MITM_DIR=${1:?no mitm dir specified}
+CERT_NAME=${2:?no cert name specified}
+
+if [[ ! -d "${MITM_DIR}" ]] ; then
+  echo "mitm dir does not exist or is not a directory: ${MITM_DIR}"
+  exit 1
+fi
+
+OPTIONS_FILE="${MITM_DIR}/mitmproxy/options.py"
+if [[ ! -f "${OPTIONS_FILE}" ]] ; then
+  echo "options.py not found in mitm dir: ${MITM_DIR}"
+  exit 1
+fi
+
+if [[ $(cat "${OPTIONS_FILE}" | egrep '^CONF_BASENAME =' | grep "${CERT_NAME}" | wc -l | tr -d ' ') -eq 0 ]] ; then
+  temp="$(mktemp /tmp/options.py.XXXXXXX)"
+  cat "${OPTIONS_FILE}" | sed -e 's/^CONF_BASENAME\s*=.*/CONF_BASENAME = "'"${CERT_NAME}"'"/' > "${temp}"
+  mv "${temp}" "${OPTIONS_FILE}"
+fi

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitmdump_monitor.conf

@@ -0,0 +1,5 @@
+
+[program:mitmdump_monitor]
+stdout_logfile = /dev/null
+stderr_logfile = /dev/null
+command=/usr/local/sbin/mitmdump_monitor.sh

bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml

@@ -0,0 +1,115 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+- name: Install python3, pip, virtualenv and required dependencies
+  apt:
+    name: [ 'python3-pip', 'python3-venv', 'libc6-dev', 'libpython3-dev', 'g++', 'libffi-dev' ]
+    state: present
+    update_cache: yes
+
+- name: Install supervisor conf file
+  template:
+    src: supervisor_mitmproxy.conf.j2
+    dest: /etc/supervisor/conf.d/mitmproxy.conf
+    owner: root
+    group: root
+    mode: 0400
+
+- name: Create mitmproxy user
+  user:
+    name: mitmproxy
+    comment: mitmdump user
+    shell: /bin/bash
+    system: yes
+    home: /home/mitmproxy
+
+- name: Creates mitmproxy dir
+  file:
+    path: /home/mitmproxy/mitmproxy
+    owner: mitmproxy
+    group: mitmproxy
+    mode: 0755
+    state: directory
+
+- name: Unzip mitmproxy.zip
+  unarchive:
+    src: mitmproxy.zip
+    dest: /home/mitmproxy/mitmproxy
+
+- name: Copy mitmdump files
+  copy:
+    src: "{{ item }}"
+    dest: "/home/mitmproxy/mitmproxy/{{ item }}"
+    owner: mitmproxy
+    group: mitmproxy
+    mode: 0500
+  with_items:
+    - bubble_api.py
+    - dns_spoofing.py
+    - bubble_passthru.py
+    - bubble_modify.py
+    - run_mitmdump.sh
+
+- name: Install cert helper scripts
+  copy:
+    src: "{{ item }}"
+    dest: "/usr/local/bin/{{ item }}"
+    owner: root
+    group: root
+    mode: 0500
+  with_items:
+    - install_cert.sh
+    - set_cert_name.sh
+    - reuse_bubble_mitm_certs.sh
+
+- name: Set the cert name
+  shell: set_cert_name.sh /home/mitmproxy/mitmproxy {{ server_alias }}
+
+- name: Set ownership of mitmproxy files
+  shell: chown -R mitmproxy /home/mitmproxy/mitmproxy
+
+- name: Reuse bubble mitm certs if available
+  shell: reuse_bubble_mitm_certs.sh
+
+- name: Copy bubble_config.py to /home/mitmproxy/mitmproxy
+  template:
+    src: bubble_config.py.j2
+    dest: /home/mitmproxy/mitmproxy/bubble_config.py
+    owner: mitmproxy
+    group: mitmproxy
+    mode: 0500
+
+- name: Fix missing symlink for libstdc++
+  file:
+    src: /usr/lib/x86_64-linux-gnu/libstdc++.so.6
+    dest: /usr/lib/x86_64-linux-gnu/libstdc++.so
+    owner: root
+    group: root
+    state: link
+
+- name: Restart dnscrypt-proxy
+  shell: service dnscrypt-proxy restart
+
+- name: restart supervisord
+  service:
+    name: supervisor
+    enabled: yes
+    state: restarted
+
+- import_tasks: route.yml
+
+- name: Install mitmdump_monitor
+  copy:
+    src: "mitmdump_monitor.sh"
+    dest: "/usr/local/sbin/mitmdump_monitor.sh"
+    owner: root
+    group: root
+    mode: 0500
+
+- name: Install mitmdump_monitor supervisor conf file
+  copy:
+    src: supervisor_mitmdump_monitor.conf
+    dest: /etc/supervisor/conf.d/mitmdump_monitor.conf
+
+- name: Ensure mitmdump_monitor is started
+  shell: supervisorctl restart mitmdump_monitor

bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/route.yml

@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+- sysctl:
+    name: net.ipv4.ip_forward
+    value: 1
+    sysctl_set: yes
+- sysctl:
+    name: net.ipv6.conf.all.forwarding
+    value: 1
+    sysctl_set: yes
+- sysctl:
+    name: net.ipv4.conf.all.send_redirects
+    value: 0
+    sysctl_set: yes
+
+- name: "Allow MITM private port"
+  iptables:
+    chain: INPUT
+    action: insert
+    rule_num: 10
+    protocol: tcp
+    destination_port: "{{ mitm_port }}"
+    ctstate: NEW
+    syn: match
+    jump: ACCEPT
+    comment: Accept new local TCP DNS connections on private port
+  become: yes
+
+- name: Route port 80 through mitmproxy
+  iptables:
+    table: nat
+    chain: PREROUTING
+    action: insert
+    rule_num: 1
+    protocol: tcp
+    destination_port: 80
+    jump: REDIRECT
+    to_ports: "{{ mitm_port }}"
+
+- name: Route port 443 through mitmproxy
+  iptables:
+    table: nat
+    chain: PREROUTING
+    action: insert
+    rule_num: 2
+    protocol: tcp
+    destination_port: 443
+    jump: REDIRECT
+    to_ports: "{{ mitm_port }}"
+
+- name: save iptables rules
+  shell: iptables-save > /etc/iptables/rules.v4
+  become: yes
+
+- name: save iptables v6 rules
+  shell: ip6tables-save > /etc/iptables/rules.v6
+  become: yes

bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/bubble_config.py.j2

@@ -0,0 +1,5 @@
+bubble_network = '{{ bubble_network }}'
+bubble_port = '{{ admin_port }}';
+bubble_host = '{{ server_name }}'
+bubble_host_alias = '{{ server_alias }}'
+bubble_ssl_port = '{{ ssl_port }}'

bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/supervisor_mitmproxy.conf.j2

@@ -0,0 +1,7 @@
+
+[program:mitmdump]
+stdout_logfile = /home/mitmproxy/mitmdump-out.log
+stderr_logfile = /home/mitmproxy/mitmdump-err.log
+command=sudo -H -u mitmproxy bash -c "/home/mitmproxy/mitmproxy/run_mitmdump.sh {{ mitm_port }}"
+stopasgroup=true
+stopsignal=QUIT

bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json

@@ -17,8 +17,8 @@
 { "percent": 76,"messageKey":"role_nginx",                  "pattern":"TASK \\[nginx : [\\w\\s]+] \\*{5,}" },
 { "percent": 81,"messageKey":"role_nginx_certbot",          "pattern":"TASK \\[nginx : Init certbot] \\*{5,}" },
 { "percent": 91,"messageKey":"role_mitmproxy",              "pattern":"TASK \\[mitmproxy : [\\w\\s]+] \\*{5,}" },
-{ "percent": 94,"messageKey":"role_bubble_finalizer",       "pattern":"TASK \\[bubble_finalizer : [\\w\\s]+] \\*{5,}" },
-{ "percent": 98,"messageKey":"role_bubble_finalizer_touch", "pattern":"TASK \\[bubble_finalizer : Touch first-time setup file] \\*{5,}" },
-{ "percent": 99,"messageKey":"role_bubble_finalizer_start", "pattern":"TASK \\[bubble_finalizer : Ensure bubble is started] \\*{5,}" },
+{ "percent": 94,"messageKey":"role_finalizer",              "pattern":"TASK \\[finalizer : [\\w\\s]+] \\*{5,}" },
+{ "percent": 98,"messageKey":"role_finalizer_touch",        "pattern":"TASK \\[finalizer : Touch first-time setup file] \\*{5,}" },
+{ "percent": 99,"messageKey":"role_finalizer_start",        "pattern":"TASK \\[finalizer : Ensure bubble is started] \\*{5,}" },
 {"percent": 100,"messageKey":"install_complete",            "pattern":"PLAY RECAP \\*{5,}" }
 ]

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -353,9 +353,9 @@ meter_tick_role_bubble_algo=Setting up VPN
 meter_tick_role_nginx=Setting up web server
 meter_tick_role_nginx_certbot=Installing SSL certificates
 meter_tick_role_mitmproxy=Setting up MITM server
-meter_tick_role_bubble_finalizer=Finalizing Bubble installation
-meter_tick_role_bubble_finalizer_touch=Turning on "first-time" setting to allow you to unlock your Bubble
-meter_tick_role_bubble_finalizer_start=Starting Bubble API services
+meter_tick_role_finalizer=Finalizing Bubble installation
+meter_tick_role_finalizer_touch=Turning on "first-time" setting to allow you to unlock your Bubble
+meter_tick_role_finalizer_start=Starting Bubble API services
 meter_tick_install_complete=Bubble installation completed
 
 # Launch progress meter: success marker

bubble-server/src/main/resources/packer/node-roles.txt

@@ -4,4 +4,4 @@ nginx
 algo
 mitmproxy
 bubble
-bubble_finalizer
+finalizer

bubble-server/src/main/resources/packer/sage-roles.txt

@@ -2,4 +2,4 @@ common
 firewall
 nginx
 bubble
-bubble_finalizer
+finalizer