fix health check, stagger mitm startup to conserve memory

bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java

@@ -161,7 +161,7 @@ public class AnsiblePrepService {
         if (installType == AnsibleInstallType.sage) return (int) (((double) memoryMB) * 0.6d);
         if (memoryMB >= 4096) return (int) (((double) memoryMB) * 0.6d);
         if (memoryMB >= 2048) return (int) (((double) memoryMB) * 0.5d);
-        if (memoryMB >= 1024) return (int) (((double) memoryMB) * 0.196d);
+        if (memoryMB >= 1024) return (int) (((double) memoryMB) * 0.24d);
         // no nodes are this small, API probably would not start, not enough memory
         return (int) (((double) memoryMB) * 0.19d);
     }

bubble-server/src/main/resources/ansible/roles/finalizer/files/bubble_role.json

@@ -6,6 +6,7 @@
     {"name": "restore_key", "value": "[[restoreKey]]"},
     {"name": "install_type", "value": "[[installType]]"},
     {"name": "bubble_java_opts", "value": "-XX:MaxRAM=[[jvmMaxRamMB]]m"},
+    {"name": "total_memory", "value": "[[nodeSize.memoryMB]]"},
     {"name": "cert_name", "value": "bubble-[[network.shortId]]"}
   ],
   "optionalConfigNames": ["restore_key"]

bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml

@@ -25,6 +25,11 @@
     src: "supervisor_bubble.conf.j2"
     dest: /etc/supervisor/conf.d/bubble.conf
 
+# Save 1% of memory, every bit counts on small nodes
+- name: Disable peer manager on small nodes
+  shell: bash -c "supervisorctl stop bubble_peer_manager && rm /etc/supervisor/conf.d/bubble_peer_manager.conf"
+  when: total_memory < 2048
+
 - name: save iptables v4 rules
   shell: iptables-save > /etc/iptables/rules.v4
   become: yes

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitm_monitor.conf

@@ -3,3 +3,4 @@
 stdout_logfile = /dev/null
 stderr_logfile = /dev/null
 command=/usr/local/sbin/mitm_monitor.sh
+stopsignal=QUIT

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py

@@ -9,11 +9,9 @@ import urllib
 import traceback
 
 from mitmproxy.net.http import Headers
-from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody
 
 from bubble_config import bubble_port, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri
-from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \
-    CTX_BUBBLE_FLEX, CTX_BUBBLE_SPECIAL, \
+from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_FLEX, \
     status_reason, get_flow_ctx, add_flow_ctx, bubble_async, async_client, cleanup_async, \
     is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \
     CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \
@@ -286,8 +284,6 @@ def responseheaders(flow):
         flex_flow = process_flex(flex_flow)
     else:
         flex_flow = None
-    if bubble_log.isEnabledFor(DEBUG):
-        bubble_log.debug('responseheaders: flex_flow = '+repr(flex_flow))
     bubble_filter_response(flow, flex_flow)
 
 

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py

@@ -30,9 +30,9 @@ import uuid
 from mitmproxy.net.http import headers as nheaders
 
 from bubble_api import bubble_matchers, bubble_activity_log, \
-    HEALTH_CHECK_URI, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \
+    CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \
     CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \
-    is_bubble_special_path, is_bubble_health_check, \
+    is_bubble_special_path, is_bubble_health_check, health_check_response, \
     is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain
 from bubble_config import bubble_host, bubble_host_alias
 from bubble_flex import new_flex_flow
@@ -49,11 +49,9 @@ class Rerouter:
         if host is None:
             return None
 
-        is_health_check = is_bubble_health_check(flow.request.path)
         if is_bubble_special_path(flow.request.path):
-            if not is_health_check:
-                if bubble_log.isEnabledFor(DEBUG):
-                    bubble_log.debug("get_matchers: not filtering special bubble path: "+flow.request.path)
+            if bubble_log.isEnabledFor(DEBUG):
+                bubble_log.debug("get_matchers: not filtering special bubble path: "+flow.request.path)
             return None
 
         client_addr = str(flow.client_conn.address[0])
@@ -138,7 +136,6 @@ class Rerouter:
                     port = int(m.group("port"))
 
         # Determine if this request should be filtered
-        is_health_check = False
         host = None
         path = flow.request.path
         if sni or host_header:
@@ -151,14 +148,17 @@ class Rerouter:
             # If http, we validate client/server here
             if is_http:
                 fqdns = [host]
-                if is_bubble_request(server_addr, fqdns):
-                    is_health_check = path.startswith(HEALTH_CHECK_URI)
-                    if not is_health_check:
-                        if bubble_log.isEnabledFor(DEBUG):
-                            bubble_log.debug('bubble_handle_request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path)
-                        add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
-                        add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path)
-                        return None
+                if is_bubble_health_check(path):
+                    # Health check
+                    health_check_response(flow)
+                    return None
+
+                elif is_bubble_request(server_addr, fqdns):
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('bubble_handle_request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path)
+                    add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301)
+                    add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path)
+                    return None
 
                 elif is_sage_request(server_addr, fqdns):
                     if bubble_log.isEnabledFor(DEBUG):
@@ -170,7 +170,7 @@ class Rerouter:
                 elif is_not_from_vpn(client_addr):
                     # todo: add to fail2ban
                     if bubble_log.isEnabledFor(WARNING):
-                        bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url)
+                        bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url+' host='+host)
                     bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns)
                     add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404)
                     return None
@@ -225,10 +225,9 @@ class Rerouter:
                             bubble_log.debug('bubble_handle_request: no rules returned, passing thru...')
                         bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url)
                 else:
-                    if not is_health_check:
-                        if bubble_log.isEnabledFor(DEBUG):
-                            bubble_log.debug('bubble_handle_request: no matcher_response returned, passing thru...')
-                        # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url)
+                    if bubble_log.isEnabledFor(DEBUG):
+                        bubble_log.debug('bubble_handle_request: no matcher_response returned, passing thru...')
+                    # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url)
 
         elif is_http and is_not_from_vpn(client_addr):
             # todo: add to fail2ban

bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh

@@ -3,6 +3,10 @@
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
 LOG=/var/log/bubble/mitm_monitor.log
+if [[ ! -f ${LOG} ]] ; then
+  touch ${LOG}
+fi
+chgrp mitmproxy ${LOG} && chmod 660 ${LOG}  # allow run_mitm.sh to write to this log
 
 function die {
   echo 1>&2 "${1}"
@@ -76,22 +80,25 @@ function fullMitmReset {
 function healthCheck {
   MITM_PORT=${1}
   START=$(date +%s)
-  HEALTH_CHECK_TIMEOUT=20
+  HEALTH_CHECK_TIMEOUT=30
   HEALTH_OK="NOT_RUN"
+  HC_URL="http://$(hostname):${MITM_PORT}/__bubble/__mitm_health"
   while [[ $(expr $(date +%s) - ${START}) -le ${HEALTH_CHECK_TIMEOUT} ]] ; do
-    # log "Performing health check on mitm${MITM_PORT}..."
-    CURL_OUT="$(curl --silent --connect-timeout 2 --max-time 2 http://$(hostname):${MITM_PORT}/__bubble/__mitm_health 2>> ${LOG})"
+    # log "Performing health check on mitm${MITM_PORT} via ${HC_URL} ..."
+    CURL_OUT="$(curl --silent --connect-timeout 2 --max-time 2 ${HC_URL} 2>> ${LOG})"
     if [[ ! -z ${CURL_OUT} && ${CURL_OUT} == "OK" ]] ; then
-      # log "Health check on mitm${MITM_PORT}: OK"
+      # log "Health check on mitm${MITM_PORT} via ${HC_URL} : OK"
       echo -n "OK"
       return
     else
-      log "Health check on mitm${MITM_PORT}: failed: ${CURL_OUT}"
+      log "Health check on mitm${MITM_PORT} via ${HC_URL} failed: ${CURL_OUT}"
       HEALTH_OK="CURL_FAIL"
     fi
-    sleep 1s
+    sleep 5s
+    DELTA=$(expr $(date +%s) - ${START})
+    log "$(expr ${HEALTH_CHECK_TIMEOUT} - ${DELTA}) seconds before restart"
   done
-  log "Health check: final failure for mitm${MITM_PORT}, returning ${HEALTH_OK}"
+  log "Health check: final failure for mitm${MITM_PORT} via ${HC_URL} returning ${HEALTH_OK}"
   echo -n "${HEALTH_OK}"
 }
 

bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh

@@ -3,12 +3,40 @@
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
 PORT=${1:-8888}
-echo "Starting mitmproxy on port ${PORT} ..."
+MITM_PORT_FILE=/home/mitmproxy/mitmproxy_port
+LOG=/var/log/bubble/mitm_monitor.log
+
+function log {
+  echo "[mitm${PORT}] $(date): ${1}" | tee -a ${LOG}
+}
+
+if [[ -f ${MITM_PORT_FILE} ]] ; then
+  MITM_PORT="$(cat ${MITM_PORT_FILE})"
+  START=$(date +%s)
+  TIMEOUT=30
+  while [[ ! -s "${MITM_PORT_FILE}" ]] ; do
+    log "MITM_PORT_FILE was empty: ${MITM_PORT_FILE} -- waiting for it to exist"
+    if [[ $(expr $(date +%s) - ${START}) -gt ${TIMEOUT} ]] ; then
+      log "timeout waiting for MITM_PORT_FILE to exist: ${MITM_PORT_FILE} -- starting anyway"
+      break
+    fi
+    sleep 5s
+  done
+  if [[ -s ${MITM_PORT_FILE} ]] ; then
+    MITM_PORT="$(cat ${MITM_PORT_FILE})"
+    if [[ ! -z "${MITM_PORT}" && ${MITM_PORT} -ne ${PORT} ]] ; then
+      log "Our port (${PORT}) is not the primary mitm port (${MITM_PORT}), delaying startup by 30 seconds"
+      sleep 30s
+    fi
+  fi
+fi
+
+log "Starting mitmproxy on port ${PORT} ..."
 
 VENV_DIR="/home/mitmproxy/mitmproxy/venv"
 SETUP_VENV=1
 if [[ -d ${VENV_DIR} && $(find ${VENV_DIR} -type f -name "redis*" | wc -c | tr -d ' ') -gt 0 ]] ; then
-  echo "venv dir looks OK, skipping venv setup"
+  log "venv dir looks OK, skipping venv setup"
   SETUP_VENV=0
 fi