use two instances of mitm and switch to the other when one runs out of memory

bubble-server/src/main/java/bubble/model/cloud/BubbleNetwork.java

@@ -216,7 +216,7 @@ public class BubbleNetwork extends IdentifiableBase implements HasNetwork, HasBu
     private static final List<String> RESERVED_NAMES = Arrays.asList(
             "root", "postmaster", "hostmaster", "webmaster",
             "dns", "dnscrypt", "dnscrypt-proxy", "ftp", "www", "www-data", "postgres", "ipfs",
-            "redis", "nginx", "mitmproxy", "mitmdump", "algo", "algovpn");
+            "redis", "nginx", "mitm", "mitmproxy", "mitmdump", "algo", "algovpn");
 
     public static boolean isReservedName(String name) { return RESERVED_NAMES.contains(name); }
 

bubble-server/src/main/java/bubble/service/account/MitmControlService.java

@@ -26,10 +26,10 @@ public class MitmControlService {
 
     @Autowired private SelfNodeService selfNodeService;
 
-    public static final File MITM_CONTROL_FILE = new File("/home/bubble/.mitmdump_monitor");
-    public static final File MITM_ROOT_CONTROL_FILE = new File("/usr/share/bubble/mitmdump_monitor");
+    public static final File MITM_CONTROL_FILE = new File("/home/bubble/.mitm_monitor");
+    public static final File MITM_ROOT_CONTROL_FILE = new File("/usr/share/bubble/mitm_monitor");
 
-    // must be longer than the sleep time in mitmdump_monitor.sh (currently 5 seconds)
+    // must be longer than the sleep time in mitm_monitor.sh (currently 5 seconds)
     private static final long MITM_CONTROL_TIMEOUT = SECONDS.toMillis(10);
     private static final long MITM_MONITOR_SLEEP = SECONDS.toMillis(1);
 

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitmdump_monitor.conf

@@ -1,5 +0,0 @@
-
-[program:mitmdump_monitor]
-stdout_logfile = /dev/null
-stderr_logfile = /dev/null
-command=/usr/local/sbin/mitmdump_monitor.sh

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitmproxy.conf

@@ -1,7 +0,0 @@
-
-[program:mitmdump]
-stdout_logfile = /home/mitmproxy/mitmdump-out.log
-stderr_logfile = /home/mitmproxy/mitmdump-err.log
-command=sudo -H -u mitmproxy bash -c "/home/mitmproxy/mitmproxy/run_mitmdump.sh"
-stopasgroup=true
-stopsignal=QUIT

bubble-server/src/main/resources/ansible/roles/mitmproxy/files/supervisor_mitmproxy_monitor.conf

@@ -0,0 +1,5 @@
+
+[program:mitm_monitor]
+stdout_logfile = /dev/null
+stderr_logfile = /dev/null
+command=/usr/local/sbin/mitm_monitor.sh

bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml

@@ -18,31 +18,53 @@
 - name: Ensure mitmproxy user owns all mitmproxy files
   shell: chown -R mitmproxy /home/mitmproxy/mitmproxy
 
-- name: Install supervisor conf file
-  copy:
-    src: supervisor_mitmproxy.conf
-    dest: /etc/supervisor/conf.d/mitmproxy.conf
+- name: Install mitmproxy1 supervisor conf file
+  template:
+    src: supervisor_mitmproxy.conf.j2
+    dest: /etc/supervisor/conf.d/mitm8888.conf
     owner: root
     group: root
     mode: 0400
+  vars:
+    port: 8888
+
+- name: Install mitmproxy2 supervisor conf file
+  template:
+    src: supervisor_mitmproxy.conf.j2
+    dest: /etc/supervisor/conf.d/mitm9999.conf
+    owner: root
+    group: root
+    mode: 0400
+  vars:
+    port: 9999
+
+- name: Install mitmproxy_port file
+  copy:
+    src: mitmproxy_port
+    dest: /home/mitmproxy/mitmproxy_port
+    owner: mitmproxy
+    group: mitmproxy
+    mode: 0600
 
-- name: Install mitmdump_monitor supervisor conf file
+- name: Install mitmproxy_monitor supervisor conf file
   copy:
-    src: supervisor_mitmdump_monitor.conf
-    dest: /etc/supervisor/conf.d/mitmdump_monitor.conf
+    src: supervisor_mitmproxy_monitor.conf
+    dest: /etc/supervisor/conf.d/mitmproxy_monitor.conf
 
-- name: Allow MITM private port
+- name: Allow mitmproxy private ports
   iptables:
     chain: INPUT
     protocol: tcp
-    destination_port: 8888
+    destination_port: "{{ item }}"
     ctstate: NEW
     syn: match
     jump: ACCEPT
     comment: Accept new local connections on mitm port
   become: yes
   tags: algo_related
-  # ensuring that algo did its work on iptables before, so rule num 5 is ok to use
+  with_items:
+    - 8888
+    - 9999
 
 - name: reload supervisord
   shell: supervisorctl reload

bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/supervisor_mitmproxy.conf.j2

@@ -0,0 +1,7 @@
+
+[program:mitm{{ port }}]
+stdout_logfile = /home/mitmproxy/mitm{{ port }}-out.log
+stderr_logfile = /home/mitmproxy/mitm{{ port }}-err.log
+command=sudo -H -u mitmproxy bash -c "/home/mitmproxy/mitmproxy/run_mitm.sh {{ port }}"
+stopasgroup=true
+stopsignal=QUIT

bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh

@@ -60,9 +60,10 @@ fi
 log "Stopping bubble service"
 supervisorctl stop bubble
 
-# stop mitmdump service
-log "Stopping mitmproxy service"
-supervisorctl stop mitmdump
+# stop mitm services
+log "Stopping mitm services"
+supervisorctl stop mitm8888
+supervisorctl stop mitm9999
 
 # restore bubble.jar
 log "Restoring bubble.jar"
@@ -133,7 +134,8 @@ fi
 
 # restart mitm proxy service
 log "Restarting mitmproxy"
-supervisorctl restart mitmdump
+supervisorctl restart mitm8888
+supervisorctl restart mitm9999
 
 # restart bubble service
 log "Restore complete: restarting bubble API"

bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py

@@ -12,12 +12,6 @@ import redis
 import json
 from bubble_config import bubble_network, bubble_port
 
-# Write python PID to file so that mitmdump_monitor.sh can check for excessive memory usage and restart if needed
-MITMDUMP_PID_FILE_PATH = '/home/mitmproxy/mitmdump.pid'
-MITMDUMP_PID_FILE = open(MITMDUMP_PID_FILE_PATH, "w")
-MITMDUMP_PID_FILE.write("%d" % os.getpid())
-MITMDUMP_PID_FILE.close()
-
 HEADER_USER_AGENT = 'User-Agent'
 HEADER_REFERER = 'Referer'
 HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru'

bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+LOG=/tmp/mitm_monitor.log
+
+function die {
+  echo 1>&2 "${1}"
+  log "${1}"
+  exit 1
+}
+
+function log {
+  echo "$(date): ${1}" >> ${LOG}
+}
+
+BUBBLE_MITM_MARKER=/home/bubble/.mitm_monitor
+ROOT_KEY_MARKER=/usr/share/bubble/mitm_monitor
+MITM_PORT_FILE=/home/mitmproxy/mitmproxy_port
+MIN_PCT_FREE=2
+
+# Start with MITM proxy turned on, or refresh value
+if [[ ! -f ${BUBBLE_MITM_MARKER} ]] ; then
+  echo -n on > ${BUBBLE_MITM_MARKER} && chown bubble ${BUBBLE_MITM_MARKER} || log "Error writing 'on' to ${ROOT_KEY_MARKER}"
+else
+  touch ${BUBBLE_MITM_MARKER}
+fi
+if [[ ! -f ${ROOT_KEY_MARKER} ]] ; then
+  sleep 1s
+  mkdir -p "$(dirname ${ROOT_KEY_MARKER})" && chmod 755 "$(dirname ${ROOT_KEY_MARKER})" || log "Error creating or setting permissions on ${ROOT_KEY_MARKER}"
+  echo -n on > ${ROOT_KEY_MARKER} && touch ${ROOT_KEY_MARKER} && chmod 644 ${ROOT_KEY_MARKER} || log "Error writing 'on' to ${ROOT_KEY_MARKER}"
+fi
+
+function ensureMitmOn {
+  PORT=${1}
+  log "Flushing PREROUTING before enabling MITM services"
+  iptables -F PREROUTING  -t nat || log "Error flushing port forwarding when enabling MITM services"
+  log "Enabling MITM port forwarding on TCP port 80 -> ${PORT}"
+  iptables -I PREROUTING 1 -t nat -p tcp --dport 80 -j REDIRECT --to-ports ${PORT} || log "Error enabling MITM port forwarding 80 -> 8888"
+  log "Enabling MITM port forwarding on TCP port 443 -> ${PORT}"
+  iptables -I PREROUTING 1 -t nat -p tcp --dport 443 -j REDIRECT --to-ports ${PORT} || log "Error enabling MITM port forwarding 443 -> 8888"
+  echo -n on > ${ROOT_KEY_MARKER}
+}
+
+function ensureMitmOff {
+  log "Flushing PREROUTING to disable MITM services"
+  iptables -F PREROUTING  -t nat || log "Error flushing port forwarding when disabling MITM services"
+  echo -n off > ${ROOT_KEY_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}"
+}
+
+function fullMitmReset {
+  log "Full mitm reset starting"
+  ensureMitmOn 8888
+  echo 8888 > ${MITM_PORT_FILE}
+  supervisorctl restart mitm8888
+  supervisorctl restart mitm9999
+  log "Full mitm reset completed"
+}
+
+log "Watching marker file ${BUBBLE_MITM_MARKER} ..."
+sleep 2s && touch ${BUBBLE_MITM_MARKER} || log "Error touching ${BUBBLE_MITM_MARKER}" # first time through, always check and set on/off state
+while : ; do
+  if [[ $(stat -c %Y ${BUBBLE_MITM_MARKER}) -gt $(stat -c %Y ${ROOT_KEY_MARKER}) ]] ; then
+    if [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "on" ]] ; then
+      if [[ ! -f "${MITM_PORT_FILE}" ]] ; then
+        log "Error: port file does not exist: ${MITM_PORT_FILE}"
+      else
+        MITM_PORT="$(cat ${MITM_PORT_FILE})"
+        if [[ -z "${MITM_PORT}" ]] ; then
+          log "Error: port file was empty: ${MITM_PORT_FILE}"
+        else
+            ensureMitmOn ${MITM_PORT}
+        fi
+      fi
+    elif [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "off" ]] ; then
+      ensureMitmOff
+    else
+      log "Error: marker file ${BUBBLE_MITM_MARKER} contained invalid value: $(cat ${BUBBLE_MITM_MARKER} | head -c 5)"
+    fi
+  fi
+
+  # Check process memory usage, restart mitm if memory goes above max % allowed
+  if [[ ! -f "${MITM_PORT_FILE}" ]] ; then
+    log "Warn: No mitm port found in file: ${MITM_PORT_FILE}, resetting mitm"
+    fullMitmReset
+  else
+    MITM_PORT="$(cat ${MITM_PORT_FILE})"
+    if [[ -z "${MITM_PORT}" ]] ; then
+      log "Warn: No mitm port found in file: ${MITM_PORT_FILE} (resetting mitm)"
+      fullMitmReset
+    else
+      MITM_PID=$(netstat -nlpt4 | grep :${MITM_PORT} | awk '{print $7}' | cut -d/ -f1)
+      if [[ -z "${MITM_PID}" ]] ; then
+        log "Warn: No mitm PID found listening on ${MITM_PORT} via netstat, may be starting up"
+      else
+        PCT_FREE=$(expr $(free | grep -m 1 Mem: | awk '{print $7"00 / "$2}'))
+        PCT_MEM="$(ps q ${MITM_PID} -o %mem --no-headers | tr -d [[:space:]] | cut -f1 -d. | sed 's/[^0-9]*//g')"
+        # log "Info: mitm pid ${MITM_PID} using ${PCT_MEM}% of memory"
+        if [[ -z "${PCT_MEM}" ]] ; then
+          log "Error: could not determine mitm % memory. pid was ${MITM_PID}"
+        else
+          if [[ ${PCT_FREE} -lt ${MIN_PCT_FREE} ]] ; then
+            log "Warn: switching mitm port: ${PCT_FREE}% free < ${MIN_PCT_FREE}% min. mitm${MITM_PORT} using ${PCT_MEM}%"
+            if [[ "${MITM_PORT}" == "8888" ]] ; then
+              ensureMitmOn 9999
+              echo 9999 > ${MITM_PORT_FILE}
+              supervisorctl restart mitm8888
+            else
+              ensureMitmOn 8888
+              echo 8888 > ${MITM_PORT_FILE}
+              supervisorctl restart mitm9999
+            fi
+          fi
+        fi
+      fi
+    fi
+  fi
+  sleep 5s
+done

bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitmdump_monitor.sh

@@ -1,81 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
-#
-LOG=/tmp/bubble.mitmdump_monitor.log
-
-function die {
-  echo 1>&2 "${1}"
-  log "${1}"
-  exit 1
-}
-
-function log {
-  echo "$(date): ${1}" >> ${LOG}
-}
-
-BUBBLE_MITM_MARKER=/home/bubble/.mitmdump_monitor
-ROOT_KEY_MARKER=/usr/share/bubble/mitmdump_monitor
-MITMDUMP_PID_FILE=/home/mitmproxy/mitmdump.pid
-MIN_PCT_FREE=3
-
-# Start with MITM proxy turned on, or refresh value
-if [[ ! -f ${BUBBLE_MITM_MARKER} ]] ; then
-  echo -n on > ${BUBBLE_MITM_MARKER} && chown bubble ${BUBBLE_MITM_MARKER} || log "Error writing 'on' to ${ROOT_KEY_MARKER}"
-else
-  touch ${BUBBLE_MITM_MARKER}
-fi
-if [[ ! -f ${ROOT_KEY_MARKER} ]] ; then
-  sleep 1s
-  mkdir -p "$(dirname ${ROOT_KEY_MARKER})" && chmod 755 "$(dirname ${ROOT_KEY_MARKER})" || log "Error creating or setting permissions on ${ROOT_KEY_MARKER}"
-  echo -n on > ${ROOT_KEY_MARKER} && touch ${ROOT_KEY_MARKER} && chmod 644 ${ROOT_KEY_MARKER} || log "Error writing 'on' to ${ROOT_KEY_MARKER}"
-fi
-
-function ensureMitmOn {
-  log "Flushing PREROUTING before enabling MITM services"
-  iptables -F PREROUTING  -t nat || log "Error flushing port forwarding when enabling MITM services"
-  log "Enabling MITM port forwarding on TCP port 80 -> 8888"
-  iptables -I PREROUTING 1 -t nat -p tcp --dport 80 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 80 -> 8888"
-  log "Enabling MITM port forwarding on TCP port 443 -> 8888"
-  iptables -I PREROUTING 1 -t nat -p tcp --dport 443 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 443 -> 8888"
-  echo -n on > ${ROOT_KEY_MARKER}
-}
-
-function ensureMitmOff {
-  log "Flushing PREROUTING to disable MITM services"
-  iptables -F PREROUTING  -t nat || log "Error flushing port forwarding when disabling MITM services"
-  echo -n off > ${ROOT_KEY_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}"
-}
-
-log "Watching marker file ${BUBBLE_MITM_MARKER} ..."
-sleep 2s && touch ${BUBBLE_MITM_MARKER} || log "Error touching ${BUBBLE_MITM_MARKER}" # first time through, always check and set on/off state
-while : ; do
-  if [[ $(stat -c %Y ${BUBBLE_MITM_MARKER}) -gt $(stat -c %Y ${ROOT_KEY_MARKER}) ]] ; then
-    if [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "on" ]] ; then
-      ensureMitmOn
-    elif [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "off" ]] ; then
-      ensureMitmOff
-    else
-      log "Error: marker file ${BUBBLE_MITM_MARKER} contained invalid value: $(cat ${BUBBLE_MITM_MARKER} | head -c 5)"
-    fi
-  fi
-
-  # Check process memory usage, restart mitmdump if memory goes above max % allowed
-  if [[ -f ${MITMDUMP_PID_FILE} && -s ${MITMDUMP_PID_FILE} ]] ; then
-    MITM_PID="$(cat ${MITMDUMP_PID_FILE})"
-    PCT_FREE=$(expr $(free | grep -m 1 Mem: | awk '{print $7"00 / "$2}'))
-    PCT_MEM="$(ps q ${MITM_PID} -o %mem --no-headers | tr -d [[:space:]] | cut -f1 -d. | sed 's/[^0-9]*//g')"
-    # log "Info: mitmdump pid ${MITM_PID} using ${PCT_MEM}% of memory"
-    if [[ ! -z "${PCT_MEM}" ]] ; then
-      if [[ ${PCT_FREE} -lt ${MIN_PCT_FREE} ]] ; then
-        log "Warn: mitmdump: less than ${MIN_PCT_FREE}% mem available, restarting: mitm used ${PCT_MEM}%, ${PCT_FREE}% free"
-        supervisorctl restart mitmdump
-      fi
-    else
-      log "Error: could not determine mitmdump % memory, maybe PID file ${MITMDUMP_PID_FILE} is out of date? pid found was ${MITM_PID}"
-    fi
-  else
-    log "Error: mitmdump PID file ${MITMDUMP_PID_FILE} not found or empty"
-  fi
-  sleep 5s
-done

bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitmdump.sh → bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh

@@ -2,11 +2,13 @@
 #
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
+PORT=${1:-8888}
+echo "Starting mitmproxy on port ${PORT} ..."
 cd /home/mitmproxy/mitmproxy && \
 ./dev.sh && . ./venv/bin/activate && \
 mitmdump \
   --listen-host 0.0.0.0 \
-  --listen-port 8888 \
+  --listen-port ${PORT} \
   --showhost \
   --no-http2 \
   --set block_global=false \

bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml

@@ -20,10 +20,10 @@
     value: 0
     sysctl_set: yes
 
-- name: Create mitmproxy user
+- name: Create mitm user
   user:
     name: mitmproxy
-    comment: mitmdump user
+    comment: mitm user
     shell: /bin/bash
     system: yes
     home: /home/mitmproxy
@@ -48,7 +48,7 @@
     src: /tmp/mitmproxy.zip
     dest: /home/mitmproxy/mitmproxy
 
-- name: Copy mitmdump files
+- name: Copy mitm files
   copy:
     src: "{{ item }}"
     dest: "/home/mitmproxy/mitmproxy/{{ item }}"
@@ -60,7 +60,7 @@
     - dns_spoofing.py
     - bubble_conn_check.py
     - bubble_modify.py
-    - run_mitmdump.sh
+    - run_mitm.sh
 
 - name: Install cert helper scripts
   copy:
@@ -88,10 +88,10 @@
 - name: Install mitmproxy dependencies
   shell: su - mitmproxy -c "bash -c 'cd /home/mitmproxy/mitmproxy && ./dev.sh'"
 
-- name: Install mitmdump_monitor
+- name: Install mitm_monitor
   copy:
-    src: "mitmdump_monitor.sh"
-    dest: "/usr/local/sbin/mitmdump_monitor.sh"
+    src: "mitm_monitor.sh"
+    dest: "/usr/local/sbin/mitm_monitor.sh"
     owner: root
     group: root
     mode: 0500