smoother interop between mitm and nginx

automation/roles/mitmproxy/files/bubble_modify.py

@@ -49,8 +49,7 @@ def bubble_filter_chunks(chunks, req_id, content_encoding, content_type, device,
 
 
 def bubble_modify(req_id, content_encoding, content_type, device, matchers):
-    return lambda chunks: bubble_filter_chunks(chunks, req_id, content_encoding, content_type,
-                                               device, matchers)
+    return lambda chunks: bubble_filter_chunks(chunks, req_id, content_encoding, content_type, device, matchers)
 
 
 def responseheaders(flow):

automation/roles/mitmproxy/files/dns_spoofing.py

@@ -1,31 +1,7 @@
-"""
-This script makes it possible to use mitmproxy in scenarios where IP spoofing
-has been used to redirect connections to mitmproxy. The way this works is that
-we rely on either the TLS Server Name Indication (SNI) or the Host header of the
-HTTP request. Of course, this is not foolproof - if an HTTPS connection comes
-without SNI, we don't know the actual target and cannot construct a certificate
-that looks valid. Similarly, if there's no Host header or a spoofed Host header,
-we're out of luck as well. Using transparent mode is the better option most of
-the time.
-
-Usage:
-    mitmproxy
-        -p 443
-        -s dns_spoofing.py
-        # Used as the target location if neither SNI nor host header are present.
-        --mode reverse:http://example.com/
-        # To avoid auto rewriting of host header by the reverse proxy target.
-        --set keep_host_header
-    mitmdump
-        -p 80
-        --mode reverse:http://localhost:443/
-
-    (Setting up a single proxy instance and using iptables to redirect to it
-    works as well)
-"""
 import json
 import re
 from bubble_api import bubble_matchers, bubble_log, HEADER_BUBBLE_MATCHERS, HEADER_BUBBLE_DEVICE
+from bubble_config import bubble_host, bubble_host_alias
 from mitmproxy import ctx
 
 # This regex extracts splits the host header into host and port.
@@ -47,6 +23,10 @@ class Rerouter:
             bubble_log("get_matchers: host "+str(host)+" could not be decoded, type="+str(type(host)))
             return None
 
+        if host == bubble_host or host == bubble_host_alias:
+            bubble_log("get_matchers: request is for bubble itself ("+host+"), not matching")
+            return None
+
         resp = bubble_matchers(remote_addr, flow, host)
         if (not resp) or (not 'matchers' in resp) or (not 'device' in resp):
             bubble_log("get_matchers: no matchers/device for remote_addr/host: "+remote_addr+'/'+str(host))

automation/roles/mitmproxy/files/mitmdump_monitor.sh

@@ -36,12 +36,8 @@ function ensureMitmOn {
 }
 
 function ensureMitmOff {
-  log "Flushing PREROUTING before disabling MITM services"
+  log "Flushing PREROUTING to disable MITM services"
   iptables -F PREROUTING  -t nat || log "Error flushing port forwarding when disabling MITM services"
-  log "Enabling MITM port forwarding on TCP port 80 -> 1080"
-  iptables -I PREROUTING 1 -t nat -p tcp --dport 80 -j REDIRECT --to-ports 1080 || log "Error enabling nginx port forwarding 80 -> 1080"
-  log "Enabling MITM port forwarding on TCP port 443 -> 1443"
-  iptables -I PREROUTING 1 -t nat -p tcp --dport 443 -j REDIRECT --to-ports 1443 || log "Error enabling nginx port forwarding 443 -> 1143"
   echo -n off > ${ROOT_KEY_MARKER}
 }
 

automation/roles/mitmproxy/files/run_mitmdump.sh

@@ -16,4 +16,4 @@ mitmdump \
   --set keep_host_header \
   -s ./dns_spoofing.py \
   -s ./bubble_modify.py \
-  --mode reverse:https://example.com:443/
+  --mode transparent

automation/roles/nginx/templates/site_node.conf.j2

@@ -1,7 +1,9 @@
 server {
     server_name {{ server_name }};
+    listen 80;
     listen 1080;
     listen {{ ssl_port }} ssl http2;
+    listen 443 ssl http2;
 
     root  /home/bubble/site/;
     index index.html;

automation/roles/nginx/templates/site_node_alias.conf.j2

@@ -1,7 +1,9 @@
 server {
     server_name {{ server_alias }};
+    listen 80;
     listen 1080;
     listen {{ ssl_port }} ssl http2;
+    listen 443 ssl http2;
 
     root  /home/bubble/site/;
     index index.html;

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -135,13 +135,34 @@ public class FilterHttpResource {
                                @PathParam("requestId") String requestId,
                                @QueryParam("device") String deviceId,
                                @QueryParam("matchers") String matchersJson,
-                               @QueryParam("encoding") HttpContentEncodingType contentEncoding,
+                               @QueryParam("encoding") String contentEncoding,
                                @QueryParam("contentType") String contentType,
                                @QueryParam("last") Boolean last) throws IOException {
 
         final String remoteHost = getRemoteHost(req);
         final String mitmAddr = req.getRemoteAddr();
 
+        // only mitmproxy is allowed to call us, and this should always be a local address
+        if (!isLocalIpv4(mitmAddr)) return forbidden();
+
+        if (empty(requestId)) {
+            if (log.isDebugEnabled()) log.debug("filterHttp: no requestId provided, returning passthru");
+            return passthru(request);
+        }
+
+        // can we handle the encoding?
+        final HttpContentEncodingType encoding;
+        if (empty(contentEncoding)) {
+            encoding = null;
+        } else {
+            try {
+                encoding = HttpContentEncodingType.fromString(contentEncoding);
+            } catch (Exception e) {
+                if (log.isWarnEnabled()) log.warn("filterHttp: invalid encoding ("+contentEncoding+"), returning passthru");
+                return passthru(request);
+            }
+        }
+
         // mitmproxy provides Content-Length, which helps us right-size the input byte buffer
         final String contentLengthHeader = req.getHeader(CONTENT_LENGTH);
         Integer contentLength;
@@ -152,14 +173,6 @@ public class FilterHttpResource {
             contentLength = null;
         }
 
-        // only mitmproxy is allowed to call us, and this should always be a local address
-        if (!isLocalIpv4(mitmAddr)) return forbidden();
-
-        if (empty(requestId)) {
-            if (log.isDebugEnabled()) log.debug("filterHttp: no requestId provided, returning passthru");
-            return passthru(request);
-        }
-
         FilterHttpRequest filterRequest = activeRequests.get(requestId);
         if (filterRequest == null) {
             if (empty(deviceId) || empty(matchersJson) || empty(contentType)) {
@@ -205,7 +218,7 @@ public class FilterHttpResource {
         }
 
         final boolean isLast = last != null && last;
-        return ruleEngine.applyRulesToChunkAndSendResponse(request, contentEncoding, contentLength,
+        return ruleEngine.applyRulesToChunkAndSendResponse(request, encoding, contentLength,
                         filterRequest.getId(), filterRequest.getAccount(), filterRequest.getDevice(),
                         filterRequest.getMatchers(), isLast);
     }