parse Referer header properly. reduce logging

automation/roles/mitmproxy/files/bubble_api.py

@@ -13,7 +13,7 @@ BUBBLE_URI_PREFIX='/__bubble/'
 def bubble_log (message):
     print(message, file=sys.stderr)
 
-# todo: cache responses by remote_addr+host for a limited time (1 minute?)
+
 def bubble_matchers (remote_addr, flow, host):
     headers = {
         'X-Forwarded-For': remote_addr,
@@ -30,7 +30,11 @@ def bubble_matchers (remote_addr, flow, host):
         bubble_log('bubble_matchers: no Referer header, setting to NONE')
         referer = 'NONE'
     else:
-        referer = flow.request.headers[HEADER_REFERER]
+        try:
+            referer = flow.request.headers[HEADER_REFERER].encode().decode()
+        except Exception as e:
+            bubble_log('bubble_matchers: error parsing Referer header: '+repr(e))
+            referer = 'NONE'
 
     try:
         data = {

automation/roles/mitmproxy/files/dns_spoofing.py

@@ -61,7 +61,7 @@ class Rerouter:
             port = 80
 
         host_header = flow.request.host_header
-        bubble_log("dns_spoofing.request: host_header is "+repr(host_header))
+        # bubble_log("dns_spoofing.request: host_header is "+repr(host_header))
         if host_header:
             m = parse_host_header.match(host_header)
             if m:
@@ -73,7 +73,7 @@ class Rerouter:
         if sni or host_header:
             matcher_response = self.get_matchers(flow, sni or host_header)
             if matcher_response and 'matchers' in matcher_response and 'device' in matcher_response and len(matcher_response['matchers']) > 0:
-                bubble_log("dns_spoofing.request: found matchers: " + ' '.join(matcher_response['matchers']))
+                # bubble_log("dns_spoofing.request: found matchers: " + ' '.join(matcher_response['matchers']))
                 flow.request.headers[HEADER_BUBBLE_MATCHERS] = json.dumps(matcher_response['matchers'])
                 flow.request.headers[HEADER_BUBBLE_DEVICE] = matcher_response['device']
             else: