add Referer to FilterMatchersRequest and TrafficRecord

automation/roles/mitmproxy/files/bubble_api.py

@@ -4,6 +4,7 @@ import sys
 from bubble_config import bubble_network, bubble_port
 
 HEADER_USER_AGENT = 'User-Agent'
+HEADER_REFERER = 'Referer'
 
 HEADER_BUBBLE_MATCHERS='X-Bubble-Matchers'
 HEADER_BUBBLE_DEVICE='X-Bubble-Device'
@@ -24,11 +25,19 @@ def bubble_matchers (remote_addr, flow, host):
         user_agent = 'UNKNOWN'
     else:
         user_agent = flow.request.headers[HEADER_USER_AGENT]
+
+    if HEADER_REFERER not in flow.request.headers:
+        bubble_log('bubble_matchers: no Referer header, setting to NONE')
+        referer = 'NONE'
+    else:
+        referer = flow.request.headers[HEADER_REFERER]
+
     try:
         data = {
             'fqdn': host,
             'uri': flow.request.path,
             'userAgent': user_agent,
+            'referer': referer,
             'remoteAddr': remote_addr
         }
         response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers', headers=headers, json=data)

bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java

@@ -13,8 +13,9 @@ public class FilterMatchersRequest {
     @Getter @Setter private String fqdn;
     @Getter @Setter private String uri;
     @Getter @Setter private String userAgent;
+    @Getter @Setter private String referer;
     @Getter @Setter private String remoteAddr;
 
-    public String cacheKey() { return hashOf(fqdn, uri, userAgent, remoteAddr); }
+    public String cacheKey() { return hashOf(fqdn, uri, userAgent, referer, remoteAddr); }
 
 }

bubble-server/src/main/java/bubble/rule/analytics/TrafficRecord.java

@@ -25,6 +25,7 @@ public class TrafficRecord {
     @Getter @Setter private String fqdn;
     @Getter @Setter private String uri;
     @Getter @Setter private String userAgent;
+    @Getter @Setter private String referer;
 
     public TrafficRecord(FilterMatchersRequest filter, Account account, Device device, Request req) {
         setAccountName(account == null ? null : account.getName());
@@ -35,5 +36,7 @@ public class TrafficRecord {
         setFqdn(filter.getFqdn());
         setUri(filter.getUri());
         setUserAgent(filter.getUserAgent());
+        setReferer(filter.getReferer());
     }
+
 }

bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json

@@ -16,6 +16,7 @@
       {"name": "fqdn"},
       {"name": "uri", "when": "view === \"recent\""},
       {"name": "userAgent", "when": "view === \"recent\""},
+      {"name": "referer", "when": "view === \"recent\""},
       {"name": "device", "when": "view !== \"recent\""},
       {"name": "data", "when": "view !== \"recent\""}
     ],
@@ -57,6 +58,7 @@
         {"name": "field.ip", "value": "From IP"},
         {"name": "field.uri", "value": "Path"},
         {"name": "field.userAgent", "value": "User Agent"},
+        {"name": "field.referer", "value": "Referer"},
         {"name": "field.data", "value": "Count"},
         {"name": "param.meta2", "value": "Site"},
         {"name": "param.device", "value": "Device"},