diff --git a/automation/roles/mitmproxy/files/bubble_api.py b/automation/roles/mitmproxy/files/bubble_api.py index 76bef7e9..e6a5726d 100644 --- a/automation/roles/mitmproxy/files/bubble_api.py +++ b/automation/roles/mitmproxy/files/bubble_api.py @@ -4,6 +4,7 @@ import sys from bubble_config import bubble_network, bubble_port HEADER_USER_AGENT = 'User-Agent' +HEADER_REFERER = 'Referer' HEADER_BUBBLE_MATCHERS='X-Bubble-Matchers' HEADER_BUBBLE_DEVICE='X-Bubble-Device' @@ -24,11 +25,19 @@ def bubble_matchers (remote_addr, flow, host): user_agent = 'UNKNOWN' else: user_agent = flow.request.headers[HEADER_USER_AGENT] + + if HEADER_REFERER not in flow.request.headers: + bubble_log('bubble_matchers: no Referer header, setting to NONE') + referer = 'NONE' + else: + referer = flow.request.headers[HEADER_REFERER] + try: data = { 'fqdn': host, 'uri': flow.request.path, 'userAgent': user_agent, + 'referer': referer, 'remoteAddr': remote_addr } response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers', headers=headers, json=data) diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java index b45e576f..31ee922b 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java @@ -13,8 +13,9 @@ public class FilterMatchersRequest { @Getter @Setter private String fqdn; @Getter @Setter private String uri; @Getter @Setter private String userAgent; + @Getter @Setter private String referer; @Getter @Setter private String remoteAddr; - public String cacheKey() { return hashOf(fqdn, uri, userAgent, remoteAddr); } + public String cacheKey() { return hashOf(fqdn, uri, userAgent, referer, remoteAddr); } } diff --git a/bubble-server/src/main/java/bubble/rule/analytics/TrafficRecord.java b/bubble-server/src/main/java/bubble/rule/analytics/TrafficRecord.java index 2f9d5b70..cfc7f731 100644 --- a/bubble-server/src/main/java/bubble/rule/analytics/TrafficRecord.java +++ b/bubble-server/src/main/java/bubble/rule/analytics/TrafficRecord.java @@ -25,6 +25,7 @@ public class TrafficRecord { @Getter @Setter private String fqdn; @Getter @Setter private String uri; @Getter @Setter private String userAgent; + @Getter @Setter private String referer; public TrafficRecord(FilterMatchersRequest filter, Account account, Device device, Request req) { setAccountName(account == null ? null : account.getName()); @@ -35,5 +36,7 @@ public class TrafficRecord { setFqdn(filter.getFqdn()); setUri(filter.getUri()); setUserAgent(filter.getUserAgent()); + setReferer(filter.getReferer()); } + } diff --git a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json index d24e8cf0..b89a333b 100644 --- a/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json +++ b/bubble-server/src/main/resources/models/apps/analytics/bubbleApp_analytics.json @@ -16,6 +16,7 @@ {"name": "fqdn"}, {"name": "uri", "when": "view === \"recent\""}, {"name": "userAgent", "when": "view === \"recent\""}, + {"name": "referer", "when": "view === \"recent\""}, {"name": "device", "when": "view !== \"recent\""}, {"name": "data", "when": "view !== \"recent\""} ], @@ -57,6 +58,7 @@ {"name": "field.ip", "value": "From IP"}, {"name": "field.uri", "value": "Path"}, {"name": "field.userAgent", "value": "User Agent"}, + {"name": "field.referer", "value": "Referer"}, {"name": "field.data", "value": "Count"}, {"name": "param.meta2", "value": "Site"}, {"name": "param.device", "value": "Device"},