add support for content-type filtering, recognize on mitm side for big efficiency boost

automation/roles/mitmproxy/files/bubble_modify.py

@@ -1,3 +1,4 @@
+import re
 import requests
 import urllib
 import traceback
@@ -42,7 +43,7 @@ def filter_chunk(chunk, req_id, last, content_encoding=None, content_type=None,
     return response.content
 
 
-def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, content_length):
+def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type):
     """
     chunks is a generator that can be used to iterate over all chunks.
     """
@@ -54,6 +55,7 @@ def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, c
                 bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT)
                 chunk_len = len(chunk)
                 last = chunk_len + bytes_sent >= content_length
+                bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent))
                 add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len)
             else:
                 last = False
@@ -70,8 +72,8 @@ def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, c
         yield None
 
 
-def bubble_modify(flow, req_id, content_encoding, content_type, content_length):
-    return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, content_length)
+def bubble_modify(flow, req_id, content_encoding, content_type):
+    return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type)
 
 
 def send_bubble_response(response):
@@ -121,6 +123,20 @@ def responseheaders(flow):
                 if HEADER_CONTENT_TYPE in flow.response.headers:
                     content_type = flow.response.headers[HEADER_CONTENT_TYPE]
                     if matchers:
+                        any_content_type_matches = False
+                        for m in matchers:
+                            if 'contentTypeRegex' in m:
+                                typeRegex = m['contentTypeRegex']
+                                if typeRegex is None:
+                                    typeRegex = '^text/html.*'
+                                if re.match(typeRegex, content_type):
+                                    any_content_type_matches = True
+                                    bubble_log('responseheaders: req_id='+req_id+' found at least one matcher for content_type ('+content_type+'), filtering')
+                                    break
+                        if not any_content_type_matches:
+                            bubble_log('responseheaders: req_id='+req_id+' no matchers for content_type ('+content_type+'), passing thru')
+                            return
+
                         if HEADER_CONTENT_ENCODING in flow.response.headers:
                             content_encoding = flow.response.headers[HEADER_CONTENT_ENCODING]
                         else:
@@ -128,7 +144,7 @@ def responseheaders(flow):
 
                         content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None)
                         bubble_log('responseheaders: req_id='+req_id+' content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type))
-                        flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type, content_length_value)
+                        flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type)
                         if content_length_value:
                             flow.response.headers['transfer-encoding'] = 'chunked'
                             # find server_conn to set fake_chunks on

automation/roles/mitmproxy/files/dns_spoofing.py

@@ -43,7 +43,7 @@ class Rerouter:
         if (not resp) or (not 'matchers' in resp) or (resp['matchers'] is None):
             bubble_log('get_matchers: no matchers for remote_addr/host: '+remote_addr+'/'+str(host))
             return None
-        matcher_ids = []
+        matchers = []
         for m in resp['matchers']:
             if 'urlRegex' in m:
                 bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex'])
@@ -52,11 +52,11 @@ class Rerouter:
                 continue
             if re.match(m['urlRegex'], flow.request.path):
                 bubble_log('get_matchers: rule matched, adding rule: '+m['rule'])
-                matcher_ids.append(m['uuid'])
+                matchers.append(m)
             else:
                 bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule'])
 
-        matcher_response = { 'matchers': matcher_ids, 'request_id': req_id }
+        matcher_response = { 'matchers': matchers, 'request_id': req_id }
         bubble_log("get_matchers: returning "+repr(matcher_response))
         return matcher_response
 
@@ -91,7 +91,7 @@ class Rerouter:
                       and 'request_id' in matcher_response
                       and len(matcher_response['matchers']) > 0):
                     req_id = matcher_response['request_id']
-                    bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + ' '.join(matcher_response['matchers']))
+                    bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers']))
                     add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers'])
                     add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id)
                 else:

bubble-server/src/main/java/bubble/model/app/AppMatcher.java

@@ -23,6 +23,7 @@ import java.util.regex.Pattern;
 
 import static bubble.ApiConstants.EP_MATCHERS;
 import static org.cobbzilla.util.daemon.ZillaRuntime.bool;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING;
 import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
@@ -42,6 +43,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H
     public static final String[] VALUE_FIELDS = {"fqdn", "urlRegex", "template", "enabled", "priority"};
     public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule");
 
+    public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE);
+
     public AppMatcher(AppMatcher other) {
         copy(this, other, CREATE_FIELDS);
         setUuid(null);
@@ -77,32 +80,42 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H
     @ECSearchable(filter=true) @ECField(index=60)
     @HasValue(message="err.urlRegex.required")
     @Size(max=1024, message="err.urlRegex.length")
-    @Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(1024+ENC_PAD)+") UNIQUE")
+    @Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(1024+ENC_PAD)+") NOT NULL")
     @Getter @Setter private String urlRegex;
 
-    @Transient @JsonIgnore public Pattern getPattern () { return Pattern.compile(getUrlRegex()); }
+    @Transient @JsonIgnore public Pattern getUrlPattern() { return Pattern.compile(getUrlRegex()); }
+    public boolean matchesUrl (String value) { return getUrlPattern().matcher(value).find(); }
+
+    @ECSearchable(filter=true) @ECField(index=70)
+    @Size(max=1024, message="err.contentTypeRegex.length")
+    @Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(200+ENC_PAD)+")")
+    @Getter @Setter private String contentTypeRegex;
+    public boolean hasContentTypeRegex() { return !empty(contentTypeRegex); }
 
-    public boolean matches (String value) { return getPattern().matcher(value).find(); }
+    @Transient @JsonIgnore public Pattern getContentTypePattern () {
+        return hasContentTypeRegex() ? Pattern.compile(getContentTypeRegex()) : DEFAULT_CONTENT_TYPE_PATTERN;
+    }
+    public boolean matchesContentType (String value) { return getContentTypePattern().matcher(value).find(); }
 
-    @ECSearchable @ECField(index=70)
+    @ECSearchable @ECField(index=80)
     @ECForeignKey(entity=AppRule.class)
     @Column(nullable=false, updatable=false, length=UUID_MAXLEN)
     @Getter @Setter private String rule;
 
-    @ECSearchable @ECField(index=80)
+    @ECSearchable @ECField(index=90)
     @Column(nullable=false)
     @Getter @Setter private Boolean blocked = false;
     public boolean blocked() { return bool(blocked); }
 
-    @ECSearchable @ECField(index=90)
+    @ECSearchable @ECField(index=100)
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean template = false;
 
-    @ECSearchable @ECField(index=100)
+    @ECSearchable @ECField(index=110)
     @ECIndex @Column(nullable=false)
     @Getter @Setter private Boolean enabled = true;
 
-    @ECSearchable @ECField(index=110)
+    @ECSearchable @ECField(index=120)
     @Column(nullable=false)
     @Getter @Setter private Integer priority = 0;
 

bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java

@@ -180,7 +180,7 @@ public class FilterHttpResource {
             retainMatchers = new HashMap<>();
             for (AppMatcher matcher : matchers) {
                 if (retainMatchers.containsKey(matcher.getUuid())) continue;
-                if (matcher.matches(uri)) {
+                if (matcher.matchesUrl(uri)) {
                     if (log.isDebugEnabled()) log.debug(prefix+"matcher "+matcher.getName()+" with pattern "+matcher.getUrlRegex()+" found match for uri: '"+uri+"'");
                     final FilterMatchDecision matchResponse = ruleEngine.preprocess(filterRequest, req, request, caller, device, matcher);
                     switch (matchResponse) {
@@ -352,6 +352,10 @@ public class FilterHttpResource {
                 if (log.isDebugEnabled()) log.debug(prefix+"filterRequest not found, and no contentType provided, returning passthru");
                 return passthru(request);
             }
+            if (!isContentTypeMatch(matchersResponse, contentType)) {
+                if (log.isInfoEnabled()) log.info(prefix+"none of the "+matchersResponse.getMatchers().size()+" matchers matched contentType="+contentType+", returning passthru");
+                return passthru(request);
+            }
 
             final Device device = findDevice(matchersResponse.getRequest().getDevice());
             if (device == null) {
@@ -376,6 +380,11 @@ public class FilterHttpResource {
             if (log.isDebugEnabled()) log.trace(prefix+"start filterRequest="+json(filterRequest, COMPACT_MAPPER));
             getActiveRequestCache().set(requestId, json(filterRequest, COMPACT_MAPPER), EX, ACTIVE_REQUEST_TIMEOUT);
         } else {
+            if (!isContentTypeMatch(matchersResponse, filterRequest.getContentType())) {
+                if (log.isInfoEnabled()) log.info(prefix+"none of the "+matchersResponse.getMatchers().size()+" matchers matched contentType="+filterRequest.getContentType()+", returning passthru");
+                return passthru(request);
+            }
+
             if (log.isTraceEnabled()) {
                 if (isLast) {
                     log.trace(prefix+"last filterRequest=" + json(filterRequest, COMPACT_MAPPER));
@@ -388,6 +397,18 @@ public class FilterHttpResource {
         return ruleEngine.applyRulesToChunkAndSendResponse(request, filterRequest, chunkLength, isLast);
     }
 
+    public boolean isContentTypeMatch(FilterMatchersResponse matchersResponse, String ct) {
+        final String prefix = "isContentTypeMatch("+matchersResponse.getRequest().getRequestId()+"): ";
+        final List<AppMatcher> matchers = matchersResponse.getMatchers();
+        for (AppMatcher m : matchers) {
+            if (log.isDebugEnabled()) log.debug(prefix+"checking contentType match, matcher.contentTypeRegex="+m.getContentTypeRegex()+", contentType="+ct);
+            if (m.getContentTypePattern().matcher(ct).matches()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     public Response passthru(@Context ContainerRequest request) { return ruleEngine.passthru(request); }
 
     @GET @Path(EP_DATA+"/{requestId}/{matcherId}"+EP_READ)

bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java

@@ -68,7 +68,7 @@ public class ReverseProxyResource {
             final Map<String, AppMatcher> matchedMatchers = new HashMap<>();
             for (AppMatcher m : matchers) {
                 // check for regex match
-                if (m.matches(ub.getFullPath())) {
+                if (m.matchesUrl(ub.getFullPath())) {
                     // is this a total block?
                     if (m.blocked()) {
                         log.debug("get: matcher("+m.getUuid()+") blocks request, returning 404 Not Found for "+ub.getFullPath());

bubble-server/src/main/resources/logback.xml

@@ -40,10 +40,10 @@
 <!--    <logger name="org.cobbzilla.util.io.multi.MultiStream" level="TRACE" />-->
 <!--    <logger name="bubble.filters.BubbleRateLimitFilter" level="TRACE" />-->
 <!--    <logger name="org.cobbzilla.wizard.filters.RateLimitFilter" level="TRACE" />-->
-    <logger name="bubble.service.stream.StandardRuleEngineService" level="DEBUG" />
+<!--    <logger name="bubble.service.stream.StandardRuleEngineService" level="DEBUG" />-->
     <logger name="bubble.service.stream.ActiveStreamState" level="INFO" />
     <logger name="bubble.resources.stream" level="DEBUG" />
-<!--    <logger name="bubble.resources.stream.FilterHttpResource" level="TRACE" />-->
+    <logger name="bubble.resources.stream.FilterHttpResource" level="INFO" />
     <logger name="bubble.service.stream" level="INFO" />
     <logger name="bubble.resources.message" level="INFO" />
     <logger name="bubble.app.analytics" level="DEBUG" />