@@ -1,3 +1,4 @@ | |||||
import re | |||||
import requests | import requests | ||||
import urllib | import urllib | ||||
import traceback | import traceback | ||||
@@ -42,7 +43,7 @@ def filter_chunk(chunk, req_id, last, content_encoding=None, content_type=None, | |||||
return response.content | return response.content | ||||
def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, content_length): | |||||
def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type): | |||||
""" | """ | ||||
chunks is a generator that can be used to iterate over all chunks. | chunks is a generator that can be used to iterate over all chunks. | ||||
""" | """ | ||||
@@ -54,6 +55,7 @@ def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, c | |||||
bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT) | bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT) | ||||
chunk_len = len(chunk) | chunk_len = len(chunk) | ||||
last = chunk_len + bytes_sent >= content_length | last = chunk_len + bytes_sent >= content_length | ||||
bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent)) | |||||
add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len) | add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len) | ||||
else: | else: | ||||
last = False | last = False | ||||
@@ -70,8 +72,8 @@ def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, c | |||||
yield None | yield None | ||||
def bubble_modify(flow, req_id, content_encoding, content_type, content_length): | |||||
return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, content_length) | |||||
def bubble_modify(flow, req_id, content_encoding, content_type): | |||||
return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type) | |||||
def send_bubble_response(response): | def send_bubble_response(response): | ||||
@@ -121,6 +123,20 @@ def responseheaders(flow): | |||||
if HEADER_CONTENT_TYPE in flow.response.headers: | if HEADER_CONTENT_TYPE in flow.response.headers: | ||||
content_type = flow.response.headers[HEADER_CONTENT_TYPE] | content_type = flow.response.headers[HEADER_CONTENT_TYPE] | ||||
if matchers: | if matchers: | ||||
any_content_type_matches = False | |||||
for m in matchers: | |||||
if 'contentTypeRegex' in m: | |||||
typeRegex = m['contentTypeRegex'] | |||||
if typeRegex is None: | |||||
typeRegex = '^text/html.*' | |||||
if re.match(typeRegex, content_type): | |||||
any_content_type_matches = True | |||||
bubble_log('responseheaders: req_id='+req_id+' found at least one matcher for content_type ('+content_type+'), filtering') | |||||
break | |||||
if not any_content_type_matches: | |||||
bubble_log('responseheaders: req_id='+req_id+' no matchers for content_type ('+content_type+'), passing thru') | |||||
return | |||||
if HEADER_CONTENT_ENCODING in flow.response.headers: | if HEADER_CONTENT_ENCODING in flow.response.headers: | ||||
content_encoding = flow.response.headers[HEADER_CONTENT_ENCODING] | content_encoding = flow.response.headers[HEADER_CONTENT_ENCODING] | ||||
else: | else: | ||||
@@ -128,7 +144,7 @@ def responseheaders(flow): | |||||
content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) | content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) | ||||
bubble_log('responseheaders: req_id='+req_id+' content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) | bubble_log('responseheaders: req_id='+req_id+' content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) | ||||
flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type, content_length_value) | |||||
flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type) | |||||
if content_length_value: | if content_length_value: | ||||
flow.response.headers['transfer-encoding'] = 'chunked' | flow.response.headers['transfer-encoding'] = 'chunked' | ||||
# find server_conn to set fake_chunks on | # find server_conn to set fake_chunks on | ||||
@@ -43,7 +43,7 @@ class Rerouter: | |||||
if (not resp) or (not 'matchers' in resp) or (resp['matchers'] is None): | if (not resp) or (not 'matchers' in resp) or (resp['matchers'] is None): | ||||
bubble_log('get_matchers: no matchers for remote_addr/host: '+remote_addr+'/'+str(host)) | bubble_log('get_matchers: no matchers for remote_addr/host: '+remote_addr+'/'+str(host)) | ||||
return None | return None | ||||
matcher_ids = [] | |||||
matchers = [] | |||||
for m in resp['matchers']: | for m in resp['matchers']: | ||||
if 'urlRegex' in m: | if 'urlRegex' in m: | ||||
bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex']) | bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex']) | ||||
@@ -52,11 +52,11 @@ class Rerouter: | |||||
continue | continue | ||||
if re.match(m['urlRegex'], flow.request.path): | if re.match(m['urlRegex'], flow.request.path): | ||||
bubble_log('get_matchers: rule matched, adding rule: '+m['rule']) | bubble_log('get_matchers: rule matched, adding rule: '+m['rule']) | ||||
matcher_ids.append(m['uuid']) | |||||
matchers.append(m) | |||||
else: | else: | ||||
bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule']) | bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule']) | ||||
matcher_response = { 'matchers': matcher_ids, 'request_id': req_id } | |||||
matcher_response = { 'matchers': matchers, 'request_id': req_id } | |||||
bubble_log("get_matchers: returning "+repr(matcher_response)) | bubble_log("get_matchers: returning "+repr(matcher_response)) | ||||
return matcher_response | return matcher_response | ||||
@@ -91,7 +91,7 @@ class Rerouter: | |||||
and 'request_id' in matcher_response | and 'request_id' in matcher_response | ||||
and len(matcher_response['matchers']) > 0): | and len(matcher_response['matchers']) > 0): | ||||
req_id = matcher_response['request_id'] | req_id = matcher_response['request_id'] | ||||
bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + ' '.join(matcher_response['matchers'])) | |||||
bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) | |||||
add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) | add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) | ||||
add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) | add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) | ||||
else: | else: | ||||
@@ -23,6 +23,7 @@ import java.util.regex.Pattern; | |||||
import static bubble.ApiConstants.EP_MATCHERS; | import static bubble.ApiConstants.EP_MATCHERS; | ||||
import static org.cobbzilla.util.daemon.ZillaRuntime.bool; | import static org.cobbzilla.util.daemon.ZillaRuntime.bool; | ||||
import static org.cobbzilla.util.daemon.ZillaRuntime.empty; | |||||
import static org.cobbzilla.util.reflect.ReflectionUtil.copy; | import static org.cobbzilla.util.reflect.ReflectionUtil.copy; | ||||
import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; | import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; | ||||
import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; | import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; | ||||
@@ -42,6 +43,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H | |||||
public static final String[] VALUE_FIELDS = {"fqdn", "urlRegex", "template", "enabled", "priority"}; | public static final String[] VALUE_FIELDS = {"fqdn", "urlRegex", "template", "enabled", "priority"}; | ||||
public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule"); | public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule"); | ||||
public static final Pattern DEFAULT_CONTENT_TYPE_PATTERN = Pattern.compile("^text/html.*", Pattern.CASE_INSENSITIVE); | |||||
public AppMatcher(AppMatcher other) { | public AppMatcher(AppMatcher other) { | ||||
copy(this, other, CREATE_FIELDS); | copy(this, other, CREATE_FIELDS); | ||||
setUuid(null); | setUuid(null); | ||||
@@ -77,32 +80,42 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H | |||||
@ECSearchable(filter=true) @ECField(index=60) | @ECSearchable(filter=true) @ECField(index=60) | ||||
@HasValue(message="err.urlRegex.required") | @HasValue(message="err.urlRegex.required") | ||||
@Size(max=1024, message="err.urlRegex.length") | @Size(max=1024, message="err.urlRegex.length") | ||||
@Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(1024+ENC_PAD)+") UNIQUE") | |||||
@Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(1024+ENC_PAD)+") NOT NULL") | |||||
@Getter @Setter private String urlRegex; | @Getter @Setter private String urlRegex; | ||||
@Transient @JsonIgnore public Pattern getPattern () { return Pattern.compile(getUrlRegex()); } | |||||
@Transient @JsonIgnore public Pattern getUrlPattern() { return Pattern.compile(getUrlRegex()); } | |||||
public boolean matchesUrl (String value) { return getUrlPattern().matcher(value).find(); } | |||||
@ECSearchable(filter=true) @ECField(index=70) | |||||
@Size(max=1024, message="err.contentTypeRegex.length") | |||||
@Type(type=ENCRYPTED_STRING) @Column(nullable=false, columnDefinition="varchar("+(200+ENC_PAD)+")") | |||||
@Getter @Setter private String contentTypeRegex; | |||||
public boolean hasContentTypeRegex() { return !empty(contentTypeRegex); } | |||||
public boolean matches (String value) { return getPattern().matcher(value).find(); } | |||||
@Transient @JsonIgnore public Pattern getContentTypePattern () { | |||||
return hasContentTypeRegex() ? Pattern.compile(getContentTypeRegex()) : DEFAULT_CONTENT_TYPE_PATTERN; | |||||
} | |||||
public boolean matchesContentType (String value) { return getContentTypePattern().matcher(value).find(); } | |||||
@ECSearchable @ECField(index=70) | |||||
@ECSearchable @ECField(index=80) | |||||
@ECForeignKey(entity=AppRule.class) | @ECForeignKey(entity=AppRule.class) | ||||
@Column(nullable=false, updatable=false, length=UUID_MAXLEN) | @Column(nullable=false, updatable=false, length=UUID_MAXLEN) | ||||
@Getter @Setter private String rule; | @Getter @Setter private String rule; | ||||
@ECSearchable @ECField(index=80) | |||||
@ECSearchable @ECField(index=90) | |||||
@Column(nullable=false) | @Column(nullable=false) | ||||
@Getter @Setter private Boolean blocked = false; | @Getter @Setter private Boolean blocked = false; | ||||
public boolean blocked() { return bool(blocked); } | public boolean blocked() { return bool(blocked); } | ||||
@ECSearchable @ECField(index=90) | |||||
@ECSearchable @ECField(index=100) | |||||
@ECIndex @Column(nullable=false) | @ECIndex @Column(nullable=false) | ||||
@Getter @Setter private Boolean template = false; | @Getter @Setter private Boolean template = false; | ||||
@ECSearchable @ECField(index=100) | |||||
@ECSearchable @ECField(index=110) | |||||
@ECIndex @Column(nullable=false) | @ECIndex @Column(nullable=false) | ||||
@Getter @Setter private Boolean enabled = true; | @Getter @Setter private Boolean enabled = true; | ||||
@ECSearchable @ECField(index=110) | |||||
@ECSearchable @ECField(index=120) | |||||
@Column(nullable=false) | @Column(nullable=false) | ||||
@Getter @Setter private Integer priority = 0; | @Getter @Setter private Integer priority = 0; | ||||
@@ -180,7 +180,7 @@ public class FilterHttpResource { | |||||
retainMatchers = new HashMap<>(); | retainMatchers = new HashMap<>(); | ||||
for (AppMatcher matcher : matchers) { | for (AppMatcher matcher : matchers) { | ||||
if (retainMatchers.containsKey(matcher.getUuid())) continue; | if (retainMatchers.containsKey(matcher.getUuid())) continue; | ||||
if (matcher.matches(uri)) { | |||||
if (matcher.matchesUrl(uri)) { | |||||
if (log.isDebugEnabled()) log.debug(prefix+"matcher "+matcher.getName()+" with pattern "+matcher.getUrlRegex()+" found match for uri: '"+uri+"'"); | if (log.isDebugEnabled()) log.debug(prefix+"matcher "+matcher.getName()+" with pattern "+matcher.getUrlRegex()+" found match for uri: '"+uri+"'"); | ||||
final FilterMatchDecision matchResponse = ruleEngine.preprocess(filterRequest, req, request, caller, device, matcher); | final FilterMatchDecision matchResponse = ruleEngine.preprocess(filterRequest, req, request, caller, device, matcher); | ||||
switch (matchResponse) { | switch (matchResponse) { | ||||
@@ -352,6 +352,10 @@ public class FilterHttpResource { | |||||
if (log.isDebugEnabled()) log.debug(prefix+"filterRequest not found, and no contentType provided, returning passthru"); | if (log.isDebugEnabled()) log.debug(prefix+"filterRequest not found, and no contentType provided, returning passthru"); | ||||
return passthru(request); | return passthru(request); | ||||
} | } | ||||
if (!isContentTypeMatch(matchersResponse, contentType)) { | |||||
if (log.isInfoEnabled()) log.info(prefix+"none of the "+matchersResponse.getMatchers().size()+" matchers matched contentType="+contentType+", returning passthru"); | |||||
return passthru(request); | |||||
} | |||||
final Device device = findDevice(matchersResponse.getRequest().getDevice()); | final Device device = findDevice(matchersResponse.getRequest().getDevice()); | ||||
if (device == null) { | if (device == null) { | ||||
@@ -376,6 +380,11 @@ public class FilterHttpResource { | |||||
if (log.isDebugEnabled()) log.trace(prefix+"start filterRequest="+json(filterRequest, COMPACT_MAPPER)); | if (log.isDebugEnabled()) log.trace(prefix+"start filterRequest="+json(filterRequest, COMPACT_MAPPER)); | ||||
getActiveRequestCache().set(requestId, json(filterRequest, COMPACT_MAPPER), EX, ACTIVE_REQUEST_TIMEOUT); | getActiveRequestCache().set(requestId, json(filterRequest, COMPACT_MAPPER), EX, ACTIVE_REQUEST_TIMEOUT); | ||||
} else { | } else { | ||||
if (!isContentTypeMatch(matchersResponse, filterRequest.getContentType())) { | |||||
if (log.isInfoEnabled()) log.info(prefix+"none of the "+matchersResponse.getMatchers().size()+" matchers matched contentType="+filterRequest.getContentType()+", returning passthru"); | |||||
return passthru(request); | |||||
} | |||||
if (log.isTraceEnabled()) { | if (log.isTraceEnabled()) { | ||||
if (isLast) { | if (isLast) { | ||||
log.trace(prefix+"last filterRequest=" + json(filterRequest, COMPACT_MAPPER)); | log.trace(prefix+"last filterRequest=" + json(filterRequest, COMPACT_MAPPER)); | ||||
@@ -388,6 +397,18 @@ public class FilterHttpResource { | |||||
return ruleEngine.applyRulesToChunkAndSendResponse(request, filterRequest, chunkLength, isLast); | return ruleEngine.applyRulesToChunkAndSendResponse(request, filterRequest, chunkLength, isLast); | ||||
} | } | ||||
public boolean isContentTypeMatch(FilterMatchersResponse matchersResponse, String ct) { | |||||
final String prefix = "isContentTypeMatch("+matchersResponse.getRequest().getRequestId()+"): "; | |||||
final List<AppMatcher> matchers = matchersResponse.getMatchers(); | |||||
for (AppMatcher m : matchers) { | |||||
if (log.isDebugEnabled()) log.debug(prefix+"checking contentType match, matcher.contentTypeRegex="+m.getContentTypeRegex()+", contentType="+ct); | |||||
if (m.getContentTypePattern().matcher(ct).matches()) { | |||||
return true; | |||||
} | |||||
} | |||||
return false; | |||||
} | |||||
public Response passthru(@Context ContainerRequest request) { return ruleEngine.passthru(request); } | public Response passthru(@Context ContainerRequest request) { return ruleEngine.passthru(request); } | ||||
@GET @Path(EP_DATA+"/{requestId}/{matcherId}"+EP_READ) | @GET @Path(EP_DATA+"/{requestId}/{matcherId}"+EP_READ) | ||||
@@ -68,7 +68,7 @@ public class ReverseProxyResource { | |||||
final Map<String, AppMatcher> matchedMatchers = new HashMap<>(); | final Map<String, AppMatcher> matchedMatchers = new HashMap<>(); | ||||
for (AppMatcher m : matchers) { | for (AppMatcher m : matchers) { | ||||
// check for regex match | // check for regex match | ||||
if (m.matches(ub.getFullPath())) { | |||||
if (m.matchesUrl(ub.getFullPath())) { | |||||
// is this a total block? | // is this a total block? | ||||
if (m.blocked()) { | if (m.blocked()) { | ||||
log.debug("get: matcher("+m.getUuid()+") blocks request, returning 404 Not Found for "+ub.getFullPath()); | log.debug("get: matcher("+m.getUuid()+") blocks request, returning 404 Not Found for "+ub.getFullPath()); | ||||
@@ -40,10 +40,10 @@ | |||||
<!-- <logger name="org.cobbzilla.util.io.multi.MultiStream" level="TRACE" />--> | <!-- <logger name="org.cobbzilla.util.io.multi.MultiStream" level="TRACE" />--> | ||||
<!-- <logger name="bubble.filters.BubbleRateLimitFilter" level="TRACE" />--> | <!-- <logger name="bubble.filters.BubbleRateLimitFilter" level="TRACE" />--> | ||||
<!-- <logger name="org.cobbzilla.wizard.filters.RateLimitFilter" level="TRACE" />--> | <!-- <logger name="org.cobbzilla.wizard.filters.RateLimitFilter" level="TRACE" />--> | ||||
<logger name="bubble.service.stream.StandardRuleEngineService" level="DEBUG" /> | |||||
<!-- <logger name="bubble.service.stream.StandardRuleEngineService" level="DEBUG" />--> | |||||
<logger name="bubble.service.stream.ActiveStreamState" level="INFO" /> | <logger name="bubble.service.stream.ActiveStreamState" level="INFO" /> | ||||
<logger name="bubble.resources.stream" level="DEBUG" /> | <logger name="bubble.resources.stream" level="DEBUG" /> | ||||
<!-- <logger name="bubble.resources.stream.FilterHttpResource" level="TRACE" />--> | |||||
<logger name="bubble.resources.stream.FilterHttpResource" level="INFO" /> | |||||
<logger name="bubble.service.stream" level="INFO" /> | <logger name="bubble.service.stream" level="INFO" /> | ||||
<logger name="bubble.resources.message" level="INFO" /> | <logger name="bubble.resources.message" level="INFO" /> | ||||
<logger name="bubble.app.analytics" level="DEBUG" /> | <logger name="bubble.app.analytics" level="DEBUG" /> | ||||