diff --git a/bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java index 88b0d9b9..5b7a2a37 100644 --- a/bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java +++ b/bubble-server/src/main/java/bubble/app/bblock/BubbleBlockAppConfigDriver.java @@ -190,7 +190,7 @@ public class BubbleBlockAppConfigDriver extends AppConfigDriverBase { try { final AppRule rule = loadRule(account, app); final RuleDriver ruleDriver = loadDriver(account, rule, BubbleBlockRuleDriver.class); - final BubbleBlockRuleDriver unwiredDriver = (BubbleBlockRuleDriver) rule.initDriver(ruleDriver, TEST_MATCHER, account, TEST_DEVICE); + final BubbleBlockRuleDriver unwiredDriver = (BubbleBlockRuleDriver) rule.initDriver(app, ruleDriver, TEST_MATCHER, account, TEST_DEVICE); final BubbleBlockRuleDriver driver = configuration.autowire(unwiredDriver); final BlockDecision decision = driver.getDecision(host, path, userAgent, primary); return getBuiltinList(account, app).setResponse(decision); diff --git a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java index 1fe10407..36c89986 100644 --- a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java +++ b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java @@ -23,6 +23,8 @@ import bubble.server.BubbleConfiguration; import bubble.service.SearchService; import bubble.service.account.SyncPasswordService; import bubble.service.boot.SelfNodeService; +import bubble.service.cloud.DeviceIdService; +import bubble.service.stream.RuleEngineService; import lombok.Getter; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -76,6 +78,8 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc @Autowired private SearchService searchService; @Autowired private SyncPasswordService syncPasswordService; @Autowired private ReferralCodeDAO referralCodeDAO; + @Autowired private DeviceIdService deviceService; + @Autowired private RuleEngineService ruleEngineService; public Account newAccount(Request req, Account caller, AccountRegistration request, Account parent) { final AccountContact contact = new AccountContact() @@ -174,6 +178,7 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc final Account current = findByUuid(account.getUuid()); if (current == null) throw notFoundEx(account.getUuid()); account.setPreviousPasswordHash(current.getHashedPassword().getHashedPassword()); + account.setRefreshShowBlockStats(current.showBlockStats() != account.showBlockStats()); return super.preUpdate(account); } @@ -189,6 +194,10 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc if (account.syncPassword() && previousState.isHashedPasswordChanged() && !previousState.skipSyncPassword()) { syncPasswordService.syncPassword(account); } + if (previousState.isRefreshShowBlockStats()) { + deviceService.initBlockStats(account); + ruleEngineService.flushCaches(); + } } return super.postUpdate(account, context); } diff --git a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java index 15f87c48..10da65cf 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppMatcherDAO.java @@ -54,6 +54,7 @@ public class AppMatcherDAO extends AppTemplateEntityDAO { @Override public Object preCreate(AppMatcher matcher) { if (matcher.getConnCheck() == null) matcher.setConnCheck(false); if (matcher.getRequestCheck() == null) matcher.setRequestCheck(false); + if (matcher.getRequestModifier() == null) matcher.setRequestModifier(false); return super.preCreate(matcher); } diff --git a/bubble-server/src/main/java/bubble/exceptionmappers/BubbleOutOfMemoryProvider.java b/bubble-server/src/main/java/bubble/exceptionmappers/BubbleOutOfMemoryProvider.java new file mode 100644 index 00000000..f5438a4c --- /dev/null +++ b/bubble-server/src/main/java/bubble/exceptionmappers/BubbleOutOfMemoryProvider.java @@ -0,0 +1,13 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.exceptionmappers; + +import org.cobbzilla.wizard.exceptionmappers.OutOfMemoryErrorMapper; +import org.springframework.stereotype.Service; + +import javax.ws.rs.ext.Provider; + +@Provider @Service +public class BubbleOutOfMemoryProvider extends OutOfMemoryErrorMapper {} diff --git a/bubble-server/src/main/java/bubble/model/account/Account.java b/bubble-server/src/main/java/bubble/model/account/Account.java index de7b8b02..2d59842b 100644 --- a/bubble-server/src/main/java/bubble/model/account/Account.java +++ b/bubble-server/src/main/java/bubble/model/account/Account.java @@ -80,7 +80,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; public class Account extends IdentifiableBaseParentEntity implements TokenPrincipal, SqlViewSearchResult { public static final String[] UPDATE_FIELDS = { - "url", "description", "autoUpdatePolicy", "syncPassword", "preferredPlan" + "url", "description", "autoUpdatePolicy", "syncPassword", "preferredPlan", "showBlockStats" }; public static final String[] ADMIN_UPDATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "suspended", "admin"); public static final String[] CREATE_FIELDS = ArrayUtil.append(ADMIN_UPDATE_FIELDS, @@ -173,6 +173,12 @@ public class Account extends IdentifiableBaseParentEntity implements TokenPrinci @Getter @Setter private Boolean syncPassword; public boolean syncPassword() { return syncPassword == null ? true : syncPassword; } + @ECField(index=140) + @Getter @Setter private Boolean showBlockStats; + public boolean showBlockStats() { return showBlockStats == null ? true : showBlockStats; } + + @JsonIgnore @Transient @Getter @Setter private boolean refreshShowBlockStats; + @JsonIgnore @Embedded @Getter private HashedPassword hashedPassword; public Account setHashedPassword (HashedPassword newPass) { this.hashedPassword = newPass; diff --git a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java index 59c5dc74..82146906 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java +++ b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java @@ -26,7 +26,6 @@ import javax.validation.constraints.Size; import java.util.regex.Pattern; import static bubble.ApiConstants.EP_MATCHERS; -import static org.cobbzilla.util.daemon.ZillaRuntime.bool; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; @@ -91,7 +90,7 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H @Getter @Setter private String urlRegex; public boolean hasUrlRegex() { return !empty(urlRegex) && !urlRegex.equals(WILDCARD_URL); } - @Transient @JsonIgnore public Pattern getUrlPattern() { return Pattern.compile(getUrlRegex()); } + @Transient @JsonIgnore @Getter(lazy=true) private final Pattern urlPattern = Pattern.compile(getUrlRegex()); public boolean matchesUrl (String value) { return getUrlPattern().matcher(value).find(); } @ECSearchable(filter=true) @ECField(index=70) @@ -100,9 +99,8 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H @Getter @Setter private String contentTypeRegex; public boolean hasContentTypeRegex() { return !empty(contentTypeRegex); } - @Transient @JsonIgnore public Pattern getContentTypePattern () { - return hasContentTypeRegex() ? Pattern.compile(getContentTypeRegex()) : DEFAULT_CONTENT_TYPE_PATTERN; - } + @Transient @JsonIgnore @Getter(lazy=true) private final Pattern contentTypePattern + = hasContentTypeRegex() ? Pattern.compile(getContentTypeRegex()) : DEFAULT_CONTENT_TYPE_PATTERN; public boolean matchesContentType (String value) { return getContentTypePattern().matcher(value).find(); } @ECSearchable(filter=true) @ECField(index=80) @@ -111,10 +109,9 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H @Getter @Setter private String userAgentRegex; public boolean hasUserAgentRegex() { return !empty(userAgentRegex); } - @Transient @JsonIgnore public Pattern getUserAgentPattern () { - return hasUserAgentRegex() ? Pattern.compile(getUserAgentRegex()) : DEFAULT_CONTENT_TYPE_PATTERN; - } - public boolean matchesUserAgent (String value) { return getUserAgentPattern().matcher(value).find(); } + @Transient @JsonIgnore @Getter(lazy=true) private final Pattern userAgentPattern + = hasUserAgentRegex() ? Pattern.compile(getUserAgentRegex()) : null; + public boolean matchesUserAgent (String value) { return !hasUserAgentRegex() || getUserAgentPattern().matcher(value).find(); } @ECSearchable @ECField(index=90) @ECForeignKey(entity=AppRule.class) @@ -132,14 +129,19 @@ public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, H @ECSearchable @ECField(index=120, required=EntityFieldRequired.optional) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean connCheck; - public boolean connCheck () { return bool(connCheck); } + public boolean connCheck () { return connCheck != null && connCheck; } @ECSearchable @ECField(index=130, required=EntityFieldRequired.optional) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean requestCheck; - public boolean requestCheck () { return bool(requestCheck); } + public boolean requestCheck () { return requestCheck != null && requestCheck; } + + @ECSearchable @ECField(index=140, required=EntityFieldRequired.optional) + @ECIndex @Column(nullable=false) + @Getter @Setter private Boolean requestModifier; + public boolean requestModifier () { return requestModifier != null && requestModifier; } - @ECSearchable @ECField(index=140) + @ECSearchable @ECField(index=150) @Column(nullable=false) @Getter @Setter private Integer priority = 0; diff --git a/bubble-server/src/main/java/bubble/model/app/AppRule.java b/bubble-server/src/main/java/bubble/model/app/AppRule.java index b51263cf..bf872391 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppRule.java +++ b/bubble-server/src/main/java/bubble/model/app/AppRule.java @@ -90,9 +90,15 @@ public class AppRule extends IdentifiableBaseParentEntity implements AppTemplate @Column(nullable=false, length=UUID_MAXLEN) @Getter @Setter private String driver; - public AppRuleDriver initDriver(RuleDriver driver, AppMatcher matcher, Account account, Device device) { + public AppRuleDriver initDriver(BubbleApp app, RuleDriver driver, AppMatcher matcher, Account account, Device device) { final AppRuleDriver d = driver.getDriver(); - d.init(json(configJson, JsonNode.class), driver.getUserConfig(), this, matcher, account, device); + d.init(json(configJson, JsonNode.class), driver.getUserConfig(), app, this, matcher, account, device); + return d; + } + + public AppRuleDriver initQuickDriver(BubbleApp app, RuleDriver driver, AppMatcher matcher, Account account, Device device) { + final AppRuleDriver d = driver.getDriver(); + d.initQuick(json(configJson, JsonNode.class), driver.getUserConfig(), app, this, matcher, account, device); return d; } diff --git a/bubble-server/src/main/java/bubble/model/device/Device.java b/bubble-server/src/main/java/bubble/model/device/Device.java index 93fff6bc..0d9e2aab 100644 --- a/bubble-server/src/main/java/bubble/model/device/Device.java +++ b/bubble-server/src/main/java/bubble/model/device/Device.java @@ -14,6 +14,7 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; import org.cobbzilla.wizard.model.Identifiable; import org.cobbzilla.wizard.model.IdentifiableBase; import org.cobbzilla.wizard.model.entityconfig.annotations.*; @@ -36,7 +37,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @Entity @ECType(root=true) @ToString(of={"name"}) @ECTypeURIs(baseURI=EP_DEVICES, listFields={"name", "enabled"}) -@NoArgsConstructor @Accessors(chain=true) +@NoArgsConstructor @Accessors(chain=true) @Slf4j @ECIndexes({ @ECIndex(unique=true, of={"account", "network", "name"}), @ECIndex(unique=true, of={"account", "name"}), diff --git a/bubble-server/src/main/java/bubble/resources/stream/AppAssetsResource.java b/bubble-server/src/main/java/bubble/resources/stream/AppAssetsResource.java index 5fe47048..a9a605cf 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/AppAssetsResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/AppAssetsResource.java @@ -38,8 +38,8 @@ import static org.cobbzilla.wizard.stream.DataUrlStreamingOutput.dataUrlBytes; @Slf4j public class AppAssetsResource { - private String locale; - private BubbleApp app; + private final String locale; + private final BubbleApp app; public AppAssetsResource(String locale, BubbleApp app) { this.locale = locale; diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java index 83a9f0e2..ca4478d4 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java @@ -6,6 +6,7 @@ package bubble.resources.stream; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang.ArrayUtils; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @@ -16,6 +17,7 @@ public class FilterConnCheckRequest { @Getter @Setter private String[] fqdns; public boolean hasFqdns() { return !empty(fqdns); } + public boolean hasFqdn(String f) { return hasFqdns() && ArrayUtils.contains(fqdns, f); } @Getter @Setter private String remoteAddr; public boolean hasRemoteAddr() { return !empty(remoteAddr); } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java index 764b78b5..90df4b36 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java @@ -5,11 +5,14 @@ package bubble.resources.stream; import bubble.dao.app.AppDataDAO; +import bubble.dao.app.AppRuleDAO; +import bubble.dao.app.BubbleAppDAO; +import bubble.dao.app.RuleDriverDAO; import bubble.model.account.Account; -import bubble.model.app.AppData; -import bubble.model.app.AppDataFormat; -import bubble.model.app.AppMatcher; +import bubble.model.app.*; import bubble.model.device.Device; +import bubble.rule.AppRuleDriver; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -29,12 +32,28 @@ import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.wizard.resources.ResourceUtil.*; +@Consumes(APPLICATION_JSON) +@Produces(APPLICATION_JSON) @Slf4j public class FilterDataResource { - private Account account; - private Device device; - private AppMatcher matcher; + @Autowired private AppDataDAO dataDAO; + @Autowired private AppRuleDAO ruleDAO; + @Autowired private RuleDriverDAO driverDAO; + @Autowired private BubbleAppDAO appDAO; + + private final Account account; + private final Device device; + private final AppMatcher matcher; + @Getter(lazy=true) private final AppRule rule = ruleDAO.findByUuid(matcher.getRule()); + @Getter(lazy=true) private final RuleDriver driver = driverDAO.findByUuid(getRule().getDriver()); + @Getter(lazy=true) private final BubbleApp app = appDAO.findByUuid(matcher.getApp()); + @Getter(lazy=true) private final AppRuleDriver ruleDriver = initAppRuleDriver(); + + private AppRuleDriver initAppRuleDriver() { + log.warn("initAppRuleDriver: initializing driver...."); + return getRule().initQuickDriver(getApp(), getDriver(), matcher, account, device); + } public FilterDataResource (Account account, Device device, AppMatcher matcher) { this.account = account; @@ -42,10 +61,7 @@ public class FilterDataResource { this.matcher = matcher; } - @Autowired private AppDataDAO dataDAO; - @GET @Path(EP_READ) - @Produces(APPLICATION_JSON) public Response readData(@Context Request req, @Context ContainerRequest ctx, @QueryParam("format") AppDataFormat format) { @@ -70,8 +86,6 @@ public class FilterDataResource { } @POST @Path(EP_WRITE) - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) public Response writeData(@Context Request req, @Context ContainerRequest ctx, AppData data) { @@ -80,7 +94,6 @@ public class FilterDataResource { } @GET @Path(EP_WRITE) - @Produces(APPLICATION_JSON) public Response writeData(@Context Request req, @Context ContainerRequest ctx, @QueryParam(Q_DATA) String dataJson, @@ -123,4 +136,12 @@ public class FilterDataResource { return dataDAO.set(data); } + @GET @Path(EP_READ+"/rule/{id}") + public Response readRuleData(@Context Request req, + @Context ContainerRequest ctx, + @PathParam("id") String id) { + final Object data = getRuleDriver().readData(id); + return data == null ? notFound(id) : ok(data); + } + } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java index a236e5d7..13aee225 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpRequest.java @@ -14,6 +14,8 @@ import lombok.Setter; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.http.HttpContentEncodingType; +import org.cobbzilla.util.http.HttpContentTypes; +import org.cobbzilla.util.http.HttpUtil; import java.util.List; import java.util.regex.Matcher; @@ -33,6 +35,8 @@ public class FilterHttpRequest { @Getter @Setter private Account account; @Getter @Setter private String contentType; + @JsonIgnore public boolean isHtml () { return HttpContentTypes.isHtml(getContentType()); } + @Getter @Setter private Long contentLength; public boolean hasContentLength () { return contentLength != null; } @@ -61,6 +65,8 @@ public class FilterHttpRequest { } public boolean hasMatchers() { return matchersResponse != null && matchersResponse.hasMatchers(); } + public boolean hasRequestMatchers() { return hasMatchers() && matchersResponse.hasRequestCheckMatchers(); } + public boolean hasRequestModifiers() { return hasMatchers() && matchersResponse.hasRequestModifiers(); } @JsonIgnore public List getMatchers() { return !hasMatchers() ? null : matchersResponse.getMatchers(); } @@ -68,6 +74,12 @@ public class FilterHttpRequest { return !hasMatchers() || !matchersResponse.hasRequest() ? null : matchersResponse.getRequest().getUrl(); } + @JsonIgnore public String getUserAgent() { + return !hasMatchers() || !matchersResponse.hasRequest() ? null : matchersResponse.getRequest().getUserAgent(); + } + + @JsonIgnore public boolean isBrowser () { return HttpUtil.isBrowser(getUserAgent()); } + public boolean hasApp(String appId) { if (!hasMatchers()) return false; for (AppMatcher m : getMatchers()) if (m.getApp().equals(appId)) return true; diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 44d75656..d46df88c 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -15,18 +15,23 @@ import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; import bubble.model.app.AppSite; import bubble.model.app.BubbleApp; +import bubble.model.cloud.BubbleNetwork; import bubble.model.cloud.BubbleNode; import bubble.model.device.Device; import bubble.rule.FilterMatchDecision; +import bubble.service.block.BlockStatsService; import bubble.server.BubbleConfiguration; +import bubble.service.block.BlockStatsSummary; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.DeviceIdService; import bubble.service.stream.ConnectionCheckResponse; import bubble.service.stream.StandardRuleEngineService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.ExpirationEvictionPolicy; import org.cobbzilla.util.collection.ExpirationMap; import org.cobbzilla.util.http.HttpContentEncodingType; +import org.cobbzilla.util.network.NetworkUtil; import org.cobbzilla.wizard.cache.redis.RedisService; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -50,6 +55,7 @@ import static java.util.Collections.emptyMap; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; +import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; @@ -75,6 +81,7 @@ public class FilterHttpResource { @Autowired private RedisService redis; @Autowired private BubbleConfiguration configuration; @Autowired private SelfNodeService selfNodeService; + @Autowired private BlockStatsService blockStats; private static final long ACTIVE_REQUEST_TIMEOUT = HOURS.toSeconds(12); @@ -143,8 +150,15 @@ public class FilterHttpResource { } validateMitmCall(req); - // if the requested IP is the same as our IP, then always passthru - if (isForUs(connCheckRequest)) return ok(); + // is the requested IP is the same as our IP? + final boolean isLocalIp = isForLocalIp(connCheckRequest); + if (isLocalIp) { + // if it is for our host or net name, passthru + if (connCheckRequest.hasFqdns() && (connCheckRequest.hasFqdn(getThisNode().getFqdn()) || connCheckRequest.hasFqdn(getThisNetwork().getNetworkDomain()))) { + if (log.isDebugEnabled()) log.debug(prefix + "returning passthru for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.passthru); + } + } final String vpnAddr = connCheckRequest.getRemoteAddr(); final Device device = deviceIdService.findDeviceByIp(vpnAddr); @@ -154,13 +168,25 @@ public class FilterHttpResource { } else if (log.isTraceEnabled()) { log.trace(prefix+"found device "+device.id()+" for IP "+vpnAddr); } - final Account account = findCaller(device.getAccount()); + final String accountUuid = device.getAccount(); + final Account account = findCaller(accountUuid); if (account == null) { - if (log.isDebugEnabled()) log.debug(prefix+"account not found for uuid "+device.getAccount()+", returning not found"); + if (log.isDebugEnabled()) log.debug(prefix+"account not found for uuid "+ accountUuid +", returning not found"); return notFound(); } - final List matchers = matcherDAO.findByAccountAndEnabledAndConnCheck(device.getAccount()); + if (isLocalIp) { + if (showStats(accountUuid)) { + // allow it for now + if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showBlockStats==true) for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.noop); + } else { + if (log.isDebugEnabled()) log.debug(prefix + "returning block (showBlockStats==false) for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.block); + } + } + + final List matchers = getConnCheckMatchers(accountUuid); final List retained = new ArrayList<>(); for (AppMatcher matcher : matchers) { final BubbleApp app = appDAO.findByUuid(matcher.getApp()); @@ -170,25 +196,44 @@ public class FilterHttpResource { retained.add(matcher); } - final String[] fqdns = connCheckRequest.getFqdns(); - for (String fqdn : fqdns) { - final ConnectionCheckResponse checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); - if (checkResponse != ConnectionCheckResponse.noop) { - if (log.isDebugEnabled()) log.debug(prefix + "returning "+checkResponse+" for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); - return ok(checkResponse); + ConnectionCheckResponse checkResponse = ConnectionCheckResponse.noop; + if (connCheckRequest.hasFqdns()) { + final String[] fqdns = connCheckRequest.getFqdns(); + for (String fqdn : fqdns) { + checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); + if (checkResponse != ConnectionCheckResponse.noop) { + if (log.isDebugEnabled()) log.debug(prefix + "found " + checkResponse + " (breaking) for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); + break; + } } + if (log.isDebugEnabled()) log.debug(prefix+"returning "+checkResponse+" for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); + return ok(checkResponse); + + } else { + if (log.isDebugEnabled()) log.debug(prefix+"returning noop for NO fqdns, addr="+connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.noop); } - if (log.isDebugEnabled()) log.debug(prefix+"returning noop for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); - return ok(ConnectionCheckResponse.noop); } - private boolean isForUs(FilterConnCheckRequest connCheckRequest) { - final BubbleNode thisNode = selfNodeService.getThisNode(); - return connCheckRequest.hasAddr() - && (thisNode.hasIp4() && thisNode.getIp4().equals(connCheckRequest.getAddr()) - || thisNode.hasIp6() && thisNode.getIp6().equals(connCheckRequest.getAddr())); + private final Map> connCheckMatcherCache = new ExpirationMap<>(10, HOURS.toMillis(1), ExpirationEvictionPolicy.atime); + public List getConnCheckMatchers(String accountUuid) { + return connCheckMatcherCache.computeIfAbsent(accountUuid, k -> matcherDAO.findByAccountAndEnabledAndConnCheck(k)); + } + + private boolean isForLocalIp(FilterConnCheckRequest connCheckRequest) { + return connCheckRequest.hasAddr() && getConfiguredIps().contains(connCheckRequest.getAddr()); + } + + private boolean isForLocalIp(FilterMatchersRequest matchersRequest) { + return matchersRequest.hasServerAddr() && getConfiguredIps().contains(matchersRequest.getServerAddr()); } + @Getter(lazy=true) private final Set configuredIps = NetworkUtil.configuredIps(); + @Getter(lazy=true) private final BubbleNode thisNode = selfNodeService.getThisNode(); + @Getter(lazy=true) private final BubbleNetwork thisNetwork = selfNodeService.getThisNetwork(); + + public boolean showStats(String accountUuid) { return deviceIdService.doShowBlockStats(accountUuid); } + @POST @Path(EP_MATCHERS+"/{requestId}") @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON) @@ -208,13 +253,13 @@ public class FilterHttpResource { if (log.isDebugEnabled()) log.debug(prefix+"starting for filterRequest="+json(filterRequest, COMPACT_MAPPER)); else if (extraLog) log.error(prefix+"starting for filterRequest="+json(filterRequest, COMPACT_MAPPER)); - if (!filterRequest.hasRemoteAddr()) { + if (!filterRequest.hasClientAddr()) { if (log.isDebugEnabled()) log.debug(prefix+"no VPN address provided, returning no matchers"); else if (extraLog) log.error(prefix+"no VPN address provided, returning no matchers"); return ok(NO_MATCHERS); } - final String vpnAddr = filterRequest.getRemoteAddr(); + final String vpnAddr = filterRequest.getClientAddr(); final Device device = deviceIdService.findDeviceByIp(vpnAddr); if (device == null) { if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning no matchers"); @@ -225,9 +270,26 @@ public class FilterHttpResource { } filterRequest.setDevice(device.getUuid()); + // if this is for a local ip, it's an automatic block + // legitimate local requests would have been passthru and never reached here + final boolean isLocalIp = isForLocalIp(filterRequest); + final boolean showStats = showStats(device.getAccount()); + if (isLocalIp) { + if (filterRequest.isBrowser() && showStats) { + blockStats.record(filterRequest, FilterMatchDecision.abort_not_found); + } + if (log.isDebugEnabled()) log.debug(prefix + "returning FORBIDDEN (showBlockStats=="+ showStats +")"); + return forbidden(); + } + final FilterMatchersResponse response = getMatchersResponse(filterRequest, req, request); if (log.isDebugEnabled()) log.debug(prefix+"returning response: "+json(response, COMPACT_MAPPER)); else if (extraLog) log.error(prefix+"returning response: "+json(response, COMPACT_MAPPER)); + + if (filterRequest.isBrowser() && showStats) { + blockStats.record(filterRequest, response.getDecision()); + } + return ok(response); } @@ -346,6 +408,7 @@ public class FilterHttpResource { public Response flushCaches(@Context ContainerRequest request) { final Account caller = userPrincipal(request); if (!caller.admin()) return forbidden(); + connCheckMatcherCache.clear(); return ok(ruleEngine.flushCaches()); } @@ -520,10 +583,24 @@ public class FilterHttpResource { final FilterSubContext filterCtx = new FilterSubContext(req, requestId); if (!filterCtx.request.hasMatcher(matcherId)) throw notFoundEx(matcherId); - final AppMatcher matcher = matcherDAO.findByAccountAndId(filterCtx.request.getAccount().getUuid(), matcherId); + final Account account = filterCtx.request.getAccount(); + account.setMtime(0); // only create one FilterDataResource + final AppMatcher matcher = matcherDAO.findByAccountAndId(account.getUuid(), matcherId); if (matcher == null) throw notFoundEx(matcherId); - return configuration.subResource(FilterDataResource.class, filterCtx.request.getAccount(), filterCtx.request.getDevice(), matcher); + final Device device = filterCtx.request.getDevice(); + device.setMtime(0); // only create one FilterDataResource + return configuration.subResource(FilterDataResource.class, account, device, matcher); + } + + @GET @Path(EP_STATUS+"/{requestId}") + public Response getRequestStatus(@Context Request req, + @Context ContainerRequest ctx, + @PathParam("requestId") String requestId) { + final FilterSubContext filterCtx = new FilterSubContext(req, requestId); + final BlockStatsSummary summary = blockStats.getSummary(requestId); + if (summary == null) return notFound(requestId); + return ok(summary); } @Path(EP_ASSETS+"/{requestId}/{appId}") diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java index 882b0a01..c23971d0 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java @@ -9,8 +9,10 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; +import org.cobbzilla.util.http.HttpUtil; import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.http.HttpSchemes.stripScheme; @NoArgsConstructor @Accessors(chain=true) public class FilterMatchersRequest { @@ -24,16 +26,27 @@ public class FilterMatchersRequest { @Getter @Setter private String fqdn; @Getter @Setter private String uri; @Getter @Setter private String userAgent; + @JsonIgnore public boolean isBrowser () { return HttpUtil.isBrowser(getUserAgent()); } @Getter @Setter private String referer; public boolean hasReferer () { return !empty(referer) && !referer.equals("NONE"); } - @Getter @Setter private String remoteAddr; - public boolean hasRemoteAddr() { return !empty(remoteAddr); } + @JsonIgnore public String getRefererFqdn () { + if (!hasReferer()) return null; + final String base = stripScheme(referer); + final int slashPos = base.indexOf('/'); + return slashPos == -1 ? base : base.substring(0, slashPos); + } + + @Getter @Setter private String clientAddr; + public boolean hasClientAddr() { return !empty(clientAddr); } + + @Getter @Setter private String serverAddr; + public boolean hasServerAddr() { return !empty(serverAddr); } // note: we do *not* include the requestId in the cache, if we did then the // FilterHttpResource.matchersCache cache would be useless, since every cache entry would be unique - public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, remoteAddr); } + public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, clientAddr, serverAddr); } @JsonIgnore public String getUrl() { return fqdn + uri; } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java index c182561e..67439b92 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersResponse.java @@ -6,6 +6,7 @@ package bubble.resources.stream; import bubble.model.app.AppMatcher; import bubble.rule.FilterMatchDecision; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -37,6 +38,10 @@ public class FilterMatchersResponse { return hasMatchers() && getMatchers().stream().anyMatch(AppMatcher::requestCheck); } + public boolean hasRequestModifiers() { + return hasMatchers() && getMatchers().stream().anyMatch(AppMatcher::requestModifier); + } + public FilterMatchersResponse setRequestId(String requestId) { if (request == null) { if (log.isInfoEnabled()) log.info("setRequestId("+requestId+"): request is null, cannot set"); @@ -56,4 +61,6 @@ public class FilterMatchersResponse { return "FilterMatchersResponse{"+decision+(hasMatchers() ? ", matchers="+names(matchers) : "")+"}"; } + @JsonIgnore public String getAccount() { return hasMatchers() ? getMatchers().get(0).getAccount() : null; } + } diff --git a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java index f1c742e4..21bdf1e0 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java @@ -90,7 +90,7 @@ public class ReverseProxyResource { .setUri(ub.getFullPath()) .setUserAgent(getUserAgent(request)) .setReferer(getReferer(request)) - .setRemoteAddr(remoteHost) + .setClientAddr(remoteHost) .setDevice(device.getUuid())) .setRequestId(id) .setDecision(FilterMatchDecision.match) diff --git a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java index f20e7452..20b924a4 100644 --- a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java @@ -11,29 +11,46 @@ import bubble.dao.device.DeviceDAO; import bubble.model.account.Account; import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; import bubble.model.device.Device; +import bubble.resources.stream.FilterHttpRequest; import bubble.server.BubbleConfiguration; +import bubble.service.cloud.StandardDeviceIdService; import bubble.service.stream.AppPrimerService; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; import lombok.Getter; import lombok.Setter; +import org.apache.commons.io.input.ReaderInputStream; +import org.cobbzilla.util.collection.ExpirationMap; +import org.cobbzilla.util.handlebars.HandlebarsUtil; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.io.regex.RegexFilterReader; +import org.cobbzilla.util.io.regex.RegexReplacementFilter; import org.cobbzilla.util.system.Bytes; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; +import java.io.File; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.Map; + +import static bubble.ApiConstants.HOME_DIR; +import static bubble.rule.RequestModifierRule.ICON_JS_TEMPLATE; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.regex.RegexReplacementFilter.DEFAULT_PREFIX_REPLACEMENT_WITH_MATCH; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.security.ShaUtil.sha256_hex; +import static org.cobbzilla.util.string.StringUtil.UTF8cs; public abstract class AbstractAppRuleDriver implements AppRuleDriver { public static final int RESPONSE_BUFSIZ = (int) (64 * Bytes.KB); - public static final String CTX_JS_PREFIX = "JS_PREFIX"; - public static final String CTX_BUBBLE_REQUEST_ID = "BUBBLE_REQUEST_ID"; - public static final String CTX_BUBBLE_DATA_ID = "BUBBLE_DATA_ID"; - public static final String CTX_BUBBLE_HOME = "BUBBLE_HOME"; - public static final String CTX_SITE = "SITE"; - @Autowired protected BubbleConfiguration configuration; @Autowired protected AppDataDAO appDataDAO; @Autowired protected AppSiteDAO appSiteDAO; @@ -41,11 +58,13 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { @Autowired protected BubbleNetworkDAO networkDAO; @Autowired protected DeviceDAO deviceDAO; @Autowired protected AppPrimerService appPrimerService; + @Autowired protected StandardDeviceIdService deviceService; @Getter @Setter private AppRuleDriver next; protected JsonNode config; protected JsonNode userConfig; + protected BubbleApp app; protected AppMatcher matcher; protected AppRule rule; protected Account account; @@ -62,12 +81,14 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { @Override public void init(JsonNode config, JsonNode userConfig, + BubbleApp app, AppRule rule, AppMatcher matcher, Account account, Device device) { this.config = config; this.userConfig = userConfig; + this.app = app; this.matcher = matcher; this.rule = rule; this.account = account; @@ -77,4 +98,129 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { } } + public static final String DEFAULT_INSERTION_REGEX = "<\\s*head[^>]*>"; + public static final String DEFAULT_SCRIPT_OPEN = ""; + + protected static String insertionRegex (String customRegex) { + return empty(customRegex) ? DEFAULT_INSERTION_REGEX : customRegex; + } + + protected static String scriptOpen (FilterHttpRequest filterRequest, String customNonceOpen, String customNoNonceOpen) { + return filterRequest.hasScriptNonce() + ? (empty(customNonceOpen) ? DEFAULT_SCRIPT_NONCE_OPEN : customNonceOpen).replace(NONCE_VAR, filterRequest.getScriptNonce()) + : (empty(customNoNonceOpen) ? DEFAULT_SCRIPT_OPEN : customNoNonceOpen); + } + + protected static String scriptClose (String customClose) { + return empty(customClose) ? DEFAULT_SCRIPT_CLOSE : customClose; + } + + protected String getSiteJsTemplate (String defaultSiteTemplate) { + return loadTemplate(defaultSiteTemplate, requestModConfig().getSiteJsTemplate()); + } + + protected String loadTemplate(String defaultTemplate, String templatePath) { + if (configuration.getEnvironment().containsKey("DEBUG_RULE_TEMPLATES")) { + final File templateFile = new File(HOME_DIR + "/debugTemplates/" + templatePath); + if (templateFile.exists()) { + log.error("loadTemplate: debug file found (using it): "+abs(templateFile)); + return FileUtil.toStringOrDie(templateFile); + } else { + log.error("loadTemplate: debug file not found (using default): "+abs(templateFile)); + } + } + return defaultTemplate; + } + + private RequestModifierConfig requestModConfig() { + if (this instanceof RequestModifierRule) return ((RequestModifierRule) this).getRequestModifierConfig(); + return die("requestModConfig: rule "+getClass().getName()+" does not implement RequestModifierRule"); + } + + @Getter(lazy=true) private final String insertionRegex = insertionRegex(requestModConfig().getInsertionRegex()); + + @Getter(lazy=true) private final String scriptClose = scriptClose(requestModConfig().getScriptClose()); + + protected InputStream filterInsertJs(InputStream in, + FilterHttpRequest filterRequest, + Map filterCtx, + String bubbleJsTemplate, + String defaultSiteTemplate, + String siteJsInsertionVar, + boolean showIcon) { + final RequestModifierConfig modConfig = requestModConfig(); + final String replacement = DEFAULT_PREFIX_REPLACEMENT_WITH_MATCH + + scriptOpen(filterRequest, modConfig.getScriptOpenNonce(), modConfig.getScriptOpenNoNonce()) + + getBubbleJs(filterRequest.getId(), filterCtx, bubbleJsTemplate, defaultSiteTemplate, siteJsInsertionVar, showIcon) + + getScriptClose(); + + final RegexReplacementFilter filter = new RegexReplacementFilter(getInsertionRegex(), replacement); + RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in), filter).setMaxMatches(1); + if (modConfig.hasAdditionalRegexReplacements()) { + for (BubbleRegexReplacement re : modConfig.getAdditionalRegexReplacements()) { + final RegexReplacementFilter f = new RegexReplacementFilter(re.getInsertionRegex(), re.getReplacement()); + reader = new RegexFilterReader(reader, f); + } + } + + return new ReaderInputStream(reader, UTF8cs); + } + + protected String getBubbleJs(String requestId, + Map filterCtx, + String bubbleJsTemplate, + String defaultSiteTemplate, + String siteJsInsertionVar, + boolean showIcon) { + final Map ctx = getBubbleJsContext(requestId, filterCtx); + + if (!empty(siteJsInsertionVar) && !empty(defaultSiteTemplate)) { + final String siteJs = HandlebarsUtil.apply(getHandlebars(), getSiteJsTemplate(defaultSiteTemplate), ctx); + ctx.put(siteJsInsertionVar, siteJs); + } + if (showIcon) { + ctx.put(CTX_ICON_JS, HandlebarsUtil.apply(getHandlebars(), ICON_JS_TEMPLATE, ctx)); + } + return HandlebarsUtil.apply(getHandlebars(), bubbleJsTemplate, ctx); + } + + public static final String CTX_JS_PREFIX = "JS_PREFIX"; + public static final String CTX_PAGE_PREFIX = "PAGE_PREFIX"; + public static final String CTX_PAGE_ONREADY_INTERVAL = "PAGE_ONREADY_INTERVAL"; + public static final String CTX_BUBBLE_REQUEST_ID = "BUBBLE_REQUEST_ID"; + public static final String CTX_BUBBLE_DATA_ID = "BUBBLE_DATA_ID"; + public static final String CTX_BUBBLE_HOME = "BUBBLE_HOME"; + public static final String CTX_BUBBLE_SITE_NAME = "BUBBLE_SITE_NAME"; + public static final String CTX_BUBBLE_APP_NAME = "BUBBLE_APP_NAME"; + public static final String CTX_ICON_JS = "ICON_JS"; + public static final String CTX_APP_CONTROLS_Z_INDEX = "APP_CONTROLS_Z_INDEX"; + + public static final int PAGE_ONREADY_INTERVAL = 50; + public static final int APP_CONTROLS_Z_INDEX = 2147483640; + + private String getPagePrefix(String requestId) { return "__bubble_page_"+sha256_hex(requestId); } + private String getJsPrefix(String requestId) { return "__bubble_js_"+sha256_hex(requestId+"_"+getClass().getName()); } + + protected Map getBubbleJsContext(String requestId, Map filterCtx) { + final Map ctx = new HashMap<>(); + ctx.put(CTX_PAGE_PREFIX, getPagePrefix(requestId)); + ctx.put(CTX_JS_PREFIX, getJsPrefix(requestId)); + ctx.put(CTX_PAGE_ONREADY_INTERVAL, PAGE_ONREADY_INTERVAL); + ctx.put(CTX_APP_CONTROLS_Z_INDEX, APP_CONTROLS_Z_INDEX); + ctx.put(CTX_BUBBLE_REQUEST_ID, requestId); + ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase()); + ctx.put(CTX_BUBBLE_SITE_NAME, getSiteName(matcher)); + ctx.put(CTX_BUBBLE_APP_NAME, app.getName()); + ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId)); + return ctx; + } + + private static final ExpirationMap siteNameCache = new ExpirationMap<>(); + protected String getSiteName(AppMatcher matcher) { + return siteNameCache.computeIfAbsent(matcher.getSite(), k -> appSiteDAO.findByAccountAndId(matcher.getAccount(), matcher.getSite()).getName()); + } + } diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 5d67e806..7457ef14 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -7,6 +7,7 @@ package bubble.rule; import bubble.model.account.Account; import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; import bubble.model.device.Device; import bubble.resources.stream.FilterHttpRequest; import bubble.resources.stream.FilterMatchersRequest; @@ -30,7 +31,6 @@ import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.cobbzilla.util.io.StreamUtil.stream2bytes; import static org.cobbzilla.util.io.StreamUtil.stream2string; -import static org.cobbzilla.util.security.ShaUtil.sha256_hex; import static org.cobbzilla.util.string.StringUtil.getPackagePath; public interface AppRuleDriver { @@ -73,11 +73,22 @@ public interface AppRuleDriver { default void init(JsonNode config, JsonNode userConfig, + BubbleApp app, AppRule rule, AppMatcher matcher, Account account, Device device) {} + default void initQuick(JsonNode config, + JsonNode userConfig, + BubbleApp app, + AppRule rule, + AppMatcher matcher, + Account account, + Device device) { + init(config, userConfig, app, rule, matcher, account, device); + } + default FilterMatchDecision preprocess(AppRuleHarness ruleHarness, FilterMatchersRequest filter, Account account, @@ -114,8 +125,6 @@ public interface AppRuleDriver { default Handlebars getHandlebars() { return null; } - static String getJsPrefix(String requestId) { return "__bubble_"+sha256_hex(requestId)+"_"; } - default String locateResource(String res) { if (!res.startsWith("@")) return res; final String prefix = getPackagePath(getClass()) + "/" + getClass().getSimpleName(); @@ -161,4 +170,6 @@ public interface AppRuleDriver { return sageRuleConfig; } + default Object readData(String id) { return null; } + } diff --git a/bubble-server/src/main/java/bubble/rule/FilterMatchDecision.java b/bubble-server/src/main/java/bubble/rule/FilterMatchDecision.java index fe2a8d29..5702adba 100644 --- a/bubble-server/src/main/java/bubble/rule/FilterMatchDecision.java +++ b/bubble-server/src/main/java/bubble/rule/FilterMatchDecision.java @@ -16,7 +16,7 @@ import static org.cobbzilla.util.http.HttpStatusCodes.OK; public enum FilterMatchDecision { no_match (OK), // associated matcher should not be included in request processing - match (OK), // associated should be included in request processing + match (OK), // associated matcher should be included in request processing abort_ok (OK), // abort request processing, return empty 200 OK response to client abort_not_found (NOT_FOUND), // abort request processing, return empty 404 Not Found response to client pass_thru (OK); // pass-through TLS request, do not intercept @@ -26,4 +26,6 @@ public enum FilterMatchDecision { @Getter private final int httpStatusCode; public int httpStatus() { return getHttpStatusCode(); } + public boolean isAbort () { return this.name().startsWith("abort"); } + } diff --git a/bubble-server/src/main/java/bubble/rule/RequestModifierConfig.java b/bubble-server/src/main/java/bubble/rule/RequestModifierConfig.java new file mode 100644 index 00000000..781dd293 --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/RequestModifierConfig.java @@ -0,0 +1,26 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule; + +import lombok.Getter; +import lombok.Setter; +import org.cobbzilla.util.collection.NameAndValue; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class RequestModifierConfig { + + @Getter @Setter private String siteJsTemplate; + @Getter @Setter private NameAndValue[] additionalJsTemplates; + + @Getter @Setter private String insertionRegex; + @Getter @Setter private String scriptOpenNonce; + @Getter @Setter private String scriptOpenNoNonce; + @Getter @Setter private String scriptClose; + + @Getter @Setter private BubbleRegexReplacement[] additionalRegexReplacements; + public boolean hasAdditionalRegexReplacements () { return !empty(additionalRegexReplacements); } + +} diff --git a/bubble-server/src/main/java/bubble/rule/RequestModifierRule.java b/bubble-server/src/main/java/bubble/rule/RequestModifierRule.java new file mode 100644 index 00000000..4319940e --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/RequestModifierRule.java @@ -0,0 +1,17 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.rule; + +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.string.StringUtil.getPackagePath; + +public interface RequestModifierRule { + + RequestModifierConfig getRequestModifierConfig (); + + Class RMR = RequestModifierRule.class; + String ICON_JS_TEMPLATE = stream2string(getPackagePath(RMR)+"/"+ RMR.getSimpleName()+"_icon.js.hbs"); + +} diff --git a/bubble-server/src/main/java/bubble/rule/TrafficRecord.java b/bubble-server/src/main/java/bubble/rule/TrafficRecord.java index 66402faf..f6af745b 100644 --- a/bubble-server/src/main/java/bubble/rule/TrafficRecord.java +++ b/bubble-server/src/main/java/bubble/rule/TrafficRecord.java @@ -36,7 +36,7 @@ public class TrafficRecord { setAccountUuid(account == null ? null : account.getUuid()); setDeviceName(device == null ? null : device.getName()); setDeviceUuid(device == null ? null : device.getUuid()); - setIp(filter.getRemoteAddr()); + setIp(filter.getServerAddr()); setFqdn(filter.getFqdn()); setUri(filter.getUri()); setUserAgent(filter.getUserAgent()); diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java index ce1d34b7..4553d2ad 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockConfig.java @@ -4,6 +4,7 @@ */ package bubble.rule.bblock; +import bubble.rule.RequestModifierConfig; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -14,13 +15,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import static org.cobbzilla.util.daemon.ZillaRuntime.bool; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @NoArgsConstructor @Slf4j -public class BubbleBlockConfig { +public class BubbleBlockConfig extends RequestModifierConfig { @Getter @Setter private Boolean inPageBlocks; - public boolean inPageBlocks() { return inPageBlocks != null && inPageBlocks; } + public boolean inPageBlocks() { return bool(inPageBlocks); } @Getter @Setter private BubbleUserAgentBlock[] userAgentBlocks; public boolean hasUserAgentBlocks () { return !empty(userAgentBlocks); } diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index 7e4ac12d..8e4474e2 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -8,28 +8,26 @@ import bubble.abp.*; import bubble.model.account.Account; import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; import bubble.model.device.Device; import bubble.resources.stream.FilterHttpRequest; import bubble.resources.stream.FilterMatchersRequest; -import bubble.rule.AppRuleDriver; import bubble.rule.FilterMatchDecision; +import bubble.rule.RequestModifierConfig; +import bubble.rule.RequestModifierRule; import bubble.rule.analytics.TrafficAnalyticsRuleDriver; import bubble.service.stream.AppRuleHarness; import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.input.ReaderInputStream; -import org.cobbzilla.util.handlebars.HandlebarsUtil; +import org.apache.commons.collections4.map.SingletonMap; import org.cobbzilla.util.http.URIUtil; -import org.cobbzilla.util.io.regex.RegexFilterReader; -import org.cobbzilla.util.io.regex.RegexReplacementFilter; import org.cobbzilla.util.string.StringUtil; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URI; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -39,15 +37,14 @@ import static bubble.service.stream.HttpStreamDebug.getLogFqdn; import static java.util.concurrent.TimeUnit.DAYS; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; -import static org.cobbzilla.util.http.HttpContentTypes.isHtml; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; -import static org.cobbzilla.util.string.StringUtil.UTF8cs; +import static org.cobbzilla.util.string.StringUtil.EMPTY; import static org.cobbzilla.util.string.StringUtil.getPackagePath; @Slf4j -public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { +public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver implements RequestModifierRule { private final AtomicReference blockList = new AtomicReference<>(new BlockList()); private BlockList getBlockList() { return blockList.get(); } @@ -60,18 +57,38 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { private final static Map blockListCache = new ConcurrentHashMap<>(); + public boolean showStats() { return deviceService.doShowBlockStats(account.getUuid()); } + @Override public Class getConfigClass() { return (Class) BubbleBlockConfig.class; } + @Override public RequestModifierConfig getRequestModifierConfig() { return getRuleConfig(); } + + @Override public boolean couldModify(FilterHttpRequest request) { + final BubbleBlockConfig config = getRuleConfig(); + return request.isHtml() && request.isBrowser() && (config.inPageBlocks() || showStats()); + } + @Override public void init(JsonNode config, JsonNode userConfig, + BubbleApp app, AppRule rule, AppMatcher matcher, Account account, Device device) { - super.init(config, userConfig, rule, matcher, account, device); + initQuick(config, userConfig, app, rule, matcher, account, device); refreshBlockLists(); } + @Override public void initQuick(JsonNode config, + JsonNode userConfig, + BubbleApp app, + AppRule rule, + AppMatcher matcher, + Account account, + Device device) { + super.init(config, userConfig, app, rule, matcher, account, device); + } + @Override public JsonNode upgradeRuleConfig(JsonNode sageRuleConfig, JsonNode localRuleConfig) { final BubbleBlockConfig sageConfig = json(sageRuleConfig, getConfigClass()); @@ -170,6 +187,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); final BlockDecision decision = getPreprocessDecision(filter.getFqdn(), filter.getUri(), filter.getUserAgent(), filter.getReferer()); final BlockDecisionType decisionType = decision.getDecisionType(); + final FilterMatchDecision subDecision; switch (decisionType) { case block: if (log.isInfoEnabled()) log.info(prefix+"decision is BLOCK"); @@ -178,19 +196,15 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { return FilterMatchDecision.abort_not_found; // block this request case allow: default: - if (filter.hasReferer()) { - final FilterMatchDecision refererDecision = checkRefererDecision(filter, account, device, app, site, prefix); - if (refererDecision != null) return refererDecision; - } + subDecision = checkRefererAndShowStats(decisionType, filter, account, device, extraLog, app, site, prefix, bubbleBlockConfig); + if (subDecision != null) return subDecision; if (log.isInfoEnabled()) log.info(prefix+"decision is ALLOW"); else if (extraLog) log.error(prefix+"decision is ALLOW"); return FilterMatchDecision.no_match; case filter: - if (filter.hasReferer()) { - final FilterMatchDecision refererDecision = checkRefererDecision(filter, account, device, app, site, prefix); - if (refererDecision != null) return refererDecision; - } + subDecision = checkRefererAndShowStats(decisionType, filter, account, device, extraLog, app, site, prefix, bubbleBlockConfig); + if (subDecision != null) return subDecision; final List specs = decision.getSpecs(); if (empty(specs)) { if (log.isWarnEnabled()) log.warn(prefix+"decision was 'filter' but no specs were found, returning no_match"); @@ -216,6 +230,23 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { } } + public FilterMatchDecision checkRefererAndShowStats(BlockDecisionType decisionType, FilterMatchersRequest filter, Account account, Device device, boolean extraLog, String app, String site, String prefix, BubbleBlockConfig bubbleBlockConfig) { + if (filter.hasReferer()) { + final FilterMatchDecision refererDecision = checkRefererDecision(filter, account, device, app, site, prefix); + if (refererDecision != null && refererDecision.isAbort()) { + if (log.isInfoEnabled()) log.info(prefix+"decision was "+decisionType+" but refererDecision was "+refererDecision+", returning "+refererDecision); + else if (extraLog) log.error(prefix+"decision was "+decisionType+" but refererDecision was "+refererDecision+", returning "+refererDecision); + return refererDecision; + } + } + if (showStats()) { + if (log.isInfoEnabled()) log.info(prefix+"decision was "+decisionType+" but showStats=true, returning match"); + else if (extraLog) log.error(prefix+"decision was "+decisionType+" but showStats=true, returning match"); + return FilterMatchDecision.match; + } + return null; + } + public FilterMatchDecision checkRefererDecision(FilterMatchersRequest filter, Account account, Device device, String app, String site, String prefix) { prefix = prefix+" (checkRefererDecision): "; final URI refererURI = URIUtil.toUriOrNull(filter.getReferer()); @@ -268,10 +299,14 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { return false; } + public static final String FILTER_CTX_DECISION = "decision"; + public static final String BLOCK_STATS_JS = "BLOCK_STATS_JS"; + @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { final FilterMatchersRequest request = filterRequest.getMatchersResponse().getRequest(); final String prefix = "doFilterResponse("+filterRequest.getId()+"): "; + final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); // todo: add support for stream blockers: we may allow the request but wrap the returned InputStream // if the wrapper detects it should be blocked, then the connection cut short @@ -281,6 +316,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { // Now that we know the content type, re-check the BlockList final String contentType = filterRequest.getContentType(); final BlockDecision decision = getBlockList().getDecision(request.getFqdn(), request.getUri(), contentType, request.getReferer(), true); + final Map filterCtx = new SingletonMap<>(FILTER_CTX_DECISION, decision); if (log.isDebugEnabled()) log.debug(prefix+"preprocess decision was "+decision+", but now we know contentType="+contentType); switch (decision.getDecisionType()) { case block: @@ -306,39 +342,51 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver { return EMPTY_STREAM; } - if (!isHtml(contentType)) { + if (!filterRequest.isHtml()) { log.warn(prefix+"cannot request non-html response ("+request.getUrl()+"), returning as-is: "+contentType); if (log.isInfoEnabled()) log.info(prefix+"SEND: unfiltered response (non-html content-type) for "+request.getUrl()); return in; } - final String replacement = ""; - final RegexReplacementFilter filter = new RegexReplacementFilter("", replacement); - final RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in, UTF8cs), filter).setMaxMatches(1); - if (log.isDebugEnabled()) { - log.debug(prefix+"filtering response for "+request.getUrl()+" - replacement.length = "+replacement.length()); - } else if (log.isInfoEnabled()) { - log.info(prefix+"SEND: filtering response for "+request.getUrl()); + final boolean showStats = showStats(); + if (!bubbleBlockConfig.inPageBlocks() && !showStats) { + if (log.isInfoEnabled()) log.info(prefix + "SEND: both inPageBlocks and showStats are false, returning as-is"); + return in; + } + if (bubbleBlockConfig.inPageBlocks() && showStats) { + return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, getBubbleJsStatsTemplate(), BLOCK_STATS_JS, showStats); + } + if (bubbleBlockConfig.inPageBlocks()) { + return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, EMPTY, BLOCK_STATS_JS, showStats); } - return new ReaderInputStream(reader, UTF8cs); + log.warn(prefix+"inserting JS for stats..."); + return filterInsertJs(in, filterRequest, filterCtx, getBubbleJsStatsTemplate(), null, null, showStats); + } + + protected String getBubbleJsStatsTemplate () { + return loadTemplate(BUBBLE_JS_STATS_TEMPLATE, BUBBLE_STATS_TEMPLATE_NAME); } public static final Class BB = BubbleBlockRuleDriver.class; public static final String BUBBLE_JS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+".js.hbs"); + + public static final String BUBBLE_STATS_TEMPLATE_NAME = BB.getSimpleName() + "_stats.js.hbs"; + public static final String BUBBLE_JS_STATS_TEMPLATE = stream2string(getPackagePath(BB) + "/" + BUBBLE_STATS_TEMPLATE_NAME); + private static final String CTX_BUBBLE_SELECTORS = "BUBBLE_SELECTORS_JSON"; private static final String CTX_BUBBLE_BLACKLIST = "BUBBLE_BLACKLIST_JSON"; private static final String CTX_BUBBLE_WHITELIST = "BUBBLE_WHITELIST_JSON"; - private String getBubbleJs(String requestId, BlockDecision decision) { - final Map ctx = new HashMap<>(); - ctx.put(CTX_JS_PREFIX, AppRuleDriver.getJsPrefix(requestId)); - ctx.put(CTX_BUBBLE_REQUEST_ID, requestId); - ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase()); - ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId)); - ctx.put(CTX_BUBBLE_SELECTORS, json(decision.getSelectors(), COMPACT_MAPPER)); - ctx.put(CTX_BUBBLE_WHITELIST, json(getBlockList().getWhitelistDomains(), COMPACT_MAPPER)); - ctx.put(CTX_BUBBLE_BLACKLIST, json(getBlockList().getBlacklistDomains(), COMPACT_MAPPER)); - return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx); + @Override protected Map getBubbleJsContext(String requestId, Map filterCtx) { + final Map ctx = super.getBubbleJsContext(requestId, filterCtx); + final BubbleBlockConfig bubbleBlockConfig = getRuleConfig(); + if (bubbleBlockConfig.inPageBlocks()) { + final BlockDecision decision = (BlockDecision) filterCtx.get(FILTER_CTX_DECISION); + ctx.put(CTX_BUBBLE_SELECTORS, json(decision.getSelectors(), COMPACT_MAPPER)); + ctx.put(CTX_BUBBLE_WHITELIST, json(getBlockList().getWhitelistDomains(), COMPACT_MAPPER)); + ctx.put(CTX_BUBBLE_BLACKLIST, json(getBlockList().getBlacklistDomains(), COMPACT_MAPPER)); + } + return ctx; } } diff --git a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerConfig.java b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerConfig.java index a2212604..b0017db6 100644 --- a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerConfig.java +++ b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerConfig.java @@ -4,26 +4,6 @@ */ package bubble.rule.social.block; -import bubble.rule.BubbleRegexReplacement; -import lombok.Getter; -import lombok.Setter; +import bubble.rule.RequestModifierConfig; -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; - -public class JsUserBlockerConfig { - - @Getter @Setter private String siteJsTemplate; - - @Getter @Setter private String insertionRegex; - public boolean hasInsertionRegex () { return !empty(insertionRegex); } - - @Getter @Setter private String scriptOpen; - public boolean hasScriptOpen () { return !empty(scriptOpen); } - - @Getter @Setter private String scriptClose; - public boolean hasScriptClose () { return !empty(scriptClose); } - - @Getter @Setter private BubbleRegexReplacement[] additionalRegexReplacements; - public boolean hasAdditionalRegexReplacements () { return !empty(additionalRegexReplacements); } - -} +public class JsUserBlockerConfig extends RequestModifierConfig {} diff --git a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerRuleDriver.java b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerRuleDriver.java index e2bab483..78167247 100644 --- a/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/social/block/JsUserBlockerRuleDriver.java @@ -4,120 +4,37 @@ */ package bubble.rule.social.block; -import bubble.model.app.AppMatcher; import bubble.resources.stream.FilterHttpRequest; import bubble.rule.AbstractAppRuleDriver; -import bubble.rule.AppRuleDriver; -import bubble.rule.BubbleRegexReplacement; +import bubble.rule.RequestModifierConfig; +import bubble.rule.RequestModifierRule; import lombok.Getter; -import org.apache.commons.io.input.ReaderInputStream; -import org.cobbzilla.util.collection.ExpirationMap; -import org.cobbzilla.util.handlebars.HandlebarsUtil; -import org.cobbzilla.util.io.FileUtil; -import org.cobbzilla.util.io.regex.RegexFilterReader; -import org.cobbzilla.util.io.regex.RegexReplacementFilter; +import lombok.extern.slf4j.Slf4j; -import java.io.File; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Map; -import static bubble.ApiConstants.HOME_DIR; -import static org.cobbzilla.util.http.HttpContentTypes.isHtml; import static org.cobbzilla.util.io.StreamUtil.stream2string; -import static org.cobbzilla.util.io.regex.RegexReplacementFilter.DEFAULT_PREFIX_REPLACEMENT_WITH_MATCH; -import static org.cobbzilla.util.json.JsonUtil.json; -import static org.cobbzilla.util.string.StringUtil.UTF8cs; import static org.cobbzilla.util.string.StringUtil.getPackagePath; -public class JsUserBlockerRuleDriver extends AbstractAppRuleDriver { +@Slf4j +public class JsUserBlockerRuleDriver extends AbstractAppRuleDriver implements RequestModifierRule { public static final Class JSB = JsUserBlockerRuleDriver.class; public static final String BUBBLE_JS_TEMPLATE = stream2string(getPackagePath(JSB)+"/"+ JSB.getSimpleName()+".js.hbs"); public static final String CTX_APPLY_BLOCKS_JS = "APPLY_BLOCKS_JS"; - public static final String DEFAULT_INSERTION_REGEX = "<\\s*head[^>]*>"; - public static final String DEFAULT_SCRIPT_OPEN = ""; - @Override public boolean couldModify(FilterHttpRequest request) { return true; } - @Getter(lazy=true) private final JsUserBlockerConfig userBlockerConfig = json(config, JsUserBlockerConfig.class); - - @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { - if (!isHtml(filterRequest.getContentType())) return in; - final String replacement = DEFAULT_PREFIX_REPLACEMENT_WITH_MATCH - + getScriptOpen(filterRequest) - + getBubbleJs(filterRequest.getId()) - + getScriptClose(); - - final RegexReplacementFilter filter = new RegexReplacementFilter(getInsertionRegex(), replacement); - RegexFilterReader reader = new RegexFilterReader(new InputStreamReader(in), filter).setMaxMatches(1); - if (getUserBlockerConfig().hasAdditionalRegexReplacements()) { - for (BubbleRegexReplacement re : getUserBlockerConfig().getAdditionalRegexReplacements()) { - final RegexReplacementFilter f = new RegexReplacementFilter(re.getInsertionRegex(), re.getReplacement()); - reader = new RegexFilterReader(reader, f); - } - } - - return new ReaderInputStream(reader, UTF8cs); - } - - @Getter(lazy=true) private final String insertionRegex = getUserBlockerConfig().hasInsertionRegex() - ? getUserBlockerConfig().getInsertionRegex() - : DEFAULT_INSERTION_REGEX; - - public String getScriptOpen(FilterHttpRequest filterRequest) { - if (filterRequest.hasScriptNonce()) { - // log.info("getScriptOpen: using nonce="+filterRequest.getScriptNonce()); - return getUserBlockerConfig().hasScriptOpen() - ? getUserBlockerConfig().getScriptOpen().replace(NONCE_VAR, filterRequest.getScriptNonce()) - : DEFAULT_SCRIPT_NONCE_OPEN.replace(NONCE_VAR, filterRequest.getScriptNonce()); - } else { - // log.info("getScriptOpen: no nonce"); - return getUserBlockerConfig().hasScriptOpen() - ? getUserBlockerConfig().getScriptOpen() - : DEFAULT_SCRIPT_OPEN; - } - } + @Override public Class getConfigClass() { return (Class) JsUserBlockerConfig.class; } - @Getter(lazy=true) private final String scriptClose = getUserBlockerConfig().hasScriptClose() - ? getUserBlockerConfig().getScriptClose() - : DEFAULT_SCRIPT_CLOSE; + @Override public RequestModifierConfig getRequestModifierConfig() { return getRuleConfig(); } - @Getter(lazy=true) private final String _siteJsTemplate = stream2string(getUserBlockerConfig().getSiteJsTemplate()); + @Getter(lazy=true) private final String defaultSiteJsTemplate = stream2string(getRequestModifierConfig().getSiteJsTemplate()); - public String getSiteJsTemplate () { - if (configuration.getEnvironment().containsKey("DEBUG_JS_SITE_TEMPLATES")) { - final File jsTemplateFile = new File(HOME_DIR + "/siteJsTemplates/" + getUserBlockerConfig().getSiteJsTemplate()); - if (jsTemplateFile.exists()) { - return FileUtil.toStringOrDie(jsTemplateFile); - } - } - return get_siteJsTemplate(); - } - - private String getBubbleJs(String requestId) { - final Map ctx = new HashMap<>(); - ctx.put(CTX_JS_PREFIX, AppRuleDriver.getJsPrefix(requestId)); - ctx.put(CTX_BUBBLE_REQUEST_ID, requestId); - ctx.put(CTX_BUBBLE_HOME, configuration.getPublicUriBase()); - ctx.put(CTX_SITE, getSiteName(matcher)); - ctx.put(CTX_BUBBLE_DATA_ID, getDataId(requestId)); - - final String siteJs = HandlebarsUtil.apply(getHandlebars(), getSiteJsTemplate(), ctx); - ctx.put(CTX_APPLY_BLOCKS_JS, siteJs); - - return HandlebarsUtil.apply(getHandlebars(), BUBBLE_JS_TEMPLATE, ctx); - } - - private ExpirationMap siteNameCache = new ExpirationMap<>(); - private String getSiteName(AppMatcher matcher) { - return siteNameCache.computeIfAbsent(matcher.getSite(), k -> appSiteDAO.findByAccountAndId(matcher.getAccount(), matcher.getSite()).getName()); + @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { + if (!filterRequest.isHtml()) return in; + log.warn("doFilterResponse("+filterRequest.getId()+"): inserting JS"); + return filterInsertJs(in, filterRequest, null, BUBBLE_JS_TEMPLATE, getDefaultSiteJsTemplate(), CTX_APPLY_BLOCKS_JS, true); } - } diff --git a/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java b/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java index 9ab2fec4..dcf3890c 100644 --- a/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/social/block/UserBlockerRuleDriver.java @@ -23,7 +23,6 @@ import java.util.Map; import java.util.Set; import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.http.HttpContentTypes.isHtml; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.string.StringUtil.UTF8cs; @@ -62,7 +61,7 @@ public class UserBlockerRuleDriver extends AbstractAppRuleDriver { protected UserBlockerConfig configObject() { return json(getFullConfig(), UserBlockerConfig.class); } @Override public InputStream doFilterResponse(FilterHttpRequest filterRequest, InputStream in) { - if (!isHtml(filterRequest.getContentType())) return in; + if (!filterRequest.isHtml()) return in; final String requestId = filterRequest.getId(); final UserBlockerStreamFilter filter = new UserBlockerStreamFilter(requestId, matcher, rule, configuration.getHttp().getBaseUri()); diff --git a/bubble-server/src/main/java/bubble/server/BubbleServer.java b/bubble-server/src/main/java/bubble/server/BubbleServer.java index 3971b5db..ed399f0b 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleServer.java +++ b/bubble-server/src/main/java/bubble/server/BubbleServer.java @@ -37,6 +37,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.network.NetworkUtil.IPv4_LOCALHOST; +import static org.cobbzilla.util.system.OutOfMemoryErrorUncaughtExceptionHandler.EXIT_ON_OOME; @NoArgsConstructor @Slf4j public class BubbleServer extends RestServerBase { @@ -78,6 +79,7 @@ public class BubbleServer extends RestServerBase { public static void main(String[] args) throws Exception { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); + Thread.setDefaultUncaughtExceptionHandler(EXIT_ON_OOME); final Map env = loadEnvironment(args); final ConfigurationSource configSource = getConfigurationSource(); @@ -142,4 +144,5 @@ public class BubbleServer extends RestServerBase { public static ConfigurationSource getConfigurationSource() { return getStreamConfigurationSource(BubbleServer.class, API_CONFIG_YML); } + } diff --git a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java index 1af25f29..098213ac 100644 --- a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java @@ -15,6 +15,7 @@ import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.DeviceIdService; import bubble.service.cloud.NetworkMonitorService; +import bubble.service.stream.AppDataCleaner; import bubble.service.stream.AppPrimerService; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.wizard.server.RestServer; @@ -106,11 +107,13 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase childRecords = new ArrayList<>(5); + + @JsonIgnore @Getter @Setter private String device; + @JsonIgnore @Getter @Setter private String fqdn; + @JsonIgnore @Getter @Setter private String userAgent; + + public BlockStatRecord(FilterMatchersRequest filter, FilterMatchDecision decision) { + this.requestId = filter.getRequestId(); + this.device = filter.getDevice(); + this.referer = filter.getReferer(); + this.fqdn = filter.getFqdn(); + this.url = filter.getUrl(); + this.userAgent = filter.getUserAgent(); + this.decision = decision; + } + + public void addChild(BlockStatRecord rec) { + synchronized (childRecords) { + childRecords.add(rec); + } + } + + public BlockStatsSummary summarize() { + return summarize(new BlockStatsSummary()); + } + + private BlockStatsSummary summarize(BlockStatsSummary summary) { + if (decision.isAbort()) { + summary.addBlock(this); + } + for (BlockStatRecord child : childRecords) { + child.summarize(summary); + } + return summary; + } + + public BlockStatRecord init() { + setFqdn(getHost(SCHEME_HTTPS+stripScheme(url))); + for (BlockStatRecord child : childRecords) child.init(); + return this; + } +} diff --git a/bubble-server/src/main/java/bubble/service/block/BlockStatsService.java b/bubble-server/src/main/java/bubble/service/block/BlockStatsService.java new file mode 100644 index 00000000..64982141 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/block/BlockStatsService.java @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.service.block; + +import bubble.resources.stream.FilterMatchersRequest; +import bubble.rule.FilterMatchDecision; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; +import org.cobbzilla.util.collection.ExpirationEvictionPolicy; +import org.cobbzilla.util.collection.ExpirationMap; +import org.springframework.stereotype.Service; + +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.cobbzilla.util.http.HttpSchemes.stripScheme; +import static org.cobbzilla.util.json.JsonUtil.json; + +@Service @Slf4j +public class BlockStatsService { + + private final ExpirationMap records + = new ExpirationMap<>(200, MINUTES.toMillis(10), ExpirationEvictionPolicy.atime); + + private final String[] EXCLUDE_FQDNS = { + "detectportal.firefox.com", "push.services.mozilla.com", + "spocs.getpocket.com", "img-getpocket.cdn.mozilla.net", + "incoming.telemetry.mozilla.org" + }; + + public void flush () { records.clear(); } + + public void record(FilterMatchersRequest filter, FilterMatchDecision decision) { + if (excludeFqdn(filter.getFqdn())) { + if (log.isDebugEnabled()) log.debug("record: excluding fqdn="+filter.getFqdn()); + return; + } + if (log.isDebugEnabled()) log.debug("record: >>>>> processing URL="+filter.getUrl()+" REFERER="+filter.getReferer()); + if (!filter.hasReferer()) { + // this must be a top-level request + final BlockStatRecord newRec = new BlockStatRecord(filter, decision); + records.put(getUrlCacheKey(filter), newRec); + records.put(getFqdnKey(filter.getFqdn(), filter.getUserAgent()), newRec); + records.put(filter.getRequestId(), newRec); + if (log.isDebugEnabled()) log.debug("record: added top-level record for device="+filter.getDevice()+"/userAgent="+filter.getUserAgent()+"/url="+filter.getUrl()); + + } else { + // find match based on device + user-agent + referer + final String cacheKey = getRefererCacheKey(filter); + BlockStatRecord rec = records.get(cacheKey); + if (rec == null) { + // try fqdn + rec = records.get(getFqdnKey(filter.getRefererFqdn(), filter.getUserAgent())); + if (rec == null) { + log.warn("record: parent not found for device=" + filter.getDevice() + "/userAgent=" + filter.getUserAgent() + "/referer=" + filter.getReferer()); + return; + } + } + final BlockStatRecord childRec = new BlockStatRecord(filter, decision); + rec.addChild(childRec); + records.put(getUrlCacheKey(filter), childRec); + records.put(getFqdnKey(filter.getFqdn(), filter.getUserAgent()), childRec); + if (log.isDebugEnabled()) log.debug("record: child("+getUrlCacheKey(filter)+", "+filter.getRequestId()+")= newRec="+json(childRec)+",\nparent="+json(rec)); + } + } + + private boolean excludeFqdn(String fqdn) { return ArrayUtils.contains(EXCLUDE_FQDNS, fqdn); } + + public String getFqdnKey(String fqdn, String userAgent) { return fqdn+"\t"+userAgent; } + + public String getRefererCacheKey(FilterMatchersRequest filter) { + return filter.getDevice()+"\t"+filter.getUserAgent()+"\t"+stripScheme(filter.getReferer()); + } + + public String getUrlCacheKey(FilterMatchersRequest filter) { + return filter.getDevice()+"\t"+filter.getUserAgent()+"\t"+stripScheme(filter.getUrl()); + } + + public BlockStatsSummary getSummary(String requestId) { + final BlockStatRecord stat = records.get(requestId); + if (stat == null) { + log.info("getSummary("+requestId+") no summary found"); + return null; + } + final BlockStatsSummary summary = stat.summarize(); + if (log.isDebugEnabled()) log.debug("getSummary("+requestId+") returning summary="+json(summary)+" for record="+json(stat)); + return summary; + } + +} diff --git a/bubble-server/src/main/java/bubble/service/block/BlockStatsSummary.java b/bubble-server/src/main/java/bubble/service/block/BlockStatsSummary.java new file mode 100644 index 00000000..db2512dd --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/block/BlockStatsSummary.java @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.service.block; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.json.JsonUtil.json; + +@Slf4j +public class BlockStatsSummary { + + private final Map blocks = new HashMap<>(); + @Getter private final long ctime = now(); + + public void addBlock(BlockStatRecord rec) { + final AtomicInteger ct = blocks.computeIfAbsent(rec.getFqdn(), k -> new AtomicInteger(0)); + ct.incrementAndGet(); + } + + public List getBlocks () { + final List fqdnBlockCounts = new ArrayList<>(); + for (Map.Entry entry : blocks.entrySet()) { + final int count = entry.getValue().get(); + if (count > 0) fqdnBlockCounts.add(new FqdnBlockCount(entry.getKey(), count)); + } + log.info("getBlocks returning counts="+json(fqdnBlockCounts)+" for blocks="+json(blocks)); + Collections.sort(fqdnBlockCounts); + return fqdnBlockCounts; + } + + public int getTotal () { + int total = 0; + for (Map.Entry entry : blocks.entrySet()) { + final String fqdn = entry.getKey(); + final int ct = entry.getValue().get(); + log.debug("getTotal: adding "+ct+" from fqdn="+fqdn); + total += ct; + } + return total; + } + + @Override public String toString () { return "BlockStatsSummary{total="+getTotal()+"}"; } + + @AllArgsConstructor @EqualsAndHashCode(of={"fqdn"}) + public static class FqdnBlockCount implements Comparable { + @Getter private final String fqdn; + @Getter private final int count; + @Override public int compareTo(FqdnBlockCount o) { return Integer.compare(o.count, count); } + } + +} diff --git a/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java b/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java index fc56d82a..8fc254b3 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java @@ -4,6 +4,7 @@ */ package bubble.service.cloud; +import bubble.model.account.Account; import bubble.model.device.Device; import bubble.model.device.DeviceStatus; @@ -16,9 +17,11 @@ public interface DeviceIdService { List findIpsByDevice(String deviceUuid); void initDeviceSecurityLevels(); - void setDeviceSecurityLevel(Device device); + void initBlockStats (Account account); + default boolean doShowBlockStats(String accountUuid) { return false; } + DeviceStatus getDeviceStatus(String deviceUuid); DeviceStatus getLiveDeviceStatus(String deviceUuid); diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java index f74c7084..d92ced05 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java @@ -7,6 +7,7 @@ package bubble.service.cloud; import bubble.dao.account.AccountDAO; import bubble.dao.app.AppSiteDAO; import bubble.dao.device.DeviceDAO; +import bubble.model.account.Account; import bubble.model.app.AppSite; import bubble.model.device.Device; import bubble.model.device.DeviceStatus; @@ -16,6 +17,8 @@ import org.cobbzilla.util.collection.ExpirationMap; import org.cobbzilla.util.collection.SingletonList; import org.cobbzilla.util.io.FileUtil; import org.cobbzilla.util.io.FilenamePrefixFilter; +import org.cobbzilla.util.network.NetworkUtil; +import org.cobbzilla.util.string.StringUtil; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -26,6 +29,7 @@ import java.net.InetAddress; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import static bubble.ApiConstants.HOME_DIR; import static bubble.model.device.DeviceStatus.NO_DEVICE_STATUS; @@ -47,6 +51,17 @@ public class StandardDeviceIdService implements DeviceIdService { // used in dnscrypt-proxy and mitmproxy to check device security level public static final String REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = "bubble_device_security_level_"; public static final String REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = "bubble_device_site_max_security_level_"; + public static final String REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS = "bubble_account_showBlockStats_"; + + // used in dnscrypt-proxy to determine how to respond to blocked requests + public static final String REDIS_KEY_DEVICE_REJECT_WITH = "bubble_device_reject_with_"; + + // used in mitmproxy to determine how to respond to blocked requests + public static final String REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = "bubble_device_showBlockStats_"; + + // used in mitmproxy to optimize passthru requests + // we flush keys with this prefix when changing showBlockStats flag + public static final String REDIS_KEY_CHUNK_FILTER_PASS = "__chunk_filter_pass__"; @Autowired private DeviceDAO deviceDAO; @Autowired private AccountDAO accountDAO; @@ -144,6 +159,45 @@ public class StandardDeviceIdService implements DeviceIdService { } } + public void initBlockStats (Account account) { + redis.set_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS+account.getUuid(), Boolean.toString(account.showBlockStats())); + redis.del_matching_withPrefix(REDIS_KEY_CHUNK_FILTER_PASS+"*"); + for (Device device : deviceDAO.findByAccount(account.getUuid())) { + if (account.showBlockStats()) { + showBlockStats(device); + } else { + hideBlockStats(device); + } + } + } + + public boolean doShowBlockStats(String accountUuid) { + return Boolean.parseBoolean(redis.get_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS + accountUuid)); + } + + public void showBlockStats (Device device) { + final Set configuredIps = NetworkUtil.configuredIps(); + final String privateIp = configuredIps.stream() + .filter(addr -> addr.startsWith("10.")) + .findFirst() + .orElse(null); + if (privateIp == null) { + log.error("showBlockStats: no system private IP found, configuredIps="+StringUtil.toString(configuredIps)); + return; + } + for (String ip : findIpsByDevice(device.getUuid())) { + redis.set_plaintext(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip, Boolean.toString(true)); + redis.set_plaintext(REDIS_KEY_DEVICE_REJECT_WITH + ip, privateIp); + } + } + + public void hideBlockStats (Device device) { + for (String ip : findIpsByDevice(device.getUuid())) { + redis.del_withPrefix(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip); + redis.del_withPrefix(REDIS_KEY_DEVICE_REJECT_WITH + ip); + } + } + private final ExpirationMap deviceStatusCache = new ExpirationMap<>(MINUTES.toMillis(2)); @Override public DeviceStatus getDeviceStatus(String deviceUuid) { diff --git a/bubble-server/src/main/java/bubble/service/stream/ActiveStreamState.java b/bubble-server/src/main/java/bubble/service/stream/ActiveStreamState.java index 39d655db..af9ca7aa 100644 --- a/bubble-server/src/main/java/bubble/service/stream/ActiveStreamState.java +++ b/bubble-server/src/main/java/bubble/service/stream/ActiveStreamState.java @@ -37,7 +37,7 @@ class ActiveStreamState { // do not wrap input with encoding stream until we have received at least this many bytes // this avoids errors when creating a GZIPInputStream when only one or a few bytes are available - public static final long MIN_BYTES_BEFORE_WRAP = 256; + public static final long MIN_BYTES_BEFORE_WRAP = Bytes.KB; private final FilterHttpRequest request; private final String requestId; diff --git a/bubble-server/src/main/java/bubble/service/stream/AppDataCleaner.java b/bubble-server/src/main/java/bubble/service/stream/AppDataCleaner.java new file mode 100644 index 00000000..f814b597 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/stream/AppDataCleaner.java @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.service.stream; + +import bubble.dao.app.AppDataDAO; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.daemon.SimpleDaemon; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import static java.util.concurrent.TimeUnit.HOURS; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; +import static org.cobbzilla.wizard.server.RestServerBase.reportError; + +@Service @Slf4j +public class AppDataCleaner extends SimpleDaemon { + + @Getter(lazy=true) private final long sleepTime = HOURS.toMillis(4); + + @Autowired private AppDataDAO dataDAO; + + @Override protected void process() { + try { + final int ct = dataDAO.bulkDeleteWhere("expiration < " + now()); + log.info("process: removed " + ct + " expired AppData records"); + } catch (Exception e) { + reportError("AppDataCleaner.process: "+shortError(e), e); + } + } + +} diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index e9df6450..204c7d18 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -80,7 +80,10 @@ public class StandardAppPrimerService implements AppPrimerService { } } - public void prime(Account account) { prime(account, (BubbleApp) null); } + public void prime(Account account) { + deviceIdService.initBlockStats(account); + prime(account, (BubbleApp) null); + } public void prime(BubbleApp app) { final Account account = accountDAO.findByUuid(app.getAccount()); @@ -137,7 +140,7 @@ public class StandardAppPrimerService implements AppPrimerService { final Set blockDomains = new HashSet<>(); final Set filterDomains = new HashSet<>(); for (AppMatcher matcher : matchers) { - final AppRuleDriver appRuleDriver = rule.initDriver(driver, matcher, account, device); + final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); final Set blocks = appRuleDriver.getPrimedBlockDomains(); if (empty(blocks)) { log.debug("_prime: no blockDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java index b36d963c..1260aa9f 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java @@ -5,10 +5,12 @@ package bubble.service.stream; import bubble.dao.app.AppRuleDAO; +import bubble.dao.app.BubbleAppDAO; import bubble.dao.app.RuleDriverDAO; import bubble.model.account.Account; import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; import bubble.model.app.RuleDriver; import bubble.model.device.Device; import bubble.resources.stream.FilterHttpRequest; @@ -75,6 +77,7 @@ public class StandardRuleEngineService implements RuleEngineService { public static final String HEADER_PASSTHRU = "X-Bubble-Passthru"; + @Autowired private BubbleAppDAO appDAO; @Autowired private AppRuleDAO ruleDAO; @Autowired private RuleDriverDAO driverDAO; @Autowired private BubbleConfiguration configuration; @@ -122,7 +125,7 @@ public class StandardRuleEngineService implements RuleEngineService { FilterHttpRequest filterRequest) throws IOException { // sanity check - if (empty(filterRequest.getMatchers())) return passthru(request.getEntityStream()); + if (empty(filterRequest.getMatchers())) return passthru(request); final List rules = initRules(filterRequest); final AppRuleHarness firstRule = rules.get(0); @@ -159,8 +162,8 @@ public class StandardRuleEngineService implements RuleEngineService { Integer chunkLength, boolean last) throws IOException { final String prefix = "applyRulesToChunkAndSendResponse("+filterRequest.getId()+"): "; - if (!filterRequest.hasMatchers()) { - if (log.isDebugEnabled()) log.debug(prefix+"adding no matchers, returning passthru"); + if (!filterRequest.hasRequestModifiers()) { + if (log.isDebugEnabled()) log.debug(prefix+"no request modifiers, returning passthru"); return passthru(request); } else { log.info(prefix+" applying matchers: "+filterRequest.getMatcherNames()); @@ -243,10 +246,15 @@ public class StandardRuleEngineService implements RuleEngineService { for (AppRuleHarness h : rules) { final RuleDriver ruleDriver = driverDAO.findByUuid(h.getRule().getDriver()); if (ruleDriver == null) { - log.warn("initRules: driver not found: "+h.getRule().getDriver()); + log.warn("initRuleHarnesses: driver not found: "+h.getRule().getDriver()); continue; } - final AppRuleDriver unwiredDriver = h.getRule().initDriver(ruleDriver, h.getMatcher(), account, device); + final BubbleApp app = appDAO.findByAccountAndId(account.getUuid(), h.getRule().getApp()); + if (app == null) { + log.warn("initRuleHarnesses: app not found: "+h.getRule().getApp()); + continue; + } + final AppRuleDriver unwiredDriver = h.getRule().initDriver(app, ruleDriver, h.getMatcher(), account, device); final AppRuleDriver driver = configuration.autowire(unwiredDriver); h.setRuleDriver(ruleDriver); h.setDriver(driver); diff --git a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java index 32692930..a33c5798 100644 --- a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java @@ -4,6 +4,7 @@ */ package bubble.service_dbfilter; +import bubble.model.account.Account; import bubble.model.device.Device; import bubble.model.device.DeviceStatus; import bubble.service.cloud.DeviceIdService; @@ -21,9 +22,10 @@ public class DbFilterDeviceIdService implements DeviceIdService { @Override public List findIpsByDevice(String deviceUuid) { return notSupported("findIpsByDevice"); } @Override public void initDeviceSecurityLevels() { notSupported("initDeviceSecurityLevels"); } - @Override public void setDeviceSecurityLevel(Device device) { notSupported("setDeviceSecurityLevel"); } + @Override public void initBlockStats(Account account) { notSupported("initBlockStats"); } + @Override public DeviceStatus getDeviceStatus(String deviceUuid) { return notSupported("getDeviceStats"); } @Override public DeviceStatus getLiveDeviceStatus(String deviceUuid) { return notSupported("getLiveDeviceStatus"); } diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index ec533667..c19b4ddd 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 0.15.7 +bubble.version=Adventure 0.16.0 diff --git a/bubble-server/src/main/resources/bubble-config.yml b/bubble-server/src/main/resources/bubble-config.yml index 5c35b0d7..63208d95 100644 --- a/bubble-server/src/main/resources/bubble-config.yml +++ b/bubble-server/src/main/resources/bubble-config.yml @@ -55,6 +55,7 @@ jersey: - org.cobbzilla.wizard.filters providerPackages: - org.cobbzilla.wizard.exceptionmappers + - bubble.exceptionmappers requestFilters: - bubble.auth.BubbleAuthFilter - bubble.filters.BubbleRateLimitFilter diff --git a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs new file mode 100644 index 00000000..c79ad23c --- /dev/null +++ b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs @@ -0,0 +1,74 @@ + +if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { + let {{PAGE_PREFIX}}_doc_ready = false; + + let {{PAGE_PREFIX}}_icon_status = []; + + {{PAGE_PREFIX}}_addBubbleApp = function (app) { + if (window.self === window.top) { + {{PAGE_PREFIX}}_icon_status.push(app); + } + } + + {{PAGE_PREFIX}}_getAppIconImgSrc = function (app) { + return '/__bubble/api/filter/assets/{{BUBBLE_REQUEST_ID}}/' + app.app + '/' + app.icon + '?raw=true'; + } + + {{PAGE_PREFIX}}_getAppIconImgId = function (app) { + return app.jsPrefix + '_app_icon_img'; + } + + {{PAGE_PREFIX}}_setAppIconImg = function (app) { + const imgId = {{PAGE_PREFIX}}_getAppIconImgId(app); + const img = document.getElementById(imgId); + if (img) { + img.src = {{PAGE_PREFIX}}_getAppIconImgSrc(app); + } else { + console.warn('setAppIconImg: img element not found: '+imgId) + } + } + + function {{PAGE_PREFIX}}_onReady(callback) { + const intervalId = window.setInterval(function() { + if (document.getElementsByTagName('body')[0] !== undefined) { + {{PAGE_PREFIX}}_doc_ready = true; + window.clearInterval(intervalId); + callback.call(this); + } + }, {{PAGE_ONREADY_INTERVAL}}); + } + + {{PAGE_PREFIX}}_onReady(function() { + const controlDivId = '{{PAGE_PREFIX}}_controlDiv'; + let bubbleControlDiv = document.getElementById(controlDivId); + if (bubbleControlDiv === null) { + bubbleControlDiv = document.createElement('div'); + bubbleControlDiv.id = controlDivId; + bubbleControlDiv.style.position = 'fixed'; + bubbleControlDiv.style.bottom = '0'; + bubbleControlDiv.style.right = '0'; + bubbleControlDiv.style.zIndex = '{{APP_CONTROLS_Z_INDEX}}'; + document.getElementsByTagName('body')[0].appendChild(bubbleControlDiv); + } + for (let i=0; i<{{PAGE_PREFIX}}_icon_status.length; i++) { + const iconSpecs = {{PAGE_PREFIX}}_icon_status[i]; + let br = document.createElement('br'); + let link = document.createElement('a'); + if (typeof iconSpecs.link === 'function') { + link.onclick = function (ev) { iconSpecs.link(ev); return false; } + } else { + link.href = '{{{BUBBLE_HOME}}}/app/' + iconSpecs.app + '/' + iconSpecs.link; + } + let img = document.createElement('img'); + img.id = {{PAGE_PREFIX}}_getAppIconImgId(iconSpecs); + img.src = {{PAGE_PREFIX}}_getAppIconImgSrc(iconSpecs); + img.width = 64; + link.appendChild(img); + bubbleControlDiv.appendChild(br); + bubbleControlDiv.appendChild(link); + if (typeof iconSpecs.onReady === 'function') { + iconSpecs.onReady(); + } + } + }); +} \ No newline at end of file diff --git a/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver.js.hbs b/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver.js.hbs index 49d7c8e1..61515be1 100644 --- a/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver.js.hbs @@ -326,3 +326,5 @@ function {{JS_PREFIX}}_process_filters() { {{JS_PREFIX}}_process_filters(); window.setInterval({{JS_PREFIX}}_process_filters, {{JS_PREFIX}}_idle_interval); }); + +{{{BLOCK_STATS_JS}}} \ No newline at end of file diff --git a/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver_stats.js.hbs b/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver_stats.js.hbs new file mode 100644 index 00000000..558832ee --- /dev/null +++ b/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver_stats.js.hbs @@ -0,0 +1,107 @@ +{{{ICON_JS}}} + +let {{JS_PREFIX}}_app_details = false; +let {{JS_PREFIX}}_last_stats = null; +let {{JS_PREFIX}}_app_stats_last_change = 0; +const {{JS_PREFIX}}_app_stats_timeout = 35000; + +function {{JS_PREFIX}}_toggle_app_details(ev) { + const detailsDivId = '{{JS_PREFIX}}_detailsDiv'; + let detailsDiv = document.getElementById(detailsDivId); + if ({{JS_PREFIX}}_app_details) { + {{JS_PREFIX}}_app_details = false; + if (detailsDiv != null) { + detailsDiv.style.display = 'none'; + while (detailsDiv.firstChild) { + detailsDiv.removeChild(detailsDiv.lastChild); + } + } + } else { + {{JS_PREFIX}}_app_details = true; + {{JS_PREFIX}}_app_refresh(function () { + if ({{JS_PREFIX}}_last_stats != null) { + if (detailsDiv === null) { + detailsDiv = document.createElement('div'); + detailsDiv.id = detailsDivId; + detailsDiv.style.backgroundColor = '#ffffff'; + detailsDiv.style.position = 'fixed'; + detailsDiv.style.bottom = '0'; + detailsDiv.style.right = '0'; + detailsDiv.style.zIndex = '{{expr APP_CONTROLS_Z_INDEX '+' 1}}'; + document.getElementsByTagName('body')[0].appendChild(detailsDiv); + detailsDiv.onclick = function (ev) { + {{JS_PREFIX}}_toggle_app_details(); + } + } + while (detailsDiv.firstChild) { + detailsDiv.removeChild(detailsDiv.lastChild); + } + detailsDiv.style.display = 'block'; + // add rows for blocked stuff... + for (let i = 0; i < {{JS_PREFIX}}_last_stats.blocks.length; i++) { + const entry = {{JS_PREFIX}}_last_stats.blocks[i]; + const entryDiv = document.createElement('div'); + const entryText = document.createTextNode(entry.fqdn + ': ' + entry.count); + entryDiv.appendChild(entryText); + detailsDiv.appendChild(entryDiv); + } + } + }) + } +} + +const {{JS_PREFIX}}_app_refresh = function (displayFunc) { + const requestOptions = { method: 'GET' }; + const block_stats_url = '/__bubble/api/filter/status/{{BUBBLE_REQUEST_ID}}'; + fetch(block_stats_url, requestOptions) + .then(resp => { + try { + return resp.json(); + } catch (error) { + console.log('cancelling window.interval, response not json: '+JSON.stringify(resp)); + window.clearInterval({{JS_PREFIX}}_app_refresh_interval); + } + }) + .then(data => { + console.log('stats = '+JSON.stringify(data)); + let icon = null; + if (typeof data.total !== 'undefined') { + if (JSON.stringify(data) !== JSON.stringify({{JS_PREFIX}}_last_stats)) { + {{JS_PREFIX}}_last_stats = data; + {{JS_PREFIX}}_app_stats_last_change = Date.now(); + if (data.total === 0) { + icon = 'icon-green'; + } else if (data.total < 5) { + icon = 'icon-yellow'; + } else { + icon = 'icon-red'; + console.log('cancelling window.interval, red status'); + window.clearInterval({{JS_PREFIX}}_app_refresh_interval); + } + {{JS_PREFIX}}_app.icon = icon; + {{PAGE_PREFIX}}_setAppIconImg({{JS_PREFIX}}_app); + + } else if (Date.now() - {{JS_PREFIX}}_app_stats_last_change > {{JS_PREFIX}}_app_stats_timeout) { + console.log('cancelling window.interval, stats unchanged for a while'); + window.clearInterval({{JS_PREFIX}}_app_refresh_interval); + } + if (typeof displayFunc === 'function') { + displayFunc(); + } + } + }).catch((error) => { + console.log('cancelling window.interval, due to error: '+error); + window.clearInterval({{JS_PREFIX}}_app_refresh_interval); + }); +} + +let {{JS_PREFIX}}_app_refresh_interval = null; +const {{JS_PREFIX}}_app = { + jsPrefix: '{{JS_PREFIX}}', + app: '{{BUBBLE_APP_NAME}}', + link: {{JS_PREFIX}}_toggle_app_details, + icon: 'icon-gray', + onReady: function () { {{JS_PREFIX}}_app_refresh_interval = window.setInterval({{JS_PREFIX}}_app_refresh, 5000); } +}; + +{{PAGE_PREFIX}}_addBubbleApp({{JS_PREFIX}}_app); diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs index aa88cbd7..f0adabaa 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs @@ -1,19 +1,10 @@ let {{JS_PREFIX}}_blocked_users = null; -let {{JS_PREFIX}}_doc_ready = false; const {{JS_PREFIX}}_request_id = '{{BUBBLE_REQUEST_ID}}'; + +let {{JS_PREFIX}}_doc_ready = false; const {{JS_PREFIX}}_interval = 50; const {{JS_PREFIX}}_idle_interval = 1000; -function {{JS_PREFIX}}_onReady(callback) { - const intervalId = window.setInterval(function() { - if (document.getElementsByTagName('body')[0] !== undefined) { - {{JS_PREFIX}}_doc_ready = true; - window.clearInterval(intervalId); - callback.call(this); - } - }, {{JS_PREFIX}}_interval); -} - function {{JS_PREFIX}}_fetch_blocks (do_apply) { const requestOptions = { method: 'GET' }; const blocked_users_url = '/__bubble/api/filter/data/{{BUBBLE_DATA_ID}}/read'; @@ -60,16 +51,11 @@ function {{JS_PREFIX}}_block_user (author) { {{{APPLY_BLOCKS_JS}}} -{{JS_PREFIX}}_onReady(function() { - const controlDivId = '{{JS_PREFIX}}_controlDiv'; - let bubbleControlDiv = document.getElementById(controlDivId); - if (bubbleControlDiv === null) { - bubbleControlDiv = document.createElement('div'); - bubbleControlDiv.id = controlDivId; - bubbleControlDiv.style.position = 'fixed'; - bubbleControlDiv.style.bottom = '0'; - bubbleControlDiv.style.right = '0'; - document.getElementsByTagName('body')[0].appendChild(bubbleControlDiv); - } - bubbleControlDiv.innerHTML = bubbleControlDiv.innerHTML + '
'; +{{{ICON_JS}}} + +{{PAGE_PREFIX}}_addBubbleApp({ + jsPrefix: '{{JS_PREFIX}}', + app: '{{BUBBLE_APP_NAME}}', + link: 'site/{{BUBBLE_SITE_NAME}}/view/blocked_users', + icon: 'icon' }); diff --git a/bubble-server/src/main/resources/db/migration/V2020081101__add_app_matcher_request_modifier.sql b/bubble-server/src/main/resources/db/migration/V2020081101__add_app_matcher_request_modifier.sql new file mode 100644 index 00000000..f211a246 --- /dev/null +++ b/bubble-server/src/main/resources/db/migration/V2020081101__add_app_matcher_request_modifier.sql @@ -0,0 +1,5 @@ +ALTER TABLE ONLY app_matcher ADD COLUMN request_modifier boolean; +UPDATE app_matcher SET request_modifier = false; +UPDATE app_matcher SET request_modifier = true WHERE fqdn != '*'; +UPDATE app_matcher SET request_modifier = true WHERE name = 'BubbleBlockMatcher'; +ALTER TABLE ONLY app_matcher ALTER COLUMN request_modifier SET NOT NULL; diff --git a/bubble-server/src/main/resources/db/migration/V2020081401__add_account_show_block_stats.sql b/bubble-server/src/main/resources/db/migration/V2020081401__add_account_show_block_stats.sql new file mode 100644 index 00000000..a08945ae --- /dev/null +++ b/bubble-server/src/main/resources/db/migration/V2020081401__add_account_show_block_stats.sql @@ -0,0 +1,3 @@ +ALTER TABLE ONLY account ADD COLUMN show_block_stats boolean; +UPDATE account SET show_block_stats = true; +ALTER TABLE ONLY account ALTER COLUMN show_block_stats SET NOT NULL; diff --git a/bubble-server/src/main/resources/logback.xml b/bubble-server/src/main/resources/logback.xml index 95187ddd..80568708 100644 --- a/bubble-server/src/main/resources/logback.xml +++ b/bubble-server/src/main/resources/logback.xml @@ -40,6 +40,7 @@ + @@ -53,8 +54,8 @@ + - diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-gray.svg b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-gray.svg new file mode 100644 index 00000000..62fc18f3 --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-gray.svg @@ -0,0 +1,336 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + Stop + 2005-10-16 + + + Andreas Nilsson + + + + + stop + halt + error + + + + + + Jakub Steiner + + + + + + + + + + + + + + + + + + diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-green.svg b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-green.svg new file mode 100644 index 00000000..7ae5ad0c --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-green.svg @@ -0,0 +1,336 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + Stop + 2005-10-16 + + + Andreas Nilsson + + + + + stop + halt + error + + + + + + Jakub Steiner + + + + + + + + + + + + + + + + + + diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-red.svg b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-red.svg new file mode 100644 index 00000000..04ce3a79 --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-red.svg @@ -0,0 +1,336 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + Stop + 2005-10-16 + + + Andreas Nilsson + + + + + stop + halt + error + + + + + + Jakub Steiner + + + + + + + + + + + + + + + + + + diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-yellow.svg b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-yellow.svg new file mode 100644 index 00000000..0fa2078e --- /dev/null +++ b/bubble-server/src/main/resources/models/apps/bubble_block/blockparty-icon-yellow.svg @@ -0,0 +1,336 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + Stop + 2005-10-16 + + + Andreas Nilsson + + + + + stop + halt + error + + + + + + Jakub Steiner + + + + + + + + + + + + + + + + + + diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json index 09cb6d6d..5bb46e60 100644 --- a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json +++ b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock.json @@ -126,6 +126,7 @@ "driver": "BubbleBlockRuleDriver", "priority": -1000, "config": { + "showStats": true, "blockLists": [ { "name": "EasyList", @@ -170,6 +171,10 @@ "messages": [ {"name": "name", "value": "Block Party!"}, {"name": "icon", "value": "classpath:models/apps/bubble_block/blockparty-icon.svg"}, + {"name": "icon-gray", "value": "classpath:models/apps/bubble_block/blockparty-icon-gray.svg"}, + {"name": "icon-red", "value": "classpath:models/apps/bubble_block/blockparty-icon-red.svg"}, + {"name": "icon-yellow", "value": "classpath:models/apps/bubble_block/blockparty-icon-yellow.svg"}, + {"name": "icon-green", "value": "classpath:models/apps/bubble_block/blockparty-icon-green.svg"}, {"name": "summary", "value": "Network Filter"}, {"name": "description", "value": "Block adware, malware, phishing/scam sites, and much more"}, {"name": "field.ctime", "value": "When"}, diff --git a/bubble-server/src/main/resources/models/apps/user_block/hn/bubbleApp_userBlock_hn_matchers.json b/bubble-server/src/main/resources/models/apps/user_block/hn/bubbleApp_userBlock_hn_matchers.json index 869f1b58..91ba2cf5 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/hn/bubbleApp_userBlock_hn_matchers.json +++ b/bubble-server/src/main/resources/models/apps/user_block/hn/bubbleApp_userBlock_hn_matchers.json @@ -6,6 +6,7 @@ "site": "HackerNews", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "news.ycombinator.com", "urlRegex": "/item\\?id=\\d+", "rule": "hn_user_blocker" @@ -14,6 +15,7 @@ "site": "HackerNews", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "news.ycombinator.com", "urlRegex": "/threads\\?id=\\w+", "rule": "hn_user_blocker" diff --git a/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json b/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json index 349eb17b..4af90acc 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json +++ b/bubble-server/src/main/resources/models/apps/user_block/mr/bubbleApp_userBlock_mr_matchers.json @@ -6,6 +6,7 @@ "site": "MarginalRevolution", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "marginalrevolution.com", "urlRegex": "(/marginalrevolution)?/20\\d{2}/\\d{2}/\\w+", "rule": "mr_user_blocker" diff --git a/bubble-server/src/main/resources/models/apps/user_block/reason/bubbleApp_userBlock_reason_matchers.json b/bubble-server/src/main/resources/models/apps/user_block/reason/bubbleApp_userBlock_reason_matchers.json index e68d5c74..830720ee 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/reason/bubbleApp_userBlock_reason_matchers.json +++ b/bubble-server/src/main/resources/models/apps/user_block/reason/bubbleApp_userBlock_reason_matchers.json @@ -6,6 +6,7 @@ "site": "Reason", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "reason.com", "urlRegex": "/20\\d{2}/\\d{2}/\\d{2}/[-\\w]+/", "rule": "reason_user_blocker" diff --git a/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter_matchers.json b/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter_matchers.json index 952cf936..9f24cf60 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter_matchers.json +++ b/bubble-server/src/main/resources/models/apps/user_block/twitter/bubbleApp_userBlock_twitter_matchers.json @@ -6,6 +6,7 @@ "site": "Twitter", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "twitter.com", "urlRegex": ".*", "rule": "twitter_user_blocker" @@ -14,6 +15,7 @@ "site": "Twitter", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "mobile.twitter.com", "urlRegex": ".*", "rule": "twitter_user_blocker" diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index a00efffe..b3a541d9 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:5a4e72d9671a38ff3ce9b5d6724c05222b343ddc408d690f1f511577d2673122 + checksum: sha256:9faacdb85b3df5d5eee7ce814a65c8d63e1975d19da06291348086aa6b1e0f01 - name: Unzip algo master.zip unarchive: diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 1ad96c23..8b6851fd 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -100,14 +100,24 @@ DEBUG_MATCHER = { 'rule': DEBUG_MATCHER_NAME }] } +BLOCK_MATCHER = { + 'decision': 'abort_not_found', + 'matchers': [{ + 'name': 'BLOCK_MATCHER', + 'contentTypeRegex': '.*', + "urlRegex": ".*", + 'rule': 'BLOCK_MATCHER' + }] +} + -def bubble_matchers(req_id, remote_addr, flow, host): +def bubble_matchers(req_id, client_addr, server_addr, flow, host): if debug_capture_fqdn and host and host in debug_capture_fqdn: bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host) return DEBUG_MATCHER headers = { - 'X-Forwarded-For': remote_addr, + 'X-Forwarded-For': client_addr, 'Accept' : 'application/json', 'Content-Type': 'application/json' } @@ -134,11 +144,15 @@ def bubble_matchers(req_id, remote_addr, flow, host): 'uri': flow.request.path, 'userAgent': user_agent, 'referer': referer, - 'remoteAddr': remote_addr + 'clientAddr': client_addr, + 'serverAddr': server_addr } response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data) if response.ok: return response.json() + elif response.status_code == 403: + bubble_log('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text)) + return BLOCK_MATCHER bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) except Exception as e: bubble_log('bubble_matchers API call failed: '+repr(e)) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index e2629f62..2bc15e6c 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -28,7 +28,7 @@ from mitmproxy.exceptions import TlsProtocolException from mitmproxy.net import tls as net_tls from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set -from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host +from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host from bubble_vpn4 import wireguard_network_ipv4 from bubble_vpn6 import wireguard_network_ipv6 from netaddr import IPAddress, IPNetwork @@ -41,6 +41,7 @@ REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = 'bubble_device_site_max_security_level_' # defined in StandardDeviceIdService +REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = 'bubble_device_showBlockStats_' FORCE_PASSTHRU = {'passthru': True} FORCE_BLOCK = {'block': True} @@ -77,6 +78,12 @@ def get_device_security_level(client_addr, fqdns): return {'level': level} +def show_block_stats(client_addr): + show = REDIS.get(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS+client_addr) + if show is None: + return False + return show.decode() == 'true' + def get_local_ips(): global local_ips if local_ips is None: @@ -86,8 +93,13 @@ def get_local_ips(): return local_ips +def is_bubble_request(ip, fqdns): + # return ip in get_local_ips() + return ip in get_local_ips() and (bubble_host in fqdns or bubble_host_alias in fqdns) + + def is_sage_request(ip, fqdns): - return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns + return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns def is_not_from_vpn(client_addr): @@ -134,11 +146,19 @@ class TlsFeedback(TlsLayer): super(TlsFeedback, self)._establish_tls_with_client() except TlsProtocolException as e: + if self.do_block: + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address) + raise e + tb = traceback.format_exc() if 'OpenSSL.SSL.ZeroReturnError' in tb: bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address) raise e + elif 'SysCallError' in tb: + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SysCallError for client '+client_address) + raise e + elif self.fqdns is not None and len(self.fqdns) > 0: for fqdn in self.fqdns: cache_key = conn_check_cache_prefix(client_address, fqdn) @@ -213,7 +233,7 @@ def next_layer(next_layer): client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile) client_addr = next_layer.client_conn.address[0] server_addr = next_layer.server_conn.address[0] - + bubble_log('next_layer: STARTING: client='+ client_addr+' server='+server_addr) if client_hello.sni: fqdn = client_hello.sni.decode() bubble_log('next_layer: using fqdn in SNI: '+ fqdn) @@ -225,19 +245,20 @@ def next_layer(next_layer): no_fqdns = fqdns is None or len(fqdns) == 0 security_level = get_device_security_level(client_addr, fqdns) next_layer.security_level = security_level - check = None - if server_addr in get_local_ips(): - bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) + next_layer.do_block = False + if is_bubble_request(server_addr, fqdns): + bubble_log('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns)) + check = FORCE_PASSTHRU + + elif is_sage_request(server_addr, fqdns): + bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU elif is_not_from_vpn(client_addr): bubble_log('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'conn_block_non_vpn', fqdns) next_layer.__class__ = TlsBlock - - elif is_sage_request(server_addr, fqdns): - bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) - check = FORCE_PASSTHRU + return elif security_level['level'] == SEC_OFF or security_level['level'] == SEC_BASIC: bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr) @@ -268,7 +289,11 @@ def next_layer(next_layer): elif 'block' in check and check['block']: bubble_log('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns) - next_layer.__class__ = TlsBlock + if show_block_stats(client_addr): + next_layer.do_block = True + next_layer.__class__ = TlsFeedback + else: + next_layer.__class__ = TlsBlock else: bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index f8c6944d..9d15c945 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -9,7 +9,7 @@ from mitmproxy.net.http import Headers from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \ - HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header + HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header BUFFER_SIZE = 4096 HEADER_CONTENT_TYPE = 'Content-Type' @@ -22,7 +22,7 @@ STANDARD_FILTER_HEADERS = {HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY} REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__' REDIS_FILTER_PASSTHRU_DURATION = 600 -def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type=None, content_length=None, csp=None): +def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, content_type=None, content_length=None, csp=None): if debug_capture_fqdn: host = None if flow.client_conn.tls_established: @@ -46,10 +46,10 @@ def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type= return chunk # should we just passthru? - redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + ':' + flow.request.url + redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + '~~~' + user_agent + ':' + flow.request.url do_pass = REDIS.get(redis_passthru_key) if do_pass: - # bubble_log('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk') + bubble_log('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk') REDIS.touch(redis_passthru_key) return chunk @@ -94,7 +94,7 @@ def filter_chunk(flow, chunk, req_id, last, content_encoding=None, content_type= return response.content -def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, csp): +def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, content_type, csp): """ chunks is a generator that can be used to iterate over all chunks. """ @@ -111,20 +111,20 @@ def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, c else: last = False if first: - yield filter_chunk(flow, chunk, req_id, last, content_encoding, content_type, content_length, csp) + yield filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding, content_type, content_length, csp) first = False else: - yield filter_chunk(flow, chunk, req_id, last) + yield filter_chunk(flow, chunk, req_id, user_agent, last) if not content_length: - yield filter_chunk(flow, None, req_id, True) # get the last bits of data + yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data except Exception as e: bubble_log('bubble_filter_chunks: exception='+repr(e)) traceback.print_exc() yield None -def bubble_modify(flow, req_id, content_encoding, content_type, csp): - return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type, csp) +def bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp): + return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, content_type, csp) def send_bubble_response(response): @@ -172,6 +172,10 @@ def responseheaders(flow): prefix = 'responseheaders(req_id='+str(req_id)+'): ' if req_id is not None and matchers is not None: bubble_log(prefix+' matchers: '+repr(matchers)) + if HEADER_USER_AGENT in flow.request.headers: + user_agent = flow.request.headers[HEADER_USER_AGENT] + else: + user_agent = '' if HEADER_CONTENT_TYPE in flow.response.headers: content_type = flow.response.headers[HEADER_CONTENT_TYPE] if matchers: @@ -201,7 +205,7 @@ def responseheaders(flow): content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) bubble_log(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) - flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type, csp) + flow.response.stream = bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp) if content_length_value: flow.response.headers['transfer-encoding'] = 'chunked' # find server_conn to set fake_chunks on diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index 78753473..a2749140 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -19,7 +19,8 @@ class Rerouter: bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path) return None - remote_addr = str(flow.client_conn.address[0]) + client_addr = str(flow.client_conn.address[0]) + server_addr = str(flow.server_conn.address[0]) try: host = host.decode() except (UnicodeDecodeError, AttributeError): @@ -35,10 +36,10 @@ class Rerouter: req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time()) bubble_log("get_matchers: requesting match decision for req_id="+req_id) - resp = bubble_matchers(req_id, remote_addr, flow, host) + resp = bubble_matchers(req_id, client_addr, server_addr, flow, host) if not resp: - bubble_log('get_matchers: no response for remote_addr/host: '+remote_addr+'/'+str(host)) + bubble_log('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host)) return None matchers = [] diff --git a/bubble-server/src/main/resources/packer/roles/nginx/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/nginx/tasks/main.yml index a6178458..0b30ae1e 100644 --- a/bubble-server/src/main/resources/packer/roles/nginx/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/nginx/tasks/main.yml @@ -44,10 +44,11 @@ - init_dhparams.sh - init_certbot.sh +# File in cron.weekly must NOT have a .sh extension, or crond will not run it - name: Install certbot_renew.sh weekly cron job copy: src: "certbot_renew.sh" - dest: /etc/cron.weekly/certbot_renew.sh + dest: /etc/cron.weekly/certbot_renew owner: root group: root mode: 0755 diff --git a/bubble-server/src/main/resources/spring.xml b/bubble-server/src/main/resources/spring.xml index 4c8a405a..9b7caf9f 100644 --- a/bubble-server/src/main/resources/spring.xml +++ b/bubble-server/src/main/resources/spring.xml @@ -28,5 +28,6 @@ + diff --git a/bubble-server/src/test/java/bubble/test/filter/BlockSummaryTest.java b/bubble-server/src/test/java/bubble/test/filter/BlockSummaryTest.java new file mode 100644 index 00000000..7b0725c0 --- /dev/null +++ b/bubble-server/src/test/java/bubble/test/filter/BlockSummaryTest.java @@ -0,0 +1,143 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.test.filter; + +import bubble.resources.stream.FilterMatchersRequest; +import bubble.rule.FilterMatchDecision; +import bubble.service.block.BlockStatRecord; +import bubble.service.block.BlockStatsService; +import bubble.service.block.BlockStatsSummary; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.http.URIUtil; +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.UUID.randomUUID; +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.junit.Assert.assertEquals; + +@Slf4j +public class BlockSummaryTest { + + public static final BlockStatsService STATS_SERVICE = new BlockStatsService(); + + public static final String BASE_FQDN = "example.com"; + public static final String BASE_URL = "https://" + BASE_FQDN + "/"; + public static final String INITIAL_URL = BASE_URL + "page.html"; + + public static final String SCRIPT_FQDN = "js.example.com"; + public static final String SCRIPT_BASE_URL = "https://" + SCRIPT_FQDN + "/"; + public static final String SCRIPT_URL = SCRIPT_BASE_URL + "script.js"; + public static final String SCRIPT2_URL = SCRIPT_BASE_URL + "script2.js"; + + public static final String TRACKER_FQDN = "tracker.example.com"; + public static final String TRACKER_BASE_URL = "https://" + TRACKER_FQDN + "/"; + public static final String TRACKER_URL = TRACKER_BASE_URL + "track.json"; + + public static final String[][] SIMPLE_TEST = { + // url referer decision + {INITIAL_URL, null, FilterMatchDecision.match.name()}, + {SCRIPT_URL, INITIAL_URL, FilterMatchDecision.abort_not_found.name()}, + {SCRIPT2_URL, INITIAL_URL, FilterMatchDecision.abort_not_found.name()} + }; + + @Test public void testSimpleSummary () throws Exception { + final BlockStatsSummary summary = runTest(SIMPLE_TEST); + assertEquals("expected total == 2", 2, summary.getTotal()); + assertEquals("expected 1 blocked fqdn", 1, summary.getBlocks().size()); + assertEquals("expected 2 blocks for fqdn", 2, findBlock(summary, SCRIPT_FQDN).getCount()); + } + + public static final String[][] BARE_REFERER_TEST = { + // url referer decision + {INITIAL_URL, null, FilterMatchDecision.match.name()}, + {SCRIPT_URL, BASE_URL, FilterMatchDecision.abort_not_found.name()}, + {SCRIPT2_URL, BASE_URL, FilterMatchDecision.abort_not_found.name()} + }; + + @Test public void testBareReferer () throws Exception { + final BlockStatsSummary summary = runTest(BARE_REFERER_TEST); + assertEquals("expected total == 2", 2, summary.getTotal()); + assertEquals("expected 1 blocked fqdn", 1, summary.getBlocks().size()); + assertEquals("expected 2 blocks for fqdn", 2, findBlock(summary, SCRIPT_FQDN).getCount()); + } + + public static final String[][] NESTED_BLOCK_TEST = { + // url referer decision + {INITIAL_URL, null, FilterMatchDecision.match.name()}, + {SCRIPT_URL, BASE_URL, FilterMatchDecision.no_match.name()}, + {TRACKER_URL, SCRIPT_URL, FilterMatchDecision.abort_not_found.name()} + }; + + @Test public void testNestedBlock () throws Exception { + final BlockStatsSummary summary = runTest(NESTED_BLOCK_TEST); + assertEquals("expected total == 1", 1, summary.getTotal()); + assertEquals("expected 1 blocked fqdn", 1, summary.getBlocks().size()); + assertEquals("expected 1 blocks for fqdn", 1, findBlock(summary, TRACKER_FQDN).getCount()); + } + + public static final String[][] NESTED_BLOCK_WITH_REPEAT_TEST = { + // url referer decision + {INITIAL_URL, null, FilterMatchDecision.match.name()}, + {SCRIPT_URL, BASE_URL, FilterMatchDecision.no_match.name()}, + {TRACKER_URL, SCRIPT_URL, FilterMatchDecision.abort_not_found.name()}, + {TRACKER_URL, SCRIPT_URL, FilterMatchDecision.abort_not_found.name()}, + {TRACKER_URL, SCRIPT_URL, FilterMatchDecision.abort_not_found.name()} + }; + + @Test public void testNestedBlockWithRepeat () throws Exception { + final BlockStatsSummary summary = runTest(NESTED_BLOCK_WITH_REPEAT_TEST); + assertEquals("expected total == 3", 3, summary.getTotal()); + assertEquals("expected 1 blocked fqdn", 1, summary.getBlocks().size()); + assertEquals("expected 3 blocks for fqdn", 3, findBlock(summary, TRACKER_FQDN).getCount()); + } + + @Test public void testComplexLiveExample () throws Exception { + final BlockStatRecord rec = json(stream2string("models/tests/filter/blockStatRecord.json"), BlockStatRecord.class).init(); + final BlockStatsSummary summary = rec.summarize(); + assertEquals("expected 11 total", 11, summary.getTotal()); + assertEquals("expected 1 googletagmanager block", 1, findBlock(summary, "www.googletagmanager.com").getCount()); + assertEquals("expected 4 googleads.g.doubleclick blocks", 4, findBlock(summary, "googleads.g.doubleclick.net").getCount()); + assertEquals("expected 2 static.doubleclick blocks", 2, findBlock(summary, "static.doubleclick.net").getCount()); + assertEquals("expected 2 youtube blocks", 2, findBlock(summary, "www.youtube.com").getCount()); + assertEquals("expected 1 d1z2jf7jlzjs58.cloudfront.net block", 1, findBlock(summary, "d1z2jf7jlzjs58.cloudfront.net").getCount()); + assertEquals("expected 1 pub.network block", 1, findBlock(summary, "a.pub.network").getCount()); + } + + public BlockStatsSummary runTest(String[][] test) { + String reqId = null; + for (String[] rec : test) { + final String id = record(STATS_SERVICE, rec[0], rec[1], FilterMatchDecision.valueOf(rec[2])); + if (reqId == null) reqId = id; + } + return STATS_SERVICE.getSummary(reqId); + } + + public BlockStatsSummary.FqdnBlockCount findBlock(BlockStatsSummary summary, String fqdn) { + final List blocks = summary.getBlocks().stream() + .filter(b -> b.getFqdn().equals(fqdn)) + .collect(Collectors.toList()); + assertEquals("fqdn not found in blocks: "+fqdn, 1, blocks.size()); + return blocks.get(0); + } + + public String record(BlockStatsService svc, String url, String referer, FilterMatchDecision decision) { + final String fqdn = URIUtil.getHost(url); + final String uri = URIUtil.getPath(url); + final String requestId = fqdn + "." + randomUUID().toString(); + svc.record(new FilterMatchersRequest() + .setRequestId(requestId) + .setFqdn(fqdn) + .setUri(uri) + .setReferer(referer) + .setUserAgent("ua"), + decision); + return requestId; + } + +} diff --git a/bubble-server/src/test/resources/models/apps/user_block/hn_test/bubbleApp_userBlock_hn_matchers.json b/bubble-server/src/test/resources/models/apps/user_block/hn_test/bubbleApp_userBlock_hn_matchers.json index 808bd836..8ecc0e0c 100644 --- a/bubble-server/src/test/resources/models/apps/user_block/hn_test/bubbleApp_userBlock_hn_matchers.json +++ b/bubble-server/src/test/resources/models/apps/user_block/hn_test/bubbleApp_userBlock_hn_matchers.json @@ -6,6 +6,7 @@ "site": "HackerNews", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "news.ycombinator.com", "urlRegex": "/item\\?id=\\d+", "rule": "hn_user_blocker" diff --git a/bubble-server/src/test/resources/models/apps/user_block/localhost/bubbleApp_userBlock_localhost_matchers.json b/bubble-server/src/test/resources/models/apps/user_block/localhost/bubbleApp_userBlock_localhost_matchers.json index dcd747ef..093e421b 100644 --- a/bubble-server/src/test/resources/models/apps/user_block/localhost/bubbleApp_userBlock_localhost_matchers.json +++ b/bubble-server/src/test/resources/models/apps/user_block/localhost/bubbleApp_userBlock_localhost_matchers.json @@ -7,6 +7,7 @@ "site": "HackerNews", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "127.0.0.1", "urlRegex": "web_mock/simple_comments.html", "rule": "local_user_blocker" @@ -15,6 +16,7 @@ "site": "HackerNews", "template": true, "requestCheck": true, + "requestModifier": true, "fqdn": "127.0.0.1", "urlRegex": "web_mock/news.ycombinator.com/item_id_\\d+\\.html", "rule": "hn_user_blocker" diff --git a/bubble-server/src/test/resources/models/include/fake_filter.json b/bubble-server/src/test/resources/models/include/fake_filter.json index 008e7880..9876f178 100644 --- a/bubble-server/src/test/resources/models/include/fake_filter.json +++ b/bubble-server/src/test/resources/models/include/fake_filter.json @@ -6,7 +6,7 @@ "fqdn": "_required", "uri": "/", "userAgent": "Test-User-Agent 1.0.0", - "remoteAddr": "127.0.0.1", + "clientAddr": "127.0.0.1", "jsCheck": "true", "requestSuffix": "<>" } @@ -21,7 +21,7 @@ "fqdn": "<>", "uri": "<>", "userAgent": "<>", - "remoteAddr": "<>" + "clientAddr": "<>" } }, "response": { diff --git a/bubble-server/src/test/resources/models/tests/filter/blockStatRecord.json b/bubble-server/src/test/resources/models/tests/filter/blockStatRecord.json new file mode 100644 index 00000000..eb654954 --- /dev/null +++ b/bubble-server/src/test/resources/models/tests/filter/blockStatRecord.json @@ -0,0 +1,475 @@ +{ + "requestId" : "reason.com.8646e759-a47c-4730-9ed6-e6dafe2515c4.1597679821.173839", + "referer" : "NONE", + "url" : "reason.com/latest/", + "decision" : "match", + "childRecords" : [ { + "requestId" : "d2eehagpk5cl65.cloudfront.net.cf3baab3-b5aa-4da2-a005-c17a97c8efa7.1597679823.885781", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/css/dist/block-library/style.min.css?ver=5.5", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.31769c6c-6a5e-4153-ae55-3251c61f6af3.1597679824.2184844", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/js/mediaelement/wp-mediaelement.min.css?ver=5.5", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "static-na.payments-amazon.com.c5555cef-5279-4551-9aad-240bda9b91dc.1597679824.4544876", + "referer" : "https://reason.com/latest/", + "url" : "static-na.payments-amazon.com/OffAmazonPayments/us/js/Widgets.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "cdnjs.cloudflare.com.048ba40f-2537-4fea-8e65-3835e85464da.1597679824.7191582", + "referer" : "https://reason.com/latest/", + "url" : "cdnjs.cloudflare.com/ajax/libs/cookieconsent2/3.1.0/cookieconsent.min.css", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "pro.fontawesome.com.a9c3331d-9b2d-4e38-b197-c18c579fd976.1597679824.9207523", + "referer" : "https://reason.com/latest/", + "url" : "pro.fontawesome.com/releases/v5.11.2/css/all.css", + "decision" : "match", + "childRecords" : [ { + "requestId" : "pro.fontawesome.com.2f42f6bc-c5ba-41d9-8e28-144e2a192ac2.1597679829.1615913", + "referer" : "https://pro.fontawesome.com/releases/v5.11.2/css/all.css", + "url" : "pro.fontawesome.com/releases/v5.11.2/webfonts/fa-regular-400.woff2", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "pro.fontawesome.com.dc4a6d29-cec5-439b-9f84-35e8097dfa54.1597679830.4538877", + "referer" : "https://pro.fontawesome.com/releases/v5.11.2/css/all.css", + "url" : "pro.fontawesome.com/releases/v5.11.2/webfonts/fa-brands-400.woff2", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "pro.fontawesome.com.623715e7-a65c-4594-99e2-3e15bfa942d1.1597679830.9182365", + "referer" : "https://pro.fontawesome.com/releases/v5.11.2/css/all.css", + "url" : "pro.fontawesome.com/releases/v5.11.2/webfonts/fa-solid-900.woff2", + "decision" : "match", + "childRecords" : [ ] + } ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.91bf780c-fee2-4a46-bdbd-a0b2ea1e3f70.1597679825.1448257", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/js/mediaelement/mediaelementplayer-legacy.min.css?ver=4.2.13-9993131", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.dc562fb9-f51f-4de1-a223-7dff12bec4cc.1597679825.493105", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/cache/min/1/wp-content/plugins/reason-forms/dist/assets/css/main_c3624a79-9de4930dc47c8f4f8e9e4028055d480a.css", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.293cd277-c502-494d-a570-4ed79044c693.1597679825.712245", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/cache/min/1/wp-content/themes/reason-com/dist/styles/main_e5f1bcd1-e3191af6573740dd1fdf95872242d36b.css", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.google.com.415d150f-21a0-4c46-9542-6878bd35c07f.1597679825.922725", + "referer" : "https://reason.com/latest/", + "url" : "www.google.com/recaptcha/api.js?onload=CaptchaCallback&render=explicit", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.bbe6be52-963e-46e9-83ad-861ca40a3698.1597679826.1044118", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/js/jquery/jquery.js?ver=1.12.4-wp", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "js.authorize.net.4312e7cf-ba51-43dc-8051-70ce8e31bc30.1597679826.277205", + "referer" : "https://reason.com/latest/", + "url" : "js.authorize.net/v1/Accept.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "cdnjs.cloudflare.com.75257aad-5093-4ddf-86ae-253cb9874d32.1597679826.4367824", + "referer" : "https://reason.com/latest/", + "url" : "cdnjs.cloudflare.com/ajax/libs/cookieconsent2/3.1.0/cookieconsent.min.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.d203ac89-c825-432e-85dd-eb2c12af8e63.1597679826.7678998", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/plugins/postup-for-reason/js/puprf-main.min.js?ver=200522-04435", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.56e4265a-f19b-49d0-b8c0-f1bd086a49f4.1597679826.9670165", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/plugins/reason-parsely/public/js/reason-parsely-public.js?ver=1.0.2", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.9fc693c9-9725-4436-a558-7d5b6d66e782.1597679827.2029767", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/plugins/reason-forms/dist/assets/js/main_c3624a79.js?ver=200724-230715", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.1637858c-e646-481f-8ffd-c10771fb42f6.1597679827.3765674", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/js/mediaelement/mediaelement-and-player.min.js?ver=4.2.13-9993131", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.a77de1e4-4d5a-46a9-a14f-a4070073fe14.1597679827.5914438", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/js/mediaelement/mediaelement-migrate.min.js?ver=5.5", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.01312be9-66e8-444a-8145-bc4845005856.1597679827.8578155", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/js/mediaelement/wp-mediaelement.min.js?ver=5.5", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "js.authorize.net.add32bb0-e5b6-4195-961b-fa0403202745.1597679828.2090032", + "referer" : "https://reason.com/latest/", + "url" : "js.authorize.net/v1/AcceptCore.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "static-na.payments-amazon.com.a45c1387-c730-449d-942b-01013bac3ae8.1597679828.4071748", + "referer" : "https://reason.com/latest/", + "url" : "static-na.payments-amazon.com/v2/login.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.27b92488-cc66-4d88-aa37-bbadd4f8221c.1597679828.8175836", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/themes/reason-com/dist/scripts/main_e5f1bcd1.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.42a9b56f-423b-44f7-bfc7-5824d0dfa4ab.1597679828.9988275", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-includes/js/wp-embed.min.js?ver=5.5", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.7401fb69-4379-4c01-8d06-48b42b2aa446.1597679829.3114657", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/cache/busting/facebook-tracking/fbpix-events-en_US-2.9.23.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.b6397573-66bf-493f-8328-4a5ad87f8b3f.1597679829.5069733", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/plugins/wp-rocket/assets/js/lazyload/16.1/lazyload.min.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.googletagmanager.com.2c9668e0-224b-454d-96f8-db86b40ad5c1.1597679830.173324", + "referer" : "https://reason.com/latest/", + "url" : "www.googletagmanager.com/gtm.js?id=GTM-5GHNJLW>m_auth=QxpctdM_XW9LaL_mGzTZ1Q>m_preview=env-1>m_cookies_win=x", + "decision" : "abort_not_found", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.80133077-30ea-4084-ba30-dd812dc6b2c1.1597679830.2052279", + "referer" : "https://reason.com/latest/", + "url" : "www.youtube.com/embed/lzka8m2PiW8?enablejsapi=1", + "decision" : "match", + "childRecords" : [ { + "requestId" : "www.youtube.com.5a939f2c-5a76-4813-9aa0-97a6b5ef882e.1597679837.8075325", + "referer" : "https://www.youtube.com/embed/lzka8m2PiW8?enablejsapi=1", + "url" : "www.youtube.com/yts/jsbin/fetch-polyfill-vfl6MZH8P/fetch-polyfill.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.3f3b2269-4c25-4c59-b1a5-d86fe5562290.1597679838.3835044", + "referer" : "https://www.youtube.com/embed/lzka8m2PiW8?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/player_ias.vflset/en_US/base.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.80888248-4632-4a4b-b50e-21253f1eced8.1597679838.575206", + "referer" : "https://www.youtube.com/embed/lzka8m2PiW8?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/www-embed-player.vflset/www-embed-player.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.a68083db-9d8d-4153-8b58-bebff0c1b036.1597679839.5476015", + "referer" : "https://www.youtube.com/embed/lzka8m2PiW8?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/player_ias.vflset/en_US/remote.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.30d0e594-5525-4a41-b903-e38eb9f2c671.1597679840.5820675", + "referer" : "https://www.youtube.com/embed/lzka8m2PiW8?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/player_ias.vflset/en_US/embed.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.5b826f52-c23a-43e3-ac0c-639f4e59b5d5.1597679850.1831374", + "referer" : "https://www.youtube.com/embed/lzka8m2PiW8?enablejsapi=1", + "url" : "www.youtube.com/youtubei/v1/log_event?alt=json&key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + "decision" : "abort_not_found", + "childRecords" : [ ] + } ] + }, { + "requestId" : "www.youtube.com.0bf4e9e9-d756-49d9-9cf6-386e068efea0.1597679830.3525195", + "referer" : "https://reason.com/latest/", + "url" : "www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "decision" : "match", + "childRecords" : [ { + "requestId" : "www.youtube.com.8a492452-b408-46d0-9391-463bf28cecb6.1597679835.165069", + "referer" : "https://www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/www-player-webp.css", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.88aee176-1c16-44ec-b29c-2f2d85bf9a0e.1597679835.3735561", + "referer" : "https://www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/www-embed-player.vflset/www-embed-player.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.eaf62562-2d9a-4e48-8d30-4c24a87edfe1.1597679835.58542", + "referer" : "https://www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/player_ias.vflset/en_US/base.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.d0112cfc-40ce-481f-8f80-482e99aca087.1597679838.176098", + "referer" : "https://www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "url" : "www.youtube.com/yts/jsbin/fetch-polyfill-vfl6MZH8P/fetch-polyfill.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.55803729-aa17-43d8-8e5e-7fd5bed566a0.1597679840.8800178", + "referer" : "https://www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/player_ias.vflset/en_US/embed.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.c3d7a820-0b74-452b-8766-067e9c558dac.1597679841.025146", + "referer" : "https://www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "url" : "www.youtube.com/s/player/0c815aae/player_ias.vflset/en_US/remote.js", + "decision" : "match", + "childRecords" : [ { + "requestId" : "googleads.g.doubleclick.net.553d006a-0315-4241-98d5-ef56d289c85f.1597679841.1790926", + "referer" : "https://www.youtube.com/", + "url" : "googleads.g.doubleclick.net/pagead/id", + "decision" : "abort_not_found", + "childRecords" : [ ] + }, { + "requestId" : "googleads.g.doubleclick.net.6da7c10d-10ab-4bec-bd5a-96b85631372a.1597679841.3626337", + "referer" : "https://www.youtube.com/", + "url" : "googleads.g.doubleclick.net/pagead/id", + "decision" : "abort_not_found", + "childRecords" : [ ] + }, { + "requestId" : "static.doubleclick.net.20fd3c88-ac6a-4e8a-890a-07a86a38e223.1597679841.3844354", + "referer" : "https://www.youtube.com/", + "url" : "static.doubleclick.net/instream/ad_status.js", + "decision" : "abort_not_found", + "childRecords" : [ ] + }, { + "requestId" : "static.doubleclick.net.ace62722-e234-4b24-b643-4a3bff7462b4.1597679841.4170556", + "referer" : "https://www.youtube.com/", + "url" : "static.doubleclick.net/instream/ad_status.js", + "decision" : "abort_not_found", + "childRecords" : [ ] + } ] + }, { + "requestId" : "www.youtube.com.cd92ec11-970a-40eb-928d-105022c5f738.1597679850.3985817", + "referer" : "https://www.youtube.com/embed/D8v_sBHo86A?enablejsapi=1", + "url" : "www.youtube.com/youtubei/v1/log_event?alt=json&key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + "decision" : "abort_not_found", + "childRecords" : [ { + "requestId" : "googleads.g.doubleclick.net.beb56c87-4031-4125-bf44-b4a9f3b6fee5.1597680087.0667331", + "referer" : "https://www.youtube.com/", + "url" : "googleads.g.doubleclick.net/pagead/id", + "decision" : "abort_not_found", + "childRecords" : [ ] + }, { + "requestId" : "googleads.g.doubleclick.net.856bd969-2e9d-4ef4-8644-4690d1908ebc.1597680087.376699", + "referer" : "https://www.youtube.com/", + "url" : "googleads.g.doubleclick.net/pagead/id", + "decision" : "abort_not_found", + "childRecords" : [ ] + } ] + } ] + }, { + "requestId" : "d1z2jf7jlzjs58.cloudfront.net.085e457a-5e1d-40f8-94e0-7499a18b45d1.1597679830.6171253", + "referer" : "https://reason.com/latest/", + "url" : "d1z2jf7jlzjs58.cloudfront.net/p.js", + "decision" : "abort_not_found", + "childRecords" : [ ] + }, { + "requestId" : "js.authorize.net.983ff3cd-be50-4cdc-bae0-22ae4aef8baf.1597679830.766471", + "referer" : "https://reason.com/latest/", + "url" : "js.authorize.net/v1/AcceptCore.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "a.pub.network.faf1cc52-7b30-46a7-afba-4f12868a0657.1597679831.1014266", + "referer" : "https://reason.com/latest/", + "url" : "a.pub.network/reason-com/pubfig.min.js", + "decision" : "abort_not_found", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.90aeebd4-c34b-4765-8eb8-96c4782bb45a.1597679831.1315837", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/kdp6eed.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.ea99f088-a4d5-45ee-8a3f-630dc136d991.1597679831.2945082", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/wp-content/cache/busting/facebook-tracking/fbpix-config-807449156089636-2.9.23.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.gstatic.com.d5aa6a6b-e214-42d9-a6cd-e72f3cd57c96.1597679831.5280895", + "referer" : "https://reason.com/latest/", + "url" : "www.gstatic.com/recaptcha/releases/TPiWapjoyMdQOtxLT9_b4n2W/recaptcha__en.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "reason.com.e7d306e8-f57d-4198-b318-b6c5abaab4e7.1597679831.7378786", + "referer" : "https://reason.com/latest/", + "url" : "reason.com/wp-content/themes/reason-com/dist/images/logo-inverted-without-tag_de403409.svg", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.47ff30db-f56a-4c06-99a4-53604e2aaefa.1597679831.9612765", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/img/c331x186-w331-q60/uploads/2020/08/Portland-protest-1-331x186.jpg", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.6b697059-4adb-4a81-be2a-1eb667bafcdd.1597679832.108128", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/img/c331x186-w331-q60/uploads/2020/08/sfphotosfour685030-331x186.jpg", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.67992054-a8f1-4c97-9c36-dce63b1663fa.1597679832.2529516", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/img/c331x186-w331-q60/uploads/2020/08/zumaamericastwentyseven151318-331x186.jpg", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "d2eehagpk5cl65.cloudfront.net.c69d62b4-32d7-44a7-8e21-a6352b255159.1597679832.4042878", + "referer" : "https://reason.com/latest/", + "url" : "d2eehagpk5cl65.cloudfront.net/img/c331x186-w331-q60/uploads/2020/08/dreamstime_xxl_176349629-331x186.jpg", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "payments.amazon.com.ffa5fb67-024c-46b8-992f-dbbe5c0b0f73.1597679832.6064692", + "referer" : "https://reason.com/latest/", + "url" : "payments.amazon.com/gp/widgets/sessionstabilizer?countryOfEstablishment=US&ledgerCurrency=USD&isSandbox=false", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "www.youtube.com.b3060764-c2a9-49a3-bd01-1196ea5f9bf9.1597679833.0756593", + "referer" : "https://reason.com/latest/", + "url" : "www.youtube.com/iframe_api", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "payments.amazon.com.46bd16a3-be9d-4dfb-b0b9-e77831e58752.1597679834.7099426", + "referer" : "https://reason.com/latest/", + "url" : "payments.amazon.com/cs/uedata", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.e0c0be7e-eda1-4c98-b0a4-e4657f528dfb.1597679836.4843092", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/a30f3c/00000000000000003b9b2245/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n5&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.1c0199db-277d-4508-9856-ff1de20170e5.1597679836.6690555", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/b825af/0000000000000000000118b1/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n7&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.63f7d0f3-3232-4509-b67b-4bc1e9548c9d.1597679836.8534153", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/a2031c/0000000000000000000118b9/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n7&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.def67027-319d-4073-9b7d-c938f1cca716.1597679837.0606365", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/bb00d4/00000000000000003b9b2244/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n4&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.c82e0501-2d57-4a9a-afbe-f553020f28bd.1597679837.2397454", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/2e6f07/000000000000000000011ce6/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n5&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.ed1d9670-9861-4b6f-85cb-6589f50cbb83.1597679837.4038742", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/9cb78a/0000000000000000000118ad/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n4&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "payments.amazon.com.d84a30c0-9d39-49f2-a6f9-f54955b9f816.1597679837.5427072", + "referer" : "https://reason.com/latest/", + "url" : "payments.amazon.com/abTestV2?countryOfEstablishment=US&ledgerCurrency=USD&isSandbox=false&encryptedSessionId=PAVJ4RnToMUH3Zqe3LMC8ysq6k7xqsxapizvmPm7oPyIJofTdsOsrapDnPFhx7I%253D", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "s.ytimg.com.88fa7148-9b54-4848-a2c7-6e3e2f48ffcb.1597679837.977823", + "referer" : "https://reason.com/latest/", + "url" : "s.ytimg.com/yts/jsbin/www-widgetapi-vfldn1jRM/www-widgetapi.js", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.0888d426-057d-411b-98e3-32d73fb345a1.1597679839.70849", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/2d0302/0000000000000000000118b6/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=i4&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.afd8f7e0-3ae9-4e9c-9453-41132a664a4e.1597679839.854518", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/726c0c/00000000000000003b9b2300/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=i7&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.6d25670f-17b2-4c8d-8bb1-78d4d9eeba00.1597679839.9960175", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/d4ccc3/00000000000000003b9b22ff/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n7&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.50f2448b-0d06-46e9-acfd-36b2cd5f798b.1597679840.1333256", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/f7c92b/00000000000000003b9b22f3/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=n4&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.346e1557-ee14-4b89-9f93-6bc52d6824db.1597679840.2907472", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/00041c/0000000000000000000118b8/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=i5&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.31f2611b-46bb-43e2-8767-cad2103c1297.1597679840.4165006", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/ce6b1d/0000000000000000000118ba/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=i7&v=3", + "decision" : "match", + "childRecords" : [ ] + }, { + "requestId" : "use.typekit.net.bfe0e19b-4c77-4213-9b8c-679c626ece40.1597679841.21301", + "referer" : "https://reason.com/latest/", + "url" : "use.typekit.net/af/494bab/00000000000000003b9b22f4/27/l?primer=7cdcb44be4a7db8877ffa5c0007b8dd865b3bbc383831fe2ea177f62257a9191&fvd=i4&v=3", + "decision" : "match", + "childRecords" : [ ] + } ] +} diff --git a/bubble-server/src/test/resources/test-bubble-config.yml b/bubble-server/src/test/resources/test-bubble-config.yml index f5374838..9dd26080 100644 --- a/bubble-server/src/test/resources/test-bubble-config.yml +++ b/bubble-server/src/test/resources/test-bubble-config.yml @@ -56,6 +56,7 @@ jersey: - org.cobbzilla.wizard.filters providerPackages: - org.cobbzilla.wizard.exceptionmappers + - bubble.exceptionmappers requestFilters: [ bubble.auth.BubbleAuthFilter ] responseFilters: - org.cobbzilla.wizard.filters.ScrubbableScrubber diff --git a/pom.xml b/pom.xml index 51b14fdf..4223f476 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,7 @@ bubble.test.system.DriverTest bubble.test.filter.ProxyTest bubble.test.filter.TrafficAnalyticsTest + bubble.test.filter.BlockSummaryTest bubble.test.system.BackupTest bubble.test.system.NetworkTest bubble.abp.spec.BlockListTest diff --git a/utils/abp-parser b/utils/abp-parser index 0940a4c1..12e1c83f 160000 --- a/utils/abp-parser +++ b/utils/abp-parser @@ -1 +1 @@ -Subproject commit 0940a4c13159434b0c8d19049ad53fb3d66a712b +Subproject commit 12e1c83ff60de7a974eec34180e18454622650c1 diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index f04e3f5f..81f18e0d 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit f04e3f5f86e1706d4bb899b4c8cdd6cdc4ac92c4 +Subproject commit 81f18e0d525d551413b8987e76faca870073264e diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index bf9e491c..9240375c 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit bf9e491c17193a54d5b53b803da82b0e5a828109 +Subproject commit 9240375cb73b47e43ad5e9a5d77f33dfef0f6925