From 699fa9192f561aff4b6ccb44db6793c5a64543e1 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 16 Aug 2020 12:09:51 -0400 Subject: [PATCH] move block stats into service, change icon color based on number of blocks --- .../BubbleOutOfMemoryProvider.java | 10 ++ .../main/java/bubble/model/app/AppRule.java | 6 + .../main/java/bubble/model/device/Device.java | 3 +- .../stream/FilterConnCheckRequest.java | 2 + .../resources/stream/FilterDataResource.java | 45 +++++-- .../resources/stream/FilterHttpResource.java | 121 ++++++++++++++---- .../stream/FilterMatchersRequest.java | 19 ++- .../stream/ReverseProxyResource.java | 2 +- .../bubble/rule/AbstractAppRuleDriver.java | 18 ++- .../main/java/bubble/rule/AppRuleDriver.java | 12 ++ .../main/java/bubble/rule/TrafficRecord.java | 2 +- .../rule/bblock/BubbleBlockRuleDriver.java | 26 +++- .../main/java/bubble/server/BubbleServer.java | 3 + .../listener/NodeInitializerListener.java | 3 + .../bubble/service/block/BlockStatRecord.java | 105 +++++++++++++++ .../service/block/BlockStatsService.java | 69 ++++++++++ .../service/block/BlockStatsSummary.java | 49 +++++++ .../bubble/service/cloud/DeviceIdService.java | 1 + .../cloud/StandardDeviceIdService.java | 9 +- .../bubble/service/stream/AppDataCleaner.java | 31 +++++ .../src/main/resources/bubble-config.yml | 1 + .../rule/RequestModifierRule_icon.js.hbs | 26 +++- .../bblock/BubbleBlockRuleDriver_stats.js.hbs | 47 ++++++- .../roles/mitmproxy/files/bubble_api.py | 20 ++- .../mitmproxy/files/bubble_conn_check.py | 39 ++++-- .../roles/mitmproxy/files/dns_spoofing.py | 7 +- bubble-server/src/main/resources/spring.xml | 1 + .../src/test/resources/test-bubble-config.yml | 1 + utils/cobbzilla-utils | 2 +- utils/cobbzilla-wizard | 2 +- 30 files changed, 608 insertions(+), 74 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/exceptionmappers/BubbleOutOfMemoryProvider.java create mode 100644 bubble-server/src/main/java/bubble/service/block/BlockStatRecord.java create mode 100644 bubble-server/src/main/java/bubble/service/block/BlockStatsService.java create mode 100644 bubble-server/src/main/java/bubble/service/block/BlockStatsSummary.java create mode 100644 bubble-server/src/main/java/bubble/service/stream/AppDataCleaner.java diff --git a/bubble-server/src/main/java/bubble/exceptionmappers/BubbleOutOfMemoryProvider.java b/bubble-server/src/main/java/bubble/exceptionmappers/BubbleOutOfMemoryProvider.java new file mode 100644 index 00000000..9f71eca1 --- /dev/null +++ b/bubble-server/src/main/java/bubble/exceptionmappers/BubbleOutOfMemoryProvider.java @@ -0,0 +1,10 @@ +package bubble.exceptionmappers; + +import org.cobbzilla.wizard.exceptionmappers.OutOfMemoryErrorMapper; +import org.springframework.stereotype.Service; + +import javax.ws.rs.ext.Provider; + +@Provider @Service +public class BubbleOutOfMemoryProvider extends OutOfMemoryErrorMapper { +} diff --git a/bubble-server/src/main/java/bubble/model/app/AppRule.java b/bubble-server/src/main/java/bubble/model/app/AppRule.java index b0afd903..bf872391 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppRule.java +++ b/bubble-server/src/main/java/bubble/model/app/AppRule.java @@ -96,6 +96,12 @@ public class AppRule extends IdentifiableBaseParentEntity implements AppTemplate return d; } + public AppRuleDriver initQuickDriver(BubbleApp app, RuleDriver driver, AppMatcher matcher, Account account, Device device) { + final AppRuleDriver d = driver.getDriver(); + d.initQuick(json(configJson, JsonNode.class), driver.getUserConfig(), app, this, matcher, account, device); + return d; + } + @ECSearchable(filter=true) @ECField(index=80) @Size(max=500000, message="err.configJson.length") @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(500000+ENC_PAD)+")") diff --git a/bubble-server/src/main/java/bubble/model/device/Device.java b/bubble-server/src/main/java/bubble/model/device/Device.java index 93fff6bc..0d9e2aab 100644 --- a/bubble-server/src/main/java/bubble/model/device/Device.java +++ b/bubble-server/src/main/java/bubble/model/device/Device.java @@ -14,6 +14,7 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; import org.cobbzilla.wizard.model.Identifiable; import org.cobbzilla.wizard.model.IdentifiableBase; import org.cobbzilla.wizard.model.entityconfig.annotations.*; @@ -36,7 +37,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @Entity @ECType(root=true) @ToString(of={"name"}) @ECTypeURIs(baseURI=EP_DEVICES, listFields={"name", "enabled"}) -@NoArgsConstructor @Accessors(chain=true) +@NoArgsConstructor @Accessors(chain=true) @Slf4j @ECIndexes({ @ECIndex(unique=true, of={"account", "network", "name"}), @ECIndex(unique=true, of={"account", "name"}), diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java index 83a9f0e2..ca4478d4 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java @@ -6,6 +6,7 @@ package bubble.resources.stream; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang.ArrayUtils; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @@ -16,6 +17,7 @@ public class FilterConnCheckRequest { @Getter @Setter private String[] fqdns; public boolean hasFqdns() { return !empty(fqdns); } + public boolean hasFqdn(String f) { return hasFqdns() && ArrayUtils.contains(fqdns, f); } @Getter @Setter private String remoteAddr; public boolean hasRemoteAddr() { return !empty(remoteAddr); } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java index 764b78b5..90df4b36 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterDataResource.java @@ -5,11 +5,14 @@ package bubble.resources.stream; import bubble.dao.app.AppDataDAO; +import bubble.dao.app.AppRuleDAO; +import bubble.dao.app.BubbleAppDAO; +import bubble.dao.app.RuleDriverDAO; import bubble.model.account.Account; -import bubble.model.app.AppData; -import bubble.model.app.AppDataFormat; -import bubble.model.app.AppMatcher; +import bubble.model.app.*; import bubble.model.device.Device; +import bubble.rule.AppRuleDriver; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -29,12 +32,28 @@ import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.wizard.resources.ResourceUtil.*; +@Consumes(APPLICATION_JSON) +@Produces(APPLICATION_JSON) @Slf4j public class FilterDataResource { - private Account account; - private Device device; - private AppMatcher matcher; + @Autowired private AppDataDAO dataDAO; + @Autowired private AppRuleDAO ruleDAO; + @Autowired private RuleDriverDAO driverDAO; + @Autowired private BubbleAppDAO appDAO; + + private final Account account; + private final Device device; + private final AppMatcher matcher; + @Getter(lazy=true) private final AppRule rule = ruleDAO.findByUuid(matcher.getRule()); + @Getter(lazy=true) private final RuleDriver driver = driverDAO.findByUuid(getRule().getDriver()); + @Getter(lazy=true) private final BubbleApp app = appDAO.findByUuid(matcher.getApp()); + @Getter(lazy=true) private final AppRuleDriver ruleDriver = initAppRuleDriver(); + + private AppRuleDriver initAppRuleDriver() { + log.warn("initAppRuleDriver: initializing driver...."); + return getRule().initQuickDriver(getApp(), getDriver(), matcher, account, device); + } public FilterDataResource (Account account, Device device, AppMatcher matcher) { this.account = account; @@ -42,10 +61,7 @@ public class FilterDataResource { this.matcher = matcher; } - @Autowired private AppDataDAO dataDAO; - @GET @Path(EP_READ) - @Produces(APPLICATION_JSON) public Response readData(@Context Request req, @Context ContainerRequest ctx, @QueryParam("format") AppDataFormat format) { @@ -70,8 +86,6 @@ public class FilterDataResource { } @POST @Path(EP_WRITE) - @Consumes(APPLICATION_JSON) - @Produces(APPLICATION_JSON) public Response writeData(@Context Request req, @Context ContainerRequest ctx, AppData data) { @@ -80,7 +94,6 @@ public class FilterDataResource { } @GET @Path(EP_WRITE) - @Produces(APPLICATION_JSON) public Response writeData(@Context Request req, @Context ContainerRequest ctx, @QueryParam(Q_DATA) String dataJson, @@ -123,4 +136,12 @@ public class FilterDataResource { return dataDAO.set(data); } + @GET @Path(EP_READ+"/rule/{id}") + public Response readRuleData(@Context Request req, + @Context ContainerRequest ctx, + @PathParam("id") String id) { + final Object data = getRuleDriver().readData(id); + return data == null ? notFound(id) : ok(data); + } + } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 44d75656..d46df88c 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -15,18 +15,23 @@ import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; import bubble.model.app.AppSite; import bubble.model.app.BubbleApp; +import bubble.model.cloud.BubbleNetwork; import bubble.model.cloud.BubbleNode; import bubble.model.device.Device; import bubble.rule.FilterMatchDecision; +import bubble.service.block.BlockStatsService; import bubble.server.BubbleConfiguration; +import bubble.service.block.BlockStatsSummary; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.DeviceIdService; import bubble.service.stream.ConnectionCheckResponse; import bubble.service.stream.StandardRuleEngineService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.ExpirationEvictionPolicy; import org.cobbzilla.util.collection.ExpirationMap; import org.cobbzilla.util.http.HttpContentEncodingType; +import org.cobbzilla.util.network.NetworkUtil; import org.cobbzilla.wizard.cache.redis.RedisService; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -50,6 +55,7 @@ import static java.util.Collections.emptyMap; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; +import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; @@ -75,6 +81,7 @@ public class FilterHttpResource { @Autowired private RedisService redis; @Autowired private BubbleConfiguration configuration; @Autowired private SelfNodeService selfNodeService; + @Autowired private BlockStatsService blockStats; private static final long ACTIVE_REQUEST_TIMEOUT = HOURS.toSeconds(12); @@ -143,8 +150,15 @@ public class FilterHttpResource { } validateMitmCall(req); - // if the requested IP is the same as our IP, then always passthru - if (isForUs(connCheckRequest)) return ok(); + // is the requested IP is the same as our IP? + final boolean isLocalIp = isForLocalIp(connCheckRequest); + if (isLocalIp) { + // if it is for our host or net name, passthru + if (connCheckRequest.hasFqdns() && (connCheckRequest.hasFqdn(getThisNode().getFqdn()) || connCheckRequest.hasFqdn(getThisNetwork().getNetworkDomain()))) { + if (log.isDebugEnabled()) log.debug(prefix + "returning passthru for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.passthru); + } + } final String vpnAddr = connCheckRequest.getRemoteAddr(); final Device device = deviceIdService.findDeviceByIp(vpnAddr); @@ -154,13 +168,25 @@ public class FilterHttpResource { } else if (log.isTraceEnabled()) { log.trace(prefix+"found device "+device.id()+" for IP "+vpnAddr); } - final Account account = findCaller(device.getAccount()); + final String accountUuid = device.getAccount(); + final Account account = findCaller(accountUuid); if (account == null) { - if (log.isDebugEnabled()) log.debug(prefix+"account not found for uuid "+device.getAccount()+", returning not found"); + if (log.isDebugEnabled()) log.debug(prefix+"account not found for uuid "+ accountUuid +", returning not found"); return notFound(); } - final List matchers = matcherDAO.findByAccountAndEnabledAndConnCheck(device.getAccount()); + if (isLocalIp) { + if (showStats(accountUuid)) { + // allow it for now + if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showBlockStats==true) for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.noop); + } else { + if (log.isDebugEnabled()) log.debug(prefix + "returning block (showBlockStats==false) for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.block); + } + } + + final List matchers = getConnCheckMatchers(accountUuid); final List retained = new ArrayList<>(); for (AppMatcher matcher : matchers) { final BubbleApp app = appDAO.findByUuid(matcher.getApp()); @@ -170,25 +196,44 @@ public class FilterHttpResource { retained.add(matcher); } - final String[] fqdns = connCheckRequest.getFqdns(); - for (String fqdn : fqdns) { - final ConnectionCheckResponse checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); - if (checkResponse != ConnectionCheckResponse.noop) { - if (log.isDebugEnabled()) log.debug(prefix + "returning "+checkResponse+" for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); - return ok(checkResponse); + ConnectionCheckResponse checkResponse = ConnectionCheckResponse.noop; + if (connCheckRequest.hasFqdns()) { + final String[] fqdns = connCheckRequest.getFqdns(); + for (String fqdn : fqdns) { + checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); + if (checkResponse != ConnectionCheckResponse.noop) { + if (log.isDebugEnabled()) log.debug(prefix + "found " + checkResponse + " (breaking) for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); + break; + } } + if (log.isDebugEnabled()) log.debug(prefix+"returning "+checkResponse+" for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); + return ok(checkResponse); + + } else { + if (log.isDebugEnabled()) log.debug(prefix+"returning noop for NO fqdns, addr="+connCheckRequest.getAddr()); + return ok(ConnectionCheckResponse.noop); } - if (log.isDebugEnabled()) log.debug(prefix+"returning noop for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); - return ok(ConnectionCheckResponse.noop); } - private boolean isForUs(FilterConnCheckRequest connCheckRequest) { - final BubbleNode thisNode = selfNodeService.getThisNode(); - return connCheckRequest.hasAddr() - && (thisNode.hasIp4() && thisNode.getIp4().equals(connCheckRequest.getAddr()) - || thisNode.hasIp6() && thisNode.getIp6().equals(connCheckRequest.getAddr())); + private final Map> connCheckMatcherCache = new ExpirationMap<>(10, HOURS.toMillis(1), ExpirationEvictionPolicy.atime); + public List getConnCheckMatchers(String accountUuid) { + return connCheckMatcherCache.computeIfAbsent(accountUuid, k -> matcherDAO.findByAccountAndEnabledAndConnCheck(k)); + } + + private boolean isForLocalIp(FilterConnCheckRequest connCheckRequest) { + return connCheckRequest.hasAddr() && getConfiguredIps().contains(connCheckRequest.getAddr()); + } + + private boolean isForLocalIp(FilterMatchersRequest matchersRequest) { + return matchersRequest.hasServerAddr() && getConfiguredIps().contains(matchersRequest.getServerAddr()); } + @Getter(lazy=true) private final Set configuredIps = NetworkUtil.configuredIps(); + @Getter(lazy=true) private final BubbleNode thisNode = selfNodeService.getThisNode(); + @Getter(lazy=true) private final BubbleNetwork thisNetwork = selfNodeService.getThisNetwork(); + + public boolean showStats(String accountUuid) { return deviceIdService.doShowBlockStats(accountUuid); } + @POST @Path(EP_MATCHERS+"/{requestId}") @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON) @@ -208,13 +253,13 @@ public class FilterHttpResource { if (log.isDebugEnabled()) log.debug(prefix+"starting for filterRequest="+json(filterRequest, COMPACT_MAPPER)); else if (extraLog) log.error(prefix+"starting for filterRequest="+json(filterRequest, COMPACT_MAPPER)); - if (!filterRequest.hasRemoteAddr()) { + if (!filterRequest.hasClientAddr()) { if (log.isDebugEnabled()) log.debug(prefix+"no VPN address provided, returning no matchers"); else if (extraLog) log.error(prefix+"no VPN address provided, returning no matchers"); return ok(NO_MATCHERS); } - final String vpnAddr = filterRequest.getRemoteAddr(); + final String vpnAddr = filterRequest.getClientAddr(); final Device device = deviceIdService.findDeviceByIp(vpnAddr); if (device == null) { if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning no matchers"); @@ -225,9 +270,26 @@ public class FilterHttpResource { } filterRequest.setDevice(device.getUuid()); + // if this is for a local ip, it's an automatic block + // legitimate local requests would have been passthru and never reached here + final boolean isLocalIp = isForLocalIp(filterRequest); + final boolean showStats = showStats(device.getAccount()); + if (isLocalIp) { + if (filterRequest.isBrowser() && showStats) { + blockStats.record(filterRequest, FilterMatchDecision.abort_not_found); + } + if (log.isDebugEnabled()) log.debug(prefix + "returning FORBIDDEN (showBlockStats=="+ showStats +")"); + return forbidden(); + } + final FilterMatchersResponse response = getMatchersResponse(filterRequest, req, request); if (log.isDebugEnabled()) log.debug(prefix+"returning response: "+json(response, COMPACT_MAPPER)); else if (extraLog) log.error(prefix+"returning response: "+json(response, COMPACT_MAPPER)); + + if (filterRequest.isBrowser() && showStats) { + blockStats.record(filterRequest, response.getDecision()); + } + return ok(response); } @@ -346,6 +408,7 @@ public class FilterHttpResource { public Response flushCaches(@Context ContainerRequest request) { final Account caller = userPrincipal(request); if (!caller.admin()) return forbidden(); + connCheckMatcherCache.clear(); return ok(ruleEngine.flushCaches()); } @@ -520,10 +583,24 @@ public class FilterHttpResource { final FilterSubContext filterCtx = new FilterSubContext(req, requestId); if (!filterCtx.request.hasMatcher(matcherId)) throw notFoundEx(matcherId); - final AppMatcher matcher = matcherDAO.findByAccountAndId(filterCtx.request.getAccount().getUuid(), matcherId); + final Account account = filterCtx.request.getAccount(); + account.setMtime(0); // only create one FilterDataResource + final AppMatcher matcher = matcherDAO.findByAccountAndId(account.getUuid(), matcherId); if (matcher == null) throw notFoundEx(matcherId); - return configuration.subResource(FilterDataResource.class, filterCtx.request.getAccount(), filterCtx.request.getDevice(), matcher); + final Device device = filterCtx.request.getDevice(); + device.setMtime(0); // only create one FilterDataResource + return configuration.subResource(FilterDataResource.class, account, device, matcher); + } + + @GET @Path(EP_STATUS+"/{requestId}") + public Response getRequestStatus(@Context Request req, + @Context ContainerRequest ctx, + @PathParam("requestId") String requestId) { + final FilterSubContext filterCtx = new FilterSubContext(req, requestId); + final BlockStatsSummary summary = blockStats.getSummary(requestId); + if (summary == null) return notFound(requestId); + return ok(summary); } @Path(EP_ASSETS+"/{requestId}/{appId}") diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java index 882b0a01..c23971d0 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterMatchersRequest.java @@ -9,8 +9,10 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; +import org.cobbzilla.util.http.HttpUtil; import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.http.HttpSchemes.stripScheme; @NoArgsConstructor @Accessors(chain=true) public class FilterMatchersRequest { @@ -24,16 +26,27 @@ public class FilterMatchersRequest { @Getter @Setter private String fqdn; @Getter @Setter private String uri; @Getter @Setter private String userAgent; + @JsonIgnore public boolean isBrowser () { return HttpUtil.isBrowser(getUserAgent()); } @Getter @Setter private String referer; public boolean hasReferer () { return !empty(referer) && !referer.equals("NONE"); } - @Getter @Setter private String remoteAddr; - public boolean hasRemoteAddr() { return !empty(remoteAddr); } + @JsonIgnore public String getRefererFqdn () { + if (!hasReferer()) return null; + final String base = stripScheme(referer); + final int slashPos = base.indexOf('/'); + return slashPos == -1 ? base : base.substring(0, slashPos); + } + + @Getter @Setter private String clientAddr; + public boolean hasClientAddr() { return !empty(clientAddr); } + + @Getter @Setter private String serverAddr; + public boolean hasServerAddr() { return !empty(serverAddr); } // note: we do *not* include the requestId in the cache, if we did then the // FilterHttpResource.matchersCache cache would be useless, since every cache entry would be unique - public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, remoteAddr); } + public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, clientAddr, serverAddr); } @JsonIgnore public String getUrl() { return fqdn + uri; } diff --git a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java index f1c742e4..21bdf1e0 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java @@ -90,7 +90,7 @@ public class ReverseProxyResource { .setUri(ub.getFullPath()) .setUserAgent(getUserAgent(request)) .setReferer(getReferer(request)) - .setRemoteAddr(remoteHost) + .setClientAddr(remoteHost) .setDevice(device.getUuid())) .setRequestId(id) .setDecision(FilterMatchDecision.match) diff --git a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java index 451b53f5..a13d7414 100644 --- a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java @@ -41,6 +41,7 @@ import static bubble.ApiConstants.HOME_DIR; import static bubble.rule.RequestModifierRule.ICON_JS_TEMPLATE; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.io.regex.RegexReplacementFilter.DEFAULT_PREFIX_REPLACEMENT_WITH_MATCH; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.security.ShaUtil.sha256_hex; @@ -118,13 +119,20 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { } protected String getSiteJsTemplate (String defaultSiteTemplate) { - if (configuration.getEnvironment().containsKey("DEBUG_JS_SITE_TEMPLATES")) { - final File jsTemplateFile = new File(HOME_DIR + "/siteJsTemplates/" + requestModConfig().getSiteJsTemplate()); - if (jsTemplateFile.exists()) { - return FileUtil.toStringOrDie(jsTemplateFile); + return loadTemplate(defaultSiteTemplate, requestModConfig().getSiteJsTemplate()); + } + + protected String loadTemplate(String defaultTemplate, String templatePath) { + if (configuration.getEnvironment().containsKey("DEBUG_RULE_TEMPLATES")) { + final File templateFile = new File(HOME_DIR + "/debugTemplates/" + templatePath); + if (templateFile.exists()) { + log.error("loadTemplate: debug file found (using it): "+abs(templateFile)); + return FileUtil.toStringOrDie(templateFile); + } else { + log.error("loadTemplate: debug file not found (using default): "+abs(templateFile)); } } - return defaultSiteTemplate; + return defaultTemplate; } private RequestModifierConfig requestModConfig() { diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index a7208e1c..7457ef14 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -79,6 +79,16 @@ public interface AppRuleDriver { Account account, Device device) {} + default void initQuick(JsonNode config, + JsonNode userConfig, + BubbleApp app, + AppRule rule, + AppMatcher matcher, + Account account, + Device device) { + init(config, userConfig, app, rule, matcher, account, device); + } + default FilterMatchDecision preprocess(AppRuleHarness ruleHarness, FilterMatchersRequest filter, Account account, @@ -160,4 +170,6 @@ public interface AppRuleDriver { return sageRuleConfig; } + default Object readData(String id) { return null; } + } diff --git a/bubble-server/src/main/java/bubble/rule/TrafficRecord.java b/bubble-server/src/main/java/bubble/rule/TrafficRecord.java index 66402faf..f6af745b 100644 --- a/bubble-server/src/main/java/bubble/rule/TrafficRecord.java +++ b/bubble-server/src/main/java/bubble/rule/TrafficRecord.java @@ -36,7 +36,7 @@ public class TrafficRecord { setAccountUuid(account == null ? null : account.getUuid()); setDeviceName(device == null ? null : device.getName()); setDeviceUuid(device == null ? null : device.getUuid()); - setIp(filter.getRemoteAddr()); + setIp(filter.getServerAddr()); setFqdn(filter.getFqdn()); setUri(filter.getUri()); setUserAgent(filter.getUserAgent()); diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index 5f5a590a..8e4474e2 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -57,7 +57,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver implements private final static Map blockListCache = new ConcurrentHashMap<>(); - public boolean showStats() { return deviceService.doShowBlockStats(account); } + public boolean showStats() { return deviceService.doShowBlockStats(account.getUuid()); } @Override public Class getConfigClass() { return (Class) BubbleBlockConfig.class; } @@ -75,10 +75,20 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver implements AppMatcher matcher, Account account, Device device) { - super.init(config, userConfig, app, rule, matcher, account, device); + initQuick(config, userConfig, app, rule, matcher, account, device); refreshBlockLists(); } + @Override public void initQuick(JsonNode config, + JsonNode userConfig, + BubbleApp app, + AppRule rule, + AppMatcher matcher, + Account account, + Device device) { + super.init(config, userConfig, app, rule, matcher, account, device); + } + @Override public JsonNode upgradeRuleConfig(JsonNode sageRuleConfig, JsonNode localRuleConfig) { final BubbleBlockConfig sageConfig = json(sageRuleConfig, getConfigClass()); @@ -344,18 +354,24 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver implements return in; } if (bubbleBlockConfig.inPageBlocks() && showStats) { - return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, BUBBLE_JS_STATS_TEMPLATE, BLOCK_STATS_JS, showStats); + return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, getBubbleJsStatsTemplate(), BLOCK_STATS_JS, showStats); } if (bubbleBlockConfig.inPageBlocks()) { return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, EMPTY, BLOCK_STATS_JS, showStats); } log.warn(prefix+"inserting JS for stats..."); - return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_STATS_TEMPLATE, null, null, showStats); + return filterInsertJs(in, filterRequest, filterCtx, getBubbleJsStatsTemplate(), null, null, showStats); + } + + protected String getBubbleJsStatsTemplate () { + return loadTemplate(BUBBLE_JS_STATS_TEMPLATE, BUBBLE_STATS_TEMPLATE_NAME); } public static final Class BB = BubbleBlockRuleDriver.class; public static final String BUBBLE_JS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+".js.hbs"); - public static final String BUBBLE_JS_STATS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+"_stats.js.hbs"); + + public static final String BUBBLE_STATS_TEMPLATE_NAME = BB.getSimpleName() + "_stats.js.hbs"; + public static final String BUBBLE_JS_STATS_TEMPLATE = stream2string(getPackagePath(BB) + "/" + BUBBLE_STATS_TEMPLATE_NAME); private static final String CTX_BUBBLE_SELECTORS = "BUBBLE_SELECTORS_JSON"; private static final String CTX_BUBBLE_BLACKLIST = "BUBBLE_BLACKLIST_JSON"; diff --git a/bubble-server/src/main/java/bubble/server/BubbleServer.java b/bubble-server/src/main/java/bubble/server/BubbleServer.java index 3971b5db..ed399f0b 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleServer.java +++ b/bubble-server/src/main/java/bubble/server/BubbleServer.java @@ -37,6 +37,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.network.NetworkUtil.IPv4_LOCALHOST; +import static org.cobbzilla.util.system.OutOfMemoryErrorUncaughtExceptionHandler.EXIT_ON_OOME; @NoArgsConstructor @Slf4j public class BubbleServer extends RestServerBase { @@ -78,6 +79,7 @@ public class BubbleServer extends RestServerBase { public static void main(String[] args) throws Exception { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); + Thread.setDefaultUncaughtExceptionHandler(EXIT_ON_OOME); final Map env = loadEnvironment(args); final ConfigurationSource configSource = getConfigurationSource(); @@ -142,4 +144,5 @@ public class BubbleServer extends RestServerBase { public static ConfigurationSource getConfigurationSource() { return getStreamConfigurationSource(BubbleServer.class, API_CONFIG_YML); } + } diff --git a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java index 1af25f29..098213ac 100644 --- a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java @@ -15,6 +15,7 @@ import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; import bubble.service.cloud.DeviceIdService; import bubble.service.cloud.NetworkMonitorService; +import bubble.service.stream.AppDataCleaner; import bubble.service.stream.AppPrimerService; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.wizard.server.RestServer; @@ -106,11 +107,13 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase> parentRecords = new AtomicReference<>(); + private final AtomicReference> subRecords = new AtomicReference<>(); + private final AtomicReference summaryRef = new AtomicReference<>(); + + public BlockStatRecord(FilterMatchersRequest filter, FilterMatchDecision decision) { + this.requestId = filter.getRequestId(); + this.device = filter.getDevice(); + this.referer = filter.getReferer(); + this.fqdn = filter.getFqdn(); + this.url = filter.getUrl(); + this.userAgent = filter.getUserAgent(); + this.decision = decision; + } + + public void addSubRecord(BlockStatRecord rec) { + synchronized (subRecords) { + if (subRecords.get() == null) { + subRecords.set(new ConcurrentHashMap<>()); + } + } + subRecords.get().put(rec.getRequestId(), rec.getRequestId()); + touch(); + } + + public void touch() { mtime = now(); } + + public void addParentRecord(BlockStatRecord parent, Map records) { + synchronized (parentRecords) { + if (parentRecords.get() == null) { + parentRecords.set(new ConcurrentHashMap<>()); + } + parentRecords.get().put(parent.getRequestId(), parent.getRequestId()); + touchParents(records); + } + } + + private void touchParents(Map records) { + touch(); + synchronized (parentRecords) { + if (parentRecords.get() == null) return; + for (String p : parentRecords.get().keySet()) { + final BlockStatRecord parent = records.get(p); + if (parent != null) { + parent.touchParents(records); + } + } + } + } + + public BlockStatsSummary summarize(Map records) { + synchronized (summaryRef) { + BlockStatsSummary sum = summaryRef.get(); + if (sum != null && sum.getCtime() > getMtime()) { + log.info("summarize("+url+"): reusing existing summary"); + return sum; + } else { + log.info("summarize("+url+"): creating new summary"); + sum = new BlockStatsSummary(); + summaryRef.set(sum); + } + return summarize(records, sum); + } + } + + private BlockStatsSummary summarize(Map records, BlockStatsSummary summary) { + final Map subRecs = subRecords.get(); + if (subRecs != null) { + for (String subRecRequestId : subRecs.values()) { + final BlockStatRecord subRec = records.get(subRecRequestId); + if (subRec.decision.isAbort()) { + summary.addBlock(subRec); + } else { + subRec.summarize(records, summary); + } + } + } + return summary; + } +} diff --git a/bubble-server/src/main/java/bubble/service/block/BlockStatsService.java b/bubble-server/src/main/java/bubble/service/block/BlockStatsService.java new file mode 100644 index 00000000..445a0f37 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/block/BlockStatsService.java @@ -0,0 +1,69 @@ +package bubble.service.block; + +import bubble.resources.stream.FilterMatchersRequest; +import bubble.rule.FilterMatchDecision; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.ExpirationEvictionPolicy; +import org.cobbzilla.util.collection.ExpirationMap; +import org.springframework.stereotype.Service; + +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.cobbzilla.util.http.HttpSchemes.stripScheme; +import static org.cobbzilla.util.json.JsonUtil.json; + +@Service @Slf4j +public class BlockStatsService { + + private final ExpirationMap records + = new ExpirationMap<>(200, MINUTES.toMillis(10), ExpirationEvictionPolicy.atime); + + public void flush () { records.clear(); } + + public void record(FilterMatchersRequest filter, FilterMatchDecision decision) { + final BlockStatRecord newRec = new BlockStatRecord(filter, decision); + synchronized (records) { + records.put(getUrlCacheKey(filter), newRec); + records.put(filter.getFqdn(), newRec); + records.put(filter.getRequestId(), newRec); + } + log.info("record: stored keys("+getUrlCacheKey(filter)+", "+filter.getRequestId()+")= newRec="+json(newRec)); + if (!filter.hasReferer()) { + // this must be a top-level request + log.info("record: added top-level record for device="+filter.getDevice()+"/userAgent="+filter.getUserAgent()+"/url="+filter.getUrl()); + } else { + // find match based on device + user-agent + referer + final String cacheKey = getRefererCacheKey(filter); + BlockStatRecord rec = records.get(cacheKey); + if (rec == null) { + // try fqdn + rec = records.get(filter.getRefererFqdn()); + if (rec == null) { + log.warn("record: rec not found for device=" + filter.getDevice() + "/userAgent=" + filter.getUserAgent() + "/referer=" + filter.getReferer()); + return; + } + } + newRec.addParentRecord(rec, records); + rec.addSubRecord(newRec); + } + } + + public String getRefererCacheKey(FilterMatchersRequest filter) { + return filter.getDevice()+"\n"+filter.getUserAgent()+"\n"+stripScheme(filter.getReferer()); + } + + public String getUrlCacheKey(FilterMatchersRequest filter) { + return filter.getDevice()+"\n"+filter.getUserAgent()+"\n"+stripScheme(filter.getUrl()); + } + + public BlockStatsSummary getSummary(String requestId) { + final BlockStatRecord stat = records.get(requestId); + if (stat == null) { + log.info("getSummary("+requestId+") no summary found"); + return null; + } + final BlockStatsSummary summary = stat.summarize(records); + log.info("getSummary("+requestId+") returning summary="+json(summary)); + return summary; + } + +} diff --git a/bubble-server/src/main/java/bubble/service/block/BlockStatsSummary.java b/bubble-server/src/main/java/bubble/service/block/BlockStatsSummary.java new file mode 100644 index 00000000..c4fa6c68 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/block/BlockStatsSummary.java @@ -0,0 +1,49 @@ +package bubble.service.block; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; + +public class BlockStatsSummary { + + private final Map blocks = new HashMap<>(); + @Getter private final long ctime = now(); + + public void addBlock(BlockStatRecord rec) { + final AtomicInteger ct = blocks.computeIfAbsent(rec.getFqdn(), k -> new AtomicInteger(0)); + ct.incrementAndGet(); + } + + public Set getBlocks () { + final Set set = new TreeSet<>(); + for (Map.Entry entry : blocks.entrySet()) { + final int count = entry.getValue().get(); + if (count > 0) set.add(new FqdnBlockCount(entry.getKey(), count)); + } + return set; + } + + public int getTotal () { + int total = 0; + for (AtomicInteger ct : blocks.values()) total += ct.get(); + return total; + } + + @Override public String toString () { return "BlockStatsSummary{total="+getTotal()+"}"; } + + @AllArgsConstructor @EqualsAndHashCode(of={"fqdn"}) + private static class FqdnBlockCount implements Comparable { + @Getter private final String fqdn; + @Getter private final int count; + @Override public int compareTo(FqdnBlockCount o) { return Integer.compare(o.count, count); } + } + +} diff --git a/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java b/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java index 8ef88ecc..8fc254b3 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java @@ -20,6 +20,7 @@ public interface DeviceIdService { void setDeviceSecurityLevel(Device device); void initBlockStats (Account account); + default boolean doShowBlockStats(String accountUuid) { return false; } DeviceStatus getDeviceStatus(String deviceUuid); DeviceStatus getLiveDeviceStatus(String deviceUuid); diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java index c828813d..d92ced05 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java @@ -56,6 +56,9 @@ public class StandardDeviceIdService implements DeviceIdService { // used in dnscrypt-proxy to determine how to respond to blocked requests public static final String REDIS_KEY_DEVICE_REJECT_WITH = "bubble_device_reject_with_"; + // used in mitmproxy to determine how to respond to blocked requests + public static final String REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = "bubble_device_showBlockStats_"; + // used in mitmproxy to optimize passthru requests // we flush keys with this prefix when changing showBlockStats flag public static final String REDIS_KEY_CHUNK_FILTER_PASS = "__chunk_filter_pass__"; @@ -168,8 +171,8 @@ public class StandardDeviceIdService implements DeviceIdService { } } - public boolean doShowBlockStats(Account account) { - return Boolean.parseBoolean(redis.get_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS + account.getUuid())); + public boolean doShowBlockStats(String accountUuid) { + return Boolean.parseBoolean(redis.get_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS + accountUuid)); } public void showBlockStats (Device device) { @@ -183,12 +186,14 @@ public class StandardDeviceIdService implements DeviceIdService { return; } for (String ip : findIpsByDevice(device.getUuid())) { + redis.set_plaintext(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip, Boolean.toString(true)); redis.set_plaintext(REDIS_KEY_DEVICE_REJECT_WITH + ip, privateIp); } } public void hideBlockStats (Device device) { for (String ip : findIpsByDevice(device.getUuid())) { + redis.del_withPrefix(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip); redis.del_withPrefix(REDIS_KEY_DEVICE_REJECT_WITH + ip); } } diff --git a/bubble-server/src/main/java/bubble/service/stream/AppDataCleaner.java b/bubble-server/src/main/java/bubble/service/stream/AppDataCleaner.java new file mode 100644 index 00000000..79665f42 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/stream/AppDataCleaner.java @@ -0,0 +1,31 @@ +package bubble.service.stream; + +import bubble.dao.app.AppDataDAO; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.daemon.SimpleDaemon; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import static java.util.concurrent.TimeUnit.HOURS; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; +import static org.cobbzilla.wizard.server.RestServerBase.reportError; + +@Service @Slf4j +public class AppDataCleaner extends SimpleDaemon { + + @Getter(lazy=true) private final long sleepTime = HOURS.toMillis(4); + + @Autowired private AppDataDAO dataDAO; + + @Override protected void process() { + try { + final int ct = dataDAO.bulkDeleteWhere("expiration < " + now()); + log.info("process: removed " + ct + " expired AppData records"); + } catch (Exception e) { + reportError("AppDataCleaner.process: "+shortError(e), e); + } + } + +} diff --git a/bubble-server/src/main/resources/bubble-config.yml b/bubble-server/src/main/resources/bubble-config.yml index 5c35b0d7..63208d95 100644 --- a/bubble-server/src/main/resources/bubble-config.yml +++ b/bubble-server/src/main/resources/bubble-config.yml @@ -55,6 +55,7 @@ jersey: - org.cobbzilla.wizard.filters providerPackages: - org.cobbzilla.wizard.exceptionmappers + - bubble.exceptionmappers requestFilters: - bubble.auth.BubbleAuthFilter - bubble.filters.BubbleRateLimitFilter diff --git a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs index fcdcff23..d92b1580 100644 --- a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs @@ -10,6 +10,24 @@ if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { } } + {{PAGE_PREFIX}}_getAppIconImgSrc = function (app) { + return '/__bubble/api/filter/assets/{{BUBBLE_REQUEST_ID}}/' + app.app + '/' + app.icon + '?raw=true'; + } + + {{PAGE_PREFIX}}_getAppIconImgId = function (app) { + return app.jsPrefix + '_app_icon_img'; + } + + {{PAGE_PREFIX}}_setAppIconImg = function (app) { + const imgId = {{PAGE_PREFIX}}_getAppIconImgId(app); + const img = document.getElementById(imgId); + if (img) { + img.src = {{PAGE_PREFIX}}_getAppIconImgSrc(app); + } else { + console.warn('setAppIconImg: img element not found: '+imgId) + } + } + function {{PAGE_PREFIX}}_onReady(callback) { const intervalId = window.setInterval(function() { if (document.getElementsByTagName('body')[0] !== undefined) { @@ -29,15 +47,17 @@ if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { bubbleControlDiv.style.position = 'fixed'; bubbleControlDiv.style.bottom = '0'; bubbleControlDiv.style.right = '0'; - bubbleControlDiv.style.zIndex = '2147483647'; + bubbleControlDiv.style.zIndex = '2147483640'; document.getElementsByTagName('body')[0].appendChild(bubbleControlDiv); } for (let i=0; i<{{PAGE_PREFIX}}_icon_status.length; i++) { + const iconSpecs = {{PAGE_PREFIX}}_icon_status[i]; let br = document.createElement('br'); let link = document.createElement('a'); - link.href = '{{{BUBBLE_HOME}}}/app/' + {{PAGE_PREFIX}}_icon_status[i].app + '/' + {{PAGE_PREFIX}}_icon_status[i].link; + link.href = '{{{BUBBLE_HOME}}}/app/' + iconSpecs.app + '/' + iconSpecs.link; let img = document.createElement('img'); - img.src = '/__bubble/api/filter/assets/{{BUBBLE_REQUEST_ID}}/' + {{PAGE_PREFIX}}_icon_status[i].app + '/' + {{PAGE_PREFIX}}_icon_status[i].icon + '?raw=true'; + img.id = {{PAGE_PREFIX}}_getAppIconImgId(iconSpecs); + img.src = {{PAGE_PREFIX}}_getAppIconImgSrc(iconSpecs); img.width = 64; link.appendChild(img); bubbleControlDiv.appendChild(br); diff --git a/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver_stats.js.hbs b/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver_stats.js.hbs index acb7a7a3..bcc2c1e2 100644 --- a/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver_stats.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/bblock/BubbleBlockRuleDriver_stats.js.hbs @@ -1,8 +1,51 @@ {{{ICON_JS}}} -{{PAGE_PREFIX}}_addBubbleApp({ +const {{JS_PREFIX}}_app = { jsPrefix: '{{JS_PREFIX}}', app: '{{BUBBLE_APP_NAME}}', link: 'view/last_24_hours', icon: 'icon-gray' -}); +}; + +{{PAGE_PREFIX}}_addBubbleApp({{JS_PREFIX}}_app); +let {{JS_PREFIX}}_app_stats_ctime = 0; +let {{JS_PREFIX}}_app_stats_last_ctime_change = 0; +const {{JS_PREFIX}}_app_stats_timeout = 35000; +const {{JS_PREFIX}}_app_refresh_interval = window.setInterval(function () { + const requestOptions = { method: 'GET' }; + const block_stats_url = '/__bubble/api/filter/status/{{BUBBLE_REQUEST_ID}}'; + fetch(block_stats_url, requestOptions) + .then(resp => { + try { + return resp.json(); + } catch (error) { + console.log('cancelling window.interval, response not json'); + window.clearInterval({{JS_PREFIX}}_app_refresh_interval); + } + }) + .then(data => { + console.log('stats = '+JSON.stringify(data)); + let icon = null; + if ((typeof data.total !== 'undefined') && (typeof data.ctime !== 'undefined')) { + if (data.ctime != {{JS_PREFIX}}_app_stats_ctime) { + if (data.total === 0) { + icon = 'icon-green'; + } else if (data.total < 5) { + icon = 'icon-yellow'; + } else { + icon = 'icon-red'; + } + {{JS_PREFIX}}_app.icon = icon; + {{PAGE_PREFIX}}_setAppIconImg({{JS_PREFIX}}_app); + {{JS_PREFIX}}_app_stats_ctime = data.ctime; + {{JS_PREFIX}}_app_stats_last_ctime_change = Date.now(); + } else if (Date.now() - {{JS_PREFIX}}_app_stats_last_ctime_change > {{JS_PREFIX}}_app_stats_timeout) { + console.log('cancelling window.interval, stats unchanged for a while'); + window.clearInterval({{JS_PREFIX}}_app_refresh_interval); + } + } + }).catch((error) => { + console.log('cancelling window.interval, due to error: '+error); + window.clearInterval({{JS_PREFIX}}_app_refresh_interval); + }); +}, 5000); diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 1ad96c23..8b6851fd 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -100,14 +100,24 @@ DEBUG_MATCHER = { 'rule': DEBUG_MATCHER_NAME }] } +BLOCK_MATCHER = { + 'decision': 'abort_not_found', + 'matchers': [{ + 'name': 'BLOCK_MATCHER', + 'contentTypeRegex': '.*', + "urlRegex": ".*", + 'rule': 'BLOCK_MATCHER' + }] +} + -def bubble_matchers(req_id, remote_addr, flow, host): +def bubble_matchers(req_id, client_addr, server_addr, flow, host): if debug_capture_fqdn and host and host in debug_capture_fqdn: bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host) return DEBUG_MATCHER headers = { - 'X-Forwarded-For': remote_addr, + 'X-Forwarded-For': client_addr, 'Accept' : 'application/json', 'Content-Type': 'application/json' } @@ -134,11 +144,15 @@ def bubble_matchers(req_id, remote_addr, flow, host): 'uri': flow.request.path, 'userAgent': user_agent, 'referer': referer, - 'remoteAddr': remote_addr + 'clientAddr': client_addr, + 'serverAddr': server_addr } response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data) if response.ok: return response.json() + elif response.status_code == 403: + bubble_log('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text)) + return BLOCK_MATCHER bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) except Exception as e: bubble_log('bubble_matchers API call failed: '+repr(e)) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index 20b284a5..766da1ac 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -28,7 +28,7 @@ from mitmproxy.exceptions import TlsProtocolException from mitmproxy.net import tls as net_tls from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set -from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host +from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host from bubble_vpn4 import wireguard_network_ipv4 from bubble_vpn6 import wireguard_network_ipv6 from netaddr import IPAddress, IPNetwork @@ -41,6 +41,7 @@ REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = 'bubble_device_site_max_security_level_' # defined in StandardDeviceIdService +REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = 'bubble_device_showBlockStats_' FORCE_PASSTHRU = {'passthru': True} FORCE_BLOCK = {'block': True} @@ -77,6 +78,12 @@ def get_device_security_level(client_addr, fqdns): return {'level': level} +def show_block_stats(client_addr): + show = REDIS.get(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS+client_addr) + if show is None: + return False + return show.decode() == 'true' + def get_local_ips(): global local_ips if local_ips is None: @@ -86,8 +93,13 @@ def get_local_ips(): return local_ips +def is_bubble_request(ip, fqdns): + # return ip in get_local_ips() + return ip in get_local_ips() and (bubble_host in fqdns or bubble_host_alias in fqdns) + + def is_sage_request(ip, fqdns): - return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns + return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns def is_not_from_vpn(client_addr): @@ -134,6 +146,10 @@ class TlsFeedback(TlsLayer): super(TlsFeedback, self)._establish_tls_with_client() except TlsProtocolException as e: + if self.do_block: + bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address) + raise e + tb = traceback.format_exc() if 'OpenSSL.SSL.ZeroReturnError' in tb: bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address) @@ -225,8 +241,13 @@ def next_layer(next_layer): no_fqdns = fqdns is None or len(fqdns) == 0 security_level = get_device_security_level(client_addr, fqdns) next_layer.security_level = security_level - if server_addr in get_local_ips(): - bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) + next_layer.do_block = False + if is_bubble_request(server_addr, fqdns): + bubble_log('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns)) + check = FORCE_PASSTHRU + + elif is_sage_request(server_addr, fqdns): + bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU elif is_not_from_vpn(client_addr): @@ -235,10 +256,6 @@ def next_layer(next_layer): next_layer.__class__ = TlsBlock return - elif is_sage_request(server_addr, fqdns): - bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) - check = FORCE_PASSTHRU - elif security_level['level'] == SEC_OFF or security_level['level'] == SEC_BASIC: bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU @@ -268,7 +285,11 @@ def next_layer(next_layer): elif 'block' in check and check['block']: bubble_log('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns) - next_layer.__class__ = TlsBlock + if show_block_stats(client_addr): + next_layer.do_block = True + next_layer.__class__ = TlsFeedback + else: + next_layer.__class__ = TlsBlock else: bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index 78753473..a2749140 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -19,7 +19,8 @@ class Rerouter: bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path) return None - remote_addr = str(flow.client_conn.address[0]) + client_addr = str(flow.client_conn.address[0]) + server_addr = str(flow.server_conn.address[0]) try: host = host.decode() except (UnicodeDecodeError, AttributeError): @@ -35,10 +36,10 @@ class Rerouter: req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time()) bubble_log("get_matchers: requesting match decision for req_id="+req_id) - resp = bubble_matchers(req_id, remote_addr, flow, host) + resp = bubble_matchers(req_id, client_addr, server_addr, flow, host) if not resp: - bubble_log('get_matchers: no response for remote_addr/host: '+remote_addr+'/'+str(host)) + bubble_log('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host)) return None matchers = [] diff --git a/bubble-server/src/main/resources/spring.xml b/bubble-server/src/main/resources/spring.xml index 4c8a405a..9b7caf9f 100644 --- a/bubble-server/src/main/resources/spring.xml +++ b/bubble-server/src/main/resources/spring.xml @@ -28,5 +28,6 @@ + diff --git a/bubble-server/src/test/resources/test-bubble-config.yml b/bubble-server/src/test/resources/test-bubble-config.yml index f5374838..9dd26080 100644 --- a/bubble-server/src/test/resources/test-bubble-config.yml +++ b/bubble-server/src/test/resources/test-bubble-config.yml @@ -56,6 +56,7 @@ jersey: - org.cobbzilla.wizard.filters providerPackages: - org.cobbzilla.wizard.exceptionmappers + - bubble.exceptionmappers requestFilters: [ bubble.auth.BubbleAuthFilter ] responseFilters: - org.cobbzilla.wizard.filters.ScrubbableScrubber diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index d2360429..81f18e0d 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit d2360429c18bc9744ee3881182ae859fbcdc4c46 +Subproject commit 81f18e0d525d551413b8987e76faca870073264e diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 37708799..9240375c 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 37708799b3a0761f18589de02d2a0f754127b5cb +Subproject commit 9240375cb73b47e43ad5e9a5d77f33dfef0f6925