@@ -0,0 +1,10 @@ | |||
package bubble.exceptionmappers; | |||
import org.cobbzilla.wizard.exceptionmappers.OutOfMemoryErrorMapper; | |||
import org.springframework.stereotype.Service; | |||
import javax.ws.rs.ext.Provider; | |||
@Provider @Service | |||
public class BubbleOutOfMemoryProvider extends OutOfMemoryErrorMapper { | |||
} |
@@ -96,6 +96,12 @@ public class AppRule extends IdentifiableBaseParentEntity implements AppTemplate | |||
return d; | |||
} | |||
public AppRuleDriver initQuickDriver(BubbleApp app, RuleDriver driver, AppMatcher matcher, Account account, Device device) { | |||
final AppRuleDriver d = driver.getDriver(); | |||
d.initQuick(json(configJson, JsonNode.class), driver.getUserConfig(), app, this, matcher, account, device); | |||
return d; | |||
} | |||
@ECSearchable(filter=true) @ECField(index=80) | |||
@Size(max=500000, message="err.configJson.length") | |||
@Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(500000+ENC_PAD)+")") | |||
@@ -14,6 +14,7 @@ import lombok.NoArgsConstructor; | |||
import lombok.Setter; | |||
import lombok.ToString; | |||
import lombok.experimental.Accessors; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.cobbzilla.wizard.model.Identifiable; | |||
import org.cobbzilla.wizard.model.IdentifiableBase; | |||
import org.cobbzilla.wizard.model.entityconfig.annotations.*; | |||
@@ -36,7 +37,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; | |||
@Entity @ECType(root=true) @ToString(of={"name"}) | |||
@ECTypeURIs(baseURI=EP_DEVICES, listFields={"name", "enabled"}) | |||
@NoArgsConstructor @Accessors(chain=true) | |||
@NoArgsConstructor @Accessors(chain=true) @Slf4j | |||
@ECIndexes({ | |||
@ECIndex(unique=true, of={"account", "network", "name"}), | |||
@ECIndex(unique=true, of={"account", "name"}), | |||
@@ -6,6 +6,7 @@ package bubble.resources.stream; | |||
import lombok.Getter; | |||
import lombok.Setter; | |||
import org.apache.commons.lang.ArrayUtils; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.empty; | |||
@@ -16,6 +17,7 @@ public class FilterConnCheckRequest { | |||
@Getter @Setter private String[] fqdns; | |||
public boolean hasFqdns() { return !empty(fqdns); } | |||
public boolean hasFqdn(String f) { return hasFqdns() && ArrayUtils.contains(fqdns, f); } | |||
@Getter @Setter private String remoteAddr; | |||
public boolean hasRemoteAddr() { return !empty(remoteAddr); } | |||
@@ -5,11 +5,14 @@ | |||
package bubble.resources.stream; | |||
import bubble.dao.app.AppDataDAO; | |||
import bubble.dao.app.AppRuleDAO; | |||
import bubble.dao.app.BubbleAppDAO; | |||
import bubble.dao.app.RuleDriverDAO; | |||
import bubble.model.account.Account; | |||
import bubble.model.app.AppData; | |||
import bubble.model.app.AppDataFormat; | |||
import bubble.model.app.AppMatcher; | |||
import bubble.model.app.*; | |||
import bubble.model.device.Device; | |||
import bubble.rule.AppRuleDriver; | |||
import lombok.Getter; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.glassfish.grizzly.http.server.Request; | |||
import org.glassfish.jersey.server.ContainerRequest; | |||
@@ -29,12 +32,28 @@ import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
import static org.cobbzilla.wizard.resources.ResourceUtil.*; | |||
@Consumes(APPLICATION_JSON) | |||
@Produces(APPLICATION_JSON) | |||
@Slf4j | |||
public class FilterDataResource { | |||
private Account account; | |||
private Device device; | |||
private AppMatcher matcher; | |||
@Autowired private AppDataDAO dataDAO; | |||
@Autowired private AppRuleDAO ruleDAO; | |||
@Autowired private RuleDriverDAO driverDAO; | |||
@Autowired private BubbleAppDAO appDAO; | |||
private final Account account; | |||
private final Device device; | |||
private final AppMatcher matcher; | |||
@Getter(lazy=true) private final AppRule rule = ruleDAO.findByUuid(matcher.getRule()); | |||
@Getter(lazy=true) private final RuleDriver driver = driverDAO.findByUuid(getRule().getDriver()); | |||
@Getter(lazy=true) private final BubbleApp app = appDAO.findByUuid(matcher.getApp()); | |||
@Getter(lazy=true) private final AppRuleDriver ruleDriver = initAppRuleDriver(); | |||
private AppRuleDriver initAppRuleDriver() { | |||
log.warn("initAppRuleDriver: initializing driver...."); | |||
return getRule().initQuickDriver(getApp(), getDriver(), matcher, account, device); | |||
} | |||
public FilterDataResource (Account account, Device device, AppMatcher matcher) { | |||
this.account = account; | |||
@@ -42,10 +61,7 @@ public class FilterDataResource { | |||
this.matcher = matcher; | |||
} | |||
@Autowired private AppDataDAO dataDAO; | |||
@GET @Path(EP_READ) | |||
@Produces(APPLICATION_JSON) | |||
public Response readData(@Context Request req, | |||
@Context ContainerRequest ctx, | |||
@QueryParam("format") AppDataFormat format) { | |||
@@ -70,8 +86,6 @@ public class FilterDataResource { | |||
} | |||
@POST @Path(EP_WRITE) | |||
@Consumes(APPLICATION_JSON) | |||
@Produces(APPLICATION_JSON) | |||
public Response writeData(@Context Request req, | |||
@Context ContainerRequest ctx, | |||
AppData data) { | |||
@@ -80,7 +94,6 @@ public class FilterDataResource { | |||
} | |||
@GET @Path(EP_WRITE) | |||
@Produces(APPLICATION_JSON) | |||
public Response writeData(@Context Request req, | |||
@Context ContainerRequest ctx, | |||
@QueryParam(Q_DATA) String dataJson, | |||
@@ -123,4 +136,12 @@ public class FilterDataResource { | |||
return dataDAO.set(data); | |||
} | |||
@GET @Path(EP_READ+"/rule/{id}") | |||
public Response readRuleData(@Context Request req, | |||
@Context ContainerRequest ctx, | |||
@PathParam("id") String id) { | |||
final Object data = getRuleDriver().readData(id); | |||
return data == null ? notFound(id) : ok(data); | |||
} | |||
} |
@@ -15,18 +15,23 @@ import bubble.model.app.AppMatcher; | |||
import bubble.model.app.AppRule; | |||
import bubble.model.app.AppSite; | |||
import bubble.model.app.BubbleApp; | |||
import bubble.model.cloud.BubbleNetwork; | |||
import bubble.model.cloud.BubbleNode; | |||
import bubble.model.device.Device; | |||
import bubble.rule.FilterMatchDecision; | |||
import bubble.service.block.BlockStatsService; | |||
import bubble.server.BubbleConfiguration; | |||
import bubble.service.block.BlockStatsSummary; | |||
import bubble.service.boot.SelfNodeService; | |||
import bubble.service.cloud.DeviceIdService; | |||
import bubble.service.stream.ConnectionCheckResponse; | |||
import bubble.service.stream.StandardRuleEngineService; | |||
import lombok.Getter; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.cobbzilla.util.collection.ExpirationEvictionPolicy; | |||
import org.cobbzilla.util.collection.ExpirationMap; | |||
import org.cobbzilla.util.http.HttpContentEncodingType; | |||
import org.cobbzilla.util.network.NetworkUtil; | |||
import org.cobbzilla.wizard.cache.redis.RedisService; | |||
import org.glassfish.grizzly.http.server.Request; | |||
import org.glassfish.jersey.server.ContainerRequest; | |||
@@ -50,6 +55,7 @@ import static java.util.Collections.emptyMap; | |||
import static java.util.concurrent.TimeUnit.HOURS; | |||
import static java.util.concurrent.TimeUnit.MINUTES; | |||
import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; | |||
import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.*; | |||
import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; | |||
import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; | |||
@@ -75,6 +81,7 @@ public class FilterHttpResource { | |||
@Autowired private RedisService redis; | |||
@Autowired private BubbleConfiguration configuration; | |||
@Autowired private SelfNodeService selfNodeService; | |||
@Autowired private BlockStatsService blockStats; | |||
private static final long ACTIVE_REQUEST_TIMEOUT = HOURS.toSeconds(12); | |||
@@ -143,8 +150,15 @@ public class FilterHttpResource { | |||
} | |||
validateMitmCall(req); | |||
// if the requested IP is the same as our IP, then always passthru | |||
if (isForUs(connCheckRequest)) return ok(); | |||
// is the requested IP is the same as our IP? | |||
final boolean isLocalIp = isForLocalIp(connCheckRequest); | |||
if (isLocalIp) { | |||
// if it is for our host or net name, passthru | |||
if (connCheckRequest.hasFqdns() && (connCheckRequest.hasFqdn(getThisNode().getFqdn()) || connCheckRequest.hasFqdn(getThisNetwork().getNetworkDomain()))) { | |||
if (log.isDebugEnabled()) log.debug(prefix + "returning passthru for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); | |||
return ok(ConnectionCheckResponse.passthru); | |||
} | |||
} | |||
final String vpnAddr = connCheckRequest.getRemoteAddr(); | |||
final Device device = deviceIdService.findDeviceByIp(vpnAddr); | |||
@@ -154,13 +168,25 @@ public class FilterHttpResource { | |||
} else if (log.isTraceEnabled()) { | |||
log.trace(prefix+"found device "+device.id()+" for IP "+vpnAddr); | |||
} | |||
final Account account = findCaller(device.getAccount()); | |||
final String accountUuid = device.getAccount(); | |||
final Account account = findCaller(accountUuid); | |||
if (account == null) { | |||
if (log.isDebugEnabled()) log.debug(prefix+"account not found for uuid "+device.getAccount()+", returning not found"); | |||
if (log.isDebugEnabled()) log.debug(prefix+"account not found for uuid "+ accountUuid +", returning not found"); | |||
return notFound(); | |||
} | |||
final List<AppMatcher> matchers = matcherDAO.findByAccountAndEnabledAndConnCheck(device.getAccount()); | |||
if (isLocalIp) { | |||
if (showStats(accountUuid)) { | |||
// allow it for now | |||
if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showBlockStats==true) for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); | |||
return ok(ConnectionCheckResponse.noop); | |||
} else { | |||
if (log.isDebugEnabled()) log.debug(prefix + "returning block (showBlockStats==false) for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); | |||
return ok(ConnectionCheckResponse.block); | |||
} | |||
} | |||
final List<AppMatcher> matchers = getConnCheckMatchers(accountUuid); | |||
final List<AppMatcher> retained = new ArrayList<>(); | |||
for (AppMatcher matcher : matchers) { | |||
final BubbleApp app = appDAO.findByUuid(matcher.getApp()); | |||
@@ -170,25 +196,44 @@ public class FilterHttpResource { | |||
retained.add(matcher); | |||
} | |||
final String[] fqdns = connCheckRequest.getFqdns(); | |||
for (String fqdn : fqdns) { | |||
final ConnectionCheckResponse checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); | |||
if (checkResponse != ConnectionCheckResponse.noop) { | |||
if (log.isDebugEnabled()) log.debug(prefix + "returning "+checkResponse+" for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); | |||
return ok(checkResponse); | |||
ConnectionCheckResponse checkResponse = ConnectionCheckResponse.noop; | |||
if (connCheckRequest.hasFqdns()) { | |||
final String[] fqdns = connCheckRequest.getFqdns(); | |||
for (String fqdn : fqdns) { | |||
checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); | |||
if (checkResponse != ConnectionCheckResponse.noop) { | |||
if (log.isDebugEnabled()) log.debug(prefix + "found " + checkResponse + " (breaking) for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); | |||
break; | |||
} | |||
} | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning "+checkResponse+" for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); | |||
return ok(checkResponse); | |||
} else { | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning noop for NO fqdns, addr="+connCheckRequest.getAddr()); | |||
return ok(ConnectionCheckResponse.noop); | |||
} | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning noop for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); | |||
return ok(ConnectionCheckResponse.noop); | |||
} | |||
private boolean isForUs(FilterConnCheckRequest connCheckRequest) { | |||
final BubbleNode thisNode = selfNodeService.getThisNode(); | |||
return connCheckRequest.hasAddr() | |||
&& (thisNode.hasIp4() && thisNode.getIp4().equals(connCheckRequest.getAddr()) | |||
|| thisNode.hasIp6() && thisNode.getIp6().equals(connCheckRequest.getAddr())); | |||
private final Map<String, List<AppMatcher>> connCheckMatcherCache = new ExpirationMap<>(10, HOURS.toMillis(1), ExpirationEvictionPolicy.atime); | |||
public List<AppMatcher> getConnCheckMatchers(String accountUuid) { | |||
return connCheckMatcherCache.computeIfAbsent(accountUuid, k -> matcherDAO.findByAccountAndEnabledAndConnCheck(k)); | |||
} | |||
private boolean isForLocalIp(FilterConnCheckRequest connCheckRequest) { | |||
return connCheckRequest.hasAddr() && getConfiguredIps().contains(connCheckRequest.getAddr()); | |||
} | |||
private boolean isForLocalIp(FilterMatchersRequest matchersRequest) { | |||
return matchersRequest.hasServerAddr() && getConfiguredIps().contains(matchersRequest.getServerAddr()); | |||
} | |||
@Getter(lazy=true) private final Set<String> configuredIps = NetworkUtil.configuredIps(); | |||
@Getter(lazy=true) private final BubbleNode thisNode = selfNodeService.getThisNode(); | |||
@Getter(lazy=true) private final BubbleNetwork thisNetwork = selfNodeService.getThisNetwork(); | |||
public boolean showStats(String accountUuid) { return deviceIdService.doShowBlockStats(accountUuid); } | |||
@POST @Path(EP_MATCHERS+"/{requestId}") | |||
@Consumes(APPLICATION_JSON) | |||
@Produces(APPLICATION_JSON) | |||
@@ -208,13 +253,13 @@ public class FilterHttpResource { | |||
if (log.isDebugEnabled()) log.debug(prefix+"starting for filterRequest="+json(filterRequest, COMPACT_MAPPER)); | |||
else if (extraLog) log.error(prefix+"starting for filterRequest="+json(filterRequest, COMPACT_MAPPER)); | |||
if (!filterRequest.hasRemoteAddr()) { | |||
if (!filterRequest.hasClientAddr()) { | |||
if (log.isDebugEnabled()) log.debug(prefix+"no VPN address provided, returning no matchers"); | |||
else if (extraLog) log.error(prefix+"no VPN address provided, returning no matchers"); | |||
return ok(NO_MATCHERS); | |||
} | |||
final String vpnAddr = filterRequest.getRemoteAddr(); | |||
final String vpnAddr = filterRequest.getClientAddr(); | |||
final Device device = deviceIdService.findDeviceByIp(vpnAddr); | |||
if (device == null) { | |||
if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning no matchers"); | |||
@@ -225,9 +270,26 @@ public class FilterHttpResource { | |||
} | |||
filterRequest.setDevice(device.getUuid()); | |||
// if this is for a local ip, it's an automatic block | |||
// legitimate local requests would have been passthru and never reached here | |||
final boolean isLocalIp = isForLocalIp(filterRequest); | |||
final boolean showStats = showStats(device.getAccount()); | |||
if (isLocalIp) { | |||
if (filterRequest.isBrowser() && showStats) { | |||
blockStats.record(filterRequest, FilterMatchDecision.abort_not_found); | |||
} | |||
if (log.isDebugEnabled()) log.debug(prefix + "returning FORBIDDEN (showBlockStats=="+ showStats +")"); | |||
return forbidden(); | |||
} | |||
final FilterMatchersResponse response = getMatchersResponse(filterRequest, req, request); | |||
if (log.isDebugEnabled()) log.debug(prefix+"returning response: "+json(response, COMPACT_MAPPER)); | |||
else if (extraLog) log.error(prefix+"returning response: "+json(response, COMPACT_MAPPER)); | |||
if (filterRequest.isBrowser() && showStats) { | |||
blockStats.record(filterRequest, response.getDecision()); | |||
} | |||
return ok(response); | |||
} | |||
@@ -346,6 +408,7 @@ public class FilterHttpResource { | |||
public Response flushCaches(@Context ContainerRequest request) { | |||
final Account caller = userPrincipal(request); | |||
if (!caller.admin()) return forbidden(); | |||
connCheckMatcherCache.clear(); | |||
return ok(ruleEngine.flushCaches()); | |||
} | |||
@@ -520,10 +583,24 @@ public class FilterHttpResource { | |||
final FilterSubContext filterCtx = new FilterSubContext(req, requestId); | |||
if (!filterCtx.request.hasMatcher(matcherId)) throw notFoundEx(matcherId); | |||
final AppMatcher matcher = matcherDAO.findByAccountAndId(filterCtx.request.getAccount().getUuid(), matcherId); | |||
final Account account = filterCtx.request.getAccount(); | |||
account.setMtime(0); // only create one FilterDataResource | |||
final AppMatcher matcher = matcherDAO.findByAccountAndId(account.getUuid(), matcherId); | |||
if (matcher == null) throw notFoundEx(matcherId); | |||
return configuration.subResource(FilterDataResource.class, filterCtx.request.getAccount(), filterCtx.request.getDevice(), matcher); | |||
final Device device = filterCtx.request.getDevice(); | |||
device.setMtime(0); // only create one FilterDataResource | |||
return configuration.subResource(FilterDataResource.class, account, device, matcher); | |||
} | |||
@GET @Path(EP_STATUS+"/{requestId}") | |||
public Response getRequestStatus(@Context Request req, | |||
@Context ContainerRequest ctx, | |||
@PathParam("requestId") String requestId) { | |||
final FilterSubContext filterCtx = new FilterSubContext(req, requestId); | |||
final BlockStatsSummary summary = blockStats.getSummary(requestId); | |||
if (summary == null) return notFound(requestId); | |||
return ok(summary); | |||
} | |||
@Path(EP_ASSETS+"/{requestId}/{appId}") | |||
@@ -9,8 +9,10 @@ import lombok.Getter; | |||
import lombok.NoArgsConstructor; | |||
import lombok.Setter; | |||
import lombok.experimental.Accessors; | |||
import org.cobbzilla.util.http.HttpUtil; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.*; | |||
import static org.cobbzilla.util.http.HttpSchemes.stripScheme; | |||
@NoArgsConstructor @Accessors(chain=true) | |||
public class FilterMatchersRequest { | |||
@@ -24,16 +26,27 @@ public class FilterMatchersRequest { | |||
@Getter @Setter private String fqdn; | |||
@Getter @Setter private String uri; | |||
@Getter @Setter private String userAgent; | |||
@JsonIgnore public boolean isBrowser () { return HttpUtil.isBrowser(getUserAgent()); } | |||
@Getter @Setter private String referer; | |||
public boolean hasReferer () { return !empty(referer) && !referer.equals("NONE"); } | |||
@Getter @Setter private String remoteAddr; | |||
public boolean hasRemoteAddr() { return !empty(remoteAddr); } | |||
@JsonIgnore public String getRefererFqdn () { | |||
if (!hasReferer()) return null; | |||
final String base = stripScheme(referer); | |||
final int slashPos = base.indexOf('/'); | |||
return slashPos == -1 ? base : base.substring(0, slashPos); | |||
} | |||
@Getter @Setter private String clientAddr; | |||
public boolean hasClientAddr() { return !empty(clientAddr); } | |||
@Getter @Setter private String serverAddr; | |||
public boolean hasServerAddr() { return !empty(serverAddr); } | |||
// note: we do *not* include the requestId in the cache, if we did then the | |||
// FilterHttpResource.matchersCache cache would be useless, since every cache entry would be unique | |||
public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, remoteAddr); } | |||
public String cacheKey() { return hashOf(device, fqdn, uri, userAgent, referer, clientAddr, serverAddr); } | |||
@JsonIgnore public String getUrl() { return fqdn + uri; } | |||
@@ -90,7 +90,7 @@ public class ReverseProxyResource { | |||
.setUri(ub.getFullPath()) | |||
.setUserAgent(getUserAgent(request)) | |||
.setReferer(getReferer(request)) | |||
.setRemoteAddr(remoteHost) | |||
.setClientAddr(remoteHost) | |||
.setDevice(device.getUuid())) | |||
.setRequestId(id) | |||
.setDecision(FilterMatchDecision.match) | |||
@@ -41,6 +41,7 @@ import static bubble.ApiConstants.HOME_DIR; | |||
import static bubble.rule.RequestModifierRule.ICON_JS_TEMPLATE; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.die; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.empty; | |||
import static org.cobbzilla.util.io.FileUtil.abs; | |||
import static org.cobbzilla.util.io.regex.RegexReplacementFilter.DEFAULT_PREFIX_REPLACEMENT_WITH_MATCH; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
import static org.cobbzilla.util.security.ShaUtil.sha256_hex; | |||
@@ -118,13 +119,20 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { | |||
} | |||
protected String getSiteJsTemplate (String defaultSiteTemplate) { | |||
if (configuration.getEnvironment().containsKey("DEBUG_JS_SITE_TEMPLATES")) { | |||
final File jsTemplateFile = new File(HOME_DIR + "/siteJsTemplates/" + requestModConfig().getSiteJsTemplate()); | |||
if (jsTemplateFile.exists()) { | |||
return FileUtil.toStringOrDie(jsTemplateFile); | |||
return loadTemplate(defaultSiteTemplate, requestModConfig().getSiteJsTemplate()); | |||
} | |||
protected String loadTemplate(String defaultTemplate, String templatePath) { | |||
if (configuration.getEnvironment().containsKey("DEBUG_RULE_TEMPLATES")) { | |||
final File templateFile = new File(HOME_DIR + "/debugTemplates/" + templatePath); | |||
if (templateFile.exists()) { | |||
log.error("loadTemplate: debug file found (using it): "+abs(templateFile)); | |||
return FileUtil.toStringOrDie(templateFile); | |||
} else { | |||
log.error("loadTemplate: debug file not found (using default): "+abs(templateFile)); | |||
} | |||
} | |||
return defaultSiteTemplate; | |||
return defaultTemplate; | |||
} | |||
private RequestModifierConfig requestModConfig() { | |||
@@ -79,6 +79,16 @@ public interface AppRuleDriver { | |||
Account account, | |||
Device device) {} | |||
default void initQuick(JsonNode config, | |||
JsonNode userConfig, | |||
BubbleApp app, | |||
AppRule rule, | |||
AppMatcher matcher, | |||
Account account, | |||
Device device) { | |||
init(config, userConfig, app, rule, matcher, account, device); | |||
} | |||
default FilterMatchDecision preprocess(AppRuleHarness ruleHarness, | |||
FilterMatchersRequest filter, | |||
Account account, | |||
@@ -160,4 +170,6 @@ public interface AppRuleDriver { | |||
return sageRuleConfig; | |||
} | |||
default Object readData(String id) { return null; } | |||
} |
@@ -36,7 +36,7 @@ public class TrafficRecord { | |||
setAccountUuid(account == null ? null : account.getUuid()); | |||
setDeviceName(device == null ? null : device.getName()); | |||
setDeviceUuid(device == null ? null : device.getUuid()); | |||
setIp(filter.getRemoteAddr()); | |||
setIp(filter.getServerAddr()); | |||
setFqdn(filter.getFqdn()); | |||
setUri(filter.getUri()); | |||
setUserAgent(filter.getUserAgent()); | |||
@@ -57,7 +57,7 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver implements | |||
private final static Map<String, BlockListSource> blockListCache = new ConcurrentHashMap<>(); | |||
public boolean showStats() { return deviceService.doShowBlockStats(account); } | |||
public boolean showStats() { return deviceService.doShowBlockStats(account.getUuid()); } | |||
@Override public <C> Class<C> getConfigClass() { return (Class<C>) BubbleBlockConfig.class; } | |||
@@ -75,10 +75,20 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver implements | |||
AppMatcher matcher, | |||
Account account, | |||
Device device) { | |||
super.init(config, userConfig, app, rule, matcher, account, device); | |||
initQuick(config, userConfig, app, rule, matcher, account, device); | |||
refreshBlockLists(); | |||
} | |||
@Override public void initQuick(JsonNode config, | |||
JsonNode userConfig, | |||
BubbleApp app, | |||
AppRule rule, | |||
AppMatcher matcher, | |||
Account account, | |||
Device device) { | |||
super.init(config, userConfig, app, rule, matcher, account, device); | |||
} | |||
@Override public JsonNode upgradeRuleConfig(JsonNode sageRuleConfig, | |||
JsonNode localRuleConfig) { | |||
final BubbleBlockConfig sageConfig = json(sageRuleConfig, getConfigClass()); | |||
@@ -344,18 +354,24 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver implements | |||
return in; | |||
} | |||
if (bubbleBlockConfig.inPageBlocks() && showStats) { | |||
return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, BUBBLE_JS_STATS_TEMPLATE, BLOCK_STATS_JS, showStats); | |||
return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, getBubbleJsStatsTemplate(), BLOCK_STATS_JS, showStats); | |||
} | |||
if (bubbleBlockConfig.inPageBlocks()) { | |||
return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_TEMPLATE, EMPTY, BLOCK_STATS_JS, showStats); | |||
} | |||
log.warn(prefix+"inserting JS for stats..."); | |||
return filterInsertJs(in, filterRequest, filterCtx, BUBBLE_JS_STATS_TEMPLATE, null, null, showStats); | |||
return filterInsertJs(in, filterRequest, filterCtx, getBubbleJsStatsTemplate(), null, null, showStats); | |||
} | |||
protected String getBubbleJsStatsTemplate () { | |||
return loadTemplate(BUBBLE_JS_STATS_TEMPLATE, BUBBLE_STATS_TEMPLATE_NAME); | |||
} | |||
public static final Class<BubbleBlockRuleDriver> BB = BubbleBlockRuleDriver.class; | |||
public static final String BUBBLE_JS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+".js.hbs"); | |||
public static final String BUBBLE_JS_STATS_TEMPLATE = stream2string(getPackagePath(BB)+"/"+ BB.getSimpleName()+"_stats.js.hbs"); | |||
public static final String BUBBLE_STATS_TEMPLATE_NAME = BB.getSimpleName() + "_stats.js.hbs"; | |||
public static final String BUBBLE_JS_STATS_TEMPLATE = stream2string(getPackagePath(BB) + "/" + BUBBLE_STATS_TEMPLATE_NAME); | |||
private static final String CTX_BUBBLE_SELECTORS = "BUBBLE_SELECTORS_JSON"; | |||
private static final String CTX_BUBBLE_BLACKLIST = "BUBBLE_BLACKLIST_JSON"; | |||
@@ -37,6 +37,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.empty; | |||
import static org.cobbzilla.util.io.FileUtil.abs; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
import static org.cobbzilla.util.network.NetworkUtil.IPv4_LOCALHOST; | |||
import static org.cobbzilla.util.system.OutOfMemoryErrorUncaughtExceptionHandler.EXIT_ON_OOME; | |||
@NoArgsConstructor @Slf4j | |||
public class BubbleServer extends RestServerBase<BubbleConfiguration> { | |||
@@ -78,6 +79,7 @@ public class BubbleServer extends RestServerBase<BubbleConfiguration> { | |||
public static void main(String[] args) throws Exception { | |||
SLF4JBridgeHandler.removeHandlersForRootLogger(); | |||
SLF4JBridgeHandler.install(); | |||
Thread.setDefaultUncaughtExceptionHandler(EXIT_ON_OOME); | |||
final Map<String, String> env = loadEnvironment(args); | |||
final ConfigurationSource configSource = getConfigurationSource(); | |||
@@ -142,4 +144,5 @@ public class BubbleServer extends RestServerBase<BubbleConfiguration> { | |||
public static ConfigurationSource getConfigurationSource() { | |||
return getStreamConfigurationSource(BubbleServer.class, API_CONFIG_YML); | |||
} | |||
} |
@@ -15,6 +15,7 @@ import bubble.server.BubbleConfiguration; | |||
import bubble.service.boot.SelfNodeService; | |||
import bubble.service.cloud.DeviceIdService; | |||
import bubble.service.cloud.NetworkMonitorService; | |||
import bubble.service.stream.AppDataCleaner; | |||
import bubble.service.stream.AppPrimerService; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.cobbzilla.wizard.server.RestServer; | |||
@@ -106,11 +107,13 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase<Bub | |||
} | |||
// ensure default devices exist, apps are primed and device security levels are set | |||
// and start AppDataCleaner | |||
if (thisNode != null) { | |||
final BubbleNetwork thisNetwork = c.getThisNetwork(); | |||
if (thisNetwork != null && thisNetwork.getInstallType() == AnsibleInstallType.node) { | |||
c.getBean(AppPrimerService.class).primeApps(); | |||
c.getBean(DeviceIdService.class).initDeviceSecurityLevels(); | |||
c.getBean(AppDataCleaner.class).start(); | |||
} | |||
} | |||
@@ -0,0 +1,105 @@ | |||
package bubble.service.block; | |||
import bubble.resources.stream.FilterMatchersRequest; | |||
import bubble.rule.FilterMatchDecision; | |||
import lombok.Getter; | |||
import lombok.Setter; | |||
import lombok.extern.slf4j.Slf4j; | |||
import java.util.Map; | |||
import java.util.concurrent.ConcurrentHashMap; | |||
import java.util.concurrent.atomic.AtomicReference; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.now; | |||
@Slf4j | |||
public class BlockStatRecord { | |||
@Getter @Setter private String requestId; | |||
@Getter @Setter private String device; | |||
@Getter @Setter private String referer; | |||
@Getter @Setter private String fqdn; | |||
@Getter @Setter private String url; | |||
@Getter @Setter private String userAgent; | |||
@Getter @Setter private FilterMatchDecision decision; | |||
@Getter private final long ctime = now(); | |||
@Getter @Setter private long mtime = now(); | |||
private final AtomicReference<Map<String, String>> parentRecords = new AtomicReference<>(); | |||
private final AtomicReference<Map<String, String>> subRecords = new AtomicReference<>(); | |||
private final AtomicReference<BlockStatsSummary> summaryRef = new AtomicReference<>(); | |||
public BlockStatRecord(FilterMatchersRequest filter, FilterMatchDecision decision) { | |||
this.requestId = filter.getRequestId(); | |||
this.device = filter.getDevice(); | |||
this.referer = filter.getReferer(); | |||
this.fqdn = filter.getFqdn(); | |||
this.url = filter.getUrl(); | |||
this.userAgent = filter.getUserAgent(); | |||
this.decision = decision; | |||
} | |||
public void addSubRecord(BlockStatRecord rec) { | |||
synchronized (subRecords) { | |||
if (subRecords.get() == null) { | |||
subRecords.set(new ConcurrentHashMap<>()); | |||
} | |||
} | |||
subRecords.get().put(rec.getRequestId(), rec.getRequestId()); | |||
touch(); | |||
} | |||
public void touch() { mtime = now(); } | |||
public void addParentRecord(BlockStatRecord parent, Map<String, BlockStatRecord> records) { | |||
synchronized (parentRecords) { | |||
if (parentRecords.get() == null) { | |||
parentRecords.set(new ConcurrentHashMap<>()); | |||
} | |||
parentRecords.get().put(parent.getRequestId(), parent.getRequestId()); | |||
touchParents(records); | |||
} | |||
} | |||
private void touchParents(Map<String, BlockStatRecord> records) { | |||
touch(); | |||
synchronized (parentRecords) { | |||
if (parentRecords.get() == null) return; | |||
for (String p : parentRecords.get().keySet()) { | |||
final BlockStatRecord parent = records.get(p); | |||
if (parent != null) { | |||
parent.touchParents(records); | |||
} | |||
} | |||
} | |||
} | |||
public BlockStatsSummary summarize(Map<String, BlockStatRecord> records) { | |||
synchronized (summaryRef) { | |||
BlockStatsSummary sum = summaryRef.get(); | |||
if (sum != null && sum.getCtime() > getMtime()) { | |||
log.info("summarize("+url+"): reusing existing summary"); | |||
return sum; | |||
} else { | |||
log.info("summarize("+url+"): creating new summary"); | |||
sum = new BlockStatsSummary(); | |||
summaryRef.set(sum); | |||
} | |||
return summarize(records, sum); | |||
} | |||
} | |||
private BlockStatsSummary summarize(Map<String, BlockStatRecord> records, BlockStatsSummary summary) { | |||
final Map<String, String> subRecs = subRecords.get(); | |||
if (subRecs != null) { | |||
for (String subRecRequestId : subRecs.values()) { | |||
final BlockStatRecord subRec = records.get(subRecRequestId); | |||
if (subRec.decision.isAbort()) { | |||
summary.addBlock(subRec); | |||
} else { | |||
subRec.summarize(records, summary); | |||
} | |||
} | |||
} | |||
return summary; | |||
} | |||
} |
@@ -0,0 +1,69 @@ | |||
package bubble.service.block; | |||
import bubble.resources.stream.FilterMatchersRequest; | |||
import bubble.rule.FilterMatchDecision; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.cobbzilla.util.collection.ExpirationEvictionPolicy; | |||
import org.cobbzilla.util.collection.ExpirationMap; | |||
import org.springframework.stereotype.Service; | |||
import static java.util.concurrent.TimeUnit.MINUTES; | |||
import static org.cobbzilla.util.http.HttpSchemes.stripScheme; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
@Service @Slf4j | |||
public class BlockStatsService { | |||
private final ExpirationMap<String, BlockStatRecord> records | |||
= new ExpirationMap<>(200, MINUTES.toMillis(10), ExpirationEvictionPolicy.atime); | |||
public void flush () { records.clear(); } | |||
public void record(FilterMatchersRequest filter, FilterMatchDecision decision) { | |||
final BlockStatRecord newRec = new BlockStatRecord(filter, decision); | |||
synchronized (records) { | |||
records.put(getUrlCacheKey(filter), newRec); | |||
records.put(filter.getFqdn(), newRec); | |||
records.put(filter.getRequestId(), newRec); | |||
} | |||
log.info("record: stored keys("+getUrlCacheKey(filter)+", "+filter.getRequestId()+")= newRec="+json(newRec)); | |||
if (!filter.hasReferer()) { | |||
// this must be a top-level request | |||
log.info("record: added top-level record for device="+filter.getDevice()+"/userAgent="+filter.getUserAgent()+"/url="+filter.getUrl()); | |||
} else { | |||
// find match based on device + user-agent + referer | |||
final String cacheKey = getRefererCacheKey(filter); | |||
BlockStatRecord rec = records.get(cacheKey); | |||
if (rec == null) { | |||
// try fqdn | |||
rec = records.get(filter.getRefererFqdn()); | |||
if (rec == null) { | |||
log.warn("record: rec not found for device=" + filter.getDevice() + "/userAgent=" + filter.getUserAgent() + "/referer=" + filter.getReferer()); | |||
return; | |||
} | |||
} | |||
newRec.addParentRecord(rec, records); | |||
rec.addSubRecord(newRec); | |||
} | |||
} | |||
public String getRefererCacheKey(FilterMatchersRequest filter) { | |||
return filter.getDevice()+"\n"+filter.getUserAgent()+"\n"+stripScheme(filter.getReferer()); | |||
} | |||
public String getUrlCacheKey(FilterMatchersRequest filter) { | |||
return filter.getDevice()+"\n"+filter.getUserAgent()+"\n"+stripScheme(filter.getUrl()); | |||
} | |||
public BlockStatsSummary getSummary(String requestId) { | |||
final BlockStatRecord stat = records.get(requestId); | |||
if (stat == null) { | |||
log.info("getSummary("+requestId+") no summary found"); | |||
return null; | |||
} | |||
final BlockStatsSummary summary = stat.summarize(records); | |||
log.info("getSummary("+requestId+") returning summary="+json(summary)); | |||
return summary; | |||
} | |||
} |
@@ -0,0 +1,49 @@ | |||
package bubble.service.block; | |||
import lombok.AllArgsConstructor; | |||
import lombok.EqualsAndHashCode; | |||
import lombok.Getter; | |||
import java.util.HashMap; | |||
import java.util.Map; | |||
import java.util.Set; | |||
import java.util.TreeSet; | |||
import java.util.concurrent.atomic.AtomicInteger; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.now; | |||
public class BlockStatsSummary { | |||
private final Map<String, AtomicInteger> blocks = new HashMap<>(); | |||
@Getter private final long ctime = now(); | |||
public void addBlock(BlockStatRecord rec) { | |||
final AtomicInteger ct = blocks.computeIfAbsent(rec.getFqdn(), k -> new AtomicInteger(0)); | |||
ct.incrementAndGet(); | |||
} | |||
public Set<FqdnBlockCount> getBlocks () { | |||
final Set<FqdnBlockCount> set = new TreeSet<>(); | |||
for (Map.Entry<String, AtomicInteger> entry : blocks.entrySet()) { | |||
final int count = entry.getValue().get(); | |||
if (count > 0) set.add(new FqdnBlockCount(entry.getKey(), count)); | |||
} | |||
return set; | |||
} | |||
public int getTotal () { | |||
int total = 0; | |||
for (AtomicInteger ct : blocks.values()) total += ct.get(); | |||
return total; | |||
} | |||
@Override public String toString () { return "BlockStatsSummary{total="+getTotal()+"}"; } | |||
@AllArgsConstructor @EqualsAndHashCode(of={"fqdn"}) | |||
private static class FqdnBlockCount implements Comparable<FqdnBlockCount> { | |||
@Getter private final String fqdn; | |||
@Getter private final int count; | |||
@Override public int compareTo(FqdnBlockCount o) { return Integer.compare(o.count, count); } | |||
} | |||
} |
@@ -20,6 +20,7 @@ public interface DeviceIdService { | |||
void setDeviceSecurityLevel(Device device); | |||
void initBlockStats (Account account); | |||
default boolean doShowBlockStats(String accountUuid) { return false; } | |||
DeviceStatus getDeviceStatus(String deviceUuid); | |||
DeviceStatus getLiveDeviceStatus(String deviceUuid); | |||
@@ -56,6 +56,9 @@ public class StandardDeviceIdService implements DeviceIdService { | |||
// used in dnscrypt-proxy to determine how to respond to blocked requests | |||
public static final String REDIS_KEY_DEVICE_REJECT_WITH = "bubble_device_reject_with_"; | |||
// used in mitmproxy to determine how to respond to blocked requests | |||
public static final String REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = "bubble_device_showBlockStats_"; | |||
// used in mitmproxy to optimize passthru requests | |||
// we flush keys with this prefix when changing showBlockStats flag | |||
public static final String REDIS_KEY_CHUNK_FILTER_PASS = "__chunk_filter_pass__"; | |||
@@ -168,8 +171,8 @@ public class StandardDeviceIdService implements DeviceIdService { | |||
} | |||
} | |||
public boolean doShowBlockStats(Account account) { | |||
return Boolean.parseBoolean(redis.get_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS + account.getUuid())); | |||
public boolean doShowBlockStats(String accountUuid) { | |||
return Boolean.parseBoolean(redis.get_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS + accountUuid)); | |||
} | |||
public void showBlockStats (Device device) { | |||
@@ -183,12 +186,14 @@ public class StandardDeviceIdService implements DeviceIdService { | |||
return; | |||
} | |||
for (String ip : findIpsByDevice(device.getUuid())) { | |||
redis.set_plaintext(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip, Boolean.toString(true)); | |||
redis.set_plaintext(REDIS_KEY_DEVICE_REJECT_WITH + ip, privateIp); | |||
} | |||
} | |||
public void hideBlockStats (Device device) { | |||
for (String ip : findIpsByDevice(device.getUuid())) { | |||
redis.del_withPrefix(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip); | |||
redis.del_withPrefix(REDIS_KEY_DEVICE_REJECT_WITH + ip); | |||
} | |||
} | |||
@@ -0,0 +1,31 @@ | |||
package bubble.service.stream; | |||
import bubble.dao.app.AppDataDAO; | |||
import lombok.Getter; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.cobbzilla.util.daemon.SimpleDaemon; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import static java.util.concurrent.TimeUnit.HOURS; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.now; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; | |||
import static org.cobbzilla.wizard.server.RestServerBase.reportError; | |||
@Service @Slf4j | |||
public class AppDataCleaner extends SimpleDaemon { | |||
@Getter(lazy=true) private final long sleepTime = HOURS.toMillis(4); | |||
@Autowired private AppDataDAO dataDAO; | |||
@Override protected void process() { | |||
try { | |||
final int ct = dataDAO.bulkDeleteWhere("expiration < " + now()); | |||
log.info("process: removed " + ct + " expired AppData records"); | |||
} catch (Exception e) { | |||
reportError("AppDataCleaner.process: "+shortError(e), e); | |||
} | |||
} | |||
} |
@@ -55,6 +55,7 @@ jersey: | |||
- org.cobbzilla.wizard.filters | |||
providerPackages: | |||
- org.cobbzilla.wizard.exceptionmappers | |||
- bubble.exceptionmappers | |||
requestFilters: | |||
- bubble.auth.BubbleAuthFilter | |||
- bubble.filters.BubbleRateLimitFilter | |||
@@ -10,6 +10,24 @@ if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { | |||
} | |||
} | |||
{{PAGE_PREFIX}}_getAppIconImgSrc = function (app) { | |||
return '/__bubble/api/filter/assets/{{BUBBLE_REQUEST_ID}}/' + app.app + '/' + app.icon + '?raw=true'; | |||
} | |||
{{PAGE_PREFIX}}_getAppIconImgId = function (app) { | |||
return app.jsPrefix + '_app_icon_img'; | |||
} | |||
{{PAGE_PREFIX}}_setAppIconImg = function (app) { | |||
const imgId = {{PAGE_PREFIX}}_getAppIconImgId(app); | |||
const img = document.getElementById(imgId); | |||
if (img) { | |||
img.src = {{PAGE_PREFIX}}_getAppIconImgSrc(app); | |||
} else { | |||
console.warn('setAppIconImg: img element not found: '+imgId) | |||
} | |||
} | |||
function {{PAGE_PREFIX}}_onReady(callback) { | |||
const intervalId = window.setInterval(function() { | |||
if (document.getElementsByTagName('body')[0] !== undefined) { | |||
@@ -29,15 +47,17 @@ if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { | |||
bubbleControlDiv.style.position = 'fixed'; | |||
bubbleControlDiv.style.bottom = '0'; | |||
bubbleControlDiv.style.right = '0'; | |||
bubbleControlDiv.style.zIndex = '2147483647'; | |||
bubbleControlDiv.style.zIndex = '2147483640'; | |||
document.getElementsByTagName('body')[0].appendChild(bubbleControlDiv); | |||
} | |||
for (let i=0; i<{{PAGE_PREFIX}}_icon_status.length; i++) { | |||
const iconSpecs = {{PAGE_PREFIX}}_icon_status[i]; | |||
let br = document.createElement('br'); | |||
let link = document.createElement('a'); | |||
link.href = '{{{BUBBLE_HOME}}}/app/' + {{PAGE_PREFIX}}_icon_status[i].app + '/' + {{PAGE_PREFIX}}_icon_status[i].link; | |||
link.href = '{{{BUBBLE_HOME}}}/app/' + iconSpecs.app + '/' + iconSpecs.link; | |||
let img = document.createElement('img'); | |||
img.src = '/__bubble/api/filter/assets/{{BUBBLE_REQUEST_ID}}/' + {{PAGE_PREFIX}}_icon_status[i].app + '/' + {{PAGE_PREFIX}}_icon_status[i].icon + '?raw=true'; | |||
img.id = {{PAGE_PREFIX}}_getAppIconImgId(iconSpecs); | |||
img.src = {{PAGE_PREFIX}}_getAppIconImgSrc(iconSpecs); | |||
img.width = 64; | |||
link.appendChild(img); | |||
bubbleControlDiv.appendChild(br); | |||
@@ -1,8 +1,51 @@ | |||
{{{ICON_JS}}} | |||
{{PAGE_PREFIX}}_addBubbleApp({ | |||
const {{JS_PREFIX}}_app = { | |||
jsPrefix: '{{JS_PREFIX}}', | |||
app: '{{BUBBLE_APP_NAME}}', | |||
link: 'view/last_24_hours', | |||
icon: 'icon-gray' | |||
}); | |||
}; | |||
{{PAGE_PREFIX}}_addBubbleApp({{JS_PREFIX}}_app); | |||
let {{JS_PREFIX}}_app_stats_ctime = 0; | |||
let {{JS_PREFIX}}_app_stats_last_ctime_change = 0; | |||
const {{JS_PREFIX}}_app_stats_timeout = 35000; | |||
const {{JS_PREFIX}}_app_refresh_interval = window.setInterval(function () { | |||
const requestOptions = { method: 'GET' }; | |||
const block_stats_url = '/__bubble/api/filter/status/{{BUBBLE_REQUEST_ID}}'; | |||
fetch(block_stats_url, requestOptions) | |||
.then(resp => { | |||
try { | |||
return resp.json(); | |||
} catch (error) { | |||
console.log('cancelling window.interval, response not json'); | |||
window.clearInterval({{JS_PREFIX}}_app_refresh_interval); | |||
} | |||
}) | |||
.then(data => { | |||
console.log('stats = '+JSON.stringify(data)); | |||
let icon = null; | |||
if ((typeof data.total !== 'undefined') && (typeof data.ctime !== 'undefined')) { | |||
if (data.ctime != {{JS_PREFIX}}_app_stats_ctime) { | |||
if (data.total === 0) { | |||
icon = 'icon-green'; | |||
} else if (data.total < 5) { | |||
icon = 'icon-yellow'; | |||
} else { | |||
icon = 'icon-red'; | |||
} | |||
{{JS_PREFIX}}_app.icon = icon; | |||
{{PAGE_PREFIX}}_setAppIconImg({{JS_PREFIX}}_app); | |||
{{JS_PREFIX}}_app_stats_ctime = data.ctime; | |||
{{JS_PREFIX}}_app_stats_last_ctime_change = Date.now(); | |||
} else if (Date.now() - {{JS_PREFIX}}_app_stats_last_ctime_change > {{JS_PREFIX}}_app_stats_timeout) { | |||
console.log('cancelling window.interval, stats unchanged for a while'); | |||
window.clearInterval({{JS_PREFIX}}_app_refresh_interval); | |||
} | |||
} | |||
}).catch((error) => { | |||
console.log('cancelling window.interval, due to error: '+error); | |||
window.clearInterval({{JS_PREFIX}}_app_refresh_interval); | |||
}); | |||
}, 5000); |
@@ -100,14 +100,24 @@ DEBUG_MATCHER = { | |||
'rule': DEBUG_MATCHER_NAME | |||
}] | |||
} | |||
BLOCK_MATCHER = { | |||
'decision': 'abort_not_found', | |||
'matchers': [{ | |||
'name': 'BLOCK_MATCHER', | |||
'contentTypeRegex': '.*', | |||
"urlRegex": ".*", | |||
'rule': 'BLOCK_MATCHER' | |||
}] | |||
} | |||
def bubble_matchers(req_id, remote_addr, flow, host): | |||
def bubble_matchers(req_id, client_addr, server_addr, flow, host): | |||
if debug_capture_fqdn and host and host in debug_capture_fqdn: | |||
bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host) | |||
return DEBUG_MATCHER | |||
headers = { | |||
'X-Forwarded-For': remote_addr, | |||
'X-Forwarded-For': client_addr, | |||
'Accept' : 'application/json', | |||
'Content-Type': 'application/json' | |||
} | |||
@@ -134,11 +144,15 @@ def bubble_matchers(req_id, remote_addr, flow, host): | |||
'uri': flow.request.path, | |||
'userAgent': user_agent, | |||
'referer': referer, | |||
'remoteAddr': remote_addr | |||
'clientAddr': client_addr, | |||
'serverAddr': server_addr | |||
} | |||
response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data) | |||
if response.ok: | |||
return response.json() | |||
elif response.status_code == 403: | |||
bubble_log('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text)) | |||
return BLOCK_MATCHER | |||
bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) | |||
except Exception as e: | |||
bubble_log('bubble_matchers API call failed: '+repr(e)) | |||
@@ -28,7 +28,7 @@ from mitmproxy.exceptions import TlsProtocolException | |||
from mitmproxy.net import tls as net_tls | |||
from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set | |||
from bubble_config import bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host | |||
from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host | |||
from bubble_vpn4 import wireguard_network_ipv4 | |||
from bubble_vpn6 import wireguard_network_ipv6 | |||
from netaddr import IPAddress, IPNetwork | |||
@@ -41,6 +41,7 @@ REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' | |||
REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout | |||
REDIS_KEY_DEVICE_SECURITY_LEVEL_PREFIX = 'bubble_device_security_level_' # defined in StandardDeviceIdService | |||
REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX = 'bubble_device_site_max_security_level_' # defined in StandardDeviceIdService | |||
REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = 'bubble_device_showBlockStats_' | |||
FORCE_PASSTHRU = {'passthru': True} | |||
FORCE_BLOCK = {'block': True} | |||
@@ -77,6 +78,12 @@ def get_device_security_level(client_addr, fqdns): | |||
return {'level': level} | |||
def show_block_stats(client_addr): | |||
show = REDIS.get(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS+client_addr) | |||
if show is None: | |||
return False | |||
return show.decode() == 'true' | |||
def get_local_ips(): | |||
global local_ips | |||
if local_ips is None: | |||
@@ -86,8 +93,13 @@ def get_local_ips(): | |||
return local_ips | |||
def is_bubble_request(ip, fqdns): | |||
# return ip in get_local_ips() | |||
return ip in get_local_ips() and (bubble_host in fqdns or bubble_host_alias in fqdns) | |||
def is_sage_request(ip, fqdns): | |||
return ip == bubble_sage_ip4 or ip == bubble_sage_ip6 or bubble_sage_host in fqdns | |||
return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns | |||
def is_not_from_vpn(client_addr): | |||
@@ -134,6 +146,10 @@ class TlsFeedback(TlsLayer): | |||
super(TlsFeedback, self)._establish_tls_with_client() | |||
except TlsProtocolException as e: | |||
if self.do_block: | |||
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address) | |||
raise e | |||
tb = traceback.format_exc() | |||
if 'OpenSSL.SSL.ZeroReturnError' in tb: | |||
bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address) | |||
@@ -225,8 +241,13 @@ def next_layer(next_layer): | |||
no_fqdns = fqdns is None or len(fqdns) == 0 | |||
security_level = get_device_security_level(client_addr, fqdns) | |||
next_layer.security_level = security_level | |||
if server_addr in get_local_ips(): | |||
bubble_log('next_layer: enabling passthru for LOCAL server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) | |||
next_layer.do_block = False | |||
if is_bubble_request(server_addr, fqdns): | |||
bubble_log('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns)) | |||
check = FORCE_PASSTHRU | |||
elif is_sage_request(server_addr, fqdns): | |||
bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) | |||
check = FORCE_PASSTHRU | |||
elif is_not_from_vpn(client_addr): | |||
@@ -235,10 +256,6 @@ def next_layer(next_layer): | |||
next_layer.__class__ = TlsBlock | |||
return | |||
elif is_sage_request(server_addr, fqdns): | |||
bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) | |||
check = FORCE_PASSTHRU | |||
elif security_level['level'] == SEC_OFF or security_level['level'] == SEC_BASIC: | |||
bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr) | |||
check = FORCE_PASSTHRU | |||
@@ -268,7 +285,11 @@ def next_layer(next_layer): | |||
elif 'block' in check and check['block']: | |||
bubble_log('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns)) | |||
bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns) | |||
next_layer.__class__ = TlsBlock | |||
if show_block_stats(client_addr): | |||
next_layer.do_block = True | |||
next_layer.__class__ = TlsFeedback | |||
else: | |||
next_layer.__class__ = TlsBlock | |||
else: | |||
bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) | |||
@@ -19,7 +19,8 @@ class Rerouter: | |||
bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path) | |||
return None | |||
remote_addr = str(flow.client_conn.address[0]) | |||
client_addr = str(flow.client_conn.address[0]) | |||
server_addr = str(flow.server_conn.address[0]) | |||
try: | |||
host = host.decode() | |||
except (UnicodeDecodeError, AttributeError): | |||
@@ -35,10 +36,10 @@ class Rerouter: | |||
req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time()) | |||
bubble_log("get_matchers: requesting match decision for req_id="+req_id) | |||
resp = bubble_matchers(req_id, remote_addr, flow, host) | |||
resp = bubble_matchers(req_id, client_addr, server_addr, flow, host) | |||
if not resp: | |||
bubble_log('get_matchers: no response for remote_addr/host: '+remote_addr+'/'+str(host)) | |||
bubble_log('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host)) | |||
return None | |||
matchers = [] | |||
@@ -28,5 +28,6 @@ | |||
<context:component-scan base-package="bubble.auth"/> | |||
<context:component-scan base-package="bubble.service"/> | |||
<context:component-scan base-package="bubble.resources"/> | |||
<context:component-scan base-package="bubble.exceptionmappers"/> | |||
</beans> |
@@ -56,6 +56,7 @@ jersey: | |||
- org.cobbzilla.wizard.filters | |||
providerPackages: | |||
- org.cobbzilla.wizard.exceptionmappers | |||
- bubble.exceptionmappers | |||
requestFilters: [ bubble.auth.BubbleAuthFilter ] | |||
responseFilters: | |||
- org.cobbzilla.wizard.filters.ScrubbableScrubber | |||
@@ -1 +1 @@ | |||
Subproject commit d2360429c18bc9744ee3881182ae859fbcdc4c46 | |||
Subproject commit 81f18e0d525d551413b8987e76faca870073264e |
@@ -1 +1 @@ | |||
Subproject commit 37708799b3a0761f18589de02d2a0f754127b5cb | |||
Subproject commit 9240375cb73b47e43ad5e9a5d77f33dfef0f6925 |