From f7b3f9cd2a7588dad2ac50a581737a66b47d44c4 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sat, 5 Sep 2020 12:35:19 -0400 Subject: [PATCH 01/78] WIP. adding flex routing --- .../src/main/java/bubble/ApiConstants.java | 9 +- .../java/bubble/dao/account/AccountDAO.java | 4 +- .../main/java/bubble/dao/app/AppSiteDAO.java | 4 +- .../java/bubble/dao/device/DeviceDAO.java | 8 +- .../java/bubble/dao/device/FlexRouterDAO.java | 21 +++++ .../java/bubble/model/device/FlexRouter.java | 84 +++++++++++++++++++ .../resources/account/AccountsResource.java | 1 + .../resources/account/AuthResource.java | 6 +- .../bubble/resources/account/MeResource.java | 8 ++ .../resources/app/AppSitesResource.java | 6 +- .../bubble/resources/app/AppsResource.java | 6 +- .../{account => device}/DevicesResource.java | 14 ++-- .../resources/device/FlexRoutersResource.java | 62 ++++++++++++++ .../resources/stream/FilterHttpResource.java | 16 ++-- .../stream/ReverseProxyResource.java | 6 +- .../bubble/rule/AbstractAppRuleDriver.java | 4 +- .../rule/bblock/BubbleBlockRuleDriver.java | 12 +-- .../listener/NodeInitializerListener.java | 6 +- .../dbfilter/FilteredEntityIterator.java | 15 ++-- .../DeviceService.java} | 4 +- .../service/device/FlexRouterService.java | 58 +++++++++++++ .../StandardDeviceService.java} | 5 +- .../stream/StandardAppPrimerService.java | 8 +- ...ervice.java => DbFilterDeviceService.java} | 4 +- bubble-server/src/main/resources/messages | 2 +- utils/cobbzilla-wizard | 2 +- 26 files changed, 310 insertions(+), 65 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java create mode 100644 bubble-server/src/main/java/bubble/model/device/FlexRouter.java rename bubble-server/src/main/java/bubble/resources/{account => device}/DevicesResource.java (92%) create mode 100644 bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java rename bubble-server/src/main/java/bubble/service/{cloud/DeviceIdService.java => device/DeviceService.java} (93%) create mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterService.java rename bubble-server/src/main/java/bubble/service/{cloud/StandardDeviceIdService.java => device/StandardDeviceService.java} (98%) rename bubble-server/src/main/java/bubble/service_dbfilter/{DbFilterDeviceIdService.java => DbFilterDeviceService.java} (91%) diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index 63296b98..96594de9 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -176,6 +176,7 @@ public class ApiConstants { public static final String EP_NODES = "/nodes"; public static final String EP_DEVICES = "/devices"; public static final String EP_DEVICE_TYPES = "/deviceTypes"; + public static final String EP_FLEX_ROUTERS = "/flexRouters"; public static final String EP_MODEL = "/model"; public static final String EP_VPN = "/vpn"; public static final String EP_IPS = "/ips"; @@ -276,8 +277,7 @@ public class ApiConstants { } public static String getRemoteHost(Request req) { - final String xff = req.getHeader("X-Forwarded-For"); - final String remoteHost = xff == null ? req.getRemoteAddr() : xff; + final String remoteHost = getRemoteAddr(req); if (isPublicIpv4(remoteHost)) return remoteHost; final String publicIp = getFirstPublicIpv4(); if (publicIp != null) return publicIp; @@ -285,6 +285,11 @@ public class ApiConstants { return isPublicIpv4(externalIp) ? externalIp : remoteHost; } + public static String getRemoteAddr(Request req) { + final String xff = req.getHeader("X-Forwarded-For"); + return xff == null ? req.getRemoteAddr() : xff; + } + public static String getUserAgent(ContainerRequest ctx) { return ctx.getHeaderString(USER_AGENT); } public static String getReferer(ContainerRequest ctx) { return ctx.getHeaderString(REFERER); } diff --git a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java index 3a5df90b..84720653 100644 --- a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java +++ b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java @@ -23,7 +23,7 @@ import bubble.server.BubbleConfiguration; import bubble.service.SearchService; import bubble.service.account.SyncPasswordService; import bubble.service.boot.SelfNodeService; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import bubble.service.stream.RuleEngineService; import lombok.Getter; import lombok.NonNull; @@ -78,7 +78,7 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc @Autowired private SearchService searchService; @Autowired private SyncPasswordService syncPasswordService; @Autowired private ReferralCodeDAO referralCodeDAO; - @Autowired private DeviceIdService deviceService; + @Autowired private DeviceService deviceService; @Autowired private RuleEngineService ruleEngineService; public Account newAccount(Request req, Account caller, AccountRegistration request, Account parent) { diff --git a/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java index bdc5992f..72f4d79e 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppSiteDAO.java @@ -5,7 +5,7 @@ package bubble.dao.app; import bubble.model.app.AppSite; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import bubble.service.stream.RuleEngineService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; @@ -14,7 +14,7 @@ import org.springframework.stereotype.Repository; public class AppSiteDAO extends AppTemplateEntityDAO { @Autowired private RuleEngineService ruleEngineService; - @Autowired private DeviceIdService deviceService; + @Autowired private DeviceService deviceService; @Override public AppSite postCreate(AppSite site, Object context) { // todo: update entities based on this template if account has updates enabled diff --git a/bubble-server/src/main/java/bubble/dao/device/DeviceDAO.java b/bubble-server/src/main/java/bubble/dao/device/DeviceDAO.java index 4926ba34..14058e81 100644 --- a/bubble-server/src/main/java/bubble/dao/device/DeviceDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/DeviceDAO.java @@ -11,7 +11,7 @@ import bubble.dao.app.AppDataDAO; import bubble.model.device.BubbleDeviceType; import bubble.model.device.Device; import bubble.server.BubbleConfiguration; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.hibernate.criterion.Order; @@ -46,7 +46,7 @@ public class DeviceDAO extends AccountOwnedEntityDAO { @Autowired private BubbleConfiguration configuration; @Autowired private AppDataDAO dataDAO; @Autowired private TrustedClientDAO trustDAO; - @Autowired private DeviceIdService deviceIdService; + @Autowired private DeviceService deviceService; @Override public Order getDefaultSortOrder() { return ORDER_CTIME_ASC; } @@ -113,7 +113,7 @@ public class DeviceDAO extends AccountOwnedEntityDAO { result = super.update(uninitialized); } - deviceIdService.setDeviceSecurityLevel(result); + deviceService.setDeviceSecurityLevel(result); return result; } } @@ -125,7 +125,7 @@ public class DeviceDAO extends AccountOwnedEntityDAO { toUpdate.update(updateRequest); final var updated = super.update(toUpdate); - deviceIdService.setDeviceSecurityLevel(updated); + deviceService.setDeviceSecurityLevel(updated); refreshVpnUsers(); return updated; } diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java new file mode 100644 index 00000000..3827b0ab --- /dev/null +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -0,0 +1,21 @@ +package bubble.dao.device; + +import bubble.dao.account.AccountOwnedEntityDAO; +import bubble.model.device.FlexRouter; +import org.springframework.stereotype.Repository; + +import java.util.List; + +import static org.hibernate.criterion.Restrictions.*; + +@Repository +public class FlexRouterDAO extends AccountOwnedEntityDAO { + + public List findEnabledAndRegistered() { + return list(criteria().add(and( + eq("enabled", true), + ne("port", 0), + isNotNull("token")))); + } + +} diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java new file mode 100644 index 00000000..2db0f715 --- /dev/null +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -0,0 +1,84 @@ +package bubble.model.device; + +import bubble.model.account.Account; +import bubble.model.account.HasAccountNoName; +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.map.SingletonMap; +import org.cobbzilla.wizard.model.Identifiable; +import org.cobbzilla.wizard.model.IdentifiableBase; +import org.cobbzilla.wizard.model.entityconfig.EntityFieldMode; +import org.cobbzilla.wizard.model.entityconfig.EntityFieldType; +import org.cobbzilla.wizard.model.entityconfig.annotations.*; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Transient; + +import static bubble.ApiConstants.EP_FLEX_ROUTERS; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@Entity +@ECType(root=true) @ToString(of={"ip", "port"}) +@ECTypeURIs(baseURI=EP_FLEX_ROUTERS, listFields={"name", "enabled"}) +@NoArgsConstructor @Accessors(chain=true) @Slf4j +@ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) }) +public class FlexRouter extends IdentifiableBase implements HasAccountNoName { + + public static final String[] CREATE_FIELDS = { "ip", "enabled" }; + public static final String[] UPDATE_FIELDS = { "enabled" }; + + public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); } + + @Override public Identifiable update(Identifiable other) { copy(this, other, UPDATE_FIELDS); return this; } + + @ECSearchable(filter=true) @ECField(index=10) + @ECIndex @Column(nullable=false, length=500) + @Getter @Setter private String ip; + + @ECSearchable(filter=true) @ECField(index=20) + @ECIndex @Column(nullable=false) + @JsonIgnore @Getter @Setter private Integer port = 0; + + public String id () { return getIp() + "/" + getUuid(); } + + @ECSearchable @ECField(index=30) + @ECForeignKey(entity=Account.class) + @Column(nullable=false, updatable=false, length=UUID_MAXLEN) + @Getter @Setter private String account; + + @ECSearchable @ECField(index=40) + @ECIndex @Column(nullable=false) + @Getter @Setter private Boolean enabled = true; + public boolean enabled () { return bool(enabled); } + + @ECSearchable @ECField(index=50) + @ECIndex @Column(nullable=false) + @Getter @Setter private Boolean active = true; + public boolean active() { return bool(active); } + + @ECSearchable @ECField(index=60, type=EntityFieldType.epoch_time, mode=EntityFieldMode.readOnly) + @Getter @Setter private Long lastSeen; + public FlexRouter setLastSeen () { return setLastSeen(now()); } + + @JsonIgnore @Transient public long getAge () { return lastSeen == null ? Long.MAX_VALUE : now() - lastSeen; } + + @ECSearchable(filter=true) @ECField(index=70) + @ECIndex @Column(length=100) + @JsonIgnore @Getter @Setter private String token; + public boolean hasToken () { return !empty(token); } + + @Transient @Getter @Setter private String serverToken; + + public String pingUrl() { return "http://" + getIp() + ":" + getPort() + "/ping"; } + + public String pingObject() { return json(new SingletonMap("token", getToken())); } + +} diff --git a/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java b/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java index dba0ecd1..3ee43dda 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java @@ -19,6 +19,7 @@ import bubble.model.device.BubbleDeviceType; import bubble.resources.app.AppsResource; import bubble.resources.bill.*; import bubble.resources.cloud.*; +import bubble.resources.device.DevicesResource; import bubble.resources.driver.DriversResource; import bubble.resources.notify.ReceivedNotificationsResource; import bubble.resources.notify.SentNotificationsResource; diff --git a/bubble-server/src/main/java/bubble/resources/account/AuthResource.java b/bubble-server/src/main/java/bubble/resources/account/AuthResource.java index ecffb009..04bd32e6 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AuthResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AuthResource.java @@ -32,7 +32,7 @@ import bubble.service.bill.PromotionService; import bubble.service.boot.ActivationService; import bubble.service.boot.NodeManagerService; import bubble.service.boot.SageHelloService; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import bubble.service.cloud.GeoService; import bubble.service.notify.NotificationService; import bubble.service.upgrade.BubbleJarUpgradeService; @@ -97,7 +97,7 @@ public class AuthResource { @Autowired private BubbleConfiguration configuration; @Autowired private StandardAuthenticatorService authenticatorService; @Autowired private PromotionService promoService; - @Autowired private DeviceIdService deviceIdService; + @Autowired private DeviceService deviceService; @Autowired private DeviceDAO deviceDAO; @Autowired private BubbleNodeKeyDAO nodeKeyDAO; @Autowired private NodeManagerService nodeManagerService; @@ -675,7 +675,7 @@ public class AuthResource { } else { final String remoteHost = getRemoteHost(req); if (!empty(remoteHost)) { - final Device device = deviceIdService.findDeviceByIp(remoteHost); + final Device device = deviceService.findDeviceByIp(remoteHost); if (device != null) { type = device.getDeviceType().getCertType(); } diff --git a/bubble-server/src/main/java/bubble/resources/account/MeResource.java b/bubble-server/src/main/java/bubble/resources/account/MeResource.java index 74110400..6853a0c3 100644 --- a/bubble-server/src/main/java/bubble/resources/account/MeResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/MeResource.java @@ -19,6 +19,8 @@ import bubble.model.device.BubbleDeviceType; import bubble.resources.app.AppsResource; import bubble.resources.bill.*; import bubble.resources.cloud.*; +import bubble.resources.device.DevicesResource; +import bubble.resources.device.FlexRoutersResource; import bubble.resources.driver.DriversResource; import bubble.resources.notify.ReceivedNotificationsResource; import bubble.resources.notify.SentNotificationsResource; @@ -371,6 +373,12 @@ public class MeResource { return ok(BubbleDeviceType.getSelectableTypes()); } + @Path(EP_FLEX_ROUTERS) + public FlexRoutersResource getFlexRouters(@Context ContainerRequest ctx) { + final Account caller = userPrincipal(ctx); + return configuration.subResource(FlexRoutersResource.class, caller); + } + @Path(EP_REFERRAL_CODES) public ReferralCodesResource getReferralCodes(@Context ContainerRequest ctx) { final Account caller = userPrincipal(ctx); diff --git a/bubble-server/src/main/java/bubble/resources/app/AppSitesResource.java b/bubble-server/src/main/java/bubble/resources/app/AppSitesResource.java index 821d9876..ce1fc92c 100644 --- a/bubble-server/src/main/java/bubble/resources/app/AppSitesResource.java +++ b/bubble-server/src/main/java/bubble/resources/app/AppSitesResource.java @@ -12,7 +12,7 @@ import bubble.model.app.config.AppDataDriver; import bubble.model.app.config.AppDataView; import bubble.model.device.Device; import bubble.resources.account.AccountOwnedTemplateResource; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import org.cobbzilla.wizard.model.search.SearchQuery; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -32,7 +32,7 @@ public class AppSitesResource extends AccountOwnedTemplateResource { } @Override protected Device populate(ContainerRequest ctx, Device device) { - return device.setStatus(deviceIdService.getDeviceStatus(device.getUuid())); + return device.setStatus(deviceService.getDeviceStatus(device.getUuid())); } @Override protected List sort(List list, Request req, ContainerRequest ctx) { @@ -106,14 +108,14 @@ public class DevicesResource extends AccountOwnedResource { return configuration.subResource(VpnConfigResource.class, device); } - @Autowired private DeviceIdService deviceIdService; + @Autowired private DeviceService deviceService; @GET @Path("/{id}"+EP_IPS) public Response getIps(@Context ContainerRequest ctx, @PathParam("id") String id) { final Device device = getDao().findByAccountAndId(getAccountUuid(ctx), id); if (device == null) return notFound(id); - return ok(deviceIdService.findIpsByDevice(device.getUuid())); + return ok(deviceService.findIpsByDevice(device.getUuid())); } @GET @Path("/{id}"+EP_STATUS) @@ -121,7 +123,7 @@ public class DevicesResource extends AccountOwnedResource { @PathParam("id") String id) { final Device device = getDao().findByAccountAndId(getAccountUuid(ctx), id); if (device == null) return notFound(id); - return ok(deviceIdService.getLiveDeviceStatus(device.getUuid())); + return ok(deviceService.getLiveDeviceStatus(device.getUuid())); } } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java new file mode 100644 index 00000000..fce13183 --- /dev/null +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -0,0 +1,62 @@ +package bubble.resources.device; + +import bubble.dao.device.FlexRouterDAO; +import bubble.model.account.Account; +import bubble.model.device.Device; +import bubble.model.device.FlexRouter; +import bubble.resources.account.AccountOwnedResource; +import bubble.service.device.DeviceService; +import lombok.extern.slf4j.Slf4j; +import org.glassfish.grizzly.http.server.Request; +import org.glassfish.jersey.server.ContainerRequest; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; + +import static bubble.ApiConstants.EP_REGISTER; +import static bubble.ApiConstants.getRemoteAddr; +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.wizard.resources.ResourceUtil.*; + +@Slf4j +public class FlexRoutersResource extends AccountOwnedResource { + + @Autowired private DeviceService deviceService; + + public FlexRoutersResource(Account account) { super(account); } + + @Override protected boolean isReadOnly(ContainerRequest ctx) { + final Account caller = userPrincipal(ctx); + return !caller.admin(); + } + + @POST @Path("{id}"+EP_REGISTER) + public Response register(@Context Request req, + @Context ContainerRequest ctx, + @PathParam("id") String id, + FlexRouter request) { + // caller must be admin + if (isReadOnly(ctx)) return forbidden(); + + // caller must come from a valid device + final String remoteAddr = getRemoteAddr(req); + final Device device = deviceService.findDeviceByIp(remoteAddr); + if (device == null) return invalid("err.device.notFound"); + + final FlexRouter flexRouter = getDao().findByUuid(id); + if (flexRouter == null) return notFound(id); + + // set token and return + final String token = randomAlphanumeric(50); + final FlexRouter updated = getDao().update(flexRouter + .setPort(request.getPort()) + .setLastSeen() + .setToken(token)); + return ok(updated.setServerToken(token)); + } + +} diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 126d16b1..f4390207 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -24,7 +24,7 @@ import bubble.server.BubbleConfiguration; import bubble.service.block.BlockStatsService; import bubble.service.block.BlockStatsSummary; import bubble.service.boot.SelfNodeService; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import bubble.service.stream.ConnectionCheckResponse; import bubble.service.stream.StandardRuleEngineService; import com.fasterxml.jackson.databind.JsonNode; @@ -79,7 +79,7 @@ public class FilterHttpResource { @Autowired private AppSiteDAO siteDAO; @Autowired private AppRuleDAO ruleDAO; @Autowired private DeviceDAO deviceDAO; - @Autowired private DeviceIdService deviceIdService; + @Autowired private DeviceService deviceService; @Autowired private RedisService redis; @Autowired private BubbleConfiguration configuration; @Autowired private SelfNodeService selfNodeService; @@ -163,7 +163,7 @@ public class FilterHttpResource { } final String vpnAddr = connCheckRequest.getRemoteAddr(); - final Device device = deviceIdService.findDeviceByIp(vpnAddr); + final Device device = deviceService.findDeviceByIp(vpnAddr); if (device == null) { if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning not found"); return notFound(); @@ -237,17 +237,17 @@ public class FilterHttpResource { @Getter(lazy=true) private final BubbleNetwork thisNetwork = selfNodeService.getThisNetwork(); public boolean showStats(String accountUuid, String ip, String[] fqdns) { - if (!deviceIdService.doShowBlockStats(accountUuid)) return false; + if (!deviceService.doShowBlockStats(accountUuid)) return false; for (String fqdn : fqdns) { - final Boolean show = deviceIdService.doShowBlockStatsForIpAndFqdn(ip, fqdn); + final Boolean show = deviceService.doShowBlockStatsForIpAndFqdn(ip, fqdn); if (show != null) return show; } return true; } public boolean showStats(String accountUuid, String ip, String fqdn) { - if (!deviceIdService.doShowBlockStats(accountUuid)) return false; - final Boolean show = deviceIdService.doShowBlockStatsForIpAndFqdn(ip, fqdn); + if (!deviceService.doShowBlockStats(accountUuid)) return false; + final Boolean show = deviceService.doShowBlockStatsForIpAndFqdn(ip, fqdn); return show == null || show; } @@ -277,7 +277,7 @@ public class FilterHttpResource { } final String vpnAddr = filterRequest.getClientAddr(); - final Device device = deviceIdService.findDeviceByIp(vpnAddr); + final Device device = deviceService.findDeviceByIp(vpnAddr); if (device == null) { if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning no matchers"); else if (extraLog) log.error(prefix+"device not found for IP "+vpnAddr+", returning no matchers"); diff --git a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java index 21bdf1e0..cf6ab7b6 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/ReverseProxyResource.java @@ -11,7 +11,7 @@ import bubble.model.app.AppMatcher; import bubble.model.device.Device; import bubble.rule.FilterMatchDecision; import bubble.server.BubbleConfiguration; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import bubble.service.stream.StandardRuleEngineService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -46,7 +46,7 @@ public class ReverseProxyResource { @Autowired private AppMatcherDAO matcherDAO; @Autowired private AppRuleDAO ruleDAO; @Autowired private StandardRuleEngineService ruleEngine; - @Autowired private DeviceIdService deviceIdService; + @Autowired private DeviceService deviceService; @Autowired private FilterHttpResource filterHttpResource; @Getter(lazy=true) private final int prefixLength = configuration.getHttp().getBaseUri().length() + PROXY_ENDPOINT.length() + 1; @@ -60,7 +60,7 @@ public class ReverseProxyResource { @PathParam("path") String path) throws URISyntaxException, IOException { final Account account = userPrincipal(request); final String remoteHost = getRemoteHost(req); - final Device device = deviceIdService.findDeviceByIp(remoteHost); + final Device device = deviceService.findDeviceByIp(remoteHost); if (device == null) return ruleEngine.passthru(request); final URIBean ub = getUriBean(request); diff --git a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java index 0a5ff0f3..f58a278d 100644 --- a/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AbstractAppRuleDriver.java @@ -16,7 +16,7 @@ import bubble.model.device.Device; import bubble.resources.stream.FilterHttpRequest; import bubble.resources.stream.FilterMatchersRequest; import bubble.server.BubbleConfiguration; -import bubble.service.cloud.StandardDeviceIdService; +import bubble.service.device.StandardDeviceService; import bubble.service.stream.AppPrimerService; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; @@ -64,7 +64,7 @@ public abstract class AbstractAppRuleDriver implements AppRuleDriver { @Autowired protected BubbleNetworkDAO networkDAO; @Autowired protected DeviceDAO deviceDAO; @Autowired protected AppPrimerService appPrimerService; - @Autowired protected StandardDeviceIdService deviceService; + @Autowired protected StandardDeviceService deviceService; @Getter @Setter private AppRuleDriver next; diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index df8f4848..9b340b7b 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -16,7 +16,7 @@ import bubble.rule.RequestModifierConfig; import bubble.rule.RequestModifierRule; import bubble.rule.analytics.TrafficAnalyticsRuleDriver; import bubble.server.BubbleConfiguration; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import bubble.service.stream.AppRuleHarness; import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; @@ -419,11 +419,11 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver } @Override public void prime(Account account, BubbleApp app, BubbleConfiguration configuration) { - final DeviceIdService deviceIdService = configuration.getBean(DeviceIdService.class); + final DeviceService deviceService = configuration.getBean(DeviceService.class); final AppDataDAO dataDAO = configuration.getBean(AppDataDAO.class); log.info("priming app="+app.getName()); dataDAO.findByAccountAndAppAndAndKeyPrefix(account.getUuid(), app.getUuid(), PREFIX_APPDATA_HIDE_STATS) - .forEach(data -> deviceIdService.setBlockStatsForFqdn(account, fqdnFromKey(data.getKey()), !Boolean.parseBoolean(data.getData()))); + .forEach(data -> deviceService.setBlockStatsForFqdn(account, fqdnFromKey(data.getKey()), !Boolean.parseBoolean(data.getData()))); } @Override public Function createCallback(Account account, @@ -433,15 +433,15 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver final String prefix = "createCallbackB("+data.getKey()+"="+data.getData()+"): "; log.info(prefix+"starting"); if (data.getKey().startsWith(PREFIX_APPDATA_HIDE_STATS)) { - final DeviceIdService deviceIdService = configuration.getBean(DeviceIdService.class); + final DeviceService deviceService = configuration.getBean(DeviceService.class); final String fqdn = fqdnFromKey(data.getKey()); if (validateRegexMatches(HOST_PATTERN, fqdn)) { if (data.deleting()) { log.info(prefix+"unsetting fqdn: "+fqdn); - deviceIdService.unsetBlockStatsForFqdn(account, fqdn); + deviceService.unsetBlockStatsForFqdn(account, fqdn); } else { log.info(prefix+"setting fqdn: "+fqdn); - deviceIdService.setBlockStatsForFqdn(account, fqdn, !Boolean.parseBoolean(data.getData())); + deviceService.setBlockStatsForFqdn(account, fqdn, !Boolean.parseBoolean(data.getData())); } } else { throw invalidEx("err.fqdn.invalid", "not a valid FQDN: "+fqdn, fqdn); diff --git a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java index 098213ac..38fd9ea7 100644 --- a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java @@ -13,8 +13,9 @@ import bubble.model.cloud.BubbleNode; import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import bubble.service.cloud.NetworkMonitorService; +import bubble.service.device.FlexRouterService; import bubble.service.stream.AppDataCleaner; import bubble.service.stream.AppPrimerService; import lombok.extern.slf4j.Slf4j; @@ -112,7 +113,8 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase> POST_COPY_ENTITIES = Arrays.asList(new Class[] { + private static final List> NO_DEFAULT_COPY_ENTITIES = Arrays.asList(new Class[] { BubbleNode.class, BubbleNodeKey.class, Device.class, AccountMessage.class, ReferralCode.class, AccountPayment.class, Bill.class, Promotion.class, - ReceivedNotification.class, SentNotification.class, TrustedClient.class + ReceivedNotification.class, SentNotification.class, TrustedClient.class, FlexRouter.class }); - private static boolean isPostCopyEntity(Class clazz) { - return POST_COPY_ENTITIES.stream().anyMatch(c -> c.isAssignableFrom(clazz)); + private static boolean isNotDefaultCopyEntity(Class clazz) { + return NO_DEFAULT_COPY_ENTITIES.stream().anyMatch(c -> c.isAssignableFrom(clazz)); } private final BubbleConfiguration configuration; @@ -82,10 +83,10 @@ public class FilteredEntityIterator extends EntityIterator { configuration.getEntityClasses().forEach(c -> { final DAO dao = configuration.getDaoForEntityClass(c); if (!AccountOwnedEntityDAO.class.isAssignableFrom(dao.getClass())) { - log.debug("iterate: skipping entity: " + c.getSimpleName()); - } else if (isPostCopyEntity(c)) { + log.debug("iterate: skipping entity, not an AccountOwnedEntityDAO: " + c.getSimpleName()); + } else if (isNotDefaultCopyEntity(c)) { log.debug("iterate: skipping " + c.getSimpleName() - + ", will copy some of these after other objects are copied"); + + ", may copy some of these after default objects are copied"); } else { // copy entities. this is how the re-keying works (decrypt using current spring config, // encrypt using new config) diff --git a/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java b/bubble-server/src/main/java/bubble/service/device/DeviceService.java similarity index 93% rename from bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java rename to bubble-server/src/main/java/bubble/service/device/DeviceService.java index ebeacae3..b0704311 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/DeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service/device/DeviceService.java @@ -2,7 +2,7 @@ * Copyright (c) 2020 Bubble, Inc. All rights reserved. * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ */ -package bubble.service.cloud; +package bubble.service.device; import bubble.model.account.Account; import bubble.model.device.Device; @@ -10,7 +10,7 @@ import bubble.model.device.DeviceStatus; import java.util.List; -public interface DeviceIdService { +public interface DeviceService { Device findDeviceByIp(String ip); diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java new file mode 100644 index 00000000..fb84f751 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -0,0 +1,58 @@ +package bubble.service.device; + +import bubble.dao.device.FlexRouterDAO; +import bubble.model.device.FlexRouter; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.daemon.SimpleDaemon; +import org.cobbzilla.util.http.HttpRequestBean; +import org.cobbzilla.util.http.HttpResponseBean; +import org.cobbzilla.util.http.HttpUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; +import static org.cobbzilla.util.http.HttpMethods.POST; + +@Service @Slf4j +public class FlexRouterService extends SimpleDaemon { + + @Getter private final long sleepTime = MINUTES.toMillis(2); + + @Autowired private FlexRouterDAO flexRouterDAO; + + @Override protected void process() { + try { + final List routers = flexRouterDAO.findEnabledAndRegistered(); + for (FlexRouter router : routers) { + FlexRouter update = pingRouter(router); + if (update != null) { + flexRouterDAO.update(update); + } + } + } catch (Exception e) { + log.error("process: "+shortError(e)); + } + } + + private FlexRouter pingRouter(FlexRouter router) { + final String prefix = "pingRouter(" + router + "): "; + final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl(), router.pingObject()); + try { + final HttpResponseBean response = HttpUtil.getResponse(request); + if (!response.isOk()) { + log.error(prefix+"response not OK, marking inactive: "+response); + return router.setActive(false); + } + return router.setActive(true).setLastSeen(); + + } catch (Exception e) { + log.error(prefix+"error (marking inactive): "+shortError(e)); + return router.setActive(false); + } + } + +} diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java b/bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java similarity index 98% rename from bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java rename to bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java index 09950246..a07e1091 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardDeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java @@ -2,7 +2,7 @@ * Copyright (c) 2020 Bubble, Inc. All rights reserved. * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ */ -package bubble.service.cloud; +package bubble.service.device; import bubble.dao.account.AccountDAO; import bubble.dao.app.AppSiteDAO; @@ -12,6 +12,7 @@ import bubble.model.app.AppSite; import bubble.model.device.Device; import bubble.model.device.DeviceStatus; import bubble.server.BubbleConfiguration; +import bubble.service.cloud.GeoService; import bubble.service.stream.StandardRuleEngineService; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ExpirationMap; @@ -40,7 +41,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; import static org.cobbzilla.wizard.server.RestServerBase.reportError; @Service @Slf4j -public class StandardDeviceIdService implements DeviceIdService { +public class StandardDeviceService implements DeviceService { public static final File WG_DEVICES_DIR = new File(HOME_DIR, "wg_devices"); diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index 683367b8..d6295f14 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -14,7 +14,7 @@ import bubble.model.cloud.BubbleNetwork; import bubble.model.device.Device; import bubble.rule.AppRuleDriver; import bubble.server.BubbleConfiguration; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.SingletonList; @@ -34,7 +34,7 @@ public class StandardAppPrimerService implements AppPrimerService { @Autowired private AccountDAO accountDAO; @Autowired private DeviceDAO deviceDAO; - @Autowired private DeviceIdService deviceIdService; + @Autowired private DeviceService deviceService; @Autowired private BubbleAppDAO appDAO; @Autowired private AppMatcherDAO matcherDAO; @Autowired private AppRuleDAO ruleDAO; @@ -77,7 +77,7 @@ public class StandardAppPrimerService implements AppPrimerService { } public void prime(Account account) { - deviceIdService.initBlockStats(account); + deviceService.initBlockStats(account); prime(account, (BubbleApp) null); } @@ -114,7 +114,7 @@ public class StandardAppPrimerService implements AppPrimerService { final Map> accountDeviceIps = new HashMap<>(); final List devices = deviceDAO.findByAccount(account.getUuid()); for (Device device : devices) { - accountDeviceIps.put(device.getUuid(), deviceIdService.findIpsByDevice(device.getUuid())); + accountDeviceIps.put(device.getUuid(), deviceService.findIpsByDevice(device.getUuid())); } if (accountDeviceIps.isEmpty()) return; diff --git a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java similarity index 91% rename from bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java rename to bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java index a33c5798..06cb4969 100644 --- a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceIdService.java +++ b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java @@ -7,7 +7,7 @@ package bubble.service_dbfilter; import bubble.model.account.Account; import bubble.model.device.Device; import bubble.model.device.DeviceStatus; -import bubble.service.cloud.DeviceIdService; +import bubble.service.device.DeviceService; import org.springframework.stereotype.Service; import java.util.List; @@ -15,7 +15,7 @@ import java.util.List; import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; @Service -public class DbFilterDeviceIdService implements DeviceIdService { +public class DbFilterDeviceService implements DeviceService { @Override public Device findDeviceByIp(String ip) { return notSupported("findDeviceByIp"); } diff --git a/bubble-server/src/main/resources/messages b/bubble-server/src/main/resources/messages index 9fcd7bc0..2b4074ab 160000 --- a/bubble-server/src/main/resources/messages +++ b/bubble-server/src/main/resources/messages @@ -1 +1 @@ -Subproject commit 9fcd7bc0a768b124b09b7e8995c27fc94dedf9d8 +Subproject commit 2b4074ab37b1e3caa0cb6382352b3391f5bb7139 diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 4dcca7b4..7727825b 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 4dcca7b4ca0240ad7f2cc1d60909c0ff5830339a +Subproject commit 7727825b5e738168a42e4148d873c3f75d572f91 -- 2.17.1 From 27c3799dbd08a04e287f64cbb1aa261c5d644ade Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sat, 5 Sep 2020 12:41:55 -0400 Subject: [PATCH 02/78] WIP. add db migration for flex_router --- .../java/bubble/model/device/FlexRouter.java | 2 +- .../service/device/FlexRouterService.java | 1 + .../V2020090501__add_flex_router.sql | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 2db0f715..da3981f6 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -71,7 +71,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccountNoName { @JsonIgnore @Transient public long getAge () { return lastSeen == null ? Long.MAX_VALUE : now() - lastSeen; } @ECSearchable(filter=true) @ECField(index=70) - @ECIndex @Column(length=100) + @Column(length=100) @JsonIgnore @Getter @Setter private String token; public boolean hasToken () { return !empty(token); } diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index fb84f751..ca6b0770 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -47,6 +47,7 @@ public class FlexRouterService extends SimpleDaemon { log.error(prefix+"response not OK, marking inactive: "+response); return router.setActive(false); } + if (log.isInfoEnabled()) log.info(prefix+"router is ok, marking active and updating lastSeen"); return router.setActive(true).setLastSeen(); } catch (Exception e) { diff --git a/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql b/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql new file mode 100644 index 00000000..f8143b11 --- /dev/null +++ b/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql @@ -0,0 +1,23 @@ +CREATE TABLE flex_router ( + uuid character varying(100) NOT NULL, + ctime bigint NOT NULL, + mtime bigint NOT NULL, + account character varying(100) NOT NULL, + active boolean NOT NULL, + enabled boolean NOT NULL, + ip character varying(500) NOT NULL, + last_seen bigint, + port integer NOT NULL, + token character varying(100) +); + +ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_pkey PRIMARY KEY (uuid); + +CREATE INDEX flex_router_idx_account ON flex_router USING btree (account); +CREATE INDEX flex_router_idx_active ON flex_router USING btree (active); +CREATE INDEX flex_router_idx_enabled ON flex_router USING btree (enabled); +CREATE INDEX flex_router_idx_ip ON flex_router USING btree (ip); +CREATE INDEX flex_router_idx_port ON flex_router USING btree (port); +CREATE UNIQUE INDEX flex_router_uniq_account_ip ON flex_router USING btree (account, ip); + +ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_fk_account FOREIGN KEY (account) REFERENCES public.account(uuid); -- 2.17.1 From ed7fb0fdc7bc8946c5c2ecd003cde3ebabf0e160 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sat, 5 Sep 2020 14:01:51 -0400 Subject: [PATCH 03/78] WIP. flex router registration --- .../java/bubble/dao/device/FlexRouterDAO.java | 4 ++ .../java/bubble/model/device/FlexRouter.java | 10 +++-- .../account/AccountOwnedResource.java | 35 ++++++++++----- .../resources/app/AppsResourceBase.java | 5 ++- .../resources/app/DataResourceBase.java | 6 ++- .../resources/bill/AccountPlansResource.java | 3 +- .../resources/device/FlexRoutersResource.java | 44 +++++++------------ bubble-server/src/main/resources/messages | 2 +- .../java/bubble/test/system/AuthTest.java | 20 ++++----- 9 files changed, 69 insertions(+), 60 deletions(-) diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index 3827b0ab..217fab64 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -18,4 +18,8 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { isNotNull("token")))); } + public FlexRouter findByAccountAndIp(String accountUuid, String ip) { + return findByUniqueFields("account", accountUuid, "ip", ip); + } + } diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index da3981f6..37a28526 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -10,6 +10,7 @@ import lombok.ToString; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.map.SingletonMap; +import org.cobbzilla.util.collection.ArrayUtil; import org.cobbzilla.wizard.model.Identifiable; import org.cobbzilla.wizard.model.IdentifiableBase; import org.cobbzilla.wizard.model.entityconfig.EntityFieldMode; @@ -32,8 +33,8 @@ import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) }) public class FlexRouter extends IdentifiableBase implements HasAccountNoName { - public static final String[] CREATE_FIELDS = { "ip", "enabled" }; - public static final String[] UPDATE_FIELDS = { "enabled" }; + public static final String[] UPDATE_FIELDS = { "enabled", "active" }; + public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip"); public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); } @@ -75,10 +76,11 @@ public class FlexRouter extends IdentifiableBase implements HasAccountNoName { @JsonIgnore @Getter @Setter private String token; public boolean hasToken () { return !empty(token); } - @Transient @Getter @Setter private String serverToken; + // used for sending the token, we never send it back + @Transient @Getter @Setter private String auth_token; + public boolean hasAuthToken () { return !empty(auth_token); } public String pingUrl() { return "http://" + getIp() + ":" + getPort() + "/ping"; } - public String pingObject() { return json(new SingletonMap("token", getToken())); } } diff --git a/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java b/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java index bae25fc1..148fd7b0 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java @@ -91,16 +91,26 @@ public class AccountOwnedResource populate(ContainerRequest ctx, List entities) { for (E e : entities) populate(ctx, e); @@ -110,14 +120,15 @@ public class AccountOwnedResource { @@ -34,29 +27,22 @@ public class FlexRoutersResource extends AccountOwnedResource Date: Sat, 5 Sep 2020 15:22:31 -0400 Subject: [PATCH 04/78] fix flex router registration --- .../java/bubble/model/device/FlexRouter.java | 7 ++++- .../resources/device/FlexRoutersResource.java | 26 +++++++++++++++---- bubble-server/src/main/resources/messages | 2 +- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 37a28526..8f8b14c1 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -33,7 +33,7 @@ import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) }) public class FlexRouter extends IdentifiableBase implements HasAccountNoName { - public static final String[] UPDATE_FIELDS = { "enabled", "active" }; + public static final String[] UPDATE_FIELDS = { "enabled", "active", "proxy_port", "auth_token" }; public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip"); public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); } @@ -47,6 +47,11 @@ public class FlexRouter extends IdentifiableBase implements HasAccountNoName { @ECSearchable(filter=true) @ECField(index=20) @ECIndex @Column(nullable=false) @JsonIgnore @Getter @Setter private Integer port = 0; + public boolean hasPort () { return port != null && port > 1024; } + + // used for sending the port, we never send it back + @Transient @Getter @Setter private Integer proxy_port; + public boolean hasProxyPort () { return proxy_port != null && proxy_port > 1024; } public String id () { return getIp() + "/" + getUuid(); } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index b52405e4..8dda2ce4 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -12,6 +12,7 @@ import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; import static bubble.ApiConstants.getRemoteAddr; +import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; import static org.cobbzilla.wizard.resources.ResourceUtil.userPrincipal; @@ -34,14 +35,29 @@ public class FlexRoutersResource extends AccountOwnedResource Date: Sat, 5 Sep 2020 20:17:18 -0400 Subject: [PATCH 05/78] ping router before successful registration --- .../java/bubble/dao/device/FlexRouterDAO.java | 2 ++ .../java/bubble/model/device/FlexRouter.java | 7 +++-- .../resources/device/FlexRoutersResource.java | 27 ++++++------------- .../service/device/FlexRouterService.java | 4 +-- 4 files changed, 17 insertions(+), 23 deletions(-) diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index 217fab64..48d2f1fe 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -11,6 +11,8 @@ import static org.hibernate.criterion.Restrictions.*; @Repository public class FlexRouterDAO extends AccountOwnedEntityDAO { + @Override protected String getNameField() { return "ip"; } + public List findEnabledAndRegistered() { return list(criteria().add(and( eq("enabled", true), diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 8f8b14c1..df377db3 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -1,7 +1,7 @@ package bubble.model.device; import bubble.model.account.Account; -import bubble.model.account.HasAccountNoName; +import bubble.model.account.HasAccount; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; @@ -31,7 +31,7 @@ import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @ECTypeURIs(baseURI=EP_FLEX_ROUTERS, listFields={"name", "enabled"}) @NoArgsConstructor @Accessors(chain=true) @Slf4j @ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) }) -public class FlexRouter extends IdentifiableBase implements HasAccountNoName { +public class FlexRouter extends IdentifiableBase implements HasAccount { public static final String[] UPDATE_FIELDS = { "enabled", "active", "proxy_port", "auth_token" }; public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip"); @@ -44,6 +44,8 @@ public class FlexRouter extends IdentifiableBase implements HasAccountNoName { @ECIndex @Column(nullable=false, length=500) @Getter @Setter private String ip; + @JsonIgnore @Transient public String getName () { return getIp(); } + @ECSearchable(filter=true) @ECField(index=20) @ECIndex @Column(nullable=false) @JsonIgnore @Getter @Setter private Integer port = 0; @@ -69,6 +71,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccountNoName { @ECIndex @Column(nullable=false) @Getter @Setter private Boolean active = true; public boolean active() { return bool(active); } + public boolean inactive() { return !active(); } @ECSearchable @ECField(index=60, type=EntityFieldType.epoch_time, mode=EntityFieldMode.readOnly) @Getter @Setter private Long lastSeen; diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 8dda2ce4..ae239b92 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -11,7 +11,8 @@ import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; -import static bubble.ApiConstants.getRemoteAddr; +import static bubble.service.device.FlexRouterService.pingFlexRouter; +import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; import static org.cobbzilla.wizard.resources.ResourceUtil.userPrincipal; @@ -28,33 +29,21 @@ public class FlexRoutersResource extends AccountOwnedResource routers = flexRouterDAO.findEnabledAndRegistered(); for (FlexRouter router : routers) { - FlexRouter update = pingRouter(router); + FlexRouter update = pingFlexRouter(router); if (update != null) { flexRouterDAO.update(update); } @@ -38,7 +38,7 @@ public class FlexRouterService extends SimpleDaemon { } } - private FlexRouter pingRouter(FlexRouter router) { + public static FlexRouter pingFlexRouter(FlexRouter router) { final String prefix = "pingRouter(" + router + "): "; final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl(), router.pingObject()); try { -- 2.17.1 From 9ba7fca2cb6f0d940230858147bc22a700eede52 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sat, 5 Sep 2020 20:20:42 -0400 Subject: [PATCH 06/78] remove unused DAO finder --- .../src/main/java/bubble/dao/device/FlexRouterDAO.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index 48d2f1fe..cc6fc013 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -20,8 +20,4 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { isNotNull("token")))); } - public FlexRouter findByAccountAndIp(String accountUuid, String ip) { - return findByUniqueFields("account", accountUuid, "ip", ip); - } - } -- 2.17.1 From 43362a6acc94eb0c5b3d67136c96da5679f1cb81 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sat, 5 Sep 2020 22:31:12 -0400 Subject: [PATCH 07/78] send ping to flex router, expect pong --- .../java/bubble/model/device/FlexRouter.java | 4 +-- .../bubble/model/device/FlexRouterPing.java | 26 +++++++++++++++++++ .../service/device/FlexRouterService.java | 15 ++++++++--- 3 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index df377db3..c67bb035 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -9,7 +9,6 @@ import lombok.Setter; import lombok.ToString; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.map.SingletonMap; import org.cobbzilla.util.collection.ArrayUtil; import org.cobbzilla.wizard.model.Identifiable; import org.cobbzilla.wizard.model.IdentifiableBase; @@ -23,7 +22,6 @@ import javax.persistence.Transient; import static bubble.ApiConstants.EP_FLEX_ROUTERS; import static org.cobbzilla.util.daemon.ZillaRuntime.*; -import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @Entity @@ -89,6 +87,6 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { public boolean hasAuthToken () { return !empty(auth_token); } public String pingUrl() { return "http://" + getIp() + ":" + getPort() + "/ping"; } - public String pingObject() { return json(new SingletonMap("token", getToken())); } + public FlexRouterPing pingObject() { return new FlexRouterPing(this); } } diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java b/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java new file mode 100644 index 00000000..7b8a639a --- /dev/null +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java @@ -0,0 +1,26 @@ +package bubble.model.device; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.util.security.ShaUtil.sha256_hex; + +@NoArgsConstructor @Accessors(chain=true) +public class FlexRouterPing { + + @Getter @Setter private String salt; + @Getter @Setter private String hash; + + public FlexRouterPing (FlexRouter router) { + salt = randomAlphanumeric(50); + hash = sha256_hex(salt + ":" + router.getToken()); + } + + public boolean validate(FlexRouter router) { + return sha256_hex(salt + ":" + router.getToken()).equals(hash); + } + +} diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index 7bfb4e69..c834fbc0 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -2,6 +2,7 @@ package bubble.service.device; import bubble.dao.device.FlexRouterDAO; import bubble.model.device.FlexRouter; +import bubble.model.device.FlexRouterPing; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.daemon.SimpleDaemon; @@ -16,6 +17,7 @@ import java.util.List; import static java.util.concurrent.TimeUnit.MINUTES; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; import static org.cobbzilla.util.http.HttpMethods.POST; +import static org.cobbzilla.util.json.JsonUtil.json; @Service @Slf4j public class FlexRouterService extends SimpleDaemon { @@ -40,15 +42,22 @@ public class FlexRouterService extends SimpleDaemon { public static FlexRouter pingFlexRouter(FlexRouter router) { final String prefix = "pingRouter(" + router + "): "; - final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl(), router.pingObject()); + final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl(), json(router.pingObject())); try { final HttpResponseBean response = HttpUtil.getResponse(request); if (!response.isOk()) { log.error(prefix+"response not OK, marking inactive: "+response); return router.setActive(false); + } else { + final FlexRouterPing pong = response.getEntity(FlexRouterPing.class); + if (pong.validate(router)) { + if (log.isInfoEnabled()) log.info(prefix+"router is ok, marking active and updating lastSeen"); + return router.setActive(true).setLastSeen(); + } else { + log.error(prefix+"pong response was invalid, marking inactive"); + return router.setActive(false); + } } - if (log.isInfoEnabled()) log.info(prefix+"router is ok, marking active and updating lastSeen"); - return router.setActive(true).setLastSeen(); } catch (Exception e) { log.error(prefix+"error (marking inactive): "+shortError(e)); -- 2.17.1 From 52abba3aa0be0f469d27b700d51ab836f15140f0 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sat, 5 Sep 2020 22:41:28 -0400 Subject: [PATCH 08/78] send ping after setting token --- .../java/bubble/resources/device/FlexRoutersResource.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index ae239b92..9ecfcf25 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -38,14 +38,14 @@ public class FlexRoutersResource extends AccountOwnedResource Date: Sat, 5 Sep 2020 22:45:14 -0400 Subject: [PATCH 09/78] remove log message --- .../main/java/bubble/resources/device/FlexRoutersResource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 9ecfcf25..c8b3e05b 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -46,7 +46,6 @@ public class FlexRoutersResource extends AccountOwnedResource Date: Sun, 6 Sep 2020 01:11:11 -0400 Subject: [PATCH 10/78] add flex config to TlsPassthru app, encrypt token column, fix update --- .../java/bubble/dao/device/FlexRouterDAO.java | 17 +++- .../java/bubble/model/device/FlexRouter.java | 9 ++- .../bubble/model/device/FlexRouterPing.java | 17 +++- .../account/AccountOwnedResource.java | 2 +- .../rule/passthru/TlsPassthruConfig.java | 81 +++++++++++++++++-- .../rule/passthru/TlsPassthruRuleDriver.java | 4 + .../stream/ConnectionCheckResponse.java | 2 +- .../V2020090501__add_flex_router.sql | 2 +- .../apps/passthru/bubbleApp_passthru.json | 4 + 9 files changed, 121 insertions(+), 17 deletions(-) diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index cc6fc013..e8261e48 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -2,13 +2,15 @@ package bubble.dao.device; import bubble.dao.account.AccountOwnedEntityDAO; import bubble.model.device.FlexRouter; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Repository; import java.util.List; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.hibernate.criterion.Restrictions.*; -@Repository +@Repository @Slf4j public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override protected String getNameField() { return "ip"; } @@ -16,7 +18,18 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { public List findEnabledAndRegistered() { return list(criteria().add(and( eq("enabled", true), - ne("port", 0), + gt("port", 1024), + le("port", 65535), + isNotNull("token")))); + } + + public List findActive(long maxAge) { + return list(criteria().add(and( + eq("active", true), + eq("enabled", true), + gt("port", 1024), + le("port", 65535), + ge("lastSeen", now()-maxAge), isNotNull("token")))); } diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index c67bb035..24ad4f5c 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -15,6 +15,7 @@ import org.cobbzilla.wizard.model.IdentifiableBase; import org.cobbzilla.wizard.model.entityconfig.EntityFieldMode; import org.cobbzilla.wizard.model.entityconfig.EntityFieldType; import org.cobbzilla.wizard.model.entityconfig.annotations.*; +import org.hibernate.annotations.Type; import javax.persistence.Column; import javax.persistence.Entity; @@ -23,6 +24,8 @@ import javax.persistence.Transient; import static bubble.ApiConstants.EP_FLEX_ROUTERS; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; +import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; +import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @Entity @ECType(root=true) @ToString(of={"ip", "port"}) @@ -31,7 +34,7 @@ import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) }) public class FlexRouter extends IdentifiableBase implements HasAccount { - public static final String[] UPDATE_FIELDS = { "enabled", "active", "proxy_port", "auth_token" }; + public static final String[] UPDATE_FIELDS = { "enabled", "active", "proxy_port", "auth_token", "token" }; public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip"); public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); } @@ -51,7 +54,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { // used for sending the port, we never send it back @Transient @Getter @Setter private Integer proxy_port; - public boolean hasProxyPort () { return proxy_port != null && proxy_port > 1024; } + public boolean hasProxyPort () { return proxy_port != null && proxy_port > 1024 && proxy_port < 65535; } public String id () { return getIp() + "/" + getUuid(); } @@ -78,7 +81,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { @JsonIgnore @Transient public long getAge () { return lastSeen == null ? Long.MAX_VALUE : now() - lastSeen; } @ECSearchable(filter=true) @ECField(index=70) - @Column(length=100) + @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(100+ENC_PAD)+")") @JsonIgnore @Getter @Setter private String token; public boolean hasToken () { return !empty(token); } diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java b/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java index 7b8a639a..3987abe6 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java @@ -5,22 +5,35 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.cobbzilla.util.security.ShaUtil.sha256_hex; @NoArgsConstructor @Accessors(chain=true) public class FlexRouterPing { + public static final long MAX_PING_AGE = SECONDS.toMillis(30); + public static final long MIN_PING_AGE = -1 * SECONDS.toMillis(5); + + @Getter @Setter private long time; @Getter @Setter private String salt; @Getter @Setter private String hash; public FlexRouterPing (FlexRouter router) { + time = now(); salt = randomAlphanumeric(50); - hash = sha256_hex(salt + ":" + router.getToken()); + hash = sha256_hex(data(router)); } public boolean validate(FlexRouter router) { - return sha256_hex(salt + ":" + router.getToken()).equals(hash); + if (empty(salt) || salt.length() < 50) return false; + final long age = now() - time; + if (age > MAX_PING_AGE || age < MIN_PING_AGE) return false; + return sha256_hex(data(router)).equals(hash); } + private String data(FlexRouter router) { return salt + ":" + time + ":" + router.getToken(); } + } diff --git a/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java b/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java index 148fd7b0..4035bba3 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java @@ -155,7 +155,7 @@ public class AccountOwnedResource> recentFeedValues = new HashMap<>(); + private final Map> recentFeedValues = new HashMap<>(); @JsonIgnore public Set getFeedSet() { final TlsPassthruFeed[] feedList = getFeedList(); return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); } + @Getter @Setter private String[] flexFqdnList; + public boolean hasFlexFqdnList () { return !empty(flexFqdnList); } + public boolean hasFlexFqdn(String flexFqdn) { return hasFlexFqdnList() && ArrayUtils.indexOf(flexFqdnList, flexFqdn) != -1; } + + public TlsPassthruConfig addFlexFqdn(String flexFqdn) { + return setFlexFqdnList(Arrays.stream(ArrayUtil.append(flexFqdnList, flexFqdn)).collect(Collectors.toSet()).toArray(String[]::new)); + } + + public TlsPassthruConfig removeFlexFqdn(String id) { + return !hasFlexFqdnList() ? this : + setFlexFqdnList(Arrays.stream(getFlexFqdnList()) + .filter(flexFqdn -> !flexFqdn.equalsIgnoreCase(id.trim())) + .toArray(String[]::new)); + } + + @Getter @Setter private TlsPassthruFeed[] flexFeedList; + public boolean hasFlexFeedList () { return !empty(flexFeedList); } + public boolean hasFlexFeed (TlsPassthruFeed flexFeed) { + return hasFlexFeedList() && Arrays.stream(flexFeedList).anyMatch(f -> f.getFeedUrl().equals(flexFeed.getFeedUrl())); + } + + public TlsPassthruConfig addFlexFeed(TlsPassthruFeed flexFeed) { + final Set flexFeeds = getFlexFeedSet(); + if (empty(flexFeeds)) return setFlexFeedList(new TlsPassthruFeed[] {flexFeed}); + flexFeeds.add(flexFeed); + return setFlexFeedList(flexFeeds.toArray(EMPTY_FEEDS)); + } + + public TlsPassthruConfig removeFlexFeed(String id) { + return setFlexFeedList(getFlexFeedSet().stream() + .filter(flexFeed -> !flexFeed.getId().equals(id)) + .toArray(TlsPassthruFeed[]::new)); + } + + private final Map> recentFlexFeedValues = new HashMap<>(); + + @JsonIgnore public Set getFlexFeedSet() { + final TlsPassthruFeed[] flexFeedList = getFlexFeedList(); + return !empty(flexFeedList) ? Arrays.stream(flexFeedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); + } + @ToString private static class TlsPassthruMatcher { @Getter @Setter private String fqdn; @@ -103,13 +145,32 @@ public class TlsPassthruConfig { @JsonIgnore public Set getPassthruSet() { return getPassthruSetRef().get(); } private Set loadPassthruSet() { + final Set set = loadFeeds(this.feedList, this.fqdnList, this.recentFeedValues); + if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", ")); + return set; + } + + @JsonIgnore @Getter(lazy=true) private final AutoRefreshingReference> flexSetRef = new AutoRefreshingReference<>() { + @Override public Set refresh() { return loadFlexSet(); } + // todo: load refresh interval from config. implement a config view with an action to set it + @Override public long getTimeout() { return DEFAULT_FLEX_FEED_REFRESH_INTERVAL; } + }; + @JsonIgnore public Set getFlexSet() { return getFlexSetRef().get(); } + + private Set loadFlexSet() { + final Set set = loadFeeds(this.flexFeedList, this.flexFqdnList, this.recentFlexFeedValues); + if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", ")); + return set; + } + + private Set loadFeeds(TlsPassthruFeed[] feedList, String[] fqdnList, Map> recentValues) { final Set set = new HashSet<>(); - if (hasFqdnList()) { - for (String val : getFqdnList()) { + if (!empty(fqdnList)) { + for (String val : fqdnList) { set.add(new TlsPassthruMatcher(val)); } } - if (hasFeedList()) { + if (!empty(feedList)) { // put in a set to avoid duplicate URLs for (TlsPassthruFeed feed : new HashSet<>(Arrays.asList(feedList))) { final TlsPassthruFeed loaded = loadFeed(feed.getFeedUrl()); @@ -118,13 +179,12 @@ public class TlsPassthruConfig { if (!feed.hasFeedName() && loaded.hasFeedName()) feed.setFeedName(loaded.getFeedName()); // add to set if anything was found - if (loaded.hasFqdnList()) recentFeedValues.put(feed.getFeedUrl(), loaded.getFqdnList()); + if (loaded.hasFqdnList()) recentValues.put(feed.getFeedUrl(), loaded.getFqdnList()); } } - for (String val : recentFeedValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet())) { + for (String val : recentValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet())) { set.add(new TlsPassthruMatcher(val)); } - if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", ")); return set; } @@ -162,4 +222,11 @@ public class TlsPassthruConfig { return false; } + public boolean isFlex(String fqdn) { + for (TlsPassthruMatcher match : getFlexSet()) { + if (match.matches(fqdn)) return true; + } + return false; + } + } diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index 7e438930..2d50b4b9 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -26,6 +26,10 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { if (log.isDebugEnabled()) log.debug("checkConnection: returning passthru for fqdn/addr="+fqdn+"/"+addr); return ConnectionCheckResponse.passthru; } + if (passthruConfig.isFlex(fqdn)) { + if (log.isDebugEnabled()) log.debug("checkConnection: returning flex for fqdn/addr="+fqdn+"/"+addr); + return ConnectionCheckResponse.flex; + } if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr); return ConnectionCheckResponse.noop; } diff --git a/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java index e1bf50cf..933474e8 100644 --- a/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java +++ b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java @@ -10,7 +10,7 @@ import static bubble.ApiConstants.enumFromString; public enum ConnectionCheckResponse { - noop, passthru, block, error; + noop, passthru, flex, block, error; @JsonCreator public static ConnectionCheckResponse fromString (String v) { return enumFromString(ConnectionCheckResponse.class, v); } diff --git a/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql b/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql index f8143b11..bb6ea1d5 100644 --- a/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql +++ b/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql @@ -8,7 +8,7 @@ CREATE TABLE flex_router ( ip character varying(500) NOT NULL, last_seen bigint, port integer NOT NULL, - token character varying(100) + token character varying(200) ); ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_pkey PRIMARY KEY (uuid); diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json index 295e72b3..2f5ffc4c 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json @@ -59,6 +59,10 @@ "fqdnList": [], "feedList": [{ "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/tls_passthru.txt" + }], + "flexFqdnList": [], + "flexFeedList": [{ + "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/flex_routing.txt" }] } }], -- 2.17.1 From a7ed61ff983613f61010e4822b96da040dda155a Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 02:39:39 -0400 Subject: [PATCH 11/78] remove FlexRouter.lastSeen, add ping retries and logging --- .../java/bubble/model/device/FlexRouter.java | 13 ++--- .../resources/device/FlexRoutersResource.java | 2 +- .../service/device/FlexRouterService.java | 50 ++++++++++++------- .../V2020090501__add_flex_router.sql | 1 - 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 24ad4f5c..24fdcc2d 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -12,8 +12,6 @@ import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ArrayUtil; import org.cobbzilla.wizard.model.Identifiable; import org.cobbzilla.wizard.model.IdentifiableBase; -import org.cobbzilla.wizard.model.entityconfig.EntityFieldMode; -import org.cobbzilla.wizard.model.entityconfig.EntityFieldType; import org.cobbzilla.wizard.model.entityconfig.annotations.*; import org.hibernate.annotations.Type; @@ -22,7 +20,8 @@ import javax.persistence.Entity; import javax.persistence.Transient; import static bubble.ApiConstants.EP_FLEX_ROUTERS; -import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.daemon.ZillaRuntime.bool; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @@ -74,13 +73,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { public boolean active() { return bool(active); } public boolean inactive() { return !active(); } - @ECSearchable @ECField(index=60, type=EntityFieldType.epoch_time, mode=EntityFieldMode.readOnly) - @Getter @Setter private Long lastSeen; - public FlexRouter setLastSeen () { return setLastSeen(now()); } - - @JsonIgnore @Transient public long getAge () { return lastSeen == null ? Long.MAX_VALUE : now() - lastSeen; } - - @ECSearchable(filter=true) @ECField(index=70) + @ECSearchable(filter=true) @ECField(index=60) @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(100+ENC_PAD)+")") @JsonIgnore @Getter @Setter private String token; public boolean hasToken () { return !empty(token); } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index c8b3e05b..0402a7e3 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -41,7 +41,7 @@ public class FlexRoutersResource extends AccountOwnedResource routers = flexRouterDAO.findEnabledAndRegistered(); for (FlexRouter router : routers) { - FlexRouter update = pingFlexRouter(router); - if (update != null) { - flexRouterDAO.update(update); + boolean active = pingFlexRouter(router); + if (active != router.active()) { + flexRouterDAO.update(router.setActive(active)); } } } catch (Exception e) { @@ -40,29 +45,36 @@ public class FlexRouterService extends SimpleDaemon { } } - public static FlexRouter pingFlexRouter(FlexRouter router) { + public static boolean pingFlexRouter(FlexRouter router) { final String prefix = "pingRouter(" + router + "): "; final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl(), json(router.pingObject())); - try { - final HttpResponseBean response = HttpUtil.getResponse(request); - if (!response.isOk()) { - log.error(prefix+"response not OK, marking inactive: "+response); - return router.setActive(false); + for (int i=0; i Date: Sun, 6 Sep 2020 02:46:47 -0400 Subject: [PATCH 12/78] create a new ping object for each ping attempt --- .../src/main/java/bubble/service/device/FlexRouterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index aeb98fea..7b9d52ab 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -47,7 +47,7 @@ public class FlexRouterService extends SimpleDaemon { public static boolean pingFlexRouter(FlexRouter router) { final String prefix = "pingRouter(" + router + "): "; - final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl(), json(router.pingObject())); + final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl()); for (int i=0; i Date: Sun, 6 Sep 2020 03:38:11 -0400 Subject: [PATCH 13/78] shorter timeout on ping, add flex responses to conn check --- .../resources/device/FlexRoutersResource.java | 2 +- .../rule/passthru/TlsPassthruRuleDriver.java | 11 ++--- .../service/device/FlexRouterService.java | 40 +++++++++++++++++-- .../stream/ConnectionCheckResponse.java | 2 +- .../mitmproxy/files/bubble_conn_check.py | 26 +++++++++--- 5 files changed, 66 insertions(+), 15 deletions(-) diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 0402a7e3..5a875a75 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -41,7 +41,7 @@ public class FlexRoutersResource extends AccountOwnedResource routers = flexRouterDAO.findEnabledAndRegistered(); for (FlexRouter router : routers) { - boolean active = pingFlexRouter(router); + boolean active = pingFlexRouter(router, flexRouterDAO, httpClient); if (active != router.active()) { flexRouterDAO.update(router.setActive(active)); } @@ -45,7 +64,17 @@ public class FlexRouterService extends SimpleDaemon { } } - public static boolean pingFlexRouter(FlexRouter router) { + public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO) { + try { + @Cleanup final CloseableHttpClient httpClient = getHttpClient(); + return pingFlexRouter(router, flexRouterDAO, httpClient); + } catch (Exception e) { + log.error("pingFlexRouter("+router+"): "+shortError(e)); + return false; + } + } + + public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO, HttpClient httpClient) { final String prefix = "pingRouter(" + router + "): "; final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl()); for (int i=0; i Date: Sun, 6 Sep 2020 03:39:50 -0400 Subject: [PATCH 14/78] use static import --- .../src/main/java/bubble/service/device/FlexRouterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index 5a0aa315..00a93489 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -19,6 +19,7 @@ import org.springframework.stereotype.Service; import java.util.List; +import static bubble.model.device.FlexRouterPing.MAX_PING_AGE; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; @@ -32,7 +33,7 @@ public class FlexRouterService extends SimpleDaemon { public static final int MAX_PING_TRIES = 5; private static final long PING_SLEEP_FACTOR = SECONDS.toMillis(2); - public static final int DEFAULT_PING_TIMEOUT = (int) SECONDS.toMillis(FlexRouterPing.MAX_PING_AGE/2); + public static final int DEFAULT_PING_TIMEOUT = (int) SECONDS.toMillis(MAX_PING_AGE/2); public static final RequestConfig DEFAULT_PING_REQUEST_CONFIG = RequestConfig.custom() .setConnectTimeout(DEFAULT_PING_TIMEOUT) -- 2.17.1 From 9365a490d487b907ffdf3fe4691f63d8cdfe65f5 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 03:44:08 -0400 Subject: [PATCH 15/78] add mitm license to dns_spoofing.py --- .../roles/mitmproxy/files/dns_spoofing.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index d3f6e270..d9789834 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -1,6 +1,29 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # +# Parts of this are borrowed from dns_spoofing.py in the mitmproxy project. The mitmproxy license is reprinted here: +# +# Copyright (c) 2013, Aldo Cortesi. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + import re import time import uuid -- 2.17.1 From 8fa2d0e1587d646264eb700d3fe43331a2bab778 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 03:57:36 -0400 Subject: [PATCH 16/78] fix http timeouts, use thread pool for concurrent pings --- .../service/device/FlexRouterService.java | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index 00a93489..6643bf25 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -10,6 +10,7 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.cobbzilla.util.daemon.AwaitResult; import org.cobbzilla.util.daemon.SimpleDaemon; import org.cobbzilla.util.http.HttpRequestBean; import org.cobbzilla.util.http.HttpResponseBean; @@ -17,11 +18,16 @@ import org.cobbzilla.util.http.HttpUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import static bubble.model.device.FlexRouterPing.MAX_PING_AGE; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.cobbzilla.util.daemon.Await.awaitAll; +import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; import static org.cobbzilla.util.http.HttpMethods.POST; import static org.cobbzilla.util.json.JsonUtil.json; @@ -33,8 +39,8 @@ public class FlexRouterService extends SimpleDaemon { public static final int MAX_PING_TRIES = 5; private static final long PING_SLEEP_FACTOR = SECONDS.toMillis(2); - public static final int DEFAULT_PING_TIMEOUT = (int) SECONDS.toMillis(MAX_PING_AGE/2); - + // HttpClient timeouts are in seconds + public static final int DEFAULT_PING_TIMEOUT = (int) SECONDS.toSeconds(MAX_PING_AGE/2); public static final RequestConfig DEFAULT_PING_REQUEST_CONFIG = RequestConfig.custom() .setConnectTimeout(DEFAULT_PING_TIMEOUT) .setSocketTimeout(DEFAULT_PING_TIMEOUT) @@ -46,6 +52,8 @@ public class FlexRouterService extends SimpleDaemon { .build(); } + private static final long PING_ALL_TIMEOUT = MINUTES.toMillis(2); + @Getter private final long sleepTime = MINUTES.toMillis(2); @Autowired private FlexRouterDAO flexRouterDAO; @@ -54,12 +62,20 @@ public class FlexRouterService extends SimpleDaemon { try { @Cleanup final CloseableHttpClient httpClient = getHttpClient(); final List routers = flexRouterDAO.findEnabledAndRegistered(); + final List> futures = new ArrayList<>(); + @Cleanup("shutdownNow") final ExecutorService exec = fixedPool(5); for (FlexRouter router : routers) { - boolean active = pingFlexRouter(router, flexRouterDAO, httpClient); - if (active != router.active()) { - flexRouterDAO.update(router.setActive(active)); - } + futures.add(exec.submit(() -> { + boolean active = pingFlexRouter(router, flexRouterDAO, httpClient); + if (active != router.active()) { + flexRouterDAO.update(router.setActive(active)); + } + return active; + })); } + final AwaitResult awaitResult = awaitAll(futures, PING_ALL_TIMEOUT); + log.debug("process: awaitResult="+awaitResult); + } catch (Exception e) { log.error("process: "+shortError(e)); } -- 2.17.1 From a33ce5af7047a0cec3033ccc9ab2b589f0e6598d Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 04:44:19 -0400 Subject: [PATCH 17/78] increase ping tries, set ping all timeout based on max tries and ping timeout --- .../main/java/bubble/service/device/FlexRouterService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index 6643bf25..2214b2d5 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -36,7 +36,8 @@ import static org.cobbzilla.util.system.Sleep.sleep; @Service @Slf4j public class FlexRouterService extends SimpleDaemon { - public static final int MAX_PING_TRIES = 5; + public static final int MAX_PING_TRIES = 10; + private static final long PING_SLEEP_FACTOR = SECONDS.toMillis(2); // HttpClient timeouts are in seconds @@ -52,7 +53,7 @@ public class FlexRouterService extends SimpleDaemon { .build(); } - private static final long PING_ALL_TIMEOUT = MINUTES.toMillis(2); + private static final long PING_ALL_TIMEOUT = (DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + SECONDS.toMillis(10); @Getter private final long sleepTime = MINUTES.toMillis(2); -- 2.17.1 From 36a6ab7c5f0563651987f31b87c2bc945bc65774 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 04:46:16 -0400 Subject: [PATCH 18/78] fix ping all timeout --- .../src/main/java/bubble/service/device/FlexRouterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index 2214b2d5..c6a061fa 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -53,7 +53,8 @@ public class FlexRouterService extends SimpleDaemon { .build(); } - private static final long PING_ALL_TIMEOUT = (DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + SECONDS.toMillis(10); + private static final long PING_ALL_TIMEOUT + = (SECONDS.toMillis(1) * DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + SECONDS.toMillis(10); @Getter private final long sleepTime = MINUTES.toMillis(2); -- 2.17.1 From fbb1a2b69965a94d147e35bcc89d5c023b7b4dd7 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 10:12:54 -0400 Subject: [PATCH 19/78] use socat to create tunnels to flex routers. name thread pools --- bin/create_flex_tunnel | 26 +++ bin/flex_tunnel_pid | 5 + bin/stop_flex_tunnel | 25 +++ .../cloud/compute/ec2/AmazonEC2Driver.java | 2 +- .../java/bubble/dao/device/FlexRouterDAO.java | 30 ++++ .../java/bubble/model/device/FlexRouter.java | 5 +- .../resources/device/FlexRoutersResource.java | 2 +- .../listener/NodeInitializerListener.java | 4 +- .../java/bubble/service/cloud/GeoService.java | 2 +- .../service/cloud/StandardNetworkService.java | 2 +- .../service/device/FlexRouterService.java | 130 +-------------- .../service/device/FlexRouterTunnel.java | 71 +++++++++ .../device/FlexRouterTunnelException.java | 13 ++ .../device/StandardFlexRouterService.java | 150 ++++++++++++++++++ .../bubble/service/packer/PackerService.java | 2 +- .../stream/StandardAppPrimerService.java | 2 +- .../DbFilterFlexRouterService.java | 16 ++ .../main/resources/ansible/bubble_scripts.txt | 5 +- bubble-server/src/main/resources/logback.xml | 1 + .../packer/roles/bubble/tasks/main.yml | 4 +- utils/cobbzilla-utils | 2 +- utils/cobbzilla-wizard | 2 +- 22 files changed, 360 insertions(+), 141 deletions(-) create mode 100755 bin/create_flex_tunnel create mode 100755 bin/flex_tunnel_pid create mode 100755 bin/stop_flex_tunnel create mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterTunnel.java create mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterTunnelException.java create mode 100644 bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java create mode 100644 bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java diff --git a/bin/create_flex_tunnel b/bin/create_flex_tunnel new file mode 100755 index 00000000..700c086a --- /dev/null +++ b/bin/create_flex_tunnel @@ -0,0 +1,26 @@ +#!/bin/bash + +BIND_ADDR=${1:?no bind address provided} +BIND_PORT=${2:?no bind port provided} +DEST_ADDR=${3:?no dest address provided} +DEST_PORT=${4:?no dest port provided} +PROTO=${5:?no protocol provided, use TCP4 or TCP6} + +if [[ "${PROTO}" != "TCP4" && "${PROTO}" != "TCP6" ]] ; then + echo "invalid protocol (use TCP4 or TCP6): ${PROTO}" + exit 1 +fi + +RANGE="127.0.0.1/8" +if [[ ${PROTO} == "TCP6" ]] ; then + RANGE="fd00::/8" +fi + +THIS_DIR=$(cd $(dirname $0) && pwd) +FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" +if [[ -z "${FLEX_PID}" ]] ; then + set -m + socat ${PROTO}-LISTEN:${BIND_PORT},bind=${BIND_ADDR},range=${RANGE},fork,reuseaddr ${PROTO}:${DEST_ADDR}:${DEST_PORT} & +else + echo "tunnel already running, pid = ${FLEX_PID}" +fi diff --git a/bin/flex_tunnel_pid b/bin/flex_tunnel_pid new file mode 100755 index 00000000..9a93b8e3 --- /dev/null +++ b/bin/flex_tunnel_pid @@ -0,0 +1,5 @@ +#!/bin/bash + +DEST_ADDR=${1:?no dest addr provided} + +ps auxww | grep socat | grep TCP[46]:${DEST_ADDR} | awk '{print $2}' diff --git a/bin/stop_flex_tunnel b/bin/stop_flex_tunnel new file mode 100755 index 00000000..a6094b2d --- /dev/null +++ b/bin/stop_flex_tunnel @@ -0,0 +1,25 @@ +#!/bin/bash + +DEST_ADDR=${1:?no dest addr provided} + +THIS_DIR=$(cd $(dirname $0) && pwd) + +FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" +if [[ ! -z "${FLEX_PID}" ]] ; then + echo "stopping tunnel with pid = ${FLEX_PID}" + kill -INT ${FLEX_PID} + sleep 2s + FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" + if [[ ! -z "${FLEX_PID}" ]] ; then + echo "killing tunnel with pid = ${FLEX_PID}" + kill -9 ${FLEX_PID} + sleep 2s + fi + FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" + if [[ ! -z "${FLEX_PID}" ]] ; then + echo 1>&2 "error killing tunnel with pid = ${FLEX_PID}" + exit 1 + fi +fi + +exit 0 diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java index 2873d9ee..3e13f559 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java @@ -95,7 +95,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { @Getter(lazy=true) private final RedisService imageCache = redis.prefixNamespace(getClass().getSimpleName()+".ec2_ubuntu_image"); public static final long IMAGE_CACHE_TIME = DAYS.toSeconds(30); - @Getter(lazy=true) private final ExecutorService perRegionExecutor = fixedPool(getRegions().size()); + @Getter(lazy=true) private final ExecutorService perRegionExecutor = fixedPool(getRegions().size(), "AmazonEC2Driver.perRegionExecutor"); @Getter(lazy=true) private final List cloudOsImages = initImages(); private List initImages() { diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index e8261e48..82b27280 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -2,7 +2,9 @@ package bubble.dao.device; import bubble.dao.account.AccountOwnedEntityDAO; import bubble.model.device.FlexRouter; +import bubble.service.device.FlexRouterService; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import java.util.List; @@ -13,8 +15,36 @@ import static org.hibernate.criterion.Restrictions.*; @Repository @Slf4j public class FlexRouterDAO extends AccountOwnedEntityDAO { + @Autowired private FlexRouterService flexRouterService; + @Override protected String getNameField() { return "ip"; } + @Override public FlexRouter postCreate(FlexRouter router, Object context) { + flexRouterService.register(router); + return super.postCreate(router, context); + } + + @Override public Object preUpdate(FlexRouter router) { + final FlexRouter existing = findByUuid(router.getUuid()); + router.setPortChanged(!existing.getPort().equals(router.getPort())); + return super.preUpdate(router); + } + + @Override public FlexRouter postUpdate(FlexRouter router, Object context) { + if (router.isPortChanged()) { + flexRouterService.unregister(router); + } + flexRouterService.register(router); + return super.postUpdate(router, context); + } + + @Override public void delete(String uuid) { + final FlexRouter router = findByUuid(uuid); + if (router == null) return; + flexRouterService.unregister(router); + super.delete(uuid); + } + public List findEnabledAndRegistered() { return list(criteria().add(and( eq("enabled", true), diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 24fdcc2d..9a5dd68b 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -20,6 +20,7 @@ import javax.persistence.Entity; import javax.persistence.Transient; import static bubble.ApiConstants.EP_FLEX_ROUTERS; +import static bubble.service.device.FlexRouterTunnel.localTunnelAddr; import static org.cobbzilla.util.daemon.ZillaRuntime.bool; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; @@ -51,6 +52,8 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { @JsonIgnore @Getter @Setter private Integer port = 0; public boolean hasPort () { return port != null && port > 1024; } + @JsonIgnore @Transient @Getter @Setter private boolean portChanged = false; + // used for sending the port, we never send it back @Transient @Getter @Setter private Integer proxy_port; public boolean hasProxyPort () { return proxy_port != null && proxy_port > 1024 && proxy_port < 65535; } @@ -82,7 +85,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { @Transient @Getter @Setter private String auth_token; public boolean hasAuthToken () { return !empty(auth_token); } - public String pingUrl() { return "http://" + getIp() + ":" + getPort() + "/ping"; } + public String pingUrl() { return "http://" + localTunnelAddr(getIp()) + ":" + getPort() + "/ping"; } public FlexRouterPing pingObject() { return new FlexRouterPing(this); } } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 5a875a75..179b02eb 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -11,7 +11,7 @@ import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; -import static bubble.service.device.FlexRouterService.pingFlexRouter; +import static bubble.service.device.StandardFlexRouterService.pingFlexRouter; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; diff --git a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java index 38fd9ea7..179653c0 100644 --- a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java @@ -15,7 +15,7 @@ import bubble.server.BubbleConfiguration; import bubble.service.boot.SelfNodeService; import bubble.service.device.DeviceService; import bubble.service.cloud.NetworkMonitorService; -import bubble.service.device.FlexRouterService; +import bubble.service.device.StandardFlexRouterService; import bubble.service.stream.AppDataCleaner; import bubble.service.stream.AppPrimerService; import lombok.extern.slf4j.Slf4j; @@ -113,7 +113,7 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase> backgroundLookups = new ConcurrentHashMap<>(); - private final ExecutorService backgroundLookupExec = fixedPool(5); + private final ExecutorService backgroundLookupExec = fixedPool(5, "GeoService.backgroundLookupExec"); public GeoLocation locate (String accountUuid, String ip, boolean cacheOnly) { final String cacheKey = hashOf(accountUuid, ip); diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index b5513902..5d34410c 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -150,7 +150,7 @@ public class StandardNetworkService implements NetworkService { String lock = nn.getLock(); NodeProgressMeter progressMeter = null; final BubbleNetwork network = nn.getNetworkObject(); - final ExecutorService backgroundJobs = DaemonThreadFactory.fixedPool(3); + final ExecutorService backgroundJobs = DaemonThreadFactory.fixedPool(3, "StandardNetworkService.backgroundJobs"); boolean killNode = false; try { progressMeter = launchMonitor.getProgressMeter(nn); diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index c6a061fa..3939b0db 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -1,134 +1,10 @@ package bubble.service.device; -import bubble.dao.device.FlexRouterDAO; import bubble.model.device.FlexRouter; -import bubble.model.device.FlexRouterPing; -import lombok.Cleanup; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.cobbzilla.util.daemon.AwaitResult; -import org.cobbzilla.util.daemon.SimpleDaemon; -import org.cobbzilla.util.http.HttpRequestBean; -import org.cobbzilla.util.http.HttpResponseBean; -import org.cobbzilla.util.http.HttpUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; +public interface FlexRouterService { -import static bubble.model.device.FlexRouterPing.MAX_PING_AGE; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.cobbzilla.util.daemon.Await.awaitAll; -import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool; -import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; -import static org.cobbzilla.util.http.HttpMethods.POST; -import static org.cobbzilla.util.json.JsonUtil.json; -import static org.cobbzilla.util.system.Sleep.sleep; - -@Service @Slf4j -public class FlexRouterService extends SimpleDaemon { - - public static final int MAX_PING_TRIES = 10; - - private static final long PING_SLEEP_FACTOR = SECONDS.toMillis(2); - - // HttpClient timeouts are in seconds - public static final int DEFAULT_PING_TIMEOUT = (int) SECONDS.toSeconds(MAX_PING_AGE/2); - public static final RequestConfig DEFAULT_PING_REQUEST_CONFIG = RequestConfig.custom() - .setConnectTimeout(DEFAULT_PING_TIMEOUT) - .setSocketTimeout(DEFAULT_PING_TIMEOUT) - .setConnectionRequestTimeout(DEFAULT_PING_TIMEOUT).build(); - - private static CloseableHttpClient getHttpClient() { - return HttpClientBuilder.create() - .setDefaultRequestConfig(DEFAULT_PING_REQUEST_CONFIG) - .build(); - } - - private static final long PING_ALL_TIMEOUT - = (SECONDS.toMillis(1) * DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + SECONDS.toMillis(10); - - @Getter private final long sleepTime = MINUTES.toMillis(2); - - @Autowired private FlexRouterDAO flexRouterDAO; - - @Override protected void process() { - try { - @Cleanup final CloseableHttpClient httpClient = getHttpClient(); - final List routers = flexRouterDAO.findEnabledAndRegistered(); - final List> futures = new ArrayList<>(); - @Cleanup("shutdownNow") final ExecutorService exec = fixedPool(5); - for (FlexRouter router : routers) { - futures.add(exec.submit(() -> { - boolean active = pingFlexRouter(router, flexRouterDAO, httpClient); - if (active != router.active()) { - flexRouterDAO.update(router.setActive(active)); - } - return active; - })); - } - final AwaitResult awaitResult = awaitAll(futures, PING_ALL_TIMEOUT); - log.debug("process: awaitResult="+awaitResult); - - } catch (Exception e) { - log.error("process: "+shortError(e)); - } - } - - public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO) { - try { - @Cleanup final CloseableHttpClient httpClient = getHttpClient(); - return pingFlexRouter(router, flexRouterDAO, httpClient); - } catch (Exception e) { - log.error("pingFlexRouter("+router+"): "+shortError(e)); - return false; - } - } - - public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO, HttpClient httpClient) { - final String prefix = "pingRouter(" + router + "): "; - final HttpRequestBean request = new HttpRequestBean(POST, router.pingUrl()); - for (int i=0; i routers = flexRouterDAO.findEnabledAndRegistered(); + if (log.isDebugEnabled()) log.debug("process: starting, will ping "+routers.size()+" routers"); + final List> futures = new ArrayList<>(); + @Cleanup("shutdownNow") final ExecutorService exec = fixedPool(DEFAULT_MAX_TUNNELS, "StandardFlexRouterService.process"); + for (FlexRouter router : routers) { + futures.add(exec.submit(() -> { + boolean active = pingFlexRouter(router, flexRouterDAO, httpClient); + if (active != router.active()) { + flexRouterDAO.update(router.setActive(active)); + } + return active; + })); + } + final AwaitResult awaitResult = awaitAll(futures, PING_ALL_TIMEOUT); + if (log.isDebugEnabled()) log.debug("process: awaitResult="+awaitResult); + + } catch (Exception e) { + log.error("process: "+shortError(e)); + } + } + + public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO) { + try { + @Cleanup final CloseableHttpClient httpClient = getHttpClient(); + return pingFlexRouter(router, flexRouterDAO, httpClient); + } catch (Exception e) { + log.error("pingFlexRouter("+router+"): "+shortError(e)); + return false; + } + } + + public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO, HttpClient httpClient) { + final String pingUrl = router.pingUrl(); + final HttpRequestBean request = new HttpRequestBean(POST, pingUrl); + final String prefix = "pingRouter(" + router + "): "; + for (int i=0; i activeJobs = new ConcurrentHashMap<>(16); private final Map> completedJobs = new ConcurrentHashMap<>(16); - private final ExecutorService pool = DaemonThreadFactory.fixedPool(5); + private final ExecutorService pool = DaemonThreadFactory.fixedPool(5, "PackerService.pool"); @Autowired private BubbleConfiguration configuration; diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index d6295f14..fa6e435f 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -99,7 +99,7 @@ public class StandardAppPrimerService implements AppPrimerService { prime(account, singleApp); } - @Getter(lazy=true) private final ExecutorService primerThread = fixedPool(1); + @Getter(lazy=true) private final ExecutorService primerThread = fixedPool(1, "StandardAppPrimerService.primerThread"); private void prime(Account account, BubbleApp singleApp) { if (!isPrimingEnabled()) { diff --git a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java new file mode 100644 index 00000000..3e38722b --- /dev/null +++ b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java @@ -0,0 +1,16 @@ +package bubble.service_dbfilter; + +import bubble.model.device.FlexRouter; +import bubble.service.device.FlexRouterService; +import org.springframework.stereotype.Service; + +import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; + +@Service +public class DbFilterFlexRouterService implements FlexRouterService { + + @Override public void register(FlexRouter router) { notSupported("register"); } + + @Override public void unregister(FlexRouter router) { notSupported("unregister"); } + +} diff --git a/bubble-server/src/main/resources/ansible/bubble_scripts.txt b/bubble-server/src/main/resources/ansible/bubble_scripts.txt index 2e4a0af2..bacfd132 100644 --- a/bubble-server/src/main/resources/ansible/bubble_scripts.txt +++ b/bubble-server/src/main/resources/ansible/bubble_scripts.txt @@ -18,4 +18,7 @@ cleanup_bubble_databases install_packer.sh rkeys rmembers -rdelkeys \ No newline at end of file +rdelkeys +create_flex_tunnel +flex_tunnel_pid +stop_flex_tunnel \ No newline at end of file diff --git a/bubble-server/src/main/resources/logback.xml b/bubble-server/src/main/resources/logback.xml index 6f12d789..f20331bc 100644 --- a/bubble-server/src/main/resources/logback.xml +++ b/bubble-server/src/main/resources/logback.xml @@ -39,6 +39,7 @@ + diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml index 2025e6f5..c7917b00 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml @@ -1,9 +1,9 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # -- name: Install OpenJDK 11 JRE (headless), redis, uuid and jq +- name: Install OpenJDK 11 JRE (headless), redis, uuid, jq, zip and socat apt: - name: [ 'openjdk-11-jre-headless', 'redis', 'uuid', 'jq', 'zip' ] + name: [ 'openjdk-11-jre-headless', 'redis', 'uuid', 'jq', 'zip', 'socat' ] state: present update_cache: yes diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index ecf004ea..d6dfc2a4 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit ecf004ea779a1e46a6ee90866fba76442562d1ed +Subproject commit d6dfc2a4e57fc787458dcd911353e38b93ba01df diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 7727825b..416d45b5 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 7727825b5e738168a42e4148d873c3f75d572f91 +Subproject commit 416d45b5ba87b5c798b2940f0bf357461c3a43c0 -- 2.17.1 From 278e0d72346129d37deab1fe61c48f378525a19e Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 10:18:50 -0400 Subject: [PATCH 20/78] move pingUrl to FlexRouterTunnel, use correct port --- .../src/main/java/bubble/model/device/FlexRouter.java | 2 +- .../main/java/bubble/service/device/FlexRouterTunnel.java | 4 ++++ .../bubble/service/device/StandardFlexRouterService.java | 5 ++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 9a5dd68b..5eab25cf 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -2,6 +2,7 @@ package bubble.model.device; import bubble.model.account.Account; import bubble.model.account.HasAccount; +import bubble.service.device.FlexRouterTunnel; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; @@ -85,7 +86,6 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { @Transient @Getter @Setter private String auth_token; public boolean hasAuthToken () { return !empty(auth_token); } - public String pingUrl() { return "http://" + localTunnelAddr(getIp()) + ":" + getPort() + "/ping"; } public FlexRouterPing pingObject() { return new FlexRouterPing(this); } } diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterTunnel.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterTunnel.java index 21b87417..4da09413 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterTunnel.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterTunnel.java @@ -68,4 +68,8 @@ public class FlexRouterTunnel { return "127" + ip.substring(firstDot); } + public static String tunnelPingUrl(FlexRouter router) { + return "http://" + localTunnelAddr(router.getIp()) + ":" + TUNNEL_PORT + "/ping"; + } + } diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index ddae2756..033a7931 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -24,8 +24,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import static bubble.model.device.FlexRouterPing.MAX_PING_AGE; -import static bubble.service.device.FlexRouterTunnel.startFlexTunnel; -import static bubble.service.device.FlexRouterTunnel.stopFlexTunnel; +import static bubble.service.device.FlexRouterTunnel.*; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.cobbzilla.util.daemon.Await.awaitAll; @@ -109,7 +108,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute } public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO, HttpClient httpClient) { - final String pingUrl = router.pingUrl(); + final String pingUrl = tunnelPingUrl(router); final HttpRequestBean request = new HttpRequestBean(POST, pingUrl); final String prefix = "pingRouter(" + router + "): "; for (int i=0; i Date: Sun, 6 Sep 2020 14:04:02 -0400 Subject: [PATCH 21/78] WIP. moving to ssh tunnel connection for flex routers --- bin/create_flex_tunnel | 26 ----- bin/flex_tunnel_pid | 5 - bin/stop_flex_tunnel | 25 ---- .../java/bubble/dao/device/FlexRouterDAO.java | 17 ++- .../bubble/model/account/AccountSshKey.java | 4 +- .../java/bubble/model/device/FlexRouter.java | 63 +++++++---- .../resources/device/FlexRoutersResource.java | 45 ++++++-- .../service/device/FlexRouterTunnel.java | 75 ------------ .../device/FlexRouterTunnelException.java | 13 --- .../device/StandardFlexRouterService.java | 107 ++++++++++++++---- .../main/resources/ansible/bubble_scripts.txt | 5 +- .../V2020090501__add_flex_router.sql | 13 ++- bubble-server/src/main/resources/messages | 2 +- .../bubble/files/refresh_flex_keys_monitor.sh | 46 ++++++++ .../supervisor_refresh_flex_keys_monitor.conf | 5 + .../packer/roles/bubble/tasks/main.yml | 10 +- .../packer/roles/common/tasks/main.yml | 9 ++ utils/cobbzilla-utils | 2 +- 18 files changed, 258 insertions(+), 214 deletions(-) delete mode 100755 bin/create_flex_tunnel delete mode 100755 bin/flex_tunnel_pid delete mode 100755 bin/stop_flex_tunnel delete mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterTunnel.java delete mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterTunnelException.java create mode 100644 bubble-server/src/main/resources/packer/roles/bubble/files/refresh_flex_keys_monitor.sh create mode 100644 bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_flex_keys_monitor.conf diff --git a/bin/create_flex_tunnel b/bin/create_flex_tunnel deleted file mode 100755 index 700c086a..00000000 --- a/bin/create_flex_tunnel +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -BIND_ADDR=${1:?no bind address provided} -BIND_PORT=${2:?no bind port provided} -DEST_ADDR=${3:?no dest address provided} -DEST_PORT=${4:?no dest port provided} -PROTO=${5:?no protocol provided, use TCP4 or TCP6} - -if [[ "${PROTO}" != "TCP4" && "${PROTO}" != "TCP6" ]] ; then - echo "invalid protocol (use TCP4 or TCP6): ${PROTO}" - exit 1 -fi - -RANGE="127.0.0.1/8" -if [[ ${PROTO} == "TCP6" ]] ; then - RANGE="fd00::/8" -fi - -THIS_DIR=$(cd $(dirname $0) && pwd) -FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" -if [[ -z "${FLEX_PID}" ]] ; then - set -m - socat ${PROTO}-LISTEN:${BIND_PORT},bind=${BIND_ADDR},range=${RANGE},fork,reuseaddr ${PROTO}:${DEST_ADDR}:${DEST_PORT} & -else - echo "tunnel already running, pid = ${FLEX_PID}" -fi diff --git a/bin/flex_tunnel_pid b/bin/flex_tunnel_pid deleted file mode 100755 index 9a93b8e3..00000000 --- a/bin/flex_tunnel_pid +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -DEST_ADDR=${1:?no dest addr provided} - -ps auxww | grep socat | grep TCP[46]:${DEST_ADDR} | awk '{print $2}' diff --git a/bin/stop_flex_tunnel b/bin/stop_flex_tunnel deleted file mode 100755 index a6094b2d..00000000 --- a/bin/stop_flex_tunnel +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -DEST_ADDR=${1:?no dest addr provided} - -THIS_DIR=$(cd $(dirname $0) && pwd) - -FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" -if [[ ! -z "${FLEX_PID}" ]] ; then - echo "stopping tunnel with pid = ${FLEX_PID}" - kill -INT ${FLEX_PID} - sleep 2s - FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" - if [[ ! -z "${FLEX_PID}" ]] ; then - echo "killing tunnel with pid = ${FLEX_PID}" - kill -9 ${FLEX_PID} - sleep 2s - fi - FLEX_PID="$(${THIS_DIR}/flex_tunnel_pid ${DEST_ADDR})" - if [[ ! -z "${FLEX_PID}" ]] ; then - echo 1>&2 "error killing tunnel with pid = ${FLEX_PID}" - exit 1 - fi -fi - -exit 0 diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index 82b27280..edb7801e 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -10,6 +10,7 @@ import org.springframework.stereotype.Repository; import java.util.List; import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; import static org.hibernate.criterion.Restrictions.*; @Repository @Slf4j @@ -19,6 +20,10 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override protected String getNameField() { return "ip"; } + @Override public Object preCreate(FlexRouter router) { + return super.preCreate(router.setInitialized(false)); + } + @Override public FlexRouter postCreate(FlexRouter router, Object context) { flexRouterService.register(router); return super.postCreate(router, context); @@ -26,14 +31,13 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public Object preUpdate(FlexRouter router) { final FlexRouter existing = findByUuid(router.getUuid()); - router.setPortChanged(!existing.getPort().equals(router.getPort())); + if (!existing.getIp().equals(router.getIp())) throw invalidEx("err.ip.cannotChange"); + if (!existing.getPort().equals(router.getPort())) throw invalidEx("err.port.cannotSetOrChange"); + if (!existing.getKeyHash().equals(router.getKeyHash())) throw invalidEx("err.sshPublicKey.cannotChange"); return super.preUpdate(router); } @Override public FlexRouter postUpdate(FlexRouter router, Object context) { - if (router.isPortChanged()) { - flexRouterService.unregister(router); - } flexRouterService.register(router); return super.postUpdate(router, context); } @@ -56,6 +60,7 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { public List findActive(long maxAge) { return list(criteria().add(and( eq("active", true), + eq("initialized", true), eq("enabled", true), gt("port", 1024), le("port", 65535), @@ -63,4 +68,8 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { isNotNull("token")))); } + public FlexRouter findByPort(int port) { return findByUniqueField("port", port); } + + public FlexRouter findByKeyHash(String keyHash) { return findByUniqueField("keyHash", keyHash); } + } diff --git a/bubble-server/src/main/java/bubble/model/account/AccountSshKey.java b/bubble-server/src/main/java/bubble/model/account/AccountSshKey.java index 9d609db2..2759798d 100644 --- a/bubble-server/src/main/java/bubble/model/account/AccountSshKey.java +++ b/bubble-server/src/main/java/bubble/model/account/AccountSshKey.java @@ -57,8 +57,8 @@ public class AccountSshKey extends IdentifiableBase implements HasAccount { @Type(type=ENCRYPTED_STRING) @Column(updatable=false, columnDefinition="varchar("+(10000+ENC_PAD)+") NOT NULL") @Getter private String sshPublicKey; public AccountSshKey setSshPublicKey(String k) { - this.sshPublicKey = k; - if (!empty(k)) this.sshPublicKeyHash = sha256_hex(k); + this.sshPublicKey = k.trim(); + if (!empty(sshPublicKey)) this.sshPublicKeyHash = sha256_hex(sshPublicKey); return this; } public boolean hasSshPublicKey () { return !empty(sshPublicKey); } diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 5eab25cf..4385e22e 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -2,7 +2,6 @@ package bubble.model.device; import bubble.model.account.Account; import bubble.model.account.HasAccount; -import bubble.service.device.FlexRouterTunnel; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; @@ -14,6 +13,7 @@ import org.cobbzilla.util.collection.ArrayUtil; import org.cobbzilla.wizard.model.Identifiable; import org.cobbzilla.wizard.model.IdentifiableBase; import org.cobbzilla.wizard.model.entityconfig.annotations.*; +import org.cobbzilla.wizard.validation.HasValue; import org.hibernate.annotations.Type; import javax.persistence.Column; @@ -21,10 +21,10 @@ import javax.persistence.Entity; import javax.persistence.Transient; import static bubble.ApiConstants.EP_FLEX_ROUTERS; -import static bubble.service.device.FlexRouterTunnel.localTunnelAddr; import static org.cobbzilla.util.daemon.ZillaRuntime.bool; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; +import static org.cobbzilla.util.security.ShaUtil.sha256_hex; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING; import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @@ -35,50 +35,65 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) }) public class FlexRouter extends IdentifiableBase implements HasAccount { - public static final String[] UPDATE_FIELDS = { "enabled", "active", "proxy_port", "auth_token", "token" }; - public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip"); + public static final String[] UPDATE_FIELDS = { "enabled", "active", "auth_token", "token", "key" }; + public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip", "port"); public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); } @Override public Identifiable update(Identifiable other) { copy(this, other, UPDATE_FIELDS); return this; } - @ECSearchable(filter=true) @ECField(index=10) - @ECIndex @Column(nullable=false, length=500) + @ECSearchable @ECField(index=10) + @ECForeignKey(entity=Account.class) + @Column(nullable=false, updatable=false, length=UUID_MAXLEN) + @Getter @Setter private String account; + + @ECSearchable(filter=true) @ECField(index=20) + @ECIndex @Column(nullable=false, updatable=false, length=50) @Getter @Setter private String ip; @JsonIgnore @Transient public String getName () { return getIp(); } - @ECSearchable(filter=true) @ECField(index=20) - @ECIndex @Column(nullable=false) - @JsonIgnore @Getter @Setter private Integer port = 0; + @ECField(index=30) @HasValue(message="err.sshPublicKey.required") + @Type(type=ENCRYPTED_STRING) @Column(updatable=false, columnDefinition="varchar("+(10000+ENC_PAD)+") NOT NULL") + @Getter private String key; + public boolean hasKey () { return !empty(key); } + public FlexRouter setKey(String k) { + this.key = k.trim(); + if (!empty(key)) this.keyHash = sha256_hex(key); + return this; + } + + @ECField(index=40) + @ECIndex(unique=true) @Column(length=100, updatable=false, nullable=false) + @Getter @Setter private String keyHash; + public boolean hasKeyHash () { return !empty(keyHash); }; + + @ECSearchable(filter=true) @ECField(index=50) + @ECIndex(unique=true) @Column(nullable=false, updatable=false) + @Getter @Setter private Integer port; public boolean hasPort () { return port != null && port > 1024; } - @JsonIgnore @Transient @Getter @Setter private boolean portChanged = false; - - // used for sending the port, we never send it back - @Transient @Getter @Setter private Integer proxy_port; - public boolean hasProxyPort () { return proxy_port != null && proxy_port > 1024 && proxy_port < 65535; } - public String id () { return getIp() + "/" + getUuid(); } - @ECSearchable @ECField(index=30) - @ECForeignKey(entity=Account.class) - @Column(nullable=false, updatable=false, length=UUID_MAXLEN) - @Getter @Setter private String account; - - @ECSearchable @ECField(index=40) + @ECSearchable @ECField(index=60) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean enabled = true; public boolean enabled () { return bool(enabled); } - @ECSearchable @ECField(index=50) + @ECSearchable @ECField(index=70) + @ECIndex @Column(nullable=false) + @Getter @Setter private Boolean initialized = true; + public boolean initialized() { return bool(initialized); } + public boolean uninitialized() { return !initialized(); } + + @ECSearchable @ECField(index=80) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean active = true; public boolean active() { return bool(active); } public boolean inactive() { return !active(); } - @ECSearchable(filter=true) @ECField(index=60) - @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(100+ENC_PAD)+")") + @ECSearchable(filter=true) @ECField(index=90) + @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(100+ENC_PAD)+") NOT NULL") @JsonIgnore @Getter @Setter private String token; public boolean hasToken () { return !empty(token); } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 179b02eb..e7daf2fc 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -7,13 +7,15 @@ import bubble.model.device.FlexRouter; import bubble.resources.account.AccountOwnedResource; import bubble.service.device.DeviceService; import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.network.PortPicker; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; -import static bubble.service.device.StandardFlexRouterService.pingFlexRouter; -import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; -import static org.cobbzilla.util.json.JsonUtil.json; +import java.math.BigInteger; +import java.net.InetAddress; + +import static org.cobbzilla.util.network.PortPicker.portIsAvailable; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; import static org.cobbzilla.wizard.resources.ResourceUtil.userPrincipal; @@ -29,10 +31,13 @@ public class FlexRoutersResource extends AccountOwnedResource { - boolean active = pingFlexRouter(router, flexRouterDAO, httpClient); - if (active != router.active()) { - flexRouterDAO.update(router.setActive(active)); + final long firstTimeDelay = now() - router.getCtime(); + if (firstTimeDelay < FIRST_TIME_WAIT) { + sleep(FIRST_TIME_WAIT - firstTimeDelay, "process: waiting for flex ssh key"); + } + boolean active = pingFlexRouter(router, httpClient); + if (active != router.active() || (active && router.uninitialized())) { + if (active && router.uninitialized()) { + router.setInitialized(true); + } + router.setActive(active); + flexRouterDAO.update(router); } return active; })); @@ -96,19 +117,9 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute log.error("process: "+shortError(e)); } } - - public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO) { - try { - @Cleanup final CloseableHttpClient httpClient = getHttpClient(); - return pingFlexRouter(router, flexRouterDAO, httpClient); - } catch (Exception e) { - log.error("pingFlexRouter("+router+"): "+shortError(e)); - return false; - } - } - - public static boolean pingFlexRouter(FlexRouter router, FlexRouterDAO flexRouterDAO, HttpClient httpClient) { - final String pingUrl = tunnelPingUrl(router); + public boolean pingFlexRouter(FlexRouter router, HttpClient httpClient) { + allowFlexKey(router.getKey()); + final String pingUrl = "http://127.0.0.1:" + router.getPort() + "/ping"; final HttpRequestBean request = new HttpRequestBean(POST, pingUrl); final String prefix = "pingRouter(" + router + "): "; for (int i=0; i line.trim().equals(trimmedKey))) { + log.info("allowKey: already present: "+trimmedKey); + } else { + @Cleanup("delete") final File temp = temp("flex_keys_", ".tmp"); + final String dataToWrite = authFileContents != null && authFileContents.endsWith("\n") ? trimmedKey + "\n" : "\n" + trimmedKey + "\n"; + toFileOrDie(temp, dataToWrite, true); + renameOrDie(temp, authFile); + log.info("allowKey: added key: "+trimmedKey); + } + } + + public synchronized void disallowFlexKey(String key) { + final String trimmedKey = key.trim(); + final File authFile = getAuthFile(); + final String authFileContents = FileUtil.toStringOrDie(authFile); + final String[] lines = authFileContents == null ? EMPTY_ARRAY : authFileContents.split("\n"); + final StringBuilder b = new StringBuilder(); + boolean found = false; + for (String line : lines) { + if (b.length() > 0) b.append("\n"); + if (line.trim().equals(trimmedKey)) { + found = true; + } else { + b.append(line); + } + } + b.append("\n"); + @Cleanup("delete") final File temp = temp("flex_keys_", ".tmp"); + toFileOrDie(temp, b.toString()); + renameOrDie(temp, authFile); + if (found) { + log.info("disallowKey: removed key from authorized_keys file: "+trimmedKey); + } else { + log.info("disallowKey: key was not found in authorized_keys file: "+trimmedKey); + } + } + + private static File getAuthFile() { + final File sshDir = new File(HOME_DIR+"/.ssh"); + if (!sshDir.exists()) { + mkdirOrDie(sshDir); + } + chmod(sshDir, "700"); + final File authFile = new File(sshDir, "flex_authorized_keys"); + if (!authFile.exists()) { + touch(authFile); + } + chmod(authFile, "600"); + return authFile; + } + } diff --git a/bubble-server/src/main/resources/ansible/bubble_scripts.txt b/bubble-server/src/main/resources/ansible/bubble_scripts.txt index bacfd132..2e4a0af2 100644 --- a/bubble-server/src/main/resources/ansible/bubble_scripts.txt +++ b/bubble-server/src/main/resources/ansible/bubble_scripts.txt @@ -18,7 +18,4 @@ cleanup_bubble_databases install_packer.sh rkeys rmembers -rdelkeys -create_flex_tunnel -flex_tunnel_pid -stop_flex_tunnel \ No newline at end of file +rdelkeys \ No newline at end of file diff --git a/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql b/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql index 16e8e0e9..e3256cfc 100644 --- a/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql +++ b/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql @@ -5,9 +5,12 @@ CREATE TABLE flex_router ( account character varying(100) NOT NULL, active boolean NOT NULL, enabled boolean NOT NULL, - ip character varying(500) NOT NULL, + initialized boolean NOT NULL, + ip character varying(50) NOT NULL, + key character varying(10100) NOT NULL, + key_hash character varying(100) NOT NULL, port integer NOT NULL, - token character varying(200) + token character varying(200) NOT NULL ); ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_pkey PRIMARY KEY (uuid); @@ -15,8 +18,10 @@ ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_pkey PRIMARY KEY (uuid); CREATE INDEX flex_router_idx_account ON flex_router USING btree (account); CREATE INDEX flex_router_idx_active ON flex_router USING btree (active); CREATE INDEX flex_router_idx_enabled ON flex_router USING btree (enabled); +CREATE INDEX flex_router_idx_initialized ON flex_router USING btree (initialized); CREATE INDEX flex_router_idx_ip ON flex_router USING btree (ip); -CREATE INDEX flex_router_idx_port ON flex_router USING btree (port); CREATE UNIQUE INDEX flex_router_uniq_account_ip ON flex_router USING btree (account, ip); +CREATE UNIQUE INDEX flex_router_uniq_key_hash ON flex_router USING btree (key_hash); +CREATE UNIQUE INDEX flex_router_uniq_port ON flex_router USING btree (port); -ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_fk_account FOREIGN KEY (account) REFERENCES public.account(uuid); +ALTER TABLE ONLY flex_router ADD CONSTRAINT flex_router_fk_account FOREIGN KEY (account) REFERENCES account(uuid); diff --git a/bubble-server/src/main/resources/messages b/bubble-server/src/main/resources/messages index a611f6fa..e19810a2 160000 --- a/bubble-server/src/main/resources/messages +++ b/bubble-server/src/main/resources/messages @@ -1 +1 @@ -Subproject commit a611f6fae14e4383455bdf2e1111e75961d8b5f9 +Subproject commit e19810a2f1afe057824b4976183fa7dc7253fd0d diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_flex_keys_monitor.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_flex_keys_monitor.sh new file mode 100644 index 00000000..6c512ffd --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_flex_keys_monitor.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +LOG=/var/log/bubble/flex_keys_monitor.log + +function die { + echo 1>&2 "${1}" + log "${1}" + exit 1 +} + +function log { + echo "$(date): ${1}" >> ${LOG} +} + +SSH_KEY_BASE=/home/bubble-flex/.ssh +if [[ ! -d ${SSH_KEY_BASE} ]] ; then + mkdir ${SSH_KEY_BASE} +fi +chown -R bubble-flex ${SSH_KEY_BASE} && chmod 700 ${SSH_KEY_BASE} + +BUBBLE_FLEX_KEYS=/home/bubble/.ssh/flex_authorized_keys +AUTH_FLEX_KEYS=${SSH_KEY_BASE}/authorized_keys + +if [[ ! -f ${AUTH_FLEX_KEYS} ]] ; then + touch ${AUTH_FLEX_KEYS} +fi +chown bubble-flex ${AUTH_FLEX_KEYS} && chmod 600 ${AUTH_FLEX_KEYS} + +if [[ ! -f ${BUBBLE_FLEX_KEYS} ]] ; then + touch ${BUBBLE_FLEX_KEYS} && chown bubble ${BUBBLE_FLEX_KEYS} && chmod 600 ${BUBBLE_FLEX_KEYS} && sleep 2s +fi + +log "Watching flex keys file ${BUBBLE_FLEX_KEYS} ..." +while : ; do + if [[ $(stat -c %Y ${BUBBLE_FLEX_KEYS}) -gt $(stat -c %Y ${AUTH_FLEX_KEYS}) ]] ; then + cat ${BUBBLE_FLEX_KEYS} > ${AUTH_FLEX_KEYS} \ + && log "Updated ${BUBBLE_FLEX_KEYS} > ${AUTH_FLEX_KEYS}" \ + || log "Error overwriting ${BUBBLE_FLEX_KEYS} > ${AUTH_FLEX_KEYS}" + # Just for sanity's sake + chown -R bubble-flex ${SSH_KEY_BASE} && chmod 700 ${SSH_KEY_BASE} + chown bubble-flex ${AUTH_FLEX_KEYS} && chmod 600 ${AUTH_FLEX_KEYS} + fi + sleep 10s +done diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_flex_keys_monitor.conf b/bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_flex_keys_monitor.conf new file mode 100644 index 00000000..20555517 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_refresh_flex_keys_monitor.conf @@ -0,0 +1,5 @@ + +[program:refresh_flex_keys_monitor] +stdout_logfile = /dev/null +stderr_logfile = /dev/null +command=/usr/local/sbin/refresh_flex_keys_monitor.sh diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml index c7917b00..f3646385 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml @@ -1,9 +1,9 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # -- name: Install OpenJDK 11 JRE (headless), redis, uuid, jq, zip and socat +- name: Install OpenJDK 11 JRE (headless), redis, uuid, jq, and zip apt: - name: [ 'openjdk-11-jre-headless', 'redis', 'uuid', 'jq', 'zip', 'socat' ] + name: [ 'openjdk-11-jre-headless', 'redis', 'uuid', 'jq', 'zip' ] state: present update_cache: yes @@ -95,6 +95,7 @@ with_items: - refresh_bubble_ssh_keys_monitor.sh - refresh_bubble_ssh_keys.sh + - refresh_flex_keys_monitor.sh - bubble_upgrade_monitor.sh - bubble_upgrade.sh - log_manager.sh @@ -104,6 +105,11 @@ src: supervisor_refresh_bubble_ssh_keys_monitor.conf dest: /etc/supervisor/conf.d/refresh_bubble_ssh_keys_monitor.conf +- name: Install refresh_flex_keys_monitor supervisor conf file + copy: + src: supervisor_refresh_flex_keys_monitor.conf + dest: /etc/supervisor/conf.d/refresh_flex_keys_monitor.conf + - name: Install bubble_upgrade_monitor supervisor conf file copy: src: supervisor_bubble_upgrade_monitor.conf diff --git a/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml index b2e4086d..756f6dcd 100644 --- a/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml @@ -84,3 +84,12 @@ group: bubble-log mode: 0770 state: directory + +- name: Create bubble flexrouting user + user: + name: bubble-flex + comment: bubble flexrouting user + shell: /bin/false + system: yes + home: /home/bubble-flex + when: install_type == 'node' diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index d6dfc2a4..df29b42b 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit d6dfc2a4e57fc787458dcd911353e38b93ba01df +Subproject commit df29b42b97be4387787e31139f0723a8c3a189ce -- 2.17.1 From 7fa891448f88dfa4256f4e8824fca7145273addc Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Sun, 6 Sep 2020 23:38:15 -0400 Subject: [PATCH 22/78] WIP. send host key to flex router --- bubble-server/src/main/java/bubble/ApiConstants.java | 7 +++++-- .../src/main/java/bubble/dao/device/FlexRouterDAO.java | 3 +++ .../src/main/java/bubble/model/device/FlexRouter.java | 5 ++++- .../java/bubble/resources/device/FlexRoutersResource.java | 1 - 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index 96594de9..338b676f 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -12,7 +12,6 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomUtils; import org.cobbzilla.util.daemon.ZillaRuntime; -import org.cobbzilla.util.io.FileUtil; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -36,6 +35,7 @@ import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.network.NetworkUtil.*; import static org.cobbzilla.util.string.StringUtil.splitAndTrim; +import static org.cobbzilla.util.system.CommandShell.execScript; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @Slf4j @@ -58,7 +58,7 @@ public class ApiConstants { private static String initDefaultDomain() { final File f = new File(HOME_DIR, ".BUBBLE_DEFAULT_DOMAIN"); - final String domain = FileUtil.toStringOrDie(f); + final String domain = toStringOrDie(f); return domain != null ? domain.trim() : die("initDefaultDomain: "+abs(f)+" not found"); } @@ -75,6 +75,9 @@ public class ApiConstants { public static final GoogleAuthenticator G_AUTH = new GoogleAuthenticator(); + public static final String HOST_KEY = toStringOrDie("/etc/ssh/ssh_host_rsa_key.pub").trim(); + public static final String KNOWN_HOST_KEY = execScript("ssh-keyscan -t rsa $(hostname -d) 2>&1 | grep -v \"^#\""); + public static final Predicate ALWAYS_TRUE = m -> true; public static final String HOME_DIR; static { diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index edb7801e..93bb76d0 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -9,6 +9,7 @@ import org.springframework.stereotype.Repository; import java.util.List; +import static bubble.ApiConstants.KNOWN_HOST_KEY; import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; import static org.hibernate.criterion.Restrictions.*; @@ -26,6 +27,7 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postCreate(FlexRouter router, Object context) { flexRouterService.register(router); + router.setHost_key(KNOWN_HOST_KEY); return super.postCreate(router, context); } @@ -39,6 +41,7 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postUpdate(FlexRouter router, Object context) { flexRouterService.register(router); + router.setHost_key(KNOWN_HOST_KEY); return super.postUpdate(router, context); } diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 4385e22e..07202880 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -35,7 +35,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; @ECIndexes({ @ECIndex(unique=true, of={"account", "ip"}) }) public class FlexRouter extends IdentifiableBase implements HasAccount { - public static final String[] UPDATE_FIELDS = { "enabled", "active", "auth_token", "token", "key" }; + public static final String[] UPDATE_FIELDS = { "enabled", "active", "auth_token", "token", "key", "host_key" }; public static final String[] CREATE_FIELDS = ArrayUtil.append(UPDATE_FIELDS, "ip", "port"); public FlexRouter (FlexRouter other) { copy(this, other, CREATE_FIELDS); } @@ -101,6 +101,9 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { @Transient @Getter @Setter private String auth_token; public boolean hasAuthToken () { return !empty(auth_token); } + // used for sending the SSH host key to flexrouter + @Transient @Getter @Setter private String host_key; + public FlexRouterPing pingObject() { return new FlexRouterPing(this); } } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index e7daf2fc..3d303f6a 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -74,7 +74,6 @@ public class FlexRoutersResource extends AccountOwnedResource Date: Mon, 7 Sep 2020 01:39:08 -0400 Subject: [PATCH 23/78] allow any rule to declare flex domains --- .../src/main/java/bubble/rule/AppRuleDriver.java | 8 ++++++-- .../bubble/rule/passthru/TlsPassthruConfig.java | 8 ++++++++ .../rule/passthru/TlsPassthruRuleDriver.java | 16 +++++++++------- .../service/stream/ConnectionCheckResponse.java | 2 +- .../service/stream/StandardAppPrimerService.java | 12 +++++++++++- .../roles/mitmproxy/files/bubble_conn_check.py | 8 +------- .../packer/roles/mitmproxy/files/dns_spoofing.py | 9 +++++---- 7 files changed, 41 insertions(+), 22 deletions(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 01925704..e9be26ca 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -5,7 +5,6 @@ package bubble.rule; import bubble.model.account.Account; -import bubble.model.app.AppData; import bubble.model.app.AppMatcher; import bubble.model.app.AppRule; import bubble.model.app.BubbleApp; @@ -27,7 +26,6 @@ import java.io.ByteArrayInputStream; import java.io.InputStream; import java.util.Map; import java.util.Set; -import java.util.function.Function; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.cobbzilla.util.daemon.ZillaRuntime.now; @@ -45,11 +43,13 @@ public interface AppRuleDriver { String REDIS_REJECT_LISTS = "rejectLists"; String REDIS_BLOCK_LISTS = "blockLists"; String REDIS_FILTER_LISTS = "filterLists"; + String REDIS_FLEX_LISTS = "flexLists"; String REDIS_LIST_SUFFIX = "~UNION"; default Set getPrimedRejectDomains () { return null; } default Set getPrimedBlockDomains () { return null; } default Set getPrimedFilterDomains () { return null; } + default Set getPrimedFlexDomains () { return null; } static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) { defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains); @@ -63,6 +63,10 @@ public interface AppRuleDriver { defineRedisSet(redis, ip, REDIS_FILTER_LISTS, list, filterDomains); } + static void defineRedisFlexSet(RedisService redis, String ip, String list, String[] filterDomains) { + defineRedisSet(redis, ip, REDIS_FLEX_LISTS, list, filterDomains); + } + static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { final String listOfListsForIp = listOfListsName + "~" + ip; final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX; diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java index 32998940..613487c8 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java @@ -124,6 +124,7 @@ public class TlsPassthruConfig { @Getter @Setter private String fqdn; @Getter @Setter private Pattern fqdnPattern; public boolean hasPattern () { return fqdnPattern != null; } + public boolean fqdnOnly () { return !hasPattern(); } public TlsPassthruMatcher (String fqdn) { this.fqdn = fqdn; if (fqdn.startsWith("/") && fqdn.endsWith("/")) { @@ -157,6 +158,13 @@ public class TlsPassthruConfig { }; @JsonIgnore public Set getFlexSet() { return getFlexSetRef().get(); } + @JsonIgnore public Set getFlexDomains() { + return getFlexSetRef().get().stream() + .filter(TlsPassthruMatcher::fqdnOnly) + .map(TlsPassthruMatcher::getFqdn) + .collect(Collectors.toSet()); + } + private Set loadFlexSet() { final Set set = loadFeeds(this.flexFeedList, this.flexFqdnList, this.recentFlexFeedValues); if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", ")); diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index ca755957..f75fe04a 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -13,6 +13,8 @@ import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ArrayUtil; +import java.util.Set; + import static org.cobbzilla.util.json.JsonUtil.json; @Slf4j @@ -20,19 +22,19 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { @Override public Class getConfigClass() { return (Class) TlsPassthruConfig.class; } + @Override public Set getPrimedFlexDomains() { + final TlsPassthruConfig passthruConfig = getRuleConfig(); + return passthruConfig.getFlexDomains(); + } + @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { final TlsPassthruConfig passthruConfig = getRuleConfig(); - boolean passthru = false; if (passthruConfig.isPassthru(fqdn) || passthruConfig.isPassthru(addr)) { if (log.isDebugEnabled()) log.debug("checkConnection: detected passthru for fqdn/addr="+fqdn+"/"+addr); - passthru = true; - } - if (passthruConfig.isFlex(fqdn)) { - if (log.isDebugEnabled()) log.debug("checkConnection: detected flex for fqdn/addr="+fqdn+"/"+addr); - return passthru ? ConnectionCheckResponse.passthru_flex : ConnectionCheckResponse.noop_flex; + return ConnectionCheckResponse.passthru; } if (log.isDebugEnabled()) log.debug("checkConnection: returning noop for fqdn/addr="+fqdn+"/"+addr); - return passthru ? ConnectionCheckResponse.passthru : ConnectionCheckResponse.noop; + return ConnectionCheckResponse.noop; } @Override public JsonNode upgradeRuleConfig(JsonNode sageRuleConfig, JsonNode localRuleConfig) { diff --git a/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java index 48d7f83c..e1bf50cf 100644 --- a/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java +++ b/bubble-server/src/main/java/bubble/service/stream/ConnectionCheckResponse.java @@ -10,7 +10,7 @@ import static bubble.ApiConstants.enumFromString; public enum ConnectionCheckResponse { - noop, noop_flex, passthru, passthru_flex, block, error; + noop, passthru, block, error; @JsonCreator public static ConnectionCheckResponse fromString (String v) { return enumFromString(ConnectionCheckResponse.class, v); } diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index fa6e435f..a3f57102 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -145,6 +145,7 @@ public class StandardAppPrimerService implements AppPrimerService { final Set rejectDomains = new HashSet<>(); final Set blockDomains = new HashSet<>(); final Set filterDomains = new HashSet<>(); + final Set flexDomains = new HashSet<>(); for (AppMatcher matcher : matchers) { final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); final Set rejects = appRuleDriver.getPrimedRejectDomains(); @@ -165,8 +166,14 @@ public class StandardAppPrimerService implements AppPrimerService { } else { filterDomains.addAll(filters); } + final Set flexes = appRuleDriver.getPrimedFlexDomains(); + if (empty(flexes)) { + log.debug("_prime: no flexDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + flexDomains.addAll(flexes); + } } - if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains)) { + if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains)) { for (String ip : accountDeviceIps.get(device.getUuid())) { if (!empty(rejectDomains)) { AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new)); @@ -177,6 +184,9 @@ public class StandardAppPrimerService implements AppPrimerService { if (!empty(filterDomains)) { AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new)); } + if (!empty(flexDomains)) { + AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new)); + } } } } diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index 6b2d2b0b..2a537843 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -266,13 +266,7 @@ def next_layer(next_layer): check = check_connection(client_addr, server_addr, fqdns, security_level) called_check_api = True - if check is None or ('passthru' in check and check['passthru']): - # make sure this is not a flex route - #if not called_check_api: - # check = check_connection(client_addr, server_addr, fqdns, security_level) - #if check is not None and 'flex' in check and check['flex']: - # # todo -- how do we route through the flex router for TLS passthru requests? - # bubble_log('next_layer: FLEX ROUTING NOT YET SUPPORTED for passthru for server=' + server_addr+', fqdns='+str(fqdns)) + if check is None or ('passthru' in check and check['passthru'] and ('flex' not in check or not check['flex'])): bubble_log('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index d9789834..931e4486 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -29,7 +29,7 @@ import time import uuid from mitmproxy.net.http import headers as nheaders -from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, HEALTH_CHECK_URI, \ +from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, REDIS, HEALTH_CHECK_URI, \ CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \ add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn from bubble_config import bubble_host, bubble_host_alias @@ -153,13 +153,14 @@ class Rerouter: matcher_response = self.get_matchers(flow, sni or host_header) if matcher_response: - if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru': + has_decision = 'decision' in matcher_response and matcher_response['decision'] is not None + if has_decision and matcher_response['decision'] == 'pass_thru': bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...') add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url) return - elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'): + elif has_decision and matcher_response['decision'].startswith('abort_'): bubble_log('dns_spoofing.request: found abort code: ' + str(matcher_response['decision']) + ', aborting') if matcher_response['decision'] == 'abort_ok': abort_code = 200 @@ -174,7 +175,7 @@ class Rerouter: bubble_activity_log(client_addr, server_addr, 'http_abort' + str(abort_code), log_url) return - elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match': + elif has_decision and matcher_response['decision'] == 'no_match': bubble_log('dns_spoofing.request: decision was no_match, passing thru...') bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url) return -- 2.17.1 From 70be50cd3e4b5a364e1cb5bb8e1c25f0e5e2eff2 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 08:46:25 -0400 Subject: [PATCH 24/78] use proper logging in mitm --- .../main/java/bubble/rule/AppRuleDriver.java | 2 +- .../roles/mitmproxy/files/bubble_api.py | 56 +++++--- .../mitmproxy/files/bubble_conn_check.py | 124 ++++++++++++------ .../roles/mitmproxy/files/bubble_debug.py | 34 +++++ .../roles/mitmproxy/files/bubble_modify.py | 91 ++++++++----- .../roles/mitmproxy/files/dns_spoofing.py | 78 +++++++---- .../packer/roles/mitmproxy/files/run_mitm.sh | 2 +- 7 files changed, 273 insertions(+), 114 deletions(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index e9be26ca..cc9effac 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -43,7 +43,7 @@ public interface AppRuleDriver { String REDIS_REJECT_LISTS = "rejectLists"; String REDIS_BLOCK_LISTS = "blockLists"; String REDIS_FILTER_LISTS = "filterLists"; - String REDIS_FLEX_LISTS = "flexLists"; + String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy for flex routing String REDIS_LIST_SUFFIX = "~UNION"; default Set getPrimedRejectDomains () { return null; } diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 2f7672af..7a0ac41a 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -3,6 +3,9 @@ # import datetime import json +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + import re import requests import redis @@ -17,6 +20,8 @@ from bubble_vpn6 import wireguard_network_ipv6 from bubble_config import bubble_network, bubble_port, debug_capture_fqdn, \ bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host +bubble_log = logging.getLogger(__name__) + HEADER_USER_AGENT = 'User-Agent' HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy' HEADER_REFERER = 'Referer' @@ -39,8 +44,8 @@ BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_' BUBBLE_ACTIVITY_LOG_EXPIRATION = 600 LOCAL_IPS = [] -for ip in subprocess.check_output(['hostname', '-I']).split(): - LOCAL_IPS.append(ip.decode()) +for local_ip in subprocess.check_output(['hostname', '-I']).split(): + LOCAL_IPS.append(local_ip.decode()) VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4) @@ -57,10 +62,6 @@ def redis_set(name, value, ex): REDIS.set(name, value, xx=True, ex=ex) -def bubble_log(message): - print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr, flush=True) - - def bubble_activity_log(client_addr, server_addr, event, data): key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4()) value = json.dumps({ @@ -70,7 +71,8 @@ def bubble_activity_log(client_addr, server_addr, event, data): 'event': event, 'data': str(data) }) - bubble_log('bubble_activity_log: setting '+key+' = '+value) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_activity_log: setting '+key+' = '+value) redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION) pass @@ -79,7 +81,8 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level): if debug_capture_fqdn and fqdns: for f in debug_capture_fqdn: if f in fqdns: - bubble_log('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f) return 'noop' headers = { @@ -96,11 +99,13 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level): response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data) if response.ok: return response.json() - bubble_log('bubble_conn_check API call failed: '+repr(response)) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_conn_check API call failed: '+repr(response)) return None except Exception as e: - bubble_log('bubble_conn_check API call failed: '+repr(e)) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_conn_check API call failed: '+repr(e)) traceback.print_exc() if security_level is not None and security_level['level'] == 'maximum': return False @@ -130,7 +135,8 @@ BLOCK_MATCHER = { def bubble_matchers(req_id, client_addr, server_addr, flow, host): if debug_capture_fqdn and host and host in debug_capture_fqdn: - bubble_log('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host) return DEBUG_MATCHER headers = { @@ -139,19 +145,22 @@ def bubble_matchers(req_id, client_addr, server_addr, flow, host): 'Content-Type': 'application/json' } if HEADER_USER_AGENT not in flow.request.headers: - bubble_log('bubble_matchers: no User-Agent header, setting to UNKNOWN') + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('bubble_matchers: no User-Agent header, setting to UNKNOWN') user_agent = 'UNKNOWN' else: user_agent = flow.request.headers[HEADER_USER_AGENT] if HEADER_REFERER not in flow.request.headers: - bubble_log('bubble_matchers: no Referer header, setting to NONE') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_matchers: no Referer header, setting to NONE') referer = 'NONE' else: try: referer = flow.request.headers[HEADER_REFERER].encode().decode() except Exception as e: - bubble_log('bubble_matchers: error parsing Referer header: '+repr(e)) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('bubble_matchers: error parsing Referer header: '+repr(e)) referer = 'NONE' try: @@ -168,11 +177,14 @@ def bubble_matchers(req_id, client_addr, server_addr, flow, host): if response.ok: return response.json() elif response.status_code == 403: - bubble_log('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text)) return BLOCK_MATCHER - bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) except Exception as e: - bubble_log('bubble_matchers API call failed: '+repr(e)) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_matchers API call failed: '+repr(e)) traceback.print_exc() return None @@ -203,3 +215,13 @@ def is_sage_request(ip, fqdns): def is_not_from_vpn(client_addr): ip = IPAddress(client_addr) return ip not in VPN_IP4_CIDR and ip not in VPN_IP6_CIDR + + +def is_flex_domain(client_addr, fqdn): + check_fqdn = fqdn + while '.' in check_fqdn: + found = REDIS.sismember("flexLists~"+client_addr+"~UNION") + if found: + return True + check_fqdn = check_fqdn[check_fqdn.index('.')+1:] + return False diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index 2a537843..a56211f4 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -28,11 +28,16 @@ from mitmproxy.exceptions import TlsProtocolException from mitmproxy.net import tls as net_tls import json +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + import traceback -from bubble_api import bubble_log, bubble_conn_check, bubble_activity_log, REDIS, redis_set, \ +from bubble_api import bubble_conn_check, bubble_activity_log, REDIS, redis_set, \ is_bubble_request, is_sage_request, is_not_from_vpn from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host +bubble_log = logging.getLogger(__name__) + REDIS_DNS_PREFIX = 'bubble_dns_' REDIS_CONN_CHECK_PREFIX = 'bubble_conn_check_' REDIS_CHECK_DURATION = 60 * 60 # 1 hour timeout @@ -56,16 +61,19 @@ def get_device_security_level(client_addr, fqdns): return {'level': SEC_MAX} level = level.decode() if level == SEC_STD: - bubble_log('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.info('get_device_security_level: checking for max_required_fqdns against fqdns='+repr(fqdns)) if fqdns: max_required_fqdns = REDIS.smembers(REDIS_KEY_DEVICE_SITE_MAX_SECURITY_LEVEL_PREFIX+client_addr) if max_required_fqdns is not None: - bubble_log('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.info('get_device_security_level: found max_required_fqdns='+repr(max_required_fqdns)) for max_required in max_required_fqdns: max_required = max_required.decode() for fqdn in fqdns: if max_required == fqdn or (max_required.startswith('*.') and fqdn.endswith(max_required[1:])): - bubble_log('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('get_device_security_level: returning maximum for fqdn '+fqdn+' based on max_required='+max_required) return {'level': SEC_MAX, 'pinned': True} return {'level': level} @@ -90,7 +98,8 @@ def fqdns_for_addr(server_addr): prefix = REDIS_DNS_PREFIX + server_addr keys = REDIS.keys(prefix + '_*') if keys is None or len(keys) == 0: - bubble_log('fqdns_for_addr: no FQDN found for addr '+str(server_addr)+', checking raw addr') + if (bubble_log.isEnabledFor(DEBUG)): + bubble_log.debug('fqdns_for_addr: no FQDN found for addr '+str(server_addr)+', checking raw addr') return '' fqdns = [] for k in keys: @@ -104,7 +113,8 @@ class TlsBlock(TlsLayer): Monkey-patch __call__ to drop this connection entirely """ def __call__(self): - bubble_log('TlsBlock: blocking') + if bubble_log.isEnabledFor(INFO): + bubble_log.info('TlsBlock: blocking') return @@ -122,16 +132,19 @@ class TlsFeedback(TlsLayer): except TlsProtocolException as e: if self.do_block: - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+' and do_block==True, raising error for client '+client_address) raise e tb = traceback.format_exc() if 'OpenSSL.SSL.ZeroReturnError' in tb: - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SSL zero return error for client '+client_address) raise e elif 'SysCallError' in tb: - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SysCallError for client '+client_address) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('_establish_tls_with_client: TLS error for '+str(server_address)+'/fqdns='+str(self.fqdns)+', raising SysCallError for client '+client_address) raise e elif self.fqdns is not None and len(self.fqdns) > 0: @@ -140,21 +153,26 @@ class TlsFeedback(TlsLayer): if security_level['level'] == SEC_MAX: if 'pinned' in security_level and security_level['pinned']: redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': False, 'reason': 'tls_failure_pinned'}), ex=REDIS_CHECK_DURATION) - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum/pinned) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) else: redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) else: redis_set(cache_key, json.dumps({'fqdns': [fqdn], 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and fqdn='+fqdn+': '+repr(e)) else: cache_key = conn_check_cache_prefix(client_address, server_address) if security_level['level'] == SEC_MAX: redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': False, 'block': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling block (security_level=maximum) for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e)) else: redis_set(cache_key, json.dumps({'fqdns': None, 'addr': server_address, 'passthru': True, 'reason': 'tls_failure'}), ex=REDIS_CHECK_DURATION) - bubble_log('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('_establish_tls_with_client: TLS error for '+str(server_address)+', enabling passthru for client '+client_address+' with cache_key='+cache_key+' and server_address='+server_address+': '+repr(e)) raise e @@ -162,30 +180,37 @@ def check_bubble_connection(client_addr, server_addr, fqdns, security_level): check_response = bubble_conn_check(client_addr, server_addr, fqdns, security_level) if check_response is None or check_response == 'error': if security_level['level'] == SEC_MAX: - bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'block': True, 'reason': 'bubble_error'} else: - bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': False, 'reason': 'bubble_error'} elif check_response == 'passthru': - bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': False, 'reason': 'bubble_passthru'} elif check_response == 'block': - bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'block': True, 'reason': 'bubble_block'} elif check_response == 'passthru_flex': - bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': True, 'reason': 'bubble_passthru_flex'} elif check_response == 'noop_flex': - bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': True, 'reason': 'bubble_no_passthru_flex'} else: - bubble_log('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False') return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'reason': 'bubble_no_passthru'} @@ -198,16 +223,20 @@ def check_connection(client_addr, server_addr, fqdns, security_level): check_json = REDIS.get(cache_key) if check_json is None or len(check_json) == 0: - bubble_log(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'not in redis or empty, calling check_bubble_connection against fqdns='+str(fqdns)) check_response = check_bubble_connection(client_addr, server_addr, fqdns, security_level) - bubble_log(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...") + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'check_bubble_connection('+str(fqdns)+') returned '+str(check_response)+", storing in redis...") redis_set(cache_key, json.dumps(check_response), ex=REDIS_CHECK_DURATION) else: - bubble_log(prefix+'found check_json='+str(check_json)+', touching key in redis') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'found check_json='+str(check_json)+', touching key in redis') check_response = json.loads(check_json) REDIS.touch(cache_key) - bubble_log(prefix+'returning '+str(check_response)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'returning '+str(check_response)) return check_response @@ -216,14 +245,17 @@ def next_layer(next_layer): client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile) client_addr = next_layer.client_conn.address[0] server_addr = next_layer.server_conn.address[0] - bubble_log('next_layer: STARTING: client='+ client_addr+' server='+server_addr) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('next_layer: STARTING: client='+ client_addr+' server='+server_addr) if client_hello.sni: fqdn = client_hello.sni.decode() - bubble_log('next_layer: using fqdn in SNI: '+ fqdn) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('next_layer: using fqdn in SNI: '+ fqdn) fqdns = [ fqdn ] else: fqdns = fqdns_for_addr(server_addr) - bubble_log('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns)) next_layer.fqdns = fqdns no_fqdns = fqdns is None or len(fqdns) == 0 security_level = get_device_security_level(client_addr, fqdns) @@ -231,49 +263,59 @@ def next_layer(next_layer): next_layer.do_block = False called_check_api = False if is_bubble_request(server_addr, fqdns): - bubble_log('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns)) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns)) check = FORCE_PASSTHRU elif is_sage_request(server_addr, fqdns): - bubble_log('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: enabling passthru for SAGE server='+server_addr+' regardless of security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU elif is_not_from_vpn(client_addr): # todo: add to fail2ban - bubble_log('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns)) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'conn_block_non_vpn', fqdns) next_layer.__class__ = TlsBlock return elif security_level['level'] == SEC_OFF: - bubble_log('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: enabling passthru for server='+server_addr+' because security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU elif fqdns is not None and len(fqdns) == 1 and cert_validation_host == fqdns[0] and security_level['level'] != SEC_BASIC: - bubble_log('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: NOT enabling passthru for server='+server_addr+' because fqdn is cert_validation_host ('+cert_validation_host+') for client='+client_addr) return elif (security_level['level'] == SEC_STD or security_level['level'] == SEC_BASIC) and no_fqdns: - bubble_log('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: enabling passthru for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_PASSTHRU elif security_level['level'] == SEC_MAX and no_fqdns: - bubble_log('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: disabling passthru (no TlsFeedback) for server='+server_addr+' because no FQDN found and security_level='+repr(security_level)+' for client='+client_addr) check = FORCE_BLOCK else: - bubble_log('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level)) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level)) check = check_connection(client_addr, server_addr, fqdns, security_level) called_check_api = True if check is None or ('passthru' in check and check['passthru'] and ('flex' not in check or not check['flex'])): - bubble_log('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) next_layer.reply.send(next_layer_replacement) elif 'block' in check and check['block']: - bubble_log('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns)) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns) if show_block_stats(client_addr, fqdns) and security_level['level'] != SEC_BASIC: next_layer.do_block = True @@ -282,12 +324,14 @@ def next_layer(next_layer): next_layer.__class__ = TlsBlock elif security_level['level'] == SEC_BASIC: - bubble_log('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) next_layer.reply.send(next_layer_replacement) else: - bubble_log('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns) next_layer.__class__ = TlsFeedback diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_debug.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_debug.py index 4baf4926..0813730c 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_debug.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_debug.py @@ -1,11 +1,42 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + +import os import threading import traceback import signal import sys +from pathlib import Path + +BUBBLE_PORT_ENV_VAR = 'BUBBLE_PORT' +BUBBLE_PORT = os.getenv(BUBBLE_PORT_ENV_VAR) +if BUBBLE_PORT is None: + BUBBLE_PORT = '(no '+BUBBLE_PORT_ENV_VAR+' env var found)' + +BUBBLE_LOG = '/var/log/bubble/mitmproxy_bubble.log' +BUBBLE_LOG_LEVEL_FILE = '/home/mitmproxy/bubble_log_level.txt' +BUBBLE_LOG_LEVEL_ENV_VAR = 'BUBBLE_LOG_LEVEL' +DEFAULT_BUBBLE_LOG_LEVEL = 'INFO' +BUBBLE_LOG_LEVEL = None +try: + BUBBLE_LOG_LEVEL = Path(BUBBLE_LOG_LEVEL_FILE).read_text().strip() +except IOError: + print('error reading log level from '+BUBBLE_LOG_LEVEL_FILE+', checking env var '+BUBBLE_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True) + BUBBLE_LOG_LEVEL = os.getenv(BUBBLE_LOG_LEVEL_ENV_VAR, DEFAULT_BUBBLE_LOG_LEVEL) + +BUBBLE_NUMERIC_LOG_LEVEL = getattr(logging, BUBBLE_LOG_LEVEL.upper(), None) +if not isinstance(BUBBLE_NUMERIC_LOG_LEVEL, int): + print('Invalid log level: ' + BUBBLE_LOG_LEVEL + ' - using default '+DEFAULT_BUBBLE_LOG_LEVEL, file=sys.stderr, flush=True) + BUBBLE_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_BUBBLE_LOG_LEVEL.upper(), None) +logging.basicConfig(format='[mitm'+BUBBLE_PORT+'] %(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=BUBBLE_LOG, level=BUBBLE_NUMERIC_LOG_LEVEL) + +bubble_log = logging.getLogger(__name__) + + # Allow SIGUSR1 to print stack traces to stderr def dumpstacks(signal, frame): id2name = dict([(th.ident, th.name) for th in threading.enumerate()]) @@ -18,5 +49,8 @@ def dumpstacks(signal, frame): code.append(" %s" % (line.strip())) print("\n------------------------------------- stack traces ------------------------------"+"\n".join(code), file=sys.stderr, flush=True) + signal.signal(signal.SIGUSR1, dumpstacks) +if bubble_log.isEnabledFor(INFO): + bubble_log.info('debug module initialized, default log level = '+logging.getLevelName(BUBBLE_NUMERIC_LOG_LEVEL)) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index b9b66c92..a38e7e96 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -2,6 +2,8 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # import json +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL import re import requests import urllib @@ -10,7 +12,7 @@ from mitmproxy.net.http import Headers from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, BUBBLE_URI_PREFIX, \ HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, \ - CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx, \ + CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, get_flow_ctx, add_flow_ctx, \ HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header BUFFER_SIZE = 4096 @@ -27,6 +29,8 @@ REDIS_FILTER_PASSTHRU_DURATION = 600 DEBUG_STREAM_COUNTERS = {} +bubble_log = logging.getLogger(__name__) + def add_csp_part(new_csp, part): if len(new_csp) > 0: @@ -53,7 +57,8 @@ def ensure_bubble_script_csp(csp): def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, content_type=None, content_length=None, csp=None): if debug_capture_fqdn: if debug_capture_fqdn in req_id: - bubble_log('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('filter_chunk: debug_capture_fqdn detected, capturing: '+debug_capture_fqdn) f = open('/tmp/bubble_capture_'+req_id, mode='ab', buffering=0) f.write(chunk) f.close() @@ -63,7 +68,8 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c redis_passthru_key = REDIS_FILTER_PASSTHRU_PREFIX + flow.request.method + '~~~' + user_agent + ':' + flow.request.url do_pass = REDIS.get(redis_passthru_key) if do_pass: - bubble_log('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('filter_chunk: req_id='+req_id+': passthru found in redis, returning chunk') REDIS.touch(redis_passthru_key) return chunk @@ -83,14 +89,15 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c url = url + '?last=true' if csp: - # bubble_log('filter_chunk: url='+url+' (csp='+csp+')') - bubble_log('filter_chunk: url='+url+' (with csp) (last='+str(last)+')') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('filter_chunk: url='+url+' (csp='+csp+')') filter_headers = { HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY, HEADER_CONTENT_SECURITY_POLICY: csp } else: - bubble_log('filter_chunk: url='+url+' (no csp) (last='+str(last)+')') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('filter_chunk: url='+url+' (no csp)') filter_headers = STANDARD_FILTER_HEADERS if debug_stream_fqdn and debug_stream_uri and debug_stream_fqdn in req_id and flow.request.path == debug_stream_uri: @@ -99,7 +106,8 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c else: count = 0 DEBUG_STREAM_COUNTERS[req_id] = count - bubble_log('filter_chunk: debug_stream detected, capturing: '+debug_stream_fqdn) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('filter_chunk: debug_stream detected, capturing: '+debug_stream_fqdn) f = open('/tmp/bubble_stream_'+req_id+'_chunk'+"{:04d}".format(count)+'.data', mode='wb', buffering=0) if chunk is not None: f.write(chunk) @@ -114,11 +122,13 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c response = requests.post(url, data=chunk, headers=filter_headers) if not response.ok: err_message = 'filter_chunk: Error fetching ' + url + ', HTTP status ' + str(response.status_code) - bubble_log(err_message) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error(err_message) return b'' elif HEADER_FILTER_PASSTHRU in response.headers: - bubble_log('filter_chunk: server returned X-Bubble-Passthru, not filtering subsequent requests') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('filter_chunk: server returned X-Bubble-Passthru, not filtering subsequent requests') redis_set(redis_passthru_key, 'passthru', ex=REDIS_FILTER_PASSTHRU_DURATION) return chunk @@ -137,7 +147,8 @@ def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, con bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT) chunk_len = len(chunk) last = chunk_len + bytes_sent >= content_length - bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent)) add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len) else: last = False @@ -149,7 +160,8 @@ def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, con if not content_length: yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data except Exception as e: - bubble_log('bubble_filter_chunks: exception='+repr(e)) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_filter_chunks: exception='+repr(e)) traceback.print_exc() yield None @@ -183,7 +195,8 @@ def responseheaders(flow): path = flow.request.path if path and path.startswith(BUBBLE_URI_PREFIX): if path.startswith(HEALTH_CHECK_URI): - # bubble_log('responseheaders: special bubble health check request, responding with OK') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('responseheaders: special bubble health check request, responding with OK') flow.response.headers = Headers() flow.response.headers[HEADER_HEALTH_CHECK] = 'OK' flow.response.headers[HEADER_CONTENT_LENGTH] = '3' @@ -191,7 +204,8 @@ def responseheaders(flow): flow.response.stream = lambda chunks: [b'OK\n'] else: uri = 'http://127.0.0.1:' + bubble_port + '/' + path[len(BUBBLE_URI_PREFIX):] - bubble_log('responseheaders: sending special bubble request to '+uri) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('responseheaders: sending special bubble request to '+uri) headers = { 'Accept' : 'application/json', 'Content-Type': 'application/json' @@ -200,13 +214,16 @@ def responseheaders(flow): if flow.request.method == 'GET': response = requests.get(uri, headers=headers, stream=True) elif flow.request.method == 'POST': - bubble_log('responseheaders: special bubble request: POST content is '+str(flow.request.content)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('responseheaders: special bubble request: POST content is '+str(flow.request.content)) headers['Content-Length'] = str(len(flow.request.content)) response = requests.post(uri, data=flow.request.content, headers=headers) else: - bubble_log('responseheaders: special bubble request: method '+flow.request.method+' not supported') + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('responseheaders: special bubble request: method '+flow.request.method+' not supported') if response is not None: - bubble_log('responseheaders: special bubble request: response status = '+str(response.status_code)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('responseheaders: special bubble request: response status = '+str(response.status_code)) flow.response.headers = Headers() for key, value in response.headers.items(): flow.response.headers[key] = value @@ -218,7 +235,8 @@ def responseheaders(flow): if abort_code is not None: abort_location = get_flow_ctx(flow, CTX_BUBBLE_LOCATION) if abort_location is not None: - bubble_log('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path) flow.response.headers = Headers() flow.response.headers[HEADER_LOCATION] = abort_location flow.response.status_code = abort_code @@ -228,21 +246,25 @@ def responseheaders(flow): content_type = flow.response.headers[HEADER_CONTENT_TYPE] else: content_type = None - bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code)+', path was: '+path) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('responseheaders: aborting request with HTTP status '+str(abort_code)+', path was: '+path) flow.response.headers = Headers() flow.response.status_code = abort_code flow.response.stream = lambda chunks: abort_data(content_type) elif flow.response.status_code // 100 != 2: - bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path) pass elif flow.response.headers is None or len(flow.response.headers) == 0: - bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path) pass elif HEADER_CONTENT_LENGTH in flow.response.headers and flow.response.headers[HEADER_CONTENT_LENGTH] == "0": - bubble_log('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path) + if bubble_log.isEnabledFor(INFO): + bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path) pass else: @@ -250,7 +272,8 @@ def responseheaders(flow): matchers = get_flow_ctx(flow, CTX_BUBBLE_MATCHERS) prefix = 'responseheaders(req_id='+str(req_id)+'): ' if req_id is not None and matchers is not None: - bubble_log(prefix+' matchers: '+repr(matchers)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+' matchers: '+repr(matchers)) if HEADER_USER_AGENT in flow.request.headers: user_agent = flow.request.headers[HEADER_USER_AGENT] else: @@ -266,10 +289,12 @@ def responseheaders(flow): typeRegex = '^text/html.*' if re.match(typeRegex, content_type): any_content_type_matches = True - bubble_log(prefix+'found at least one matcher for content_type ('+content_type+'), filtering: '+path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'found at least one matcher for content_type ('+content_type+'), filtering: '+path) break if not any_content_type_matches: - bubble_log(prefix+'no matchers for content_type ('+content_type+'), passing thru: '+path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'no matchers for content_type ('+content_type+'), passing thru: '+path) return if HEADER_CONTENT_ENCODING in flow.response.headers: @@ -284,7 +309,8 @@ def responseheaders(flow): csp = None content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) - # bubble_log(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) flow.response.stream = bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp) if content_length_value: flow.response.headers['transfer-encoding'] = 'chunked' @@ -295,10 +321,12 @@ def responseheaders(flow): if hasattr(ctx, 'ctx'): ctx = ctx.ctx else: - bubble_log(prefix+'error finding server_conn for path '+path+'. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx))) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error(prefix+'error finding server_conn for path '+path+'. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx))) return if not hasattr(ctx, 'server_conn'): - bubble_log(prefix+'error finding server_conn for path '+path+'. ctx type='+str(type(ctx))+' vars='+str(vars(ctx))) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error(prefix+'error finding server_conn for path '+path+'. ctx type='+str(type(ctx))+' vars='+str(vars(ctx))) return content_length = int(content_length_value) ctx.server_conn.rfile.fake_chunks = content_length @@ -306,11 +334,14 @@ def responseheaders(flow): add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, 0) else: - bubble_log(prefix+'no matchers, passing thru: '+path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'no matchers, passing thru: '+path) pass else: - bubble_log(prefix+'no '+HEADER_CONTENT_TYPE+' header, passing thru: '+path) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning(prefix+'no '+HEADER_CONTENT_TYPE+' header, passing thru: '+path) pass else: - bubble_log(prefix+'no '+CTX_BUBBLE_MATCHERS+' in ctx, passing thru: '+path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug(prefix+'no '+CTX_BUBBLE_MATCHERS+' in ctx, passing thru: '+path) pass diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index 931e4486..6df53dc3 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -25,15 +25,21 @@ # import re +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + import time import uuid from mitmproxy.net.http import headers as nheaders -from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, REDIS, HEALTH_CHECK_URI, \ +from bubble_api import bubble_matchers, bubble_activity_log, HEALTH_CHECK_URI, \ CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \ add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn from bubble_config import bubble_host, bubble_host_alias +bubble_log = logging.getLogger(__name__) + + class Rerouter: @staticmethod def get_matchers(flow, host): @@ -43,7 +49,8 @@ class Rerouter: is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI) if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX): if not is_health_check: - bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path) + if bubble_log.isEnabledFor(INFO): + bubble_log.info("get_matchers: not filtering special bubble path: "+flow.request.path) return None client_addr = str(flow.client_conn.address[0]) @@ -54,43 +61,53 @@ class Rerouter: try: host = str(host) except Exception as e: - bubble_log('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e)) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e)) return None if host == bubble_host or host == bubble_host_alias: - bubble_log('get_matchers: request is for bubble itself ('+host+'), not matching') + if bubble_log.isEnabledFor(INFO): + bubble_log.info('get_matchers: request is for bubble itself ('+host+'), not matching') return None req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time()) - bubble_log("get_matchers: requesting match decision for req_id="+req_id) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug("get_matchers: requesting match decision for req_id="+req_id) resp = bubble_matchers(req_id, client_addr, server_addr, flow, host) if not resp: - bubble_log('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host)) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('get_matchers: no response for client_addr/host: '+client_addr+'/'+str(host)) return None matchers = [] if 'matchers' in resp and resp['matchers'] is not None: for m in resp['matchers']: if 'urlRegex' in m: - bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex']) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex']) else: - bubble_log('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping') continue if re.match(m['urlRegex'], flow.request.path): - bubble_log('get_matchers: rule matched, adding rule: '+m['rule']) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('get_matchers: rule matched, adding rule: '+m['rule']) matchers.append(m) else: - bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule']) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule']) else: - bubble_log('get_matchers: no matchers. response='+repr(resp)) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('get_matchers: no matchers. response='+repr(resp)) decision = None if 'decision' in resp: decision = resp['decision'] matcher_response = {'decision': decision, 'matchers': matchers, 'request_id': req_id} - bubble_log("get_matchers: returning "+repr(matcher_response)) + if bubble_log.isEnabledFor(INFO): + bubble_log.info("get_matchers: returning "+repr(matcher_response)) return matcher_response def request(self, flow): @@ -110,7 +127,6 @@ class Rerouter: # check if https and sni is missing but we have a host header, fill in the sni host_header = flow.request.host_header - # bubble_log("dns_spoofing.request: host_header is "+repr(host_header)) if host_header: m = parse_host_header.match(host_header) if m: @@ -133,20 +149,23 @@ class Rerouter: if is_bubble_request(server_addr, fqdns): is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI) if not is_health_check: - bubble_log('dns_spoofing.request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns)+', path='+flow.request.path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns)+', path='+flow.request.path) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path) return elif is_sage_request(server_addr, fqdns): - bubble_log('dns_spoofing.request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path) return elif is_not_from_vpn(client_addr): # todo: add to fail2ban - bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns)) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) return @@ -155,19 +174,22 @@ class Rerouter: if matcher_response: has_decision = 'decision' in matcher_response and matcher_response['decision'] is not None if has_decision and matcher_response['decision'] == 'pass_thru': - bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: passthru response returned, passing thru and NOT performing TLS interception...') add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url) return elif has_decision and matcher_response['decision'].startswith('abort_'): - bubble_log('dns_spoofing.request: found abort code: ' + str(matcher_response['decision']) + ', aborting') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: found abort code: ' + str(matcher_response['decision']) + ', aborting') if matcher_response['decision'] == 'abort_ok': abort_code = 200 elif matcher_response['decision'] == 'abort_not_found': abort_code = 404 else: - bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') abort_code = 404 flow.request.headers = nheaders.Headers([]) flow.request.content = b'' @@ -176,7 +198,8 @@ class Rerouter: return elif has_decision and matcher_response['decision'] == 'no_match': - bubble_log('dns_spoofing.request: decision was no_match, passing thru...') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: decision was no_match, passing thru...') bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url) return @@ -184,27 +207,32 @@ class Rerouter: and 'request_id' in matcher_response and len(matcher_response['matchers']) > 0): req_id = matcher_response['request_id'] - bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug("request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) bubble_activity_log(client_addr, server_addr, 'http_match', log_url) else: - bubble_log('dns_spoofing.request: no rules returned, passing thru...') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: no rules returned, passing thru...') bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url) else: if not is_health_check: - bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...') + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: no matcher_response returned, passing thru...') # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) elif is_http and is_not_from_vpn(client_addr): # todo: add to fail2ban - bubble_log('dns_spoofing.request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr) bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr]) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) return else: - bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path) + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('request: no sni/host found, not applying rules to path: ' + flow.request.path) bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr]) flow.request.host_header = host_header diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh index c5b056fb..6ca204e3 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh @@ -14,7 +14,7 @@ fi cd /home/mitmproxy/mitmproxy && \ ./dev.sh ${SETUP_VENV} && . ./venv/bin/activate && \ -mitmdump \ +BUBBLE_PORT=${PORT} mitmdump \ --listen-host 0.0.0.0 \ --listen-port ${PORT} \ --showhost \ -- 2.17.1 From 2e75033ab4c5cc3b95065566965ea6e03e684a47 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 09:49:38 -0400 Subject: [PATCH 25/78] update bubble-web --- bubble-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-web b/bubble-web index 6e0c4081..63128400 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 6e0c408113364e9ddf631db95dfcdc139bd09024 +Subproject commit 631284006bb691d3d6d88aa2c3544ebc6cfa2893 -- 2.17.1 From a6801493d6bd3c39cd804c293617c2d638ddeaaa Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 13:14:31 -0400 Subject: [PATCH 26/78] interrupt flex router after creating/updating. add status endpoint for flex router --- .../java/bubble/dao/device/FlexRouterDAO.java | 2 + .../resources/device/FlexRoutersResource.java | 22 ++++++++++- .../service/device/FlexRouterService.java | 5 ++- .../device/StandardFlexRouterService.java | 38 +++++++++++++++---- .../DbFilterFlexRouterService.java | 11 +----- 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index 93bb76d0..bcfe72a5 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -28,6 +28,7 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postCreate(FlexRouter router, Object context) { flexRouterService.register(router); router.setHost_key(KNOWN_HOST_KEY); + flexRouterService.interruptSoon(); return super.postCreate(router, context); } @@ -42,6 +43,7 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postUpdate(FlexRouter router, Object context) { flexRouterService.register(router); router.setHost_key(KNOWN_HOST_KEY); + if (router.doInterrupt()) flexRouterService.interruptSoon(); return super.postUpdate(router, context); } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 3d303f6a..028d5a4f 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -6,23 +6,31 @@ import bubble.model.device.Device; import bubble.model.device.FlexRouter; import bubble.resources.account.AccountOwnedResource; import bubble.service.device.DeviceService; +import bubble.service.device.FlexRouterStatus; +import bubble.service.device.StandardFlexRouterService; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.network.PortPicker; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; import java.math.BigInteger; import java.net.InetAddress; +import static bubble.ApiConstants.EP_STATUS; import static org.cobbzilla.util.network.PortPicker.portIsAvailable; -import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; -import static org.cobbzilla.wizard.resources.ResourceUtil.userPrincipal; +import static org.cobbzilla.wizard.resources.ResourceUtil.*; @Slf4j public class FlexRoutersResource extends AccountOwnedResource { @Autowired private DeviceService deviceService; + @Autowired private StandardFlexRouterService flexRouterService; public FlexRoutersResource(Account account) { super(account); } @@ -35,6 +43,16 @@ public class FlexRoutersResource extends AccountOwnedResource { + sleep(FIRST_TIME_WAIT); + log.debug("interruptSoon: interrupting..."); + interrupt(); + }, "StandardFlexRouterService.interruptSoon"); + } + public void register (FlexRouter router) { allowFlexKey(router.getKey()); } public void unregister (FlexRouter router) { @@ -86,6 +98,13 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute disallowFlexKey(router.getKey()); } + private final Map statusMap = new ConcurrentHashMap<>(DEFAULT_MAX_TUNNELS); + + public FlexRouterStatus status(String uuid) { + final FlexRouterStatus stat = statusMap.get(uuid); + return stat == null ? FlexRouterStatus.none : stat; + } + @Override protected void process() { try { @Cleanup final CloseableHttpClient httpClient = getHttpClient(); @@ -105,7 +124,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute router.setInitialized(true); } router.setActive(active); - flexRouterDAO.update(router); + flexRouterDAO.update(router.noInterrupt()); } return active; })); @@ -127,10 +146,13 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute if (i == 0) { if (log.isInfoEnabled()) log.info(prefix+"pinging router at "+pingUrl+" ..."); } else { - router = flexRouterDAO.findByUuid(router.getUuid()); - if (router == null) { - log.error(prefix+"router no longer exists"); + final FlexRouter existing = flexRouterDAO.findByUuid(router.getUuid()); + if (existing == null) { + log.error(prefix+"router no longer exists: "+router.getUuid()); + statusMap.put(router.getUuid(), FlexRouterStatus.deleted); return false; + } else { + router = existing; } if (log.isWarnEnabled()) log.warn(prefix+"attempting to ping again (try="+(i+1)+"/"+MAX_PING_TRIES+")"); } @@ -143,6 +165,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute final FlexRouterPing pong = response.getEntity(FlexRouterPing.class); if (pong.validate(router)) { if (log.isInfoEnabled()) log.info(prefix+"router is ok"); + statusMap.put(router.getUuid(), FlexRouterStatus.active); return true; } else { log.error(prefix+"pong response was invalid"); @@ -152,6 +175,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute } catch (Exception e) { log.error(prefix+"error: "+shortError(e)); } + statusMap.put(router.getUuid(), FlexRouterStatus.unreachable); } log.error(prefix+"error: router failed after "+MAX_PING_TRIES+" attempts, returning false"); return false; diff --git a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java index 3e38722b..68de51e5 100644 --- a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java @@ -1,16 +1,7 @@ package bubble.service_dbfilter; -import bubble.model.device.FlexRouter; import bubble.service.device.FlexRouterService; import org.springframework.stereotype.Service; -import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; - @Service -public class DbFilterFlexRouterService implements FlexRouterService { - - @Override public void register(FlexRouter router) { notSupported("register"); } - - @Override public void unregister(FlexRouter router) { notSupported("unregister"); } - -} +public class DbFilterFlexRouterService implements FlexRouterService {} -- 2.17.1 From f6399665f4140b6bff10c9c314769ba16eccbbfc Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 13:14:54 -0400 Subject: [PATCH 27/78] interrupt flex router after creating/updating. add status endpoint for flex router --- .../src/main/java/bubble/model/device/FlexRouter.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 07202880..8bb87558 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -106,4 +106,10 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { public FlexRouterPing pingObject() { return new FlexRouterPing(this); } + // used by StandardFlexRouterService to signal that its updates should not trigger an interrupt, + // otherwise it would never sleep + private boolean noInterrupt = false; + public FlexRouter noInterrupt () { noInterrupt = true; return this; } + public boolean doInterrupt () { !noInterrupt; } + } -- 2.17.1 From 36e0b16d2e6497300132fa6cb3689ffa465fa30c Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 13:15:02 -0400 Subject: [PATCH 28/78] interrupt flex router after creating/updating. add status endpoint for flex router --- .../bubble/service/device/FlexRouterStatus.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java new file mode 100644 index 00000000..4e83d6cf --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java @@ -0,0 +1,13 @@ +package bubble.service.device; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import static bubble.ApiConstants.enumFromString; + +public enum FlexRouterStatus { + + none, active, unreachable, deleted; + + @JsonCreator public static FlexRouterStatus fromString (String v) { return enumFromString(FlexRouterStatus.class, v); } + +} -- 2.17.1 From 147a33f84fd0794d16079b070f81ea0c82042ef4 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 13:15:34 -0400 Subject: [PATCH 29/78] fix typo --- bubble-server/src/main/java/bubble/model/device/FlexRouter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 8bb87558..06b608b2 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -110,6 +110,6 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { // otherwise it would never sleep private boolean noInterrupt = false; public FlexRouter noInterrupt () { noInterrupt = true; return this; } - public boolean doInterrupt () { !noInterrupt; } + public boolean doInterrupt () { return !noInterrupt; } } -- 2.17.1 From 89cc1fccba9f907ba18d01d418c99ea51ecb95de Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 13:15:46 -0400 Subject: [PATCH 30/78] name background threads --- .../src/main/java/bubble/dao/bill/AccountPlanDAO.java | 2 +- .../src/main/java/bubble/main/RekeyDatabaseMain.java | 4 ++-- .../src/main/java/bubble/resources/account/MeResource.java | 2 +- .../main/java/bubble/resources/bill/AccountPlansResource.java | 2 +- .../src/main/java/bubble/server/BubbleConfiguration.java | 2 +- .../java/bubble/server/listener/NodeInitializerListener.java | 2 +- .../main/java/bubble/service/backup/NetworkKeysService.java | 2 +- .../src/main/java/bubble/service/boot/ActivationService.java | 2 +- .../java/bubble/service/boot/StandardSelfNodeService.java | 2 +- .../src/main/java/bubble/service/cloud/NodeProgressMeter.java | 4 ++-- .../java/bubble/service/cloud/StandardNetworkService.java | 2 +- .../java/bubble/service/dbfilter/DatabaseFilterService.java | 2 +- .../src/main/java/bubble/service/dbfilter/EntityIterator.java | 2 +- utils/cobbzilla-utils | 2 +- 14 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java b/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java index b78bbbc4..8b5a5d66 100644 --- a/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java +++ b/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java @@ -164,7 +164,7 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO { background(() -> { sleep(PURCHASE_DELAY, "AccountPlanDAO.postCreate: waiting to finalize purchase"); paymentDriver.purchase(accountPlanUuid, paymentMethodUuid, billUuid); - }); + }, "AccountPlanDAO.postCreate"); } return super.postCreate(accountPlan, context); } diff --git a/bubble-server/src/main/java/bubble/main/RekeyDatabaseMain.java b/bubble-server/src/main/java/bubble/main/RekeyDatabaseMain.java index d057e269..c2d34efb 100644 --- a/bubble-server/src/main/java/bubble/main/RekeyDatabaseMain.java +++ b/bubble-server/src/main/java/bubble/main/RekeyDatabaseMain.java @@ -34,7 +34,7 @@ public class RekeyDatabaseMain extends BaseMain { } catch (Exception e) { die("READ ERROR: " + e); } - }); + }, "RekeyDatabaseMain.run.reader"); final AtomicReference writeResult = new AtomicReference<>(); final Thread writer = runWriter(options, writeResult, options.getEnv()); @@ -58,7 +58,7 @@ public class RekeyDatabaseMain extends BaseMain { } catch (Exception e) { writeResult.set(new CommandResult(e).setExitStatus(-1)); } - }, e -> writeResult.set(new CommandResult(e).setExitStatus(-1))); + }, "RekeyDatabaseMain.runWriter", e -> writeResult.set(new CommandResult(e).setExitStatus(-1))); } public static Command readerCommand(RekeyDatabaseOptions options, Map env) { diff --git a/bubble-server/src/main/java/bubble/resources/account/MeResource.java b/bubble-server/src/main/java/bubble/resources/account/MeResource.java index 6853a0c3..4d7e2132 100644 --- a/bubble-server/src/main/java/bubble/resources/account/MeResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/MeResource.java @@ -452,7 +452,7 @@ public class MeResource { if (!caller.admin()) return forbidden(); authenticatorService.ensureAuthenticated(ctx); - background(() -> jarUpgradeService.upgrade()); + background(() -> jarUpgradeService.upgrade(), "MeResource.upgrade"); return ok(configuration.getPublicSystemConfigs()); } diff --git a/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java b/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java index b1c78f1a..3aa040a3 100644 --- a/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java +++ b/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java @@ -327,7 +327,7 @@ public class AccountPlansResource extends AccountOwnedResource getDao().delete(planUuid)); + background(() -> getDao().delete(planUuid), "AccountPlansResource.delete"); return ok(found.setDeletedNetwork(found.getNetwork())); } diff --git a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java index 37860387..cc97755a 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java +++ b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java @@ -385,7 +385,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration // called after activation, because now thisNetwork will be defined public void refreshPublicSystemConfigs () { synchronized (publicSystemConfigs) { publicSystemConfigs.set(null); } - background(this::getPublicSystemConfigs); + background(this::getPublicSystemConfigs, "BubbleConfiguration.refreshPublicSystemConfigs"); } public boolean paymentsEnabled () { diff --git a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java index 179653c0..46d3f4f1 100644 --- a/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java @@ -103,7 +103,7 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase cloud.wireAndSetup(c)); +// background(() -> cloud.wireAndSetup(c), "NodeInitializerListener.onStart.cloudInit); } } diff --git a/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java b/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java index 075aecbc..1132de90 100644 --- a/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java +++ b/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java @@ -102,7 +102,7 @@ public class NetworkKeysService { log.error("Cannot delete tmp backup folder " + backupDir, e); } } - }); + }, "NetworkKeysService.startBackupDownload"); } @NonNull public BackupPackagingStatus backupDownloadStatus(@NonNull final String keysCode) { diff --git a/bubble-server/src/main/java/bubble/service/boot/ActivationService.java b/bubble-server/src/main/java/bubble/service/boot/ActivationService.java index 8c13fefb..779a4c5e 100644 --- a/bubble-server/src/main/java/bubble/service/boot/ActivationService.java +++ b/bubble-server/src/main/java/bubble/service/boot/ActivationService.java @@ -226,7 +226,7 @@ public class ActivationService { final Map> objects = modelSetupService.setupModel(api, account, "manifest-defaults"); log.info("bootstrapThisNode: created default objects\n"+json(objects)); - }); + }, "ActivationService.bootstrapThisNode.createDefaultObjects"); } return node; diff --git a/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java b/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java index 8686f12e..d00afe2e 100644 --- a/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java +++ b/bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java @@ -161,7 +161,7 @@ public class StandardSelfNodeService implements SelfNodeService { .booleanValue()) { deviceDAO.refreshVpnUsers(); } - }); + }, "StandardSelfNodeService.onStart.spareDevices"); } // start RefundService if payments are enabled and this is a SageLauncher diff --git a/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java index 11bcbca8..31c09f5f 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java +++ b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java @@ -209,7 +209,7 @@ public class NodeProgressMeter extends PipedOutputStream implements Runnable { .setAccount(nn.getAccount()) .setMessageKey(METER_COMPLETED_OK) .setPercent(100)); - background(this::close); + background(this::close, "NodeProgressMeter.completed"); } public NodeProgressMeter uncloseable() throws IOException { @@ -225,7 +225,7 @@ public class NodeProgressMeter extends PipedOutputStream implements Runnable { .setAccount(nn.getAccount()) .setMessageKey(METER_ERROR_CANCELED) .setPercent(0)); - background(this::close); + background(this::close, "NodeProgressMeter.cancel"); } private class UncloseableNodeProgressMeter extends NodeProgressMeter { diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index 5d34410c..7f8dd2ff 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -883,7 +883,7 @@ public class StandardNetworkService implements NetworkService { } finally { if (lock != null) unlockNetwork(networkUuid, lock); } - }); + }, "StandardNetworkService.stopNetwork"); return true; } diff --git a/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java b/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java index ea656e49..a0f69c4e 100644 --- a/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java +++ b/bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java @@ -114,7 +114,7 @@ public class DatabaseFilterService { ? new FullEntityIterator(configuration, network, readerError) : new FilteredEntityIterator(configuration, account, network, node, planApps, readerError); } - }.runInBackground(readerError::set); + }.runInBackground("RekeyReaderMain.reader", readerError::set); // start a RekeyWriter to pull objects from RekeyReader final AtomicReference writeResult = new AtomicReference<>(); diff --git a/bubble-server/src/main/java/bubble/service/dbfilter/EntityIterator.java b/bubble-server/src/main/java/bubble/service/dbfilter/EntityIterator.java index a64c155d..67a6b46d 100644 --- a/bubble-server/src/main/java/bubble/service/dbfilter/EntityIterator.java +++ b/bubble-server/src/main/java/bubble/service/dbfilter/EntityIterator.java @@ -51,7 +51,7 @@ public abstract class EntityIterator implements Iterator { public EntityIterator(AtomicReference error) { this.error = error; - this.thread = background(this::_iterate, this.error::set); + this.thread = background(this::_iterate, "EntityIterator", this.error::set); } @Override public boolean hasNext() { diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index df29b42b..c66f9e16 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit df29b42b97be4387787e31139f0723a8c3a189ce +Subproject commit c66f9e167ca32887374ec07a05fd3bddaeb8c2d5 -- 2.17.1 From 2477522e5ffa039c0247a52e4850cd57be51fc6a Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 13:19:18 -0400 Subject: [PATCH 31/78] mark noInterrupt field as ignored by json and hibernate --- bubble-server/src/main/java/bubble/model/device/FlexRouter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 06b608b2..d858b680 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -108,7 +108,7 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { // used by StandardFlexRouterService to signal that its updates should not trigger an interrupt, // otherwise it would never sleep - private boolean noInterrupt = false; + @JsonIgnore @Transient private boolean noInterrupt = false; public FlexRouter noInterrupt () { noInterrupt = true; return this; } public boolean doInterrupt () { return !noInterrupt; } -- 2.17.1 From 80c65c739f587e0b461e8160ea9baa57680c41e9 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 13:27:16 -0400 Subject: [PATCH 32/78] if interrupted, do not sleep --- .../bubble/service/device/StandardFlexRouterService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 61d6c072..08b28dd8 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -4,7 +4,6 @@ import bubble.dao.device.FlexRouterDAO; import bubble.model.device.FlexRouter; import bubble.model.device.FlexRouterPing; import lombok.Cleanup; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; @@ -27,6 +26,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; import static bubble.ApiConstants.HOME_DIR; import static bubble.model.device.FlexRouterPing.MAX_PING_AGE; @@ -71,9 +71,10 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute private static final long PING_ALL_TIMEOUT = (SECONDS.toMillis(1) * DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + FIRST_TIME_WAIT; - @Getter private final long sleepTime = MINUTES.toMillis(2); + public static final long DEFAULT_SLEEP_TIME = MINUTES.toMillis(2); @Autowired private FlexRouterDAO flexRouterDAO; + private final AtomicBoolean interrupted = new AtomicBoolean(false); @Override public void onStart() { flexRouterDAO.findEnabledAndRegistered().forEach(this::register); @@ -82,8 +83,11 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute @Override protected boolean canInterruptSleep() { return true; } + @Override protected long getSleepTime() { return interrupted.get() ? 0 : DEFAULT_SLEEP_TIME; } + @Override public void interruptSoon() { log.debug("interruptSoon: will interrupt in "+FIRST_TIME_WAIT+" millis"); + interrupted.set(true); background(() -> { sleep(FIRST_TIME_WAIT); log.debug("interruptSoon: interrupting..."); @@ -107,6 +111,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute @Override protected void process() { try { + interrupted.set(false); @Cleanup final CloseableHttpClient httpClient = getHttpClient(); final List routers = flexRouterDAO.findEnabledAndRegistered(); if (log.isDebugEnabled()) log.debug("process: starting, will ping "+routers.size()+" routers"); -- 2.17.1 From 4c9cac1bf8299e8b3f65d2e61a1723e55b39e824 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 14:27:19 -0400 Subject: [PATCH 33/78] fix flex service interrupts --- .../java/bubble/dao/device/FlexRouterDAO.java | 2 -- .../java/bubble/model/device/FlexRouter.java | 6 ------ .../resources/account/AccountOwnedResource.java | 5 +++-- .../resources/device/FlexRoutersResource.java | 12 ++++++++++++ .../device/StandardFlexRouterService.java | 17 ++++++++++++----- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index bcfe72a5..93bb76d0 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -28,7 +28,6 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postCreate(FlexRouter router, Object context) { flexRouterService.register(router); router.setHost_key(KNOWN_HOST_KEY); - flexRouterService.interruptSoon(); return super.postCreate(router, context); } @@ -43,7 +42,6 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postUpdate(FlexRouter router, Object context) { flexRouterService.register(router); router.setHost_key(KNOWN_HOST_KEY); - if (router.doInterrupt()) flexRouterService.interruptSoon(); return super.postUpdate(router, context); } diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index d858b680..07202880 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -106,10 +106,4 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { public FlexRouterPing pingObject() { return new FlexRouterPing(this); } - // used by StandardFlexRouterService to signal that its updates should not trigger an interrupt, - // otherwise it would never sleep - @JsonIgnore @Transient private boolean noInterrupt = false; - public FlexRouter noInterrupt () { noInterrupt = true; return this; } - public boolean doInterrupt () { return !noInterrupt; } - } diff --git a/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java b/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java index 4035bba3..0b34d695 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AccountOwnedResource.java @@ -157,7 +157,7 @@ public class AccountOwnedResource { - sleep(FIRST_TIME_WAIT); + sleep(INTERRUPT_WAIT); log.debug("interruptSoon: interrupting..."); + synchronized (interrupted) { interrupted.set(false); } interrupt(); }, "StandardFlexRouterService.interruptSoon"); } @@ -110,8 +117,8 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute } @Override protected void process() { + synchronized (interrupted) { interrupted.set(false); } try { - interrupted.set(false); @Cleanup final CloseableHttpClient httpClient = getHttpClient(); final List routers = flexRouterDAO.findEnabledAndRegistered(); if (log.isDebugEnabled()) log.debug("process: starting, will ping "+routers.size()+" routers"); @@ -129,7 +136,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute router.setInitialized(true); } router.setActive(active); - flexRouterDAO.update(router.noInterrupt()); + flexRouterDAO.update(router); } return active; })); -- 2.17.1 From 17ee3b533c567295a17271ad499c83cd73fd4605 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 14:31:24 -0400 Subject: [PATCH 34/78] remove stacktrace --- .../java/bubble/service/device/StandardFlexRouterService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 0d5390ff..22eb1308 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -87,7 +87,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute @Override protected long getSleepTime() { return interrupted.get() ? 0 : DEFAULT_SLEEP_TIME; } @Override public void interruptSoon() { - log.debug("interruptSoon: will interrupt in "+INTERRUPT_WAIT+" millis from: "+stacktrace()); + log.debug("interruptSoon: will interrupt in "+INTERRUPT_WAIT+" millis"); synchronized (interrupted) { if (interrupted.get()) { log.debug("interruptSoon: interrupt flag already set, not setting again"); -- 2.17.1 From 1cbfa40cdd96b05abdb4b0453d1a831ced85d340 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 16:19:33 -0400 Subject: [PATCH 35/78] do not reset interrupted flag before interrupting --- .../java/bubble/service/device/StandardFlexRouterService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 22eb1308..990065d8 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -97,7 +97,6 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute background(() -> { sleep(INTERRUPT_WAIT); log.debug("interruptSoon: interrupting..."); - synchronized (interrupted) { interrupted.set(false); } interrupt(); }, "StandardFlexRouterService.interruptSoon"); } -- 2.17.1 From 0cb9d598293f842d5dbc7e23cdc2dbfc528b4a79 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 17:42:00 -0400 Subject: [PATCH 36/78] only interrupt flex router service if stat is unreachable --- .../java/bubble/service/device/StandardFlexRouterService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 990065d8..dca18a31 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -112,6 +112,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute public FlexRouterStatus status(String uuid) { final FlexRouterStatus stat = statusMap.get(uuid); + if (stat == FlexRouterStatus.unreachable) interruptSoon(); return stat == null ? FlexRouterStatus.none : stat; } -- 2.17.1 From b6bf98a8e773f19491fb12fbd70aa6c5984f18e6 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 17:51:04 -0400 Subject: [PATCH 37/78] lazy-init known-host key --- bubble-server/src/main/java/bubble/ApiConstants.java | 8 ++++++-- .../src/main/java/bubble/dao/device/FlexRouterDAO.java | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index 338b676f..b56f4c55 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -75,8 +75,12 @@ public class ApiConstants { public static final GoogleAuthenticator G_AUTH = new GoogleAuthenticator(); - public static final String HOST_KEY = toStringOrDie("/etc/ssh/ssh_host_rsa_key.pub").trim(); - public static final String KNOWN_HOST_KEY = execScript("ssh-keyscan -t rsa $(hostname -d) 2>&1 | grep -v \"^#\""); + private static final AtomicReference knownHostKey = new AtomicReference<>(); + public static String getKnownHostKey () { + return lazyGet(knownHostKey, + () -> execScript("ssh-keyscan -t rsa $(hostname -d) 2>&1 | grep -v \"^#\""), + () -> die("getKnownHostKey")); + } public static final Predicate ALWAYS_TRUE = m -> true; public static final String HOME_DIR; diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index 93bb76d0..5c6e48c7 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -9,7 +9,7 @@ import org.springframework.stereotype.Repository; import java.util.List; -import static bubble.ApiConstants.KNOWN_HOST_KEY; +import static bubble.ApiConstants.getKnownHostKey; import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; import static org.hibernate.criterion.Restrictions.*; @@ -27,7 +27,7 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postCreate(FlexRouter router, Object context) { flexRouterService.register(router); - router.setHost_key(KNOWN_HOST_KEY); + router.setHost_key(getKnownHostKey()); return super.postCreate(router, context); } @@ -41,7 +41,7 @@ public class FlexRouterDAO extends AccountOwnedEntityDAO { @Override public FlexRouter postUpdate(FlexRouter router, Object context) { flexRouterService.register(router); - router.setHost_key(KNOWN_HOST_KEY); + router.setHost_key(getKnownHostKey()); return super.postUpdate(router, context); } -- 2.17.1 From bc68798f62a1ae5e7a74f17f1c5a08d1fb6ae951 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 23:05:32 -0400 Subject: [PATCH 38/78] WIP. add flex router selection algoritm. start integrating with mitm --- .../bubble/cloud/geoLocation/GeoLocation.java | 3 +- .../bubble/model/device/DeviceStatus.java | 3 +- .../java/bubble/model/device/FlexRouter.java | 22 +++-- .../resources/device/FlexRoutersResource.java | 1 + .../resources/stream/FilterHttpResource.java | 15 +++ .../bubble/service/device/FlexRouterInfo.java | 38 ++++++++ .../device/FlexRouterProximityComparator.java | 30 ++++++ .../device/StandardFlexRouterService.java | 54 +++++++++-- .../V2020090501__add_flex_router.sql | 5 +- .../roles/mitmproxy/files/bubble_api.py | 2 - .../roles/mitmproxy/files/bubble_modify.py | 4 + .../roles/mitmproxy/files/dns_spoofing.py | 41 +++++--- .../FlexRouterProximityComparatorTest.java | 97 +++++++++++++++++++ pom.xml | 1 + 14 files changed, 284 insertions(+), 32 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java create mode 100644 bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java create mode 100644 bubble-server/src/test/java/bubble/test/filter/FlexRouterProximityComparatorTest.java diff --git a/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java b/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java index c70bdad8..9704387f 100644 --- a/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java +++ b/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.ToString; import lombok.experimental.Accessors; import org.cobbzilla.util.math.Haversine; @@ -16,7 +17,7 @@ import javax.persistence.Transient; import static org.cobbzilla.util.daemon.ZillaRuntime.*; -@NoArgsConstructor @Accessors(chain=true) +@NoArgsConstructor @Accessors(chain=true) @ToString(of={"lat", "lon"}) public class GeoLocation { @Getter @Setter private String country; diff --git a/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java b/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java index 3900af22..bd85c589 100644 --- a/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java +++ b/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java @@ -9,6 +9,7 @@ import bubble.service.cloud.GeoService; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.ToString; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.wizard.cache.redis.RedisService; @@ -17,7 +18,7 @@ import static java.util.concurrent.TimeUnit.*; import static org.cobbzilla.util.daemon.ZillaRuntime.now; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; -@NoArgsConstructor @Accessors(chain=true) @Slf4j +@NoArgsConstructor @Accessors(chain=true) @ToString(of={"ip", "location"}) @Slf4j public class DeviceStatus { public static final DeviceStatus NO_DEVICE_STATUS = new DeviceStatus(); diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index 07202880..b5992e72 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -47,13 +47,18 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { @Column(nullable=false, updatable=false, length=UUID_MAXLEN) @Getter @Setter private String account; - @ECSearchable(filter=true) @ECField(index=20) + @ECSearchable @ECField(index=20) + @ECForeignKey(entity=Device.class) + @Column(nullable=false, updatable=false, length=UUID_MAXLEN) + @Getter @Setter private String device; + + @ECSearchable(filter=true) @ECField(index=30) @ECIndex @Column(nullable=false, updatable=false, length=50) @Getter @Setter private String ip; @JsonIgnore @Transient public String getName () { return getIp(); } - @ECField(index=30) @HasValue(message="err.sshPublicKey.required") + @ECField(index=40) @HasValue(message="err.sshPublicKey.required") @Type(type=ENCRYPTED_STRING) @Column(updatable=false, columnDefinition="varchar("+(10000+ENC_PAD)+") NOT NULL") @Getter private String key; public boolean hasKey () { return !empty(key); } @@ -63,36 +68,36 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { return this; } - @ECField(index=40) + @ECField(index=50) @ECIndex(unique=true) @Column(length=100, updatable=false, nullable=false) @Getter @Setter private String keyHash; public boolean hasKeyHash () { return !empty(keyHash); }; - @ECSearchable(filter=true) @ECField(index=50) + @ECSearchable(filter=true) @ECField(index=60) @ECIndex(unique=true) @Column(nullable=false, updatable=false) @Getter @Setter private Integer port; public boolean hasPort () { return port != null && port > 1024; } public String id () { return getIp() + "/" + getUuid(); } - @ECSearchable @ECField(index=60) + @ECSearchable @ECField(index=70) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean enabled = true; public boolean enabled () { return bool(enabled); } - @ECSearchable @ECField(index=70) + @ECSearchable @ECField(index=80) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean initialized = true; public boolean initialized() { return bool(initialized); } public boolean uninitialized() { return !initialized(); } - @ECSearchable @ECField(index=80) + @ECSearchable @ECField(index=90) @ECIndex @Column(nullable=false) @Getter @Setter private Boolean active = true; public boolean active() { return bool(active); } public boolean inactive() { return !active(); } - @ECSearchable(filter=true) @ECField(index=90) + @ECSearchable(filter=true) @ECField(index=100) @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(100+ENC_PAD)+") NOT NULL") @JsonIgnore @Getter @Setter private String token; public boolean hasToken () { return !empty(token); } @@ -105,5 +110,6 @@ public class FlexRouter extends IdentifiableBase implements HasAccount { @Transient @Getter @Setter private String host_key; public FlexRouterPing pingObject() { return new FlexRouterPing(this); } + public String proxyBaseUri() { return "http://127.0.0.1:" + getPort(); } } diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 699f7011..0baa568b 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -72,6 +72,7 @@ public class FlexRoutersResource extends AccountOwnedResource { + + private final GeoLocation geoLocation; + private final String preferredIp; + + @Override public int compare(FlexRouterInfo r1, FlexRouterInfo r2) { + if (r1.getVpnIp().equals(preferredIp)) { + return Integer.MIN_VALUE; + } + if (r2.getVpnIp().equals(preferredIp)) { + return Integer.MAX_VALUE; + } + if (r1.hasNoGeoLocation()) return Integer.MAX_VALUE; + if (r2.hasNoGeoLocation()) return Integer.MIN_VALUE; + final double distance1 = r1.distance(geoLocation); + final double distance2 = r2.distance(geoLocation); + final int delta = (int) (1000.0d * (distance1 - distance2)); + return delta != 0 ? delta : r1.getPort() - r2.getPort(); + } + +} diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index dca18a31..8e2d021a 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -1,14 +1,18 @@ package bubble.service.device; +import bubble.cloud.geoLocation.GeoLocation; import bubble.dao.device.FlexRouterDAO; +import bubble.model.device.DeviceStatus; import bubble.model.device.FlexRouter; import bubble.model.device.FlexRouterPing; +import bubble.service.cloud.GeoService; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.cobbzilla.util.collection.SingletonSet; import org.cobbzilla.util.daemon.AwaitResult; import org.cobbzilla.util.daemon.SimpleDaemon; import org.cobbzilla.util.http.HttpRequestBean; @@ -19,10 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -75,6 +76,9 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute public static final long DEFAULT_SLEEP_TIME = MINUTES.toMillis(2); @Autowired private FlexRouterDAO flexRouterDAO; + @Autowired private GeoService geoService; + @Autowired private DeviceService deviceService; + private final AtomicBoolean interrupted = new AtomicBoolean(false); @Override public void onStart() { @@ -109,6 +113,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute } private final Map statusMap = new ConcurrentHashMap<>(DEFAULT_MAX_TUNNELS); + private final Map activeRouters = new ConcurrentHashMap<>(DEFAULT_MAX_TUNNELS); public FlexRouterStatus status(String uuid) { final FlexRouterStatus stat = statusMap.get(uuid); @@ -116,6 +121,38 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute return stat == null ? FlexRouterStatus.none : stat; } + public FlexRouterStatus setStatus(FlexRouter router, FlexRouterStatus status) { + statusMap.put(router.getUuid(), status); + if (status == FlexRouterStatus.active && router.initialized()) { + final FlexRouterInfo info = activeRouters.get(router.getUuid()); + if (info == null || info.hasNoDeviceStatus()) { + try { + final DeviceStatus deviceStatus = deviceService.getDeviceStatus(router.getDevice()); + activeRouters.put(router.getUuid(), new FlexRouterInfo(router, deviceStatus)); + } catch (Exception e) { + log.error("setStatus: error creating FlexRouterInfo: "+shortError(e)); + } + } + } else { + activeRouters.remove(router.getUuid()); + } + return status; + } + + public Set selectClosestRouter (String accountUuid, String publicIp, String vpnIp) { + if (log.isDebugEnabled()) log.debug("selectClosestRouter: publicIp="+publicIp+", vpnIp="+vpnIp); + final GeoLocation geoLocation = geoService.locate(accountUuid, publicIp); + final Collection values = activeRouters.values(); + switch (values.size()) { + case 0: return Collections.emptySet(); + case 1: return new SingletonSet<>(values.iterator().next()); + default: + final Set candidates = new TreeSet<>(new FlexRouterProximityComparator(geoLocation, vpnIp)); + candidates.addAll(values); + return candidates; + } + } + @Override protected void process() { synchronized (interrupted) { interrupted.set(false); } try { @@ -148,9 +185,10 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute log.error("process: "+shortError(e)); } } + public boolean pingFlexRouter(FlexRouter router, HttpClient httpClient) { allowFlexKey(router.getKey()); - final String pingUrl = "http://127.0.0.1:" + router.getPort() + "/ping"; + final String pingUrl = router.proxyBaseUri() + "/ping"; final HttpRequestBean request = new HttpRequestBean(POST, pingUrl); final String prefix = "pingRouter(" + router + "): "; for (int i=0; i TEST_INFO = Arrays.asList( + ROUTER_NEW_YORK, + ROUTER_SINGAPORE, + ROUTER_LONDON, + ROUTER_ATLANTA, + ROUTER_CHICAGO, + ROUTER_NULL); + + private static final List EXPECTED_ATLANTA = Arrays.asList( + ROUTER_ATLANTA, + ROUTER_CHICAGO, + ROUTER_NEW_YORK, + ROUTER_LONDON, + ROUTER_SINGAPORE, + ROUTER_NULL); + + private static final List EXPECTED_LONDON = Arrays.asList( + ROUTER_LONDON, + ROUTER_NEW_YORK, + ROUTER_CHICAGO, + ROUTER_ATLANTA, + ROUTER_SINGAPORE, + ROUTER_NULL); + + private static final List EXPECTED_ATLANTA_PREFER_SINGAPORE = Arrays.asList( + ROUTER_SINGAPORE, + ROUTER_ATLANTA, + ROUTER_CHICAGO, + ROUTER_NEW_YORK, + ROUTER_LONDON, + ROUTER_NULL); + + @Test public void testProximitySortAtlanta () throws Exception { + testProximitySort(GEO_ATLANTA, EXPECTED_ATLANTA); + } + + @Test public void testProximitySortLondon () throws Exception { + testProximitySort(GEO_LONDON, EXPECTED_LONDON); + } + + @Test public void testProximitySortFromAtlantaWithPreferredIpInSingapore () throws Exception { + testProximitySort(GEO_ATLANTA, EXPECTED_ATLANTA_PREFER_SINGAPORE, ROUTER_SINGAPORE.getVpnIp()); + } + + private void testProximitySort(GeoLocation geo, List expected) throws Exception { + testProximitySort(geo, expected, "127.3.3.3"); + } + + private void testProximitySort(GeoLocation geo, List expected, String preferredIp) throws Exception { + final List test = new ArrayList<>(TEST_INFO); + Collections.shuffle(test); + final FlexRouterProximityComparator comparator = new FlexRouterProximityComparator(geo, preferredIp); + final Set sorted = new TreeSet<>(comparator); + sorted.addAll(test); + final List actual = new ArrayList<>(sorted); + assertEquals("wrong number of results", expected.size(), actual.size()); + for (int i=0; ibubble.test.filter.ProxyTest bubble.test.filter.TrafficAnalyticsTest bubble.test.filter.BlockSummaryTest + bubble.test.filter.FlexRouterProximityComparatorTest bubble.test.system.BackupTest bubble.test.system.NetworkTest bubble.abp.spec.BlockListTest -- 2.17.1 From 86758590f1ad60e328580345771cd2cfddde2105 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Sep 2020 23:14:16 -0400 Subject: [PATCH 39/78] fix stub calls to is_flex_domain --- .../roles/mitmproxy/files/bubble_api.py | 2 +- .../roles/mitmproxy/files/dns_spoofing.py | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index b8359458..f569f3fd 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -218,7 +218,7 @@ def is_not_from_vpn(client_addr): def is_flex_domain(client_addr, fqdn): check_fqdn = fqdn while '.' in check_fqdn: - found = REDIS.sismember("flexLists~"+client_addr+"~UNION") + found = REDIS.sismember(fqdn, "flexLists~"+client_addr+"~UNION") if found: return True check_fqdn = check_fqdn[check_fqdn.index('.')+1:] diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index 78d79945..c805fec7 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -247,16 +247,18 @@ class Rerouter: def request(self, flow): check_flex_host = self.bubble_handle_request(flow) - if check_flex_host is not None and is_flex_domain(check_flex_host): - if bubble_log.isEnabledFor(INFO): - bubble_log.info('request: check_flex_host is '+check_flex_host+' locating flex router') - # select flex router - # send request to flex router - # set status line - # populate headers - # flow.response.headers = - # populate body stream, ensure ".iter_content" works - bubble_filter_response(flow) + if check_flex_host is not None: + client_addr = flow.client_conn.address[0] + if is_flex_domain(client_addr, check_flex_host): + if bubble_log.isEnabledFor(INFO): + bubble_log.info('request: check_flex_host is '+check_flex_host+' locating flex router') + # select flex router + # send request to flex router + # set status line + # populate headers + # flow.response.headers = + # populate body stream, ensure ".iter_content" works + bubble_filter_response(flow) addons = [Rerouter()] -- 2.17.1 From 544eb7fc2042521b7c5c1ed491dace0f1af167ad Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 01:16:15 -0400 Subject: [PATCH 40/78] WIP. starting to use flex router from mitm --- .../stream/FilterConnCheckRequest.java | 10 +- .../resources/stream/FilterHttpResource.java | 49 ++++++---- .../bubble/service/device/FlexRouterInfo.java | 29 ++++-- .../device/FlexRouterProximityComparator.java | 17 ++-- .../device/StandardFlexRouterService.java | 2 +- .../roles/mitmproxy/files/bubble_api.py | 38 ++++++-- .../roles/mitmproxy/files/bubble_flex.py | 94 +++++++++++++++++++ .../roles/mitmproxy/files/bubble_modify.py | 11 ++- .../roles/mitmproxy/files/dns_spoofing.py | 20 ++-- 9 files changed, 210 insertions(+), 60 deletions(-) create mode 100644 bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java index ca4478d4..9016d9c7 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterConnCheckRequest.java @@ -12,14 +12,14 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.empty; public class FilterConnCheckRequest { - @Getter @Setter private String addr; - public boolean hasAddr() { return !empty(addr); } + @Getter @Setter private String clientAddr; + public boolean hasClientAddr() { return !empty(clientAddr); } + + @Getter @Setter private String serverAddr; + public boolean hasServerAddr() { return !empty(serverAddr); } @Getter @Setter private String[] fqdns; public boolean hasFqdns() { return !empty(fqdns); } public boolean hasFqdn(String f) { return hasFqdns() && ArrayUtils.contains(fqdns, f); } - @Getter @Setter private String remoteAddr; - public boolean hasRemoteAddr() { return !empty(remoteAddr); } - } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 6aa3468a..321f7d78 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -26,6 +26,7 @@ import bubble.service.block.BlockStatsService; import bubble.service.block.BlockStatsSummary; import bubble.service.boot.SelfNodeService; import bubble.service.device.DeviceService; +import bubble.service.device.FlexRouterInfo; import bubble.service.device.StandardFlexRouterService; import bubble.service.stream.ConnectionCheckResponse; import bubble.service.stream.StandardRuleEngineService; @@ -149,7 +150,7 @@ public class FilterHttpResource { @Context ContainerRequest request, FilterConnCheckRequest connCheckRequest) { final String prefix = "checkConnection: "; - if (connCheckRequest == null || !connCheckRequest.hasAddr() || !connCheckRequest.hasRemoteAddr()) { + if (connCheckRequest == null || !connCheckRequest.hasServerAddr() || !connCheckRequest.hasClientAddr()) { if (log.isDebugEnabled()) log.debug(prefix+"invalid connCheckRequest, returning forbidden"); return forbidden(); } @@ -160,12 +161,12 @@ public class FilterHttpResource { if (isLocalIp) { // if it is for our host or net name, passthru if (connCheckRequest.hasFqdns() && (connCheckRequest.hasFqdn(getThisNode().getFqdn()) || connCheckRequest.hasFqdn(getThisNetwork().getNetworkDomain()))) { - if (log.isDebugEnabled()) log.debug(prefix + "returning passthru for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + if (log.isDebugEnabled()) log.debug(prefix + "returning passthru for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr()); return ok(ConnectionCheckResponse.passthru); } } - final String vpnAddr = connCheckRequest.getRemoteAddr(); + final String vpnAddr = connCheckRequest.getClientAddr(); final Device device = deviceService.findDeviceByIp(vpnAddr); if (device == null) { if (log.isDebugEnabled()) log.debug(prefix+"device not found for IP "+vpnAddr+", returning not found"); @@ -181,14 +182,14 @@ public class FilterHttpResource { } if (isLocalIp) { - final boolean showStats = showStats(accountUuid, connCheckRequest.getAddr(), connCheckRequest.getFqdns()); + final boolean showStats = showStats(accountUuid, connCheckRequest.getServerAddr(), connCheckRequest.getFqdns()); final DeviceSecurityLevel secLevel = device.getSecurityLevel(); if (showStats && secLevel.supportsRequestModification()) { // allow it for now - if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showStats=true, secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showStats=true, secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr()); return ok(ConnectionCheckResponse.noop); } else { - if (log.isDebugEnabled()) log.debug(prefix + "returning block (showStats="+showStats+", secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getAddr()); + if (log.isDebugEnabled()) log.debug(prefix + "returning block (showStats="+showStats+", secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr()); return ok(ConnectionCheckResponse.block); } } @@ -207,17 +208,17 @@ public class FilterHttpResource { if (connCheckRequest.hasFqdns()) { final String[] fqdns = connCheckRequest.getFqdns(); for (String fqdn : fqdns) { - checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getAddr(), fqdn); + checkResponse = ruleEngine.checkConnection(account, device, retained, connCheckRequest.getServerAddr(), fqdn); if (checkResponse != ConnectionCheckResponse.noop) { - if (log.isDebugEnabled()) log.debug(prefix + "found " + checkResponse + " (breaking) for fqdn/addr=" + fqdn + "/" + connCheckRequest.getAddr()); + if (log.isDebugEnabled()) log.debug(prefix + "found " + checkResponse + " (breaking) for fqdn/addr=" + fqdn + "/" + connCheckRequest.getServerAddr()); break; } } - if (log.isDebugEnabled()) log.debug(prefix+"returning "+checkResponse+" for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getAddr()); + if (log.isDebugEnabled()) log.debug(prefix+"returning "+checkResponse+" for fqdns/addr="+Arrays.toString(fqdns)+"/"+ connCheckRequest.getServerAddr()); return ok(checkResponse); } else { - if (log.isDebugEnabled()) log.debug(prefix+"returning noop for NO fqdns, addr="+connCheckRequest.getAddr()); + if (log.isDebugEnabled()) log.debug(prefix+"returning noop for NO fqdns, addr="+connCheckRequest.getServerAddr()); return ok(ConnectionCheckResponse.noop); } } @@ -228,7 +229,7 @@ public class FilterHttpResource { } private boolean isForLocalIp(FilterConnCheckRequest connCheckRequest) { - return connCheckRequest.hasAddr() && getConfiguredIps().contains(connCheckRequest.getAddr()); + return connCheckRequest.hasServerAddr() && getConfiguredIps().contains(connCheckRequest.getServerAddr()); } private boolean isForLocalIp(FilterMatchersRequest matchersRequest) { @@ -631,16 +632,24 @@ public class FilterHttpResource { return ok(summary); } - @GET @Path(EP_FLEX_ROUTERS+"/{requestId}") - public Response getFlexRouters(@Context Request req, - @Context ContainerRequest ctx, - @PathParam("requestId") String requestId) { - final FilterSubContext filterCtx = new FilterSubContext(req, requestId); - final FilterHttpRequest request = filterCtx.request; - final DeviceStatus deviceStatus = deviceService.getDeviceStatus(request.getDevice().getUuid()); + @GET @Path(EP_FLEX_ROUTERS) + public Response getFlexRouter(@Context Request req, + @Context ContainerRequest ctx) { + final String vpnIp = getRemoteAddr(req); + final Device device = deviceService.findDeviceByIp(vpnIp); + if (device == null) { + log.warn("getFlexRouters: device not found: "+vpnIp); + return ok(Collections.emptySet()); + } + + if (log.isDebugEnabled()) log.debug("getFlexRouters: finding routers for vpnIp="+vpnIp); + final DeviceStatus deviceStatus = deviceService.getDeviceStatus(device.getUuid()); final String publicIp = deviceStatus.getIp(); - final String vpnIp = request.getMatchersResponse().getRequest().getClientAddr(); - return ok(flexRouterService.selectClosestRouter(request.getAccount().getUuid(), vpnIp, publicIp)); + Collection routers = flexRouterService.selectClosestRouter(device.getAccount(), vpnIp, publicIp); + + if (log.isDebugEnabled()) log.debug("getFlexRouters: found router(s) for vpnIp="+vpnIp+": "+json(routers)); + if (routers.isEmpty()) return ok(Collections.emptySet()); + return ok(routers.iterator().next().initAuth()); } @POST @Path(EP_LOGS+"/{requestId}") diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java index 6ae7e057..12d0e398 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java @@ -3,25 +3,32 @@ package bubble.service.device; import bubble.cloud.geoLocation.GeoLocation; import bubble.model.device.DeviceStatus; import bubble.model.device.FlexRouter; -import lombok.EqualsAndHashCode; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; +import lombok.Setter; import lombok.ToString; import static bubble.model.device.DeviceStatus.NO_DEVICE_STATUS; +import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; +import static org.cobbzilla.util.json.JsonUtil.json; -@EqualsAndHashCode(of="port") @ToString +@ToString public class FlexRouterInfo { - @Getter private final String vpnIp; - @Getter private final int port; - @Getter private final DeviceStatus deviceStatus; + @JsonIgnore @Getter private final FlexRouter router; + @JsonIgnore @Getter private final DeviceStatus deviceStatus; + @Getter @Setter private String auth; public FlexRouterInfo (FlexRouter router, DeviceStatus deviceStatus) { - this.vpnIp = router.getIp(); - this.port = router.getPort(); + this.router = router; this.deviceStatus = deviceStatus; } + @JsonIgnore public String getVpnIp () { return router.getIp(); } + @JsonIgnore public int getPort () { return router.getPort(); } + + public String getProxyUrl () { return router.proxyBaseUri(); } + public boolean hasGeoLocation () { return hasDeviceStatus() && deviceStatus.getLocation() != null && deviceStatus.getLocation().hasLatLon(); } public boolean hasNoGeoLocation () { return !hasGeoLocation(); } @@ -35,4 +42,12 @@ public class FlexRouterInfo { public boolean hasIp () { return hasDeviceStatus() && deviceStatus.hasIp(); } public String ip () { return hasIp() ? deviceStatus.getIp() : null; } + public FlexRouterInfo initAuth () { auth = json(router.pingObject(), COMPACT_MAPPER); return this; } + + @Override public int hashCode() { return getPort(); } + + @Override public boolean equals(Object obj) { + return obj instanceof FlexRouterInfo && ((FlexRouterInfo) obj).getPort() == getPort(); + } + } diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java index 815a1099..8956f475 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java @@ -13,14 +13,19 @@ public class FlexRouterProximityComparator implements Comparator private final String preferredIp; @Override public int compare(FlexRouterInfo r1, FlexRouterInfo r2) { - if (r1.getVpnIp().equals(preferredIp)) { - return Integer.MIN_VALUE; - } - if (r2.getVpnIp().equals(preferredIp)) { - return Integer.MAX_VALUE; - } + + // if preferred ip matches, that takes precedence over everything + if (r1.getVpnIp().equals(preferredIp)) return Integer.MIN_VALUE; + if (r2.getVpnIp().equals(preferredIp)) return Integer.MAX_VALUE; + + // if a router has no location info, it goes last if (r1.hasNoGeoLocation()) return Integer.MAX_VALUE; if (r2.hasNoGeoLocation()) return Integer.MIN_VALUE; + + // if WE have no location info, just compare ports (we choose randomly) + if (geoLocation == null) return r1.getPort() - r2.getPort(); + + // compare distances. if they are equals, just compare ports (we choose randomly) final double distance1 = r1.distance(geoLocation); final double distance2 = r2.distance(geoLocation); final int delta = (int) (1000.0d * (distance1 - distance2)); diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 8e2d021a..113dd445 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -141,7 +141,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute public Set selectClosestRouter (String accountUuid, String publicIp, String vpnIp) { if (log.isDebugEnabled()) log.debug("selectClosestRouter: publicIp="+publicIp+", vpnIp="+vpnIp); - final GeoLocation geoLocation = geoService.locate(accountUuid, publicIp); + final GeoLocation geoLocation = publicIp == null ? null : geoService.locate(accountUuid, publicIp); final Collection values = activeRouters.values(); switch (values.size()) { case 0: return Collections.emptySet(); diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index f569f3fd..e9f111c5 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -12,6 +12,7 @@ import subprocess import time import traceback import uuid +from http import HTTPStatus from netaddr import IPAddress, IPNetwork from bubble_vpn4 import wireguard_network_ipv4 from bubble_vpn6 import wireguard_network_ipv6 @@ -24,6 +25,7 @@ HEADER_USER_AGENT = 'User-Agent' HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy' HEADER_REFERER = 'Referer' HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru' +HEADER_FLEX_AUTH = 'X-Bubble-Flex-Auth' CTX_BUBBLE_MATCHERS = 'X-Bubble-Matchers' CTX_BUBBLE_ABORT = 'X-Bubble-Abort' @@ -55,6 +57,10 @@ VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6) parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") +def status_reason(status_code): + return HTTPStatus(status_code).phrase + + def redis_set(name, value, ex): REDIS.set(name, value, nx=True, ex=ex) REDIS.set(name, value, xx=True, ex=ex) @@ -75,7 +81,7 @@ def bubble_activity_log(client_addr, server_addr, event, data): pass -def bubble_conn_check(remote_addr, addr, fqdns, security_level): +def bubble_conn_check(client_addr, server_addr, fqdns, security_level): if debug_capture_fqdn and fqdns: for f in debug_capture_fqdn: if f in fqdns: @@ -84,15 +90,15 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level): return 'noop' headers = { - 'X-Forwarded-For': remote_addr, - 'Accept' : 'application/json', + 'X-Forwarded-For': client_addr, + 'Accept': 'application/json', 'Content-Type': 'application/json' } try: data = { - 'addr': str(addr), + 'serverAddr': str(server_addr), 'fqdns': fqdns, - 'remoteAddr': remote_addr + 'clientAddr': client_addr } response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data) if response.ok: @@ -110,6 +116,26 @@ def bubble_conn_check(remote_addr, addr, fqdns, security_level): return None +def bubble_get_flex_router(client_addr): + headers = { + 'X-Forwarded-For': client_addr, + 'Accept': 'application/json' + } + try: + response = requests.get('http://127.0.0.1:'+bubble_port+'/api/filter/flexRouters', headers=headers) + if response.ok: + return response.json() + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_get_flex_routes API call failed: '+repr(response)) + return None + + except Exception as e: + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_get_flex_routes API call failed: '+repr(e)) + traceback.print_exc() + return None + + DEBUG_MATCHER_NAME = 'DebugCaptureMatcher' DEBUG_MATCHER = { 'decision': 'match', @@ -139,7 +165,7 @@ def bubble_matchers(req_id, client_addr, server_addr, flow, host): headers = { 'X-Forwarded-For': client_addr, - 'Accept' : 'application/json', + 'Accept': 'application/json', 'Content-Type': 'application/json' } if HEADER_USER_AGENT not in flow.request.headers: diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py new file mode 100644 index 00000000..43dab2af --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +import requests +from mitmproxy.net.http import headers as nheaders + +from bubble_api import HEADER_FLEX_AUTH, bubble_get_flex_router + +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + +bubble_log = logging.getLogger(__name__) + + +def prepend_remainder_to_stream(remainder, raw): + first = True + while True: + chunk = raw.read(8192) + if first and chunk and len(remainder) > 0: + yield remainder + chunk + elif chunk: + yield chunk + else: + break + if first: + first = False + + +def set_flex_response(client_addr, flex_host, flow): + if bubble_log.isEnabledFor(INFO): + bubble_log.info('set_flex_response: checking for flex router for host: '+flex_host) + router = bubble_get_flex_router(client_addr) + if router is None or 'auth' not in router: + if bubble_log.isEnabledFor(INFO): + bubble_log.info('set_flex_response: no flex router for host: '+flex_host) + return + + if bubble_log.isEnabledFor(INFO): + bubble_log.info('set_flex_response: found router '+repr(router)+' for flex host: '+flex_host) + + # build the request + url = flow.request.scheme + '://' + flex_host + flow.request.path + headers = flow.request.headers + headers[HEADER_FLEX_AUTH] = router['auth'] + proxy_url = router['proxyUrl'] + proxies = { "http": proxy_url, "https": proxy_url } + + # send request to flex router + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: sending flex request for '+url+' to '+proxy_url) + response = requests.request(flow.request.method, url, + headers=headers, + timeout=(15, 15), + stream=True, + data=flow.request.stream, + proxies=proxies) + + # Parse the response, we have to do this raw to capture the full status line + + # Status line + next_bytes = response.raw.read(16384) # enough to read all headers + response_text = next_bytes.decode() + lines = response_text.splitlines() + status_line = lines[0] + status_line_parts = status_line.split() + flow.response.http_version = status_line_parts[0] + flow.response.status_code = int(status_line_parts[1]) + flow.response.reason = status_line_parts[2] + + # Headers + response_headers = nheaders.Headers() + end_of_headers = False + lines = lines[1:] + bytes_consumed = len(status_line) + 1 + while True: + for header_line in lines[:-1]: + if header_line == '': + bytes_consumed = bytes_consumed + 1 + end_of_headers = True + break + header_parts = header_line.split(':', 1) + response_headers[header_parts[0].strip()] = header_parts[1].strip() + bytes_consumed = bytes_consumed + len(header_line) + 1 + if end_of_headers: + break + next_bytes = response.raw.read(8192) + next_text = lines[-1] + '\n' + next_bytes.decode() + lines = next_text.splitlines() + + flow.response.headers = response_headers + + # Body -- prepend remainder left over from parsing headers + remainder = next_bytes[bytes_consumed:] + flow.response.stream = prepend_remainder_to_stream(remainder, response.raw) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 3d1e4e21..d39a18d4 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -11,7 +11,7 @@ import traceback from mitmproxy.net.http import Headers from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, BUBBLE_URI_PREFIX, \ - HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, \ + HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, status_reason, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, get_flow_ctx, add_flow_ctx, \ HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header @@ -196,6 +196,11 @@ def bubble_filter_response(flow): def responseheaders(flow): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.info('flow.response.http_version='+flow.response.http_version) + bubble_log.info('flow.response.status_code='+str(flow.response.status_code)) + bubble_log.info('flow.response.reason='+flow.response.reason) + path = flow.request.path if path and path.startswith(BUBBLE_URI_PREFIX): if path.startswith(HEALTH_CHECK_URI): @@ -205,6 +210,7 @@ def responseheaders(flow): flow.response.headers[HEADER_HEALTH_CHECK] = 'OK' flow.response.headers[HEADER_CONTENT_LENGTH] = '3' flow.response.status_code = 200 + flow.response.reason = 'OK' flow.response.stream = lambda chunks: [b'OK\n'] else: uri = 'http://127.0.0.1:' + bubble_port + '/' + path[len(BUBBLE_URI_PREFIX):] @@ -232,6 +238,7 @@ def responseheaders(flow): for key, value in response.headers.items(): flow.response.headers[key] = value flow.response.status_code = response.status_code + flow.response.reason = status_reason(response.status_code) flow.response.stream = lambda chunks: send_bubble_response(response) else: @@ -244,6 +251,7 @@ def responseheaders(flow): flow.response.headers = Headers() flow.response.headers[HEADER_LOCATION] = abort_location flow.response.status_code = abort_code + flow.response.reason = status_reason(abort_code) flow.response.stream = lambda chunks: [] else: if HEADER_CONTENT_TYPE in flow.response.headers: @@ -254,6 +262,7 @@ def responseheaders(flow): bubble_log.info('responseheaders: aborting request with HTTP status '+str(abort_code)+', path was: '+path) flow.response.headers = Headers() flow.response.status_code = abort_code + flow.response.reason = status_reason(abort_code) flow.response.stream = lambda chunks: abort_data(content_type) elif flow.response.status_code // 100 != 2: diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index c805fec7..7585ad7a 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -32,11 +32,11 @@ import time import uuid from mitmproxy.net.http import headers as nheaders -from bubble_api import bubble_matchers, bubble_activity_log, HEALTH_CHECK_URI, REDIS, \ +from bubble_api import bubble_matchers, bubble_activity_log, HEALTH_CHECK_URI, \ CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \ add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain from bubble_config import bubble_host, bubble_host_alias -from bubble_modify import bubble_filter_response +from bubble_flex import set_flex_response bubble_log = logging.getLogger(__name__) @@ -246,19 +246,11 @@ class Rerouter: return host def request(self, flow): - check_flex_host = self.bubble_handle_request(flow) - if check_flex_host is not None: + flex_host = self.bubble_handle_request(flow) + if flex_host is not None: client_addr = flow.client_conn.address[0] - if is_flex_domain(client_addr, check_flex_host): - if bubble_log.isEnabledFor(INFO): - bubble_log.info('request: check_flex_host is '+check_flex_host+' locating flex router') - # select flex router - # send request to flex router - # set status line - # populate headers - # flow.response.headers = - # populate body stream, ensure ".iter_content" works - bubble_filter_response(flow) + if is_flex_domain(client_addr, flex_host): + set_flex_response(flow) addons = [Rerouter()] -- 2.17.1 From 5710f7a64ccbf4e3934c99e2a1c1f8ea410df1b5 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 01:20:13 -0400 Subject: [PATCH 41/78] remove trailing comma when parsing device last handshake time --- .../src/main/java/bubble/model/device/DeviceStatus.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java b/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java index bd85c589..53bb1aa4 100644 --- a/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java +++ b/bubble-server/src/main/java/bubble/model/device/DeviceStatus.java @@ -98,7 +98,8 @@ public class DeviceStatus { } else { for (int i=0; i Date: Tue, 8 Sep 2020 01:38:20 -0400 Subject: [PATCH 42/78] remove debug logging --- .../resources/packer/roles/mitmproxy/files/bubble_modify.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index d39a18d4..420b13ce 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -196,11 +196,6 @@ def bubble_filter_response(flow): def responseheaders(flow): - if bubble_log.isEnabledFor(DEBUG): - bubble_log.info('flow.response.http_version='+flow.response.http_version) - bubble_log.info('flow.response.status_code='+str(flow.response.status_code)) - bubble_log.info('flow.response.reason='+flow.response.reason) - path = flow.request.path if path and path.startswith(BUBBLE_URI_PREFIX): if path.startswith(HEALTH_CHECK_URI): -- 2.17.1 From c8ee1fa84646e980615d728d2ca84b686ed3d3d8 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 01:38:42 -0400 Subject: [PATCH 43/78] update code comments --- .../packer/roles/mitmproxy/files/bubble_flex.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index 43dab2af..13310bdd 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -43,7 +43,7 @@ def set_flex_response(client_addr, flex_host, flow): headers = flow.request.headers headers[HEADER_FLEX_AUTH] = router['auth'] proxy_url = router['proxyUrl'] - proxies = { "http": proxy_url, "https": proxy_url } + proxies = {"http": proxy_url, "https": proxy_url} # send request to flex router if bubble_log.isEnabledFor(DEBUG): @@ -58,7 +58,9 @@ def set_flex_response(client_addr, flex_host, flow): # Parse the response, we have to do this raw to capture the full status line # Status line - next_bytes = response.raw.read(16384) # enough to read all headers + # 16K should be enough to capture status line and all headers + # see https://stackoverflow.com/questions/686217/maximum-on-http-header-values + next_bytes = response.raw.read(16384) response_text = next_bytes.decode() lines = response_text.splitlines() status_line = lines[0] @@ -83,12 +85,13 @@ def set_flex_response(client_addr, flex_host, flow): bytes_consumed = bytes_consumed + len(header_line) + 1 if end_of_headers: break - next_bytes = response.raw.read(8192) + next_bytes = response.raw.read(8192) # wow, headers are big! continue reading in 8K chunks next_text = lines[-1] + '\n' + next_bytes.decode() lines = next_text.splitlines() flow.response.headers = response_headers - # Body -- prepend remainder left over from parsing headers + # Body + # Determine the remainder left over from parsing headers, send that first remainder = next_bytes[bytes_consumed:] flow.response.stream = prepend_remainder_to_stream(remainder, response.raw) -- 2.17.1 From 7285e414952ff331faf5928818ef47e5f6b6b36d Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 01:46:18 -0400 Subject: [PATCH 44/78] enable priming on passthru app --- .../main/resources/models/apps/passthru/bubbleApp_passthru.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json index 2f5ffc4c..73b54acb 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json @@ -5,7 +5,7 @@ "template": true, "enabled": true, "priority": 1000000, - "canPrime": false, + "canPrime": true, "dataConfig": { "dataDriver": "bubble.app.passthru.TlsPassthruAppDataDriver", "presentation": "none", -- 2.17.1 From 6bcb99979bd6935266ad798975f8696ff8bc0980 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 02:14:07 -0400 Subject: [PATCH 45/78] WIP. getting flex routing to work in mitm --- .../java/bubble/resources/stream/FilterHttpResource.java | 5 +++++ bubble-server/src/main/java/bubble/rule/AppRuleDriver.java | 5 +++-- .../resources/packer/roles/mitmproxy/files/bubble_api.py | 6 +++++- .../resources/packer/roles/mitmproxy/files/dns_spoofing.py | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 321f7d78..5edbf382 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -63,6 +63,7 @@ import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; +import static org.cobbzilla.util.http.HttpContentTypes.TEXT_PLAIN; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.network.NetworkUtil.isLocalIpv4; @@ -623,6 +624,7 @@ public class FilterHttpResource { } @GET @Path(EP_STATUS+"/{requestId}") + @Produces(APPLICATION_JSON) public Response getRequestStatus(@Context Request req, @Context ContainerRequest ctx, @PathParam("requestId") String requestId) { @@ -633,6 +635,7 @@ public class FilterHttpResource { } @GET @Path(EP_FLEX_ROUTERS) + @Produces(APPLICATION_JSON) public Response getFlexRouter(@Context Request req, @Context ContainerRequest ctx) { final String vpnIp = getRemoteAddr(req); @@ -653,6 +656,7 @@ public class FilterHttpResource { } @POST @Path(EP_LOGS+"/{requestId}") + @Produces(APPLICATION_JSON) public Response requestLog(@Context Request req, @Context ContainerRequest ctx, @PathParam("requestId") String requestId, @@ -666,6 +670,7 @@ public class FilterHttpResource { = new ExpirationMap<>(1000, DAYS.toMillis(3), ExpirationEvictionPolicy.atime); @POST @Path(EP_FOLLOW+"/{requestId}") + @Produces(TEXT_PLAIN) public Response followLink(@Context Request req, @Context ContainerRequest ctx, @PathParam("requestId") String requestId, diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index cc9effac..e370c917 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -15,6 +15,7 @@ import bubble.service.stream.AppRuleHarness; import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; +import com.mchange.v1.util.ArrayUtils; import org.cobbzilla.util.handlebars.HandlebarsUtil; import org.cobbzilla.wizard.cache.redis.RedisService; import org.glassfish.grizzly.http.server.Request; @@ -63,8 +64,8 @@ public interface AppRuleDriver { defineRedisSet(redis, ip, REDIS_FILTER_LISTS, list, filterDomains); } - static void defineRedisFlexSet(RedisService redis, String ip, String list, String[] filterDomains) { - defineRedisSet(redis, ip, REDIS_FLEX_LISTS, list, filterDomains); + static void defineRedisFlexSet(RedisService redis, String ip, String list, String[] flexDomains) { + defineRedisSet(redis, ip, REDIS_FLEX_LISTS, list, flexDomains); } static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index e9f111c5..51e906b6 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -242,10 +242,14 @@ def is_not_from_vpn(client_addr): def is_flex_domain(client_addr, fqdn): + if fqdn == bubble_host or fqdn == bubble_host_alias or fqdn == bubble_sage_host: + return False check_fqdn = fqdn while '.' in check_fqdn: - found = REDIS.sismember(fqdn, "flexLists~"+client_addr+"~UNION") + found = REDIS.sismember('flexLists~'+client_addr+'~UNION', fqdn) if found: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('is_flex_domain: returning True for: '+fqdn) return True check_fqdn = check_fqdn[check_fqdn.index('.')+1:] return False diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index 7585ad7a..2706308e 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -250,7 +250,7 @@ class Rerouter: if flex_host is not None: client_addr = flow.client_conn.address[0] if is_flex_domain(client_addr, flex_host): - set_flex_response(flow) + set_flex_response(client_addr, flex_host, flow) addons = [Rerouter()] -- 2.17.1 From 62f0398ca61ed607855e49255205329ac0ddbc3e Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 02:22:36 -0400 Subject: [PATCH 46/78] remove unused import --- bubble-server/src/main/java/bubble/rule/AppRuleDriver.java | 1 - 1 file changed, 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index e370c917..18071c8c 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -15,7 +15,6 @@ import bubble.service.stream.AppRuleHarness; import bubble.service.stream.ConnectionCheckResponse; import com.fasterxml.jackson.databind.JsonNode; import com.github.jknack.handlebars.Handlebars; -import com.mchange.v1.util.ArrayUtils; import org.cobbzilla.util.handlebars.HandlebarsUtil; import org.cobbzilla.wizard.cache.redis.RedisService; import org.glassfish.grizzly.http.server.Request; -- 2.17.1 From 43d08c52946caab59f4fbaa2c8e198b626e48620 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 02:39:13 -0400 Subject: [PATCH 47/78] WIP. working on flex routing --- .../resources/stream/FilterHttpResource.java | 23 ++++++++++++------- ...r.sql => V2020090801__add_flex_router.sql} | 0 bubble-server/src/main/resources/messages | 2 +- bubble-web | 2 +- utils/cobbzilla-wizard | 2 +- 5 files changed, 18 insertions(+), 11 deletions(-) rename bubble-server/src/main/resources/db/migration/{V2020090501__add_flex_router.sql => V2020090801__add_flex_router.sql} (100%) diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 5edbf382..34ba0077 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -90,6 +90,8 @@ public class FilterHttpResource { @Autowired private BlockStatsService blockStats; @Autowired private StandardFlexRouterService flexRouterService; + private static final Response NO_FLEX_ROUTER = OK_EMPTY_LIST; + private static final long ACTIVE_REQUEST_TIMEOUT = HOURS.toSeconds(12); @Getter(lazy=true) private final RedisService activeRequestCache = redis.prefixNamespace(getClass().getSimpleName()+".requests"); @@ -638,20 +640,25 @@ public class FilterHttpResource { @Produces(APPLICATION_JSON) public Response getFlexRouter(@Context Request req, @Context ContainerRequest ctx) { - final String vpnIp = getRemoteAddr(req); - final Device device = deviceService.findDeviceByIp(vpnIp); + final String publicIp = getRemoteAddr(req); + final Device device = deviceService.findDeviceByIp(publicIp); if (device == null) { - log.warn("getFlexRouters: device not found: "+vpnIp); - return ok(Collections.emptySet()); + log.warn("getFlexRouter: device not found with IP: "+publicIp); + return NO_FLEX_ROUTER; } - if (log.isDebugEnabled()) log.debug("getFlexRouters: finding routers for vpnIp="+vpnIp); final DeviceStatus deviceStatus = deviceService.getDeviceStatus(device.getUuid()); - final String publicIp = deviceStatus.getIp(); + if (!deviceStatus.hasIp()) { + log.error("getFlexRouter: no device status for device: "+device); + return NO_FLEX_ROUTER; + } + final String vpnIp = deviceStatus.getIp(); + + if (log.isDebugEnabled()) log.debug("getFlexRouter: finding routers for vpnIp="+vpnIp); Collection routers = flexRouterService.selectClosestRouter(device.getAccount(), vpnIp, publicIp); - if (log.isDebugEnabled()) log.debug("getFlexRouters: found router(s) for vpnIp="+vpnIp+": "+json(routers)); - if (routers.isEmpty()) return ok(Collections.emptySet()); + if (log.isDebugEnabled()) log.debug("getFlexRouter: found router(s) for vpnIp="+vpnIp+": "+json(routers, COMPACT_MAPPER)); + if (routers.isEmpty()) return NO_FLEX_ROUTER; return ok(routers.iterator().next().initAuth()); } diff --git a/bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql b/bubble-server/src/main/resources/db/migration/V2020090801__add_flex_router.sql similarity index 100% rename from bubble-server/src/main/resources/db/migration/V2020090501__add_flex_router.sql rename to bubble-server/src/main/resources/db/migration/V2020090801__add_flex_router.sql diff --git a/bubble-server/src/main/resources/messages b/bubble-server/src/main/resources/messages index d97340e3..74c7c7f2 160000 --- a/bubble-server/src/main/resources/messages +++ b/bubble-server/src/main/resources/messages @@ -1 +1 @@ -Subproject commit d97340e367e1b15ab71dc1ccd564af3a0dc7a361 +Subproject commit 74c7c7f2efa41dd12d74cffff5013149136d3867 diff --git a/bubble-web b/bubble-web index 63128400..2d0df5ec 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 631284006bb691d3d6d88aa2c3544ebc6cfa2893 +Subproject commit 2d0df5ecd846bc81f72d37ec385ed8b46ae17fd9 diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 416d45b5..f57899c2 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 416d45b5ba87b5c798b2940f0bf357461c3a43c0 +Subproject commit f57899c2022b6dcf6d991fe4a1b0c748ca3a56b3 -- 2.17.1 From f42662db5c30430470a432664aafd12d7601c43b Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 02:52:12 -0400 Subject: [PATCH 48/78] fix empty return value for flex router --- .../bubble/resources/stream/FilterHttpResource.java | 8 +++----- .../packer/roles/mitmproxy/files/bubble_api.py | 10 +++++++--- .../packer/roles/mitmproxy/files/bubble_flex.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 34ba0077..e93fabbc 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -90,8 +90,6 @@ public class FilterHttpResource { @Autowired private BlockStatsService blockStats; @Autowired private StandardFlexRouterService flexRouterService; - private static final Response NO_FLEX_ROUTER = OK_EMPTY_LIST; - private static final long ACTIVE_REQUEST_TIMEOUT = HOURS.toSeconds(12); @Getter(lazy=true) private final RedisService activeRequestCache = redis.prefixNamespace(getClass().getSimpleName()+".requests"); @@ -644,13 +642,13 @@ public class FilterHttpResource { final Device device = deviceService.findDeviceByIp(publicIp); if (device == null) { log.warn("getFlexRouter: device not found with IP: "+publicIp); - return NO_FLEX_ROUTER; + return notFound(); } final DeviceStatus deviceStatus = deviceService.getDeviceStatus(device.getUuid()); if (!deviceStatus.hasIp()) { log.error("getFlexRouter: no device status for device: "+device); - return NO_FLEX_ROUTER; + return notFound(); } final String vpnIp = deviceStatus.getIp(); @@ -658,7 +656,7 @@ public class FilterHttpResource { Collection routers = flexRouterService.selectClosestRouter(device.getAccount(), vpnIp, publicIp); if (log.isDebugEnabled()) log.debug("getFlexRouter: found router(s) for vpnIp="+vpnIp+": "+json(routers, COMPACT_MAPPER)); - if (routers.isEmpty()) return NO_FLEX_ROUTER; + if (routers.isEmpty()) return notFound(); return ok(routers.iterator().next().initAuth()); } diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 51e906b6..d4e01da2 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -125,13 +125,17 @@ def bubble_get_flex_router(client_addr): response = requests.get('http://127.0.0.1:'+bubble_port+'/api/filter/flexRouters', headers=headers) if response.ok: return response.json() - if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_get_flex_routes API call failed: '+repr(response)) + elif response.status_code == 404: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_get_flex_routes: no router found for '+client_addr) + else: + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_get_flex_routes: API call failed with status: '+response.status_code) return None except Exception as e: if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_get_flex_routes API call failed: '+repr(e)) + bubble_log.error('bubble_get_flex_routes: API call failed with exception: '+repr(e)) traceback.print_exc() return None diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index 13310bdd..0d1f1e4a 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -47,7 +47,7 @@ def set_flex_response(client_addr, flex_host, flow): # send request to flex router if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: sending flex request for '+url+' to '+proxy_url) + bubble_log.debug('request: sending flex request for '+url+' to '+proxy_url+' with headers='+repr(headers)) response = requests.request(flow.request.method, url, headers=headers, timeout=(15, 15), -- 2.17.1 From 648574c6175239967e5d9889c53d671a799d6c09 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 18:28:45 -0400 Subject: [PATCH 49/78] WIP. flex routing basically works in mitm --- .../roles/mitmproxy/files/bubble_api.py | 1 - .../roles/mitmproxy/files/bubble_flex.py | 118 ++++++++++-------- .../roles/mitmproxy/files/bubble_modify.py | 32 +++-- 3 files changed, 89 insertions(+), 62 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index d4e01da2..a9513fe3 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -25,7 +25,6 @@ HEADER_USER_AGENT = 'User-Agent' HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy' HEADER_REFERER = 'Referer' HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru' -HEADER_FLEX_AUTH = 'X-Bubble-Flex-Auth' CTX_BUBBLE_MATCHERS = 'X-Bubble-Matchers' CTX_BUBBLE_ABORT = 'X-Bubble-Abort' diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index 0d1f1e4a..f4028944 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -1,16 +1,20 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # -import requests +from mitmproxy import http from mitmproxy.net.http import headers as nheaders +from bubble_api import bubble_get_flex_router +from bubble_modify import bubble_filter_response -from bubble_api import HEADER_FLEX_AUTH, bubble_get_flex_router +import requests import logging from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL bubble_log = logging.getLogger(__name__) +FLEX_TIMEOUT = 20 + def prepend_remainder_to_stream(remainder, raw): first = True @@ -37,61 +41,73 @@ def set_flex_response(client_addr, flex_host, flow): if bubble_log.isEnabledFor(INFO): bubble_log.info('set_flex_response: found router '+repr(router)+' for flex host: '+flex_host) + try: + process_flex(flex_host, flow, router) + except Exception as e: + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('set_flex_response: error processing: '+repr(e)) + + +def process_flex(flex_host, flow, router): + + # build the request URL + scheme = flow.request.scheme + url = scheme + '://' + flex_host + flow.request.path + + # copy request headers + # see: https://stackoverflow.com/questions/16789840/python-requests-cant-send-multiple-headers-with-same-key + request_headers = {} + for name in flow.request.headers: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_flex: processing request header: '+repr(name)) + if name in request_headers: + request_headers[name] = request_headers[name] + "," + flow.request.headers[name] + else: + request_headers[name] = flow.request.headers[name] - # build the request - url = flow.request.scheme + '://' + flex_host + flow.request.path - headers = flow.request.headers - headers[HEADER_FLEX_AUTH] = router['auth'] + # setup proxies proxy_url = router['proxyUrl'] proxies = {"http": proxy_url, "https": proxy_url} # send request to flex router if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: sending flex request for '+url+' to '+proxy_url+' with headers='+repr(headers)) - response = requests.request(flow.request.method, url, - headers=headers, - timeout=(15, 15), - stream=True, - data=flow.request.stream, - proxies=proxies) - - # Parse the response, we have to do this raw to capture the full status line - - # Status line - # 16K should be enough to capture status line and all headers - # see https://stackoverflow.com/questions/686217/maximum-on-http-header-values - next_bytes = response.raw.read(16384) - response_text = next_bytes.decode() - lines = response_text.splitlines() - status_line = lines[0] - status_line_parts = status_line.split() - flow.response.http_version = status_line_parts[0] - flow.response.status_code = int(status_line_parts[1]) - flow.response.reason = status_line_parts[2] - - # Headers + bubble_log.debug('process_flex: sending flex request for '+url+' to '+proxy_url+' with headers='+repr(request_headers)) + try: + response = requests.request(flow.request.method, url, + headers=request_headers, + timeout=(15, 15), + stream=True, + data=flow.request.stream, # use the original request body, if there is one + proxies=proxies) + except Exception as e: + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('process_flex: error sending request to '+url+': '+repr(e)) + return + + # Status line -- http version is buried in response.raw.version + raw_version = response.raw.version + if raw_version == 10: + http_version = 'HTTP/1.0' + elif raw_version == 11: + http_version = 'HTTP/1.1' + else: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_flex: invalid HTTP version detected, response.raw.version=='+repr(raw_version)) + return + + # Headers -- copy from requests dict to Headers multimap response_headers = nheaders.Headers() - end_of_headers = False - lines = lines[1:] - bytes_consumed = len(status_line) + 1 - while True: - for header_line in lines[:-1]: - if header_line == '': - bytes_consumed = bytes_consumed + 1 - end_of_headers = True - break - header_parts = header_line.split(':', 1) - response_headers[header_parts[0].strip()] = header_parts[1].strip() - bytes_consumed = bytes_consumed + len(header_line) + 1 - if end_of_headers: - break - next_bytes = response.raw.read(8192) # wow, headers are big! continue reading in 8K chunks - next_text = lines[-1] + '\n' + next_bytes.decode() - lines = next_text.splitlines() + for name in response.headers: + response_headers[name] = response.headers[name] - flow.response.headers = response_headers + # Construct the response -- use the raw response stream + flow.response = http.HTTPResponse(http_version=http_version, + status_code=response.status_code, + reason=response.reason, + headers=response_headers) - # Body - # Determine the remainder left over from parsing headers, send that first - remainder = next_bytes[bytes_consumed:] - flow.response.stream = prepend_remainder_to_stream(remainder, response.raw) + # Apply filters + bubble_filter_response(flow, response.raw) + + if bubble_log.isEnabledFor(INFO): + bubble_log.info('process_flex: successfully requested url '+url+' from flex router, proceeding ...') diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 420b13ce..e2c22789 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -135,14 +135,22 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c return response.content -def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, content_type, csp): +def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_encoding, content_type, csp): """ chunks is a generator that can be used to iterate over all chunks. """ + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_filter_chunks: starting...') first = True content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_filter_chunks: found content_length='+str(content_length)) + if flex_stream is not None: + chunks = flex_stream try: for chunk in chunks: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_filter_chunks: filtering chunk of size '+str(len(chunk))) if content_length: bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT) chunk_len = len(chunk) @@ -166,8 +174,11 @@ def bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, con yield None -def bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp): - return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, user_agent, content_encoding, content_type, csp) +def bubble_modify(flow, flex_stream, req_id, user_agent, content_encoding, content_type, csp): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_modify: modifying req_id='+req_id) + return lambda chunks: bubble_filter_chunks(flow, chunks, flex_stream, req_id, + user_agent, content_encoding, content_type, csp) def send_bubble_response(response): @@ -190,17 +201,16 @@ def abort_data(content_type): return EMPTY_DEFAULT -def bubble_filter_response(flow): - return responseheaders(flow) - - def responseheaders(flow): + bubble_filter_response(flow, None) + +def bubble_filter_response(flow, flex_stream): path = flow.request.path if path and path.startswith(BUBBLE_URI_PREFIX): if path.startswith(HEALTH_CHECK_URI): - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('responseheaders: special bubble health check request, responding with OK') + #if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('responseheaders: special bubble health check request, responding with OK') flow.response.headers = Headers() flow.response.headers[HEADER_HEALTH_CHECK] = 'OK' flow.response.headers[HEADER_CONTENT_LENGTH] = '3' @@ -319,7 +329,9 @@ def responseheaders(flow): content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) if bubble_log.isEnabledFor(DEBUG): bubble_log.debug(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) - flow.response.stream = bubble_modify(flow, req_id, user_agent, content_encoding, content_type, csp) + + flow.response.stream = bubble_modify(flow, flex_stream, req_id, + user_agent, content_encoding, content_type, csp) if content_length_value: flow.response.headers['transfer-encoding'] = 'chunked' # find server_conn to set fake_chunks on -- 2.17.1 From 55ab35e1091464568ba2116a8fde6d0c2caa41ab Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 23:21:01 -0400 Subject: [PATCH 50/78] flex routing works. still some cleanup to do --- .../src/main/java/bubble/ApiConstants.java | 15 ++ .../java/bubble/dao/account/AccountDAO.java | 2 +- .../resources/stream/FilterHttpResource.java | 17 ++- .../main/java/bubble/rule/AppRuleDriver.java | 7 + .../bubble/service/device/DeviceService.java | 2 +- .../service/device/StandardDeviceService.java | 29 ++-- .../stream/StandardAppPrimerService.java | 2 +- .../DbFilterDeviceService.java | 2 +- .../packer/roles/algo/tasks/main.yml | 2 +- .../roles/mitmproxy/files/bubble_api.py | 110 ++++++++++++++- .../roles/mitmproxy/files/bubble_flex.py | 19 +-- .../roles/mitmproxy/files/bubble_modify.py | 73 +++------- .../roles/mitmproxy/files/dns_spoofing.py | 132 ++++++++++-------- utils/cobbzilla-wizard | 2 +- 14 files changed, 260 insertions(+), 154 deletions(-) diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index b56f4c55..40bea4c7 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -12,6 +12,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomUtils; import org.cobbzilla.util.daemon.ZillaRuntime; +import org.cobbzilla.util.string.StringUtil; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -82,6 +83,20 @@ public class ApiConstants { () -> die("getKnownHostKey")); } + private static final AtomicReference privateIp = new AtomicReference<>(); + public static String getPrivateIp() { + return lazyGet(privateIp, + () -> configuredIps().stream() + .filter(addr -> addr.startsWith("10.")) + .findFirst() + .orElse(null), + () -> { + final String msg = "getPrivateIp: no system private IP found, configuredIps=" + StringUtil.toString(configuredIps()); + log.error(msg); + return die(msg); + }); + } + public static final Predicate ALWAYS_TRUE = m -> true; public static final String HOME_DIR; static { diff --git a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java index 68d8449d..7ccd77f7 100644 --- a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java +++ b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java @@ -196,7 +196,7 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc syncAccountService.syncAccount(account); } if (previousState.isRefreshShowBlockStats()) { - deviceService.initBlockStats(account); + deviceService.initBlocksAndFlexRoutes(account); ruleEngineService.flushCaches(); } } diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index e93fabbc..aac489bb 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -54,6 +54,7 @@ import java.util.stream.Collectors; import static bubble.ApiConstants.*; import static bubble.resources.stream.FilterMatchersResponse.NO_MATCHERS; +import static bubble.rule.AppRuleDriver.isFlexRouteFqdn; import static bubble.service.stream.HttpStreamDebug.getLogFqdn; import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIMEOUT; import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY; @@ -292,16 +293,20 @@ public class FilterHttpResource { } filterRequest.setDevice(device.getUuid()); - // if this is for a local ip, it's an automatic block - // legitimate local requests would have been passthru and never reached here + // if this is for a local ip, it's either a flex route or an automatic block + // legitimate local requests would have otherwise been "passthru" and never reached here final boolean isLocalIp = isForLocalIp(filterRequest); final boolean showStats = showStats(device.getAccount(), filterRequest.getClientAddr(), filterRequest.getFqdn()); if (isLocalIp) { - if (filterRequest.isBrowser() && showStats) { - blockStats.record(filterRequest, FilterMatchDecision.abort_not_found); + if (isFlexRouteFqdn(redis, vpnAddr, filterRequest.getFqdn())) { + if (log.isDebugEnabled()) log.debug(prefix + "detected flex route, not blocking"); + } else { + if (filterRequest.isBrowser() && showStats) { + blockStats.record(filterRequest, FilterMatchDecision.abort_not_found); + } + if (log.isDebugEnabled()) log.debug(prefix + "returning FORBIDDEN (showBlockStats==" + showStats + ")"); + return forbidden(); } - if (log.isDebugEnabled()) log.debug(prefix + "returning FORBIDDEN (showBlockStats=="+ showStats +")"); - return forbidden(); } final FilterMatchersResponse response = getMatchersResponse(filterRequest, req, request); diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 18071c8c..70be8f3b 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -79,6 +79,13 @@ public interface AppRuleDriver { log.debug("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count); } + static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) { + final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX; + final boolean found = redis.sismember_plaintext(key, fqdn); + if (log.isErrorEnabled()) log.error("isFlexRouteFqdn("+ip+", "+fqdn+") == "+found+ "(key="+key+")"); + return found; + } + AppRuleDriver getNext(); void setNext(AppRuleDriver next); default boolean hasNext() { return getNext() != null; } diff --git a/bubble-server/src/main/java/bubble/service/device/DeviceService.java b/bubble-server/src/main/java/bubble/service/device/DeviceService.java index b0704311..e8a98eae 100644 --- a/bubble-server/src/main/java/bubble/service/device/DeviceService.java +++ b/bubble-server/src/main/java/bubble/service/device/DeviceService.java @@ -19,7 +19,7 @@ public interface DeviceService { void initDeviceSecurityLevels(); void setDeviceSecurityLevel(Device device); - void initBlockStats (Account account); + void initBlocksAndFlexRoutes(Account account); default boolean doShowBlockStats(String accountUuid) { return false; } default Boolean doShowBlockStatsForIpAndFqdn(String ip, String fqdn) { return false; } default void setBlockStatsForFqdn (Account account, String fqdn, boolean value) {} diff --git a/bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java b/bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java index a07e1091..073e44a4 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardDeviceService.java @@ -19,8 +19,6 @@ import org.cobbzilla.util.collection.ExpirationMap; import org.cobbzilla.util.collection.SingletonList; import org.cobbzilla.util.io.FileUtil; import org.cobbzilla.util.io.FilenamePrefixFilter; -import org.cobbzilla.util.network.NetworkUtil; -import org.cobbzilla.util.string.StringUtil; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -31,9 +29,9 @@ import java.net.InetAddress; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import static bubble.ApiConstants.HOME_DIR; +import static bubble.ApiConstants.getPrivateIp; import static bubble.model.device.DeviceStatus.NO_DEVICE_STATUS; import static java.util.concurrent.TimeUnit.MINUTES; import static org.cobbzilla.util.daemon.ZillaRuntime.*; @@ -58,6 +56,9 @@ public class StandardDeviceService implements DeviceService { // used in dnscrypt-proxy to determine how to respond to blocked requests public static final String REDIS_KEY_DEVICE_REJECT_WITH = "bubble_device_reject_with_"; + // used in dnscrypt-proxy to determine how to respond to flex routed requests + public static final String REDIS_KEY_DEVICE_FLEX_WITH = "bubble_device_flex_with_"; + // used in mitmproxy to determine how to respond to blocked requests public static final String REDIS_KEY_DEVICE_SHOW_BLOCK_STATS = "bubble_device_showBlockStats_"; @@ -162,7 +163,7 @@ public class StandardDeviceService implements DeviceService { } } - @Override public void initBlockStats (Account account) { + @Override public void initBlocksAndFlexRoutes(Account account) { final boolean showBlockStats = configuration.showBlockStatsSupported() && account.showBlockStats(); redis.set_plaintext(REDIS_KEY_ACCOUNT_SHOW_BLOCK_STATS+account.getUuid(), Boolean.toString(showBlockStats)); redis.del_matching_withPrefix(REDIS_KEY_CHUNK_FILTER_PASS+"*"); @@ -172,6 +173,7 @@ public class StandardDeviceService implements DeviceService { } else { hideBlockStats(device); } + initFlexRoutes(device); } } @@ -193,15 +195,8 @@ public class StandardDeviceService implements DeviceService { } public void showBlockStats (Device device) { - final Set configuredIps = NetworkUtil.configuredIps(); - final String privateIp = configuredIps.stream() - .filter(addr -> addr.startsWith("10.")) - .findFirst() - .orElse(null); - if (privateIp == null) { - log.error("showBlockStats: no system private IP found, configuredIps="+StringUtil.toString(configuredIps)); - return; - } + final String privateIp = getPrivateIp(); + if (privateIp == null) return; for (String ip : findIpsByDevice(device.getUuid())) { redis.set_plaintext(REDIS_KEY_DEVICE_SHOW_BLOCK_STATS + ip, Boolean.toString(true)); redis.set_plaintext(REDIS_KEY_DEVICE_REJECT_WITH + ip, privateIp); @@ -215,6 +210,14 @@ public class StandardDeviceService implements DeviceService { } } + public void initFlexRoutes (Device device) { + final String privateIp = getPrivateIp(); + if (privateIp == null) return; + for (String ip : findIpsByDevice(device.getUuid())) { + redis.set_plaintext(REDIS_KEY_DEVICE_FLEX_WITH + ip, privateIp); + } + } + @Override public void setBlockStatsForFqdn(Account account, String fqdn, boolean value) { for (Device device : deviceDAO.findByAccount(account.getUuid())) { for (String ip : findIpsByDevice(device.getUuid())) { diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index a3f57102..862c0057 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -77,7 +77,7 @@ public class StandardAppPrimerService implements AppPrimerService { } public void prime(Account account) { - deviceService.initBlockStats(account); + deviceService.initBlocksAndFlexRoutes(account); prime(account, (BubbleApp) null); } diff --git a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java index 06cb4969..609d6c86 100644 --- a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java +++ b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterDeviceService.java @@ -24,7 +24,7 @@ public class DbFilterDeviceService implements DeviceService { @Override public void initDeviceSecurityLevels() { notSupported("initDeviceSecurityLevels"); } @Override public void setDeviceSecurityLevel(Device device) { notSupported("setDeviceSecurityLevel"); } - @Override public void initBlockStats(Account account) { notSupported("initBlockStats"); } + @Override public void initBlocksAndFlexRoutes(Account account) { notSupported("initBlocksAndFlexRoutes"); } @Override public DeviceStatus getDeviceStatus(String deviceUuid) { return notSupported("getDeviceStats"); } @Override public DeviceStatus getLiveDeviceStatus(String deviceUuid) { return notSupported("getLiveDeviceStatus"); } diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index 01a4ff4d..365d3917 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:44584be79375d94714f0e5c5772a76dee17eebb465f015685ff9df79f32fc809 + checksum: sha256:e73e04bb5aecf9b6b4543b0d5111d43c64a196bf42c6926770cc5678abb18a0c - name: Unzip algo master.zip unarchive: diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index a9513fe3..60117410 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -1,10 +1,13 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # -import json import logging from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL +from mitmproxy import http +from mitmproxy.net.http import headers as nheaders + +import json import re import requests import redis @@ -22,17 +25,20 @@ from bubble_config import bubble_network, bubble_port, debug_capture_fqdn, \ bubble_log = logging.getLogger(__name__) HEADER_USER_AGENT = 'User-Agent' +HEADER_CONTENT_LENGTH = 'Content-Length' HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy' HEADER_REFERER = 'Referer' HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru' CTX_BUBBLE_MATCHERS = 'X-Bubble-Matchers' CTX_BUBBLE_ABORT = 'X-Bubble-Abort' +CTX_BUBBLE_SPECIAL = 'X-Bubble-Special' CTX_BUBBLE_LOCATION = 'X-Bubble-Location' CTX_BUBBLE_PASSTHRU = 'X-Bubble-Passthru' CTX_BUBBLE_REQUEST_ID = 'X-Bubble-RequestId' CTX_CONTENT_LENGTH = 'X-Bubble-Content-Length' CTX_CONTENT_LENGTH_SENT = 'X-Bubble-Content-Length-Sent' +CTX_BUBBLE_FILTERED = 'X-Bubble-Filtered' BUBBLE_URI_PREFIX = '/__bubble/' HEADER_HEALTH_CHECK = 'X-Mitm-Health' @@ -235,6 +241,18 @@ def is_bubble_request(ip, fqdns): return ip in LOCAL_IPS and (bubble_host in fqdns or bubble_host_alias in fqdns) +def is_bubble_special_path(path): + return path and path.startswith(BUBBLE_URI_PREFIX) + + +def make_bubble_special_path(path): + return 'http://127.0.0.1:' + bubble_port + '/' + path[len(BUBBLE_URI_PREFIX):] + + +def is_bubble_health_check(path): + return path and path.startswith(HEALTH_CHECK_URI) + + def is_sage_request(ip, fqdns): return (ip == bubble_sage_ip4 or ip == bubble_sage_ip6) and bubble_sage_host in fqdns @@ -256,3 +274,93 @@ def is_flex_domain(client_addr, fqdn): return True check_fqdn = check_fqdn[check_fqdn.index('.')+1:] return False + + +def health_check_response(flow): + #if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('health_check_response: special bubble health check request, responding with OK') + response_headers = nheaders.Headers() + response_headers[HEADER_HEALTH_CHECK] = 'OK' + response_headers[HEADER_CONTENT_LENGTH] = '3' + if flow.response is None: + flow.response = http.HTTPResponse(http_version='HTTP/1.1', + status_code=200, + reason='OK', + headers=response_headers, + content=b'OK\n') + else: + flow.response.headers = nheaders.Headers() + flow.response.headers = response_headers + flow.response.status_code = 200 + flow.response.reason = 'OK' + flow.response.stream = lambda chunks: [b'OK\n'] + + +def special_bubble_response(flow): + path = flow.request.path + if is_bubble_health_check(path): + health_check_response(flow) + return + uri = make_bubble_special_path(path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('special_bubble_response: sending special bubble request to '+uri) + headers = { + 'Accept': 'application/json', + 'Content-Type': 'application/json' + } + response = None + if flow.request.method == 'GET': + response = requests.get(uri, headers=headers, stream=True) + elif flow.request.method == 'POST': + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('special_bubble_response: special bubble request: POST content is '+str(flow.request.content)) + headers['Content-Length'] = str(len(flow.request.content)) + response = requests.post(uri, data=flow.request.content, headers=headers, stream=True) + else: + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('special_bubble_response: special bubble request: method '+flow.request.method+' not supported') + if flow.response is None: + http_version = get_http_version(response) + if http_version is None: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('special_bubble_response: invalid HTTP version, bailing out') + return + + response_headers = collect_response_headers(response) + + flow.response = http.HTTPResponse(http_version=http_version, + status_code=response.status_code, + reason=response.reason, + headers=response_headers, + content=None) + if response is not None: + #if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('special_bubble_response: special bubble request: response status = '+str(response.status_code)) + flow.response.headers = collect_response_headers(response) + flow.response.status_code = response.status_code + flow.response.reason = status_reason(response.status_code) + flow.response.stream = lambda chunks: send_bubble_response(response) + + +def send_bubble_response(response): + for chunk in response.iter_content(8192): + yield chunk + + +def get_http_version(response): + raw_version = response.raw.version + if raw_version == 10: + return 'HTTP/1.0' + elif raw_version == 11: + return 'HTTP/1.1' + else: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('get_http_version: invalid HTTP version detected, response.raw.version=='+repr(raw_version)) + return None + + +def collect_response_headers(response): + response_headers = nheaders.Headers() + for name in response.headers: + response_headers[name] = response.headers[name] + return response_headers diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index f4028944..264baf4c 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -2,8 +2,7 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # from mitmproxy import http -from mitmproxy.net.http import headers as nheaders -from bubble_api import bubble_get_flex_router +from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, add_flow_ctx from bubble_modify import bubble_filter_response import requests @@ -58,8 +57,6 @@ def process_flex(flex_host, flow, router): # see: https://stackoverflow.com/questions/16789840/python-requests-cant-send-multiple-headers-with-same-key request_headers = {} for name in flow.request.headers: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: processing request header: '+repr(name)) if name in request_headers: request_headers[name] = request_headers[name] + "," + flow.request.headers[name] else: @@ -85,20 +82,14 @@ def process_flex(flex_host, flow, router): return # Status line -- http version is buried in response.raw.version - raw_version = response.raw.version - if raw_version == 10: - http_version = 'HTTP/1.0' - elif raw_version == 11: - http_version = 'HTTP/1.1' - else: + http_version = get_http_version(response) + if http_version is None: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: invalid HTTP version detected, response.raw.version=='+repr(raw_version)) + bubble_log.debug('process_flex: invalid HTTP version, bailing out') return # Headers -- copy from requests dict to Headers multimap - response_headers = nheaders.Headers() - for name in response.headers: - response_headers[name] = response.headers[name] + response_headers = collect_response_headers(response) # Construct the response -- use the raw response stream flow.response = http.HTTPResponse(http_version=http_version, diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index e2c22789..8bb8be82 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -10,14 +10,14 @@ import urllib import traceback from mitmproxy.net.http import Headers from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri -from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, BUBBLE_URI_PREFIX, \ - HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, status_reason, \ - CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, get_flow_ctx, add_flow_ctx, \ - HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header +from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ + HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, status_reason, get_flow_ctx, add_flow_ctx, \ + is_bubble_special_path, is_bubble_health_check, health_check_response, make_bubble_special_path, special_bubble_response, \ + CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \ + HEADER_CONTENT_LENGTH, HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header BUFFER_SIZE = 4096 HEADER_CONTENT_TYPE = 'Content-Type' -HEADER_CONTENT_LENGTH = 'Content-Length' HEADER_CONTENT_ENCODING = 'Content-Encoding' HEADER_TRANSFER_ENCODING = 'Transfer-Encoding' HEADER_LOCATION = 'Location' @@ -181,11 +181,6 @@ def bubble_modify(flow, flex_stream, req_id, user_agent, content_encoding, conte user_agent, content_encoding, content_type, csp) -def send_bubble_response(response): - for chunk in response.iter_content(8192): - yield chunk - - EMPTY_XML = [b''] EMPTY_JSON = [b'null'] EMPTY_DEFAULT = [] @@ -206,45 +201,17 @@ def responseheaders(flow): def bubble_filter_response(flow, flex_stream): + # only filter once -- flex routing may have pre-filtered + if get_flow_ctx(flow, CTX_BUBBLE_FILTERED): + return + add_flow_ctx(flow, CTX_BUBBLE_FILTERED, True) + path = flow.request.path - if path and path.startswith(BUBBLE_URI_PREFIX): - if path.startswith(HEALTH_CHECK_URI): - #if bubble_log.isEnabledFor(DEBUG): - # bubble_log.debug('responseheaders: special bubble health check request, responding with OK') - flow.response.headers = Headers() - flow.response.headers[HEADER_HEALTH_CHECK] = 'OK' - flow.response.headers[HEADER_CONTENT_LENGTH] = '3' - flow.response.status_code = 200 - flow.response.reason = 'OK' - flow.response.stream = lambda chunks: [b'OK\n'] + if is_bubble_special_path(path): + if is_bubble_health_check(path): + health_check_response(flow) else: - uri = 'http://127.0.0.1:' + bubble_port + '/' + path[len(BUBBLE_URI_PREFIX):] - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('responseheaders: sending special bubble request to '+uri) - headers = { - 'Accept' : 'application/json', - 'Content-Type': 'application/json' - } - response = None - if flow.request.method == 'GET': - response = requests.get(uri, headers=headers, stream=True) - elif flow.request.method == 'POST': - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('responseheaders: special bubble request: POST content is '+str(flow.request.content)) - headers['Content-Length'] = str(len(flow.request.content)) - response = requests.post(uri, data=flow.request.content, headers=headers) - else: - if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('responseheaders: special bubble request: method '+flow.request.method+' not supported') - if response is not None: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('responseheaders: special bubble request: response status = '+str(response.status_code)) - flow.response.headers = Headers() - for key, value in response.headers.items(): - flow.response.headers[key] = value - flow.response.status_code = response.status_code - flow.response.reason = status_reason(response.status_code) - flow.response.stream = lambda chunks: send_bubble_response(response) + special_bubble_response(flow) else: abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT) @@ -252,7 +219,7 @@ def bubble_filter_response(flow, flex_stream): abort_location = get_flow_ctx(flow, CTX_BUBBLE_LOCATION) if abort_location is not None: if bubble_log.isEnabledFor(INFO): - bubble_log.info('responseheaders: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path) + bubble_log.info('bubble_filter_response: redirecting request with HTTP status '+str(abort_code)+' to: '+abort_location+', path was: '+path) flow.response.headers = Headers() flow.response.headers[HEADER_LOCATION] = abort_location flow.response.status_code = abort_code @@ -264,7 +231,7 @@ def bubble_filter_response(flow, flex_stream): else: content_type = None if bubble_log.isEnabledFor(INFO): - bubble_log.info('responseheaders: aborting request with HTTP status '+str(abort_code)+', path was: '+path) + bubble_log.info('bubble_filter_response: aborting request with HTTP status '+str(abort_code)+', path was: '+path) flow.response.headers = Headers() flow.response.status_code = abort_code flow.response.reason = status_reason(abort_code) @@ -272,23 +239,23 @@ def bubble_filter_response(flow, flex_stream): elif flow.response.status_code // 100 != 2: if bubble_log.isEnabledFor(INFO): - bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path) + bubble_log.info('bubble_filter_response: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path) pass elif flow.response.headers is None or len(flow.response.headers) == 0: if bubble_log.isEnabledFor(INFO): - bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path) + bubble_log.info('bubble_filter_response: response had HTTP status '+str(flow.response.status_code)+', and NO response headers, returning as-is: '+path) pass elif HEADER_CONTENT_LENGTH in flow.response.headers and flow.response.headers[HEADER_CONTENT_LENGTH] == "0": if bubble_log.isEnabledFor(INFO): - bubble_log.info('responseheaders: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path) + bubble_log.info('bubble_filter_response: response had HTTP status '+str(flow.response.status_code)+', and '+HEADER_CONTENT_LENGTH+' was zero, returning as-is: '+path) pass else: req_id = get_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID) matchers = get_flow_ctx(flow, CTX_BUBBLE_MATCHERS) - prefix = 'responseheaders(req_id='+str(req_id)+'): ' + prefix = 'bubble_filter_response(req_id='+str(req_id)+'): ' if req_id is not None and matchers is not None: if bubble_log.isEnabledFor(DEBUG): bubble_log.debug(prefix+' matchers: '+repr(matchers)) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py index 2706308e..30dc83a2 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py @@ -32,9 +32,11 @@ import time import uuid from mitmproxy.net.http import headers as nheaders -from bubble_api import bubble_matchers, bubble_activity_log, HEALTH_CHECK_URI, \ - CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, \ - add_flow_ctx, parse_host_header, is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain +from bubble_api import bubble_matchers, bubble_activity_log, \ + HEALTH_CHECK_URI, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ + CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, get_flow_ctx, parse_host_header, \ + is_bubble_special_path, special_bubble_response, is_bubble_health_check, \ + is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain from bubble_config import bubble_host, bubble_host_alias from bubble_flex import set_flex_response @@ -47,11 +49,11 @@ class Rerouter: if host is None: return None - is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI) - if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX): + is_health_check = is_bubble_health_check(flow.request.path) + if is_bubble_special_path(flow.request.path): if not is_health_check: - if bubble_log.isEnabledFor(INFO): - bubble_log.info("get_matchers: not filtering special bubble path: "+flow.request.path) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug("get_matchers: not filtering special bubble path: "+flow.request.path) return None client_addr = str(flow.client_conn.address[0]) @@ -138,91 +140,95 @@ class Rerouter: # Determine if this request should be filtered is_health_check = False host = None + path = flow.request.path if sni or host_header: host = str(sni or host_header) if host.startswith("b'"): host = host[2:-1] - log_url = flow.request.scheme + '://' + host + flow.request.path + log_url = flow.request.scheme + '://' + host + path # If https, we have already checked that the client/server are legal in bubble_conn_check.py # If http, we validate client/server here if is_http: fqdns = [host] if is_bubble_request(server_addr, fqdns): - is_health_check = flow.request.path.startswith(HEALTH_CHECK_URI) + is_health_check = path.startswith(HEALTH_CHECK_URI) if not is_health_check: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: redirecting to https for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) for client='+client_addr+', fqdns='+repr(fqdns)+', path='+flow.request.path) + bubble_log.debug('request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) - add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path) + add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path) return None elif is_sage_request(server_addr, fqdns): if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) - add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://'+host+flow.request.path) + add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path) return None elif is_not_from_vpn(client_addr): # todo: add to fail2ban if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', fqdns='+str(fqdns)) + bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', url='+log_url) bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) return None - matcher_response = self.get_matchers(flow, sni or host_header) - if matcher_response: - has_decision = 'decision' in matcher_response and matcher_response['decision'] is not None - if has_decision and matcher_response['decision'] == 'pass_thru': - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: passthru response returned, passing thru and NOT performing TLS interception...') - add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) - bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url) - return host + if is_bubble_special_path(path): + add_flow_ctx(flow, CTX_BUBBLE_SPECIAL, True) + else: + matcher_response = self.get_matchers(flow, sni or host_header) + if matcher_response: + has_decision = 'decision' in matcher_response and matcher_response['decision'] is not None + if has_decision and matcher_response['decision'] == 'pass_thru': + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: passthru response returned, passing thru...') + add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) + bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url) + return host - elif has_decision and matcher_response['decision'].startswith('abort_'): - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: found abort code: ' + str(matcher_response['decision']) + ', aborting') - if matcher_response['decision'] == 'abort_ok': - abort_code = 200 - elif matcher_response['decision'] == 'abort_not_found': - abort_code = 404 - else: + elif has_decision and matcher_response['decision'].startswith('abort_'): if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') - abort_code = 404 - flow.request.headers = nheaders.Headers([]) - flow.request.content = b'' - add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code) - bubble_activity_log(client_addr, server_addr, 'http_abort' + str(abort_code), log_url) - return None + bubble_log.debug('request: found abort code: ' + str(matcher_response['decision']) + ', aborting') + if matcher_response['decision'] == 'abort_ok': + abort_code = 200 + elif matcher_response['decision'] == 'abort_not_found': + abort_code = 404 + else: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') + abort_code = 404 + flow.request.headers = nheaders.Headers([]) + flow.request.content = b'' + add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code) + bubble_activity_log(client_addr, server_addr, 'http_abort' + str(abort_code), log_url) + return None - elif has_decision and matcher_response['decision'] == 'no_match': - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: decision was no_match, passing thru...') - bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url) - return host + elif has_decision and matcher_response['decision'] == 'no_match': + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: decision was no_match, passing thru...') + bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url) + return host - elif ('matchers' in matcher_response - and 'request_id' in matcher_response - and len(matcher_response['matchers']) > 0): - req_id = matcher_response['request_id'] - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug("request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) - add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) - add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) - bubble_activity_log(client_addr, server_addr, 'http_match', log_url) + elif ('matchers' in matcher_response + and 'request_id' in matcher_response + and len(matcher_response['matchers']) > 0): + req_id = matcher_response['request_id'] + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug("request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) + add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) + add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) + bubble_activity_log(client_addr, server_addr, 'http_match', log_url) + else: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: no rules returned, passing thru...') + bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url) else: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: no rules returned, passing thru...') - bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url) - else: - if not is_health_check: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: no matcher_response returned, passing thru...') - # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) + if not is_health_check: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: no matcher_response returned, passing thru...') + # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) elif is_http and is_not_from_vpn(client_addr): # todo: add to fail2ban @@ -234,7 +240,7 @@ class Rerouter: else: if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('request: no sni/host found, not applying rules to path: ' + flow.request.path) + bubble_log.warning('request: no sni/host found, not applying rules to path: ' + path) bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr]) flow.request.host_header = host_header @@ -247,7 +253,11 @@ class Rerouter: def request(self, flow): flex_host = self.bubble_handle_request(flow) - if flex_host is not None: + path = flow.request.path + if is_bubble_special_path(path): + special_bubble_response(flow) + + elif flex_host is not None: client_addr = flow.client_conn.address[0] if is_flex_domain(client_addr, flex_host): set_flex_response(client_addr, flex_host, flow) diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 47b03e4b..87283443 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 47b03e4b00490119357d8972ce7ed727266b6c35 +Subproject commit 872834434be7e62553ff3d42623ccfa3d82d07b4 -- 2.17.1 From 97e590695a22a435f8445e5ead691a790ea0c560 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 8 Sep 2020 23:28:44 -0400 Subject: [PATCH 51/78] rename dns_spoofing to bubble_request --- .../mitmproxy/files/{dns_spoofing.py => bubble_request.py} | 0 .../src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh | 2 +- .../src/main/resources/packer/roles/mitmproxy/tasks/main.yml | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename bubble-server/src/main/resources/packer/roles/mitmproxy/files/{dns_spoofing.py => bubble_request.py} (100%) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py similarity index 100% rename from bubble-server/src/main/resources/packer/roles/mitmproxy/files/dns_spoofing.py rename to bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh index 6ca204e3..6ad5747a 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh @@ -26,7 +26,7 @@ BUBBLE_PORT=${PORT} mitmdump \ --set stream_large_bodies=5m \ --set keep_host_header \ -s ./bubble_debug.py \ - -s ./dns_spoofing.py \ -s ./bubble_conn_check.py \ + -s ./bubble_request.py \ -s ./bubble_modify.py \ --mode transparent diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml index b5620538..a218b8bc 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml @@ -59,7 +59,7 @@ with_items: - bubble_api.py - bubble_debug.py - - dns_spoofing.py + - bubble_request.py - bubble_conn_check.py - bubble_modify.py - run_mitm.sh -- 2.17.1 From 309485ab361b12474737f12f07e152794373bb86 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 00:01:30 -0400 Subject: [PATCH 52/78] do not follow redirects, fix content/stream --- .../resources/packer/roles/mitmproxy/files/bubble_flex.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index 264baf4c..be809abf 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -75,7 +75,8 @@ def process_flex(flex_host, flow, router): timeout=(15, 15), stream=True, data=flow.request.stream, # use the original request body, if there is one - proxies=proxies) + proxies=proxies, + allow_redirects=False) except Exception as e: if bubble_log.isEnabledFor(ERROR): bubble_log.error('process_flex: error sending request to '+url+': '+repr(e)) @@ -95,7 +96,10 @@ def process_flex(flex_host, flow, router): flow.response = http.HTTPResponse(http_version=http_version, status_code=response.status_code, reason=response.reason, - headers=response_headers) + headers=response_headers, + content=response.raw) + flow.response.raw_content = None + flow.response.stream = True # Apply filters bubble_filter_response(flow, response.raw) -- 2.17.1 From ee3ff4889502de0ab49fc491283820a8d474534a Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 00:01:44 -0400 Subject: [PATCH 53/78] check parent domains for flex routing --- .../src/main/java/bubble/rule/AppRuleDriver.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 70be8f3b..5b84d086 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -81,9 +81,14 @@ public interface AppRuleDriver { static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) { final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX; - final boolean found = redis.sismember_plaintext(key, fqdn); - if (log.isErrorEnabled()) log.error("isFlexRouteFqdn("+ip+", "+fqdn+") == "+found+ "(key="+key+")"); - return found; + String check = fqdn; + while (true) { + final boolean found = redis.sismember_plaintext(key, check); + if (found) return true; + final int dotPos = check.indexOf('.'); + check = check.substring(dotPos+1); + if (!check.contains(".")) return false; + } } AppRuleDriver getNext(); -- 2.17.1 From 73c78ed60cec6c4fda191d9cbbfc6ea1c1732c65 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 02:54:16 -0400 Subject: [PATCH 54/78] flex routing works --- .../main/java/bubble/rule/AppRuleDriver.java | 1 + .../bubbleApp_bubbleBlock_data.json | 6 ++++ .../roles/mitmproxy/files/bubble_api.py | 16 +++++++-- .../roles/mitmproxy/files/bubble_flex.py | 34 +++++++++++++++---- .../roles/mitmproxy/files/bubble_modify.py | 10 +++--- .../roles/mitmproxy/files/bubble_request.py | 28 ++++++++------- 6 files changed, 68 insertions(+), 27 deletions(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 5b84d086..81cc3a59 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -86,6 +86,7 @@ public interface AppRuleDriver { final boolean found = redis.sismember_plaintext(key, check); if (found) return true; final int dotPos = check.indexOf('.'); + if (dotPos == check.length()) return false; check = check.substring(dotPos+1); if (!check.contains(".")) return false; } diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json index ad7ebcc7..e0d36a41 100644 --- a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json +++ b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json @@ -2,6 +2,12 @@ "name": "BubbleBlock", "children": { "AppData": [{ + "site": "All_Sites", + "template": true, + "matcher": "BubbleBlockMatcher", + "key": "hideStats_abercrombie.com", + "data": "true" + }, { "site": "All_Sites", "template": true, "matcher": "BubbleBlockMatcher", diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 60117410..065a0f6a 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -26,6 +26,10 @@ bubble_log = logging.getLogger(__name__) HEADER_USER_AGENT = 'User-Agent' HEADER_CONTENT_LENGTH = 'Content-Length' +HEADER_CONTENT_TYPE = 'Content-Type' +HEADER_CONTENT_ENCODING = 'Content-Encoding' +HEADER_TRANSFER_ENCODING = 'Transfer-Encoding' +HEADER_LOCATION = 'Location' HEADER_CONTENT_SECURITY_POLICY = 'Content-Security-Policy' HEADER_REFERER = 'Referer' HEADER_FILTER_PASSTHRU = 'X-Bubble-Passthru' @@ -264,15 +268,19 @@ def is_not_from_vpn(client_addr): def is_flex_domain(client_addr, fqdn): if fqdn == bubble_host or fqdn == bubble_host_alias or fqdn == bubble_sage_host: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('is_flex_domain: (early) returning False for: '+fqdn) return False check_fqdn = fqdn while '.' in check_fqdn: - found = REDIS.sismember('flexLists~'+client_addr+'~UNION', fqdn) + found = REDIS.sismember('flexLists~'+client_addr+'~UNION', check_fqdn) if found: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('is_flex_domain: returning True for: '+fqdn) + bubble_log.debug('is_flex_domain: returning True for: '+fqdn+' (check='+check_fqdn+')') return True check_fqdn = check_fqdn[check_fqdn.index('.')+1:] + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('is_flex_domain: (early) returning False for: '+fqdn) return False @@ -359,8 +367,10 @@ def get_http_version(response): return None -def collect_response_headers(response): +def collect_response_headers(response, omit=None): response_headers = nheaders.Headers() for name in response.headers: + if omit and name in omit: + continue response_headers[name] = response.headers[name] return response_headers diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index be809abf..ff8883dc 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -2,7 +2,8 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # from mitmproxy import http -from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, add_flow_ctx +from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, add_flow_ctx, \ + HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH from bubble_modify import bubble_filter_response import requests @@ -77,6 +78,8 @@ def process_flex(flex_host, flow, router): data=flow.request.stream, # use the original request body, if there is one proxies=proxies, allow_redirects=False) + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_flex: response returned HTTP status '+str(response.status_code)+' for '+url) except Exception as e: if bubble_log.isEnabledFor(ERROR): bubble_log.error('process_flex: error sending request to '+url+': '+repr(e)) @@ -90,19 +93,38 @@ def process_flex(flex_host, flow, router): return # Headers -- copy from requests dict to Headers multimap - response_headers = collect_response_headers(response) + # Remove Content-Length, Content-Encoding and Transfer-Encoding + # We will rechunk the output + response_headers = collect_response_headers(response, [HEADER_CONTENT_LENGTH, HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING]) + response_headers[HEADER_TRANSFER_ENCODING] = 'chunked' - # Construct the response -- use the raw response stream + # Construct the response flow.response = http.HTTPResponse(http_version=http_version, status_code=response.status_code, reason=response.reason, headers=response_headers, - content=response.raw) - flow.response.raw_content = None - flow.response.stream = True + content=None) + + # If Content-Length header did not exist, or did exist and was > 0, then chunk the content + content_length = None + if HEADER_CONTENT_LENGTH in response.headers: + content_length = response.headers[HEADER_CONTENT_LENGTH] + if content_length is None or int(content_length) > 0: + flow.response.stream = lambda chunk: make_chunk(response.iter_content(chunk_size=8192)) + + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_flex: before setting stream, flow.response.stream='+repr(flow.response.stream)) # Apply filters + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_flex: bubble filtering: '+url) bubble_filter_response(flow, response.raw) if bubble_log.isEnabledFor(INFO): bubble_log.info('process_flex: successfully requested url '+url+' from flex router, proceeding ...') + + +def make_chunk(iter_content): + for chunk in iter_content: + yield chunk + yield None diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 8bb8be82..72126efc 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -14,13 +14,10 @@ from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATIO HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, status_reason, get_flow_ctx, add_flow_ctx, \ is_bubble_special_path, is_bubble_health_check, health_check_response, make_bubble_special_path, special_bubble_response, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \ - HEADER_CONTENT_LENGTH, HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header + HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \ + HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header BUFFER_SIZE = 4096 -HEADER_CONTENT_TYPE = 'Content-Type' -HEADER_CONTENT_ENCODING = 'Content-Encoding' -HEADER_TRANSFER_ENCODING = 'Transfer-Encoding' -HEADER_LOCATION = 'Location' CONTENT_TYPE_BINARY = 'application/octet-stream' STANDARD_FILTER_HEADERS = {HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY} @@ -207,6 +204,7 @@ def bubble_filter_response(flow, flex_stream): add_flow_ctx(flow, CTX_BUBBLE_FILTERED, True) path = flow.request.path + client_addr = flow.client_conn.address[0] if is_bubble_special_path(path): if is_bubble_health_check(path): health_check_response(flow) @@ -231,7 +229,7 @@ def bubble_filter_response(flow, flex_stream): else: content_type = None if bubble_log.isEnabledFor(INFO): - bubble_log.info('bubble_filter_response: aborting request with HTTP status '+str(abort_code)+', path was: '+path) + bubble_log.info('bubble_filter_response: aborting request from '+client_addr+' with HTTP status '+str(abort_code)+', path was: '+path) flow.response.headers = Headers() flow.response.status_code = abort_code flow.response.reason = status_reason(abort_code) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 30dc83a2..4abb6f77 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -155,14 +155,14 @@ class Rerouter: is_health_check = path.startswith(HEALTH_CHECK_URI) if not is_health_check: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path) + bubble_log.debug('bubble_handle_request: redirecting to https for LOCAL bubble=' + server_addr +' (bubble_host (' + bubble_host +') in fqdns or bubble_host_alias (' + bubble_host_alias +') in fqdns) for client=' + client_addr +', fqdns=' + repr(fqdns) +', path=' + path) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path) return None elif is_sage_request(server_addr, fqdns): if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr) + bubble_log.debug('bubble_handle_request: redirecting to https for SAGE server='+server_addr+' for client='+client_addr) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 301) add_flow_ctx(flow, CTX_BUBBLE_LOCATION, 'https://' + host + path) return None @@ -170,7 +170,7 @@ class Rerouter: elif is_not_from_vpn(client_addr): # todo: add to fail2ban if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', url='+log_url) + bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url) bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) return None @@ -183,21 +183,21 @@ class Rerouter: has_decision = 'decision' in matcher_response and matcher_response['decision'] is not None if has_decision and matcher_response['decision'] == 'pass_thru': if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: passthru response returned, passing thru...') + bubble_log.debug('bubble_handle_request: passthru response returned, passing thru...') add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) bubble_activity_log(client_addr, server_addr, 'http_passthru', log_url) return host elif has_decision and matcher_response['decision'].startswith('abort_'): if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: found abort code: ' + str(matcher_response['decision']) + ', aborting') + bubble_log.debug('bubble_handle_request: found abort code: ' + str(matcher_response['decision']) + ', aborting') if matcher_response['decision'] == 'abort_ok': abort_code = 200 elif matcher_response['decision'] == 'abort_not_found': abort_code = 404 else: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') + bubble_log.debug('bubble_handle_request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') abort_code = 404 flow.request.headers = nheaders.Headers([]) flow.request.content = b'' @@ -207,7 +207,7 @@ class Rerouter: elif has_decision and matcher_response['decision'] == 'no_match': if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: decision was no_match, passing thru...') + bubble_log.debug('bubble_handle_request: decision was no_match, passing thru...') bubble_activity_log(client_addr, server_addr, 'http_no_match', log_url) return host @@ -216,31 +216,31 @@ class Rerouter: and len(matcher_response['matchers']) > 0): req_id = matcher_response['request_id'] if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug("request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) + bubble_log.debug("bubble_handle_request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) bubble_activity_log(client_addr, server_addr, 'http_match', log_url) else: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: no rules returned, passing thru...') + bubble_log.debug('bubble_handle_request: no rules returned, passing thru...') bubble_activity_log(client_addr, server_addr, 'http_no_rules', log_url) else: if not is_health_check: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: no matcher_response returned, passing thru...') + bubble_log.debug('bubble_handle_request: no matcher_response returned, passing thru...') # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) elif is_http and is_not_from_vpn(client_addr): # todo: add to fail2ban if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr) + bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr) bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr]) add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) return None else: if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('request: no sni/host found, not applying rules to path: ' + path) + bubble_log.warning('bubble_handle_request: no sni/host found, not applying rules to path: ' + path) bubble_activity_log(client_addr, server_addr, 'http_no_sni_or_host', [server_addr]) flow.request.host_header = host_header @@ -255,11 +255,15 @@ class Rerouter: flex_host = self.bubble_handle_request(flow) path = flow.request.path if is_bubble_special_path(path): + #if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('request: is_bubble_special_path('+path+') returned true, sending special bubble response') special_bubble_response(flow) elif flex_host is not None: client_addr = flow.client_conn.address[0] if is_flex_domain(client_addr, flex_host): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('request: is_flex_domain('+flex_host+') returned true, sending flex response') set_flex_response(client_addr, flex_host, flow) -- 2.17.1 From cd6191c952341b2fe0bb8ef731a9d14ad9cb7134 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 04:13:20 -0400 Subject: [PATCH 55/78] WIP. make flex routing faster. POSTs still don't work. --- .../roles/mitmproxy/files/bubble_flex.py | 23 ++++++++----------- .../roles/mitmproxy/files/bubble_modify.py | 22 ++++++++++++++---- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index ff8883dc..f574d2f1 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -96,7 +96,6 @@ def process_flex(flex_host, flow, router): # Remove Content-Length, Content-Encoding and Transfer-Encoding # We will rechunk the output response_headers = collect_response_headers(response, [HEADER_CONTENT_LENGTH, HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING]) - response_headers[HEADER_TRANSFER_ENCODING] = 'chunked' # Construct the response flow.response = http.HTTPResponse(http_version=http_version, @@ -109,22 +108,20 @@ def process_flex(flex_host, flow, router): content_length = None if HEADER_CONTENT_LENGTH in response.headers: content_length = response.headers[HEADER_CONTENT_LENGTH] - if content_length is None or int(content_length) > 0: - flow.response.stream = lambda chunk: make_chunk(response.iter_content(chunk_size=8192)) - - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: before setting stream, flow.response.stream='+repr(flow.response.stream)) + if response.status_code // 100 != 2: + response_headers[HEADER_CONTENT_LENGTH] = '0' + flow.response.stream = lambda chunks: [] + elif content_length is None or int(content_length) > 0: + response_headers[HEADER_TRANSFER_ENCODING] = 'chunked' + flow.response.stream = lambda chunks: response.iter_content(8192) + else: + response_headers[HEADER_CONTENT_LENGTH] = '0' + flow.response.stream = lambda chunks: [] # Apply filters if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('process_flex: bubble filtering: '+url) - bubble_filter_response(flow, response.raw) + bubble_filter_response(flow, response) if bubble_log.isEnabledFor(INFO): bubble_log.info('process_flex: successfully requested url '+url+' from flex router, proceeding ...') - - -def make_chunk(iter_content): - for chunk in iter_content: - yield chunk - yield None diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 72126efc..7f851b67 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -25,6 +25,7 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__' REDIS_FILTER_PASSTHRU_DURATION = 600 DEBUG_STREAM_COUNTERS = {} +MIN_FILTER_CHUNK_SIZE = 4096 bubble_log = logging.getLogger(__name__) @@ -139,18 +140,23 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('bubble_filter_chunks: starting...') first = True + last = False content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('bubble_filter_chunks: found content_length='+str(content_length)) if flex_stream is not None: - chunks = flex_stream + chunks = flex_stream.iter_content(8192) try: + buffer = b'' for chunk in chunks: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_filter_chunks: filtering chunk of size '+str(len(chunk))) + buffer = buffer + chunk + if len(buffer) < MIN_FILTER_CHUNK_SIZE: + continue + chunk_len = len(buffer) + chunk = buffer + buffer = b'' if content_length: bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT) - chunk_len = len(chunk) last = chunk_len + bytes_sent >= content_length if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent)) @@ -162,7 +168,12 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ first = False else: yield filter_chunk(flow, chunk, req_id, user_agent, last) - if not content_length: + # send whatever is left in the buffer + if len(buffer) > 0: + # bubble_log.debug('bubble_filter_chunks(end): sending remainder buffer of size '+str(len(buffer))) + yield filter_chunk(flow, buffer, req_id, user_agent, last) + if not content_length or not last: + # bubble_log.debug('bubble_filter_chunks(end): sending last empty chunk') yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data except Exception as e: if bubble_log.isEnabledFor(ERROR): @@ -238,6 +249,7 @@ def bubble_filter_response(flow, flex_stream): elif flow.response.status_code // 100 != 2: if bubble_log.isEnabledFor(INFO): bubble_log.info('bubble_filter_response: response had HTTP status '+str(flow.response.status_code)+', returning as-is: '+path) + flow.response.headers[HEADER_CONTENT_LENGTH] = '0' pass elif flow.response.headers is None or len(flow.response.headers) == 0: -- 2.17.1 From 5907ccd97a440c4bc09770564abe5122d7e6440d Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 11:11:45 -0400 Subject: [PATCH 56/78] lower log level --- .../java/bubble/service/device/StandardFlexRouterService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 113dd445..fbcdb539 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -194,7 +194,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute for (int i=0; i Date: Wed, 9 Sep 2020 11:14:52 -0400 Subject: [PATCH 57/78] fix support for flex requests with request body --- .../packer/roles/mitmproxy/files/bubble_flex.py | 14 +++++++++----- .../packer/roles/mitmproxy/files/bubble_modify.py | 11 +++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index f574d2f1..866bd0e5 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -2,8 +2,9 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # from mitmproxy import http -from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, add_flow_ctx, \ - HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH +from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, \ + get_flow_ctx, add_flow_ctx, \ + HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_FILTERED from bubble_modify import bubble_filter_response import requests @@ -51,6 +52,7 @@ def set_flex_response(client_addr, flex_host, flow): def process_flex(flex_host, flow, router): # build the request URL + method = flow.request.method scheme = flow.request.scheme url = scheme + '://' + flex_host + flow.request.path @@ -68,14 +70,15 @@ def process_flex(flex_host, flow, router): proxies = {"http": proxy_url, "https": proxy_url} # send request to flex router + request_body = flow.request.content if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: sending flex request for '+url+' to '+proxy_url+' with headers='+repr(request_headers)) + bubble_log.debug('process_flex: sending flex request for ' + method +' ' + url +' to ' + proxy_url +' with headers=' + repr(request_headers) +' and body=' + repr(request_body)) try: - response = requests.request(flow.request.method, url, + response = requests.request(method, url, headers=request_headers, timeout=(15, 15), stream=True, - data=flow.request.stream, # use the original request body, if there is one + data=request_body, # use the original request body, if there is one proxies=proxies, allow_redirects=False) if bubble_log.isEnabledFor(DEBUG): @@ -121,6 +124,7 @@ def process_flex(flex_host, flow, router): # Apply filters if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('process_flex: bubble filtering: '+url) + # flow.response.stream = lambda chunks: response.iter_content(8192) bubble_filter_response(flow, response) if bubble_log.isEnabledFor(INFO): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 7f851b67..eadbcd6e 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -25,7 +25,7 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__' REDIS_FILTER_PASSTHRU_DURATION = 600 DEBUG_STREAM_COUNTERS = {} -MIN_FILTER_CHUNK_SIZE = 4096 +MIN_FILTER_CHUNK_SIZE = 16384 bubble_log = logging.getLogger(__name__) @@ -138,7 +138,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ chunks is a generator that can be used to iterate over all chunks. """ if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_filter_chunks: starting...') + bubble_log.debug('bubble_filter_chunks: starting with content_type='+content_type) first = True last = False content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) @@ -171,7 +171,10 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ # send whatever is left in the buffer if len(buffer) > 0: # bubble_log.debug('bubble_filter_chunks(end): sending remainder buffer of size '+str(len(buffer))) - yield filter_chunk(flow, buffer, req_id, user_agent, last) + if first: + yield filter_chunk(flow, buffer, req_id, user_agent, last, content_encoding, content_type, content_length, csp) + else: + yield filter_chunk(flow, buffer, req_id, user_agent, last) if not content_length or not last: # bubble_log.debug('bubble_filter_chunks(end): sending last empty chunk') yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data @@ -184,7 +187,7 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ def bubble_modify(flow, flex_stream, req_id, user_agent, content_encoding, content_type, csp): if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_modify: modifying req_id='+req_id) + bubble_log.debug('bubble_modify: modifying req_id='+req_id+' with content_type='+content_type) return lambda chunks: bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_encoding, content_type, csp) -- 2.17.1 From 0fc1f840afcbbf1161f5171a00cbc5ef9d9d64d8 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 14:05:57 -0400 Subject: [PATCH 58/78] check for flex domain during checkConnection, fix whitelist support --- .../resources/stream/FilterHttpResource.java | 21 ++++++++++++------- .../main/java/bubble/rule/AppRuleDriver.java | 13 ++++++++++++ .../rule/bblock/BubbleBlockRuleDriver.java | 6 ++++++ .../device/StandardFlexRouterService.java | 1 - .../stream/StandardAppPrimerService.java | 10 +++++++++ .../packer/roles/algo/tasks/main.yml | 2 +- utils/abp-parser | 2 +- 7 files changed, 44 insertions(+), 11 deletions(-) diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index aac489bb..af7ce528 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -183,16 +183,21 @@ public class FilterHttpResource { return notFound(); } + // if this is for a local ip, it's either a flex route or an automatic block + // legitimate local requests would have otherwise never reached here if (isLocalIp) { - final boolean showStats = showStats(accountUuid, connCheckRequest.getServerAddr(), connCheckRequest.getFqdns()); - final DeviceSecurityLevel secLevel = device.getSecurityLevel(); - if (showStats && secLevel.supportsRequestModification()) { - // allow it for now - if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showStats=true, secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr()); - return ok(ConnectionCheckResponse.noop); + if (isFlexRouteFqdn(redis, vpnAddr, connCheckRequest.getFqdns())) { + if (log.isDebugEnabled()) log.debug(prefix + "detected flex route, allowing processing to continue"); } else { - if (log.isDebugEnabled()) log.debug(prefix + "returning block (showStats="+showStats+", secLevel="+secLevel+") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr()); - return ok(ConnectionCheckResponse.block); + final boolean showStats = showStats(accountUuid, connCheckRequest.getServerAddr(), connCheckRequest.getFqdns()); + final DeviceSecurityLevel secLevel = device.getSecurityLevel(); + if (showStats && secLevel.supportsRequestModification()) { + // allow it for now + if (log.isDebugEnabled()) log.debug(prefix + "returning noop (showStats=true, secLevel=" + secLevel + ") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr()); + } else { + if (log.isDebugEnabled()) log.debug(prefix + "returning block (showStats=" + showStats + ", secLevel=" + secLevel + ") for LOCAL fqdn/addr=" + arrayToString(connCheckRequest.getFqdns()) + "/" + connCheckRequest.getServerAddr()); + return ok(ConnectionCheckResponse.block); + } } } diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 81cc3a59..99f0bedc 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -42,12 +42,14 @@ public interface AppRuleDriver { // also used in dnscrypt-proxy/plugin_reverse_resolve_cache.go String REDIS_REJECT_LISTS = "rejectLists"; String REDIS_BLOCK_LISTS = "blockLists"; + String REDIS_WHITE_LISTS = "whiteLists"; String REDIS_FILTER_LISTS = "filterLists"; String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy for flex routing String REDIS_LIST_SUFFIX = "~UNION"; default Set getPrimedRejectDomains () { return null; } default Set getPrimedBlockDomains () { return null; } + default Set getPrimedWhiteListDomains() { return null; } default Set getPrimedFilterDomains () { return null; } default Set getPrimedFlexDomains () { return null; } @@ -59,6 +61,10 @@ public interface AppRuleDriver { defineRedisSet(redis, ip, REDIS_BLOCK_LISTS, list, fullyBlockedDomains); } + static void defineRedisWhiteListSet(RedisService redis, String ip, String list, String[] fullyBlockedDomains) { + defineRedisSet(redis, ip, REDIS_WHITE_LISTS, list, fullyBlockedDomains); + } + static void defineRedisFilterSet(RedisService redis, String ip, String list, String[] filterDomains) { defineRedisSet(redis, ip, REDIS_FILTER_LISTS, list, filterDomains); } @@ -79,6 +85,13 @@ public interface AppRuleDriver { log.debug("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count); } + static boolean isFlexRouteFqdn(RedisService redis, String ip, String[] fqdns) { + for (String fqdn : fqdns) { + if (isFlexRouteFqdn(redis, ip, fqdn)) return true; + } + return false; + } + static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) { final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX; String check = fqdn; diff --git a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java index 9b340b7b..7cf6fb5b 100644 --- a/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/bblock/BubbleBlockRuleDriver.java @@ -58,6 +58,9 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver private final AtomicReference> fullyBlockedDomains = new AtomicReference<>(Collections.emptySet()); @Override public Set getPrimedBlockDomains() { return fullyBlockedDomains.get(); } + private final AtomicReference> whiteListDomains = new AtomicReference<>(Collections.emptySet()); + @Override public Set getPrimedWhiteListDomains() { return whiteListDomains.get(); } + private final AtomicReference> rejectDomains = new AtomicReference<>(Collections.emptySet()); @Override public Set getPrimedRejectDomains() { return rejectDomains.get(); } @@ -165,6 +168,9 @@ public class BubbleBlockRuleDriver extends TrafficAnalyticsRuleDriver if (!newBlockList.getPartiallyBlockedDomains().equals(partiallyBlockedDomains.get())) { partiallyBlockedDomains.set(newBlockList.getPartiallyBlockedDomains()); } + if (!newBlockList.getWhitelistDomains().equals(whiteListDomains.get())) { + whiteListDomains.set(newBlockList.getWhitelistDomainNames()); + } log.debug("refreshBlockLists: rejectDomains="+rejectDomains.get().size()); log.debug("refreshBlockLists: fullyBlockedDomains="+fullyBlockedDomains.get().size()); diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index fbcdb539..0a4e118e 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -140,7 +140,6 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute } public Set selectClosestRouter (String accountUuid, String publicIp, String vpnIp) { - if (log.isDebugEnabled()) log.debug("selectClosestRouter: publicIp="+publicIp+", vpnIp="+vpnIp); final GeoLocation geoLocation = publicIp == null ? null : geoService.locate(accountUuid, publicIp); final Collection values = activeRouters.values(); switch (values.size()) { diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index 862c0057..2e5af4e3 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -144,6 +144,7 @@ public class StandardAppPrimerService implements AppPrimerService { for (Device device : devices) { final Set rejectDomains = new HashSet<>(); final Set blockDomains = new HashSet<>(); + final Set whiteListDomains = new HashSet<>(); final Set filterDomains = new HashSet<>(); final Set flexDomains = new HashSet<>(); for (AppMatcher matcher : matchers) { @@ -160,6 +161,12 @@ public class StandardAppPrimerService implements AppPrimerService { } else { blockDomains.addAll(blocks); } + final Set whiteList = appRuleDriver.getPrimedWhiteListDomains(); + if (empty(whiteList)) { + log.debug("_prime: no whiteListDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + whiteListDomains.addAll(whiteList); + } final Set filters = appRuleDriver.getPrimedFilterDomains(); if (empty(filters)) { log.debug("_prime: no filterDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); @@ -181,6 +188,9 @@ public class StandardAppPrimerService implements AppPrimerService { if (!empty(blockDomains)) { AppRuleDriver.defineRedisBlockSet(redis, ip, app.getName() + ":" + app.getUuid(), blockDomains.toArray(String[]::new)); } + if (!empty(whiteListDomains)) { + AppRuleDriver.defineRedisWhiteListSet(redis, ip, app.getName() + ":" + app.getUuid(), whiteListDomains.toArray(String[]::new)); + } if (!empty(filterDomains)) { AppRuleDriver.defineRedisFilterSet(redis, ip, app.getName() + ":" + app.getUuid(), filterDomains.toArray(String[]::new)); } diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index 365d3917..e6a53d84 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:e73e04bb5aecf9b6b4543b0d5111d43c64a196bf42c6926770cc5678abb18a0c + checksum: sha256:9ce1c0df1075e9ba23585daa897f4f05ca90f9f99e8b1d208427aa11dc62fcdc - name: Unzip algo master.zip unarchive: diff --git a/utils/abp-parser b/utils/abp-parser index 23f140fc..072a11de 160000 --- a/utils/abp-parser +++ b/utils/abp-parser @@ -1 +1 @@ -Subproject commit 23f140fc2f99df9a5712df4faf08e1068421a57b +Subproject commit 072a11decff65461f12f47e5dae763b56a5a3247 -- 2.17.1 From f912965a8db4b0539a94bcc0d6f9a864b2b69587 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 14:19:49 -0400 Subject: [PATCH 59/78] adjust logging --- .../device/StandardFlexRouterService.java | 22 +++++++++---------- bubble-server/src/main/resources/logback.xml | 1 - 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 0a4e118e..f1f49a98 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -91,16 +91,16 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute @Override protected long getSleepTime() { return interrupted.get() ? 0 : DEFAULT_SLEEP_TIME; } @Override public void interruptSoon() { - log.debug("interruptSoon: will interrupt in "+INTERRUPT_WAIT+" millis"); + if (log.isTraceEnabled()) log.trace("interruptSoon: will interrupt in "+INTERRUPT_WAIT+" millis"); synchronized (interrupted) { if (interrupted.get()) { - log.debug("interruptSoon: interrupt flag already set, not setting again"); + if (log.isTraceEnabled()) log.trace("interruptSoon: interrupt flag already set, not setting again"); } interrupted.set(true); } background(() -> { sleep(INTERRUPT_WAIT); - log.debug("interruptSoon: interrupting..."); + if (log.isTraceEnabled()) log.trace("interruptSoon: interrupting..."); interrupt(); }, "StandardFlexRouterService.interruptSoon"); } @@ -157,7 +157,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute try { @Cleanup final CloseableHttpClient httpClient = getHttpClient(); final List routers = flexRouterDAO.findEnabledAndRegistered(); - if (log.isDebugEnabled()) log.debug("process: starting, will ping "+routers.size()+" routers"); + if (log.isTraceEnabled()) log.trace("process: starting, will ping "+routers.size()+" routers"); final List> futures = new ArrayList<>(); @Cleanup("shutdownNow") final ExecutorService exec = fixedPool(DEFAULT_MAX_TUNNELS, "StandardFlexRouterService.process"); for (FlexRouter router : routers) { @@ -178,7 +178,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute })); } final AwaitResult awaitResult = awaitAll(futures, PING_ALL_TIMEOUT); - if (log.isDebugEnabled()) log.debug("process: awaitResult="+awaitResult); + if (log.isTraceEnabled()) log.trace("process: awaitResult="+awaitResult); } catch (Exception e) { log.error("process: "+shortError(e)); @@ -193,7 +193,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute for (int i=0; i line.trim().equals(trimmedKey))) { - log.info("allowKey: already present: "+trimmedKey); + if (log.isDebugEnabled()) log.debug("allowKey: already present: "+trimmedKey); } else { @Cleanup("delete") final File temp = temp("flex_keys_", ".tmp"); final String dataToWrite = authFileContents != null && authFileContents.endsWith("\n") ? trimmedKey + "\n" : "\n" + trimmedKey + "\n"; toFileOrDie(temp, dataToWrite, true); renameOrDie(temp, authFile); - log.info("allowKey: added key: "+trimmedKey); + if (log.isInfoEnabled()) log.info("allowKey: added key: "+trimmedKey); } } @@ -266,9 +266,9 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute toFileOrDie(temp, b.toString()); renameOrDie(temp, authFile); if (found) { - log.info("disallowKey: removed key from authorized_keys file: "+trimmedKey); + if (log.isInfoEnabled()) log.info("disallowKey: removed key from authorized_keys file: "+trimmedKey); } else { - log.info("disallowKey: key was not found in authorized_keys file: "+trimmedKey); + if (log.isInfoEnabled()) log.info("disallowKey: key was not found in authorized_keys file: "+trimmedKey); } } diff --git a/bubble-server/src/main/resources/logback.xml b/bubble-server/src/main/resources/logback.xml index f20331bc..6f12d789 100644 --- a/bubble-server/src/main/resources/logback.xml +++ b/bubble-server/src/main/resources/logback.xml @@ -39,7 +39,6 @@ - -- 2.17.1 From 62f122b7b4eda7788d38753522eee825c99165e0 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 14:28:10 -0400 Subject: [PATCH 60/78] add file headers --- .../src/main/java/bubble/dao/device/FlexRouterDAO.java | 4 ++++ .../src/main/java/bubble/model/device/FlexRouter.java | 4 ++++ .../src/main/java/bubble/model/device/FlexRouterPing.java | 4 ++++ .../java/bubble/resources/device/FlexRoutersResource.java | 4 ++++ .../src/main/java/bubble/service/device/FlexRouterInfo.java | 4 ++++ .../bubble/service/device/FlexRouterProximityComparator.java | 4 ++++ .../main/java/bubble/service/device/FlexRouterService.java | 4 ++++ .../src/main/java/bubble/service/device/FlexRouterStatus.java | 4 ++++ .../java/bubble/service/device/StandardFlexRouterService.java | 4 ++++ .../bubble/service_dbfilter/DbFilterFlexRouterService.java | 4 ++++ .../bubble/test/filter/FlexRouterProximityComparatorTest.java | 4 ++++ 11 files changed, 44 insertions(+) diff --git a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java index 5c6e48c7..f734f2e8 100644 --- a/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java +++ b/bubble-server/src/main/java/bubble/dao/device/FlexRouterDAO.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.dao.device; import bubble.dao.account.AccountOwnedEntityDAO; diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java index b5992e72..3f28acdf 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouter.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouter.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.model.device; import bubble.model.account.Account; diff --git a/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java b/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java index 3987abe6..0eaf6b78 100644 --- a/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java +++ b/bubble-server/src/main/java/bubble/model/device/FlexRouterPing.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.model.device; import lombok.Getter; diff --git a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java index 0baa568b..b524fb7b 100644 --- a/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java +++ b/bubble-server/src/main/java/bubble/resources/device/FlexRoutersResource.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.resources.device; import bubble.dao.device.FlexRouterDAO; diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java index 12d0e398..a83727fe 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.service.device; import bubble.cloud.geoLocation.GeoLocation; diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java index 8956f475..cd34f6a2 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterProximityComparator.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.service.device; import bubble.cloud.geoLocation.GeoLocation; diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java index 87b55fcc..cda5ecc8 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterService.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.service.device; import bubble.model.device.FlexRouter; diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java index 4e83d6cf..11ace6a6 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterStatus.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.service.device; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index f1f49a98..4dd99cfd 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.service.device; import bubble.cloud.geoLocation.GeoLocation; diff --git a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java index 68de51e5..33eed4f8 100644 --- a/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service_dbfilter/DbFilterFlexRouterService.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.service_dbfilter; import bubble.service.device.FlexRouterService; diff --git a/bubble-server/src/test/java/bubble/test/filter/FlexRouterProximityComparatorTest.java b/bubble-server/src/test/java/bubble/test/filter/FlexRouterProximityComparatorTest.java index 7f48faba..22ab81a9 100644 --- a/bubble-server/src/test/java/bubble/test/filter/FlexRouterProximityComparatorTest.java +++ b/bubble-server/src/test/java/bubble/test/filter/FlexRouterProximityComparatorTest.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.test.filter; import bubble.cloud.geoLocation.GeoLocation; -- 2.17.1 From 7528f87097b22d11cec77211d4a84dfac4413ba1 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 22:44:15 -0400 Subject: [PATCH 61/78] proper refreshing of tls passthru and flex sets after updating --- .../passthru/TlsPassthruAppConfigDriver.java | 9 +++---- .../main/java/bubble/dao/app/AppRuleDAO.java | 24 +++++++++++++++---- .../main/java/bubble/model/app/AppRule.java | 5 ++++ .../rule/passthru/TlsPassthruConfig.java | 4 ++-- .../rule/passthru/TlsPassthruRuleDriver.java | 18 ++++++++++++++ .../service/stream/RuleEngineService.java | 1 + .../stream/StandardRuleEngineService.java | 6 ++++- .../service/upgrade/AppUpgradeService.java | 14 +++++------ 8 files changed, 62 insertions(+), 19 deletions(-) diff --git a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java index a5a6f229..2049c708 100644 --- a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java +++ b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java @@ -78,7 +78,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { case ACTION_addFeed: return addFeed(account, app, params, data); } - log.debug("takeAppAction: action not found: "+action); + if (log.isWarnEnabled()) log.warn("takeAppAction: action not found: "+action); throw notFoundEx(action); } @@ -95,6 +95,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { final AppRule rule = loadRule(account, app); loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver + if (log.isDebugEnabled()) log.debug("addFqdn: updating rule: "+rule.getName()+", adding fqdn: "+fqdn); ruleDAO.update(rule.setConfigJson(json(config))); return getFqdnList(config); @@ -134,7 +135,7 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { case ACTION_removeFeed: return removeFeed(account, app, id); } - log.debug("takeItemAction: action not found: "+action); + if (log.isWarnEnabled()) log.warn("takeItemAction: action not found: "+action); throw notFoundEx(action); } @@ -142,10 +143,10 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { final AppRule rule = loadRule(account, app); loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver final TlsPassthruConfig config = getConfig(account, app); - log.debug("removeFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getFqdnList())); + if (log.isDebugEnabled()) log.debug("removeFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getFqdnList())); final TlsPassthruConfig updated = config.removeFqdn(id); - log.debug("removeFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getFqdnList())); + if (log.isDebugEnabled()) log.debug("removeFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getFqdnList())); ruleDAO.update(rule.setConfigJson(json(updated))); return getFqdnList(updated); } diff --git a/bubble-server/src/main/java/bubble/dao/app/AppRuleDAO.java b/bubble-server/src/main/java/bubble/dao/app/AppRuleDAO.java index 1c774797..4287f884 100644 --- a/bubble-server/src/main/java/bubble/dao/app/AppRuleDAO.java +++ b/bubble-server/src/main/java/bubble/dao/app/AppRuleDAO.java @@ -5,20 +5,36 @@ package bubble.dao.app; import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; +import bubble.service.stream.AppPrimerService; import bubble.service.stream.RuleEngineService; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; -@Repository public class AppRuleDAO extends AppTemplateEntityDAO { +@Repository @Slf4j +public class AppRuleDAO extends AppTemplateEntityDAO { @Autowired private RuleEngineService ruleEngineService; + @Autowired private BubbleAppDAO appDAO; + @Autowired private AppPrimerService appPrimerService; - @Override public AppRule postUpdate(AppRule entity, Object context) { + @Override public Object preUpdate(AppRule rule) { + final AppRule existing = findByUuid(rule.getUuid()); + rule.setPreviousConfigJson(existing.getConfigJson()); + return super.preUpdate(rule); + } + + @Override public AppRule postUpdate(AppRule rule, Object context) { - ruleEngineService.flushCaches(); + if (rule.configJsonChanged()) { + final BubbleApp app = appDAO.findByUuid(rule.getApp()); + appPrimerService.prime(app); + } + ruleEngineService.flushCaches(false); // todo: update entities based on this template if account has updates enabled - return super.postUpdate(entity, context); + return super.postUpdate(rule, context); } @Override public void delete(String uuid) { diff --git a/bubble-server/src/main/java/bubble/model/app/AppRule.java b/bubble-server/src/main/java/bubble/model/app/AppRule.java index bf872391..b9ad336e 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppRule.java +++ b/bubble-server/src/main/java/bubble/model/app/AppRule.java @@ -107,6 +107,11 @@ public class AppRule extends IdentifiableBaseParentEntity implements AppTemplate @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(500000+ENC_PAD)+")") @JsonIgnore @Getter @Setter private String configJson; + @JsonIgnore @Transient @Getter @Setter private String previousConfigJson; + public boolean configJsonChanged () { + return (previousConfigJson == null && configJson != null) || (previousConfigJson != null && !previousConfigJson.equals(configJson)); + } + public AppRule(AppRule other) { copy(this, other, CREATE_FIELDS); setUuid(null); diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java index 613487c8..7fea520c 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java @@ -147,7 +147,7 @@ public class TlsPassthruConfig { private Set loadPassthruSet() { final Set set = loadFeeds(this.feedList, this.fqdnList, this.recentFeedValues); - if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning fqdnList: "+StringUtil.toString(set, ", ")); + if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning set: "+StringUtil.toString(set, ", ")+" -- fqdnList="+Arrays.toString(this.fqdnList)); return set; } @@ -207,7 +207,7 @@ public class TlsPassthruConfig { if (trimmed.startsWith("#")) { if (!loaded.hasFeedName() && trimmed.toLowerCase().startsWith(FEED_NAME_PREFIX.toLowerCase())) { final String name = trimmed.substring(FEED_NAME_PREFIX.length()).trim(); - if (log.isDebugEnabled()) log.debug("loadFeed("+url+"): setting name="+name+" from special comment: "+trimmed); + if (log.isTraceEnabled()) log.trace("loadFeed("+url+"): setting name="+name+" from special comment: "+trimmed); loaded.setFeedName(name); } else { if (log.isDebugEnabled()) log.debug("loadFeed("+url+"): ignoring comment: "+trimmed); diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index f75fe04a..1abfedc9 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -5,6 +5,9 @@ package bubble.rule.passthru; import bubble.model.account.Account; +import bubble.model.app.AppMatcher; +import bubble.model.app.AppRule; +import bubble.model.app.BubbleApp; import bubble.model.device.Device; import bubble.rule.AbstractAppRuleDriver; import bubble.service.stream.AppRuleHarness; @@ -27,6 +30,21 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { return passthruConfig.getFlexDomains(); } + @Override public void init(JsonNode config, + JsonNode userConfig, + BubbleApp app, + AppRule rule, + AppMatcher matcher, + Account account, + Device device) { + super.init(config, userConfig, app, rule, matcher, account, device); + + // refresh lists + final TlsPassthruConfig passthruConfig = getRuleConfig(); + passthruConfig.getPassthruSet(); + passthruConfig.getFlexSet(); + } + @Override public ConnectionCheckResponse checkConnection(AppRuleHarness harness, Account account, Device device, String addr, String fqdn) { final TlsPassthruConfig passthruConfig = getRuleConfig(); if (passthruConfig.isPassthru(fqdn) || passthruConfig.isPassthru(addr)) { diff --git a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java index 4ea0ca53..a14f3cd2 100644 --- a/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/RuleEngineService.java @@ -11,5 +11,6 @@ import static java.util.Collections.emptyMap; public interface RuleEngineService { default Map flushCaches() { return emptyMap(); } + default Map flushCaches(boolean prime) { return emptyMap(); } } diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java index 160ce1ab..10fedbca 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardRuleEngineService.java @@ -193,7 +193,11 @@ public class StandardRuleEngineService implements RuleEngineService { = new ExpirationMap<>(HOURS.toMillis(1), ExpirationEvictionPolicy.atime); private final AtomicBoolean cachedFlushingEnabled = new AtomicBoolean(true); - public void enableCacheFlushing () { cachedFlushingEnabled.set(true); } + public void enableCacheFlushing () { + if (log.isDebugEnabled()) log.debug("enableCacheFlushing: caching re-enabled, flushing"); + cachedFlushingEnabled.set(true); + flushCaches(false); + } public void disableCacheFlushing () { cachedFlushingEnabled.set(false); } public Map flushCaches() { return flushCaches(true); } diff --git a/bubble-server/src/main/java/bubble/service/upgrade/AppUpgradeService.java b/bubble-server/src/main/java/bubble/service/upgrade/AppUpgradeService.java index 17762389..33ba3d0c 100644 --- a/bubble-server/src/main/java/bubble/service/upgrade/AppUpgradeService.java +++ b/bubble-server/src/main/java/bubble/service/upgrade/AppUpgradeService.java @@ -114,14 +114,7 @@ public class AppUpgradeService extends SimpleDaemon { return; } - try { - ruleEngine.disableCacheFlushing(); - handleAdminUpgrades(admin, sageNode); - - } finally { - ruleEngine.enableCacheFlushing(); - ruleEngine.flushCaches(); - } + handleAdminUpgrades(admin, sageNode); } private void handleAdminUpgrades(Account admin, BubbleNode sageNode) { @@ -146,6 +139,8 @@ public class AppUpgradeService extends SimpleDaemon { final List myDrivers = Arrays.asList(upgradeRequest.getDrivers()); accountDrivers.put(admin.getUuid(), myDrivers); + + ruleEngine.disableCacheFlushing(); for (RuleDriver sageDriver : sageDrivers) { log.info("handleAdminUpgrades: updating admin driver: "+sageDriver.getName()); updateDriver(admin, myDrivers, sageDriver); @@ -166,6 +161,9 @@ public class AppUpgradeService extends SimpleDaemon { } } catch (Exception e) { log.error("handleAdminUpgrades: "+shortError(e), e); + + } finally { + ruleEngine.enableCacheFlushing(); } } -- 2.17.1 From edd6325bf384115c751bb91af6bc4fc502d1e84f Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 9 Sep 2020 23:39:56 -0400 Subject: [PATCH 62/78] refactor passthru app config, add support for managing flex domains/feeds --- .../passthru/TlsPassthruAppConfigDriver.java | 192 +++++++++++++----- .../rule/passthru/TlsPassthruConfig.java | 44 ++-- .../rule/passthru/TlsPassthruRuleDriver.java | 16 +- bubble-server/src/main/resources/messages | 2 +- .../apps/passthru/bubbleApp_passthru.json | 101 ++++++--- 5 files changed, 251 insertions(+), 104 deletions(-) diff --git a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java index 2049c708..ee1bbd99 100644 --- a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java +++ b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java @@ -30,31 +30,51 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; @Slf4j public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { - public static final String VIEW_manageDomains = "manageDomains"; - public static final String VIEW_manageFeeds = "manageFeeds"; + public static final String VIEW_managePassthruDomains = "managePassthruDomains"; + public static final String VIEW_managePassthruFeeds = "managePassthruFeeds"; + public static final String VIEW_manageFlexDomains = "manageFlexDomains"; + public static final String VIEW_manageFlexFeeds = "manageFlexFeeds"; @Autowired @Getter private AppRuleDAO ruleDAO; @Override public Object getView(Account account, BubbleApp app, String view, Map params) { switch (view) { - case VIEW_manageDomains: - return loadManageDomains(account, app); - case VIEW_manageFeeds: - return loadManageFeeds(account, app); + case VIEW_managePassthruDomains: + return loadManagePassthruDomains(account, app); + case VIEW_managePassthruFeeds: + return loadManagePassthuFeeds(account, app); + case VIEW_manageFlexDomains: + return loadManageFlexDomains(account, app); + case VIEW_manageFlexFeeds: + return loadManageFlexFeeds(account, app); } throw notFoundEx(view); } - private Set loadManageFeeds(Account account, BubbleApp app) { + private Set loadManagePassthuFeeds(Account account, BubbleApp app) { final TlsPassthruConfig config = getConfig(account, app); config.getPassthruSet(); // ensure names are initialized - return config.getFeedSet(); + return config.getPassthruFeedSet(); } - private Set loadManageDomains(Account account, BubbleApp app) { + private Set loadManagePassthruDomains(Account account, BubbleApp app) { final TlsPassthruConfig config = getConfig(account, app); - return !config.hasFqdnList() ? Collections.emptySet() : - Arrays.stream(config.getFqdnList()) + return !config.hasPassthruFqdnList() ? Collections.emptySet() : + Arrays.stream(config.getPassthruFqdnList()) + .map(TlsPassthruFqdn::new) + .collect(Collectors.toCollection(TreeSet::new)); + } + + private Set loadManageFlexFeeds(Account account, BubbleApp app) { + final TlsPassthruConfig config = getConfig(account, app); + config.getFlexSet(); // ensure names are initialized + return config.getFlexFeedSet(); + } + + private Set loadManageFlexDomains(Account account, BubbleApp app) { + final TlsPassthruConfig config = getConfig(account, app); + return !config.hasFlexFqdnList() ? Collections.emptySet() : + Arrays.stream(config.getFlexFqdnList()) .map(TlsPassthruFqdn::new) .collect(Collectors.toCollection(TreeSet::new)); } @@ -63,54 +83,106 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { return getConfig(account, app, TlsPassthruRuleDriver.class, TlsPassthruConfig.class); } - public static final String ACTION_addFqdn = "addFqdn"; - public static final String ACTION_removeFqdn = "removeFqdn"; - public static final String ACTION_addFeed = "addFeed"; - public static final String ACTION_removeFeed = "removeFeed"; + public static final String ACTION_addPassthruFqdn = "addPassthruFqdn"; + public static final String ACTION_addPassthruFeed = "addPassthruFeed"; + public static final String ACTION_removePassthruFqdn = "removePassthruFqdn"; + public static final String ACTION_removePassthruFeed = "removePassthruFeed"; + + public static final String ACTION_addFlexFqdn = "addFlexFqdn"; + public static final String ACTION_addFlexFeed = "addFlexFeed"; + public static final String ACTION_removeFlexFqdn = "removeFlexFqdn"; + public static final String ACTION_removeFlexFeed = "removeFlexFeed"; - public static final String PARAM_FQDN = "passthruFqdn"; - public static final String PARAM_FEED_URL = "feedUrl"; + public static final String PARAM_PASSTHRU_FQDN = "passthruFqdn"; + public static final String PARAM_PASSTHRU_FEED_URL = "passthruFeedUrl"; + public static final String PARAM_FLEX_FQDN = "flexFqdn"; + public static final String PARAM_FLEX_FEED_URL = "flexFeedUrl"; @Override public Object takeAppAction(Account account, BubbleApp app, String view, String action, Map params, JsonNode data) { switch (action) { - case ACTION_addFqdn: - return addFqdn(account, app, data); - case ACTION_addFeed: - return addFeed(account, app, params, data); + case ACTION_addPassthruFqdn: + return addPassthruFqdn(account, app, data); + case ACTION_addPassthruFeed: + return addPassthruFeed(account, app, params, data); + case ACTION_addFlexFqdn: + return addFlexFqdn(account, app, data); + case ACTION_addFlexFeed: + return addFlexFeed(account, app, params, data); } if (log.isWarnEnabled()) log.warn("takeAppAction: action not found: "+action); throw notFoundEx(action); } - private List addFqdn(Account account, BubbleApp app, JsonNode data) { - final JsonNode fqdnNode = data.get(PARAM_FQDN); + private List addPassthruFqdn(Account account, BubbleApp app, JsonNode data) { + final JsonNode fqdnNode = data.get(PARAM_PASSTHRU_FQDN); if (fqdnNode == null || fqdnNode.textValue() == null || empty(fqdnNode.textValue().trim())) { - throw invalidEx("err.addFqdn.passthruFqdnRequired"); + throw invalidEx("err.passthruFqdn.passthruFqdnRequired"); } final String fqdn = fqdnNode.textValue().trim().toLowerCase(); + final TlsPassthruConfig config = getConfig(account, app).addPassthruFqdn(fqdn); - final TlsPassthruConfig config = getConfig(account, app) - .addFqdn(fqdn); + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver + if (log.isDebugEnabled()) log.debug("addPassthruFqdn: updating rule: "+rule.getName()+", adding fqdn: "+fqdn); + ruleDAO.update(rule.setConfigJson(json(config))); + + return getPassthruFqdnList(config); + } + + private List getPassthruFqdnList(TlsPassthruConfig config) { + return Arrays.stream(config.getPassthruFqdnList()) + .map(TlsPassthruFqdn::new) + .collect(Collectors.toList()); + } + + private Set addPassthruFeed(Account account, BubbleApp app, Map params, JsonNode data) { + final JsonNode urlNode = data.get(PARAM_PASSTHRU_FEED_URL); + if (urlNode == null || urlNode.textValue() == null || empty(urlNode.textValue().trim())) { + throw invalidEx("err.passthruFeedUrl.feedUrlRequired"); + } + + final String url = urlNode.textValue().trim().toLowerCase(); + final TlsPassthruConfig config = getConfig(account, app); + + final TlsPassthruFeed feed = config.loadFeed(url); + if (!feed.hasFqdnList()) throw invalidEx("err.passthruFeedUrl.emptyFqdnList"); + config.addPassthruFeed(feed); + + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver + ruleDAO.update(rule.setConfigJson(json(config))); + + return config.getPassthruFeedSet(); + } + + private List addFlexFqdn(Account account, BubbleApp app, JsonNode data) { + final JsonNode fqdnNode = data.get(PARAM_FLEX_FQDN); + if (fqdnNode == null || fqdnNode.textValue() == null || empty(fqdnNode.textValue().trim())) { + throw invalidEx("err.flexFqdn.flexFqdnRequired"); + } + + final String fqdn = fqdnNode.textValue().trim().toLowerCase(); + final TlsPassthruConfig config = getConfig(account, app).addFlexFqdn(fqdn); final AppRule rule = loadRule(account, app); loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver - if (log.isDebugEnabled()) log.debug("addFqdn: updating rule: "+rule.getName()+", adding fqdn: "+fqdn); + if (log.isDebugEnabled()) log.debug("addFlexFqdn: updating rule: "+rule.getName()+", adding fqdn: "+fqdn); ruleDAO.update(rule.setConfigJson(json(config))); - return getFqdnList(config); + return getFlexFqdnList(config); } - private List getFqdnList(TlsPassthruConfig config) { - return Arrays.stream(config.getFqdnList()) + private List getFlexFqdnList(TlsPassthruConfig config) { + return Arrays.stream(config.getFlexFqdnList()) .map(TlsPassthruFqdn::new) .collect(Collectors.toList()); } - private Set addFeed(Account account, BubbleApp app, Map params, JsonNode data) { - final JsonNode urlNode = data.get(PARAM_FEED_URL); + private Set addFlexFeed(Account account, BubbleApp app, Map params, JsonNode data) { + final JsonNode urlNode = data.get(PARAM_FLEX_FEED_URL); if (urlNode == null || urlNode.textValue() == null || empty(urlNode.textValue().trim())) { - throw invalidEx("err.addFeed.feedUrlRequired"); + throw invalidEx("err.flexFeedUrl.feedUrlRequired"); } final String url = urlNode.textValue().trim().toLowerCase(); @@ -118,45 +190,69 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { final TlsPassthruConfig config = getConfig(account, app); final TlsPassthruFeed feed = config.loadFeed(url); - if (!feed.hasFqdnList()) throw invalidEx("err.addFeed.emptyFqdnList"); - config.addFeed(feed); + if (!feed.hasFqdnList()) throw invalidEx("err.flexFeedUrl.emptyFqdnList"); + config.addPassthruFeed(feed); final AppRule rule = loadRule(account, app); loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver ruleDAO.update(rule.setConfigJson(json(config))); - return config.getFeedSet(); + return config.getPassthruFeedSet(); } @Override public Object takeItemAction(Account account, BubbleApp app, String view, String action, String id, Map params, JsonNode data) { switch (action) { - case ACTION_removeFqdn: - return removeFqdn(account, app, id); - case ACTION_removeFeed: - return removeFeed(account, app, id); + case ACTION_removePassthruFqdn: + return removePassthruFqdn(account, app, id); + case ACTION_removePassthruFeed: + return removePassthruFeed(account, app, id); + case ACTION_removeFlexFqdn: + return removeFlexFqdn(account, app, id); + case ACTION_removeFlexFeed: + return removeFlexFeed(account, app, id); } if (log.isWarnEnabled()) log.warn("takeItemAction: action not found: "+action); throw notFoundEx(action); } - private List removeFqdn(Account account, BubbleApp app, String id) { + private List removePassthruFqdn(Account account, BubbleApp app, String id) { + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver + final TlsPassthruConfig config = getConfig(account, app); + if (log.isDebugEnabled()) log.debug("removePassthruFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getPassthruFqdnList())); + + final TlsPassthruConfig updated = config.removePassthruFqdn(id); + if (log.isDebugEnabled()) log.debug("removePassthruFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getPassthruFqdnList())); + ruleDAO.update(rule.setConfigJson(json(updated))); + return getPassthruFqdnList(updated); + } + + public Set removePassthruFeed(Account account, BubbleApp app, String id) { + final AppRule rule = loadRule(account, app); + loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver + final TlsPassthruConfig config = getConfig(account, app).removePassthruFeed(id); + ruleDAO.update(rule.setConfigJson(json(config))); + return config.getPassthruFeedSet(); + } + + private List removeFlexFqdn(Account account, BubbleApp app, String id) { final AppRule rule = loadRule(account, app); loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver final TlsPassthruConfig config = getConfig(account, app); - if (log.isDebugEnabled()) log.debug("removeFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getFqdnList())); + if (log.isDebugEnabled()) log.debug("removeFlexFqdn: removing id: "+id+" from config.fqdnList: "+ ArrayUtil.arrayToString(config.getPassthruFqdnList())); - final TlsPassthruConfig updated = config.removeFqdn(id); - if (log.isDebugEnabled()) log.debug("removeFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getFqdnList())); + final TlsPassthruConfig updated = config.removeFlexFqdn(id); + if (log.isDebugEnabled()) log.debug("removeFlexFqdn: updated.fqdnList: "+ ArrayUtil.arrayToString(updated.getPassthruFqdnList())); ruleDAO.update(rule.setConfigJson(json(updated))); - return getFqdnList(updated); + return getFlexFqdnList(updated); } - public Set removeFeed(Account account, BubbleApp app, String id) { + public Set removeFlexFeed(Account account, BubbleApp app, String id) { final AppRule rule = loadRule(account, app); loadDriver(account, rule, TlsPassthruRuleDriver.class); // validate proper driver - final TlsPassthruConfig config = getConfig(account, app).removeFeed(id); + final TlsPassthruConfig config = getConfig(account, app).removeFlexFeed(id); ruleDAO.update(rule.setConfigJson(json(config))); - return config.getFeedSet(); + return config.getPassthruFeedSet(); } } diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java index 7fea520c..ac54d559 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java @@ -37,44 +37,44 @@ public class TlsPassthruConfig { public static final long DEFAULT_FLEX_FEED_REFRESH_INTERVAL = HOURS.toMillis(1); public static final String FEED_NAME_PREFIX = "# Name:"; - @Getter @Setter private String[] fqdnList; - public boolean hasFqdnList () { return !empty(fqdnList); } - public boolean hasFqdn(String fqdn) { return hasFqdnList() && ArrayUtils.indexOf(fqdnList, fqdn) != -1; } + @Getter @Setter private String[] passthruFqdnList; + public boolean hasPassthruFqdnList() { return !empty(passthruFqdnList); } + public boolean hasPassthruFqdn(String fqdn) { return hasPassthruFqdnList() && ArrayUtils.indexOf(passthruFqdnList, fqdn) != -1; } - public TlsPassthruConfig addFqdn(String fqdn) { - return setFqdnList(Arrays.stream(ArrayUtil.append(fqdnList, fqdn)).collect(Collectors.toSet()).toArray(String[]::new)); + public TlsPassthruConfig addPassthruFqdn(String fqdn) { + return setPassthruFqdnList(Arrays.stream(ArrayUtil.append(passthruFqdnList, fqdn)).collect(Collectors.toSet()).toArray(String[]::new)); } - public TlsPassthruConfig removeFqdn(String id) { - return !hasFqdnList() ? this : - setFqdnList(Arrays.stream(getFqdnList()) + public TlsPassthruConfig removePassthruFqdn(String id) { + return !hasPassthruFqdnList() ? this : + setPassthruFqdnList(Arrays.stream(getPassthruFqdnList()) .filter(fqdn -> !fqdn.equalsIgnoreCase(id.trim())) .toArray(String[]::new)); } - @Getter @Setter private TlsPassthruFeed[] feedList; - public boolean hasFeedList () { return !empty(feedList); } - public boolean hasFeed (TlsPassthruFeed feed) { - return hasFeedList() && Arrays.stream(feedList).anyMatch(f -> f.getFeedUrl().equals(feed.getFeedUrl())); + @Getter @Setter private TlsPassthruFeed[] passthruFeedList; + public boolean hasPassthruFeedList() { return !empty(passthruFeedList); } + public boolean hasPassthruFeed(TlsPassthruFeed feed) { + return hasPassthruFeedList() && Arrays.stream(passthruFeedList).anyMatch(f -> f.getFeedUrl().equals(feed.getFeedUrl())); } - public TlsPassthruConfig addFeed(TlsPassthruFeed feed) { - final Set feeds = getFeedSet(); - if (empty(feeds)) return setFeedList(new TlsPassthruFeed[] {feed}); + public TlsPassthruConfig addPassthruFeed(TlsPassthruFeed feed) { + final Set feeds = getPassthruFeedSet(); + if (empty(feeds)) return setPassthruFeedList(new TlsPassthruFeed[] {feed}); feeds.add(feed); - return setFeedList(feeds.toArray(EMPTY_FEEDS)); + return setPassthruFeedList(feeds.toArray(EMPTY_FEEDS)); } - public TlsPassthruConfig removeFeed(String id) { - return setFeedList(getFeedSet().stream() + public TlsPassthruConfig removePassthruFeed(String id) { + return setPassthruFeedList(getPassthruFeedSet().stream() .filter(feed -> !feed.getId().equals(id)) .toArray(TlsPassthruFeed[]::new)); } private final Map> recentFeedValues = new HashMap<>(); - @JsonIgnore public Set getFeedSet() { - final TlsPassthruFeed[] feedList = getFeedList(); + @JsonIgnore public Set getPassthruFeedSet() { + final TlsPassthruFeed[] feedList = getPassthruFeedList(); return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); } @@ -146,8 +146,8 @@ public class TlsPassthruConfig { @JsonIgnore public Set getPassthruSet() { return getPassthruSetRef().get(); } private Set loadPassthruSet() { - final Set set = loadFeeds(this.feedList, this.fqdnList, this.recentFeedValues); - if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning set: "+StringUtil.toString(set, ", ")+" -- fqdnList="+Arrays.toString(this.fqdnList)); + final Set set = loadFeeds(this.passthruFeedList, this.passthruFqdnList, this.recentFeedValues); + if (log.isDebugEnabled()) log.debug("loadPassthruSet: returning set: "+StringUtil.toString(set, ", ")+" -- fqdnList="+Arrays.toString(this.passthruFqdnList)); return set; } diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index 1abfedc9..56977ccd 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -58,17 +58,17 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { @Override public JsonNode upgradeRuleConfig(JsonNode sageRuleConfig, JsonNode localRuleConfig) { final TlsPassthruConfig sageConfig = json(sageRuleConfig, getConfigClass()); final TlsPassthruConfig localConfig = json(sageRuleConfig, getConfigClass()); - if (sageConfig.hasFqdnList()) { - for (String fqdn : sageConfig.getFqdnList()) { - if (!localConfig.hasFqdnList() || localConfig.hasFqdn(fqdn)) { - localConfig.setFqdnList(ArrayUtil.append(localConfig.getFqdnList(), fqdn)); + if (sageConfig.hasPassthruFqdnList()) { + for (String fqdn : sageConfig.getPassthruFqdnList()) { + if (!localConfig.hasPassthruFqdnList() || localConfig.hasPassthruFqdn(fqdn)) { + localConfig.setPassthruFqdnList(ArrayUtil.append(localConfig.getPassthruFqdnList(), fqdn)); } } } - if (sageConfig.hasFeedList()) { - for (TlsPassthruFeed feed : sageConfig.getFeedList()) { - if (!localConfig.hasFeed(feed)) { - localConfig.setFeedList(ArrayUtil.append(localConfig.getFeedList(), feed)); + if (sageConfig.hasPassthruFeedList()) { + for (TlsPassthruFeed feed : sageConfig.getPassthruFeedList()) { + if (!localConfig.hasPassthruFeed(feed)) { + localConfig.setPassthruFeedList(ArrayUtil.append(localConfig.getPassthruFeedList(), feed)); } } } diff --git a/bubble-server/src/main/resources/messages b/bubble-server/src/main/resources/messages index 74c7c7f2..b84f708d 160000 --- a/bubble-server/src/main/resources/messages +++ b/bubble-server/src/main/resources/messages @@ -1 +1 @@ -Subproject commit 74c7c7f2efa41dd12d74cffff5013149136d3867 +Subproject commit b84f708deb8b80b3850af575a4341999d6dbb601 diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json index 73b54acb..c90f0df9 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json @@ -12,33 +12,62 @@ "configDriver": "bubble.app.passthru.TlsPassthruAppConfigDriver", "configFields": [ {"name": "passthruFqdn", "type": "hostname", "truncate": false}, - {"name": "feedName", "truncate": false}, - {"name": "feedUrl", "type": "http_url"} + {"name": "passthruFeedName", "truncate": false}, + {"name": "passthruFeedUrl", "type": "http_url"}, + {"name": "flexFqdn", "type": "hostname", "truncate": false}, + {"name": "flexFeedName", "truncate": false}, + {"name": "flexFeedUrl", "type": "http_url"} ], "configViews": [{ - "name": "manageDomains", + "name": "managePassthruDomains", "scope": "app", "root": "true", "fields": ["passthruFqdn"], "actions": [ - {"name": "removeFqdn", "index": 10}, + {"name": "removePassthruFqdn", "index": 10}, { - "name": "addFqdn", "scope": "app", "index": 10, + "name": "addPassthruFqdn", "scope": "app", "index": 10, "params": ["passthruFqdn"], - "button": "addFqdn" + "button": "addPassthruFqdn" } ] }, { - "name": "manageFeeds", + "name": "managePassthruFeeds", "scope": "app", "root": "true", - "fields": ["feedName", "feedUrl"], + "fields": ["passthruFeedName", "passthruFeedUrl"], "actions": [ - {"name": "removeFeed", "index": 10}, + {"name": "removePassthruFeed", "index": 10}, { - "name": "addFeed", "scope": "app", "index": 10, - "params": ["feedUrl"], - "button": "addFeed" + "name": "addPassthruFeed", "scope": "app", "index": 10, + "params": ["passthruFeedUrl"], + "button": "addPassthruFeed" + } + ] + }, { + "name": "manageFlexDomains", + "scope": "app", + "root": "true", + "fields": ["flexFqdn"], + "actions": [ + {"name": "removeFlexFqdn", "index": 10}, + { + "name": "addFlexFqdn", "scope": "app", "index": 10, + "params": ["flexFqdn"], + "button": "addFlexFqdn" + } + ] + }, { + "name": "manageFlexFeeds", + "scope": "app", + "root": "true", + "fields": ["flexFeedName", "flexFeedUrl"], + "actions": [ + {"name": "removeFlexFeed", "index": 10}, + { + "name": "addFlexFeed", "scope": "app", "index": 10, + "params": ["flexFeedUrl"], + "button": "addFlexFeed" } ] }] @@ -56,8 +85,8 @@ "driver": "TlsPassthruRuleDriver", "priority": -1000000, "config": { - "fqdnList": [], - "feedList": [{ + "passthruFqdnList": [], + "passthruFeedList": [{ "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/tls_passthru.txt" }], "flexFqdnList": [], @@ -74,19 +103,41 @@ {"name": "summary", "value": "Network Bypass"}, {"name": "description", "value": "Do not perform SSL interception for certificate-pinned domains"}, - {"name": "config.view.manageDomains", "value": "Manage Bypass Domains"}, - {"name": "config.view.manageFeeds", "value": "Manage Bypass Domain Feeds"}, + {"name": "config.view.managePassthruDomains", "value": "Manage Bypass Domains"}, + {"name": "config.view.managePassthruFeeds", "value": "Manage Bypass Domain Feeds"}, {"name": "config.field.passthruFqdn", "value": "Domain"}, {"name": "config.field.passthruFqdn.description", "value": "Bypass traffic interception for this hostname"}, - {"name": "config.field.feedName", "value": "Name"}, - {"name": "config.field.feedUrl", "value": "Bypass Domains List URL"}, - {"name": "config.field.feedUrl.description", "value": "URL returning a list of bypass domains and/or hostnames, one per line"}, - {"name": "config.action.addFqdn", "value": "Add New Bypass Domain"}, - {"name": "config.button.addFqdn", "value": "Add"}, - {"name": "config.action.removeFqdn", "value": "Remove"}, - {"name": "config.action.addFeed", "value": "Add New Bypass Domain Feed"}, - {"name": "config.button.addFeed", "value": "Add"}, - {"name": "config.action.removeFeed", "value": "Remove"} + {"name": "config.field.passthruFeedName", "value": "Name"}, + {"name": "config.field.passthruFeedUrl", "value": "Bypass Domains List URL"}, + {"name": "config.field.passthruFeedUrl.description", "value": "URL returning a list of bypass domains and/or hostnames, one per line"}, + {"name": "config.action.addPassthruFqdn", "value": "Add New Bypass Domain"}, + {"name": "config.button.addPassthruFqdn", "value": "Add"}, + {"name": "config.action.removePassthruFqdn", "value": "Remove"}, + {"name": "config.action.addPassthruFeed", "value": "Add New Bypass Domain Feed"}, + {"name": "config.button.addPassthruFeed", "value": "Add"}, + {"name": "config.action.removePassthruFeed", "value": "Remove"}, + + {"name": "config.view.manageFlexDomains", "value": "Manage Flex Routing Domains"}, + {"name": "config.view.manageFlexFeeds", "value": "Manage Flex Routing Domain Feeds"}, + {"name": "config.field.flexFqdn", "value": "Domain"}, + {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this hostname"}, + {"name": "config.field.flexFeedName", "value": "Name"}, + {"name": "config.field.flexFeedUrl", "value": "Flex Routing Domains List URL"}, + {"name": "config.field.flexFeedUrl.description", "value": "URL returning a list of domains and/or hostnames to flex route, one per line"}, + {"name": "config.action.addFlexFqdn", "value": "Add New Flex Routing Domain"}, + {"name": "config.button.addFlexFqdn", "value": "Add"}, + {"name": "config.action.removeFlexFqdn", "value": "Remove"}, + {"name": "config.action.addFlexFeed", "value": "Add New Flex Routing Domain Feed"}, + {"name": "config.button.addFlexFeed", "value": "Add"}, + {"name": "config.action.removeFlexFeed", "value": "Remove"}, + + {"name": "err.passthruFqdn.passthruFqdnRequired", "value": "Domain or Hostname field is required"}, + {"name": "err.passthruFeedUrl.feedUrlRequired", "value": "Feed URL is required"}, + {"name": "err.passthruFeedUrl.emptyFqdnList", "value": "Feed URL was not found or contained no data"}, + + {"name": "err.flexFqdn.flexFqdnRequired", "value": "Domain or Hostname field is required"}, + {"name": "err.flexFeedUrl.feedUrlRequired", "value": "Feed URL is required"}, + {"name": "err.flexFeedUrl.emptyFqdnList", "value": "Feed URL was not found or contained no data"} ] }] } -- 2.17.1 From 2a1c1f546d4a28560e6da3e27542f3716fe7c5a8 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 10 Sep 2020 00:09:25 -0400 Subject: [PATCH 63/78] use separate classes for passthru vs flex feeds/fqdns --- .../passthru/TlsPassthruAppConfigDriver.java | 11 ++--- .../rule/passthru/BasePassthruFeed.java | 35 +++++++++++++++ .../java/bubble/rule/passthru/FlexFeed.java | 28 ++++++++++++ .../java/bubble/rule/passthru/FlexFqdn.java | 22 ++++++++++ .../rule/passthru/TlsPassthruConfig.java | 44 ++++++++++--------- .../bubble/rule/passthru/TlsPassthruFeed.java | 32 +++++--------- bubble-server/src/main/resources/logback.xml | 8 ++-- .../apps/passthru/bubbleApp_passthru.json | 4 +- 8 files changed, 128 insertions(+), 56 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java create mode 100644 bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java create mode 100644 bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java diff --git a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java index ee1bbd99..1e7a8547 100644 --- a/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java +++ b/bubble-server/src/main/java/bubble/app/passthru/TlsPassthruAppConfigDriver.java @@ -9,10 +9,7 @@ import bubble.model.account.Account; import bubble.model.app.AppRule; import bubble.model.app.BubbleApp; import bubble.model.app.config.AppConfigDriverBase; -import bubble.rule.passthru.TlsPassthruConfig; -import bubble.rule.passthru.TlsPassthruFeed; -import bubble.rule.passthru.TlsPassthruFqdn; -import bubble.rule.passthru.TlsPassthruRuleDriver; +import bubble.rule.passthru.*; import com.fasterxml.jackson.databind.JsonNode; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -65,17 +62,17 @@ public class TlsPassthruAppConfigDriver extends AppConfigDriverBase { .collect(Collectors.toCollection(TreeSet::new)); } - private Set loadManageFlexFeeds(Account account, BubbleApp app) { + private Set loadManageFlexFeeds(Account account, BubbleApp app) { final TlsPassthruConfig config = getConfig(account, app); config.getFlexSet(); // ensure names are initialized return config.getFlexFeedSet(); } - private Set loadManageFlexDomains(Account account, BubbleApp app) { + private Set loadManageFlexDomains(Account account, BubbleApp app) { final TlsPassthruConfig config = getConfig(account, app); return !config.hasFlexFqdnList() ? Collections.emptySet() : Arrays.stream(config.getFlexFqdnList()) - .map(TlsPassthruFqdn::new) + .map(FlexFqdn::new) .collect(Collectors.toCollection(TreeSet::new)); } diff --git a/bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java b/bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java new file mode 100644 index 00000000..84944c3e --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java @@ -0,0 +1,35 @@ +package bubble.rule.passthru; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.util.Set; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.security.ShaUtil.sha256_hex; + +@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of="feedUrl") +public class BasePassthruFeed implements Comparable { + + public BasePassthruFeed (String url) { setFeedUrl(url); } + + public String getId() { return sha256_hex(getFeedUrl()); } + public void setId(String id) {} // noop + + @JsonIgnore @Getter @Setter private String feedName; + public boolean hasFeedName() { return !empty(feedName); } + + @JsonIgnore @Getter @Setter private String feedUrl; + + @JsonIgnore @Getter @Setter private Set fqdnList; + public boolean hasFqdnList() { return !empty(fqdnList); } + + @Override public int compareTo(BasePassthruFeed o) { + return getFeedUrl().toLowerCase().compareTo(o.getFeedUrl().toLowerCase()); + } + +} diff --git a/bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java b/bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java new file mode 100644 index 00000000..ebe427f6 --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java @@ -0,0 +1,28 @@ +package bubble.rule.passthru; + +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; + +import java.util.Set; + +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@NoArgsConstructor @Accessors(chain=true) +public class FlexFeed extends BasePassthruFeed { + + public static final FlexFeed[] EMPTY_FLEX_FEEDS = new FlexFeed[0]; + + public String getFlexFeedName () { return getFeedName(); } + public FlexFeed setFlexFeedName (String name) { return (FlexFeed) setFeedName(name); } + + public String getFlexFeedUrl () { return getFeedUrl(); } + public FlexFeed setFlexFeedUrl (String url) { return (FlexFeed) setFeedUrl(url); } + + public Set getFlexFqdnList () { return getFqdnList(); } + public FlexFeed setFlexFqdnList (Set set) { return (FlexFeed) setFqdnList(set); } + + public FlexFeed(String url) { super(url); } + + public FlexFeed(FlexFeed feed) { copy(this, feed); } + +} diff --git a/bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java b/bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java new file mode 100644 index 00000000..b6566edd --- /dev/null +++ b/bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java @@ -0,0 +1,22 @@ +package bubble.rule.passthru; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +@NoArgsConstructor @Accessors(chain=true) +public class FlexFqdn implements Comparable { + + public String getId() { return flexFqdn; } + public void setId(String id) {} // noop + + @Getter @Setter private String flexFqdn; + + public FlexFqdn(String fqdn) { setFlexFqdn(fqdn); } + + @Override public int compareTo(FlexFqdn o) { + return getFlexFqdn().toLowerCase().compareTo(o.getFlexFqdn().toLowerCase()); + } + +} diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java index ac54d559..d9e3025f 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruConfig.java @@ -21,7 +21,8 @@ import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; -import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_FEEDS; +import static bubble.rule.passthru.FlexFeed.EMPTY_FLEX_FEEDS; +import static bubble.rule.passthru.TlsPassthruFeed.EMPTY_PASSTHRU_FEEDS; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.regex.Pattern.CASE_INSENSITIVE; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @@ -55,14 +56,14 @@ public class TlsPassthruConfig { @Getter @Setter private TlsPassthruFeed[] passthruFeedList; public boolean hasPassthruFeedList() { return !empty(passthruFeedList); } public boolean hasPassthruFeed(TlsPassthruFeed feed) { - return hasPassthruFeedList() && Arrays.stream(passthruFeedList).anyMatch(f -> f.getFeedUrl().equals(feed.getFeedUrl())); + return hasPassthruFeedList() && Arrays.stream(passthruFeedList).anyMatch(f -> f.getPassthruFeedUrl().equals(feed.getPassthruFeedUrl())); } public TlsPassthruConfig addPassthruFeed(TlsPassthruFeed feed) { final Set feeds = getPassthruFeedSet(); if (empty(feeds)) return setPassthruFeedList(new TlsPassthruFeed[] {feed}); feeds.add(feed); - return setPassthruFeedList(feeds.toArray(EMPTY_FEEDS)); + return setPassthruFeedList(feeds.toArray(EMPTY_PASSTHRU_FEEDS)); } public TlsPassthruConfig removePassthruFeed(String id) { @@ -75,7 +76,8 @@ public class TlsPassthruConfig { @JsonIgnore public Set getPassthruFeedSet() { final TlsPassthruFeed[] feedList = getPassthruFeedList(); - return !empty(feedList) ? Arrays.stream(feedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); + return !empty(feedList) ? Arrays.stream(feedList) + .collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); } @Getter @Setter private String[] flexFqdnList; @@ -93,29 +95,29 @@ public class TlsPassthruConfig { .toArray(String[]::new)); } - @Getter @Setter private TlsPassthruFeed[] flexFeedList; + @Getter @Setter private FlexFeed[] flexFeedList; public boolean hasFlexFeedList () { return !empty(flexFeedList); } - public boolean hasFlexFeed (TlsPassthruFeed flexFeed) { - return hasFlexFeedList() && Arrays.stream(flexFeedList).anyMatch(f -> f.getFeedUrl().equals(flexFeed.getFeedUrl())); + public boolean hasFlexFeed (FlexFeed flexFeed) { + return hasFlexFeedList() && Arrays.stream(flexFeedList).anyMatch(f -> f.getFlexFeedUrl().equals(flexFeed.getFlexFeedUrl())); } - public TlsPassthruConfig addFlexFeed(TlsPassthruFeed flexFeed) { - final Set flexFeeds = getFlexFeedSet(); - if (empty(flexFeeds)) return setFlexFeedList(new TlsPassthruFeed[] {flexFeed}); + public TlsPassthruConfig addFlexFeed(FlexFeed flexFeed) { + final Set flexFeeds = getFlexFeedSet(); + if (empty(flexFeeds)) return setFlexFeedList(new FlexFeed[] {flexFeed}); flexFeeds.add(flexFeed); - return setFlexFeedList(flexFeeds.toArray(EMPTY_FEEDS)); + return setFlexFeedList(flexFeeds.toArray(EMPTY_FLEX_FEEDS)); } public TlsPassthruConfig removeFlexFeed(String id) { return setFlexFeedList(getFlexFeedSet().stream() .filter(flexFeed -> !flexFeed.getId().equals(id)) - .toArray(TlsPassthruFeed[]::new)); + .toArray(FlexFeed[]::new)); } private final Map> recentFlexFeedValues = new HashMap<>(); - @JsonIgnore public Set getFlexFeedSet() { - final TlsPassthruFeed[] flexFeedList = getFlexFeedList(); + @JsonIgnore public Set getFlexFeedSet() { + final FlexFeed[] flexFeedList = getFlexFeedList(); return !empty(flexFeedList) ? Arrays.stream(flexFeedList).collect(Collectors.toCollection(TreeSet::new)) : Collections.emptySet(); } @@ -171,7 +173,7 @@ public class TlsPassthruConfig { return set; } - private Set loadFeeds(TlsPassthruFeed[] feedList, String[] fqdnList, Map> recentValues) { + private Set loadFeeds(BasePassthruFeed[] feedList, String[] fqdnList, Map> recentValues) { final Set set = new HashSet<>(); if (!empty(fqdnList)) { for (String val : fqdnList) { @@ -180,14 +182,14 @@ public class TlsPassthruConfig { } if (!empty(feedList)) { // put in a set to avoid duplicate URLs - for (TlsPassthruFeed feed : new HashSet<>(Arrays.asList(feedList))) { + for (BasePassthruFeed feed : new HashSet<>(Arrays.asList(feedList))) { final TlsPassthruFeed loaded = loadFeed(feed.getFeedUrl()); // set name if found in special comment - if (!feed.hasFeedName() && loaded.hasFeedName()) feed.setFeedName(loaded.getFeedName()); + if (!feed.hasFeedName() && loaded.hasFeedName()) feed.setFeedName(loaded.getPassthruFeedName()); // add to set if anything was found - if (loaded.hasFqdnList()) recentValues.put(feed.getFeedUrl(), loaded.getFqdnList()); + if (loaded.hasFqdnList()) recentValues.put(feed.getFeedUrl(), loaded.getPassthruFqdnList()); } } for (String val : recentValues.values().stream().flatMap(Collection::stream).collect(Collectors.toSet())) { @@ -197,7 +199,7 @@ public class TlsPassthruConfig { } public TlsPassthruFeed loadFeed(String url) { - final TlsPassthruFeed loaded = new TlsPassthruFeed().setFeedUrl(url); + final TlsPassthruFeed loaded = new TlsPassthruFeed().setPassthruFeedUrl(url); try (final InputStream in = getUrlInputStream(url)) { final List lines = StringUtil.split(IOUtils.toString(in), "\r\n"); final Set fqdnList = new HashSet<>(); @@ -208,7 +210,7 @@ public class TlsPassthruConfig { if (!loaded.hasFeedName() && trimmed.toLowerCase().startsWith(FEED_NAME_PREFIX.toLowerCase())) { final String name = trimmed.substring(FEED_NAME_PREFIX.length()).trim(); if (log.isTraceEnabled()) log.trace("loadFeed("+url+"): setting name="+name+" from special comment: "+trimmed); - loaded.setFeedName(name); + loaded.setPassthruFeedName(name); } else { if (log.isDebugEnabled()) log.debug("loadFeed("+url+"): ignoring comment: "+trimmed); } @@ -216,7 +218,7 @@ public class TlsPassthruConfig { fqdnList.add(trimmed); } } - loaded.setFqdnList(fqdnList); + loaded.setPassthruFqdnList(fqdnList); } catch (Exception e) { reportError("loadFeed("+url+"): "+shortError(e), e); } diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruFeed.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruFeed.java index b8008ca3..621991b9 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruFeed.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruFeed.java @@ -4,41 +4,29 @@ */ package bubble.rule.passthru; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.EqualsAndHashCode; -import lombok.Getter; import lombok.NoArgsConstructor; -import lombok.Setter; import lombok.experimental.Accessors; import java.util.Set; -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; -import static org.cobbzilla.util.security.ShaUtil.sha256_hex; -@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of="feedUrl") -public class TlsPassthruFeed implements Comparable { +@NoArgsConstructor @Accessors(chain=true) +public class TlsPassthruFeed extends BasePassthruFeed { - public static final TlsPassthruFeed[] EMPTY_FEEDS = new TlsPassthruFeed[0]; + public static final TlsPassthruFeed[] EMPTY_PASSTHRU_FEEDS = new TlsPassthruFeed[0]; - public String getId() { return sha256_hex(getFeedUrl()); } - public void setId(String id) {} // noop + public String getPassthruFeedName () { return getFeedName(); } + public TlsPassthruFeed setPassthruFeedName (String name) { return (TlsPassthruFeed) setFeedName(name); } - @Getter @Setter private String feedName; - public boolean hasFeedName() { return !empty(feedName); } + public String getPassthruFeedUrl () { return getFeedUrl(); } + public TlsPassthruFeed setPassthruFeedUrl (String url) { return (TlsPassthruFeed) setFeedUrl(url); } - @Getter @Setter private String feedUrl; + public Set getPassthruFqdnList () { return getFqdnList(); } + public TlsPassthruFeed setPassthruFqdnList (Set set) { return (TlsPassthruFeed) setFqdnList(set); } - @JsonIgnore @Getter @Setter private Set fqdnList; - public boolean hasFqdnList () { return !empty(fqdnList); } - - public TlsPassthruFeed(String url) { setFeedUrl(url); } + public TlsPassthruFeed(String url) { super(url); } public TlsPassthruFeed(TlsPassthruFeed feed) { copy(this, feed); } - @Override public int compareTo(TlsPassthruFeed o) { - return getFeedUrl().toLowerCase().compareTo(o.getFeedUrl().toLowerCase()); - } - } diff --git a/bubble-server/src/main/resources/logback.xml b/bubble-server/src/main/resources/logback.xml index 6f12d789..3b6c9e5e 100644 --- a/bubble-server/src/main/resources/logback.xml +++ b/bubble-server/src/main/resources/logback.xml @@ -42,6 +42,8 @@ + + @@ -57,8 +59,8 @@ - - + + @@ -70,8 +72,6 @@ - - diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json index c90f0df9..c112e5fa 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json @@ -87,11 +87,11 @@ "config": { "passthruFqdnList": [], "passthruFeedList": [{ - "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/tls_passthru.txt" + "passthruFeedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/tls_passthru.txt" }], "flexFqdnList": [], "flexFeedList": [{ - "feedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/flex_routing.txt" + "flexFeedUrl": "https://raw.githubusercontent.com/getbubblenow/bubble-filter-lists/master/flex_routing.txt" }] } }], -- 2.17.1 From a86180c7f45f3a18d9febdbeb9abb8fe05a7d719 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 10 Sep 2020 00:24:28 -0400 Subject: [PATCH 64/78] add log conditional --- bubble-server/src/main/java/bubble/rule/AppRuleDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 99f0bedc..e0290c83 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -82,7 +82,7 @@ public interface AppRuleDriver { redis.rename(tempList, ipList); redis.sadd_plaintext(listOfListsForIp, ipList); final Long count = redis.sunionstore(unionSetName, redis.smembers(listOfListsForIp)); - log.debug("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count); + if (log.isDebugEnabled()) log.debug("defineRedisSet("+ip+","+listOfListsName+","+listName+"): unionSetName="+unionSetName+" size="+count); } static boolean isFlexRouteFqdn(RedisService redis, String ip, String[] fqdns) { -- 2.17.1 From 60e256275203f7709a0898e0b5e24c7ac15d92fd Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 10 Sep 2020 05:20:52 -0400 Subject: [PATCH 65/78] WIP. passthru + flex routing works. code cleanup is next --- .../bubble/service/device/FlexRouterInfo.java | 2 +- .../roles/mitmproxy/files/bubble_api.py | 10 +- .../mitmproxy/files/bubble_conn_check.py | 96 ++++++++------ .../mitmproxy/files/bubble_flex_passthru.py | 123 ++++++++++++++++++ .../roles/mitmproxy/files/bubble_request.py | 10 +- .../packer/roles/mitmproxy/files/run_mitm.sh | 2 +- 6 files changed, 198 insertions(+), 45 deletions(-) create mode 100644 bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java index a83727fe..a8e3f2c8 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java @@ -29,8 +29,8 @@ public class FlexRouterInfo { } @JsonIgnore public String getVpnIp () { return router.getIp(); } - @JsonIgnore public int getPort () { return router.getPort(); } + public int getPort () { return router.getPort(); } public String getProxyUrl () { return router.proxyBaseUri(); } public boolean hasGeoLocation () { return hasDeviceStatus() && deviceStatus.getLocation() != null && deviceStatus.getLocation().hasLatLon(); } diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 065a0f6a..af29259c 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -139,7 +139,7 @@ def bubble_get_flex_router(client_addr): bubble_log.debug('bubble_get_flex_routes: no router found for '+client_addr) else: if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_get_flex_routes: API call failed with status: '+response.status_code) + bubble_log.error('bubble_get_flex_routes: API call failed with HTTP status: '+str(response.status_code)) return None except Exception as e: @@ -284,6 +284,14 @@ def is_flex_domain(client_addr, fqdn): return False +def original_flex_ip(client_addr, fqdns): + for fqdn in fqdns: + ip = REDIS.get("flexOriginal~"+client_addr+"~"+fqdn) + if ip is not None: + return ip.decode() + return None + + def health_check_response(flow): #if bubble_log.isEnabledFor(DEBUG): # bubble_log.debug('health_check_response: special bubble health check request, responding with OK') diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index a56211f4..65e5f867 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -33,8 +33,9 @@ from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL import traceback from bubble_api import bubble_conn_check, bubble_activity_log, REDIS, redis_set, \ - is_bubble_request, is_sage_request, is_not_from_vpn + is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, bubble_get_flex_router, original_flex_ip from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host +from bubble_flex_passthru import BubbleFlexPassthruLayer bubble_log = logging.getLogger(__name__) @@ -182,36 +183,26 @@ def check_bubble_connection(client_addr, server_addr, fqdns, security_level): if security_level['level'] == SEC_MAX: if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', security_level=maximum, returning Block') - return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'block': True, 'reason': 'bubble_error'} + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_error'} else: if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') - return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': False, 'reason': 'bubble_error'} + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_error'} elif check_response == 'passthru': if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') - return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': False, 'reason': 'bubble_passthru'} + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'reason': 'bubble_passthru'} elif check_response == 'block': if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning Block') - return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'block': True, 'reason': 'bubble_block'} - - elif check_response == 'passthru_flex': - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') - return {'fqdns': fqdns, 'addr': server_addr, 'passthru': True, 'flex': True, 'reason': 'bubble_passthru_flex'} - - elif check_response == 'noop_flex': - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning True') - return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': True, 'reason': 'bubble_no_passthru_flex'} + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'block': True, 'reason': 'bubble_block'} else: if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('check_bubble_connection: bubble API returned ' + str(check_response) +' for FQDN/addr ' + str(fqdns) +'/' + str(server_addr) + ', returning False') - return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'flex': False, 'reason': 'bubble_no_passthru'} + return {'fqdns': fqdns, 'addr': server_addr, 'passthru': False, 'reason': 'bubble_no_passthru'} def check_connection(client_addr, server_addr, fqdns, security_level): @@ -240,28 +231,50 @@ def check_connection(client_addr, server_addr, fqdns, security_level): return check_response -def next_layer(next_layer): - if isinstance(next_layer, TlsLayer) and next_layer._client_tls: - client_hello = net_tls.ClientHello.from_file(next_layer.client_conn.rfile) - client_addr = next_layer.client_conn.address[0] - server_addr = next_layer.server_conn.address[0] +def check_passthru_flex(client_addr, server_addr, fqdns): + if fqdns: + for fqdn in fqdns: + if is_flex_domain(client_addr, fqdn): + return True + else: + return is_flex_domain(client_addr, server_addr) + + +def passthru_flex_port(client_addr, fqdns): + router = bubble_get_flex_router(client_addr) + if router is None or 'auth' not in router: + if bubble_log.isEnabledFor(INFO): + bubble_log.info('apply_passthru_flex: no flex router for fqdn(s): '+repr(fqdns)) + elif 'port' in router: + return router['port'] + else: + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('apply_passthru_flex: flex router found but has no port ('+repr(router)+') for fqdn(s): '+repr(fqdns)) + return None + + +def next_layer(layer): + if isinstance(layer, TlsLayer) and layer._client_tls: + client_hello = net_tls.ClientHello.from_file(layer.client_conn.rfile) + client_addr = layer.client_conn.address[0] + server_addr = layer.server_conn.address[0] if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('next_layer: STARTING: client='+ client_addr+' server='+server_addr) if client_hello.sni: fqdn = client_hello.sni.decode() if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('next_layer: using fqdn in SNI: '+ fqdn) - fqdns = [ fqdn ] + fqdns = [fqdn] else: fqdns = fqdns_for_addr(server_addr) if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('next_layer: NO fqdn in sni, using fqdns from DNS: '+ str(fqdns)) - next_layer.fqdns = fqdns + layer.fqdns = fqdns no_fqdns = fqdns is None or len(fqdns) == 0 security_level = get_device_security_level(client_addr, fqdns) - next_layer.security_level = security_level - next_layer.do_block = False - called_check_api = False + layer.security_level = security_level + layer.do_block = False + check_for_flex = False if is_bubble_request(server_addr, fqdns): if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns)) @@ -277,7 +290,7 @@ def next_layer(next_layer): if bubble_log.isEnabledFor(WARNING): bubble_log.warning('next_layer: enabling block for non-VPN client='+client_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'conn_block_non_vpn', fqdns) - next_layer.__class__ = TlsBlock + layer.__class__ = TlsBlock return elif security_level['level'] == SEC_OFF: @@ -304,34 +317,43 @@ def next_layer(next_layer): if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: calling check_connection for server='+server_addr+', fqdns='+str(fqdns)+', client='+client_addr+' with security_level='+repr(security_level)) check = check_connection(client_addr, server_addr, fqdns, security_level) - called_check_api = True - if check is None or ('passthru' in check and check['passthru'] and ('flex' not in check or not check['flex'])): + if check is None or ('passthru' in check and check['passthru']): if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) - next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) - next_layer.reply.send(next_layer_replacement) + flex_port = None + if check_passthru_flex(client_addr, server_addr, fqdns): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('next_layer: applying flex passthru for server=' + server_addr+', fqdns='+str(fqdns)) + flex_port = passthru_flex_port(client_addr, fqdns) + if flex_port: + layer_replacement = BubbleFlexPassthruLayer(layer.ctx, ('127.0.0.1', flex_port), fqdns[0], 443) + layer.reply.send(layer_replacement) + if flex_port is None: + layer_replacement = RawTCPLayer(layer.ctx, ignore=True) + layer.reply.send(layer_replacement) elif 'block' in check and check['block']: if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: enabling block for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'conn_block', fqdns) if show_block_stats(client_addr, fqdns) and security_level['level'] != SEC_BASIC: - next_layer.do_block = True - next_layer.__class__ = TlsFeedback + layer.do_block = True + layer.__class__ = TlsFeedback else: - next_layer.__class__ = TlsBlock + layer.__class__ = TlsBlock elif security_level['level'] == SEC_BASIC: if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) - next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) - next_layer.reply.send(next_layer_replacement) + # todo + layer_replacement = RawTCPLayer(layer.ctx, ignore=True) + layer.reply.send(layer_replacement) else: if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: disabling passthru (with TlsFeedback) for client_addr='+client_addr+', server_addr='+server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_intercept', fqdns) - next_layer.__class__ = TlsFeedback + layer.__class__ = TlsFeedback diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py new file mode 100644 index 00000000..2c661f5a --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py @@ -0,0 +1,123 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Parts of this are borrowed from rawtcp.py in the mitmproxy project. The mitmproxy license is reprinted here: +# +# Copyright (c) 2013, Aldo Cortesi. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +import socket + +from OpenSSL import SSL + +import mitmproxy.net.tcp +from mitmproxy.exceptions import MitmproxyException +from mitmproxy import tcp +from mitmproxy import flow +from mitmproxy import exceptions +from mitmproxy.proxy.protocol import base +from mitmproxy.connections import ServerConnection +from mitmproxy.http import make_connect_request +from mitmproxy.net.http.http1 import assemble_request, read_response + +import traceback +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + +bubble_log = logging.getLogger(__name__) + + +class BubbleFlexPassthruException(MitmproxyException): + pass + + +class BubbleFlexPassthruLayer(base.Layer): + chunk_size = 4096 + proxy_addr = None + host = None + port = None + + def __init__(self, ctx, proxy_addr, host, port): + bubble_log.info('__init__ called with ctx='+repr(ctx)+' and ctx.server_conn=('+repr(ctx.server_conn)+') and proxy_addr='+repr(proxy_addr)) + self.ignore = True + self.proxy_addr = proxy_addr + self.server_conn = ServerConnection(proxy_addr) + self.host = host + self.port = port + ctx.server_conn = self.server_conn + super().__init__(ctx) + bubble_log.info('__init__ finished, self.server_conn='+repr(self.server_conn)) + + def __call__(self): + bubble_log.info('__call__ starting, self.server_conn='+repr(self.server_conn)) + self.connect() + client = self.client_conn.connection + server = self.server_conn.connection + + buf = memoryview(bytearray(self.chunk_size)) + + connect_req = make_connect_request((self.host, self.port)) + server.send(assemble_request(connect_req)) + resp = server.recv(1024).decode() + if not resp.startswith('HTTP/1.1 200 OK'): + raise BubbleFlexPassthruException('CONNECT request error: '+resp) + + conns = [client, server] + + # https://github.com/openssl/openssl/issues/6234 + for conn in conns: + if isinstance(conn, SSL.Connection) and hasattr(SSL._lib, "SSL_clear_mode"): + SSL._lib.SSL_clear_mode(conn._ssl, SSL._lib.SSL_MODE_AUTO_RETRY) + + try: + while not self.channel.should_exit.is_set(): + r = mitmproxy.net.tcp.ssl_read_select(conns, 10) + for conn in r: + dst = server if conn == client else client + try: + size = conn.recv_into(buf, self.chunk_size) + except (SSL.WantReadError, SSL.WantWriteError): + continue + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + tcp_message = tcp.TCPMessage(dst == server, buf[:size].tobytes()) + dst.sendall(tcp_message.content) + + except (socket.error, exceptions.TcpException, SSL.Error) as e: + just_the_string = traceback.format_exc() + bubble_log.info('~~~~~~~~~~~~~~~~~~~~~~~~~~~~ __call__ exception: '+repr(e)+' from '+just_the_string) + if not self.ignore: + f.error = flow.Error("TCP connection closed unexpectedly: {}".format(repr(e))) + self.channel.tell("tcp_error", f) + finally: + if not self.ignore: + self.channel.tell("tcp_end", f) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 4abb6f77..2f918a84 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -252,19 +252,19 @@ class Rerouter: return host def request(self, flow): - flex_host = self.bubble_handle_request(flow) + host = self.bubble_handle_request(flow) path = flow.request.path if is_bubble_special_path(path): #if bubble_log.isEnabledFor(DEBUG): # bubble_log.debug('request: is_bubble_special_path('+path+') returned true, sending special bubble response') special_bubble_response(flow) - elif flex_host is not None: + elif host is not None: client_addr = flow.client_conn.address[0] - if is_flex_domain(client_addr, flex_host): + if is_flex_domain(client_addr, host): if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: is_flex_domain('+flex_host+') returned true, sending flex response') - set_flex_response(client_addr, flex_host, flow) + bubble_log.debug('request: is_flex_domain('+host+') returned true, sending flex response') + set_flex_response(client_addr, host, flow) addons = [Rerouter()] diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh index 6ad5747a..f2eae5cb 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh @@ -23,7 +23,7 @@ BUBBLE_PORT=${PORT} mitmdump \ --set block_private=false \ --set termlog_verbosity=warn \ --set flow_detail=0 \ - --set stream_large_bodies=5m \ + --set stream_large_bodies=1 \ --set keep_host_header \ -s ./bubble_debug.py \ -s ./bubble_conn_check.py \ -- 2.17.1 From db25825db58c3f5348b065919adc153ffc038b96 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 10 Sep 2020 05:45:37 -0400 Subject: [PATCH 66/78] clean up mitm code --- .../roles/mitmproxy/files/bubble_api.py | 10 ++--- .../mitmproxy/files/bubble_conn_check.py | 37 ++++++++++--------- .../roles/mitmproxy/files/bubble_flex.py | 1 - .../mitmproxy/files/bubble_flex_passthru.py | 21 +++-------- .../roles/mitmproxy/files/bubble_modify.py | 27 +++++++------- .../roles/mitmproxy/files/bubble_request.py | 12 +++--- 6 files changed, 49 insertions(+), 59 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index af29259c..b0be50d0 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -19,7 +19,7 @@ from http import HTTPStatus from netaddr import IPAddress, IPNetwork from bubble_vpn4 import wireguard_network_ipv4 from bubble_vpn6 import wireguard_network_ipv6 -from bubble_config import bubble_network, bubble_port, debug_capture_fqdn, \ +from bubble_config import bubble_port, debug_capture_fqdn, \ bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host bubble_log = logging.getLogger(__name__) @@ -293,8 +293,8 @@ def original_flex_ip(client_addr, fqdns): def health_check_response(flow): - #if bubble_log.isEnabledFor(DEBUG): - # bubble_log.debug('health_check_response: special bubble health check request, responding with OK') + # if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('health_check_response: special bubble health check request, responding with OK') response_headers = nheaders.Headers() response_headers[HEADER_HEALTH_CHECK] = 'OK' response_headers[HEADER_CONTENT_LENGTH] = '3' @@ -350,8 +350,8 @@ def special_bubble_response(flow): headers=response_headers, content=None) if response is not None: - #if bubble_log.isEnabledFor(DEBUG): - # bubble_log.debug('special_bubble_response: special bubble request: response status = '+str(response.status_code)) + # if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('special_bubble_response: special bubble request: response status = '+str(response.status_code)) flow.response.headers = collect_response_headers(response) flow.response.status_code = response.status_code flow.response.reason = status_reason(response.status_code) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py index 65e5f867..bfb9a410 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_conn_check.py @@ -33,8 +33,8 @@ from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL import traceback from bubble_api import bubble_conn_check, bubble_activity_log, REDIS, redis_set, \ - is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, bubble_get_flex_router, original_flex_ip -from bubble_config import bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host + is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain, bubble_get_flex_router +from bubble_config import bubble_host, bubble_host_alias, cert_validation_host from bubble_flex_passthru import BubbleFlexPassthruLayer bubble_log = logging.getLogger(__name__) @@ -99,7 +99,7 @@ def fqdns_for_addr(server_addr): prefix = REDIS_DNS_PREFIX + server_addr keys = REDIS.keys(prefix + '_*') if keys is None or len(keys) == 0: - if (bubble_log.isEnabledFor(DEBUG)): + if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('fqdns_for_addr: no FQDN found for addr '+str(server_addr)+', checking raw addr') return '' fqdns = [] @@ -253,6 +253,20 @@ def passthru_flex_port(client_addr, fqdns): return None +def do_passthru(client_addr, server_addr, fqdns, layer): + flex_port = None + if check_passthru_flex(client_addr, server_addr, fqdns): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('do_passthru: applying flex passthru for server=' + server_addr + ', fqdns=' + str(fqdns)) + flex_port = passthru_flex_port(client_addr, fqdns) + if flex_port: + layer_replacement = BubbleFlexPassthruLayer(layer.ctx, ('127.0.0.1', flex_port), fqdns[0], 443) + layer.reply.send(layer_replacement) + if flex_port is None: + layer_replacement = RawTCPLayer(layer.ctx, ignore=True) + layer.reply.send(layer_replacement) + + def next_layer(layer): if isinstance(layer, TlsLayer) and layer._client_tls: client_hello = net_tls.ClientHello.from_file(layer.client_conn.rfile) @@ -274,7 +288,6 @@ def next_layer(layer): security_level = get_device_security_level(client_addr, fqdns) layer.security_level = security_level layer.do_block = False - check_for_flex = False if is_bubble_request(server_addr, fqdns): if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: enabling passthru for LOCAL bubble='+server_addr+' (bubble_host ('+bubble_host+') in fqdns or bubble_host_alias ('+bubble_host_alias+') in fqdns) regardless of security_level='+repr(security_level)+' for client='+client_addr+', fqdns='+repr(fqdns)) @@ -322,17 +335,7 @@ def next_layer(layer): if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) - flex_port = None - if check_passthru_flex(client_addr, server_addr, fqdns): - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('next_layer: applying flex passthru for server=' + server_addr+', fqdns='+str(fqdns)) - flex_port = passthru_flex_port(client_addr, fqdns) - if flex_port: - layer_replacement = BubbleFlexPassthruLayer(layer.ctx, ('127.0.0.1', flex_port), fqdns[0], 443) - layer.reply.send(layer_replacement) - if flex_port is None: - layer_replacement = RawTCPLayer(layer.ctx, ignore=True) - layer.reply.send(layer_replacement) + do_passthru(client_addr, server_addr, fqdns, layer) elif 'block' in check and check['block']: if bubble_log.isEnabledFor(INFO): @@ -348,9 +351,7 @@ def next_layer(layer): if bubble_log.isEnabledFor(INFO): bubble_log.info('next_layer: check='+repr(check)+' but security_level='+repr(security_level)+', enabling passthru for server=' + server_addr+', fqdns='+str(fqdns)) bubble_activity_log(client_addr, server_addr, 'tls_passthru', fqdns) - # todo - layer_replacement = RawTCPLayer(layer.ctx, ignore=True) - layer.reply.send(layer_replacement) + do_passthru(client_addr, server_addr, fqdns, layer) else: if bubble_log.isEnabledFor(INFO): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index 866bd0e5..4a76d20d 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -3,7 +3,6 @@ # from mitmproxy import http from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, \ - get_flow_ctx, add_flow_ctx, \ HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_FILTERED from bubble_modify import bubble_filter_response diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py index 2c661f5a..cf28bbae 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex_passthru.py @@ -27,17 +27,15 @@ import socket from OpenSSL import SSL -import mitmproxy.net.tcp from mitmproxy.exceptions import MitmproxyException from mitmproxy import tcp -from mitmproxy import flow from mitmproxy import exceptions from mitmproxy.proxy.protocol import base from mitmproxy.connections import ServerConnection from mitmproxy.http import make_connect_request -from mitmproxy.net.http.http1 import assemble_request, read_response +from mitmproxy.net.http.http1 import assemble_request +from mitmproxy.net.tcp import ssl_read_select -import traceback import logging from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL @@ -55,7 +53,6 @@ class BubbleFlexPassthruLayer(base.Layer): port = None def __init__(self, ctx, proxy_addr, host, port): - bubble_log.info('__init__ called with ctx='+repr(ctx)+' and ctx.server_conn=('+repr(ctx.server_conn)+') and proxy_addr='+repr(proxy_addr)) self.ignore = True self.proxy_addr = proxy_addr self.server_conn = ServerConnection(proxy_addr) @@ -63,16 +60,15 @@ class BubbleFlexPassthruLayer(base.Layer): self.port = port ctx.server_conn = self.server_conn super().__init__(ctx) - bubble_log.info('__init__ finished, self.server_conn='+repr(self.server_conn)) def __call__(self): - bubble_log.info('__call__ starting, self.server_conn='+repr(self.server_conn)) self.connect() client = self.client_conn.connection server = self.server_conn.connection buf = memoryview(bytearray(self.chunk_size)) + # send CONNECT, expect 200 OK connect_req = make_connect_request((self.host, self.port)) server.send(assemble_request(connect_req)) resp = server.recv(1024).decode() @@ -88,7 +84,7 @@ class BubbleFlexPassthruLayer(base.Layer): try: while not self.channel.should_exit.is_set(): - r = mitmproxy.net.tcp.ssl_read_select(conns, 10) + r = ssl_read_select(conns, 10) for conn in r: dst = server if conn == client else client try: @@ -113,11 +109,4 @@ class BubbleFlexPassthruLayer(base.Layer): dst.sendall(tcp_message.content) except (socket.error, exceptions.TcpException, SSL.Error) as e: - just_the_string = traceback.format_exc() - bubble_log.info('~~~~~~~~~~~~~~~~~~~~~~~~~~~~ __call__ exception: '+repr(e)+' from '+just_the_string) - if not self.ignore: - f.error = flow.Error("TCP connection closed unexpectedly: {}".format(repr(e))) - self.channel.tell("tcp_error", f) - finally: - if not self.ignore: - self.channel.tell("tcp_end", f) + bubble_log.error('exception: '+repr(e)) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index eadbcd6e..3c47e013 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -2,20 +2,23 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # import json -import logging -from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL import re import requests import urllib import traceback from mitmproxy.net.http import Headers -from bubble_config import bubble_port, bubble_host_alias, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri +from bubble_config import bubble_port, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ - HEADER_HEALTH_CHECK, HEALTH_CHECK_URI, status_reason, get_flow_ctx, add_flow_ctx, \ - is_bubble_special_path, is_bubble_health_check, health_check_response, make_bubble_special_path, special_bubble_response, \ + status_reason, get_flow_ctx, add_flow_ctx, \ + is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \ - HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \ - HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set, parse_host_header + HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \ + HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set + +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + +bubble_log = logging.getLogger(__name__) BUFFER_SIZE = 4096 CONTENT_TYPE_BINARY = 'application/octet-stream' @@ -27,8 +30,6 @@ REDIS_FILTER_PASSTHRU_DURATION = 600 DEBUG_STREAM_COUNTERS = {} MIN_FILTER_CHUNK_SIZE = 16384 -bubble_log = logging.getLogger(__name__) - def add_csp_part(new_csp, part): if len(new_csp) > 0: @@ -282,10 +283,10 @@ def bubble_filter_response(flow, flex_stream): any_content_type_matches = False for m in matchers: if 'contentTypeRegex' in m: - typeRegex = m['contentTypeRegex'] - if typeRegex is None: - typeRegex = '^text/html.*' - if re.match(typeRegex, content_type): + type_regex = m['contentTypeRegex'] + if type_regex is None: + type_regex = '^text/html.*' + if re.match(type_regex, content_type): any_content_type_matches = True if bubble_log.isEnabledFor(DEBUG): bubble_log.debug(prefix+'found at least one matcher for content_type ('+content_type+'), filtering: '+path) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 2f918a84..cc675982 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -25,21 +25,21 @@ # import re -import logging -from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL - import time import uuid from mitmproxy.net.http import headers as nheaders from bubble_api import bubble_matchers, bubble_activity_log, \ HEALTH_CHECK_URI, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ - CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, get_flow_ctx, parse_host_header, \ + CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ is_bubble_special_path, special_bubble_response, is_bubble_health_check, \ is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain from bubble_config import bubble_host, bubble_host_alias from bubble_flex import set_flex_response +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + bubble_log = logging.getLogger(__name__) @@ -255,8 +255,8 @@ class Rerouter: host = self.bubble_handle_request(flow) path = flow.request.path if is_bubble_special_path(path): - #if bubble_log.isEnabledFor(DEBUG): - # bubble_log.debug('request: is_bubble_special_path('+path+') returned true, sending special bubble response') + # if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('request: is_bubble_special_path('+path+') returned true, sending special bubble response') special_bubble_response(flow) elif host is not None: -- 2.17.1 From 1f61c8cec16e120b3be85774867e4d403bf70e1e Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 10 Sep 2020 05:51:24 -0400 Subject: [PATCH 67/78] update algo and mitm hashes --- .../src/main/resources/packer/roles/algo/tasks/main.yml | 2 +- .../src/main/resources/packer/roles/mitmproxy/tasks/main.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index e6a53d84..ac83c14b 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:9ce1c0df1075e9ba23585daa897f4f05ca90f9f99e8b1d208427aa11dc62fcdc + checksum: sha256:895d28911907d8f7f79cca6b70a6eda6ca4c892553cd02c0fd95060f392970a3 - name: Unzip algo master.zip unarchive: diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml index a218b8bc..afe93dc1 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml @@ -41,7 +41,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/mitmproxy/mitmproxy.zip dest: /tmp/mitmproxy.zip - checksum: sha256:b288b55e0b25e453fbc08ec2ad130ea22033f200cab1319e7bc8d2332e538ec5 + checksum: sha256:9883696cc304326d0d94f6d0a498721de9b8c77c833b30f4b1661646467172bf - name: Unzip mitmproxy.zip unarchive: @@ -62,6 +62,8 @@ - bubble_request.py - bubble_conn_check.py - bubble_modify.py + - bubble_flex.py + - bubble_flex_passthru.py - run_mitm.sh - name: Install cert helper scripts -- 2.17.1 From 81a6c63cef5f8124965ede2868b77c50085e1a5d Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 10 Sep 2020 06:17:23 -0400 Subject: [PATCH 68/78] allow flex POST with no content, extend flex timeout --- .../main/resources/packer/roles/mitmproxy/files/bubble_api.py | 4 +++- .../resources/packer/roles/mitmproxy/files/bubble_flex.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index b0be50d0..cecf45f8 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -330,8 +330,10 @@ def special_bubble_response(flow): elif flow.request.method == 'POST': if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('special_bubble_response: special bubble request: POST content is '+str(flow.request.content)) - headers['Content-Length'] = str(len(flow.request.content)) + if flow.request.content: + headers['Content-Length'] = str(len(flow.request.content)) response = requests.post(uri, data=flow.request.content, headers=headers, stream=True) + else: if bubble_log.isEnabledFor(WARNING): bubble_log.warning('special_bubble_response: special bubble request: method '+flow.request.method+' not supported') diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index 4a76d20d..95d72968 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -75,7 +75,7 @@ def process_flex(flex_host, flow, router): try: response = requests.request(method, url, headers=request_headers, - timeout=(15, 15), + timeout=(20, 120), stream=True, data=request_body, # use the original request body, if there is one proxies=proxies, -- 2.17.1 From 5d3fd4029e51847ce30bd05f1912ec9ed339a888 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 04:11:49 -0400 Subject: [PATCH 69/78] WIP. use async io, fix support for flex requests with bodies like POST/PUT --- .../roles/mitmproxy/files/bubble_api.py | 220 ++++++++++++------ .../roles/mitmproxy/files/bubble_flex.py | 152 +++++++----- .../roles/mitmproxy/files/bubble_modify.py | 51 ++-- .../roles/mitmproxy/files/bubble_request.py | 12 +- 4 files changed, 285 insertions(+), 150 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index cecf45f8..6757ab43 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -1,29 +1,33 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # -import logging -from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL - -from mitmproxy import http -from mitmproxy.net.http import headers as nheaders - +import asyncio import json +import logging import re -import requests -import redis import subprocess import time import traceback import uuid from http import HTTPStatus -from netaddr import IPAddress, IPNetwork +from logging import INFO, DEBUG, WARNING, ERROR + +import httpx +import nest_asyncio +import redis from bubble_vpn4 import wireguard_network_ipv4 from bubble_vpn6 import wireguard_network_ipv6 +from netaddr import IPAddress, IPNetwork + from bubble_config import bubble_port, debug_capture_fqdn, \ - bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6, cert_validation_host + bubble_host, bubble_host_alias, bubble_sage_host, bubble_sage_ip4, bubble_sage_ip6 +from mitmproxy import http +from mitmproxy.net.http import headers as nheaders bubble_log = logging.getLogger(__name__) +nest_asyncio.apply() + HEADER_USER_AGENT = 'User-Agent' HEADER_CONTENT_LENGTH = 'Content-Length' HEADER_CONTENT_TYPE = 'Content-Type' @@ -43,6 +47,7 @@ CTX_BUBBLE_REQUEST_ID = 'X-Bubble-RequestId' CTX_CONTENT_LENGTH = 'X-Bubble-Content-Length' CTX_CONTENT_LENGTH_SENT = 'X-Bubble-Content-Length-Sent' CTX_BUBBLE_FILTERED = 'X-Bubble-Filtered' +CTX_BUBBLE_FLEX = 'X-Bubble-Flex' BUBBLE_URI_PREFIX = '/__bubble/' HEADER_HEALTH_CHECK = 'X-Mitm-Health' @@ -90,6 +95,107 @@ def bubble_activity_log(client_addr, server_addr, event, data): pass +def async_client(proxies=None, + timeout=5, + max_redirects=0): + return httpx.AsyncClient(timeout=timeout, max_redirects=max_redirects, proxies=proxies) + + +async def async_response(client, name, url, + headers=None, + method='GET', + data=None, + json=None): + if bubble_log.isEnabledFor(INFO): + bubble_log.info('bubble_async_request(' + name + '): starting async: ' + method + ' ' + url) + + response = await client.request(method=method, url=url, headers=headers, json=json, data=data) + + if bubble_log.isEnabledFor(INFO): + bubble_log.info('bubble_async_request(' + name + '): async request returned HTTP status ' + str(response.status_code)) + + if response.status_code != 200: + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('bubble_async_request(' + name + '): API call failed: ' + repr(response)) + + return response + + +def async_stream(client, name, url, + headers=None, + method='GET', + data=None, + json=None, + timeout=5, + max_redirects=0, + loop=asyncio.get_running_loop()): + try: + return loop.run_until_complete(_async_stream(client, name, url, + headers=headers, + method=method, + data=data, + json=json, + timeout=timeout, + max_redirects=max_redirects)) + except Exception as e: + bubble_log.error('async_stream('+name+'): error with url='+url+' -- '+repr(e)) + + +async def _async_stream(client, name, url, + headers=None, + method='GET', + data=None, + json=None, + timeout=5, + max_redirects=0): + request = client.build_request(method=method, url=url, headers=headers, json=json, data=data) + return await client.send(request, stream=True, allow_redirects=(max_redirects > 0), timeout=timeout) + + +async def _bubble_async(name, url, + headers=None, + method='GET', + data=None, + json=None, + proxies=None, + timeout=5, + max_redirects=0): + async with async_client(proxies=proxies, timeout=timeout, max_redirects=max_redirects) as client: + return await async_response(client, name, url, headers=headers, method=method, data=data, json=json) + + +def bubble_async(name, url, + headers=None, + method='GET', + data=None, + json=None, + proxies=None, + timeout=5, + max_redirects=0, + loop=asyncio.get_running_loop()): + try: + return loop.run_until_complete(_bubble_async(name, url, + headers=headers, + method=method, + data=data, + json=json, + proxies=proxies, + timeout=timeout, + max_redirects=max_redirects)) + except Exception as e: + bubble_log.error('bubble_async('+name+'): error: '+repr(e)) + + +def bubble_async_request_json(name, url, headers, method='GET', json=None): + response = bubble_async(name, url, headers, method=method, json=json) + if response and response.status_code == 200: + return response.json() + else: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_async_request_json('+name+'): received invalid HTTP status: '+str(response.status_code)) + return None + + def bubble_conn_check(client_addr, server_addr, fqdns, security_level): if debug_capture_fqdn and fqdns: for f in debug_capture_fqdn: @@ -98,27 +204,24 @@ def bubble_conn_check(client_addr, server_addr, fqdns, security_level): bubble_log.debug('bubble_conn_check: debug_capture_fqdn detected, returning noop: '+f) return 'noop' + name = 'bubble_conn_check' + url = 'http://127.0.0.1:'+bubble_port+'/api/filter/check' headers = { 'X-Forwarded-For': client_addr, 'Accept': 'application/json', 'Content-Type': 'application/json' } + data = { + 'serverAddr': str(server_addr), + 'fqdns': fqdns, + 'clientAddr': client_addr + } try: - data = { - 'serverAddr': str(server_addr), - 'fqdns': fqdns, - 'clientAddr': client_addr - } - response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/check', headers=headers, json=data) - if response.ok: - return response.json() - if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_conn_check API call failed: '+repr(response)) - return None + return bubble_async_request_json(name, url, headers, method='POST', json=data) except Exception as e: if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_conn_check API call failed: '+repr(e)) + bubble_log.error('bubble_conn_check: API call failed: '+repr(e)) traceback.print_exc() if security_level is not None and security_level['level'] == 'maximum': return False @@ -126,21 +229,14 @@ def bubble_conn_check(client_addr, server_addr, fqdns, security_level): def bubble_get_flex_router(client_addr): + name = 'bubble_get_flex_router' + url = 'http://127.0.0.1:' + bubble_port + '/api/filter/flexRouters' headers = { 'X-Forwarded-For': client_addr, 'Accept': 'application/json' } try: - response = requests.get('http://127.0.0.1:'+bubble_port+'/api/filter/flexRouters', headers=headers) - if response.ok: - return response.json() - elif response.status_code == 404: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_get_flex_routes: no router found for '+client_addr) - else: - if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_get_flex_routes: API call failed with HTTP status: '+str(response.status_code)) - return None + return bubble_async_request_json(name, url, headers) except Exception as e: if bubble_log.isEnabledFor(ERROR): @@ -176,6 +272,8 @@ def bubble_matchers(req_id, client_addr, server_addr, flow, host): bubble_log.info('bubble_matchers: debug_capture_fqdn detected, returning DEBUG_MATCHER: '+host) return DEBUG_MATCHER + name = 'bubble_matchers' + url = 'http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id headers = { 'X-Forwarded-For': client_addr, 'Accept': 'application/json', @@ -200,28 +298,29 @@ def bubble_matchers(req_id, client_addr, server_addr, flow, host): bubble_log.warning('bubble_matchers: error parsing Referer header: '+repr(e)) referer = 'NONE' + data = { + 'requestId': req_id, + 'fqdn': host, + 'uri': flow.request.path, + 'userAgent': user_agent, + 'referer': referer, + 'clientAddr': client_addr, + 'serverAddr': server_addr + } + try: - data = { - 'requestId': req_id, - 'fqdn': host, - 'uri': flow.request.path, - 'userAgent': user_agent, - 'referer': referer, - 'clientAddr': client_addr, - 'serverAddr': server_addr - } - response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data) - if response.ok: + response = bubble_async(name, url, headers=headers, method='POST', json=data) + if response.status_code == 200: return response.json() elif response.status_code == 403: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('bubble_matchers response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text)) + bubble_log.debug('bubble_matchers: response was FORBIDDEN, returning block: '+str(response.status_code)+' / '+repr(response.text)) return BLOCK_MATCHER if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) + bubble_log.warning('bubble_matchers: response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) except Exception as e: if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_matchers API call failed: '+repr(e)) + bubble_log.error('bubble_matchers: API call failed: '+repr(e)) traceback.print_exc() return None @@ -313,6 +412,7 @@ def health_check_response(flow): def special_bubble_response(flow): + name = 'special_bubble_response' path = flow.request.path if is_bubble_health_check(path): health_check_response(flow) @@ -324,28 +424,24 @@ def special_bubble_response(flow): 'Accept': 'application/json', 'Content-Type': 'application/json' } - response = None if flow.request.method == 'GET': - response = requests.get(uri, headers=headers, stream=True) + response = bubble_async(name, uri, headers=headers) + elif flow.request.method == 'POST': if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('special_bubble_response: special bubble request: POST content is '+str(flow.request.content)) if flow.request.content: headers['Content-Length'] = str(len(flow.request.content)) - response = requests.post(uri, data=flow.request.content, headers=headers, stream=True) + response = bubble_async(name, uri, json=flow.request.content, headers=headers) else: if bubble_log.isEnabledFor(WARNING): bubble_log.warning('special_bubble_response: special bubble request: method '+flow.request.method+' not supported') - if flow.response is None: - http_version = get_http_version(response) - if http_version is None: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('special_bubble_response: invalid HTTP version, bailing out') - return + return + if flow.response is None: + http_version = response.http_version response_headers = collect_response_headers(response) - flow.response = http.HTTPResponse(http_version=http_version, status_code=response.status_code, reason=response.reason, @@ -365,18 +461,6 @@ def send_bubble_response(response): yield chunk -def get_http_version(response): - raw_version = response.raw.version - if raw_version == 10: - return 'HTTP/1.0' - elif raw_version == 11: - return 'HTTP/1.1' - else: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('get_http_version: invalid HTTP version detected, response.raw.version=='+repr(raw_version)) - return None - - def collect_response_headers(response, omit=None): response_headers = nheaders.Headers() for name in response.headers: diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index 95d72968..cf12304b 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -1,54 +1,66 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # +import asyncio + +from httpx._types import RequestData + +from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody + from mitmproxy import http -from bubble_api import bubble_get_flex_router, get_http_version, collect_response_headers, \ - HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_FILTERED -from bubble_modify import bubble_filter_response +from mitmproxy.proxy.protocol.request_capture import RequestCapture -import requests +from bubble_api import bubble_get_flex_router, collect_response_headers, bubble_async, async_client, async_stream, \ + HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH import logging from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + bubble_log = logging.getLogger(__name__) FLEX_TIMEOUT = 20 -def prepend_remainder_to_stream(remainder, raw): - first = True - while True: - chunk = raw.read(8192) - if first and chunk and len(remainder) > 0: - yield remainder + chunk - elif chunk: - yield chunk - else: - break - if first: - first = False +class FlexFlow(RequestCapture): + flex_host: None + mitm_flow: None + router: None + request_chunks: None + def __init__(self, flex_host, mitm_flow, router): + super().__init__() + self.flex_host = flex_host + self.mitm_flow = mitm_flow + self.router = router + mitm_flow.request.stream = self + mitm_flow.response = http.HTTPResponse(http_version='HTTP/1.1', + status_code=523, + reason='FlexFlow Not Initialized', + headers={}, + content=None) -def set_flex_response(client_addr, flex_host, flow): - if bubble_log.isEnabledFor(INFO): - bubble_log.info('set_flex_response: checking for flex router for host: '+flex_host) + def capture(self, chunks): + self.request_chunks = chunks + + +def new_flex_flow(client_addr, flex_host, flow): router = bubble_get_flex_router(client_addr) if router is None or 'auth' not in router: if bubble_log.isEnabledFor(INFO): - bubble_log.info('set_flex_response: no flex router for host: '+flex_host) - return + bubble_log.info('new_flex_flow: no flex router for host: '+flex_host) + return None if bubble_log.isEnabledFor(INFO): - bubble_log.info('set_flex_response: found router '+repr(router)+' for flex host: '+flex_host) - try: - process_flex(flex_host, flow, router) - except Exception as e: - if bubble_log.isEnabledFor(ERROR): - bubble_log.error('set_flex_response: error processing: '+repr(e)) + bubble_log.info('new_flex_flow: found router '+repr(router)+' for flex host: '+flex_host) + return FlexFlow(flex_host, flow, router) + +def process_flex(flex_flow): -def process_flex(flex_host, flow, router): + flex_host = flex_flow.flex_host + flow = flex_flow.mitm_flow + router = flex_flow.router # build the request URL method = flow.request.method @@ -68,41 +80,39 @@ def process_flex(flex_host, flow, router): proxy_url = router['proxyUrl'] proxies = {"http": proxy_url, "https": proxy_url} - # send request to flex router - request_body = flow.request.content if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('process_flex: sending flex request for ' + method +' ' + url +' to ' + proxy_url +' with headers=' + repr(request_headers) +' and body=' + repr(request_body)) + + loop = asyncio.new_event_loop() + client = async_client(proxies=proxies, timeout=30) try: - response = requests.request(method, url, - headers=request_headers, - timeout=(20, 120), - stream=True, - data=request_body, # use the original request body, if there is one - proxies=proxies, - allow_redirects=False) + response = async_stream(client, 'process_flex', url, + method=method, + headers=request_headers, + timeout=30, + data=async_chunk_iter(flex_flow.request_chunks), + loop=loop) if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('process_flex: response returned HTTP status '+str(response.status_code)+' for '+url) except Exception as e: if bubble_log.isEnabledFor(ERROR): bubble_log.error('process_flex: error sending request to '+url+': '+repr(e)) - return + return None - # Status line -- http version is buried in response.raw.version - http_version = get_http_version(response) - if http_version is None: - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: invalid HTTP version, bailing out') - return + if response is None: + return None + + # Status line + http_version = response.http_version # Headers -- copy from requests dict to Headers multimap - # Remove Content-Length, Content-Encoding and Transfer-Encoding - # We will rechunk the output - response_headers = collect_response_headers(response, [HEADER_CONTENT_LENGTH, HEADER_CONTENT_ENCODING, HEADER_TRANSFER_ENCODING]) + # Remove Content-Length and Content-Encoding, we will rechunk the output + response_headers = collect_response_headers(response, [HEADER_CONTENT_LENGTH, HEADER_TRANSFER_ENCODING]) - # Construct the response + # Construct the real response flow.response = http.HTTPResponse(http_version=http_version, status_code=response.status_code, - reason=response.reason, + reason=response.reason_phrase, headers=response_headers, content=None) @@ -113,18 +123,48 @@ def process_flex(flex_host, flow, router): if response.status_code // 100 != 2: response_headers[HEADER_CONTENT_LENGTH] = '0' flow.response.stream = lambda chunks: [] + elif content_length is None or int(content_length) > 0: response_headers[HEADER_TRANSFER_ENCODING] = 'chunked' - flow.response.stream = lambda chunks: response.iter_content(8192) + flow.response.stream = AsyncStreamBody(owner=client, loop=loop, chunks=response.aiter_raw(), finalize=cleanup_flex(url, loop, client, response)) + else: response_headers[HEADER_CONTENT_LENGTH] = '0' flow.response.stream = lambda chunks: [] # Apply filters - if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: bubble filtering: '+url) - # flow.response.stream = lambda chunks: response.iter_content(8192) - bubble_filter_response(flow, response) - if bubble_log.isEnabledFor(INFO): - bubble_log.info('process_flex: successfully requested url '+url+' from flex router, proceeding ...') + bubble_log.info('process_flex: successfully requested url '+url+' from flex router, proceeding...') + return response + + +async def async_chunk_iter(chunks): + for chunk in chunks: + yield chunk + + +def cleanup_flex(url, loop, client, response): + def cleanup(): + + errors = False + + try: + loop.run_until_complete(response.aclose()) + except Exception as e: + bubble_log.error('cleanup_flex: error closing response: '+repr(e)) + errors = True + + try: + loop.run_until_complete(client.aclose()) + except Exception as e: + bubble_log.error('cleanup_flex: error: '+repr(e)) + errors = True + + if not errors: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('cleanup_flex: successfully completed: '+url) + else: + if bubble_log.isEnabledFor(WARNING): + bubble_log.warning('cleanup_flex: successfully completed (but had errors closing): ' + url) + + return cleanup diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index 3c47e013..f77afe52 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -1,19 +1,20 @@ # # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # +import asyncio import json import re -import requests import urllib import traceback from mitmproxy.net.http import Headers from bubble_config import bubble_port, debug_capture_fqdn, debug_stream_fqdn, debug_stream_uri -from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ - status_reason, get_flow_ctx, add_flow_ctx, \ +from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, CTX_BUBBLE_FLEX, \ + status_reason, get_flow_ctx, add_flow_ctx, bubble_async, \ is_bubble_special_path, is_bubble_health_check, health_check_response, special_bubble_response, \ CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, CTX_BUBBLE_FILTERED, \ HEADER_CONTENT_TYPE, HEADER_CONTENT_ENCODING, HEADER_LOCATION, HEADER_CONTENT_LENGTH, \ HEADER_USER_AGENT, HEADER_FILTER_PASSTHRU, HEADER_CONTENT_SECURITY_POLICY, REDIS, redis_set +from bubble_flex import process_flex import logging from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL @@ -28,7 +29,7 @@ REDIS_FILTER_PASSTHRU_PREFIX = '__chunk_filter_pass__' REDIS_FILTER_PASSTHRU_DURATION = 600 DEBUG_STREAM_COUNTERS = {} -MIN_FILTER_CHUNK_SIZE = 16384 +MIN_FILTER_CHUNK_SIZE = 1024 * 32 # Filter data in 32KB chunks def add_csp_part(new_csp, part): @@ -53,7 +54,8 @@ def ensure_bubble_script_csp(csp): return new_csp -def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, content_type=None, content_length=None, csp=None): +def filter_chunk(loop, flow, chunk, req_id, user_agent, last, content_encoding=None, content_type=None, content_length=None, csp=None): + name = 'filter_chunk' if debug_capture_fqdn: if debug_capture_fqdn in req_id: if bubble_log.isEnabledFor(DEBUG): @@ -87,17 +89,21 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c else: url = url + '?last=true' + chunk_len = 0 + if bubble_log.isEnabledFor(DEBUG): + if chunk is not None: + chunk_len = len(chunk) if csp: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('filter_chunk: url='+url+' (csp='+csp+')') - filter_headers = { + bubble_log.debug('filter_chunk: url='+url+' (csp='+csp+') size='+str(chunk_len)) + headers = { HEADER_CONTENT_TYPE: CONTENT_TYPE_BINARY, HEADER_CONTENT_SECURITY_POLICY: csp } else: if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('filter_chunk: url='+url+' (no csp)') - filter_headers = STANDARD_FILTER_HEADERS + bubble_log.debug('filter_chunk: url='+url+' (no csp) size='+str(chunk_len)) + headers = STANDARD_FILTER_HEADERS if debug_stream_fqdn and debug_stream_uri and debug_stream_fqdn in req_id and flow.request.path == debug_stream_uri: if req_id in DEBUG_STREAM_COUNTERS: @@ -112,14 +118,14 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c f.write(chunk) f.close() f = open('/tmp/bubble_stream_'+req_id+'_chunk'+"{:04d}".format(count)+'.headers.json', mode='w') - f.write(json.dumps(filter_headers)) + f.write(json.dumps(headers)) f.close() f = open('/tmp/bubble_stream_'+req_id+'_chunk'+"{:04d}".format(count)+'.url', mode='w') f.write(url) f.close() - response = requests.post(url, data=chunk, headers=filter_headers) - if not response.ok: + response = bubble_async(name, url, headers=headers, method='POST', data=chunk, loop=loop) + if not response.status_code == 200: err_message = 'filter_chunk: Error fetching ' + url + ', HTTP status ' + str(response.status_code) if bubble_log.isEnabledFor(ERROR): bubble_log.error(err_message) @@ -135,9 +141,7 @@ def filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding=None, c def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_encoding, content_type, csp): - """ - chunks is a generator that can be used to iterate over all chunks. - """ + loop = asyncio.new_event_loop() if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('bubble_filter_chunks: starting with content_type='+content_type) first = True @@ -165,20 +169,20 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ else: last = False if first: - yield filter_chunk(flow, chunk, req_id, user_agent, last, content_encoding, content_type, content_length, csp) + yield filter_chunk(loop, flow, chunk, req_id, user_agent, last, content_encoding, content_type, content_length, csp) first = False else: - yield filter_chunk(flow, chunk, req_id, user_agent, last) + yield filter_chunk(loop, flow, chunk, req_id, user_agent, last) # send whatever is left in the buffer if len(buffer) > 0: # bubble_log.debug('bubble_filter_chunks(end): sending remainder buffer of size '+str(len(buffer))) if first: - yield filter_chunk(flow, buffer, req_id, user_agent, last, content_encoding, content_type, content_length, csp) + yield filter_chunk(loop, flow, buffer, req_id, user_agent, last, content_encoding, content_type, content_length, csp) else: - yield filter_chunk(flow, buffer, req_id, user_agent, last) + yield filter_chunk(loop, flow, buffer, req_id, user_agent, last) if not content_length or not last: # bubble_log.debug('bubble_filter_chunks(end): sending last empty chunk') - yield filter_chunk(flow, None, req_id, user_agent, True) # get the last bits of data + yield filter_chunk(loop, flow, None, req_id, user_agent, True) # get the last bits of data except Exception as e: if bubble_log.isEnabledFor(ERROR): bubble_log.error('bubble_filter_chunks: exception='+repr(e)) @@ -209,7 +213,12 @@ def abort_data(content_type): def responseheaders(flow): - bubble_filter_response(flow, None) + flex_flow = get_flow_ctx(flow, CTX_BUBBLE_FLEX) + if flex_flow: + flex_stream = process_flex(flex_flow) + else: + flex_stream = None + bubble_filter_response(flow, flex_stream) def bubble_filter_response(flow, flex_stream): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index cc675982..7054a0f3 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -31,11 +31,11 @@ from mitmproxy.net.http import headers as nheaders from bubble_api import bubble_matchers, bubble_activity_log, \ HEALTH_CHECK_URI, CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ - CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ + CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ is_bubble_special_path, special_bubble_response, is_bubble_health_check, \ is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain from bubble_config import bubble_host, bubble_host_alias -from bubble_flex import set_flex_response +from bubble_flex import new_flex_flow import logging from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL @@ -251,9 +251,11 @@ class Rerouter: flow.request.port = port return host - def request(self, flow): + def requestheaders(self, flow): host = self.bubble_handle_request(flow) path = flow.request.path + flow.request.capture_stream = True + if is_bubble_special_path(path): # if bubble_log.isEnabledFor(DEBUG): # bubble_log.debug('request: is_bubble_special_path('+path+') returned true, sending special bubble response') @@ -262,9 +264,9 @@ class Rerouter: elif host is not None: client_addr = flow.client_conn.address[0] if is_flex_domain(client_addr, host): + add_flow_ctx(flow, CTX_BUBBLE_FLEX, new_flex_flow(client_addr, host, flow)) if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('request: is_flex_domain('+host+') returned true, sending flex response') - set_flex_response(client_addr, host, flow) + bubble_log.debug('request: is_flex_domain('+host+') returned true, setting ctx: '+CTX_BUBBLE_FLEX) addons = [Rerouter()] -- 2.17.1 From a41cdc5be13cd2d586495598865b418aea43765f Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 10:30:34 -0400 Subject: [PATCH 70/78] emit router OK info message if it was previously inactive --- .../java/bubble/service/device/StandardFlexRouterService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index 4dd99cfd..eb418b8e 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -217,7 +217,8 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute } else { final FlexRouterPing pong = response.getEntity(FlexRouterPing.class); if (pong.validate(router)) { - if (log.isTraceEnabled()) log.trace(prefix+"router is ok"); + // emit message if loglevel is tracing, or info message if router was previously inactive + if (log.isTraceEnabled() || (router.inactive() && log.isInfoEnabled())) log.trace(prefix+"router is ok"); setStatus(router, FlexRouterStatus.active); return true; } else { -- 2.17.1 From 37fb6d077cff6fd56d7ff5c5671d518aa3f984a5 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 10:39:33 -0400 Subject: [PATCH 71/78] add mitm_pid script --- bin/mitm_pid | 8 ++++++++ .../src/main/resources/ansible/bubble_scripts.txt | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100755 bin/mitm_pid diff --git a/bin/mitm_pid b/bin/mitm_pid new file mode 100755 index 00000000..8832b405 --- /dev/null +++ b/bin/mitm_pid @@ -0,0 +1,8 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Print PID of currently-active mitmproxy +# Note: this command only works on a running bubble node +# +ps auxwww | grep /home/mitmproxy/mitmproxy/venv/bin/mitmdump | grep $(cat /home/mitmproxy/mitmproxy_port) | grep -v grep | awk '{print $2}' diff --git a/bubble-server/src/main/resources/ansible/bubble_scripts.txt b/bubble-server/src/main/resources/ansible/bubble_scripts.txt index 2e4a0af2..2248261e 100644 --- a/bubble-server/src/main/resources/ansible/bubble_scripts.txt +++ b/bubble-server/src/main/resources/ansible/bubble_scripts.txt @@ -18,4 +18,5 @@ cleanup_bubble_databases install_packer.sh rkeys rmembers -rdelkeys \ No newline at end of file +rdelkeys +mitm_pid \ No newline at end of file -- 2.17.1 From 9ca2a9eff8fa01448f19d82c97e88da958c43cbb Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 12:50:05 -0400 Subject: [PATCH 72/78] display warning page if flex route detected and no flex router running --- .../resources/stream/FilterHttpResource.java | 13 +++-- .../bubble/service/device/FlexRouterInfo.java | 44 ++++++++++++++-- .../device/StandardFlexRouterService.java | 16 +++--- .../service/message/MessageService.java | 13 ++++- bubble-server/src/main/resources/logback.xml | 5 +- bubble-server/src/main/resources/messages | 2 +- .../roles/mitmproxy/files/bubble_api.py | 4 +- .../roles/mitmproxy/files/bubble_flex.py | 52 +++++++++++++++---- .../roles/mitmproxy/files/bubble_modify.py | 28 ++++++---- .../roles/mitmproxy/files/bubble_request.py | 3 +- 10 files changed, 140 insertions(+), 40 deletions(-) diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index af7ce528..54d4a69c 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -28,6 +28,7 @@ import bubble.service.boot.SelfNodeService; import bubble.service.device.DeviceService; import bubble.service.device.FlexRouterInfo; import bubble.service.device.StandardFlexRouterService; +import bubble.service.message.MessageService; import bubble.service.stream.ConnectionCheckResponse; import bubble.service.stream.StandardRuleEngineService; import com.fasterxml.jackson.databind.JsonNode; @@ -55,6 +56,7 @@ import java.util.stream.Collectors; import static bubble.ApiConstants.*; import static bubble.resources.stream.FilterMatchersResponse.NO_MATCHERS; import static bubble.rule.AppRuleDriver.isFlexRouteFqdn; +import static bubble.service.device.FlexRouterInfo.missingFlexRouter; import static bubble.service.stream.HttpStreamDebug.getLogFqdn; import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIMEOUT; import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY; @@ -90,6 +92,7 @@ public class FilterHttpResource { @Autowired private SelfNodeService selfNodeService; @Autowired private BlockStatsService blockStats; @Autowired private StandardFlexRouterService flexRouterService; + @Autowired private MessageService messageService; private static final long ACTIVE_REQUEST_TIMEOUT = HOURS.toSeconds(12); @@ -644,10 +647,11 @@ public class FilterHttpResource { return ok(summary); } - @GET @Path(EP_FLEX_ROUTERS) + @GET @Path(EP_FLEX_ROUTERS+"/{fqdn}") @Produces(APPLICATION_JSON) public Response getFlexRouter(@Context Request req, - @Context ContainerRequest ctx) { + @Context ContainerRequest ctx, + @PathParam("fqdn") String fqdn) { final String publicIp = getRemoteAddr(req); final Device device = deviceService.findDeviceByIp(publicIp); if (device == null) { @@ -666,7 +670,10 @@ public class FilterHttpResource { Collection routers = flexRouterService.selectClosestRouter(device.getAccount(), vpnIp, publicIp); if (log.isDebugEnabled()) log.debug("getFlexRouter: found router(s) for vpnIp="+vpnIp+": "+json(routers, COMPACT_MAPPER)); - if (routers.isEmpty()) return notFound(); + if (routers.isEmpty()) { + final Account account = accountDAO.findByUuid(device.getAccount()); + return ok(missingFlexRouter(account, device, fqdn, messageService, configuration.getHandlebars())); + } return ok(routers.iterator().next().initAuth()); } diff --git a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java index a8e3f2c8..0eafc270 100644 --- a/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java +++ b/bubble-server/src/main/java/bubble/service/device/FlexRouterInfo.java @@ -5,24 +5,36 @@ package bubble.service.device; import bubble.cloud.geoLocation.GeoLocation; +import bubble.model.account.Account; +import bubble.model.device.Device; import bubble.model.device.DeviceStatus; import bubble.model.device.FlexRouter; +import bubble.service.message.MessageService; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.github.jknack.handlebars.Handlebars; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import lombok.experimental.Accessors; +import org.cobbzilla.util.handlebars.HandlebarsUtil; + +import java.util.HashMap; +import java.util.Map; import static bubble.model.device.DeviceStatus.NO_DEVICE_STATUS; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; -@ToString +@Accessors(chain=true) @ToString public class FlexRouterInfo { @JsonIgnore @Getter private final FlexRouter router; @JsonIgnore @Getter private final DeviceStatus deviceStatus; @Getter @Setter private String auth; + // set by missingFlexRouter method when there is no flex router but there should be one + @Getter @Setter private String error_html; + public FlexRouterInfo (FlexRouter router, DeviceStatus deviceStatus) { this.router = router; this.deviceStatus = deviceStatus; @@ -30,8 +42,8 @@ public class FlexRouterInfo { @JsonIgnore public String getVpnIp () { return router.getIp(); } - public int getPort () { return router.getPort(); } - public String getProxyUrl () { return router.proxyBaseUri(); } + public int getPort () { return router == null ? -1 : router.getPort(); } + public String getProxyUrl () { return router == null ? null : router.proxyBaseUri(); } public boolean hasGeoLocation () { return hasDeviceStatus() && deviceStatus.getLocation() != null && deviceStatus.getLocation().hasLatLon(); } public boolean hasNoGeoLocation () { return !hasGeoLocation(); } @@ -54,4 +66,30 @@ public class FlexRouterInfo { return obj instanceof FlexRouterInfo && ((FlexRouterInfo) obj).getPort() == getPort(); } + public static final String CTX_ACCOUNT = "account"; + public static final String CTX_DEVICE = "device"; + public static final String CTX_MESSAGES = "messages"; + public static final String CTX_FLEX_FQDN = "flex_fqdn"; + public static final String CTX_DEVICE_TYPE_LABEL = "device_type_label"; + + public static FlexRouterInfo missingFlexRouter(Account account, + Device device, + String fqdn, + MessageService messageService, + Handlebars handlebars) { + final String locale = account.getLocale(); + final String template = messageService.loadPageTemplate(locale, "no_flex_router"); + final Map ctx = new HashMap<>(); + ctx.put(CTX_ACCOUNT, account); + ctx.put(CTX_DEVICE, device); + ctx.put(CTX_FLEX_FQDN, fqdn); + + final Map messages = messageService.formatStandardMessages(locale); + ctx.put(CTX_MESSAGES, messages); + ctx.put(CTX_DEVICE_TYPE_LABEL, messages.get("device_type_"+device.getDeviceType().name())); + + final String html = HandlebarsUtil.apply(handlebars, template, ctx); + return new FlexRouterInfo(null, null).setError_html(html); + } + } diff --git a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java index eb418b8e..ea777943 100644 --- a/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java +++ b/bubble-server/src/main/java/bubble/service/device/StandardFlexRouterService.java @@ -50,7 +50,7 @@ import static org.cobbzilla.util.system.Sleep.sleep; @Service @Slf4j public class StandardFlexRouterService extends SimpleDaemon implements FlexRouterService { - public static final int MAX_PING_TRIES = 10; + public static final int MAX_PING_TRIES = 5; private static final long PING_SLEEP_FACTOR = SECONDS.toMillis(2); @@ -61,22 +61,22 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute .setSocketTimeout(DEFAULT_PING_TIMEOUT) .setConnectionRequestTimeout(DEFAULT_PING_TIMEOUT).build(); - // thread pool size - public static final int DEFAULT_MAX_TUNNELS = 5; - // wait for ssh key to be written private static final long FIRST_TIME_WAIT = SECONDS.toMillis(10); private static final long INTERRUPT_WAIT = FIRST_TIME_WAIT/2; + public static final long PING_ALL_TIMEOUT + = (SECONDS.toMillis(1) * DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + FIRST_TIME_WAIT; + + // thread pool size + public static final int DEFAULT_MAX_TUNNELS = 5; + private static CloseableHttpClient getHttpClient() { return HttpClientBuilder.create() .setDefaultRequestConfig(DEFAULT_PING_REQUEST_CONFIG) .build(); } - private static final long PING_ALL_TIMEOUT - = (SECONDS.toMillis(1) * DEFAULT_PING_TIMEOUT * MAX_PING_TRIES) + FIRST_TIME_WAIT; - public static final long DEFAULT_SLEEP_TIME = MINUTES.toMillis(2); @Autowired private FlexRouterDAO flexRouterDAO; @@ -227,7 +227,7 @@ public class StandardFlexRouterService extends SimpleDaemon implements FlexRoute } } catch (Exception e) { - log.error(prefix+"error: "+shortError(e)); + log.warn(prefix+"error: "+shortError(e)); } setStatus(router, FlexRouterStatus.unreachable); } diff --git a/bubble-server/src/main/java/bubble/service/message/MessageService.java b/bubble-server/src/main/java/bubble/service/message/MessageService.java index cec46f6e..c05cb145 100644 --- a/bubble-server/src/main/java/bubble/service/message/MessageService.java +++ b/bubble-server/src/main/java/bubble/service/message/MessageService.java @@ -19,13 +19,15 @@ import java.util.concurrent.ConcurrentHashMap; import static bubble.ApiConstants.MESSAGE_RESOURCE_BASE; import static org.cobbzilla.util.daemon.ZillaRuntime.*; -import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; +import static org.cobbzilla.util.io.StreamUtil.*; import static org.cobbzilla.util.string.StringUtil.UTF8cs; @Service @Slf4j public class MessageService { public static final String MESSAGE_RESOURCE_PATH = "/server/"; + public static final String PAGE_TEMPLATES_PATH = "pages/"; + public static final String PAGE_TEMPLATES_SUFFIX = ".html.hbs"; public static final String RESOURCE_MESSAGES_PROPS = "ResourceMessages.properties"; public static final String[] PRE_AUTH_MESSAGE_GROUPS = {"pre_auth", "countries", "timezones"}; @@ -81,4 +83,13 @@ public class MessageService { }); } + private final Map pageTemplateCache = new ConcurrentHashMap<>(10); + + public String loadPageTemplate(String locale, String templatePath) { + final String key = locale + ":" + templatePath; + return pageTemplateCache.computeIfAbsent(key, k -> { + final String path = MESSAGE_RESOURCE_BASE + locale + MESSAGE_RESOURCE_PATH + PAGE_TEMPLATES_PATH + templatePath + PAGE_TEMPLATES_SUFFIX; + return loadResourceAsStringOrDie(path); + }); + } } diff --git a/bubble-server/src/main/resources/logback.xml b/bubble-server/src/main/resources/logback.xml index 3b6c9e5e..42728e5b 100644 --- a/bubble-server/src/main/resources/logback.xml +++ b/bubble-server/src/main/resources/logback.xml @@ -40,6 +40,7 @@ + @@ -59,8 +60,8 @@ - - + + diff --git a/bubble-server/src/main/resources/messages b/bubble-server/src/main/resources/messages index 4f834554..04db2238 160000 --- a/bubble-server/src/main/resources/messages +++ b/bubble-server/src/main/resources/messages @@ -1 +1 @@ -Subproject commit 4f8345542b29228db0dba3e845f0662ab9cf6693 +Subproject commit 04db22382ddffa08b3f05c024603da75cdfb8b55 diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 6757ab43..e38e615e 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -228,9 +228,9 @@ def bubble_conn_check(client_addr, server_addr, fqdns, security_level): return None -def bubble_get_flex_router(client_addr): +def bubble_get_flex_router(client_addr, host): name = 'bubble_get_flex_router' - url = 'http://127.0.0.1:' + bubble_port + '/api/filter/flexRouters' + url = 'http://127.0.0.1:' + bubble_port + '/api/filter/flexRouters/' + host headers = { 'X-Forwarded-For': client_addr, 'Accept': 'application/json' diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py index cf12304b..df64708d 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_flex.py @@ -3,15 +3,14 @@ # import asyncio -from httpx._types import RequestData - from mitmproxy.proxy.protocol.async_stream_body import AsyncStreamBody from mitmproxy import http +from mitmproxy.net.http import headers as nheaders from mitmproxy.proxy.protocol.request_capture import RequestCapture -from bubble_api import bubble_get_flex_router, collect_response_headers, bubble_async, async_client, async_stream, \ - HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH +from bubble_api import bubble_get_flex_router, collect_response_headers, async_client, async_stream, \ + HEADER_TRANSFER_ENCODING, HEADER_CONTENT_LENGTH, HEADER_CONTENT_TYPE import logging from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL @@ -27,6 +26,7 @@ class FlexFlow(RequestCapture): mitm_flow: None router: None request_chunks: None + response_stream: None def __init__(self, flex_host, mitm_flow, router): super().__init__() @@ -40,15 +40,39 @@ class FlexFlow(RequestCapture): headers={}, content=None) + def is_error(self): + return 'error_html' in self.router and self.router['error_html'] and len(self.router['error_html']) > 0 + def capture(self, chunks): self.request_chunks = chunks +def process_no_flex(flex_flow): + + flow = flex_flow.mitm_flow + + response_headers = nheaders.Headers() + response_headers[HEADER_CONTENT_TYPE] = 'text/html' + response_headers[HEADER_CONTENT_LENGTH] = str(len(flex_flow.router['error_html'])) + + flow.response = http.HTTPResponse(http_version='HTTP/1.1', + status_code=200, + reason='OK', + headers=response_headers, + content=None) + error_html = flex_flow.router['error_html'] + flex_flow.response_stream = lambda chunks: error_html + flow.response.stream = lambda chunks: error_html + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_no_flex: no router found, returning error_html') + return flex_flow + + def new_flex_flow(client_addr, flex_host, flow): - router = bubble_get_flex_router(client_addr) + router = bubble_get_flex_router(client_addr, flex_host) if router is None or 'auth' not in router: - if bubble_log.isEnabledFor(INFO): - bubble_log.info('new_flex_flow: no flex router for host: '+flex_host) + if bubble_log.isEnabledFor(ERROR): + bubble_log.error('new_flex_flow: no flex router for host: '+flex_host) return None if bubble_log.isEnabledFor(INFO): @@ -58,6 +82,14 @@ def new_flex_flow(client_addr, flex_host, flow): def process_flex(flex_flow): + if flex_flow.is_error(): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_flex: no router found, returning default flow') + return process_no_flex(flex_flow) + else: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('process_flex: using router: '+repr(flex_flow.router)) + flex_host = flex_flow.flex_host flow = flex_flow.mitm_flow router = flex_flow.router @@ -81,7 +113,7 @@ def process_flex(flex_flow): proxies = {"http": proxy_url, "https": proxy_url} if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('process_flex: sending flex request for ' + method +' ' + url +' to ' + proxy_url +' with headers=' + repr(request_headers) +' and body=' + repr(request_body)) + bubble_log.debug('process_flex: sending flex request for '+method+' '+url+' to '+proxy_url) loop = asyncio.new_event_loop() client = async_client(proxies=proxies, timeout=30) @@ -135,7 +167,9 @@ def process_flex(flex_flow): # Apply filters if bubble_log.isEnabledFor(INFO): bubble_log.info('process_flex: successfully requested url '+url+' from flex router, proceeding...') - return response + + flex_flow.response_stream = response + return flex_flow async def async_chunk_iter(chunks): diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py index f77afe52..d7a43c95 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_modify.py @@ -140,7 +140,7 @@ def filter_chunk(loop, flow, chunk, req_id, user_agent, last, content_encoding=N return response.content -def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_encoding, content_type, csp): +def bubble_filter_chunks(flow, chunks, flex_flow, req_id, user_agent, content_encoding, content_type, csp): loop = asyncio.new_event_loop() if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('bubble_filter_chunks: starting with content_type='+content_type) @@ -149,8 +149,9 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('bubble_filter_chunks: found content_length='+str(content_length)) - if flex_stream is not None: - chunks = flex_stream.iter_content(8192) + if flex_flow is not None: + # flex flows with errors are handled before we get here + chunks = flex_flow.response_stream.iter_content(8192) try: buffer = b'' for chunk in chunks: @@ -190,10 +191,10 @@ def bubble_filter_chunks(flow, chunks, flex_stream, req_id, user_agent, content_ yield None -def bubble_modify(flow, flex_stream, req_id, user_agent, content_encoding, content_type, csp): +def bubble_modify(flow, flex_flow, req_id, user_agent, content_encoding, content_type, csp): if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('bubble_modify: modifying req_id='+req_id+' with content_type='+content_type) - return lambda chunks: bubble_filter_chunks(flow, chunks, flex_stream, req_id, + return lambda chunks: bubble_filter_chunks(flow, chunks, flex_flow, req_id, user_agent, content_encoding, content_type, csp) @@ -215,13 +216,15 @@ def abort_data(content_type): def responseheaders(flow): flex_flow = get_flow_ctx(flow, CTX_BUBBLE_FLEX) if flex_flow: - flex_stream = process_flex(flex_flow) + flex_flow = process_flex(flex_flow) else: - flex_stream = None - bubble_filter_response(flow, flex_stream) + flex_flow = None + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('responseheaders: flex_flow = '+repr(flex_flow)) + bubble_filter_response(flow, flex_flow) -def bubble_filter_response(flow, flex_stream): +def bubble_filter_response(flow, flex_flow): # only filter once -- flex routing may have pre-filtered if get_flow_ctx(flow, CTX_BUBBLE_FILTERED): return @@ -235,6 +238,11 @@ def bubble_filter_response(flow, flex_stream): else: special_bubble_response(flow) + elif flex_flow and flex_flow.is_error(): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('bubble_filter_response: flex_flow had error, returning error_html: ' + repr(flex_flow.response_stream)) + flow.response.stream = flex_flow.response_stream + else: abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT) if abort_code is not None: @@ -320,7 +328,7 @@ def bubble_filter_response(flow, flex_stream): if bubble_log.isEnabledFor(DEBUG): bubble_log.debug(prefix+'content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) - flow.response.stream = bubble_modify(flow, flex_stream, req_id, + flow.response.stream = bubble_modify(flow, flex_flow, req_id, user_agent, content_encoding, content_type, csp) if content_length_value: flow.response.headers['transfer-encoding'] = 'chunked' diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 7054a0f3..4d5b8702 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -264,7 +264,8 @@ class Rerouter: elif host is not None: client_addr = flow.client_conn.address[0] if is_flex_domain(client_addr, host): - add_flow_ctx(flow, CTX_BUBBLE_FLEX, new_flex_flow(client_addr, host, flow)) + flex_flow = new_flex_flow(client_addr, host, flow) + add_flow_ctx(flow, CTX_BUBBLE_FLEX, flex_flow) if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('request: is_flex_domain('+host+') returned true, setting ctx: '+CTX_BUBBLE_FLEX) -- 2.17.1 From a029ed80f937e3ee88fbd4fc983617c0fc05dc2e Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 15:09:32 -0400 Subject: [PATCH 73/78] patch bug with redirect handling in httpx client library --- .../packer/roles/mitmproxy/files/_client.py | 1725 +++++++++++++++++ .../packer/roles/mitmproxy/tasks/main.yml | 7 + 2 files changed, 1732 insertions(+) create mode 100644 bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py new file mode 100644 index 00000000..fa9b9630 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/_client.py @@ -0,0 +1,1725 @@ +import functools +import typing +from types import TracebackType + +import httpcore + +from ._auth import Auth, BasicAuth, FunctionAuth +from ._config import ( + DEFAULT_LIMITS, + DEFAULT_MAX_REDIRECTS, + DEFAULT_TIMEOUT_CONFIG, + UNSET, + Limits, + Proxy, + Timeout, + UnsetType, + create_ssl_context, +) +from ._content_streams import ContentStream +from ._exceptions import ( + HTTPCORE_EXC_MAP, + InvalidURL, + RemoteProtocolError, + RequestBodyUnavailable, + TooManyRedirects, + map_exceptions, +) +from ._models import URL, Cookies, Headers, QueryParams, Request, Response +from ._status_codes import codes +from ._transports.asgi import ASGITransport +from ._transports.wsgi import WSGITransport +from ._types import ( + AuthTypes, + CertTypes, + CookieTypes, + HeaderTypes, + ProxiesTypes, + QueryParamTypes, + RequestData, + RequestFiles, + TimeoutTypes, + URLTypes, + VerifyTypes, +) +from ._utils import ( + NetRCInfo, + URLPattern, + get_environment_proxies, + get_logger, + same_origin, + warn_deprecated, +) + +logger = get_logger(__name__) + +KEEPALIVE_EXPIRY = 5.0 + + +class BaseClient: + def __init__( + self, + *, + auth: AuthTypes = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + base_url: URLTypes = "", + trust_env: bool = True, + ): + self._base_url = self._enforce_trailing_slash(URL(base_url)) + + self._auth = self._build_auth(auth) + self._params = QueryParams(params) + self._headers = Headers(headers) + self._cookies = Cookies(cookies) + self._timeout = Timeout(timeout) + self.max_redirects = max_redirects + self._trust_env = trust_env + self._netrc = NetRCInfo() + + @property + def trust_env(self) -> bool: + return self._trust_env + + def _enforce_trailing_slash(self, url: URL) -> URL: + if url.path.endswith("/"): + return url + return url.copy_with(path=url.path + "/") + + def _get_proxy_map( + self, proxies: typing.Optional[ProxiesTypes], allow_env_proxies: bool + ) -> typing.Dict[str, typing.Optional[Proxy]]: + if proxies is None: + if allow_env_proxies: + return { + key: None if url is None else Proxy(url=url) + for key, url in get_environment_proxies().items() + } + return {} + if isinstance(proxies, dict): + new_proxies = {} + for key, value in proxies.items(): + proxy = Proxy(url=value) if isinstance(value, (str, URL)) else value + new_proxies[str(key)] = proxy + return new_proxies + else: + proxy = Proxy(url=proxies) if isinstance(proxies, (str, URL)) else proxies + return {"all://": proxy} + + @property + def timeout(self) -> Timeout: + return self._timeout + + @timeout.setter + def timeout(self, timeout: TimeoutTypes) -> None: + self._timeout = Timeout(timeout) + + @property + def auth(self) -> typing.Optional[Auth]: + """ + Authentication class used when none is passed at the request-level. + + See also [Authentication][0]. + + [0]: /quickstart/#authentication + """ + return self._auth + + @auth.setter + def auth(self, auth: AuthTypes) -> None: + self._auth = self._build_auth(auth) + + @property + def base_url(self) -> URL: + """ + Base URL to use when sending requests with relative URLs. + """ + return self._base_url + + @base_url.setter + def base_url(self, url: URLTypes) -> None: + self._base_url = self._enforce_trailing_slash(URL(url)) + + @property + def headers(self) -> Headers: + """ + HTTP headers to include when sending requests. + """ + return self._headers + + @headers.setter + def headers(self, headers: HeaderTypes) -> None: + self._headers = Headers(headers) + + @property + def cookies(self) -> Cookies: + """ + Cookie values to include when sending requests. + """ + return self._cookies + + @cookies.setter + def cookies(self, cookies: CookieTypes) -> None: + self._cookies = Cookies(cookies) + + @property + def params(self) -> QueryParams: + """ + Query parameters to include in the URL when sending requests. + """ + return self._params + + @params.setter + def params(self, params: QueryParamTypes) -> None: + self._params = QueryParams(params) + + def stream( + self, + method: str, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: AuthTypes = None, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> "StreamContextManager": + """ + Alternative to `httpx.request()` that streams the response body + instead of loading it into memory at once. + + **Parameters**: See `httpx.request`. + + See also: [Streaming Responses][0] + + [0]: /quickstart#streaming-responses + """ + request = self.build_request( + method=method, + url=url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + ) + return StreamContextManager( + client=self, + request=request, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def build_request( + self, + method: str, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + ) -> Request: + """ + Build and return a request instance. + + * The `params`, `headers` and `cookies` arguments + are merged with any values set on the client. + * The `url` argument is merged with any `base_url` set on the client. + + See also: [Request instances][0] + + [0]: /advanced/#request-instances + """ + url = self._merge_url(url) + headers = self._merge_headers(headers) + cookies = self._merge_cookies(cookies) + params = self._merge_queryparams(params) + return Request( + method, + url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + ) + + def _merge_url(self, url: URLTypes) -> URL: + """ + Merge a URL argument together with any 'base_url' on the client, + to create the URL used for the outgoing request. + """ + merge_url = URL(url) + if merge_url.is_relative_url: + # We always ensure the base_url paths include the trailing '/', + # and always strip any leading '/' from the merge URL. + merge_url = merge_url.copy_with(path=merge_url.path.lstrip("/")) + return self.base_url.join(merge_url) + return merge_url + + def _merge_cookies( + self, cookies: CookieTypes = None + ) -> typing.Optional[CookieTypes]: + """ + Merge a cookies argument together with any cookies on the client, + to create the cookies used for the outgoing request. + """ + if cookies or self.cookies: + merged_cookies = Cookies(self.cookies) + merged_cookies.update(cookies) + return merged_cookies + return cookies + + def _merge_headers( + self, headers: HeaderTypes = None + ) -> typing.Optional[HeaderTypes]: + """ + Merge a headers argument together with any headers on the client, + to create the headers used for the outgoing request. + """ + if headers or self.headers: + merged_headers = Headers(self.headers) + merged_headers.update(headers) + return merged_headers + return headers + + def _merge_queryparams( + self, params: QueryParamTypes = None + ) -> typing.Optional[QueryParamTypes]: + """ + Merge a queryparams argument together with any queryparams on the client, + to create the queryparams used for the outgoing request. + """ + if params or self.params: + merged_queryparams = QueryParams(self.params) + merged_queryparams.update(params) + return merged_queryparams + return params + + def _build_auth(self, auth: AuthTypes) -> typing.Optional[Auth]: + if auth is None: + return None + elif isinstance(auth, tuple): + return BasicAuth(username=auth[0], password=auth[1]) + elif isinstance(auth, Auth): + return auth + elif callable(auth): + return FunctionAuth(func=auth) + else: + raise TypeError('Invalid "auth" argument.') + + def _build_request_auth( + self, request: Request, auth: typing.Union[AuthTypes, UnsetType] = UNSET + ) -> Auth: + auth = self._auth if isinstance(auth, UnsetType) else self._build_auth(auth) + + if auth is not None: + return auth + + username, password = request.url.username, request.url.password + if username or password: + return BasicAuth(username=username, password=password) + + if self.trust_env and "Authorization" not in request.headers: + credentials = self._netrc.get_credentials(request.url.authority) + if credentials is not None: + return BasicAuth(username=credentials[0], password=credentials[1]) + + return Auth() + + def _build_redirect_request(self, request: Request, response: Response) -> Request: + """ + Given a request and a redirect response, return a new request that + should be used to effect the redirect. + """ + method = self._redirect_method(request, response) + url = self._redirect_url(request, response) + headers = self._redirect_headers(request, url, method) + stream = self._redirect_stream(request, method) + cookies = Cookies(self.cookies) + return Request( + method=method, url=url, headers=headers, cookies=cookies, stream=stream + ) + + def _redirect_method(self, request: Request, response: Response) -> str: + """ + When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + """ + method = request.method + + # https://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.SEE_OTHER and method != "HEAD": + method = "GET" + + # Do what the browsers do, despite standards... + # Turn 302s into GETs. + if response.status_code == codes.FOUND and method != "HEAD": + method = "GET" + + # If a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in 'requests' issue 1704. + if response.status_code == codes.MOVED_PERMANENTLY and method == "POST": + method = "GET" + + return method + + def _redirect_url(self, request: Request, response: Response) -> URL: + """ + Return the URL for the redirect to follow. + """ + location = response.headers["Location"] + + try: + url = URL(location) + except InvalidURL as exc: + raise RemoteProtocolError( + f"Invalid URL in location header: {exc}.", request=request + ) from None + + # Handle malformed 'Location' headers that are "absolute" form, have no host. + # See: https://github.com/encode/httpx/issues/771 + if url.scheme and not url.host: + url = url.copy_with(host=request.url.host) + + # Facilitate relative 'Location' headers, as allowed by RFC 7231. + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if url.is_relative_url: + url = request.url.join(url) + + # Attach previous fragment if needed (RFC 7231 7.1.2) + if request.url.fragment and not url.fragment: + url = url.copy_with(fragment=request.url.fragment) + + return url + + def _redirect_headers(self, request: Request, url: URL, method: str) -> Headers: + """ + Return the headers that should be used for the redirect request. + """ + headers = Headers(request.headers) + + if not same_origin(url, request.url): + # Strip Authorization headers when responses are redirected away from + # the origin. + headers.pop("Authorization", None) + headers["Host"] = url.authority + + if method != request.method and method == "GET": + # If we've switch to a 'GET' request, then strip any headers which + # are only relevant to the request body. + headers.pop("Content-Length", None) + headers.pop("Transfer-Encoding", None) + + # We should use the client cookie store to determine any cookie header, + # rather than whatever was on the original outgoing request. + headers.pop("Cookie", None) + + return headers + + def _redirect_stream( + self, request: Request, method: str + ) -> typing.Optional[ContentStream]: + """ + Return the body that should be used for the redirect request. + """ + if method != request.method and method == "GET": + return None + + if not request.stream.can_replay(): + raise RequestBodyUnavailable( + "Got a redirect response, but the request body was streaming " + "and is no longer available.", + request=request, + ) + + return request.stream + + +class Client(BaseClient): + """ + An HTTP client, with connection pooling, HTTP/2, redirects, cookie persistence, etc. + + Usage: + + ```python + >>> client = httpx.Client() + >>> response = client.get('https://example.org') + ``` + + **Parameters:** + + * **auth** - *(optional)* An authentication class to use when sending + requests. + * **params** - *(optional)* Query parameters to include in request URLs, as + a string, dictionary, or list of two-tuples. + * **headers** - *(optional)* Dictionary of HTTP headers to include when + sending requests. + * **cookies** - *(optional)* Dictionary of Cookie items to include when + sending requests. + * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to + verify the identity of requested hosts. Either `True` (default CA bundle), + a path to an SSL certificate file, or `False` (disable verification). + * **cert** - *(optional)* An SSL certificate used by the requested host + to authenticate the client. Either a path to an SSL certificate file, or + two-tuple of (certificate file, key file), or a three-tuple of (certificate + file, key file, password). + * **proxies** - *(optional)* A dictionary mapping proxy keys to proxy + URLs. + * **timeout** - *(optional)* The timeout configuration to use when sending + requests. + * **limits** - *(optional)* The limits configuration to use. + * **max_redirects** - *(optional)* The maximum number of redirect responses + that should be followed. + * **base_url** - *(optional)* A URL to use as the base when building + request URLs. + * **transport** - *(optional)* A transport class to use for sending requests + over the network. + * **app** - *(optional)* An WSGI application to send requests to, + rather than sending actual network requests. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + """ + + def __init__( + self, + *, + auth: AuthTypes = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + verify: VerifyTypes = True, + cert: CertTypes = None, + http2: bool = False, + proxies: ProxiesTypes = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + limits: Limits = DEFAULT_LIMITS, + pool_limits: Limits = None, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + base_url: URLTypes = "", + transport: httpcore.SyncHTTPTransport = None, + app: typing.Callable = None, + trust_env: bool = True, + ): + super().__init__( + auth=auth, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + max_redirects=max_redirects, + base_url=base_url, + trust_env=trust_env, + ) + + if http2: + try: + import h2 # noqa + except ImportError: # pragma: nocover + raise ImportError( + "Using http2=True, but the 'h2' package is not installed. " + "Make sure to install httpx using `pip install httpx[http2]`." + ) from None + + if pool_limits is not None: + warn_deprecated( + "Client(..., pool_limits=...) is deprecated and will raise " + "errors in the future. Use Client(..., limits=...) instead." + ) + limits = pool_limits + + allow_env_proxies = trust_env and app is None and transport is None + proxy_map = self._get_proxy_map(proxies, allow_env_proxies) + + self._transport = self._init_transport( + verify=verify, + cert=cert, + http2=http2, + limits=limits, + transport=transport, + app=app, + trust_env=trust_env, + ) + self._proxies: typing.Dict[ + URLPattern, typing.Optional[httpcore.SyncHTTPTransport] + ] = { + URLPattern(key): None + if proxy is None + else self._init_proxy_transport( + proxy, + verify=verify, + cert=cert, + http2=http2, + limits=limits, + trust_env=trust_env, + ) + for key, proxy in proxy_map.items() + } + self._proxies = dict(sorted(self._proxies.items())) + + def _init_transport( + self, + verify: VerifyTypes = True, + cert: CertTypes = None, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + transport: httpcore.SyncHTTPTransport = None, + app: typing.Callable = None, + trust_env: bool = True, + ) -> httpcore.SyncHTTPTransport: + if transport is not None: + return transport + + if app is not None: + return WSGITransport(app=app) + + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + return httpcore.SyncConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=KEEPALIVE_EXPIRY, + http2=http2, + ) + + def _init_proxy_transport( + self, + proxy: Proxy, + verify: VerifyTypes = True, + cert: CertTypes = None, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + trust_env: bool = True, + ) -> httpcore.SyncHTTPTransport: + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + return httpcore.SyncHTTPProxy( + proxy_url=proxy.url.raw, + proxy_headers=proxy.headers.raw, + proxy_mode=proxy.mode, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=KEEPALIVE_EXPIRY, + http2=http2, + ) + + def _transport_for_url(self, url: URL) -> httpcore.SyncHTTPTransport: + """ + Returns the transport instance that should be used for a given URL. + This will either be the standard connection pool, or a proxy. + """ + for pattern, transport in self._proxies.items(): + if pattern.matches(url): + return self._transport if transport is None else transport + + return self._transport + + def request( + self, + method: str, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Build and send a request. + + Equivalent to: + + ```python + request = client.build_request(...) + response = client.send(request, ...) + ``` + + See `Client.build_request()`, `Client.send()` and + [Merging of configuration][0] for how the various parameters + are merged with client-level configuration. + + [0]: /advanced/#merging-of-configuration + """ + request = self.build_request( + method=method, + url=url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + ) + return self.send( + request, auth=auth, allow_redirects=allow_redirects, timeout=timeout + ) + + def send( + self, + request: Request, + *, + stream: bool = False, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a request. + + The request is sent as-is, unmodified. + + Typically you'll want to build one with `Client.build_request()` + so that any client-level configuration is merged into the request, + but passing an explicit `httpx.Request()` is supported as well. + + See also: [Request instances][0] + + [0]: /advanced/#request-instances + """ + timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) + + auth = self._build_request_auth(request, auth) + + response = self._send_handling_redirects( + request, auth=auth, timeout=timeout, allow_redirects=allow_redirects + ) + + if not stream: + try: + response.read() + finally: + response.close() + + return response + + def _send_handling_redirects( + self, + request: Request, + auth: Auth, + timeout: Timeout, + allow_redirects: bool = True, + history: typing.List[Response] = None, + ) -> Response: + if history is None: + history = [] + + while True: + if len(history) > self.max_redirects: + raise TooManyRedirects( + "Exceeded maximum allowed redirects.", request=request + ) + + response = self._send_handling_auth( + request, auth=auth, timeout=timeout, history=history + ) + response.history = list(history) + + if not response.is_redirect: + return response + + if allow_redirects: + response.read() + request = self._build_redirect_request(request, response) + history = history + [response] + + if not allow_redirects: + response.call_next = functools.partial( + self._send_handling_redirects, + request=request, + auth=auth, + timeout=timeout, + allow_redirects=False, + history=history, + ) + return response + + def _send_handling_auth( + self, + request: Request, + history: typing.List[Response], + auth: Auth, + timeout: Timeout, + ) -> Response: + if auth.requires_request_body: + request.read() + + auth_flow = auth.auth_flow(request) + request = next(auth_flow) + while True: + response = self._send_single_request(request, timeout) + if auth.requires_response_body: + response.read() + try: + next_request = auth_flow.send(response) + except StopIteration: + return response + except BaseException as exc: + response.close() + raise exc from None + else: + response.history = list(history) + response.read() + request = next_request + history.append(response) + + def _send_single_request(self, request: Request, timeout: Timeout) -> Response: + """ + Sends a single request, without handling any redirections. + """ + transport = self._transport_for_url(request.url) + + with map_exceptions(HTTPCORE_EXC_MAP, request=request): + ( + http_version, + status_code, + reason_phrase, + headers, + stream, + ) = transport.request( + request.method.encode(), + request.url.raw, + headers=request.headers.raw, + stream=request.stream, + timeout=timeout.as_dict(), + ) + response = Response( + status_code, + http_version=http_version.decode("ascii"), + headers=headers, + stream=stream, # type: ignore + request=request, + ) + + self.cookies.extract_cookies(response) + + status = f"{response.status_code} {response.reason_phrase}" + response_line = f"{response.http_version} {status}" + logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"') + + return response + + def get( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `GET` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def options( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def head( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `HEAD` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def post( + self, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "POST", + url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def put( + self, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "PUT", + url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def patch( + self, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "PATCH", + url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def delete( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `DELETE` request. + + **Parameters**: See `httpx.request`. + """ + return self.request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + def close(self) -> None: + """ + Close transport and proxies. + """ + self._transport.close() + for proxy in self._proxies.values(): + if proxy is not None: + proxy.close() + + def __enter__(self) -> "Client": + self._transport.__enter__() + for proxy in self._proxies.values(): + if proxy is not None: + proxy.__enter__() + return self + + def __exit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + self._transport.__exit__(exc_type, exc_value, traceback) + for proxy in self._proxies.values(): + if proxy is not None: + proxy.__exit__(exc_type, exc_value, traceback) + + +class AsyncClient(BaseClient): + """ + An asynchronous HTTP client, with connection pooling, HTTP/2, redirects, + cookie persistence, etc. + + Usage: + + ```python + >>> async with httpx.AsyncClient() as client: + >>> response = await client.get('https://example.org') + ``` + + **Parameters:** + + * **auth** - *(optional)* An authentication class to use when sending + requests. + * **params** - *(optional)* Query parameters to include in request URLs, as + a string, dictionary, or list of two-tuples. + * **headers** - *(optional)* Dictionary of HTTP headers to include when + sending requests. + * **cookies** - *(optional)* Dictionary of Cookie items to include when + sending requests. + * **verify** - *(optional)* SSL certificates (a.k.a CA bundle) used to + verify the identity of requested hosts. Either `True` (default CA bundle), + a path to an SSL certificate file, or `False` (disable verification). + * **cert** - *(optional)* An SSL certificate used by the requested host + to authenticate the client. Either a path to an SSL certificate file, or + two-tuple of (certificate file, key file), or a three-tuple of (certificate + file, key file, password). + * **http2** - *(optional)* A boolean indicating if HTTP/2 support should be + enabled. Defaults to `False`. + * **proxies** - *(optional)* A dictionary mapping HTTP protocols to proxy + URLs. + * **timeout** - *(optional)* The timeout configuration to use when sending + requests. + * **limits** - *(optional)* The limits configuration to use. + * **max_redirects** - *(optional)* The maximum number of redirect responses + that should be followed. + * **base_url** - *(optional)* A URL to use as the base when building + request URLs. + * **transport** - *(optional)* A transport class to use for sending requests + over the network. + * **app** - *(optional)* An ASGI application to send requests to, + rather than sending actual network requests. + * **trust_env** - *(optional)* Enables or disables usage of environment + variables for configuration. + """ + + def __init__( + self, + *, + auth: AuthTypes = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + verify: VerifyTypes = True, + cert: CertTypes = None, + http2: bool = False, + proxies: ProxiesTypes = None, + timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, + limits: Limits = DEFAULT_LIMITS, + pool_limits: Limits = None, + max_redirects: int = DEFAULT_MAX_REDIRECTS, + base_url: URLTypes = "", + transport: httpcore.AsyncHTTPTransport = None, + app: typing.Callable = None, + trust_env: bool = True, + ): + super().__init__( + auth=auth, + params=params, + headers=headers, + cookies=cookies, + timeout=timeout, + max_redirects=max_redirects, + base_url=base_url, + trust_env=trust_env, + ) + + if http2: + try: + import h2 # noqa + except ImportError: # pragma: nocover + raise ImportError( + "Using http2=True, but the 'h2' package is not installed. " + "Make sure to install httpx using `pip install httpx[http2]`." + ) from None + + if pool_limits is not None: + warn_deprecated( + "AsyncClient(..., pool_limits=...) is deprecated and will raise " + "errors in the future. Use AsyncClient(..., limits=...) instead." + ) + limits = pool_limits + + allow_env_proxies = trust_env and app is None and transport is None + proxy_map = self._get_proxy_map(proxies, allow_env_proxies) + + self._transport = self._init_transport( + verify=verify, + cert=cert, + http2=http2, + limits=limits, + transport=transport, + app=app, + trust_env=trust_env, + ) + + self._proxies: typing.Dict[ + URLPattern, typing.Optional[httpcore.AsyncHTTPTransport] + ] = { + URLPattern(key): None + if proxy is None + else self._init_proxy_transport( + proxy, + verify=verify, + cert=cert, + http2=http2, + limits=limits, + trust_env=trust_env, + ) + for key, proxy in proxy_map.items() + } + self._proxies = dict(sorted(self._proxies.items())) + + def _init_transport( + self, + verify: VerifyTypes = True, + cert: CertTypes = None, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + transport: httpcore.AsyncHTTPTransport = None, + app: typing.Callable = None, + trust_env: bool = True, + ) -> httpcore.AsyncHTTPTransport: + if transport is not None: + return transport + + if app is not None: + return ASGITransport(app=app) + + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + return httpcore.AsyncConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=KEEPALIVE_EXPIRY, + http2=http2, + ) + + def _init_proxy_transport( + self, + proxy: Proxy, + verify: VerifyTypes = True, + cert: CertTypes = None, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + trust_env: bool = True, + ) -> httpcore.AsyncHTTPTransport: + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + return httpcore.AsyncHTTPProxy( + proxy_url=proxy.url.raw, + proxy_headers=proxy.headers.raw, + proxy_mode=proxy.mode, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=KEEPALIVE_EXPIRY, + http2=http2, + ) + + def _transport_for_url(self, url: URL) -> httpcore.AsyncHTTPTransport: + """ + Returns the transport instance that should be used for a given URL. + This will either be the standard connection pool, or a proxy. + """ + for pattern, transport in self._proxies.items(): + if pattern.matches(url): + return self._transport if transport is None else transport + + return self._transport + + async def request( + self, + method: str, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Build and send a request. + + Equivalent to: + + ```python + request = client.build_request(...) + response = await client.send(request, ...) + ``` + + See `AsyncClient.build_request()`, `AsyncClient.send()` + and [Merging of configuration][0] for how the various parameters + are merged with client-level configuration. + + [0]: /advanced/#merging-of-configuration + """ + request = self.build_request( + method=method, + url=url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + ) + response = await self.send( + request, auth=auth, allow_redirects=allow_redirects, timeout=timeout + ) + return response + + async def send( + self, + request: Request, + *, + stream: bool = False, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a request. + + The request is sent as-is, unmodified. + + Typically you'll want to build one with `AsyncClient.build_request()` + so that any client-level configuration is merged into the request, + but passing an explicit `httpx.Request()` is supported as well. + + See also: [Request instances][0] + + [0]: /advanced/#request-instances + """ + timeout = self.timeout if isinstance(timeout, UnsetType) else Timeout(timeout) + + auth = self._build_request_auth(request, auth) + + response = await self._send_handling_redirects( + request, auth=auth, timeout=timeout, allow_redirects=allow_redirects + ) + + if not stream: + try: + await response.aread() + finally: + await response.aclose() + + return response + + async def _send_handling_redirects( + self, + request: Request, + auth: Auth, + timeout: Timeout, + allow_redirects: bool = True, + history: typing.List[Response] = None, + ) -> Response: + if history is None: + history = [] + + while True: + if len(history) > self.max_redirects: + raise TooManyRedirects( + "Exceeded maximum allowed redirects.", request=request + ) + + response = await self._send_handling_auth( + request, auth=auth, timeout=timeout, history=history + ) + response.history = list(history) + + if not response.is_redirect: + return response + + if allow_redirects: + await response.aread() + request = self._build_redirect_request(request, response) + history = history + [response] + + if not allow_redirects: + response.call_next = functools.partial( + self._send_handling_redirects, + request=request, + auth=auth, + timeout=timeout, + allow_redirects=False, + history=history, + ) + return response + + async def _send_handling_auth( + self, + request: Request, + history: typing.List[Response], + auth: Auth, + timeout: Timeout, + ) -> Response: + if auth.requires_request_body: + await request.aread() + + auth_flow = auth.auth_flow(request) + request = next(auth_flow) + while True: + response = await self._send_single_request(request, timeout) + if auth.requires_response_body: + await response.aread() + try: + next_request = auth_flow.send(response) + except StopIteration: + return response + except BaseException as exc: + await response.aclose() + raise exc from None + else: + response.history = list(history) + await response.aread() + request = next_request + history.append(response) + + async def _send_single_request( + self, request: Request, timeout: Timeout + ) -> Response: + """ + Sends a single request, without handling any redirections. + """ + transport = self._transport_for_url(request.url) + + with map_exceptions(HTTPCORE_EXC_MAP, request=request): + ( + http_version, + status_code, + reason_phrase, + headers, + stream, + ) = await transport.request( + request.method.encode(), + request.url.raw, + headers=request.headers.raw, + stream=request.stream, + timeout=timeout.as_dict(), + ) + response = Response( + status_code, + http_version=http_version.decode("ascii"), + headers=headers, + stream=stream, # type: ignore + request=request, + ) + + self.cookies.extract_cookies(response) + + status = f"{response.status_code} {response.reason_phrase}" + response_line = f"{response.http_version} {status}" + logger.debug(f'HTTP Request: {request.method} {request.url} "{response_line}"') + + return response + + async def get( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `GET` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "GET", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + async def options( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send an `OPTIONS` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "OPTIONS", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + async def head( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `HEAD` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "HEAD", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + async def post( + self, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `POST` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "POST", + url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + async def put( + self, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `PUT` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "PUT", + url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + async def patch( + self, + url: URLTypes, + *, + data: RequestData = None, + files: RequestFiles = None, + json: typing.Any = None, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `PATCH` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "PATCH", + url, + data=data, + files=files, + json=json, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + async def delete( + self, + url: URLTypes, + *, + params: QueryParamTypes = None, + headers: HeaderTypes = None, + cookies: CookieTypes = None, + auth: typing.Union[AuthTypes, UnsetType] = UNSET, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + ) -> Response: + """ + Send a `DELETE` request. + + **Parameters**: See `httpx.request`. + """ + return await self.request( + "DELETE", + url, + params=params, + headers=headers, + cookies=cookies, + auth=auth, + allow_redirects=allow_redirects, + timeout=timeout, + ) + + async def aclose(self) -> None: + """ + Close transport and proxies. + """ + await self._transport.aclose() + for proxy in self._proxies.values(): + if proxy is not None: + await proxy.aclose() + + async def __aenter__(self) -> "AsyncClient": + await self._transport.__aenter__() + for proxy in self._proxies.values(): + if proxy is not None: + await proxy.__aenter__() + return self + + async def __aexit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + await self._transport.__aexit__(exc_type, exc_value, traceback) + for proxy in self._proxies.values(): + if proxy is not None: + await proxy.__aexit__(exc_type, exc_value, traceback) + + +class StreamContextManager: + def __init__( + self, + client: BaseClient, + request: Request, + *, + auth: AuthTypes = None, + allow_redirects: bool = True, + timeout: typing.Union[TimeoutTypes, UnsetType] = UNSET, + close_client: bool = False, + ) -> None: + self.client = client + self.request = request + self.auth = auth + self.allow_redirects = allow_redirects + self.timeout = timeout + self.close_client = close_client + + def __enter__(self) -> "Response": + assert isinstance(self.client, Client) + self.response = self.client.send( + request=self.request, + auth=self.auth, + allow_redirects=self.allow_redirects, + timeout=self.timeout, + stream=True, + ) + return self.response + + def __exit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + assert isinstance(self.client, Client) + self.response.close() + if self.close_client: + self.client.close() + + async def __aenter__(self) -> "Response": + assert isinstance(self.client, AsyncClient) + self.response = await self.client.send( + request=self.request, + auth=self.auth, + allow_redirects=self.allow_redirects, + timeout=self.timeout, + stream=True, + ) + return self.response + + async def __aexit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + assert isinstance(self.client, AsyncClient) + await self.response.aclose() diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml index afe93dc1..51ff3543 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml @@ -92,6 +92,13 @@ - name: Install mitmproxy dependencies shell: su - mitmproxy -c "bash -c 'cd /home/mitmproxy/mitmproxy && ./dev.sh'" +- name: Overwrite _client.py from httpx to fix bug with HTTP/2 redirects + file: + src: _client.py + dest: /home/mitmproxxy/mitmproxy/venv/lib/python3.8/site-packages/httpx/_client.py + owner: mitmproxy + group: mitmproxy + - name: Install mitm_monitor copy: src: "mitm_monitor.sh" -- 2.17.1 From 36e06cf31dd4bea5802e3f89703cf828060f4a88 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 16:56:42 -0400 Subject: [PATCH 74/78] add support for flex exclude domains --- .../src/main/java/bubble/rule/AppRuleDriver.java | 14 ++++++++++++-- .../rule/passthru/TlsPassthruRuleDriver.java | 13 ++++++++++++- .../service/stream/StandardAppPrimerService.java | 12 +++++++++++- .../models/apps/passthru/bubbleApp_passthru.json | 2 +- .../resources/packer/roles/algo/tasks/main.yml | 2 +- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index e0290c83..3d3e2fb4 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -44,7 +44,8 @@ public interface AppRuleDriver { String REDIS_BLOCK_LISTS = "blockLists"; String REDIS_WHITE_LISTS = "whiteLists"; String REDIS_FILTER_LISTS = "filterLists"; - String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy for flex routing + String REDIS_FLEX_LISTS = "flexLists"; // used in mitmproxy and dnscrypt-proxy for flex routing + String REDIS_FLEX_EXCLUDE_LISTS = "flexExcludeLists"; // used in mitmproxy and dnscrypt-proxy for flex routing String REDIS_LIST_SUFFIX = "~UNION"; default Set getPrimedRejectDomains () { return null; } @@ -52,6 +53,7 @@ public interface AppRuleDriver { default Set getPrimedWhiteListDomains() { return null; } default Set getPrimedFilterDomains () { return null; } default Set getPrimedFlexDomains () { return null; } + default Set getPrimedFlexExcludeDomains () { return null; } static void defineRedisRejectSet(RedisService redis, String ip, String list, String[] rejectDomains) { defineRedisSet(redis, ip, REDIS_REJECT_LISTS, list, rejectDomains); @@ -73,6 +75,10 @@ public interface AppRuleDriver { defineRedisSet(redis, ip, REDIS_FLEX_LISTS, list, flexDomains); } + static void defineRedisFlexExcludeSet(RedisService redis, String ip, String list, String[] flexExcludeDomains) { + defineRedisSet(redis, ip, REDIS_FLEX_EXCLUDE_LISTS, list, flexExcludeDomains); + } + static void defineRedisSet(RedisService redis, String ip, String listOfListsName, String listName, String[] domains) { final String listOfListsForIp = listOfListsName + "~" + ip; final String unionSetName = listOfListsForIp + REDIS_LIST_SUFFIX; @@ -94,10 +100,14 @@ public interface AppRuleDriver { static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) { final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX; + final String excludeKey = REDIS_FLEX_EXCLUDE_LISTS + "~" + ip + REDIS_LIST_SUFFIX; String check = fqdn; while (true) { final boolean found = redis.sismember_plaintext(key, check); - if (found) return true; + if (found) { + final boolean excluded = redis.sismember_plaintext(excludeKey, check); + if (!excluded) return true; + } final int dotPos = check.indexOf('.'); if (dotPos == check.length()) return false; check = check.substring(dotPos+1); diff --git a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java index 56977ccd..ece93a1d 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/TlsPassthruRuleDriver.java @@ -17,6 +17,7 @@ import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.collection.ArrayUtil; import java.util.Set; +import java.util.stream.Collectors; import static org.cobbzilla.util.json.JsonUtil.json; @@ -27,7 +28,17 @@ public class TlsPassthruRuleDriver extends AbstractAppRuleDriver { @Override public Set getPrimedFlexDomains() { final TlsPassthruConfig passthruConfig = getRuleConfig(); - return passthruConfig.getFlexDomains(); + return passthruConfig.getFlexDomains().stream() + .filter(d -> !d.startsWith("!")) + .collect(Collectors.toSet()); + } + + @Override public Set getPrimedFlexExcludeDomains() { + final TlsPassthruConfig passthruConfig = getRuleConfig(); + return passthruConfig.getFlexDomains().stream() + .filter(d -> d.startsWith("!")) + .map(d -> d.substring(1)) + .collect(Collectors.toSet()); } @Override public void init(JsonNode config, diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index 2e5af4e3..a17f3fe7 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -147,6 +147,7 @@ public class StandardAppPrimerService implements AppPrimerService { final Set whiteListDomains = new HashSet<>(); final Set filterDomains = new HashSet<>(); final Set flexDomains = new HashSet<>(); + final Set flexExcludeDomains = new HashSet<>(); for (AppMatcher matcher : matchers) { final AppRuleDriver appRuleDriver = rule.initDriver(app, driver, matcher, account, device); final Set rejects = appRuleDriver.getPrimedRejectDomains(); @@ -179,8 +180,14 @@ public class StandardAppPrimerService implements AppPrimerService { } else { flexDomains.addAll(flexes); } + final Set flexExcludes = appRuleDriver.getPrimedFlexExcludeDomains(); + if (empty(flexExcludes)) { + log.debug("_prime: no flexExcludeDomains for device/app/rule/matcher: " + device.getName() + "/" + app.getName() + "/" + rule.getName() + "/" + matcher.getName()); + } else { + flexExcludeDomains.addAll(flexExcludes); + } } - if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains)) { + if (!empty(rejectDomains) || !empty(blockDomains) || !empty(filterDomains) || !empty(flexDomains) || !empty(flexExcludeDomains)) { for (String ip : accountDeviceIps.get(device.getUuid())) { if (!empty(rejectDomains)) { AppRuleDriver.defineRedisRejectSet(redis, ip, app.getName() + ":" + app.getUuid(), rejectDomains.toArray(String[]::new)); @@ -197,6 +204,9 @@ public class StandardAppPrimerService implements AppPrimerService { if (!empty(flexDomains)) { AppRuleDriver.defineRedisFlexSet(redis, ip, app.getName() + ":" + app.getUuid(), flexDomains.toArray(String[]::new)); } + if (!empty(flexExcludeDomains)) { + AppRuleDriver.defineRedisFlexExcludeSet(redis, ip, app.getName() + ":" + app.getUuid(), flexExcludeDomains.toArray(String[]::new)); + } } } } diff --git a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json index c112e5fa..1619d202 100644 --- a/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json +++ b/bubble-server/src/main/resources/models/apps/passthru/bubbleApp_passthru.json @@ -120,7 +120,7 @@ {"name": "config.view.manageFlexDomains", "value": "Manage Flex Routing Domains"}, {"name": "config.view.manageFlexFeeds", "value": "Manage Flex Routing Domain Feeds"}, {"name": "config.field.flexFqdn", "value": "Domain"}, - {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this hostname"}, + {"name": "config.field.flexFqdn.description", "value": "Use flex routing for this domain and all subdomains. Prefix with ! to exclude from flex routing."}, {"name": "config.field.flexFeedName", "value": "Name"}, {"name": "config.field.flexFeedUrl", "value": "Flex Routing Domains List URL"}, {"name": "config.field.flexFeedUrl.description", "value": "URL returning a list of domains and/or hostnames to flex route, one per line"}, diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index ac83c14b..d2a0f8a5 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:895d28911907d8f7f79cca6b70a6eda6ca4c892553cd02c0fd95060f392970a3 + checksum: sha256:1be58465d27dd8b40bc8ef9fe33c4c1dbad8dec6abb0b0c68d19754786562add - name: Unzip algo master.zip unarchive: -- 2.17.1 From 1a661d12282d7f648e2950bb4c44ad43ba104466 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 17:48:21 -0400 Subject: [PATCH 75/78] add support for flex exclusions --- .../src/main/java/bubble/rule/AppRuleDriver.java | 12 +++++++----- .../main/resources/packer/roles/algo/tasks/main.yml | 2 +- .../packer/roles/mitmproxy/files/bubble_api.py | 11 ++++++++++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java index 3d3e2fb4..beea7bef 100644 --- a/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java +++ b/bubble-server/src/main/java/bubble/rule/AppRuleDriver.java @@ -99,15 +99,17 @@ public interface AppRuleDriver { } static boolean isFlexRouteFqdn(RedisService redis, String ip, String fqdn) { - final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX; + final String excludeKey = REDIS_FLEX_EXCLUDE_LISTS + "~" + ip + REDIS_LIST_SUFFIX; + if (redis.sismember_plaintext(excludeKey, fqdn)) { + return false; + } + + final String key = REDIS_FLEX_LISTS + "~" + ip + REDIS_LIST_SUFFIX; String check = fqdn; while (true) { final boolean found = redis.sismember_plaintext(key, check); - if (found) { - final boolean excluded = redis.sismember_plaintext(excludeKey, check); - if (!excluded) return true; - } + if (found) return true; final int dotPos = check.indexOf('.'); if (dotPos == check.length()) return false; check = check.substring(dotPos+1); diff --git a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml index d2a0f8a5..b9d44a0a 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/algo/tasks/main.yml @@ -13,7 +13,7 @@ get_url: url: https://github.com/getbubblenow/bubble-dist/raw/master/algo/master.zip dest: /tmp/algo.zip - checksum: sha256:1be58465d27dd8b40bc8ef9fe33c4c1dbad8dec6abb0b0c68d19754786562add + checksum: sha256:af3e8856626248646ea496919b7bae5974e552e24a7603460e7eebc7f5c7f93f - name: Unzip algo master.zip unarchive: diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index e38e615e..19a5b1ad 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -371,8 +371,17 @@ def is_flex_domain(client_addr, fqdn): bubble_log.debug('is_flex_domain: (early) returning False for: '+fqdn) return False check_fqdn = fqdn + + exclusion_set = 'flexExcludeLists~' + client_addr + '~UNION' + excluded = REDIS.sismember(exclusion_set, fqdn) + if excluded: + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('is_flex_domain: returning False for excluded flex domain: ' + fqdn + ' (check=' + check_fqdn + ')') + return False + + flex_set = 'flexLists~' + client_addr + '~UNION' while '.' in check_fqdn: - found = REDIS.sismember('flexLists~'+client_addr+'~UNION', check_fqdn) + found = REDIS.sismember(flex_set, check_fqdn) if found: if bubble_log.isEnabledFor(DEBUG): bubble_log.debug('is_flex_domain: returning True for: '+fqdn+' (check='+check_fqdn+')') -- 2.17.1 From 52341da85a5f4f7e63c518c5c81970adf708a3ad Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 18:06:26 -0400 Subject: [PATCH 76/78] log failing url --- .../main/resources/packer/roles/mitmproxy/files/bubble_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 19a5b1ad..753213a3 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -116,7 +116,7 @@ async def async_response(client, name, url, if response.status_code != 200: if bubble_log.isEnabledFor(ERROR): - bubble_log.error('bubble_async_request(' + name + '): API call failed: ' + repr(response)) + bubble_log.error('bubble_async_request(' + name + '): API call failed ('+url+'): ' + repr(response)) return response -- 2.17.1 From df85583cf595d80891e1e80e1480718756e01dcd Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 18:10:19 -0400 Subject: [PATCH 77/78] bump version --- .../src/main/resources/META-INF/bubble/bubble.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index 0283ec9a..4c37e475 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 1.0.7 +bubble.version=Adventure 1.1.0 -- 2.17.1 From 5c3e55242190533735f98604d2f798af5cdcf2cc Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 11 Sep 2020 18:12:35 -0400 Subject: [PATCH 78/78] add file headers --- .../src/main/java/bubble/rule/passthru/BasePassthruFeed.java | 4 ++++ .../src/main/java/bubble/rule/passthru/FlexFeed.java | 4 ++++ .../src/main/java/bubble/rule/passthru/FlexFqdn.java | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java b/bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java index 84944c3e..853977e3 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/BasePassthruFeed.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.rule.passthru; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java b/bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java index ebe427f6..b8692014 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/FlexFeed.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.rule.passthru; import lombok.NoArgsConstructor; diff --git a/bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java b/bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java index b6566edd..94156e54 100644 --- a/bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java +++ b/bubble-server/src/main/java/bubble/rule/passthru/FlexFqdn.java @@ -1,3 +1,7 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ package bubble.rule.passthru; import lombok.Getter; -- 2.17.1