From e680b3a4ca189a7a21b4b5cd31ae2681e2cb10dd Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 16 Sep 2020 09:51:54 -0400 Subject: [PATCH 01/16] update web --- bubble-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-web b/bubble-web index 8845990e..fb7b574c 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 8845990e046eb27dc71da14d9348c84fed6bd7c1 +Subproject commit fb7b574c64654b22afe40e0abf23c38f99df0d4e From 55d0b21d5c6afefee7add46e69096cb0e517e483 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 16 Sep 2020 11:17:33 -0400 Subject: [PATCH 02/16] get more aggressive about disabling unattended upgrades during setup --- .../src/main/resources/ansible/install_local.sh.hbs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/resources/ansible/install_local.sh.hbs b/bubble-server/src/main/resources/ansible/install_local.sh.hbs index e5d81e09..f13e8d5a 100644 --- a/bubble-server/src/main/resources/ansible/install_local.sh.hbs +++ b/bubble-server/src/main/resources/ansible/install_local.sh.hbs @@ -7,6 +7,8 @@ LOG=/var/log/bubble/ansible.log # Stop unattended upgrades so that apt installs will work # unattended upgrades are re-enabled at the end of the ansible run systemctl stop unattended-upgrades +UNATTENDED_UPGRADES_DISABLED=20auto-upgrades-disabled +cp /usr/share/unattended-upgrades/${UNATTENDED_UPGRADES_DISABLED} /etc/apt/apt.conf.d/${UNATTENDED_UPGRADES_DISABLED} # Enable job control. Allows us to start creating dhparam in the background right now. {{#if isNode}}# For node, also allows us to install AlgoVPN in the background.{{/if}} @@ -104,6 +106,7 @@ fi kill_bg_jobs # ansible should have already restarted unattended-upgrades, but just in case -systemctl start unattended-upgrades +rm -f /etc/apt/apt.conf.d/${UNATTENDED_UPGRADES_DISABLED} +systemctl restart unattended-upgrades exit 0 From 65ed8153d104521527d81d2fd92d392932423e92 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 16 Sep 2020 18:10:51 -0400 Subject: [PATCH 03/16] shadowban work on desktop linkedin --- .../src/main/java/bubble/ApiConstants.java | 1 + .../resources/stream/FilterHttpResource.java | 97 +++++++---- .../stream/FollowThenApplyRegex.java | 16 ++ .../rule/RequestModifierRule_icon.js.hbs | 53 ++++-- .../bubble/rule/social/block/site/FB.js.hbs | 4 +- .../bubble/rule/social/block/site/LI.js.hbs | 157 ++++++++++++++---- .../roles/mitmproxy/files/bubble_api.py | 20 ++- utils/cobbzilla-utils | 2 +- 8 files changed, 270 insertions(+), 80 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/resources/stream/FollowThenApplyRegex.java diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index 70b56700..379f2d14 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -253,6 +253,7 @@ public class ApiConstants { public static final String EP_UPGRADE = "/upgrade"; public static final String EP_LOGS = "/logs"; public static final String EP_FOLLOW = "/follow"; + public static final String EP_FOLLOW_AND_APPLY_REGEX = "/followAndApplyRegex"; public static final String DETECT_ENDPOINT = "/detect"; public static final String EP_LOCALE = "/locale"; diff --git a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java index 3525f5ea..d68c43a0 100644 --- a/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java +++ b/bubble-server/src/main/java/bubble/resources/stream/FilterHttpResource.java @@ -34,13 +34,12 @@ import bubble.service.stream.StandardRuleEngineService; import com.fasterxml.jackson.databind.JsonNode; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ArrayUtils; import org.cobbzilla.util.collection.ExpirationEvictionPolicy; import org.cobbzilla.util.collection.ExpirationMap; import org.cobbzilla.util.collection.NameAndValue; import org.cobbzilla.util.http.HttpContentEncodingType; -import org.cobbzilla.util.http.HttpUtil; import org.cobbzilla.util.network.NetworkUtil; -import org.cobbzilla.util.string.StringUtil; import org.cobbzilla.wizard.cache.redis.RedisService; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -64,15 +63,16 @@ import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIM import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY; import static java.util.Collections.emptyMap; import static java.util.concurrent.TimeUnit.*; -import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; +import static org.apache.http.HttpHeaders.*; import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; import static org.cobbzilla.util.http.HttpContentTypes.TEXT_PLAIN; import static org.cobbzilla.util.http.HttpUtil.applyRegexToUrl; -import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; -import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.http.HttpUtil.chaseRedirects; +import static org.cobbzilla.util.json.JsonUtil.*; import static org.cobbzilla.util.network.NetworkUtil.isLocalIpv4; +import static org.cobbzilla.util.security.ShaUtil.sha256_hex; import static org.cobbzilla.util.string.StringUtil.trimQuotes; import static org.cobbzilla.wizard.cache.redis.RedisService.EX; import static org.cobbzilla.wizard.model.NamedEntity.names; @@ -446,8 +446,17 @@ public class FilterHttpResource { public Response flushCaches(@Context ContainerRequest request) { final Account caller = userPrincipal(request); if (!caller.admin()) return forbidden(); + + final int connCheckMatcherCacheSize = connCheckMatcherCache.size(); connCheckMatcherCache.clear(); - return ok(ruleEngine.flushCaches()); + + // disable redirect flushing for now -- it works well and it's a lot of work + // final Long redirectCacheSize = getRedirectCache().del_matching("*"); + + final Map flushes = ruleEngine.flushCaches(); + flushes.put("connCheckMatchersCache", connCheckMatcherCacheSize); + // flushes.put("redirectCache", redirectCacheSize == null ? 0 : redirectCacheSize); + return ok(flushes); } @DELETE @Path(EP_MATCHERS) @@ -690,40 +699,68 @@ public class FilterHttpResource { return ok_empty(); } - private final Map redirectCache - = new ExpirationMap<>(1000, DAYS.toMillis(3), ExpirationEvictionPolicy.atime); + public static final String REDIS_PREFIX_REDIRECT_CACHE = "followLink_"; + @Getter(lazy=true) private final RedisService redirectCache = redis.prefixNamespace(REDIS_PREFIX_REDIRECT_CACHE); @POST @Path(EP_FOLLOW+"/{requestId}") + @Consumes(APPLICATION_JSON) @Produces(TEXT_PLAIN) public Response followLink(@Context Request req, @Context ContainerRequest ctx, @PathParam("requestId") String requestId, - JsonNode followSpec) { + JsonNode urlNode) { final FilterSubContext filterCtx = new FilterSubContext(req, requestId); + final RedisService cache = getRedirectCache(); + final String url = urlNode.textValue(); + final String cacheKey = sha256_hex(url); + final String cachedValue = cache.get(cacheKey); + if (cachedValue != null) return ok(cachedValue); + + final String result = chaseRedirects(url); + cache.set(cacheKey, result, EX, DAYS.toMillis(365)); + return ok(result); + } - // is this a request to parse regexes from a URL? - if (followSpec.has("regex")) { - return ok(redirectCache.computeIfAbsent(json(followSpec), k -> { - final String url = followSpec.get("url").textValue(); - final String regex = followSpec.get("regex").textValue(); - final Integer group = followSpec.has("group") ? followSpec.get("group").asInt() : null; - final List headers = new ArrayList<>(); - for (String name : req.getHeaderNames()) { - final String value = req.getHeader(name); - headers.add(new NameAndValue(name, value)); - } - final List matches = applyRegexToUrl(url, headers, regex, group); - return matches == null ? null : StringUtil.toString(matches, "\n"); - })); + public static final String CLIENT_HEADER_PREFIX = "X-Bubble-Client-Header-"; - } else if (followSpec.isTextual()) { - // just a regular follow -- chase redirects - return ok(redirectCache.computeIfAbsent(followSpec.textValue(), HttpUtil::chaseRedirects)); - } else { - final String json = json(followSpec); - log.error("followLink: invalid json (expected String or {regex, url}): "+json); - return notFound(json); + public static final String[] EXCLUDED_CLIENT_HEADERS = { + ACCEPT.toLowerCase(), + CONTENT_TYPE.toLowerCase(), CONTENT_LENGTH.toLowerCase(), + CONTENT_ENCODING.toLowerCase(), TRANSFER_ENCODING.toLowerCase() + }; + + @POST @Path(EP_FOLLOW_AND_APPLY_REGEX+"/{requestId}") + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + public Response followLinkThenApplyRegex(@Context Request req, + @Context ContainerRequest ctx, + @PathParam("requestId") String requestId, + FollowThenApplyRegex follow) { + final FilterSubContext filterCtx = new FilterSubContext(req, requestId); + final RedisService cache = getRedirectCache(); + final String followJson = json(follow); + final String cacheKey = sha256_hex(followJson); + final String cachedValue = cache.get(cacheKey); + if (cachedValue != null) return ok(cachedValue); + + // collect client headers + final List headers = new ArrayList<>(); + for (String name : req.getHeaderNames()) { + if (name.toLowerCase().startsWith(CLIENT_HEADER_PREFIX.toLowerCase())) { + final String value = req.getHeader(name); + final String realName = name.substring(CLIENT_HEADER_PREFIX.length()); + if (ArrayUtils.indexOf(EXCLUDED_CLIENT_HEADERS, realName.toLowerCase()) == -1) { + headers.add(new NameAndValue(realName, value)); + } + } } + headers.add(new NameAndValue(ACCEPT, "*/*")); + final List> matches + = applyRegexToUrl(follow.getUrl(), headers, follow.getRegex(), Arrays.asList(follow.getGroups())); + if (log.isWarnEnabled()) log.warn("followLink(" + follow.getUrl() + ") returning: " + json(matches)); + final String result = matches == null ? EMPTY_JSON_ARRAY : json(matches); + cache.set(cacheKey, result, EX, DAYS.toMillis(365)); + return ok(result); } @Path(EP_ASSETS+"/{requestId}/{appId}") diff --git a/bubble-server/src/main/java/bubble/resources/stream/FollowThenApplyRegex.java b/bubble-server/src/main/java/bubble/resources/stream/FollowThenApplyRegex.java new file mode 100644 index 00000000..3fed138b --- /dev/null +++ b/bubble-server/src/main/java/bubble/resources/stream/FollowThenApplyRegex.java @@ -0,0 +1,16 @@ +/** + * Copyright (c) 2020 Bubble, Inc. All rights reserved. + * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ + */ +package bubble.resources.stream; + +import lombok.Getter; +import lombok.Setter; + +public class FollowThenApplyRegex { + + @Getter @Setter private String url; + @Getter @Setter private String regex; + @Getter @Setter private Integer[] groups; + +} diff --git a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs index 52706e65..7c5c3ed7 100644 --- a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs @@ -87,9 +87,12 @@ function {{JS_PREFIX}}_create_button(labelKey, labelDefault, onclick, labelForma return btn; } -if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { +{{JS_PREFIX}}_follow_url = '/__bubble/api/filter/follow/{{BUBBLE_REQUEST_ID}}'; +{{JS_PREFIX}}_follow_and_apply_regex_url = '/__bubble/api/filter/followAndApplyRegex/{{BUBBLE_REQUEST_ID}}'; + +{{JS_PREFIX}}_url_chasers = {}; - let {{PAGE_PREFIX}}_url_chasers = {}; +if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { {{PAGE_PREFIX}}_screenWidth = function () { return window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth }; @@ -232,10 +235,10 @@ if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { }); } -function {{JS_PREFIX}}_chase_redirects (a, removeParams) { +function {{JS_PREFIX}}_chase_redirects (a, removeParams, regex, groups, callback) { const initial_href = a.href; - if (initial_href in {{PAGE_PREFIX}}_url_chasers) { - a.href = {{PAGE_PREFIX}}_url_chasers[initial_href]; + if (initial_href in {{JS_PREFIX}}_url_chasers) { + a.href = {{JS_PREFIX}}_url_chasers[initial_href]; return; } if (a.className && a.className.indexOf('{{JS_PREFIX}}_followed') !== -1) return; @@ -247,20 +250,38 @@ function {{JS_PREFIX}}_chase_redirects (a, removeParams) { a.rel = 'noopener noreferrer nofollow'; - fetch('/__bubble/api/filter/follow/{{BUBBLE_REQUEST_ID}}', {method: 'POST', body: JSON.stringify(initial_href)}) - .then(response => response.text()) + let is_regex = (typeof regex !== 'undefined'); + const follow_body = !is_regex ? initial_href : + { + 'url': initial_href, + 'regex': regex, + 'groups': (typeof groups === 'undefined' || groups === null ? null : groups) + }; + const request_opts = { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify(follow_body) + } + const follow_url = is_regex ? {{JS_PREFIX}}_follow_and_apply_regex_url : {{JS_PREFIX}}_follow_url; + console.log('>>>>>>>>>>>> requesting follow_url ('+follow_url+') with request_opts: '+JSON.stringify(request_opts)); + fetch(follow_url, request_opts) + .then(response => is_regex ? response.json() : response.text()) .then(data => { - if (data && (data.startsWith('http://') || data.startsWith('https://'))) { - if (typeof removeParams === 'undefined' || removeParams === null || removeParams) { - const qPos = data.indexOf('?'); - a.href = qPos === -1 ? data : data.substring(0, qPos); + if (is_regex) { + callback(data); + } else { + if (data && (data.startsWith('http://') || data.startsWith('https://'))) { + if (typeof removeParams === 'undefined' || removeParams === null || removeParams) { + const qPos = data.indexOf('?'); + a.href = qPos === -1 ? data : data.substring(0, qPos); + } else { + a.href = data; + } + {{JS_PREFIX}}_url_chasers[initial_href] = a.href; + // console.log('chase_redirect: rewrote '+initial_href+' -> '+a.href); } else { - a.href = data; + console.warn('chase_redirects: ' + a.href + ' returned non-URL response: ' + data); } - {{PAGE_PREFIX}}_url_chasers[initial_href] = a.href; - // console.log('chase_redirect: rewrote '+initial_href+' -> '+a.href); - } else { - console.warn('chase_redirects: '+a.href+' returned non-URL response: '+data); } }) .catch((error) => { diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/site/FB.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/site/FB.js.hbs index cd2938b7..058c6513 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/site/FB.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/site/FB.js.hbs @@ -32,7 +32,7 @@ Element.prototype.appendChild = function() { const block = {{JS_PREFIX}}_should_block({{JS_PREFIX}}_blocked_users, node) if (block) { // log('>>> BLOCKING via appendChild: '+block); - {{JS_PREFIX}}_appendChild.apply({{JS_PREFIX}}_jail, arguments); + return {{JS_PREFIX}}_appendChild.apply({{JS_PREFIX}}_jail, arguments); } } } @@ -40,7 +40,7 @@ Element.prototype.appendChild = function() { console.log('>>> error inspecting: e='+e); } try { - {{JS_PREFIX}}_appendChild.apply(this, arguments); + return {{JS_PREFIX}}_appendChild.apply(this, arguments); } catch (e) { console.log('>>> error calling document.appendChild: arg[0].tagName = '+node.tagName+' e='+e); } diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs index d38f958a..c4922e96 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs @@ -1,8 +1,21 @@ {{JS_PREFIX}}_supports_keywords = true; +{{JS_PREFIX}}_idle_interval = 5000; const {{JS_PREFIX}}_site_host = location.protocol + '//' + window.location.hostname + '/'; +const {{JS_PREFIX}}_jail = document.createElement('div'); +{{JS_PREFIX}}_jail.style.display = 'none'; + function {{JS_PREFIX}}_apply_blocks(blocked_users) { + const adBanner = Array.from(document.getElementsByTagName('iframe')).find(i => i.className && i.className === 'ad-banner'); + if (typeof adBanner !== 'undefined') { + let adParent = adBanner.parentNode; + if (adParent != null) { + adParent.innerHTML = ''; + adParent.style.display = 'none'; + } else { + } + } const articles = Array.from(document.getElementsByClassName('feed-shared-update-v2')); if (articles === null || articles.length === 0) { console.warn('No articles found, not filtering'); @@ -11,7 +24,21 @@ function {{JS_PREFIX}}_apply_blocks(blocked_users) { {{JS_PREFIX}}_consider_block(articles, blocked_users); } -function {{JS_PREFIX}}_author_from_href(href) { +function {{JS_PREFIX}}_is_valid_author_name(name) { + return !(name.startsWith('ACoAA') || name.length >= 38); +} + +function {{JS_PREFIX}}_author_from_href(linkId, callback) { + if (typeof linkId === 'undefined' || linkId === null || linkId.length === 0) { + console.log('author_from_href: invalid link ID: '+linkId); + return; + } + const link = document.getElementById(linkId); + if (link === null) { + // console.log('author_from_href: link with ID '+linkId+' not found'); + return; + } + const href = link.href; if (typeof href === 'undefined' || href === null) return null; let h = href.startsWith({{JS_PREFIX}}_site_host) ? href.substring({{JS_PREFIX}}_site_host.length) : href; const qPos = h.indexOf('?'); @@ -19,14 +46,51 @@ function {{JS_PREFIX}}_author_from_href(href) { h = h.substring(0, qPos); } if (h.endsWith('/')) h = h.substring(0, h.length - 1); - if (!h.startsWith('in/') && !h.startsWith('company/')) { - return null; + let profile_type = null; + if (h.startsWith('in/')) { + profile_type = 'in/'; + } else if (h.startsWith('company/')) { + profile_type = 'company/'; + } else { + // console.log("author_from_href: skipping (not in/ or company/) href: "+href); + return; } const slashPos = h.indexOf('/'); - const name = h.substring(slashPos); - if (name.length > 35 && name.indexOf('-') === -1 && name.indexOf('_') === -1) return null; - console.log("author_from_href: found "+name+' from '+href); - return name; + const name = h.substring(slashPos+1); + if ({{JS_PREFIX}}_is_valid_author_name(name)) { + // console.log("author_from_href: found " + name + ' from ' + href); + callback(linkId, name); + } else { + // only chase a link once + let linkClass = link.className; + const chaseClass = '{{JS_PREFIX}}_link_chased'; + if (linkClass && linkClass.indexOf(chaseClass) !== -1) { + return; + } else { + link.className = link.className ? link.className + ' '+chaseClass : chaseClass; + } + + {{JS_PREFIX}}_chase_redirects(link, true, '/voyager/api/identity/profiles/([^/]+)/privacySettings', [1], function (matches) { + if (typeof matches.length !== 'undefined') { + for (let i=0; i 0) { for (let i=0; i 0) { + appendToSpan = authorSpans[0]; + } else { + appendToSpan = Array.from(realLink.getElementsByTagName('span')) + .find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type')); + if (typeof appendToSpan === 'undefined') { + console.log('consider_block: found no span to attach block control for author: '+author); + return; + } + } + + let b = {{JS_PREFIX}}_create_block_control(article, author, realLink); + if (b !== null) { + console.log('consider_block: inserting span='+b.id+' for article by '+author); + appendToSpan.parentNode.appendChild(b); + {{JS_PREFIX}}_tally_allow(); + } else { + console.log('consider_block: create_block_control returned null for author '+author) + } } - } + }); } } } diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index e16d3ac1..7ffaea77 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -469,6 +469,10 @@ def health_check_response(flow): flow.response.stream = lambda chunks: [b'OK\n'] +def include_request_headers(path): + return '/followAndApplyRegex' in path + + def special_bubble_response(flow): name = 'special_bubble_response' path = flow.request.path @@ -478,7 +482,7 @@ def special_bubble_response(flow): uri = make_bubble_special_path(path) if bubble_log.isEnabledFor(DEBUG): - bubble_log.debug('special_bubble_response: sending special bubble request to '+uri) + bubble_log.debug('special_bubble_response: sending special bubble '+flow.request.method+' to '+uri) headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' @@ -489,12 +493,22 @@ def special_bubble_response(flow): response = async_stream(client, name, uri, headers=headers, loop=loop) elif flow.request.method == 'POST': - loop = asyncio.new_event_loop() - client = async_client(timeout=30) + if include_request_headers(flow.request.path): + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('special_bubble_request: including client headers: '+repr(flow.request.headers)) + # add client request headers + for name, value in flow.request.headers.items(): + headers['X-Bubble-Client-Header-'+name] = value + if bubble_log.isEnabledFor(DEBUG): + bubble_log.debug('special_bubble_request: NOW headers='+repr(headers)) + data = None if flow.request.content and flow.request.content: headers[HEADER_CONTENT_LENGTH] = str(len(flow.request.content)) data = flow.request.content + + loop = asyncio.new_event_loop() + client = async_client(timeout=30) response = async_stream(client, name, uri, headers=headers, method='POST', data=data, loop=loop) else: diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index ea72ac4a..dfafe62c 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit ea72ac4a1619c4f5915047650cdd18b8a6202681 +Subproject commit dfafe62c7eb3413cf1210e40e551094458f4d9d0 From 6b2928af44c9f98fad3ff6286ff7b1b6e53691e3 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 16 Sep 2020 18:13:26 -0400 Subject: [PATCH 04/16] update web --- bubble-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-web b/bubble-web index fb7b574c..0748d749 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit fb7b574c64654b22afe40e0abf23c38f99df0d4e +Subproject commit 0748d749df4bddac2ed011f2f256e9ca7f2a7e31 From 78ac4ff05d415d130728b5f6eedfb16bd86082e1 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 16 Sep 2020 18:16:40 -0400 Subject: [PATCH 05/16] disable in-page stats for more app-heavy sites --- .../bubbleApp_bubbleBlock_data.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json index e0d36a41..cad539c0 100644 --- a/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json +++ b/bubble-server/src/main/resources/models/apps/bubble_block/bubbleApp_bubbleBlock_data.json @@ -247,6 +247,12 @@ "matcher": "BubbleBlockMatcher", "key": "hideStats_reddit.com", "data": "true" + }, { + "site": "All_Sites", + "template": true, + "matcher": "BubbleBlockMatcher", + "key": "hideStats_redfin.com", + "data": "true" }, { "site": "All_Sites", "template": true, @@ -319,6 +325,12 @@ "matcher": "BubbleBlockMatcher", "key": "hideStats_tripactions.com", "data": "true" + }, { + "site": "All_Sites", + "template": true, + "matcher": "BubbleBlockMatcher", + "key": "hideStats_trulia.com", + "data": "true" }, { "site": "All_Sites", "template": true, @@ -373,6 +385,12 @@ "matcher": "BubbleBlockMatcher", "key": "hideStats_zendesk.com", "data": "true" + }, { + "site": "All_Sites", + "template": true, + "matcher": "BubbleBlockMatcher", + "key": "hideStats_zillow.com", + "data": "true" }, { "site": "All_Sites", "template": true, From d0ba76f027b83f374a3c0ddf9a396ddb25c4da62 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 16 Sep 2020 19:26:25 -0400 Subject: [PATCH 06/16] shadowban for linkedin working on mobile --- .../java/bubble/model/app/AppMatcher.java | 3 +- .../META-INF/bubble/bubble.properties | 2 +- .../rule/RequestModifierRule_icon.js.hbs | 1 - .../block/JsUserBlockerRuleDriver.js.hbs | 5 +- .../bubble/rule/social/block/site/LI.js.hbs | 76 +++++++++++++++---- .../li/bubbleApp_userBlock_li_data.json | 8 +- .../li/bubbleApp_userBlock_li_matchers.json | 10 +++ 7 files changed, 78 insertions(+), 27 deletions(-) diff --git a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java index 3acced4c..859df5a0 100644 --- a/bubble-server/src/main/java/bubble/model/app/AppMatcher.java +++ b/bubble-server/src/main/java/bubble/model/app/AppMatcher.java @@ -44,7 +44,8 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, HasPriority { public static final String[] VALUE_FIELDS = { - "fqdn", "urlRegex", "template", "enabled", "priority", "connCheck", "requestCheck", "requestModifier" + "fqdn", "urlRegex", "userAgentRegex", "template", "enabled", "priority", + "connCheck", "requestCheck", "requestModifier" }; public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule"); diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index 443625cc..15362a1d 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 1.1.3 +bubble.version=Adventure 1.1.4 diff --git a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs index 7c5c3ed7..70c249c4 100644 --- a/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/RequestModifierRule_icon.js.hbs @@ -263,7 +263,6 @@ function {{JS_PREFIX}}_chase_redirects (a, removeParams, regex, groups, callback body: JSON.stringify(follow_body) } const follow_url = is_regex ? {{JS_PREFIX}}_follow_and_apply_regex_url : {{JS_PREFIX}}_follow_url; - console.log('>>>>>>>>>>>> requesting follow_url ('+follow_url+') with request_opts: '+JSON.stringify(request_opts)); fetch(follow_url, request_opts) .then(response => is_regex ? response.json() : response.text()) .then(data => { diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs index be698074..2532a89c 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/JsUserBlockerRuleDriver.js.hbs @@ -72,8 +72,11 @@ function {{JS_PREFIX}}_uuidv4() { const {{JS_PREFIX}}_create_block_img = function(size) { const img = document.createElement('img'); + img.style.all = 'revert'; + img.style.fontSize = 'x-small'; img.src = {{JS_PREFIX}}_asset_img_url('icon'); - img.width = typeof size !== 'undefined' ? size : 24; + img.width = typeof size !== 'undefined' && size !== null ? size : 24; + img.style.width = img.width+'px'; return img; } diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs index c4922e96..9070df8c 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs @@ -3,6 +3,14 @@ const {{JS_PREFIX}}_site_host = location.protocol + '//' + window.location.hostname + '/'; +function {{JS_PREFIX}}_mobile() { + const html = Array.from(document.getElementsByTagName('html')); + if (html.length !== 0) { + return html[0].className && html[0].className.indexOf(' mobile ') !== -1; + } + return false; +} + const {{JS_PREFIX}}_jail = document.createElement('div'); {{JS_PREFIX}}_jail.style.display = 'none'; @@ -16,7 +24,9 @@ function {{JS_PREFIX}}_apply_blocks(blocked_users) { } else { } } - const articles = Array.from(document.getElementsByClassName('feed-shared-update-v2')); + const articles = {{JS_PREFIX}}_mobile() + ? Array.from(document.getElementsByClassName('feed-item')) + : Array.from(document.getElementsByClassName('feed-shared-update-v2')); if (articles === null || articles.length === 0) { console.warn('No articles found, not filtering'); return; @@ -35,7 +45,7 @@ function {{JS_PREFIX}}_author_from_href(linkId, callback) { } const link = document.getElementById(linkId); if (link === null) { - // console.log('author_from_href: link with ID '+linkId+' not found'); + console.log('author_from_href: link with ID '+linkId+' not found'); return; } const href = link.href; @@ -47,12 +57,17 @@ function {{JS_PREFIX}}_author_from_href(linkId, callback) { } if (h.endsWith('/')) h = h.substring(0, h.length - 1); let profile_type = null; + + const mobile = {{JS_PREFIX}}_mobile(); + if (mobile && h.startsWith('mwlite/')) { + h = h.substring('mwlite/'.length); + } if (h.startsWith('in/')) { profile_type = 'in/'; } else if (h.startsWith('company/')) { profile_type = 'company/'; } else { - // console.log("author_from_href: skipping (not in/ or company/) href: "+href); + // console.log("author_from_href: skipping (not in/ or company/) href: "+href+', h='+h); return; } const slashPos = h.indexOf('/'); @@ -127,20 +142,56 @@ function {{JS_PREFIX}}_create_block_control(article, authorName, articleLink) { function {{JS_PREFIX}}_hash_url(url) { return btoa(url).replaceAll('=', ''); } +function {{JS_PREFIX}}_is_ad(article) { + const mobile = {{JS_PREFIX}}_mobile(); + return (mobile && article.getAttribute('data-is-sponsored') && article.getAttribute('data-is-sponsored') !== "false") + || (article.innerHTML.indexOf('Promoted') !== -1) + || (article.innerHTML.indexOf('Promoted') !== -1); +} + +function {{JS_PREFIX}}_find_append_span(link) { + const mobile = {{JS_PREFIX}}_mobile(); + if (!mobile) { + let authorSpans = Array.from(link.getElementsByClassName('feed-shared-actor__name')); + if (authorSpans.length > 0) { + return authorSpans[0]; + } else { + return Array.from(link.getElementsByTagName('span')) + .find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type')); + } + } else { + const ltrSpan = Array.from(link.getElementsByTagName('span')) + .find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type')); + if (ltrSpan) return ltrSpan; + if (link.className && link.className.indexOf('profile-link') !== -1) { + return link.lastChild; + } + } +} + function {{JS_PREFIX}}_consider_block(articles, blocked_users) { + const mobile = {{JS_PREFIX}}_mobile(); if (articles && articles.length && articles.length > 0) { for (let i=0; i !a.hasAttribute('aria-hidden')) + : Array.from(article.getElementsByTagName('a')); + // console.log('consider_block: found '+articleLinks.length+' articleLinks'); for (let j=0; j 0) { - appendToSpan = authorSpans[0]; - } else { - appendToSpan = Array.from(realLink.getElementsByTagName('span')) - .find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type')); - if (typeof appendToSpan === 'undefined') { - console.log('consider_block: found no span to attach block control for author: '+author); - return; - } + let appendToSpan = {{JS_PREFIX}}_find_append_span(realLink); + if (!appendToSpan) { + console.log('consider_block: no span found to append to for author '+author); + return; } let b = {{JS_PREFIX}}_create_block_control(article, author, realLink); diff --git a/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_data.json b/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_data.json index bb4b1aef..ca5fe4bf 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_data.json +++ b/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_data.json @@ -1,12 +1,6 @@ [{ "name": "UserBlocker", "children": { - "AppData": [{ - "site": "LinkedIn", - "template": true, - "matcher": "LIMatcher", - "key": "kw:_Promoted", - "data": "true" - }] + "AppData": [] } }] diff --git a/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json b/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json index fba42acd..8ddaa55a 100644 --- a/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json +++ b/bubble-server/src/main/resources/models/apps/user_block/li/bubbleApp_userBlock_li_matchers.json @@ -10,6 +10,16 @@ "fqdn": "www.linkedin.com", "urlRegex": "/feed/", "rule": "li_user_blocker" + }, { + "name": "LIMobileMatcher", + "site": "LinkedIn", + "template": true, + "requestCheck": true, + "requestModifier": true, + "fqdn": "www.linkedin.com", + "urlRegex": "/", + "userAgentRegex": "Mobi|Android", + "rule": "li_user_blocker" }] } }] From f151af308d25f389be9e7a7f187f9680700ec924 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 16 Sep 2020 19:28:04 -0400 Subject: [PATCH 07/16] remove logs --- .../bubble/rule/social/block/site/LI.js.hbs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs index 9070df8c..999c03c3 100644 --- a/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs +++ b/bubble-server/src/main/resources/bubble/rule/social/block/site/LI.js.hbs @@ -40,12 +40,12 @@ function {{JS_PREFIX}}_is_valid_author_name(name) { function {{JS_PREFIX}}_author_from_href(linkId, callback) { if (typeof linkId === 'undefined' || linkId === null || linkId.length === 0) { - console.log('author_from_href: invalid link ID: '+linkId); + // console.log('author_from_href: invalid link ID: '+linkId); return; } const link = document.getElementById(linkId); if (link === null) { - console.log('author_from_href: link with ID '+linkId+' not found'); + // console.log('author_from_href: link with ID '+linkId+' not found'); return; } const href = link.href; @@ -136,7 +136,7 @@ function {{JS_PREFIX}}_create_block_control(article, authorName, articleLink) { blockSpan.appendChild(document.createTextNode('\u00A0\u00A0')); blockSpan.appendChild(blockLink); blockSpan.id = 'blockSpan_'+{{JS_PREFIX}}_uuidv4(); - console.log('adding block control on '+authorName); + // console.log('adding block control on '+authorName); return blockSpan; } @@ -214,7 +214,7 @@ function {{JS_PREFIX}}_consider_block(articles, blocked_users) { const realLink = document.getElementById(linkId); if (realLink === null) { - console.log('consider_block: link with id '+linkId+' seems to have disappeared from the document'); + // console.log('consider_block: link with id '+linkId+' seems to have disappeared from the document'); return; } @@ -227,17 +227,17 @@ function {{JS_PREFIX}}_consider_block(articles, blocked_users) { } else if (!seenBefore) { let appendToSpan = {{JS_PREFIX}}_find_append_span(realLink); if (!appendToSpan) { - console.log('consider_block: no span found to append to for author '+author); + // console.log('consider_block: no span found to append to for author '+author); return; } let b = {{JS_PREFIX}}_create_block_control(article, author, realLink); if (b !== null) { - console.log('consider_block: inserting span='+b.id+' for article by '+author); + // console.log('consider_block: inserting span='+b.id+' for article by '+author); appendToSpan.parentNode.appendChild(b); {{JS_PREFIX}}_tally_allow(); } else { - console.log('consider_block: create_block_control returned null for author '+author) + // console.log('consider_block: create_block_control returned null for author '+author) } } }); From 1c1d5c380f278bd870f6677c50842161249a4db4 Mon Sep 17 00:00:00 2001 From: jonathan Date: Thu, 17 Sep 2020 11:40:07 +0000 Subject: [PATCH 08/16] feature/tarpit (#56) add file header bump version add ssh tarpit, add fail2ban ssh config, harden ssh fix bug, supervisor conf is a file not a template initial http tarpit Co-authored-by: Jonathan Cobb Reviewed-on: https://git.bubblev.org/bubblev/bubble/pulls/56 --- .../META-INF/bubble/bubble.properties | 2 +- .../ansible/roles/mitmproxy/tasks/main.yml | 4 +- .../src/main/resources/packer/node-roles.txt | 1 + .../roles/firewall/files/bubble_sshd.conf | 9 ++ .../packer/roles/firewall/files/jail.local | 5 ++ .../packer/roles/firewall/tasks/main.yml | 23 ++++- .../packer/roles/firewall/tasks/rules.yml | 7 +- .../roles/mitmproxy/files/bubble_api.py | 19 ++++ .../roles/mitmproxy/files/bubble_request.py | 8 +- .../roles/tarpit/files/bubble_http_tarpit.py | 89 +++++++++++++++++++ .../roles/tarpit/files/bubble_ssh_tarpit.py | 87 ++++++++++++++++++ .../tarpit/files/supervisor_http_tarpit.conf | 6 ++ .../tarpit/files/supervisor_ssh_tarpit.conf | 6 ++ .../packer/roles/tarpit/tasks/main.yml | 54 +++++++++++ 14 files changed, 309 insertions(+), 11 deletions(-) create mode 100644 bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf create mode 100644 bubble-server/src/main/resources/packer/roles/firewall/files/jail.local create mode 100644 bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_http_tarpit.py create mode 100644 bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_ssh_tarpit.py create mode 100644 bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_http_tarpit.conf create mode 100644 bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_ssh_tarpit.conf create mode 100644 bubble-server/src/main/resources/packer/roles/tarpit/tasks/main.yml diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index 15362a1d..dba5e398 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 1.1.4 +bubble.version=Adventure 1.2.0 diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml index 50188ca6..f37289dc 100644 --- a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml @@ -18,7 +18,7 @@ - name: Ensure mitmproxy user owns all mitmproxy files shell: chown -R mitmproxy /home/mitmproxy/mitmproxy -- name: Install mitmproxy1 supervisor conf file +- name: Install mitm8888 supervisor conf file template: src: supervisor_mitmproxy.conf.j2 dest: /etc/supervisor/conf.d/mitm8888.conf @@ -28,7 +28,7 @@ vars: port: 8888 -- name: Install mitmproxy2 supervisor conf file +- name: Install mitm9999 supervisor conf file template: src: supervisor_mitmproxy.conf.j2 dest: /etc/supervisor/conf.d/mitm9999.conf diff --git a/bubble-server/src/main/resources/packer/node-roles.txt b/bubble-server/src/main/resources/packer/node-roles.txt index 1fbe509d..77889ca5 100644 --- a/bubble-server/src/main/resources/packer/node-roles.txt +++ b/bubble-server/src/main/resources/packer/node-roles.txt @@ -1,5 +1,6 @@ common firewall +tarpit nginx algo mitmproxy diff --git a/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf b/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf new file mode 100644 index 00000000..db18fb25 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf @@ -0,0 +1,9 @@ +Port 1202 +LoginGraceTime 10 +PasswordAuthentication no +PermitEmptyPasswords no +ChallengeResponseAuthentication no +KerberosAuthentication no +GSSAPIAuthentication no +X11Forwarding no +PermitUserEnvironment no diff --git a/bubble-server/src/main/resources/packer/roles/firewall/files/jail.local b/bubble-server/src/main/resources/packer/roles/firewall/files/jail.local new file mode 100644 index 00000000..cd957ea7 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/firewall/files/jail.local @@ -0,0 +1,5 @@ +[sshd] +mode = aggressive +port = 1202 +logpath = %(sshd_log)s +backend = %(sshd_backend)s diff --git a/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml index 97b04e6d..8646fb2c 100644 --- a/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml @@ -90,15 +90,34 @@ dest: /usr/local/bin/bubble_peer_manager.py owner: root group: root - mode: 0555 + mode: 0550 when: fw_enable_admin -- name: Install supervisor conf file for port manager +- name: Install supervisor conf file for peer manager copy: src: supervisor_bubble_peer_manager.conf dest: /etc/supervisor/conf.d/bubble_peer_manager.conf + owner: root + group: root + mode: 0550 when: fw_enable_admin +- name: Install SSH hardening settings + copy: + src: bubble_sshd.conf + dest: /etc/ssh/ssh_config.d/bubble_sshd.conf + owner: root + group: root + mode: 0400 + +- name: Install SSH fail2ban settings + copy: + src: jail.local + dest: /etc/fail2ban/jail.local + owner: root + group: root + mode: 0400 + - include: rules.yml - supervisorctl: diff --git a/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml b/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml index bff800b7..f3a1cf3f 100644 --- a/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml +++ b/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml @@ -17,15 +17,18 @@ comment: Allow related and established connections become: yes -- name: Allow SSH +- name: Allow SSH on ports 22 and 1202 iptables: chain: INPUT protocol: tcp - destination_port: 22 + destination_port: "{{ item }}" ctstate: NEW syn: match jump: ACCEPT comment: Accept new SSH connections + with_items: + - 22 + - 1202 become: yes when: fw_enable_ssh diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py index 7ffaea77..47f88908 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_api.py @@ -64,6 +64,7 @@ LOCAL_IPS = [] for local_ip in subprocess.check_output(['hostname', '-I']).split(): LOCAL_IPS.append(local_ip.decode()) +TARPIT_PORT = 8080 VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4) VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6) @@ -469,6 +470,24 @@ def health_check_response(flow): flow.response.stream = lambda chunks: [b'OK\n'] +def tarpit_response(flow, host): + # if bubble_log.isEnabledFor(DEBUG): + # bubble_log.debug('health_check_response: special bubble health check request, responding with OK') + response_headers = nheaders.Headers() + response_headers[HEADER_LOCATION] = 'http://'+host+':'+str(TARPIT_PORT)+'/admin/index.php' + if flow.response is None: + flow.response = http.HTTPResponse(http_version='HTTP/1.1', + status_code=301, + reason='Moved Permanently', + headers=response_headers, + content=b'') + else: + flow.response.headers = nheaders.Headers() + flow.response.headers = response_headers + flow.response.status_code = 301 + flow.response.reason = 'Moved Permanently' + + def include_request_headers(path): return '/followAndApplyRegex' in path diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index 9615a56f..b62d0110 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -32,7 +32,7 @@ from mitmproxy.net.http import headers as nheaders from bubble_api import bubble_matchers, bubble_activity_log, \ CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ - is_bubble_special_path, is_bubble_health_check, health_check_response, \ + is_bubble_special_path, is_bubble_health_check, health_check_response, tarpit_response,\ is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain from bubble_config import bubble_host, bubble_host_alias from bubble_flex import new_flex_flow @@ -170,9 +170,9 @@ class Rerouter: elif is_not_from_vpn(client_addr): # todo: add to fail2ban if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url+' host='+host) - bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns) - add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) + bubble_log.warning('bubble_handle_request: sending to tarpit: non-VPN client='+client_addr+', url='+log_url+' host='+host) + bubble_activity_log(client_addr, server_addr, 'http_tarpit_non_vpn', fqdns) + tarpit_response(flow, host) return None if is_bubble_special_path(path): diff --git a/bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_http_tarpit.py b/bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_http_tarpit.py new file mode 100644 index 00000000..289ecc0f --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_http_tarpit.py @@ -0,0 +1,89 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Adapted from: https://nullprogram.com/blog/2019/03/22/ +# +import asyncio +import random +import os +import sys + +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + +from pathlib import Path + +TARPIT_LOG = '/var/log/bubble/http_tarpit.log' +TARPIT_LOG_LEVEL_FILE = '/home/tarpit/http_tarpit_log_level.txt' +TARPIT_LOG_LEVEL_ENV_VAR = 'HTTP_TARPIT_LOG_LEVEL' +DEFAULT_TARPIT_LOG_LEVEL = 'INFO' +TARPIT_LOG_LEVEL = None + +TARPIT_PORT_FILE = '/home/tarpit/http_tarpit_port.txt' +TARPIT_PORT_ENV_VAR = 'HTTP_TARPIT_PORT' +DEFAULT_TARPIT_PORT = '8080' + +tarpit_log = logging.getLogger(__name__) + +try: + TARPIT_LOG_LEVEL = Path(TARPIT_LOG_LEVEL_FILE).read_text().strip() +except IOError: + print('error reading log level from '+TARPIT_LOG_LEVEL_FILE+', checking env var '+TARPIT_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True) + TARPIT_LOG_LEVEL = os.getenv(TARPIT_LOG_LEVEL_ENV_VAR, DEFAULT_TARPIT_LOG_LEVEL) + +TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, TARPIT_LOG_LEVEL.upper(), None) +if not isinstance(TARPIT_NUMERIC_LOG_LEVEL, int): + print('Invalid log level: ' + TARPIT_LOG_LEVEL + ' - using default '+DEFAULT_TARPIT_LOG_LEVEL, file=sys.stderr, flush=True) + TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_TARPIT_LOG_LEVEL.upper(), None) + +try: + with open(TARPIT_LOG, 'w+') as f: + logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=TARPIT_LOG, level=TARPIT_NUMERIC_LOG_LEVEL) +except IOError: + logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', stream=sys.stdout, level=TARPIT_NUMERIC_LOG_LEVEL) + +tarpit_log = logging.getLogger(__name__) + +if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('tarpit initialized, default log level = '+logging.getLevelName(TARPIT_NUMERIC_LOG_LEVEL)) + +TARPIT_PORT = 8080 +try: + TARPIT_PORT = int(Path(TARPIT_PORT_FILE).read_text().strip()) +except IOError: + print('error reading port from '+TARPIT_PORT_FILE+', checking env var '+TARPIT_PORT_ENV_VAR, file=sys.stderr, flush=True) + TARPIT_PORT = int(os.getenv(TARPIT_PORT_ENV_VAR, DEFAULT_TARPIT_PORT)) + +TRAP_COUNT = 0 + + +async def handler(_reader, writer): + global TRAP_COUNT + TRAP_COUNT = TRAP_COUNT + 1 + peer_addr = writer.get_extra_info('socket').getpeername()[0] + if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('trapped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) + writer.write(b'HTTP/1.1 200 OK\r\n') + try: + while True: + header = random.randint(0, 2**32) + value = random.randint(0, 2**32) + await asyncio.sleep(3 + (header % 4)) + writer.write(b'X-WOPR-%x: %x\r\n' % (header, value)) + await writer.drain() + except ConnectionResetError: + TRAP_COUNT = TRAP_COUNT - 1 + if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('dropped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) + pass + + +async def main(): + if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('starting HTTP tarpit on port '+str(TARPIT_PORT)) + server = await asyncio.start_server(handler, '0.0.0.0', TARPIT_PORT) + async with server: + await server.serve_forever() + +asyncio.run(main()) diff --git a/bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_ssh_tarpit.py b/bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_ssh_tarpit.py new file mode 100644 index 00000000..03539f89 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/tarpit/files/bubble_ssh_tarpit.py @@ -0,0 +1,87 @@ +#!/usr/bin/python3 +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Adapted from: https://nullprogram.com/blog/2019/03/22/ +# +import asyncio +import random +import os +import sys + +import logging +from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL + +from pathlib import Path + +TARPIT_LOG = '/var/log/bubble/ssh_tarpit.log' +TARPIT_LOG_LEVEL_FILE = '/home/tarpit/ssh_tarpit_log_level.txt' +TARPIT_LOG_LEVEL_ENV_VAR = 'SSH_TARPIT_LOG_LEVEL' +DEFAULT_TARPIT_LOG_LEVEL = 'INFO' +TARPIT_LOG_LEVEL = None + +TARPIT_PORT_FILE = '/home/tarpit/ssh_tarpit_port.txt' +TARPIT_PORT_ENV_VAR = 'SSH_TARPIT_PORT' +DEFAULT_TARPIT_PORT = '22' + +tarpit_log = logging.getLogger(__name__) + +try: + TARPIT_LOG_LEVEL = Path(TARPIT_LOG_LEVEL_FILE).read_text().strip() +except IOError: + print('error reading log level from '+TARPIT_LOG_LEVEL_FILE+', checking env var '+TARPIT_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True) + TARPIT_LOG_LEVEL = os.getenv(TARPIT_LOG_LEVEL_ENV_VAR, DEFAULT_TARPIT_LOG_LEVEL) + +TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, TARPIT_LOG_LEVEL.upper(), None) +if not isinstance(TARPIT_NUMERIC_LOG_LEVEL, int): + print('Invalid log level: ' + TARPIT_LOG_LEVEL + ' - using default '+DEFAULT_TARPIT_LOG_LEVEL, file=sys.stderr, flush=True) + TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_TARPIT_LOG_LEVEL.upper(), None) + +try: + with open(TARPIT_LOG, 'w+') as f: + logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=TARPIT_LOG, level=TARPIT_NUMERIC_LOG_LEVEL) +except IOError: + logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', stream=sys.stdout, level=TARPIT_NUMERIC_LOG_LEVEL) + +tarpit_log = logging.getLogger(__name__) + +if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('tarpit initialized, default log level = '+logging.getLevelName(TARPIT_NUMERIC_LOG_LEVEL)) + +TARPIT_PORT = 8080 +try: + TARPIT_PORT = int(Path(TARPIT_PORT_FILE).read_text().strip()) +except IOError: + print('error reading port from '+TARPIT_PORT_FILE+', checking env var '+TARPIT_PORT_ENV_VAR, file=sys.stderr, flush=True) + TARPIT_PORT = int(os.getenv(TARPIT_PORT_ENV_VAR, DEFAULT_TARPIT_PORT)) + +TRAP_COUNT = 0 + + +async def handler(_reader, writer): + global TRAP_COUNT + TRAP_COUNT = TRAP_COUNT + 1 + peer_addr = writer.get_extra_info('socket').getpeername()[0] + if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('trapped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) + try: + while True: + val = random.randint(0, 2 ** 32) + await asyncio.sleep(6 + (val % 4)) + writer.write(b'%x\r\n' % val) + await writer.drain() + except ConnectionResetError: + TRAP_COUNT = TRAP_COUNT - 1 + if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('dropped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) + pass + + +async def main(): + if tarpit_log.isEnabledFor(INFO): + tarpit_log.info('starting SSH tarpit on port '+str(TARPIT_PORT)) + server = await asyncio.start_server(handler, '0.0.0.0', TARPIT_PORT) + async with server: + await server.serve_forever() + +asyncio.run(main()) diff --git a/bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_http_tarpit.conf b/bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_http_tarpit.conf new file mode 100644 index 00000000..34275426 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_http_tarpit.conf @@ -0,0 +1,6 @@ + +[program:http_tarpit] +stdout_logfile = /dev/null +stderr_logfile = /dev/null +command=sudo -u tarpit /home/tarpit/bubble_http_tarpit.py +stopsignal=QUIT \ No newline at end of file diff --git a/bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_ssh_tarpit.conf b/bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_ssh_tarpit.conf new file mode 100644 index 00000000..82b15c76 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/tarpit/files/supervisor_ssh_tarpit.conf @@ -0,0 +1,6 @@ + +[program:ssh_tarpit] +stdout_logfile = /dev/null +stderr_logfile = /dev/null +command=/home/tarpit/bubble_ssh_tarpit.py +stopsignal=QUIT \ No newline at end of file diff --git a/bubble-server/src/main/resources/packer/roles/tarpit/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/tarpit/tasks/main.yml new file mode 100644 index 00000000..ceafd876 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/tarpit/tasks/main.yml @@ -0,0 +1,54 @@ +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +- name: Create tarpit user + user: + name: tarpit + comment: tarpit user + shell: /bin/false + system: yes + home: /home/tarpit + groups: bubble-log + +- name: Copy bubble_ssh_tarpit script + copy: + src: bubble_ssh_tarpit.py + dest: /home/tarpit/bubble_ssh_tarpit.py + owner: tarpit + group: tarpit + mode: 0500 + +- name: Copy bubble_http_tarpit script + copy: + src: bubble_http_tarpit.py + dest: /home/tarpit/bubble_http_tarpit.py + owner: tarpit + group: tarpit + mode: 0500 + +- name: Install ssh tarpit supervisor conf file + copy: + src: supervisor_ssh_tarpit.conf + dest: /etc/supervisor/conf.d/ssh_tarpit.conf + owner: root + group: root + mode: 0400 + +- name: Install http tarpit supervisor conf file + copy: + src: supervisor_http_tarpit.conf + dest: /etc/supervisor/conf.d/http_tarpit.conf + owner: root + group: root + mode: 0400 + +- name: Allow HTTP tarpit port + iptables: + chain: INPUT + protocol: tcp + destination_port: 8080 + ctstate: NEW + syn: match + jump: ACCEPT + comment: Accept new connections on HTTP tarpit port + become: yes From f98a6c7e69ee6e240f795c51f2b63533168a4acd Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 07:48:13 -0400 Subject: [PATCH 09/16] pass ssh audit --- .../resources/packer/roles/firewall/files/bubble_sshd.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf b/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf index db18fb25..58981a88 100644 --- a/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf +++ b/bubble-server/src/main/resources/packer/roles/firewall/files/bubble_sshd.conf @@ -7,3 +7,8 @@ KerberosAuthentication no GSSAPIAuthentication no X11Forwarding no PermitUserEnvironment no +HostKey /etc/ssh/ssh_host_ed25519_key +HostKey /etc/ssh/ssh_host_rsa_key +KexAlgorithms curve25519-sha256@libssh.org +Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr +MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com From 6d54bd734efd682a1d4767399b5c5f2a6e6b182a Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 08:12:42 -0400 Subject: [PATCH 10/16] additional tarpit point --- .../packer/roles/mitmproxy/files/bubble_request.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py index b62d0110..658f2745 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/bubble_request.py @@ -168,7 +168,6 @@ class Rerouter: return None elif is_not_from_vpn(client_addr): - # todo: add to fail2ban if bubble_log.isEnabledFor(WARNING): bubble_log.warning('bubble_handle_request: sending to tarpit: non-VPN client='+client_addr+', url='+log_url+' host='+host) bubble_activity_log(client_addr, server_addr, 'http_tarpit_non_vpn', fqdns) @@ -230,11 +229,10 @@ class Rerouter: # bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) elif is_http and is_not_from_vpn(client_addr): - # todo: add to fail2ban if bubble_log.isEnabledFor(WARNING): - bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr) - bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr]) - add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) + bubble_log.warning('bubble_handle_request: sending to tarpit: non-VPN client='+client_addr) + bubble_activity_log(client_addr, server_addr, 'http_tarpit_non_vpn', [server_addr]) + tarpit_response(flow, host) return None else: From c0e6c6257621a1ddf8e41c032219100ed932084a Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 08:22:04 -0400 Subject: [PATCH 11/16] change ssh port --- .../java/bubble/service/cloud/StandardNetworkService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index 7f8dd2ff..58dcc47f 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -290,7 +290,9 @@ public class StandardNetworkService implements NetworkService { prepareLaunchFiles(nn, computeDriver, node, progressMeter, network, sageKey, account, plan, sageNode, automation, errors, sshKeyFile); // run ansible - final String sshArgs = "-o UserKnownHostsFile=/dev/null " + final String sshArgs + = "-p 1202 " + + "-o UserKnownHostsFile=/dev/null " + "-o StrictHostKeyChecking=no " + "-o PreferredAuthentications=publickey " + "-i " + abs(sshKeyFile); From 92705696512b3f98a81e5a3ec5d17d627cbf51d0 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 10:38:34 -0400 Subject: [PATCH 12/16] fix sshd config location --- .../src/main/resources/packer/roles/firewall/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml index 8646fb2c..c6d4ff79 100644 --- a/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml @@ -105,7 +105,7 @@ - name: Install SSH hardening settings copy: src: bubble_sshd.conf - dest: /etc/ssh/ssh_config.d/bubble_sshd.conf + dest: /etc/ssh/sshd_config.d/bubble_sshd.conf owner: root group: root mode: 0400 From 4f0e797896b2f42d8003d4dd1b5e4a3b5b6561b8 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 10:55:36 -0400 Subject: [PATCH 13/16] when removing account ssh key, update account plan and network to null key --- .../bubble/dao/account/AccountSshKeyDAO.java | 19 +++++++++++++++++++ .../java/bubble/dao/bill/AccountPlanDAO.java | 4 ++++ .../bubble/dao/cloud/BubbleNetworkDAO.java | 4 ++++ 3 files changed, 27 insertions(+) diff --git a/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java b/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java index 317c7416..56b3e012 100644 --- a/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java +++ b/bubble-server/src/main/java/bubble/dao/account/AccountSshKeyDAO.java @@ -4,8 +4,11 @@ */ package bubble.dao.account; +import bubble.dao.bill.AccountPlanDAO; +import bubble.dao.cloud.BubbleNetworkDAO; import bubble.model.account.Account; import bubble.model.account.AccountSshKey; +import bubble.model.bill.AccountPlan; import bubble.model.cloud.AnsibleInstallType; import bubble.model.cloud.BubbleNetwork; import bubble.server.BubbleConfiguration; @@ -14,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import java.io.File; +import java.util.List; import static bubble.ApiConstants.HOME_DIR; import static org.cobbzilla.util.io.FileUtil.touch; @@ -25,6 +29,8 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; public class AccountSshKeyDAO extends AccountOwnedEntityDAO { @Autowired private AccountDAO accountDAO; + @Autowired private AccountPlanDAO accountPlanDAO; + @Autowired private BubbleNetworkDAO networkDAO; @Autowired private BubbleConfiguration configuration; public AccountSshKey findByAccountAndHash(String accountUuid, String hash) { @@ -90,6 +96,19 @@ public class AccountSshKeyDAO extends AccountOwnedEntityDAO { @Override public void delete(String uuid) { final AccountSshKey key = findByUuid(uuid); + + // remove from any AccountPlans that reference it + final List accountPlans = accountPlanDAO.findByAccountAndSshKey(key.getAccount(), key.getUuid()); + for (AccountPlan plan : accountPlans) { + accountPlanDAO.update(plan.setSshKey(null)); + } + + // remove from any BubbleNetworks that reference it + final List bubbleNetworks = networkDAO.findByAccountAndSshKey(key.getAccount(), key.getUuid()); + for (BubbleNetwork network : bubbleNetworks) { + networkDAO.update(network.setSshKey(null)); + } + super.delete(uuid); if (key.installSshKey()) refreshInstalledKeys(); } diff --git a/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java b/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java index 8b5a5d66..bca75f6f 100644 --- a/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java +++ b/bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java @@ -63,6 +63,10 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO { public AccountPlan findByNetwork(String networkUuid) { return findByUniqueField("network", networkUuid); } + public List findByAccountAndSshKey(String account, String keyUuid) { + return findByFields("account", account, "sshKey", keyUuid); + } + public List findByAccountAndNotDeleted(String account) { return findByFields("account", account, "deleting", false, "deleted", null); } diff --git a/bubble-server/src/main/java/bubble/dao/cloud/BubbleNetworkDAO.java b/bubble-server/src/main/java/bubble/dao/cloud/BubbleNetworkDAO.java index 7e76140a..13d68c3c 100644 --- a/bubble-server/src/main/java/bubble/dao/cloud/BubbleNetworkDAO.java +++ b/bubble-server/src/main/java/bubble/dao/cloud/BubbleNetworkDAO.java @@ -92,6 +92,10 @@ public class BubbleNetworkDAO extends AccountOwnedEntityDAO { return findByUniqueFields("name", name, "domain", domainUuid); } + public List findByAccountAndSshKey(String account, String keyUuid) { + return findByFields("account", account, "sshKey", keyUuid); + } + @Override public void delete(String uuid) { final BubbleNetwork network = findByUuid(uuid); if (network == null) return; From 929571af24c440fc2279171737127fe3a76d1eec Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 10:57:46 -0400 Subject: [PATCH 14/16] update web --- bubble-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-web b/bubble-web index 0748d749..66d46695 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 0748d749df4bddac2ed011f2f256e9ca7f2a7e31 +Subproject commit 66d46695a64ff58934560c4b35aa43a0ab32fbe2 From ca95a31f7a04298691aedb42269b65084e2d4e86 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 12:18:06 -0400 Subject: [PATCH 15/16] re-enable proper ssh port in iptables after algo overwrites --- .../ansible/roles/algo/tasks/algo_firewall.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml b/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml index 145c7cbb..90204f31 100644 --- a/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml +++ b/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml @@ -3,7 +3,7 @@ # # Insert additional firewall rules to allow required services to function # Insert them all on rule_num 5, and insert them in reverse order here: -- name: Allow SSH +- name: Allow SSH tarpit iptables: chain: INPUT protocol: tcp @@ -11,6 +11,17 @@ ctstate: NEW syn: match jump: ACCEPT + comment: Accept new SSH tarpit connections + become: yes + +- name: Allow SSH + iptables: + chain: INPUT + protocol: tcp + destination_port: 1202 + ctstate: NEW + syn: match + jump: ACCEPT comment: Accept new SSH connections become: yes From 3cb85aff84087122b6ee8ebcf4c2eae1921e237f Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 17 Sep 2020 12:34:19 -0400 Subject: [PATCH 16/16] bump version --- .../src/main/resources/META-INF/bubble/bubble.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index dba5e398..beb64e55 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 1.2.0 +bubble.version=Adventure 1.2.1