# Conflicts: # utils/cobbzilla-utilspull/58/head
@@ -253,6 +253,7 @@ public class ApiConstants { | |||
public static final String EP_UPGRADE = "/upgrade"; | |||
public static final String EP_LOGS = "/logs"; | |||
public static final String EP_FOLLOW = "/follow"; | |||
public static final String EP_FOLLOW_AND_APPLY_REGEX = "/followAndApplyRegex"; | |||
public static final String DETECT_ENDPOINT = "/detect"; | |||
public static final String EP_LOCALE = "/locale"; | |||
@@ -4,8 +4,11 @@ | |||
*/ | |||
package bubble.dao.account; | |||
import bubble.dao.bill.AccountPlanDAO; | |||
import bubble.dao.cloud.BubbleNetworkDAO; | |||
import bubble.model.account.Account; | |||
import bubble.model.account.AccountSshKey; | |||
import bubble.model.bill.AccountPlan; | |||
import bubble.model.cloud.AnsibleInstallType; | |||
import bubble.model.cloud.BubbleNetwork; | |||
import bubble.server.BubbleConfiguration; | |||
@@ -14,6 +17,7 @@ import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Repository; | |||
import java.io.File; | |||
import java.util.List; | |||
import static bubble.ApiConstants.HOME_DIR; | |||
import static org.cobbzilla.util.io.FileUtil.touch; | |||
@@ -25,6 +29,8 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; | |||
public class AccountSshKeyDAO extends AccountOwnedEntityDAO<AccountSshKey> { | |||
@Autowired private AccountDAO accountDAO; | |||
@Autowired private AccountPlanDAO accountPlanDAO; | |||
@Autowired private BubbleNetworkDAO networkDAO; | |||
@Autowired private BubbleConfiguration configuration; | |||
public AccountSshKey findByAccountAndHash(String accountUuid, String hash) { | |||
@@ -90,6 +96,19 @@ public class AccountSshKeyDAO extends AccountOwnedEntityDAO<AccountSshKey> { | |||
@Override public void delete(String uuid) { | |||
final AccountSshKey key = findByUuid(uuid); | |||
// remove from any AccountPlans that reference it | |||
final List<AccountPlan> accountPlans = accountPlanDAO.findByAccountAndSshKey(key.getAccount(), key.getUuid()); | |||
for (AccountPlan plan : accountPlans) { | |||
accountPlanDAO.update(plan.setSshKey(null)); | |||
} | |||
// remove from any BubbleNetworks that reference it | |||
final List<BubbleNetwork> bubbleNetworks = networkDAO.findByAccountAndSshKey(key.getAccount(), key.getUuid()); | |||
for (BubbleNetwork network : bubbleNetworks) { | |||
networkDAO.update(network.setSshKey(null)); | |||
} | |||
super.delete(uuid); | |||
if (key.installSshKey()) refreshInstalledKeys(); | |||
} | |||
@@ -63,6 +63,10 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO<AccountPlan> { | |||
public AccountPlan findByNetwork(String networkUuid) { return findByUniqueField("network", networkUuid); } | |||
public List<AccountPlan> findByAccountAndSshKey(String account, String keyUuid) { | |||
return findByFields("account", account, "sshKey", keyUuid); | |||
} | |||
public List<AccountPlan> findByAccountAndNotDeleted(String account) { | |||
return findByFields("account", account, "deleting", false, "deleted", null); | |||
} | |||
@@ -92,6 +92,10 @@ public class BubbleNetworkDAO extends AccountOwnedEntityDAO<BubbleNetwork> { | |||
return findByUniqueFields("name", name, "domain", domainUuid); | |||
} | |||
public List<BubbleNetwork> findByAccountAndSshKey(String account, String keyUuid) { | |||
return findByFields("account", account, "sshKey", keyUuid); | |||
} | |||
@Override public void delete(String uuid) { | |||
final BubbleNetwork network = findByUuid(uuid); | |||
if (network == null) return; | |||
@@ -44,7 +44,8 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD; | |||
public class AppMatcher extends IdentifiableBase implements AppTemplateEntity, HasPriority { | |||
public static final String[] VALUE_FIELDS = { | |||
"fqdn", "urlRegex", "template", "enabled", "priority", "connCheck", "requestCheck", "requestModifier" | |||
"fqdn", "urlRegex", "userAgentRegex", "template", "enabled", "priority", | |||
"connCheck", "requestCheck", "requestModifier" | |||
}; | |||
public static final String[] CREATE_FIELDS = ArrayUtil.append(VALUE_FIELDS, "name", "site", "rule"); | |||
@@ -34,13 +34,12 @@ import bubble.service.stream.StandardRuleEngineService; | |||
import com.fasterxml.jackson.databind.JsonNode; | |||
import lombok.Getter; | |||
import lombok.extern.slf4j.Slf4j; | |||
import org.apache.commons.lang3.ArrayUtils; | |||
import org.cobbzilla.util.collection.ExpirationEvictionPolicy; | |||
import org.cobbzilla.util.collection.ExpirationMap; | |||
import org.cobbzilla.util.collection.NameAndValue; | |||
import org.cobbzilla.util.http.HttpContentEncodingType; | |||
import org.cobbzilla.util.http.HttpUtil; | |||
import org.cobbzilla.util.network.NetworkUtil; | |||
import org.cobbzilla.util.string.StringUtil; | |||
import org.cobbzilla.wizard.cache.redis.RedisService; | |||
import org.glassfish.grizzly.http.server.Request; | |||
import org.glassfish.jersey.server.ContainerRequest; | |||
@@ -64,15 +63,16 @@ import static bubble.service.stream.StandardRuleEngineService.MATCHERS_CACHE_TIM | |||
import static com.google.common.net.HttpHeaders.CONTENT_SECURITY_POLICY; | |||
import static java.util.Collections.emptyMap; | |||
import static java.util.concurrent.TimeUnit.*; | |||
import static javax.ws.rs.core.HttpHeaders.CONTENT_LENGTH; | |||
import static org.apache.http.HttpHeaders.*; | |||
import static org.cobbzilla.util.collection.ArrayUtil.arrayToString; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.*; | |||
import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; | |||
import static org.cobbzilla.util.http.HttpContentTypes.TEXT_PLAIN; | |||
import static org.cobbzilla.util.http.HttpUtil.applyRegexToUrl; | |||
import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
import static org.cobbzilla.util.http.HttpUtil.chaseRedirects; | |||
import static org.cobbzilla.util.json.JsonUtil.*; | |||
import static org.cobbzilla.util.network.NetworkUtil.isLocalIpv4; | |||
import static org.cobbzilla.util.security.ShaUtil.sha256_hex; | |||
import static org.cobbzilla.util.string.StringUtil.trimQuotes; | |||
import static org.cobbzilla.wizard.cache.redis.RedisService.EX; | |||
import static org.cobbzilla.wizard.model.NamedEntity.names; | |||
@@ -446,8 +446,17 @@ public class FilterHttpResource { | |||
public Response flushCaches(@Context ContainerRequest request) { | |||
final Account caller = userPrincipal(request); | |||
if (!caller.admin()) return forbidden(); | |||
final int connCheckMatcherCacheSize = connCheckMatcherCache.size(); | |||
connCheckMatcherCache.clear(); | |||
return ok(ruleEngine.flushCaches()); | |||
// disable redirect flushing for now -- it works well and it's a lot of work | |||
// final Long redirectCacheSize = getRedirectCache().del_matching("*"); | |||
final Map<Object, Object> flushes = ruleEngine.flushCaches(); | |||
flushes.put("connCheckMatchersCache", connCheckMatcherCacheSize); | |||
// flushes.put("redirectCache", redirectCacheSize == null ? 0 : redirectCacheSize); | |||
return ok(flushes); | |||
} | |||
@DELETE @Path(EP_MATCHERS) | |||
@@ -690,40 +699,68 @@ public class FilterHttpResource { | |||
return ok_empty(); | |||
} | |||
private final Map<String, String> redirectCache | |||
= new ExpirationMap<>(1000, DAYS.toMillis(3), ExpirationEvictionPolicy.atime); | |||
public static final String REDIS_PREFIX_REDIRECT_CACHE = "followLink_"; | |||
@Getter(lazy=true) private final RedisService redirectCache = redis.prefixNamespace(REDIS_PREFIX_REDIRECT_CACHE); | |||
@POST @Path(EP_FOLLOW+"/{requestId}") | |||
@Consumes(APPLICATION_JSON) | |||
@Produces(TEXT_PLAIN) | |||
public Response followLink(@Context Request req, | |||
@Context ContainerRequest ctx, | |||
@PathParam("requestId") String requestId, | |||
JsonNode followSpec) { | |||
JsonNode urlNode) { | |||
final FilterSubContext filterCtx = new FilterSubContext(req, requestId); | |||
final RedisService cache = getRedirectCache(); | |||
final String url = urlNode.textValue(); | |||
final String cacheKey = sha256_hex(url); | |||
final String cachedValue = cache.get(cacheKey); | |||
if (cachedValue != null) return ok(cachedValue); | |||
final String result = chaseRedirects(url); | |||
cache.set(cacheKey, result, EX, DAYS.toMillis(365)); | |||
return ok(result); | |||
} | |||
// is this a request to parse regexes from a URL? | |||
if (followSpec.has("regex")) { | |||
return ok(redirectCache.computeIfAbsent(json(followSpec), k -> { | |||
final String url = followSpec.get("url").textValue(); | |||
final String regex = followSpec.get("regex").textValue(); | |||
final Integer group = followSpec.has("group") ? followSpec.get("group").asInt() : null; | |||
final List<NameAndValue> headers = new ArrayList<>(); | |||
for (String name : req.getHeaderNames()) { | |||
final String value = req.getHeader(name); | |||
headers.add(new NameAndValue(name, value)); | |||
} | |||
final List<String> matches = applyRegexToUrl(url, headers, regex, group); | |||
return matches == null ? null : StringUtil.toString(matches, "\n"); | |||
})); | |||
public static final String CLIENT_HEADER_PREFIX = "X-Bubble-Client-Header-"; | |||
} else if (followSpec.isTextual()) { | |||
// just a regular follow -- chase redirects | |||
return ok(redirectCache.computeIfAbsent(followSpec.textValue(), HttpUtil::chaseRedirects)); | |||
} else { | |||
final String json = json(followSpec); | |||
log.error("followLink: invalid json (expected String or {regex, url}): "+json); | |||
return notFound(json); | |||
public static final String[] EXCLUDED_CLIENT_HEADERS = { | |||
ACCEPT.toLowerCase(), | |||
CONTENT_TYPE.toLowerCase(), CONTENT_LENGTH.toLowerCase(), | |||
CONTENT_ENCODING.toLowerCase(), TRANSFER_ENCODING.toLowerCase() | |||
}; | |||
@POST @Path(EP_FOLLOW_AND_APPLY_REGEX+"/{requestId}") | |||
@Consumes(APPLICATION_JSON) | |||
@Produces(APPLICATION_JSON) | |||
public Response followLinkThenApplyRegex(@Context Request req, | |||
@Context ContainerRequest ctx, | |||
@PathParam("requestId") String requestId, | |||
FollowThenApplyRegex follow) { | |||
final FilterSubContext filterCtx = new FilterSubContext(req, requestId); | |||
final RedisService cache = getRedirectCache(); | |||
final String followJson = json(follow); | |||
final String cacheKey = sha256_hex(followJson); | |||
final String cachedValue = cache.get(cacheKey); | |||
if (cachedValue != null) return ok(cachedValue); | |||
// collect client headers | |||
final List<NameAndValue> headers = new ArrayList<>(); | |||
for (String name : req.getHeaderNames()) { | |||
if (name.toLowerCase().startsWith(CLIENT_HEADER_PREFIX.toLowerCase())) { | |||
final String value = req.getHeader(name); | |||
final String realName = name.substring(CLIENT_HEADER_PREFIX.length()); | |||
if (ArrayUtils.indexOf(EXCLUDED_CLIENT_HEADERS, realName.toLowerCase()) == -1) { | |||
headers.add(new NameAndValue(realName, value)); | |||
} | |||
} | |||
} | |||
headers.add(new NameAndValue(ACCEPT, "*/*")); | |||
final List<Map<Integer, String>> matches | |||
= applyRegexToUrl(follow.getUrl(), headers, follow.getRegex(), Arrays.asList(follow.getGroups())); | |||
if (log.isWarnEnabled()) log.warn("followLink(" + follow.getUrl() + ") returning: " + json(matches)); | |||
final String result = matches == null ? EMPTY_JSON_ARRAY : json(matches); | |||
cache.set(cacheKey, result, EX, DAYS.toMillis(365)); | |||
return ok(result); | |||
} | |||
@Path(EP_ASSETS+"/{requestId}/{appId}") | |||
@@ -0,0 +1,16 @@ | |||
/** | |||
* Copyright (c) 2020 Bubble, Inc. All rights reserved. | |||
* For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
*/ | |||
package bubble.resources.stream; | |||
import lombok.Getter; | |||
import lombok.Setter; | |||
public class FollowThenApplyRegex { | |||
@Getter @Setter private String url; | |||
@Getter @Setter private String regex; | |||
@Getter @Setter private Integer[] groups; | |||
} |
@@ -290,7 +290,9 @@ public class StandardNetworkService implements NetworkService { | |||
prepareLaunchFiles(nn, computeDriver, node, progressMeter, network, sageKey, account, plan, sageNode, automation, errors, sshKeyFile); | |||
// run ansible | |||
final String sshArgs = "-o UserKnownHostsFile=/dev/null " | |||
final String sshArgs | |||
= "-p 1202 " | |||
+ "-o UserKnownHostsFile=/dev/null " | |||
+ "-o StrictHostKeyChecking=no " | |||
+ "-o PreferredAuthentications=publickey " | |||
+ "-i " + abs(sshKeyFile); | |||
@@ -1 +1 @@ | |||
bubble.version=Adventure 1.1.3 | |||
bubble.version=Adventure 1.2.1 |
@@ -7,6 +7,8 @@ LOG=/var/log/bubble/ansible.log | |||
# Stop unattended upgrades so that apt installs will work | |||
# unattended upgrades are re-enabled at the end of the ansible run | |||
systemctl stop unattended-upgrades | |||
UNATTENDED_UPGRADES_DISABLED=20auto-upgrades-disabled | |||
cp /usr/share/unattended-upgrades/${UNATTENDED_UPGRADES_DISABLED} /etc/apt/apt.conf.d/${UNATTENDED_UPGRADES_DISABLED} | |||
# Enable job control. Allows us to start creating dhparam in the background right now. | |||
{{#if isNode}}# For node, also allows us to install AlgoVPN in the background.{{/if}} | |||
@@ -104,6 +106,7 @@ fi | |||
kill_bg_jobs | |||
# ansible should have already restarted unattended-upgrades, but just in case | |||
systemctl start unattended-upgrades | |||
rm -f /etc/apt/apt.conf.d/${UNATTENDED_UPGRADES_DISABLED} | |||
systemctl restart unattended-upgrades | |||
exit 0 |
@@ -3,7 +3,7 @@ | |||
# | |||
# Insert additional firewall rules to allow required services to function | |||
# Insert them all on rule_num 5, and insert them in reverse order here: | |||
- name: Allow SSH | |||
- name: Allow SSH tarpit | |||
iptables: | |||
chain: INPUT | |||
protocol: tcp | |||
@@ -11,6 +11,17 @@ | |||
ctstate: NEW | |||
syn: match | |||
jump: ACCEPT | |||
comment: Accept new SSH tarpit connections | |||
become: yes | |||
- name: Allow SSH | |||
iptables: | |||
chain: INPUT | |||
protocol: tcp | |||
destination_port: 1202 | |||
ctstate: NEW | |||
syn: match | |||
jump: ACCEPT | |||
comment: Accept new SSH connections | |||
become: yes | |||
@@ -18,7 +18,7 @@ | |||
- name: Ensure mitmproxy user owns all mitmproxy files | |||
shell: chown -R mitmproxy /home/mitmproxy/mitmproxy | |||
- name: Install mitmproxy1 supervisor conf file | |||
- name: Install mitm8888 supervisor conf file | |||
template: | |||
src: supervisor_mitmproxy.conf.j2 | |||
dest: /etc/supervisor/conf.d/mitm8888.conf | |||
@@ -28,7 +28,7 @@ | |||
vars: | |||
port: 8888 | |||
- name: Install mitmproxy2 supervisor conf file | |||
- name: Install mitm9999 supervisor conf file | |||
template: | |||
src: supervisor_mitmproxy.conf.j2 | |||
dest: /etc/supervisor/conf.d/mitm9999.conf | |||
@@ -87,9 +87,12 @@ function {{JS_PREFIX}}_create_button(labelKey, labelDefault, onclick, labelForma | |||
return btn; | |||
} | |||
if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { | |||
{{JS_PREFIX}}_follow_url = '/__bubble/api/filter/follow/{{BUBBLE_REQUEST_ID}}'; | |||
{{JS_PREFIX}}_follow_and_apply_regex_url = '/__bubble/api/filter/followAndApplyRegex/{{BUBBLE_REQUEST_ID}}'; | |||
{{JS_PREFIX}}_url_chasers = {}; | |||
let {{PAGE_PREFIX}}_url_chasers = {}; | |||
if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { | |||
{{PAGE_PREFIX}}_screenWidth = function () { return window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth }; | |||
@@ -232,10 +235,10 @@ if (typeof {{PAGE_PREFIX}}_icon_status === 'undefined') { | |||
}); | |||
} | |||
function {{JS_PREFIX}}_chase_redirects (a, removeParams) { | |||
function {{JS_PREFIX}}_chase_redirects (a, removeParams, regex, groups, callback) { | |||
const initial_href = a.href; | |||
if (initial_href in {{PAGE_PREFIX}}_url_chasers) { | |||
a.href = {{PAGE_PREFIX}}_url_chasers[initial_href]; | |||
if (initial_href in {{JS_PREFIX}}_url_chasers) { | |||
a.href = {{JS_PREFIX}}_url_chasers[initial_href]; | |||
return; | |||
} | |||
if (a.className && a.className.indexOf('{{JS_PREFIX}}_followed') !== -1) return; | |||
@@ -247,20 +250,37 @@ function {{JS_PREFIX}}_chase_redirects (a, removeParams) { | |||
a.rel = 'noopener noreferrer nofollow'; | |||
fetch('/__bubble/api/filter/follow/{{BUBBLE_REQUEST_ID}}', {method: 'POST', body: JSON.stringify(initial_href)}) | |||
.then(response => response.text()) | |||
let is_regex = (typeof regex !== 'undefined'); | |||
const follow_body = !is_regex ? initial_href : | |||
{ | |||
'url': initial_href, | |||
'regex': regex, | |||
'groups': (typeof groups === 'undefined' || groups === null ? null : groups) | |||
}; | |||
const request_opts = { | |||
method: 'POST', | |||
headers: {'Content-Type': 'application/json'}, | |||
body: JSON.stringify(follow_body) | |||
} | |||
const follow_url = is_regex ? {{JS_PREFIX}}_follow_and_apply_regex_url : {{JS_PREFIX}}_follow_url; | |||
fetch(follow_url, request_opts) | |||
.then(response => is_regex ? response.json() : response.text()) | |||
.then(data => { | |||
if (data && (data.startsWith('http://') || data.startsWith('https://'))) { | |||
if (typeof removeParams === 'undefined' || removeParams === null || removeParams) { | |||
const qPos = data.indexOf('?'); | |||
a.href = qPos === -1 ? data : data.substring(0, qPos); | |||
if (is_regex) { | |||
callback(data); | |||
} else { | |||
if (data && (data.startsWith('http://') || data.startsWith('https://'))) { | |||
if (typeof removeParams === 'undefined' || removeParams === null || removeParams) { | |||
const qPos = data.indexOf('?'); | |||
a.href = qPos === -1 ? data : data.substring(0, qPos); | |||
} else { | |||
a.href = data; | |||
} | |||
{{JS_PREFIX}}_url_chasers[initial_href] = a.href; | |||
// console.log('chase_redirect: rewrote '+initial_href+' -> '+a.href); | |||
} else { | |||
a.href = data; | |||
console.warn('chase_redirects: ' + a.href + ' returned non-URL response: ' + data); | |||
} | |||
{{PAGE_PREFIX}}_url_chasers[initial_href] = a.href; | |||
// console.log('chase_redirect: rewrote '+initial_href+' -> '+a.href); | |||
} else { | |||
console.warn('chase_redirects: '+a.href+' returned non-URL response: '+data); | |||
} | |||
}) | |||
.catch((error) => { | |||
@@ -72,8 +72,11 @@ function {{JS_PREFIX}}_uuidv4() { | |||
const {{JS_PREFIX}}_create_block_img = function(size) { | |||
const img = document.createElement('img'); | |||
img.style.all = 'revert'; | |||
img.style.fontSize = 'x-small'; | |||
img.src = {{JS_PREFIX}}_asset_img_url('icon'); | |||
img.width = typeof size !== 'undefined' ? size : 24; | |||
img.width = typeof size !== 'undefined' && size !== null ? size : 24; | |||
img.style.width = img.width+'px'; | |||
return img; | |||
} | |||
@@ -32,7 +32,7 @@ Element.prototype.appendChild = function() { | |||
const block = {{JS_PREFIX}}_should_block({{JS_PREFIX}}_blocked_users, node) | |||
if (block) { | |||
// log('>>> BLOCKING via appendChild: '+block); | |||
{{JS_PREFIX}}_appendChild.apply({{JS_PREFIX}}_jail, arguments); | |||
return {{JS_PREFIX}}_appendChild.apply({{JS_PREFIX}}_jail, arguments); | |||
} | |||
} | |||
} | |||
@@ -40,7 +40,7 @@ Element.prototype.appendChild = function() { | |||
console.log('>>> error inspecting: e='+e); | |||
} | |||
try { | |||
{{JS_PREFIX}}_appendChild.apply(this, arguments); | |||
return {{JS_PREFIX}}_appendChild.apply(this, arguments); | |||
} catch (e) { | |||
console.log('>>> error calling document.appendChild: arg[0].tagName = '+node.tagName+' e='+e); | |||
} | |||
@@ -1,9 +1,32 @@ | |||
{{JS_PREFIX}}_supports_keywords = true; | |||
{{JS_PREFIX}}_idle_interval = 5000; | |||
const {{JS_PREFIX}}_site_host = location.protocol + '//' + window.location.hostname + '/'; | |||
function {{JS_PREFIX}}_mobile() { | |||
const html = Array.from(document.getElementsByTagName('html')); | |||
if (html.length !== 0) { | |||
return html[0].className && html[0].className.indexOf(' mobile ') !== -1; | |||
} | |||
return false; | |||
} | |||
const {{JS_PREFIX}}_jail = document.createElement('div'); | |||
{{JS_PREFIX}}_jail.style.display = 'none'; | |||
function {{JS_PREFIX}}_apply_blocks(blocked_users) { | |||
const articles = Array.from(document.getElementsByClassName('feed-shared-update-v2')); | |||
const adBanner = Array.from(document.getElementsByTagName('iframe')).find(i => i.className && i.className === 'ad-banner'); | |||
if (typeof adBanner !== 'undefined') { | |||
let adParent = adBanner.parentNode; | |||
if (adParent != null) { | |||
adParent.innerHTML = ''; | |||
adParent.style.display = 'none'; | |||
} else { | |||
} | |||
} | |||
const articles = {{JS_PREFIX}}_mobile() | |||
? Array.from(document.getElementsByClassName('feed-item')) | |||
: Array.from(document.getElementsByClassName('feed-shared-update-v2')); | |||
if (articles === null || articles.length === 0) { | |||
console.warn('No articles found, not filtering'); | |||
return; | |||
@@ -11,7 +34,21 @@ function {{JS_PREFIX}}_apply_blocks(blocked_users) { | |||
{{JS_PREFIX}}_consider_block(articles, blocked_users); | |||
} | |||
function {{JS_PREFIX}}_author_from_href(href) { | |||
function {{JS_PREFIX}}_is_valid_author_name(name) { | |||
return !(name.startsWith('ACoAA') || name.length >= 38); | |||
} | |||
function {{JS_PREFIX}}_author_from_href(linkId, callback) { | |||
if (typeof linkId === 'undefined' || linkId === null || linkId.length === 0) { | |||
// console.log('author_from_href: invalid link ID: '+linkId); | |||
return; | |||
} | |||
const link = document.getElementById(linkId); | |||
if (link === null) { | |||
// console.log('author_from_href: link with ID '+linkId+' not found'); | |||
return; | |||
} | |||
const href = link.href; | |||
if (typeof href === 'undefined' || href === null) return null; | |||
let h = href.startsWith({{JS_PREFIX}}_site_host) ? href.substring({{JS_PREFIX}}_site_host.length) : href; | |||
const qPos = h.indexOf('?'); | |||
@@ -19,14 +56,56 @@ function {{JS_PREFIX}}_author_from_href(href) { | |||
h = h.substring(0, qPos); | |||
} | |||
if (h.endsWith('/')) h = h.substring(0, h.length - 1); | |||
if (!h.startsWith('in/') && !h.startsWith('company/')) { | |||
return null; | |||
let profile_type = null; | |||
const mobile = {{JS_PREFIX}}_mobile(); | |||
if (mobile && h.startsWith('mwlite/')) { | |||
h = h.substring('mwlite/'.length); | |||
} | |||
if (h.startsWith('in/')) { | |||
profile_type = 'in/'; | |||
} else if (h.startsWith('company/')) { | |||
profile_type = 'company/'; | |||
} else { | |||
// console.log("author_from_href: skipping (not in/ or company/) href: "+href+', h='+h); | |||
return; | |||
} | |||
const slashPos = h.indexOf('/'); | |||
const name = h.substring(slashPos); | |||
if (name.length > 35 && name.indexOf('-') === -1 && name.indexOf('_') === -1) return null; | |||
console.log("author_from_href: found "+name+' from '+href); | |||
return name; | |||
const name = h.substring(slashPos+1); | |||
if ({{JS_PREFIX}}_is_valid_author_name(name)) { | |||
// console.log("author_from_href: found " + name + ' from ' + href); | |||
callback(linkId, name); | |||
} else { | |||
// only chase a link once | |||
let linkClass = link.className; | |||
const chaseClass = '{{JS_PREFIX}}_link_chased'; | |||
if (linkClass && linkClass.indexOf(chaseClass) !== -1) { | |||
return; | |||
} else { | |||
link.className = link.className ? link.className + ' '+chaseClass : chaseClass; | |||
} | |||
{{JS_PREFIX}}_chase_redirects(link, true, '/voyager/api/identity/profiles/([^/]+)/privacySettings', [1], function (matches) { | |||
if (typeof matches.length !== 'undefined') { | |||
for (let i=0; i<matches.length; i++) { | |||
const match = matches[i]; | |||
if (!('1' in match)) continue; | |||
const updated_name = matches[i]['1']; | |||
if ({{JS_PREFIX}}_is_valid_author_name(updated_name)) { | |||
const realLink = document.getElementById(linkId); | |||
if (realLink === null) { | |||
console.log('author_from_href: link with id '+linkId+' seems to have disappeared from the document'); | |||
return; | |||
} | |||
link.href = {{JS_PREFIX}}_site_host + profile_type + updated_name; | |||
console.log('author_from_href: updated link.href from '+href+' to: '+link.href); | |||
callback(linkId, updated_name); | |||
return; | |||
} | |||
} | |||
} | |||
}); | |||
} | |||
} | |||
function {{JS_PREFIX}}_remove_article_from_dom(article) { | |||
@@ -57,45 +136,111 @@ function {{JS_PREFIX}}_create_block_control(article, authorName, articleLink) { | |||
blockSpan.appendChild(document.createTextNode('\u00A0\u00A0')); | |||
blockSpan.appendChild(blockLink); | |||
blockSpan.id = 'blockSpan_'+{{JS_PREFIX}}_uuidv4(); | |||
console.log('adding block control on '+authorName); | |||
// console.log('adding block control on '+authorName); | |||
return blockSpan; | |||
} | |||
function {{JS_PREFIX}}_hash_url(url) { return btoa(url).replaceAll('=', ''); } | |||
function {{JS_PREFIX}}_is_ad(article) { | |||
const mobile = {{JS_PREFIX}}_mobile(); | |||
return (mobile && article.getAttribute('data-is-sponsored') && article.getAttribute('data-is-sponsored') !== "false") | |||
|| (article.innerHTML.indexOf('<span>Promoted</span>') !== -1) | |||
|| (article.innerHTML.indexOf('<span dir="ltr">Promoted</span>') !== -1); | |||
} | |||
function {{JS_PREFIX}}_find_append_span(link) { | |||
const mobile = {{JS_PREFIX}}_mobile(); | |||
if (!mobile) { | |||
let authorSpans = Array.from(link.getElementsByClassName('feed-shared-actor__name')); | |||
if (authorSpans.length > 0) { | |||
return authorSpans[0]; | |||
} else { | |||
return Array.from(link.getElementsByTagName('span')) | |||
.find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type')); | |||
} | |||
} else { | |||
const ltrSpan = Array.from(link.getElementsByTagName('span')) | |||
.find(s => s.getAttribute('dir') === 'ltr' || s.getAttribute('data-entity-type')); | |||
if (ltrSpan) return ltrSpan; | |||
if (link.className && link.className.indexOf('profile-link') !== -1) { | |||
return link.lastChild; | |||
} | |||
} | |||
} | |||
function {{JS_PREFIX}}_consider_block(articles, blocked_users) { | |||
const mobile = {{JS_PREFIX}}_mobile(); | |||
if (articles && articles.length && articles.length > 0) { | |||
for (let i=0; i<articles.length; i++) { | |||
const article = articles[i]; | |||
if ({{JS_PREFIX}}_is_ad(article)) { | |||
{{JS_PREFIX}}_tally_author_block({{PAGE_PREFIX}}_msg_or_default({{JS_PREFIX}}_messages, 'web_advertOrOtherBlock', 'ad/other')); | |||
{{JS_PREFIX}}_remove_article_from_dom(article); | |||
continue; | |||
} | |||
const firstEval = {{JS_PREFIX}}_mark_evaluated(article); | |||
if ({{JS_PREFIX}}_includes_block_keyword(article, firstEval)) { | |||
{{JS_PREFIX}}_remove_article_from_dom(article); | |||
continue; | |||
} | |||
const articleLinks = Array.from(article.getElementsByTagName('a')); | |||
const articleLinks = mobile | |||
? Array.from(article.getElementsByTagName('a')).filter(a => !a.hasAttribute('aria-hidden')) | |||
: Array.from(article.getElementsByTagName('a')); | |||
// console.log('consider_block: found '+articleLinks.length+' articleLinks'); | |||
for (let j=0; j<articleLinks.length; j++) { | |||
const articleLink = articleLinks[i]; | |||
console.log('consider_block: examining articleLink with href='+articleLink.href); | |||
const author = {{JS_PREFIX}}_author_from_href(articleLink.href); | |||
if (author === null) continue; | |||
if (author in blocked_users) { | |||
{{JS_PREFIX}}_tally_author_block(author); | |||
if (!firstEval) {{JS_PREFIX}}_untally_allow(); | |||
{{JS_PREFIX}}_remove_article_from_dom(article); | |||
} else if (firstEval) { | |||
const authorSpans = Array.from(articleLink.getElementsByClassName('feed-shared-actor__name')); | |||
if (authorSpans.length === 0) { | |||
continue; | |||
const articleLink = articleLinks[j]; | |||
if (typeof articleLink === 'undefined' || articleLink === null || typeof articleLink.href === 'undefined') { | |||
// console.log('consider_block: skipping invalid articleLink: '+JSON.stringify(articleLink)); | |||
continue; | |||
} | |||
if (typeof articleLink.id === 'undefined' || articleLink.id === null || articleLink.id.length === 0) { | |||
articleLink.id = {{JS_PREFIX}}_uuidv4(); | |||
} | |||
const href = articleLink.href; | |||
if (href === null || href.trim().length === 0) continue; | |||
let examinedClass = articleLink.className; | |||
const seenClass = '{{JS_PREFIX}}_link_examined_'+{{JS_PREFIX}}_hash_url(href)+'_'+articleLink.id; | |||
let seenBefore = false; | |||
if (examinedClass && examinedClass.indexOf(seenClass) !== -1) { | |||
seenBefore = true; | |||
} else { | |||
articleLink.className = articleLink.className ? articleLink.className + ' '+seenClass : seenClass; | |||
} | |||
{{JS_PREFIX}}_author_from_href(articleLink.id, function (linkId, author) { | |||
if (author === null) return; | |||
const realLink = document.getElementById(linkId); | |||
if (realLink === null) { | |||
// console.log('consider_block: link with id '+linkId+' seems to have disappeared from the document'); | |||
return; | |||
} | |||
let b = {{JS_PREFIX}}_create_block_control(article, author, articleLink); | |||
if (b !== null) { | |||
console.log('consider_block: inserting span='+b.id+' for article by '+author); | |||
authorSpans[0].parentNode.appendChild(b); | |||
{{JS_PREFIX}}_tally_allow(); | |||
} else { | |||
console.log('consider_block: create_block_control returned null for author '+author) | |||
// console.log('consider_block: examining linkId with author='+author); | |||
if (author in blocked_users) { | |||
{{JS_PREFIX}}_tally_author_block(author); | |||
if (!firstEval) {{JS_PREFIX}}_untally_allow(); | |||
{{JS_PREFIX}}_remove_article_from_dom(article); | |||
} else if (!seenBefore) { | |||
let appendToSpan = {{JS_PREFIX}}_find_append_span(realLink); | |||
if (!appendToSpan) { | |||
// console.log('consider_block: no span found to append to for author '+author); | |||
return; | |||
} | |||
let b = {{JS_PREFIX}}_create_block_control(article, author, realLink); | |||
if (b !== null) { | |||
// console.log('consider_block: inserting span='+b.id+' for article by '+author); | |||
appendToSpan.parentNode.appendChild(b); | |||
{{JS_PREFIX}}_tally_allow(); | |||
} else { | |||
// console.log('consider_block: create_block_control returned null for author '+author) | |||
} | |||
} | |||
} | |||
}); | |||
} | |||
} | |||
} | |||
@@ -247,6 +247,12 @@ | |||
"matcher": "BubbleBlockMatcher", | |||
"key": "hideStats_reddit.com", | |||
"data": "true" | |||
}, { | |||
"site": "All_Sites", | |||
"template": true, | |||
"matcher": "BubbleBlockMatcher", | |||
"key": "hideStats_redfin.com", | |||
"data": "true" | |||
}, { | |||
"site": "All_Sites", | |||
"template": true, | |||
@@ -319,6 +325,12 @@ | |||
"matcher": "BubbleBlockMatcher", | |||
"key": "hideStats_tripactions.com", | |||
"data": "true" | |||
}, { | |||
"site": "All_Sites", | |||
"template": true, | |||
"matcher": "BubbleBlockMatcher", | |||
"key": "hideStats_trulia.com", | |||
"data": "true" | |||
}, { | |||
"site": "All_Sites", | |||
"template": true, | |||
@@ -373,6 +385,12 @@ | |||
"matcher": "BubbleBlockMatcher", | |||
"key": "hideStats_zendesk.com", | |||
"data": "true" | |||
}, { | |||
"site": "All_Sites", | |||
"template": true, | |||
"matcher": "BubbleBlockMatcher", | |||
"key": "hideStats_zillow.com", | |||
"data": "true" | |||
}, { | |||
"site": "All_Sites", | |||
"template": true, | |||
@@ -1,12 +1,6 @@ | |||
[{ | |||
"name": "UserBlocker", | |||
"children": { | |||
"AppData": [{ | |||
"site": "LinkedIn", | |||
"template": true, | |||
"matcher": "LIMatcher", | |||
"key": "kw:_<span>Promoted</span>", | |||
"data": "true" | |||
}] | |||
"AppData": [] | |||
} | |||
}] |
@@ -10,6 +10,16 @@ | |||
"fqdn": "www.linkedin.com", | |||
"urlRegex": "/feed/", | |||
"rule": "li_user_blocker" | |||
}, { | |||
"name": "LIMobileMatcher", | |||
"site": "LinkedIn", | |||
"template": true, | |||
"requestCheck": true, | |||
"requestModifier": true, | |||
"fqdn": "www.linkedin.com", | |||
"urlRegex": "/", | |||
"userAgentRegex": "Mobi|Android", | |||
"rule": "li_user_blocker" | |||
}] | |||
} | |||
}] |
@@ -1,5 +1,6 @@ | |||
common | |||
firewall | |||
tarpit | |||
nginx | |||
algo | |||
mitmproxy | |||
@@ -0,0 +1,14 @@ | |||
Port 1202 | |||
LoginGraceTime 10 | |||
PasswordAuthentication no | |||
PermitEmptyPasswords no | |||
ChallengeResponseAuthentication no | |||
KerberosAuthentication no | |||
GSSAPIAuthentication no | |||
X11Forwarding no | |||
PermitUserEnvironment no | |||
HostKey /etc/ssh/ssh_host_ed25519_key | |||
HostKey /etc/ssh/ssh_host_rsa_key | |||
KexAlgorithms curve25519-sha256@libssh.org | |||
Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr | |||
MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com |
@@ -0,0 +1,5 @@ | |||
[sshd] | |||
mode = aggressive | |||
port = 1202 | |||
logpath = %(sshd_log)s | |||
backend = %(sshd_backend)s |
@@ -90,15 +90,34 @@ | |||
dest: /usr/local/bin/bubble_peer_manager.py | |||
owner: root | |||
group: root | |||
mode: 0555 | |||
mode: 0550 | |||
when: fw_enable_admin | |||
- name: Install supervisor conf file for port manager | |||
- name: Install supervisor conf file for peer manager | |||
copy: | |||
src: supervisor_bubble_peer_manager.conf | |||
dest: /etc/supervisor/conf.d/bubble_peer_manager.conf | |||
owner: root | |||
group: root | |||
mode: 0550 | |||
when: fw_enable_admin | |||
- name: Install SSH hardening settings | |||
copy: | |||
src: bubble_sshd.conf | |||
dest: /etc/ssh/sshd_config.d/bubble_sshd.conf | |||
owner: root | |||
group: root | |||
mode: 0400 | |||
- name: Install SSH fail2ban settings | |||
copy: | |||
src: jail.local | |||
dest: /etc/fail2ban/jail.local | |||
owner: root | |||
group: root | |||
mode: 0400 | |||
- include: rules.yml | |||
- supervisorctl: | |||
@@ -17,15 +17,18 @@ | |||
comment: Allow related and established connections | |||
become: yes | |||
- name: Allow SSH | |||
- name: Allow SSH on ports 22 and 1202 | |||
iptables: | |||
chain: INPUT | |||
protocol: tcp | |||
destination_port: 22 | |||
destination_port: "{{ item }}" | |||
ctstate: NEW | |||
syn: match | |||
jump: ACCEPT | |||
comment: Accept new SSH connections | |||
with_items: | |||
- 22 | |||
- 1202 | |||
become: yes | |||
when: fw_enable_ssh | |||
@@ -64,6 +64,7 @@ LOCAL_IPS = [] | |||
for local_ip in subprocess.check_output(['hostname', '-I']).split(): | |||
LOCAL_IPS.append(local_ip.decode()) | |||
TARPIT_PORT = 8080 | |||
VPN_IP4_CIDR = IPNetwork(wireguard_network_ipv4) | |||
VPN_IP6_CIDR = IPNetwork(wireguard_network_ipv6) | |||
@@ -469,6 +470,28 @@ def health_check_response(flow): | |||
flow.response.stream = lambda chunks: [b'OK\n'] | |||
def tarpit_response(flow, host): | |||
# if bubble_log.isEnabledFor(DEBUG): | |||
# bubble_log.debug('health_check_response: special bubble health check request, responding with OK') | |||
response_headers = nheaders.Headers() | |||
response_headers[HEADER_LOCATION] = 'http://'+host+':'+str(TARPIT_PORT)+'/admin/index.php' | |||
if flow.response is None: | |||
flow.response = http.HTTPResponse(http_version='HTTP/1.1', | |||
status_code=301, | |||
reason='Moved Permanently', | |||
headers=response_headers, | |||
content=b'') | |||
else: | |||
flow.response.headers = nheaders.Headers() | |||
flow.response.headers = response_headers | |||
flow.response.status_code = 301 | |||
flow.response.reason = 'Moved Permanently' | |||
def include_request_headers(path): | |||
return '/followAndApplyRegex' in path | |||
def special_bubble_response(flow): | |||
name = 'special_bubble_response' | |||
path = flow.request.path | |||
@@ -478,7 +501,7 @@ def special_bubble_response(flow): | |||
uri = make_bubble_special_path(path) | |||
if bubble_log.isEnabledFor(DEBUG): | |||
bubble_log.debug('special_bubble_response: sending special bubble request to '+uri) | |||
bubble_log.debug('special_bubble_response: sending special bubble '+flow.request.method+' to '+uri) | |||
headers = { | |||
'Accept': 'application/json', | |||
'Content-Type': 'application/json' | |||
@@ -489,12 +512,22 @@ def special_bubble_response(flow): | |||
response = async_stream(client, name, uri, headers=headers, loop=loop) | |||
elif flow.request.method == 'POST': | |||
loop = asyncio.new_event_loop() | |||
client = async_client(timeout=30) | |||
if include_request_headers(flow.request.path): | |||
if bubble_log.isEnabledFor(DEBUG): | |||
bubble_log.debug('special_bubble_request: including client headers: '+repr(flow.request.headers)) | |||
# add client request headers | |||
for name, value in flow.request.headers.items(): | |||
headers['X-Bubble-Client-Header-'+name] = value | |||
if bubble_log.isEnabledFor(DEBUG): | |||
bubble_log.debug('special_bubble_request: NOW headers='+repr(headers)) | |||
data = None | |||
if flow.request.content and flow.request.content: | |||
headers[HEADER_CONTENT_LENGTH] = str(len(flow.request.content)) | |||
data = flow.request.content | |||
loop = asyncio.new_event_loop() | |||
client = async_client(timeout=30) | |||
response = async_stream(client, name, uri, headers=headers, method='POST', data=data, loop=loop) | |||
else: | |||
@@ -32,7 +32,7 @@ from mitmproxy.net.http import headers as nheaders | |||
from bubble_api import bubble_matchers, bubble_activity_log, \ | |||
CTX_BUBBLE_MATCHERS, CTX_BUBBLE_SPECIAL, CTX_BUBBLE_ABORT, CTX_BUBBLE_LOCATION, \ | |||
CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_FLEX, CTX_BUBBLE_REQUEST_ID, add_flow_ctx, parse_host_header, \ | |||
is_bubble_special_path, is_bubble_health_check, health_check_response, \ | |||
is_bubble_special_path, is_bubble_health_check, health_check_response, tarpit_response,\ | |||
is_bubble_request, is_sage_request, is_not_from_vpn, is_flex_domain | |||
from bubble_config import bubble_host, bubble_host_alias | |||
from bubble_flex import new_flex_flow | |||
@@ -168,11 +168,10 @@ class Rerouter: | |||
return None | |||
elif is_not_from_vpn(client_addr): | |||
# todo: add to fail2ban | |||
if bubble_log.isEnabledFor(WARNING): | |||
bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', url='+log_url+' host='+host) | |||
bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', fqdns) | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) | |||
bubble_log.warning('bubble_handle_request: sending to tarpit: non-VPN client='+client_addr+', url='+log_url+' host='+host) | |||
bubble_activity_log(client_addr, server_addr, 'http_tarpit_non_vpn', fqdns) | |||
tarpit_response(flow, host) | |||
return None | |||
if is_bubble_special_path(path): | |||
@@ -230,11 +229,10 @@ class Rerouter: | |||
# bubble_activity_log(client_addr, server_addr, 'http_no_matcher_response', log_url) | |||
elif is_http and is_not_from_vpn(client_addr): | |||
# todo: add to fail2ban | |||
if bubble_log.isEnabledFor(WARNING): | |||
bubble_log.warning('bubble_handle_request: returning 404 for non-VPN client='+client_addr+', server_addr='+server_addr) | |||
bubble_activity_log(client_addr, server_addr, 'http_abort_non_vpn', [server_addr]) | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, 404) | |||
bubble_log.warning('bubble_handle_request: sending to tarpit: non-VPN client='+client_addr) | |||
bubble_activity_log(client_addr, server_addr, 'http_tarpit_non_vpn', [server_addr]) | |||
tarpit_response(flow, host) | |||
return None | |||
else: | |||
@@ -0,0 +1,89 @@ | |||
#!/usr/bin/python3 | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
# Adapted from: https://nullprogram.com/blog/2019/03/22/ | |||
# | |||
import asyncio | |||
import random | |||
import os | |||
import sys | |||
import logging | |||
from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL | |||
from pathlib import Path | |||
TARPIT_LOG = '/var/log/bubble/http_tarpit.log' | |||
TARPIT_LOG_LEVEL_FILE = '/home/tarpit/http_tarpit_log_level.txt' | |||
TARPIT_LOG_LEVEL_ENV_VAR = 'HTTP_TARPIT_LOG_LEVEL' | |||
DEFAULT_TARPIT_LOG_LEVEL = 'INFO' | |||
TARPIT_LOG_LEVEL = None | |||
TARPIT_PORT_FILE = '/home/tarpit/http_tarpit_port.txt' | |||
TARPIT_PORT_ENV_VAR = 'HTTP_TARPIT_PORT' | |||
DEFAULT_TARPIT_PORT = '8080' | |||
tarpit_log = logging.getLogger(__name__) | |||
try: | |||
TARPIT_LOG_LEVEL = Path(TARPIT_LOG_LEVEL_FILE).read_text().strip() | |||
except IOError: | |||
print('error reading log level from '+TARPIT_LOG_LEVEL_FILE+', checking env var '+TARPIT_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True) | |||
TARPIT_LOG_LEVEL = os.getenv(TARPIT_LOG_LEVEL_ENV_VAR, DEFAULT_TARPIT_LOG_LEVEL) | |||
TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, TARPIT_LOG_LEVEL.upper(), None) | |||
if not isinstance(TARPIT_NUMERIC_LOG_LEVEL, int): | |||
print('Invalid log level: ' + TARPIT_LOG_LEVEL + ' - using default '+DEFAULT_TARPIT_LOG_LEVEL, file=sys.stderr, flush=True) | |||
TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_TARPIT_LOG_LEVEL.upper(), None) | |||
try: | |||
with open(TARPIT_LOG, 'w+') as f: | |||
logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=TARPIT_LOG, level=TARPIT_NUMERIC_LOG_LEVEL) | |||
except IOError: | |||
logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', stream=sys.stdout, level=TARPIT_NUMERIC_LOG_LEVEL) | |||
tarpit_log = logging.getLogger(__name__) | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('tarpit initialized, default log level = '+logging.getLevelName(TARPIT_NUMERIC_LOG_LEVEL)) | |||
TARPIT_PORT = 8080 | |||
try: | |||
TARPIT_PORT = int(Path(TARPIT_PORT_FILE).read_text().strip()) | |||
except IOError: | |||
print('error reading port from '+TARPIT_PORT_FILE+', checking env var '+TARPIT_PORT_ENV_VAR, file=sys.stderr, flush=True) | |||
TARPIT_PORT = int(os.getenv(TARPIT_PORT_ENV_VAR, DEFAULT_TARPIT_PORT)) | |||
TRAP_COUNT = 0 | |||
async def handler(_reader, writer): | |||
global TRAP_COUNT | |||
TRAP_COUNT = TRAP_COUNT + 1 | |||
peer_addr = writer.get_extra_info('socket').getpeername()[0] | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('trapped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) | |||
writer.write(b'HTTP/1.1 200 OK\r\n') | |||
try: | |||
while True: | |||
header = random.randint(0, 2**32) | |||
value = random.randint(0, 2**32) | |||
await asyncio.sleep(3 + (header % 4)) | |||
writer.write(b'X-WOPR-%x: %x\r\n' % (header, value)) | |||
await writer.drain() | |||
except ConnectionResetError: | |||
TRAP_COUNT = TRAP_COUNT - 1 | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('dropped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) | |||
pass | |||
async def main(): | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('starting HTTP tarpit on port '+str(TARPIT_PORT)) | |||
server = await asyncio.start_server(handler, '0.0.0.0', TARPIT_PORT) | |||
async with server: | |||
await server.serve_forever() | |||
asyncio.run(main()) |
@@ -0,0 +1,87 @@ | |||
#!/usr/bin/python3 | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
# Adapted from: https://nullprogram.com/blog/2019/03/22/ | |||
# | |||
import asyncio | |||
import random | |||
import os | |||
import sys | |||
import logging | |||
from logging import INFO, DEBUG, WARNING, ERROR, CRITICAL | |||
from pathlib import Path | |||
TARPIT_LOG = '/var/log/bubble/ssh_tarpit.log' | |||
TARPIT_LOG_LEVEL_FILE = '/home/tarpit/ssh_tarpit_log_level.txt' | |||
TARPIT_LOG_LEVEL_ENV_VAR = 'SSH_TARPIT_LOG_LEVEL' | |||
DEFAULT_TARPIT_LOG_LEVEL = 'INFO' | |||
TARPIT_LOG_LEVEL = None | |||
TARPIT_PORT_FILE = '/home/tarpit/ssh_tarpit_port.txt' | |||
TARPIT_PORT_ENV_VAR = 'SSH_TARPIT_PORT' | |||
DEFAULT_TARPIT_PORT = '22' | |||
tarpit_log = logging.getLogger(__name__) | |||
try: | |||
TARPIT_LOG_LEVEL = Path(TARPIT_LOG_LEVEL_FILE).read_text().strip() | |||
except IOError: | |||
print('error reading log level from '+TARPIT_LOG_LEVEL_FILE+', checking env var '+TARPIT_LOG_LEVEL_ENV_VAR, file=sys.stderr, flush=True) | |||
TARPIT_LOG_LEVEL = os.getenv(TARPIT_LOG_LEVEL_ENV_VAR, DEFAULT_TARPIT_LOG_LEVEL) | |||
TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, TARPIT_LOG_LEVEL.upper(), None) | |||
if not isinstance(TARPIT_NUMERIC_LOG_LEVEL, int): | |||
print('Invalid log level: ' + TARPIT_LOG_LEVEL + ' - using default '+DEFAULT_TARPIT_LOG_LEVEL, file=sys.stderr, flush=True) | |||
TARPIT_NUMERIC_LOG_LEVEL = getattr(logging, DEFAULT_TARPIT_LOG_LEVEL.upper(), None) | |||
try: | |||
with open(TARPIT_LOG, 'w+') as f: | |||
logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', filename=TARPIT_LOG, level=TARPIT_NUMERIC_LOG_LEVEL) | |||
except IOError: | |||
logging.basicConfig(format='%(asctime)s - [%(module)s:%(lineno)d] - %(levelname)s: %(message)s', stream=sys.stdout, level=TARPIT_NUMERIC_LOG_LEVEL) | |||
tarpit_log = logging.getLogger(__name__) | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('tarpit initialized, default log level = '+logging.getLevelName(TARPIT_NUMERIC_LOG_LEVEL)) | |||
TARPIT_PORT = 8080 | |||
try: | |||
TARPIT_PORT = int(Path(TARPIT_PORT_FILE).read_text().strip()) | |||
except IOError: | |||
print('error reading port from '+TARPIT_PORT_FILE+', checking env var '+TARPIT_PORT_ENV_VAR, file=sys.stderr, flush=True) | |||
TARPIT_PORT = int(os.getenv(TARPIT_PORT_ENV_VAR, DEFAULT_TARPIT_PORT)) | |||
TRAP_COUNT = 0 | |||
async def handler(_reader, writer): | |||
global TRAP_COUNT | |||
TRAP_COUNT = TRAP_COUNT + 1 | |||
peer_addr = writer.get_extra_info('socket').getpeername()[0] | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('trapped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) | |||
try: | |||
while True: | |||
val = random.randint(0, 2 ** 32) | |||
await asyncio.sleep(6 + (val % 4)) | |||
writer.write(b'%x\r\n' % val) | |||
await writer.drain() | |||
except ConnectionResetError: | |||
TRAP_COUNT = TRAP_COUNT - 1 | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('dropped '+peer_addr+' - trap count: ' + str(TRAP_COUNT)) | |||
pass | |||
async def main(): | |||
if tarpit_log.isEnabledFor(INFO): | |||
tarpit_log.info('starting SSH tarpit on port '+str(TARPIT_PORT)) | |||
server = await asyncio.start_server(handler, '0.0.0.0', TARPIT_PORT) | |||
async with server: | |||
await server.serve_forever() | |||
asyncio.run(main()) |
@@ -0,0 +1,6 @@ | |||
[program:http_tarpit] | |||
stdout_logfile = /dev/null | |||
stderr_logfile = /dev/null | |||
command=sudo -u tarpit /home/tarpit/bubble_http_tarpit.py | |||
stopsignal=QUIT |
@@ -0,0 +1,6 @@ | |||
[program:ssh_tarpit] | |||
stdout_logfile = /dev/null | |||
stderr_logfile = /dev/null | |||
command=/home/tarpit/bubble_ssh_tarpit.py | |||
stopsignal=QUIT |
@@ -0,0 +1,54 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
- name: Create tarpit user | |||
user: | |||
name: tarpit | |||
comment: tarpit user | |||
shell: /bin/false | |||
system: yes | |||
home: /home/tarpit | |||
groups: bubble-log | |||
- name: Copy bubble_ssh_tarpit script | |||
copy: | |||
src: bubble_ssh_tarpit.py | |||
dest: /home/tarpit/bubble_ssh_tarpit.py | |||
owner: tarpit | |||
group: tarpit | |||
mode: 0500 | |||
- name: Copy bubble_http_tarpit script | |||
copy: | |||
src: bubble_http_tarpit.py | |||
dest: /home/tarpit/bubble_http_tarpit.py | |||
owner: tarpit | |||
group: tarpit | |||
mode: 0500 | |||
- name: Install ssh tarpit supervisor conf file | |||
copy: | |||
src: supervisor_ssh_tarpit.conf | |||
dest: /etc/supervisor/conf.d/ssh_tarpit.conf | |||
owner: root | |||
group: root | |||
mode: 0400 | |||
- name: Install http tarpit supervisor conf file | |||
copy: | |||
src: supervisor_http_tarpit.conf | |||
dest: /etc/supervisor/conf.d/http_tarpit.conf | |||
owner: root | |||
group: root | |||
mode: 0400 | |||
- name: Allow HTTP tarpit port | |||
iptables: | |||
chain: INPUT | |||
protocol: tcp | |||
destination_port: 8080 | |||
ctstate: NEW | |||
syn: match | |||
jump: ACCEPT | |||
comment: Accept new connections on HTTP tarpit port | |||
become: yes |
@@ -1 +1 @@ | |||
Subproject commit 8845990e046eb27dc71da14d9348c84fed6bd7c1 | |||
Subproject commit 66d46695a64ff58934560c4b35aa43a0ab32fbe2 |
@@ -1 +1 @@ | |||
Subproject commit cad625431e357e94647a1d99da2efc171740d8e3 | |||
Subproject commit dfafe62c7eb3413cf1210e40e551094458f4d9d0 |