@@ -327,7 +327,7 @@ | |||
</executions> | |||
</plugin> | |||
<!-- update ansible/default_roles.json, copy scripts into jar --> | |||
<!-- copy scripts and web ui into jar --> | |||
<plugin> | |||
<groupId>org.codehaus.mojo</groupId> | |||
<artifactId>exec-maven-plugin</artifactId> | |||
@@ -44,8 +44,8 @@ public class ApiConstants { | |||
public static final String DEFAULT_LOCALE = "en_US"; | |||
public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "bubble_finalizer"}; | |||
public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "bubble_finalizer"}; | |||
public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "finalizer"}; | |||
public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "finalizer"}; | |||
public static final String ANSIBLE_DIR = "ansible"; | |||
public static final List<String> BUBBLE_SCRIPTS = splitAndTrim(stream2string(ANSIBLE_DIR + "/bubble_scripts.txt"), "\n") | |||
@@ -47,7 +47,7 @@ public class RestoreService { | |||
// this is how long bubble_restore_monitor.sh will allow a restore after it starts | |||
// we add some time because, in the ansible setup, the script starts (in role bubble) before the | |||
// API is started (in role bubble_finalizer) | |||
// API is started (in role finalizer) | |||
public static final long RESTORE_MONITOR_SCRIPT_TIMEOUT_SECONDS = RESTORE_WINDOW_SECONDS + MINUTES.toSeconds(5); | |||
private static final long RESTORE_LOCK_TIMEOUT = MINUTES.toMillis(31); | |||
@@ -30,11 +30,9 @@ import org.cobbzilla.wizard.validation.ValidationResult; | |||
import org.springframework.beans.factory.annotation.Autowired; | |||
import org.springframework.stereotype.Service; | |||
import java.io.File; | |||
import java.util.*; | |||
import static bubble.ApiConstants.ROOT_NETWORK_UUID; | |||
import static bubble.cloud.storage.StorageServiceDriver.STORAGE_PREFIX; | |||
import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE; | |||
import static bubble.model.cloud.BubbleFootprint.DEFAULT_FOOTPRINT; | |||
import static bubble.model.cloud.BubbleFootprint.DEFAULT_FOOTPRINT_OBJECT; | |||
@@ -42,12 +40,10 @@ import static bubble.model.cloud.BubbleNetwork.TAG_ALLOW_REGISTRATION; | |||
import static bubble.model.cloud.BubbleNetwork.TAG_PARENT_ACCOUNT; | |||
import static java.util.concurrent.TimeUnit.SECONDS; | |||
import static org.cobbzilla.util.daemon.ZillaRuntime.*; | |||
import static org.cobbzilla.util.io.FileUtil.toStringOrDie; | |||
import static org.cobbzilla.util.io.StreamUtil.stream2string; | |||
import static org.cobbzilla.util.json.JsonUtil.json; | |||
import static org.cobbzilla.util.network.NetworkUtil.getFirstPublicIpv4; | |||
import static org.cobbzilla.util.network.NetworkUtil.getLocalhostIpv4; | |||
import static org.cobbzilla.util.system.CommandShell.execScript; | |||
import static org.cobbzilla.util.system.Sleep.sleep; | |||
import static org.cobbzilla.wizard.model.entityconfig.ModelSetup.scrubSpecial; | |||
import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; | |||
@@ -55,8 +51,6 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; | |||
@Service @Slf4j | |||
public class ActivationService { | |||
public static final String DEFAULT_ROLES = "ansible/default_roles.json"; | |||
public static final long ACTIVATION_TIMEOUT = SECONDS.toMillis(10); | |||
@Autowired private AccountSshKeyDAO sshKeyDAO; | |||
@@ -233,17 +227,6 @@ public class ActivationService { | |||
return node; | |||
} | |||
public String loadDefaultRoles() { | |||
if (configuration.testMode()) { | |||
final File roleFile = new File("target/classes/"+DEFAULT_ROLES); | |||
final String rolesJson = toStringOrDie(roleFile); | |||
if (rolesJson == null || !rolesJson.contains(STORAGE_PREFIX)) execScript("../bin/prep_bubble_jar"); | |||
return toStringOrDie(roleFile); | |||
} else { | |||
return stream2string(DEFAULT_ROLES); | |||
} | |||
} | |||
public BubbleNetwork createRootNetwork(BubbleNetwork network) { | |||
network.setUuid(ROOT_NETWORK_UUID); | |||
return networkDAO.create(network); | |||
@@ -119,7 +119,9 @@ public class AnsiblePrepService { | |||
for (String roleName : installRoles) { | |||
final TempDir roleTemp = copyClasspathDirectory("ansible/roles/"+roleName); | |||
final File roleDir = new File(rolesDir, roleName); | |||
if (!roleTemp.renameTo(roleDir)) return die("prepAnsible: error renaming role dir "+abs(roleTemp)+" -> "+abs(roleDir)); | |||
if (!roleTemp.renameTo(roleDir)) { | |||
return die("prepAnsible: error renaming role dir "+abs(roleTemp)+" -> "+abs(roleDir)); | |||
} | |||
final File bubbleRoleJson = new File(abs(roleDir)+"/files/bubble_role.json"); | |||
if (bubbleRoleJson.exists()) { | |||
final File varsDir = mkdirOrDie(new File(abs(roleDir)+"/vars")); | |||
@@ -1,108 +0,0 @@ | |||
[ | |||
{ | |||
"name": "common-0.0.1", | |||
"priority": 100, | |||
"template": true, | |||
"config": [ | |||
{"name": "hostname", "value": "[[node.fqdn]]"} | |||
], | |||
"tgzB64": "" | |||
}, | |||
{ | |||
"name": "firewall-0.0.1", | |||
"priority": 200, | |||
"template": true, | |||
"config": [ | |||
{"name": "ssl_port", "value": "[[node.sslPort]]"}, | |||
{"name": "admin_port", "value": "[[node.adminPort]]"}, | |||
{"name": "dns_port", "value": "[[configuration.dnsPort]]"} | |||
], | |||
"tgzB64": "" | |||
}, | |||
{ | |||
"name": "bubble-0.0.1", | |||
"priority": 300, | |||
"template": true, | |||
"config": [ | |||
{"name": "node_uuid", "value": "[[node.uuid]]"}, | |||
{"name": "network_uuid", "value": "[[node.network]]"}, | |||
{"name": "admin_port", "value": "[[node.adminPort]]"}, | |||
{"name": "ssl_port", "value": "[[node.sslPort]]"}, | |||
{"name": "public_base_uri", "value": "[[publicBaseUri]]"}, | |||
{"name": "sage_node", "value": "[[sageNode]]"}, | |||
{"name": "promo_code_policy", "value": "[[#compare fork '==' true]][[configuration.promoCodePolicy]][[else]]disabled[[/compare]]"}, | |||
{"name": "install_type", "value": "[[installType]]"}, | |||
{"name": "default_locale", "value": "[[network.locale]]"}, | |||
{"name": "time_zone", "value": "[[network.timezone]]"}, | |||
{"name": "bubble_version", "value": "[[configuration.version]]"}, | |||
{"name": "bubble_host", "value": "[[node.fqdn]]"}, | |||
{"name": "bubble_cname", "value": "[[network.networkDomain]]"}, | |||
{"name": "admin_user", "value": "[[node.ansibleUser]]"}, | |||
{"name": "db_encoding", "value": "UTF-8"}, | |||
{"name": "db_locale", "value": "en_US"}, | |||
{"name": "db_user", "value": "bubble"}, | |||
{"name": "db_name", "value": "bubble"}, | |||
{"name": "db_key", "value": "[[dbEncryptionKey]]"}, | |||
{"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"}, | |||
{"name": "is_fork", "value": "[[fork]]"}, | |||
{"name": "restore_key", "value": "[[restoreKey]]"}, | |||
{"name": "restore_timeout", "value": "[[restoreTimeoutSeconds]]"}, | |||
{"name": "test_mode", "value": "[[testMode]]"}, | |||
{"name": "error_url", "value": "[[error_url]]"}, | |||
{"name": "error_key", "value": "[[error_key]]"}, | |||
{"name": "error_env", "value": "[[error_env]]"} | |||
], | |||
"optionalConfigNames": ["restore_key", "restore_timeout", "error_url", "error_key", "error_env"], | |||
"tgzB64": "" | |||
}, | |||
{ | |||
"name": "algo-0.0.1", | |||
"priority": 400, | |||
"template": true, | |||
"install": "node", | |||
"config": [ | |||
{"name": "server_name", "value": "[[node.fqdn]]"}, | |||
{"name": "endpoint", "value": "[[node.ip4]]"}, | |||
{"name": "dns_port", "value": "[[configuration.dnsPort]]"}, | |||
{"name": "ssl_port", "value": "[[node.sslPort]]"} | |||
], | |||
"tgzB64": "" | |||
}, | |||
{ | |||
"name": "nginx-0.0.1", | |||
"priority": 500, | |||
"template": true, | |||
"config": [ | |||
{"name": "server_name", "value": "[[node.fqdn]]"}, | |||
{"name": "letsencrypt_email", "value": "[[configuration.letsencryptEmail]]"}, | |||
{"name": "ssl_port", "value": "[[node.sslPort]]"}, | |||
{"name": "admin_port", "value": "[[node.adminPort]]"} | |||
], | |||
"tgzB64": "" | |||
}, | |||
{ | |||
"name": "mitmproxy-0.0.1", | |||
"priority": 600, | |||
"template": true, | |||
"install": "node", | |||
"config": [ | |||
{"name": "admin_port", "value": "[[node.adminPort]]"}, | |||
{"name": "bubble_network", "value": "[[node.network]]"}, | |||
{"name": "mitm_port", "value": "[[configuration.defaultMitmProxyPort]]"} | |||
], | |||
"tgzB64": "" | |||
}, | |||
{ | |||
"name": "bubble_finalizer-0.0.1", | |||
"priority": 9999999, | |||
"template": true, | |||
"config": [ | |||
{"name": "server_alias", "value": "[[network.networkDomain]]"}, | |||
{"name": "restore_key", "value": "[[restoreKey]]"}, | |||
{"name": "install_type", "value": "[[installType]]"}, | |||
{"name": "bubble_java_opts", "value": "-XX:MaxRAM=[[expr nodeSize.memoryMB '//' '2.625']]m"} | |||
], | |||
"optionalConfigNames": ["restore_key"], | |||
"tgzB64": "" | |||
} | |||
] |
@@ -1,5 +1,5 @@ | |||
{ | |||
"name": "bubble_finalizer", | |||
"name": "finalizer", | |||
"config": [ | |||
{"name": "server_alias", "value": "[[network.networkDomain]]"}, | |||
{"name": "restore_key", "value": "[[restoreKey]]"}, |
@@ -0,0 +1,129 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
import requests | |||
import traceback | |||
import sys | |||
import os | |||
import time | |||
import uuid | |||
import datetime | |||
import redis | |||
import json | |||
from bubble_config import bubble_network, bubble_port | |||
# Write python PID to file so that mitmdump_monitor.sh can check for excessive memory usage and restart if needed | |||
MITMDUMP_PID_FILE_PATH = '/home/mitmproxy/mitmdump.pid' | |||
MITMDUMP_PID_FILE = open(MITMDUMP_PID_FILE_PATH, "w") | |||
MITMDUMP_PID_FILE.write("%d" % os.getpid()) | |||
MITMDUMP_PID_FILE.close() | |||
HEADER_USER_AGENT = 'User-Agent' | |||
HEADER_REFERER = 'Referer' | |||
CTX_BUBBLE_MATCHERS='X-Bubble-Matchers' | |||
CTX_BUBBLE_ABORT='X-Bubble-Abort' | |||
CTX_BUBBLE_PASSTHRU='X-Bubble-Passthru' | |||
CTX_BUBBLE_REQUEST_ID='X-Bubble-RequestId' | |||
CTX_CONTENT_LENGTH='X-Bubble-Content-Length' | |||
CTX_CONTENT_LENGTH_SENT='X-Bubble-Content-Length-Sent' | |||
BUBBLE_URI_PREFIX='/__bubble/' | |||
REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) | |||
BUBBLE_ACTIVITY_LOG_PREFIX = 'bubble_activity_log_' | |||
BUBBLE_ACTIVITY_LOG_EXPIRATION = 600 | |||
def redis_set(name, value, ex): | |||
REDIS.set(name, value, nx=True, ex=ex) | |||
REDIS.set(name, value, xx=True, ex=ex) | |||
def bubble_log(message): | |||
print(str(datetime.datetime.time(datetime.datetime.now()))+': ' + message, file=sys.stderr) | |||
def bubble_activity_log(client_addr, server_addr, event, data): | |||
key = BUBBLE_ACTIVITY_LOG_PREFIX + str(time.time() * 1000.0) + '_' + str(uuid.uuid4()) | |||
value = json.dumps({ | |||
'source': 'mitmproxy', | |||
'client_addr': client_addr, | |||
'server_addr': server_addr, | |||
'event': event, | |||
'data': data | |||
}) | |||
bubble_log('bubble_activity_log: setting '+key+' = '+value) | |||
redis_set(key, value, BUBBLE_ACTIVITY_LOG_EXPIRATION) | |||
pass | |||
def bubble_passthru(remote_addr, addr, fqdn): | |||
headers = { | |||
'X-Forwarded-For': remote_addr, | |||
'Accept' : 'application/json', | |||
'Content-Type': 'application/json' | |||
} | |||
try: | |||
data = { | |||
'addr': str(addr), | |||
'fqdn': str(fqdn), | |||
'remoteAddr': remote_addr | |||
} | |||
response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/passthru', headers=headers, json=data) | |||
return response.ok | |||
except Exception as e: | |||
bubble_log('bubble_passthru API call failed: '+repr(e)) | |||
traceback.print_exc() | |||
return False | |||
def bubble_matchers(req_id, remote_addr, flow, host): | |||
headers = { | |||
'X-Forwarded-For': remote_addr, | |||
'Accept' : 'application/json', | |||
'Content-Type': 'application/json' | |||
} | |||
if HEADER_USER_AGENT not in flow.request.headers: | |||
bubble_log('bubble_matchers: no User-Agent header, setting to UNKNOWN') | |||
user_agent = 'UNKNOWN' | |||
else: | |||
user_agent = flow.request.headers[HEADER_USER_AGENT] | |||
if HEADER_REFERER not in flow.request.headers: | |||
bubble_log('bubble_matchers: no Referer header, setting to NONE') | |||
referer = 'NONE' | |||
else: | |||
try: | |||
referer = flow.request.headers[HEADER_REFERER].encode().decode() | |||
except Exception as e: | |||
bubble_log('bubble_matchers: error parsing Referer header: '+repr(e)) | |||
referer = 'NONE' | |||
try: | |||
data = { | |||
'requestId': req_id, | |||
'fqdn': host, | |||
'uri': flow.request.path, | |||
'userAgent': user_agent, | |||
'referer': referer, | |||
'remoteAddr': remote_addr | |||
} | |||
response = requests.post('http://127.0.0.1:'+bubble_port+'/api/filter/matchers/'+req_id, headers=headers, json=data) | |||
if response.ok: | |||
return response.json() | |||
bubble_log('bubble_matchers response not OK, returning empty matchers array: '+str(response.status_code)+' / '+repr(response.text)) | |||
except Exception as e: | |||
bubble_log('bubble_matchers API call failed: '+repr(e)) | |||
traceback.print_exc() | |||
return None | |||
def add_flow_ctx(flow, name, value): | |||
if not hasattr(flow, 'bubble_ctx'): | |||
flow.bubble_ctx = {} | |||
flow.bubble_ctx[name] = value | |||
def get_flow_ctx(flow, name): | |||
if not hasattr(flow, 'bubble_ctx'): | |||
return None | |||
if not name in flow.bubble_ctx: | |||
return None | |||
return flow.bubble_ctx[name] |
@@ -0,0 +1,178 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
import re | |||
import requests | |||
import urllib | |||
import traceback | |||
from mitmproxy.net.http import Headers | |||
from bubble_config import bubble_port, bubble_host_alias | |||
from bubble_api import CTX_BUBBLE_MATCHERS, CTX_BUBBLE_ABORT, BUBBLE_URI_PREFIX, \ | |||
CTX_BUBBLE_REQUEST_ID, CTX_CONTENT_LENGTH, CTX_CONTENT_LENGTH_SENT, bubble_log, get_flow_ctx, add_flow_ctx | |||
BUFFER_SIZE = 4096 | |||
HEADER_CONTENT_TYPE = 'Content-Type' | |||
HEADER_CONTENT_LENGTH = 'Content-Length' | |||
HEADER_CONTENT_ENCODING = 'Content-Encoding' | |||
HEADER_TRANSFER_ENCODING = 'Transfer-Encoding' | |||
BINARY_DATA_HEADER = {HEADER_CONTENT_TYPE: 'application/octet-stream'} | |||
def filter_chunk(chunk, req_id, last, content_encoding=None, content_type=None, content_length=None): | |||
url = 'http://127.0.0.1:' + bubble_port + '/api/filter/apply/' + req_id | |||
params_added = False | |||
if chunk and content_type: | |||
params_added = True | |||
url = (url | |||
+ '?type=' + urllib.parse.quote_plus(content_type)) | |||
if content_encoding: | |||
url = url + '&encoding=' + urllib.parse.quote_plus(content_encoding) | |||
if content_length: | |||
url = url + '&length=' + str(content_length) | |||
if last: | |||
if params_added: | |||
url = url + '&last=true' | |||
else: | |||
url = url + '?last=true' | |||
bubble_log('filter_chunk: url='+url) | |||
response = requests.post(url, data=chunk, headers=BINARY_DATA_HEADER) | |||
if not response.ok: | |||
err_message = 'filter_chunk: Error fetching ' + url + ', HTTP status ' + str(response.status_code) | |||
bubble_log(err_message) | |||
return b'' | |||
return response.content | |||
def bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type): | |||
""" | |||
chunks is a generator that can be used to iterate over all chunks. | |||
""" | |||
first = True | |||
content_length = get_flow_ctx(flow, CTX_CONTENT_LENGTH) | |||
try: | |||
for chunk in chunks: | |||
if content_length: | |||
bytes_sent = get_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT) | |||
chunk_len = len(chunk) | |||
last = chunk_len + bytes_sent >= content_length | |||
bubble_log('bubble_filter_chunks: content_length = '+str(content_length)+', bytes_sent = '+str(bytes_sent)) | |||
add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, bytes_sent + chunk_len) | |||
else: | |||
last = False | |||
if first: | |||
yield filter_chunk(chunk, req_id, last, content_encoding, content_type, content_length) | |||
first = False | |||
else: | |||
yield filter_chunk(chunk, req_id, last) | |||
if not content_length: | |||
yield filter_chunk(None, req_id, True) # get the last bits of data | |||
except Exception as e: | |||
bubble_log('bubble_filter_chunks: exception='+repr(e)) | |||
traceback.print_exc() | |||
yield None | |||
def bubble_modify(flow, req_id, content_encoding, content_type): | |||
return lambda chunks: bubble_filter_chunks(flow, chunks, req_id, content_encoding, content_type) | |||
def send_bubble_response(response): | |||
for chunk in response.iter_content(8192): | |||
yield chunk | |||
def responseheaders(flow): | |||
if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX): | |||
uri = 'http://127.0.0.1:' + bubble_port + '/' + flow.request.path[len(BUBBLE_URI_PREFIX):] | |||
bubble_log('responseheaders: sending special bubble request to '+uri) | |||
headers = { | |||
'Accept' : 'application/json', | |||
'Content-Type': 'application/json' | |||
} | |||
response = None | |||
if flow.request.method == 'GET': | |||
response = requests.get(uri, headers=headers, stream=True) | |||
elif flow.request.method == 'POST': | |||
bubble_log('responseheaders: special bubble request: POST content is '+str(flow.request.content)) | |||
headers['Content-Length'] = str(len(flow.request.content)) | |||
response = requests.post(uri, data=flow.request.content, headers=headers) | |||
else: | |||
bubble_log('responseheaders: special bubble request: method '+flow.request.method+' not supported') | |||
if response is not None: | |||
bubble_log('responseheaders: special bubble request: response status = '+str(response.status_code)) | |||
flow.response.headers = Headers() | |||
for key, value in response.headers.items(): | |||
flow.response.headers[key] = value | |||
flow.response.status_code = response.status_code | |||
flow.response.stream = lambda chunks: send_bubble_response(response) | |||
else: | |||
abort_code = get_flow_ctx(flow, CTX_BUBBLE_ABORT) | |||
if abort_code is not None: | |||
bubble_log('responseheaders: aborting request with HTTP status '+str(abort_code)) | |||
flow.response.headers = Headers() | |||
flow.response.status_code = abort_code | |||
flow.response.stream = lambda chunks: [] | |||
else: | |||
req_id = get_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID) | |||
matchers = get_flow_ctx(flow, CTX_BUBBLE_MATCHERS) | |||
if req_id is not None and matchers is not None: | |||
bubble_log('responseheaders: req_id='+req_id+' with matchers: '+repr(matchers)) | |||
if HEADER_CONTENT_TYPE in flow.response.headers: | |||
content_type = flow.response.headers[HEADER_CONTENT_TYPE] | |||
if matchers: | |||
any_content_type_matches = False | |||
for m in matchers: | |||
if 'contentTypeRegex' in m: | |||
typeRegex = m['contentTypeRegex'] | |||
if typeRegex is None: | |||
typeRegex = '^text/html.*' | |||
if re.match(typeRegex, content_type): | |||
any_content_type_matches = True | |||
bubble_log('responseheaders: req_id='+req_id+' found at least one matcher for content_type ('+content_type+'), filtering') | |||
break | |||
if not any_content_type_matches: | |||
bubble_log('responseheaders: req_id='+req_id+' no matchers for content_type ('+content_type+'), passing thru') | |||
return | |||
if HEADER_CONTENT_ENCODING in flow.response.headers: | |||
content_encoding = flow.response.headers[HEADER_CONTENT_ENCODING] | |||
else: | |||
content_encoding = None | |||
content_length_value = flow.response.headers.pop(HEADER_CONTENT_LENGTH, None) | |||
bubble_log('responseheaders: req_id='+req_id+' content_encoding='+repr(content_encoding) + ', content_type='+repr(content_type)) | |||
flow.response.stream = bubble_modify(flow, req_id, content_encoding, content_type) | |||
if content_length_value: | |||
flow.response.headers['transfer-encoding'] = 'chunked' | |||
# find server_conn to set fake_chunks on | |||
if flow.live and flow.live.ctx: | |||
ctx = flow.live.ctx | |||
while not hasattr(ctx, 'server_conn'): | |||
if hasattr(ctx, 'ctx'): | |||
ctx = ctx.ctx | |||
else: | |||
bubble_log('responseheaders: error finding server_conn. last ctx has no further ctx. type='+str(type(ctx))+' vars='+str(vars(ctx))) | |||
return | |||
if not hasattr(ctx, 'server_conn'): | |||
bubble_log('responseheaders: error finding server_conn. ctx type='+str(type(ctx))+' vars='+str(vars(ctx))) | |||
return | |||
content_length = int(content_length_value) | |||
ctx.server_conn.rfile.fake_chunks = content_length | |||
add_flow_ctx(flow, CTX_CONTENT_LENGTH, content_length) | |||
add_flow_ctx(flow, CTX_CONTENT_LENGTH_SENT, 0) | |||
else: | |||
bubble_log('responseheaders: no matchers, passing thru') | |||
pass | |||
else: | |||
bubble_log('responseheaders: no '+HEADER_CONTENT_TYPE +' header, passing thru') | |||
pass | |||
else: | |||
bubble_log('responseheaders: no '+CTX_BUBBLE_MATCHERS +' in ctx, passing thru') | |||
pass |
@@ -0,0 +1,112 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
# Parts of this are borrowed from tls_passthrough.py in the mitmproxy project. The mitmproxy license is reprinted here: | |||
# | |||
# Copyright (c) 2013, Aldo Cortesi. All rights reserved. | |||
# | |||
# Permission is hereby granted, free of charge, to any person obtaining a copy | |||
# of this software and associated documentation files (the "Software"), to deal | |||
# in the Software without restriction, including without limitation the rights | |||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | |||
# copies of the Software, and to permit persons to whom the Software is | |||
# furnished to do so, subject to the following conditions: | |||
# | |||
# The above copyright notice and this permission notice shall be included in | |||
# all copies or substantial portions of the Software. | |||
# | |||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | |||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |||
# SOFTWARE. | |||
# | |||
from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer | |||
from mitmproxy.exceptions import TlsProtocolException | |||
from bubble_api import bubble_log, bubble_passthru, bubble_activity_log, redis_set | |||
import redis | |||
import json | |||
REDIS_DNS_PREFIX = 'bubble_dns_' | |||
REDIS_PASSTHRU_PREFIX = 'bubble_passthru_' | |||
REDIS_PASSTHRU_DURATION = 60 * 60 # 1 hour timeout on passthru | |||
REDIS = redis.Redis(host='127.0.0.1', port=6379, db=0) | |||
def passthru_cache_prefix(client_addr, server_addr): | |||
return REDIS_PASSTHRU_PREFIX + client_addr + '_' + server_addr | |||
class TlsFeedback(TlsLayer): | |||
""" | |||
Monkey-patch _establish_tls_with_client to get feedback if TLS could be established | |||
successfully on the client connection (which may fail due to cert pinning). | |||
""" | |||
def _establish_tls_with_client(self): | |||
client_address = self.client_conn.address[0] | |||
server_address = self.server_conn.address[0] | |||
try: | |||
super(TlsFeedback, self)._establish_tls_with_client() | |||
except TlsProtocolException as e: | |||
bubble_log('_establish_tls_with_client: TLS error for '+repr(server_address)+', enabling passthru') | |||
cache_key = passthru_cache_prefix(client_address, server_address) | |||
fqdn = fqdn_for_addr(server_address) | |||
redis_set(cache_key, json.dumps({'fqdn': fqdn, 'addr': server_address, 'passthru': True}), ex=REDIS_PASSTHRU_DURATION) | |||
raise e | |||
def fqdn_for_addr(addr): | |||
fqdn = REDIS.get(REDIS_DNS_PREFIX + addr) | |||
if fqdn is None or len(fqdn) == 0: | |||
bubble_log('check_bubble_passthru: no FQDN found for addr '+repr(addr)+', checking raw addr') | |||
fqdn = b'' | |||
return fqdn.decode() | |||
def check_bubble_passthru(remote_addr, addr, fqdn): | |||
if bubble_passthru(remote_addr, addr, fqdn): | |||
bubble_log('check_bubble_passthru: bubble_passthru returned True for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning True') | |||
return {'fqdn': fqdn, 'addr': addr, 'passthru': True} | |||
bubble_log('check_bubble_passthru: bubble_passthru returned False for FQDN/addr '+repr(fqdn)+'/'+repr(addr)+', returning False') | |||
return {'fqdn': fqdn, 'addr': addr, 'passthru': False} | |||
def should_passthru(remote_addr, addr): | |||
prefix = 'should_passthru: '+repr(addr)+' ' | |||
bubble_log(prefix+'starting...') | |||
cache_key = passthru_cache_prefix(remote_addr, addr) | |||
passthru_json = REDIS.get(cache_key) | |||
if passthru_json is None or len(passthru_json) == 0: | |||
bubble_log(prefix+' not in redis or empty, calling check_bubble_passthru...') | |||
fqdn = fqdn_for_addr(addr) | |||
if fqdn is None or len(fqdn) == 0: | |||
bubble_log(prefix+' no fqdn found for addr '+addr+', returning (uncached) passthru = False') | |||
return {'fqdn': None, 'addr': addr, 'passthru': False} | |||
passthru = check_bubble_passthru(remote_addr, addr, fqdn) | |||
bubble_log(prefix+'check_bubble_passthru returned '+repr(passthru)+", storing in redis...") | |||
redis_set(cache_key, json.dumps(passthru), ex=REDIS_PASSTHRU_DURATION) | |||
else: | |||
bubble_log('found passthru_json='+str(passthru_json)+', touching key in redis') | |||
passthru = json.loads(passthru_json) | |||
REDIS.touch(cache_key) | |||
bubble_log(prefix+'returning '+repr(passthru)) | |||
return passthru | |||
def next_layer(next_layer): | |||
if isinstance(next_layer, TlsLayer) and next_layer._client_tls: | |||
client_address = next_layer.client_conn.address[0] | |||
server_address = next_layer.server_conn.address[0] | |||
passthru = should_passthru(client_address, server_address) | |||
if passthru['passthru']: | |||
bubble_log('next_layer: TLS passthru for ' + repr(next_layer.server_conn.address)) | |||
bubble_activity_log(client_address, server_address, 'tls_passthru', passthru['fqdn']) | |||
next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) | |||
next_layer.reply.send(next_layer_replacement) | |||
else: | |||
bubble_activity_log(client_address, server_address, 'tls_intercept', passthru['fqdn']) | |||
next_layer.__class__ = TlsFeedback |
@@ -0,0 +1,15 @@ | |||
{ | |||
"name": "mitmproxy-0.0.1", | |||
"priority": 600, | |||
"template": true, | |||
"install": "node", | |||
"config": [ | |||
{"name": "admin_port", "value": "[[node.adminPort]]"}, | |||
{"name": "bubble_network", "value": "[[node.network]]"}, | |||
{"name": "mitm_port", "value": "[[configuration.defaultMitmProxyPort]]"}, | |||
{"name": "server_name", "value": "[[node.fqdn]]"}, | |||
{"name": "server_alias", "value": "[[network.networkDomain]]"}, | |||
{"name": "ssl_port", "value": "[[node.sslPort]]"} | |||
], | |||
"tgzB64": "" | |||
} |
@@ -0,0 +1,148 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
import re | |||
import time | |||
import uuid | |||
from bubble_api import bubble_matchers, bubble_log, bubble_activity_log, CTX_BUBBLE_MATCHERS, BUBBLE_URI_PREFIX, CTX_BUBBLE_ABORT, CTX_BUBBLE_PASSTHRU, CTX_BUBBLE_REQUEST_ID, add_flow_ctx | |||
from bubble_config import bubble_host, bubble_host_alias | |||
# This regex extracts splits the host header into host and port. | |||
# Handles the edge case of IPv6 addresses containing colons. | |||
# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 | |||
parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$") | |||
class Rerouter: | |||
@staticmethod | |||
def get_matchers(flow, host): | |||
if host is None: | |||
return None | |||
if flow.request.path and flow.request.path.startswith(BUBBLE_URI_PREFIX): | |||
bubble_log("get_matchers: not filtering special bubble path: "+flow.request.path) | |||
return None | |||
remote_addr = str(flow.client_conn.address[0]) | |||
try: | |||
host = host.decode() | |||
except (UnicodeDecodeError, AttributeError): | |||
try: | |||
host = str(host) | |||
except Exception as e: | |||
bubble_log('get_matchers: host '+repr(host)+' could not be decoded, type='+str(type(host))+' e='+repr(e)) | |||
return None | |||
if host == bubble_host or host == bubble_host_alias: | |||
bubble_log('get_matchers: request is for bubble itself ('+host+'), not matching') | |||
return None | |||
req_id = str(host) + '.' + str(uuid.uuid4()) + '.' + str(time.time()) | |||
bubble_log("get_matchers: requesting match decision for req_id="+req_id) | |||
resp = bubble_matchers(req_id, remote_addr, flow, host) | |||
if not resp: | |||
bubble_log('get_matchers: no response for remote_addr/host: '+remote_addr+'/'+str(host)) | |||
return None | |||
matchers = [] | |||
if 'matchers' in resp and resp['matchers'] is not None: | |||
for m in resp['matchers']: | |||
if 'urlRegex' in m: | |||
bubble_log('get_matchers: checking for match of path='+flow.request.path+' against regex: '+m['urlRegex']) | |||
else: | |||
bubble_log('get_matchers: checking for match of path='+flow.request.path+' -- NO regex, skipping') | |||
continue | |||
if re.match(m['urlRegex'], flow.request.path): | |||
bubble_log('get_matchers: rule matched, adding rule: '+m['rule']) | |||
matchers.append(m) | |||
else: | |||
bubble_log('get_matchers: rule (regex='+m['urlRegex']+') did NOT match, skipping rule: '+m['rule']) | |||
else: | |||
bubble_log('get_matchers: no matchers. response='+repr(resp)) | |||
decision = None | |||
if 'decision' in resp: | |||
decision = resp['decision'] | |||
matcher_response = { 'decision': decision, 'matchers': matchers, 'request_id': req_id } | |||
bubble_log("get_matchers: returning "+repr(matcher_response)) | |||
return matcher_response | |||
def request(self, flow): | |||
client_address = flow.client_conn.address[0] | |||
server_address = flow.server_conn.address[0] | |||
if flow.client_conn.tls_established: | |||
flow.request.scheme = "https" | |||
sni = flow.client_conn.connection.get_servername() | |||
port = 443 | |||
else: | |||
flow.request.scheme = "http" | |||
sni = None | |||
port = 80 | |||
host_header = flow.request.host_header | |||
# bubble_log("dns_spoofing.request: host_header is "+repr(host_header)) | |||
if host_header: | |||
m = parse_host_header.match(host_header) | |||
if m: | |||
host_header = m.group("host").strip("[]") | |||
if m.group("port"): | |||
port = int(m.group("port")) | |||
# Determine if this request should be filtered | |||
if sni or host_header: | |||
host = str(sni or host_header) | |||
if host.startswith("b'"): | |||
host = host[2:-1] | |||
log_url = flow.request.scheme + '://' + host + flow.request.path | |||
matcher_response = self.get_matchers(flow, sni or host_header) | |||
if matcher_response: | |||
if 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'passthru': | |||
bubble_log('dns_spoofing.request: passthru response returned, passing thru and NOT performing TLS interception...') | |||
add_flow_ctx(flow, CTX_BUBBLE_PASSTHRU, True) | |||
bubble_activity_log(client_address, server_address, 'http_passthru', log_url) | |||
return | |||
elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'].startswith('abort_'): | |||
bubble_log('dns_spoofing.request: found abort code: ' + str(matcher_response['decision']) + ', aborting') | |||
if matcher_response['decision'] == 'abort_ok': | |||
abort_code = 200 | |||
elif matcher_response['decision'] == 'abort_not_found': | |||
abort_code = 404 | |||
else: | |||
bubble_log('dns_spoofing.request: unknown abort code: ' + str(matcher_response['decision']) + ', aborting with 404 Not Found') | |||
abort_code = 404 | |||
add_flow_ctx(flow, CTX_BUBBLE_ABORT, abort_code) | |||
bubble_activity_log(client_address, server_address, 'http_abort' + str(abort_code), log_url) | |||
return | |||
elif 'decision' in matcher_response and matcher_response['decision'] is not None and matcher_response['decision'] == 'no_match': | |||
bubble_log('dns_spoofing.request: decision was no_match, passing thru...') | |||
bubble_activity_log(client_address, server_address, 'http_no_match', log_url) | |||
return | |||
elif ('matchers' in matcher_response | |||
and 'request_id' in matcher_response | |||
and len(matcher_response['matchers']) > 0): | |||
req_id = matcher_response['request_id'] | |||
bubble_log("dns_spoofing.request: found request_id: " + req_id + ' with matchers: ' + repr(matcher_response['matchers'])) | |||
add_flow_ctx(flow, CTX_BUBBLE_MATCHERS, matcher_response['matchers']) | |||
add_flow_ctx(flow, CTX_BUBBLE_REQUEST_ID, req_id) | |||
bubble_activity_log(client_address, server_address, 'http_match', log_url) | |||
else: | |||
bubble_log('dns_spoofing.request: no rules returned, passing thru...') | |||
bubble_activity_log(client_address, server_address, 'http_no_rules', log_url) | |||
else: | |||
bubble_log('dns_spoofing.request: no matcher_response returned, passing thru...') | |||
# bubble_activity_log(client_address, server_address, 'http_no_matcher_response', log_url) | |||
else: | |||
bubble_log('dns_spoofing.request: no sni/host found, not applying rules to path: ' + flow.request.path) | |||
bubble_activity_log(client_address, server_address, 'http_no_sni_or_host', 'n/a') | |||
flow.request.host_header = host_header | |||
flow.request.host = sni or host_header | |||
flow.request.port = port | |||
addons = [Rerouter()] |
@@ -0,0 +1,34 @@ | |||
#!/bin/bash | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
CERT="${1:?no cert provided}" | |||
TIMEOUT=${2:-0} | |||
function die { | |||
echo 1>&2 "${1}" | |||
exit 1 | |||
} | |||
START=$(date +%s) | |||
while [[ ! -f "${CERT}" ]] ; do | |||
ELAPSED=$(expr $(date +%s) - ${START}) | |||
if [[ ${ELAPSED} -gt ${TIMEOUT} ]] ; then | |||
break | |||
fi | |||
echo "Cert file does not exist, sleeping then rechecking: ${CERT}" | |||
sleep 5s | |||
done | |||
if [[ ! -f "${CERT}" ]] ; then | |||
die "Cert file does not exist: ${CERT}" | |||
fi | |||
if [[ "${CERT}" == *.pem || "${CERT}" == *.p12 ]] ; then | |||
openssl x509 -in "${CERT}" -inform PEM -out "${CERT}.crt" || die "Error converting certificate" | |||
CERT="${CERT}.crt" | |||
fi | |||
mkdir -p /usr/local/share/ca-certificates || die "Error ensuring CA certs directory exists" | |||
cp "${CERT}" /usr/local/share/ca-certificates || die "Error installing certificate" | |||
update-ca-certificates || die "Error updating CA certificates" |
@@ -0,0 +1,80 @@ | |||
#!/bin/bash | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
LOG=/tmp/bubble.mitmdump_monitor.log | |||
function die { | |||
echo 1>&2 "${1}" | |||
log "${1}" | |||
exit 1 | |||
} | |||
function log { | |||
echo "$(date): ${1}" >> ${LOG} | |||
} | |||
BUBBLE_MITM_MARKER=/home/bubble/.mitmdump_monitor | |||
ROOT_KEY_MARKER=/usr/share/bubble/mitmdump_monitor | |||
MITMDUMP_PID_FILE=/home/mitmproxy/mitmdump.pid | |||
MAX_MITM_PCT_MEM=18 | |||
# Start with MITM proxy turned off | |||
if [[ ! -f ${BUBBLE_MITM_MARKER} ]] ; then | |||
echo -n off > ${BUBBLE_MITM_MARKER} && chown bubble ${BUBBLE_MITM_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}" | |||
fi | |||
if [[ ! -f ${ROOT_KEY_MARKER} ]] ; then | |||
sleep 1s | |||
mkdir -p "$(dirname ${ROOT_KEY_MARKER})" && chmod 755 "$(dirname ${ROOT_KEY_MARKER})" || log "Error creating or setting permissions on ${ROOT_KEY_MARKER}" | |||
echo -n on > ${ROOT_KEY_MARKER} && touch ${ROOT_KEY_MARKER} && chmod 644 ${ROOT_KEY_MARKER} || log "Error writing 'on' to ${ROOT_KEY_MARKER}" | |||
fi | |||
function ensureMitmOn { | |||
log "Flushing PREROUTING before enabling MITM services" | |||
iptables -F PREROUTING -t nat || log "Error flushing port forwarding when enabling MITM services" | |||
log "Enabling MITM port forwarding on TCP port 80 -> 8888" | |||
iptables -I PREROUTING 1 -t nat -p tcp --dport 80 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 80 -> 8888" | |||
log "Enabling MITM port forwarding on TCP port 443 -> 8888" | |||
iptables -I PREROUTING 1 -t nat -p tcp --dport 443 -j REDIRECT --to-ports 8888 || log "Error enabling MITM port forwarding 443 -> 8888" | |||
echo -n on > ${ROOT_KEY_MARKER} | |||
} | |||
function ensureMitmOff { | |||
log "Flushing PREROUTING to disable MITM services" | |||
iptables -F PREROUTING -t nat || log "Error flushing port forwarding when disabling MITM services" | |||
echo -n off > ${ROOT_KEY_MARKER} || log "Error writing 'off' to ${ROOT_KEY_MARKER}" | |||
} | |||
log "Watching marker file ${BUBBLE_MITM_MARKER} ..." | |||
sleep 2s && touch ${BUBBLE_MITM_MARKER} || log "Error touching ${BUBBLE_MITM_MARKER}" # first time through, always check and set on/off state | |||
while : ; do | |||
if [[ $(stat -c %Y ${BUBBLE_MITM_MARKER}) -gt $(stat -c %Y ${ROOT_KEY_MARKER}) ]] ; then | |||
if [[ ! -z "$(cmp -b ${ROOT_KEY_MARKER} ${BUBBLE_MITM_MARKER})" ]] ; then | |||
if [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "on" ]] ; then | |||
ensureMitmOn | |||
elif [[ "$(cat ${BUBBLE_MITM_MARKER} | tr -d [[:space:]])" == "off" ]] ; then | |||
ensureMitmOff | |||
else | |||
log "Error: marker file ${BUBBLE_MITM_MARKER} contained invalid value: $(cat ${BUBBLE_MITM_MARKER} | head -c 5)" | |||
fi | |||
fi | |||
fi | |||
# Check process memory usage, restart mitmdump if memory goes above max % allowed | |||
if [[ -f ${MITMDUMP_PID_FILE} && -s ${MITMDUMP_PID_FILE} ]] ; then | |||
MITM_PID="$(cat ${MITMDUMP_PID_FILE})" | |||
PCT_MEM="$(ps q ${MITM_PID} -o %mem --no-headers | tr -d [[:space:]] | cut -f1 -d. | sed 's/[^0-9]*//g')" | |||
# log "Info: mitmdump pid ${MITM_PID} using ${PCT_MEM}% of memory" | |||
if [[ ! -z "${PCT_MEM}" ]] ; then | |||
if [[ ${PCT_MEM} -ge ${MAX_MITM_PCT_MEM} ]] ; then | |||
log "Warn: mitmdump: pid=$(cat ${MITMDUMP_PID_FILE}) memory used > max, restarting: ${PCT_MEM}% >= ${MAX_MITM_PCT_MEM}%" | |||
supervisorctl restart mitmdump | |||
fi | |||
else | |||
log "Error: could not determine mitmdump % memory, maybe PID file ${MITMDUMP_PID_FILE} is out of date? pid found was ${MITM_PID}" | |||
fi | |||
else | |||
log "Error: mitmdump PID file ${MITMDUMP_PID_FILE} not found or empty" | |||
fi | |||
sleep 5s | |||
done |
@@ -0,0 +1,30 @@ | |||
#!/bin/bash | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
function die { | |||
echo 1>&2 "${1}" | |||
exit 1 | |||
} | |||
CERTS_BACKUP=/home/bubble/mitm_certs | |||
if [[ ! -d ${CERTS_BACKUP} ]] ; then | |||
echo "No mitm_certs backup found, skipping restore" | |||
exit 0 | |||
fi | |||
MITM_CERTS=/home/mitmproxy/.mitmproxy | |||
if [[ -d ${MITM_CERTS} ]] ; then | |||
echo "Removing obsolete mitm certs: ${MITM_CERTS}" | |||
rm -rf ${MITM_CERTS} || die "Error removing obsolete mitm certs" | |||
if [[ -d ${MITM_CERTS} ]] ; then | |||
die "Error removing obsolete mitm certs: dir still exists: ${MITM_CERTS}" | |||
fi | |||
fi | |||
mkdir -p ${MITM_CERTS} || die "Error creating mitm certs dir: ${MITM_CERTS}" | |||
chmod 750 ${MITM_CERTS} || die "Error setting permissions on mitm certs dir: ${MITM_CERTS}" | |||
cp -R ${CERTS_BACKUP}/* ${MITM_CERTS}/ || die "Error restoring mitm certs" | |||
chown -R mitmproxy ${MITM_CERTS} || die "Error changing ownership of ${MITM_CERTS}" | |||
chgrp -R root ${MITM_CERTS} || die "Error changing group ownership of ${MITM_CERTS}" | |||
chmod 440 ${MITM_CERTS}/* || die "Error setting permissions on mitm certs files" |
@@ -0,0 +1,22 @@ | |||
#!/bin/bash | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
MITM_PORT=${1:?no port provided} | |||
cd /home/mitmproxy/mitmproxy && \ | |||
./dev.sh && . ./venv/bin/activate && \ | |||
mitmdump \ | |||
--listen-host 0.0.0.0 \ | |||
--listen-port ${MITM_PORT} \ | |||
--showhost \ | |||
--no-http2 \ | |||
--set block_global=true \ | |||
--set block_private=false \ | |||
--set termlog_verbosity=debug \ | |||
--set flow_detail=3 \ | |||
--set stream_large_bodies=5m \ | |||
--set keep_host_header \ | |||
-s ./dns_spoofing.py \ | |||
-s ./bubble_passthru.py \ | |||
-s ./bubble_modify.py \ | |||
--mode transparent |
@@ -0,0 +1,23 @@ | |||
#!/bin/bash | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
MITM_DIR=${1:?no mitm dir specified} | |||
CERT_NAME=${2:?no cert name specified} | |||
if [[ ! -d "${MITM_DIR}" ]] ; then | |||
echo "mitm dir does not exist or is not a directory: ${MITM_DIR}" | |||
exit 1 | |||
fi | |||
OPTIONS_FILE="${MITM_DIR}/mitmproxy/options.py" | |||
if [[ ! -f "${OPTIONS_FILE}" ]] ; then | |||
echo "options.py not found in mitm dir: ${MITM_DIR}" | |||
exit 1 | |||
fi | |||
if [[ $(cat "${OPTIONS_FILE}" | egrep '^CONF_BASENAME =' | grep "${CERT_NAME}" | wc -l | tr -d ' ') -eq 0 ]] ; then | |||
temp="$(mktemp /tmp/options.py.XXXXXXX)" | |||
cat "${OPTIONS_FILE}" | sed -e 's/^CONF_BASENAME\s*=.*/CONF_BASENAME = "'"${CERT_NAME}"'"/' > "${temp}" | |||
mv "${temp}" "${OPTIONS_FILE}" | |||
fi |
@@ -0,0 +1,5 @@ | |||
[program:mitmdump_monitor] | |||
stdout_logfile = /dev/null | |||
stderr_logfile = /dev/null | |||
command=/usr/local/sbin/mitmdump_monitor.sh |
@@ -0,0 +1,115 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
- name: Install python3, pip, virtualenv and required dependencies | |||
apt: | |||
name: [ 'python3-pip', 'python3-venv', 'libc6-dev', 'libpython3-dev', 'g++', 'libffi-dev' ] | |||
state: present | |||
update_cache: yes | |||
- name: Install supervisor conf file | |||
template: | |||
src: supervisor_mitmproxy.conf.j2 | |||
dest: /etc/supervisor/conf.d/mitmproxy.conf | |||
owner: root | |||
group: root | |||
mode: 0400 | |||
- name: Create mitmproxy user | |||
user: | |||
name: mitmproxy | |||
comment: mitmdump user | |||
shell: /bin/bash | |||
system: yes | |||
home: /home/mitmproxy | |||
- name: Creates mitmproxy dir | |||
file: | |||
path: /home/mitmproxy/mitmproxy | |||
owner: mitmproxy | |||
group: mitmproxy | |||
mode: 0755 | |||
state: directory | |||
- name: Unzip mitmproxy.zip | |||
unarchive: | |||
src: mitmproxy.zip | |||
dest: /home/mitmproxy/mitmproxy | |||
- name: Copy mitmdump files | |||
copy: | |||
src: "{{ item }}" | |||
dest: "/home/mitmproxy/mitmproxy/{{ item }}" | |||
owner: mitmproxy | |||
group: mitmproxy | |||
mode: 0500 | |||
with_items: | |||
- bubble_api.py | |||
- dns_spoofing.py | |||
- bubble_passthru.py | |||
- bubble_modify.py | |||
- run_mitmdump.sh | |||
- name: Install cert helper scripts | |||
copy: | |||
src: "{{ item }}" | |||
dest: "/usr/local/bin/{{ item }}" | |||
owner: root | |||
group: root | |||
mode: 0500 | |||
with_items: | |||
- install_cert.sh | |||
- set_cert_name.sh | |||
- reuse_bubble_mitm_certs.sh | |||
- name: Set the cert name | |||
shell: set_cert_name.sh /home/mitmproxy/mitmproxy {{ server_alias }} | |||
- name: Set ownership of mitmproxy files | |||
shell: chown -R mitmproxy /home/mitmproxy/mitmproxy | |||
- name: Reuse bubble mitm certs if available | |||
shell: reuse_bubble_mitm_certs.sh | |||
- name: Copy bubble_config.py to /home/mitmproxy/mitmproxy | |||
template: | |||
src: bubble_config.py.j2 | |||
dest: /home/mitmproxy/mitmproxy/bubble_config.py | |||
owner: mitmproxy | |||
group: mitmproxy | |||
mode: 0500 | |||
- name: Fix missing symlink for libstdc++ | |||
file: | |||
src: /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | |||
dest: /usr/lib/x86_64-linux-gnu/libstdc++.so | |||
owner: root | |||
group: root | |||
state: link | |||
- name: Restart dnscrypt-proxy | |||
shell: service dnscrypt-proxy restart | |||
- name: restart supervisord | |||
service: | |||
name: supervisor | |||
enabled: yes | |||
state: restarted | |||
- import_tasks: route.yml | |||
- name: Install mitmdump_monitor | |||
copy: | |||
src: "mitmdump_monitor.sh" | |||
dest: "/usr/local/sbin/mitmdump_monitor.sh" | |||
owner: root | |||
group: root | |||
mode: 0500 | |||
- name: Install mitmdump_monitor supervisor conf file | |||
copy: | |||
src: supervisor_mitmdump_monitor.conf | |||
dest: /etc/supervisor/conf.d/mitmdump_monitor.conf | |||
- name: Ensure mitmdump_monitor is started | |||
shell: supervisorctl restart mitmdump_monitor |
@@ -0,0 +1,58 @@ | |||
# | |||
# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ | |||
# | |||
- sysctl: | |||
name: net.ipv4.ip_forward | |||
value: 1 | |||
sysctl_set: yes | |||
- sysctl: | |||
name: net.ipv6.conf.all.forwarding | |||
value: 1 | |||
sysctl_set: yes | |||
- sysctl: | |||
name: net.ipv4.conf.all.send_redirects | |||
value: 0 | |||
sysctl_set: yes | |||
- name: "Allow MITM private port" | |||
iptables: | |||
chain: INPUT | |||
action: insert | |||
rule_num: 10 | |||
protocol: tcp | |||
destination_port: "{{ mitm_port }}" | |||
ctstate: NEW | |||
syn: match | |||
jump: ACCEPT | |||
comment: Accept new local TCP DNS connections on private port | |||
become: yes | |||
- name: Route port 80 through mitmproxy | |||
iptables: | |||
table: nat | |||
chain: PREROUTING | |||
action: insert | |||
rule_num: 1 | |||
protocol: tcp | |||
destination_port: 80 | |||
jump: REDIRECT | |||
to_ports: "{{ mitm_port }}" | |||
- name: Route port 443 through mitmproxy | |||
iptables: | |||
table: nat | |||
chain: PREROUTING | |||
action: insert | |||
rule_num: 2 | |||
protocol: tcp | |||
destination_port: 443 | |||
jump: REDIRECT | |||
to_ports: "{{ mitm_port }}" | |||
- name: save iptables rules | |||
shell: iptables-save > /etc/iptables/rules.v4 | |||
become: yes | |||
- name: save iptables v6 rules | |||
shell: ip6tables-save > /etc/iptables/rules.v6 | |||
become: yes |
@@ -0,0 +1,5 @@ | |||
bubble_network = '{{ bubble_network }}' | |||
bubble_port = '{{ admin_port }}'; | |||
bubble_host = '{{ server_name }}' | |||
bubble_host_alias = '{{ server_alias }}' | |||
bubble_ssl_port = '{{ ssl_port }}' |
@@ -0,0 +1,7 @@ | |||
[program:mitmdump] | |||
stdout_logfile = /home/mitmproxy/mitmdump-out.log | |||
stderr_logfile = /home/mitmproxy/mitmdump-err.log | |||
command=sudo -H -u mitmproxy bash -c "/home/mitmproxy/mitmproxy/run_mitmdump.sh {{ mitm_port }}" | |||
stopasgroup=true | |||
stopsignal=QUIT |
@@ -17,8 +17,8 @@ | |||
{ "percent": 76,"messageKey":"role_nginx", "pattern":"TASK \\[nginx : [\\w\\s]+] \\*{5,}" }, | |||
{ "percent": 81,"messageKey":"role_nginx_certbot", "pattern":"TASK \\[nginx : Init certbot] \\*{5,}" }, | |||
{ "percent": 91,"messageKey":"role_mitmproxy", "pattern":"TASK \\[mitmproxy : [\\w\\s]+] \\*{5,}" }, | |||
{ "percent": 94,"messageKey":"role_bubble_finalizer", "pattern":"TASK \\[bubble_finalizer : [\\w\\s]+] \\*{5,}" }, | |||
{ "percent": 98,"messageKey":"role_bubble_finalizer_touch", "pattern":"TASK \\[bubble_finalizer : Touch first-time setup file] \\*{5,}" }, | |||
{ "percent": 99,"messageKey":"role_bubble_finalizer_start", "pattern":"TASK \\[bubble_finalizer : Ensure bubble is started] \\*{5,}" }, | |||
{ "percent": 94,"messageKey":"role_finalizer", "pattern":"TASK \\[finalizer : [\\w\\s]+] \\*{5,}" }, | |||
{ "percent": 98,"messageKey":"role_finalizer_touch", "pattern":"TASK \\[finalizer : Touch first-time setup file] \\*{5,}" }, | |||
{ "percent": 99,"messageKey":"role_finalizer_start", "pattern":"TASK \\[finalizer : Ensure bubble is started] \\*{5,}" }, | |||
{"percent": 100,"messageKey":"install_complete", "pattern":"PLAY RECAP \\*{5,}" } | |||
] |
@@ -353,9 +353,9 @@ meter_tick_role_bubble_algo=Setting up VPN | |||
meter_tick_role_nginx=Setting up web server | |||
meter_tick_role_nginx_certbot=Installing SSL certificates | |||
meter_tick_role_mitmproxy=Setting up MITM server | |||
meter_tick_role_bubble_finalizer=Finalizing Bubble installation | |||
meter_tick_role_bubble_finalizer_touch=Turning on "first-time" setting to allow you to unlock your Bubble | |||
meter_tick_role_bubble_finalizer_start=Starting Bubble API services | |||
meter_tick_role_finalizer=Finalizing Bubble installation | |||
meter_tick_role_finalizer_touch=Turning on "first-time" setting to allow you to unlock your Bubble | |||
meter_tick_role_finalizer_start=Starting Bubble API services | |||
meter_tick_install_complete=Bubble installation completed | |||
# Launch progress meter: success marker | |||
@@ -4,4 +4,4 @@ nginx | |||
algo | |||
mitmproxy | |||
bubble | |||
bubble_finalizer | |||
finalizer |
@@ -2,4 +2,4 @@ common | |||
firewall | |||
nginx | |||
bubble | |||
bubble_finalizer | |||
finalizer |