From f805953f6d80461e52d5e0f7cf13f3a7123bfa14 Mon Sep 17 00:00:00 2001 From: Kristijan Mitrovic Date: Tue, 7 Jul 2020 19:27:49 +0000 Subject: [PATCH] Add needed stuff and fixes for bubble restore process (#20) Merge branch 'master' of git.bubblev.org:bubblev/bubble into kris/add_support_for_restore_ui Rename isWaitingRestoring Use == instead of equals on enums Use ctime instead of creationTime in backup objects Fix bin scripts after CR Merge branch 'master' into kris/add_support_for_restore_ui # Conflicts: # bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties # bubble-web Add missing label Update web Merge branch 'master' into kris/add_support_for_restore_ui # Conflicts: # bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java # bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml # bubble-web Update web Use better check if restore is started Merge branch 'master' into kris/add_support_for_restore_ui # Conflicts: # bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java Init sage node on init self node if needed Update log message on post copy entities Update web Remove another word from host prefixes Mark restoring node as ready Add ctime to ticks-stats returned to FE Merge branch 'master' into kris/add_support_for_restore_ui # Conflicts: # bubble-web Try resetting progress meter on each new node Add auth/ready ep in skip auth for restore node Add more logs of api exception Update web Remove some bad host prefixes Update web Deploy web if it is included in jar Merge branch 'master' into kris/add_support_for_restore_ui Add support for showing latest backup on FE Merge branch 'master' into kris/add_support_for_restore_ui Update web Use proper flag for waiting restoring bubble Use separate bash to avoid continuing within venv Start restore monitor on instance when needed Save iptables in packer instance Merge branch 'master' into kris/add_support_for_restore_ui Save iptables before corresponding service restart Merge branch 'master' into kris/add_support_for_restore_ui # Conflicts: # bubble-server/src/main/java/bubble/server/BubbleConfiguration.java # bubble-web Fix iptables entries again Echo error to stderr Fix iptable rules creation Add back needed tags in algo related ansible tasks Update web Merge branch 'master' into kris/add_support_for_restore_ui # Conflicts: # bubble-web Add new labels and update web Use network state as restore mode tag Create full jar with web on full patching Update first_time_marker file with correct value Run first time listener for restoring instances also Merge branch 'master' into kris/add_support_for_restore_ui # Conflicts: # bubble-web Add new lib to utils pom Co-authored-by: Jonathan Cobb Co-authored-by: Kristijan Mitrovic Reviewed-on: https://git.bubblev.org/bubblev/bubble/pulls/20 --- bin/bpatchfull | 7 ++- bin/bscript | 2 +- bin/vultr/vcurl | 2 +- .../java/bubble/auth/BubbleAuthFilter.java | 3 +- .../java/bubble/model/cloud/BubbleBackup.java | 3 ++ .../resources/account/AuthResource.java | 1 + .../bubble/server/BubbleConfiguration.java | 10 ++++- .../main/java/bubble/server/BubbleServer.java | 3 +- .../listener/BubbleFirstTimeListener.java | 42 +++++++++++++----- .../bubble/service/backup/RestoreService.java | 3 ++ .../service/boot/StandardSelfNodeService.java | 43 +++++++++--------- .../service/cloud/NodeProgressMeterTick.java | 10 +++++ .../service/cloud/StandardNetworkService.java | 23 ++++++++-- .../dbfilter/FilteredEntityIterator.java | 3 +- .../roles/algo/tasks/algo_firewall.yml | 44 +++++++------------ .../ansible/roles/algo/tasks/main.yml | 13 +++--- .../bubble/files/bubble_restore_monitor.sh | 17 ++++--- .../ansible/roles/bubble/tasks/main.yml | 4 ++ .../ansible/roles/bubble/tasks/restore.yml | 2 - .../ansible/roles/finalizer/tasks/main.yml | 21 +++++++-- .../ansible/roles/mitmproxy/tasks/main.yml | 7 +-- .../main/resources/bubble/host-prefixes.txt | 8 ---- .../post_auth/ResourceMessages.properties | 14 ++++-- .../pre_auth/ResourceMessages.properties | 9 ++++ .../packer/roles/firewall/tasks/rules.yml | 29 ++++++++---- .../resources/models/include/new_bubble.json | 6 +++ .../models/tests/live/backup_and_restore.json | 23 ++++++++-- utils/pom.xml | 1 + 28 files changed, 238 insertions(+), 115 deletions(-) rename bubble-server/src/main/resources/{packer => ansible}/roles/bubble/files/bubble_restore_monitor.sh (89%) diff --git a/bin/bpatchfull b/bin/bpatchfull index 4c465a69..aa121ed0 100755 --- a/bin/bpatchfull +++ b/bin/bpatchfull @@ -45,7 +45,7 @@ else echo "Files changed, rebuilding bubble jar: " find "./src/main" -type f -newer "$(find "./target" -type f -name "bubble*.jar" | head -1)" fi - mvn -DskipTests=true -Dcheckstyle.skip=true clean package || die "Error packaging jar" + BUBBLE_PRODUCTION=1 mvn -DskipTests=true -Dcheckstyle.skip=true clean package || die "Error packaging jar" scp ./target/bubble*.jar ${HOST}:/tmp/bubble.jar || die "Error copying file to remote host ${HOST}" fi @@ -56,3 +56,8 @@ else echo "Patching and restarting..." ssh ${HOST} "cat /tmp/bubble.jar > ~bubble/api/bubble.jar && supervisorctl restart bubble" fi + +if [[ $(jar tf ./target/bubble*.jar | grep "^site/$") ]] ; then + echo "Deploying new web..." + ssh ${HOST} "cd ~bubble && jar xf /tmp/bubble.jar site && chown -R bubble:bubble site" +fi diff --git a/bin/bscript b/bin/bscript index d179b88a..c092a3d7 100755 --- a/bin/bscript +++ b/bin/bscript @@ -7,7 +7,7 @@ # # Usage: # -# run-script script-file [options] [args] +# bscript script-file [options] [args] # # script-file : a JSON API script # options : script options, see bubble.main.BubbleScriptOptions (and parent classes) for more info diff --git a/bin/vultr/vcurl b/bin/vultr/vcurl index 44e7b5ab..b9a5b1a4 100755 --- a/bin/vultr/vcurl +++ b/bin/vultr/vcurl @@ -3,7 +3,7 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # if [[ -z "${VULTR_API_KEY}" ]] ; then - echo "VULTR_API_KEY not defined in environment" + echo 1>&2 "VULTR_API_KEY not defined in environment" exit 1 fi diff --git a/bubble-server/src/main/java/bubble/auth/BubbleAuthFilter.java b/bubble-server/src/main/java/bubble/auth/BubbleAuthFilter.java index 7e52e28b..a083b1bf 100644 --- a/bubble-server/src/main/java/bubble/auth/BubbleAuthFilter.java +++ b/bubble-server/src/main/java/bubble/auth/BubbleAuthFilter.java @@ -37,7 +37,8 @@ public class BubbleAuthFilter extends AuthFilter { public static final Set SKIP_AUTH_PATHS = new SingletonSet<>(AUTH_ENDPOINT); public static final Set SKIP_ALL_AUTH = new SingletonSet<>("/"); public static final Set SKIP_AUTH_RESTORE = new HashSet<>(Arrays.asList(new String[] { - AUTH_ENDPOINT, BUBBLE_MAGIC_ENDPOINT, NOTIFY_ENDPOINT, MESSAGES_ENDPOINT + AUTH_ENDPOINT + EP_RESTORE + "/", AUTH_ENDPOINT + EP_CONFIGS, AUTH_ENDPOINT + EP_READY, + BUBBLE_MAGIC_ENDPOINT, NOTIFY_ENDPOINT, MESSAGES_ENDPOINT })); public static final Set SKIP_AUTH_TEST = new HashSet<>(Arrays.asList(ArrayUtil.append(SKIP_AUTH_PREFIXES.toArray(new String[0]), DEBUG_ENDPOINT diff --git a/bubble-server/src/main/java/bubble/model/cloud/BubbleBackup.java b/bubble-server/src/main/java/bubble/model/cloud/BubbleBackup.java index e9534f67..8c7fed8d 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/BubbleBackup.java +++ b/bubble-server/src/main/java/bubble/model/cloud/BubbleBackup.java @@ -7,6 +7,7 @@ package bubble.model.cloud; import bubble.model.account.Account; import bubble.model.account.HasAccount; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -75,4 +76,6 @@ public class BubbleBackup extends IdentifiableBase implements HasAccount { public boolean hasError () { return !empty(error); } public boolean canDelete() { return status.isDeletable() || getCtimeAge() > BR_STATE_LOCK_TIMEOUT; } + + @JsonProperty public long getCtime () { return super.getCtime(); } } diff --git a/bubble-server/src/main/java/bubble/resources/account/AuthResource.java b/bubble-server/src/main/java/bubble/resources/account/AuthResource.java index 1327ee61..523c04e0 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AuthResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AuthResource.java @@ -112,6 +112,7 @@ public class AuthResource { @GET @Path(EP_READY) public Response getNodeIsReady(@Context ContainerRequest ctx) { try { + if (configuration.getThisNetwork().getState() == BubbleNetworkState.restoring) return ok(); if (deviceDAO.findByAccountAndUninitialized(accountDAO.getFirstAdmin().getUuid()) .stream() .anyMatch(Device::configsOk)) { diff --git a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java index cc650417..4f1bfca8 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java +++ b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java @@ -13,9 +13,11 @@ import bubble.dao.account.AccountDAO; import bubble.dao.cloud.CloudServiceDAO; import bubble.model.cloud.AnsibleInstallType; import bubble.model.cloud.BubbleNetwork; +import bubble.model.cloud.BubbleNetworkState; import bubble.model.cloud.BubbleNode; import bubble.model.device.DeviceSecurityLevel; import bubble.server.listener.BubbleFirstTimeListener; +import bubble.service.backup.RestoreService; import bubble.service.boot.ActivationService; import bubble.service.boot.StandardSelfNodeService; import bubble.service.notify.LocalNotificationStrategy; @@ -90,6 +92,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration public static final String TAG_REQUIRE_SEND_METRICS = "requireSendMetrics"; public static final String TAG_SUPPORT = "support"; public static final String TAG_SECURITY_LEVELS = "securityLevels"; + public static final String TAG_RESTORE_MODE = "awaitingRestore"; public static final String DEFAULT_LOCAL_STORAGE_DIR = HOME_DIR + "/.bubble_local_storage"; @@ -290,11 +293,12 @@ public class BubbleConfiguration extends PgRestServerConfiguration public Map getPublicSystemConfigs () { synchronized (publicSystemConfigs) { if (publicSystemConfigs.get() == null) { - final BubbleNode thisNode = getThisNode(); final BubbleNetwork thisNetwork = getThisNetwork(); final AccountDAO accountDAO = getBean(AccountDAO.class); final CloudServiceDAO cloudDAO = getBean(CloudServiceDAO.class); final ActivationService activationService = getBean(ActivationService.class); + final RestoreService restoreService = getBean(RestoreService.class); + publicSystemConfigs.set(MapBuilder.build(new Object[][]{ {TAG_ALLOW_REGISTRATION, thisNetwork == null ? null : thisNetwork.getBooleanTag(TAG_ALLOW_REGISTRATION, false)}, {TAG_NETWORK_UUID, thisNetwork == null ? null : thisNetwork.getUuid()}, @@ -308,6 +312,10 @@ public class BubbleConfiguration extends PgRestServerConfiguration {TAG_CLOUD_CONFIGS, accountDAO.activated() ? null : activationService.getCloudDefaults()}, {TAG_LOCKED, accountDAO.locked()}, {TAG_LAUNCH_LOCK, isSageLauncher() || thisNetwork == null ? null : thisNetwork.launchLock()}, + {TAG_RESTORE_MODE, thisNetwork == null + ? false + : thisNetwork.getState() == BubbleNetworkState.restoring + && !restoreService.isRestoreStarted(thisNetwork.getUuid())}, {TAG_SSL_PORT, getDefaultSslPort()}, {TAG_SUPPORT, getSupport()}, {TAG_SECURITY_LEVELS, DeviceSecurityLevel.values()} diff --git a/bubble-server/src/main/java/bubble/server/BubbleServer.java b/bubble-server/src/main/java/bubble/server/BubbleServer.java index c56a9daa..3971b5db 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleServer.java +++ b/bubble-server/src/main/java/bubble/server/BubbleServer.java @@ -54,7 +54,8 @@ public class BubbleServer extends RestServerBase { }); public static final List RESTORE_LIFECYCLE_LISTENERS = Arrays.asList(new RestServerLifecycleListener[] { - new NodeInitializerListener() + new NodeInitializerListener(), + new BubbleFirstTimeListener() }); public static final String[] DEFAULT_ENV_FILE_PATHS = { diff --git a/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java b/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java index 941517a3..3082c1f3 100644 --- a/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java @@ -7,6 +7,7 @@ package bubble.server.listener; import bubble.ApiConstants; import bubble.dao.account.AccountDAO; import bubble.dao.account.AccountPolicyDAO; +import bubble.dao.cloud.BubbleNetworkDAO; import bubble.model.account.Account; import bubble.model.account.AccountPolicy; import bubble.model.account.message.AccountAction; @@ -14,8 +15,10 @@ import bubble.model.account.message.AccountMessage; import bubble.model.account.message.AccountMessageType; import bubble.model.account.message.ActionTarget; import bubble.model.cloud.BubbleNetwork; +import bubble.model.cloud.BubbleNetworkState; import bubble.server.BubbleConfiguration; import bubble.service.boot.SageHelloService; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.wizard.cache.redis.RedisService; import org.cobbzilla.wizard.server.RestServer; @@ -27,6 +30,7 @@ import java.util.concurrent.atomic.AtomicReference; import static java.util.concurrent.TimeUnit.HOURS; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.toStringOrDie; import static org.cobbzilla.wizard.cache.redis.RedisService.EX; @Slf4j @@ -37,6 +41,8 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase redis = new AtomicReference<>(); public static String getUnlockKey () { final RedisService r = redis.get(); @@ -49,15 +55,24 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase nodes = nodeDAO.findByNetwork(network.getUuid()); if (!nodes.isEmpty()) { - throw invalidEx("err.network.restore.nodesExist"); + throw invalidEx("err.networkRestore.nodesExist"); } if (network.getState() != BubbleNetworkState.stopped) { - throw invalidEx("err.network.restore.notStopped"); + throw invalidEx("err.networkRestore.notStopped"); } network.setState(BubbleNetworkState.starting); networkDAO.update(network); @@ -688,8 +699,14 @@ public class StandardNetworkService implements NetworkService { return newNodeRequest; + } catch (SimpleViolationException e) { + // TODO: should this go here, or just in some specific cases within above try block? + // also, should this go into other method here that are locking network? + try { unlockNetwork(network.getUuid(), lock); } catch (Exception e1) { } + log.error("startNetwork: original SimpleViolationException: ", e); + throw e; } catch (Exception e) { - return die("startNetwork: "+e, e); + return die("startNetwork: " + e, e); } } diff --git a/bubble-server/src/main/java/bubble/service/dbfilter/FilteredEntityIterator.java b/bubble-server/src/main/java/bubble/service/dbfilter/FilteredEntityIterator.java index 6ba58cba..227a9072 100644 --- a/bubble-server/src/main/java/bubble/service/dbfilter/FilteredEntityIterator.java +++ b/bubble-server/src/main/java/bubble/service/dbfilter/FilteredEntityIterator.java @@ -80,7 +80,8 @@ public class FilteredEntityIterator extends EntityIterator { if (!AccountOwnedEntityDAO.class.isAssignableFrom(dao.getClass())) { log.debug("iterate: skipping entity: " + c.getSimpleName()); } else if (isPostCopyEntity(c)) { - log.debug("iterate: skipping " + c.getSimpleName() + ", will copy after other objects are copied"); + log.debug("iterate: skipping " + c.getSimpleName() + + ", will copy some of these after other objects are copied"); } else { // copy entities. this is how the re-keying works (decrypt using current spring config, // encrypt using new config) diff --git a/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml b/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml index 9ddd4f27..145c7cbb 100644 --- a/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml +++ b/bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml @@ -2,54 +2,42 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # # Insert additional firewall rules to allow required services to function -- name: Allow HTTP +# Insert them all on rule_num 5, and insert them in reverse order here: +- name: Allow SSH iptables: chain: INPUT - action: insert - rule_num: 5 protocol: tcp - destination_port: 80 + destination_port: 22 ctstate: NEW syn: match jump: ACCEPT - comment: Accept new HTTP connections + comment: Accept new SSH connections become: yes -- name: Allow HTTPS +- name: "Allow HTTP on port {{ item }}" iptables: chain: INPUT - action: insert - rule_num: 6 protocol: tcp - destination_port: 443 + destination_port: "{{ item }}" ctstate: NEW syn: match jump: ACCEPT - comment: Accept new HTTPS connections + comment: "Accept new HTTP ({{ item }}) connections" + with_items: + - 80 + - 1080 become: yes -- name: Allow admin HTTPS on port 1443 +- name: "Allow HTTPS on port {{ item }}" iptables: chain: INPUT - action: insert - rule_num: 7 protocol: tcp - destination_port: 1443 + destination_port: "{{ item }}" ctstate: NEW syn: match jump: ACCEPT - comment: Accept new admin SSL connections - become: yes - -- name: Allow admin HTTP on port 1080 - iptables: - chain: INPUT - action: insert - rule_num: 8 - protocol: tcp - destination_port: 1080 - ctstate: NEW - syn: match - jump: ACCEPT - comment: Accept new admin SSL connections + comment: "Accept new HTTPS ({{ item }}) connections" + with_items: + - 443 + - 1443 become: yes diff --git a/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml index 62caaf05..6f51ac9c 100644 --- a/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml @@ -9,15 +9,14 @@ group: root mode: 0500 +- name: Stop algo monitors just in case + shell: bash -c "supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections" + # Don't setup algo when in restore mode, bubble_restore_monitor.sh will set it up after the CA key has been restored - name: Run algo playbook to install algo shell: bash -c "/root/ansible/roles/algo/algo/install_algo.sh 2>&1 >> /tmp/install_algo.log" - when: restore_key is not defined - -# Don't start monitors when in restore mode, bubble_restore_monitor.sh will start it after algo is installed -- name: Stop algo monitors (in restore mode) - shell: bash -c "supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections" - when: restore_key is defined + tags: algo_related -# Add bubble rules +# Algo installation clears out iptable rules. Add needed bubble rules back: - include: algo_firewall.yml + tags: algo_related diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh b/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_restore_monitor.sh similarity index 89% rename from bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh rename to bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_restore_monitor.sh index 499567be..3dde23d4 100755 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh +++ b/bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_restore_monitor.sh @@ -99,8 +99,13 @@ log "Removing node keys" echo "DELETE FROM bubble_node_key" | bsql.sh # restore local storage -log "Restoring bubble LocalStorage" -rm -rf ${BUBBLE_HOME}/.bubble_local_storage/* && rsync -ac ${RESTORE_BASE}/LocalStorage/* ${BUBBLE_HOME}/.bubble_local_storage/ || die "Error restoring LocalStorage" +LOCAL_STORAGE_DIR="${RESTORE_BASE}/LocalStorage" +if [[ -d LOCAL_STORAGE_DIR ]] ; then + log "Restoring bubble LocalStorage" + rm -rf ${BUBBLE_HOME}/.bubble_local_storage/* \ + && rsync -ac ${LOCAL_STORAGE_DIR}/* ${BUBBLE_HOME}/.bubble_local_storage/ \ + || die "Error restoring LocalStorage" +fi # flush redis log "Flushing redis" @@ -121,11 +126,9 @@ else fi cd ${ALGO_BASE} && tar xzf ${CONFIGS_BACKUP} || die "Error restoring algo VPN configs" - cd "${ANSIBLE_DIR}" && \ - . ./venv/bin/activate && \ - bash -c \ - "ansible-playbook --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \ - || die "Error running ansible in post-restore. journalctl -xe = $(journalctl -xe | tail -n 50)" + ANSIBLE_CMD="ansible-playbook --tags 'algo_related,always' --inventory ./hosts ./playbook.yml" + bash -c "cd '${ANSIBLE_DIR}' && . ./venv/bin/activate && ${ANSIBLE_CMD} 2>&1 >> ${LOG}" \ + || die "Error running ansible in post-restore. journalctl -xe = $(journalctl -xe | tail -n 50)" fi # restart mitm proxy service diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml index 3bef1179..01cb80ad 100644 --- a/bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml @@ -45,3 +45,7 @@ - sage_key.json - import_tasks: postgresql_data.yml + +- name: Start monitor for restoring this bubble if applicable + include: restore.yml + when: restore_key is defined diff --git a/bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml index 61f57fd1..aed4a2c6 100644 --- a/bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml +++ b/bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml @@ -10,8 +10,6 @@ mode: 0550 with_items: - "bubble_restore_monitor.sh" - when: restore_key is defined - name: Start restore monitor shell: bash -c 'nohup /usr/local/bin/bubble_restore_monitor.sh {{ admin_port }} {{ restore_timeout }} > /dev/null &' - when: restore_key is defined diff --git a/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml index fc9b27f3..a6a12795 100644 --- a/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml @@ -4,12 +4,12 @@ - name: Snapshot ansible roles in the background command: bash -c "/usr/local/bin/snapshot_ansible.sh &" -- name: Touch first-time setup file - shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n install > /home/bubble/first_time_marker ; fi" +- name: Create first-time setup file + shell: su - bubble bash -c "echo -n install > /home/bubble/first_time_marker" when: restore_key is not defined -- name: Touch first-time setup file (restore) - shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n restore > /home/bubble/first_time_marker ; fi" +- name: Create first-time setup file (restore) + shell: su - bubble bash -c "echo -n restore > /home/bubble/first_time_marker" when: restore_key is defined - name: Install mitmproxy CA cert in local CA store @@ -25,6 +25,19 @@ src: "supervisor_bubble.conf.j2" dest: /etc/supervisor/conf.d/bubble.conf +- name: save iptables v4 rules + shell: iptables-save > /etc/iptables/rules.v4 + become: yes + +- name: save iptables v6 rules + shell: ip6tables-save > /etc/iptables/rules.v6 + become: yes + +- name: Restart iptables + service: + name: netfilter-persistent + state: restarted + # We cannot receive notifications until nginx is running, so start bubble API as the very last step - name: reload supervisord shell: supervisorctl reload diff --git a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml index 33c490da..499957d2 100644 --- a/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml @@ -31,11 +31,9 @@ src: supervisor_mitmdump_monitor.conf dest: /etc/supervisor/conf.d/mitmdump_monitor.conf -- name: "Allow MITM private port" +- name: Allow MITM private port iptables: chain: INPUT - action: insert - rule_num: 7 protocol: tcp destination_port: 8888 ctstate: NEW @@ -43,6 +41,9 @@ jump: ACCEPT comment: Accept new local connections on mitm port become: yes + tags: algo_related + # ensuring that algo did its work on iptables before, so rule num 5 is ok to use - name: reload supervisord shell: supervisorctl reload + tags: always diff --git a/bubble-server/src/main/resources/bubble/host-prefixes.txt b/bubble-server/src/main/resources/bubble/host-prefixes.txt index 394fac72..0fee916b 100644 --- a/bubble-server/src/main/resources/bubble/host-prefixes.txt +++ b/bubble-server/src/main/resources/bubble/host-prefixes.txt @@ -1418,7 +1418,6 @@ briny brios brise brisk -briss brith brits britt @@ -3496,7 +3495,6 @@ fease feast feats feaze -fecal feces fecht fecit @@ -3548,7 +3546,6 @@ fesse festa fests festy -fetal fetas fetch feted @@ -3557,7 +3554,6 @@ fetid fetor fetta fetts -fetus fetwa feuar feuds @@ -4915,7 +4911,6 @@ horde horis horme horns -horny horse horst horsy @@ -8060,9 +8055,6 @@ porge porgy porks porky -porno -porns -porny porta ports porty diff --git a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties index 7716cc27..2a7d4461 100644 --- a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties +++ b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties @@ -225,6 +225,7 @@ message_plan_node_apps=Your Bubble will include these apps: # Network Page - Connect message_network_connect=Connect to Bubble +message_network_restore=Connect to Bubble to actually start restoring process # Network Page - Restore Keys link_network_action_request_keys=Request Bubble Restore Key @@ -236,6 +237,10 @@ field_network_key_download_code=Download Code field_network_key_download_password=Encrypt with password button_label_retrieve_keys=Download err.retrieveNetworkKeys.notFound=Download Code Not Found +restore_key_label=Your restore short key is: +button_label_restore=Restore +button_description_restore=You will need full network restore key build from this bubble while if was running. +restore_not_possible_nodes_exist_html=Cannot restore bubbles with existing nodes. Please contact support@getbubblenow.com # Network Page - Danger Zone title_network_danger_zone=Danger Zone @@ -243,6 +248,9 @@ link_network_action_stop=Stop link_network_action_stop_description=Stop this Bubble. If you have downloaded your restore key, you can later restore it. network_action_stop_not_ready=Still loading, try again in a moment. confirmation_network_action_stop=Please confirm stop.\nConnected devices will lose Internet access until they are disconnected from the Bubble\nYou can restore this Bubble later. +label_latest_backup=Latest Backup: +label_no_latest_backup=No backups available +link_backup_network=Queue new backup link_network_action_delete=Delete link_network_action_delete_description=Delete this Bubble and all backups. You will not be able to restore this Bubble. This action cannot be undone. confirmation_network_action_delete=Please confirm deletion.\nYou will not be able to restore this Bubble.\nThis action cannot be undone. @@ -712,8 +720,8 @@ err.networkKeys.invalid=Bubble Restore Key was not valid err.networkName.required=Network name is required err.network.cannotStartInCurrentState=Cannot proceed: network cannot be started in its current state err.network.required=Network is required -err.network.restore.nodesExist=Cannot restore when active nodes exist -err.network.restore.notStopped=Cannot restore when network is running +err.networkRestore.nodesExist=Cannot restore when active nodes exist +err.networkRestore.notStopped=Cannot restore when network is running err.nick.alreadyInUse=Nickname is already in use by another contact err.nick.tooLong=Nickname cannot be longer than 100 characters err.node.notInitialized=Node is not initialized @@ -868,4 +876,4 @@ err.addFilter.analyticsFilterRequired=Filter pattern is required err.nodemanager.error=Error calling nodemanager err.nodemanager.noPasswordSet=No nodemanager password is set err.nodemanager.invalidPath=Path is invalid -err.nodemanager.nodeNotLocal=Target node must be this node \ No newline at end of file +err.nodemanager.nodeNotLocal=Target node must be this node diff --git a/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties b/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties index fd62aa41..3ff441be 100644 --- a/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties +++ b/bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties @@ -132,6 +132,7 @@ label_menu_apps=Apps label_menu_apps_icon=fa fa-smile label_menu_notifications=Notifications label_menu_notifications_icon=fa fa-flag +label_menu_network=My Bubble label_menu_networks=Bubbles label_menu_networks_icon=fa fa-cloud label_menu_bills=Bills @@ -281,6 +282,13 @@ field_label_password_guidance=Password must be at least 8 characters long.
P field_label_confirm_password=Confirm Password field_label_unlock_key=Unlock Key form_title_register=Register +form_title_restore=Restore +message_restore_not_applicable=Restore already started or not applicable. +message_back_to_root=Back to your Bubble +field_label_restore_short_key=Short Key (6 letters) +field_label_restore_long_key=Long Network Key +err.restoreShortKey.required=Short Key is required +err.restoreLongNetworkKey.required=Long Network Key is required field_label_contactType=Contact Type field_label_email=Email field_label_promoCode=Beta Invite Code @@ -302,6 +310,7 @@ button_label_login=Login button_label_register=Register button_label_forgotPassword=Forgot Password button_label_cancel=Cancel +button_label_restore=Restore alert_registration_success=Registration successful form_title_forgotPassword=Forgot Password button_label_resetPassword=Request Password Reset diff --git a/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml b/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml index dcb24c85..bff800b7 100644 --- a/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml +++ b/bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml @@ -29,33 +29,46 @@ become: yes when: fw_enable_ssh -- name: Allow HTTP +- name: "Allow HTTP on port {{ item }}" iptables: chain: INPUT protocol: tcp - destination_port: 80 + destination_port: "{{ item }}" ctstate: NEW syn: match jump: ACCEPT - comment: Accept new HTTP connections + comment: "Accept new HTTP ({{ item }}) connections" + with_items: + - 80 + - 1080 become: yes when: fw_enable_http -- name: Allow HTTPS +- name: "Allow HTTPS on port {{ item }}" iptables: chain: INPUT protocol: tcp - destination_port: 443 + destination_port: "{{ item }}" ctstate: NEW syn: match jump: ACCEPT - comment: Accept new HTTPS connections + comment: "Accept new HTTPS ({{ item }}) connections" + with_items: + - 443 + - 1443 become: yes when: fw_enable_http - name: Drop everything else iptables: chain: INPUT - jump: DROP - comment: Drop anything else + policy: DROP + become: yes + +- name: save iptables v4 rules + shell: iptables-save > /etc/iptables/rules.v4 + become: yes + +- name: save iptables v6 rules + shell: ip6tables-save > /etc/iptables/rules.v6 become: yes diff --git a/bubble-server/src/test/resources/models/include/new_bubble.json b/bubble-server/src/test/resources/models/include/new_bubble.json index a7768fde..8764048c 100644 --- a/bubble-server/src/test/resources/models/include/new_bubble.json +++ b/bubble-server/src/test/resources/models/include/new_bubble.json @@ -223,5 +223,11 @@ {"condition": "json.admin() == true"} ] } + }, + + { + "comment": "network should be in running state by now", + "request": { "uri": "me/networks/<>" }, + "response": { "check": [{ "condition": "json.getState().name() == 'running'" }] } } ] diff --git a/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json b/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json index 9a8bbff7..33716f47 100644 --- a/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json +++ b/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json @@ -151,18 +151,29 @@ "response": { "store": "restoreNN", "check": [ - {"condition": "restoreNN.getNetwork() == bubbleNetwork.getNetwork()"} + { "condition": "restoreNN.getNetwork() == bubbleNetwork.getNetwork()" }, + { "condition": "restoreNN.getState().name() == 'starting'" } ] } }, { - "comment": "restore node using restoreKey", - "before": "await_url .bubble 16m:20m 20s", + "comment": "wait for network and then try to login - cannot do that as network is in restoring state", "connection": { "name": "restoredBubbleConnection", "baseUri": "https://{{restoreNN.fqdn}}:{{serverConfig.nginxPort}}/api" }, + "before": "await_url .bubble 16m:20m 20s", + "request": { + "session": "new", + "uri": "auth/login", + "entity": { "name": "{{username}}", "password": "password1!" } + }, + "response": { "status": "401" } + }, + + { + "comment": "restore node using restoreKey", "request": { "uri": "auth/restore/{{restoreNN.restoreKey}}", "entity": { @@ -191,6 +202,12 @@ } }, + { + "comment": "check again for bubble's status - should be running", + "request": { "uri": "me/networks/{{ restoreNN.getNetwork() }}" }, + "response": { "check": [{ "condition": "json.getState().name() == 'running'" }] } + }, + { "comment": "verify account we added has been restored", "request": { diff --git a/utils/pom.xml b/utils/pom.xml index 1bab2aa1..16e17d54 100644 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -26,6 +26,7 @@ This code is available under the GNU Affero General Public License, version 3: h cobbzilla-wizard restex templated-mail-sender + abp-parser