Add needed stuff and fixes for bubble restore process (#20)

Merge branch 'master' of git.bubblev.org:bubblev/bubble into kris/add_support_for_restore_ui

Rename isWaitingRestoring

Use == instead of equals on enums

Use ctime instead of creationTime in backup objects

Fix bin scripts after CR

Merge branch 'master' into kris/add_support_for_restore_ui

# Conflicts:
#	bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties
#	bubble-web

Add missing label

Update web

Merge branch 'master' into kris/add_support_for_restore_ui

# Conflicts:
#	bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java
#	bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml
#	bubble-web

Update web

Use better check if restore is started

Merge branch 'master' into kris/add_support_for_restore_ui

# Conflicts:
#	bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

Init sage node on init self node if needed

Update log message on post copy entities

Update web

Remove another word from host prefixes

Mark restoring node as ready

Add ctime to ticks-stats returned to FE

Merge branch 'master' into kris/add_support_for_restore_ui

# Conflicts:
#	bubble-web

Try resetting progress meter on each new node

Add auth/ready ep in skip auth for restore node

Add more logs of api exception

Update web

Remove some bad host prefixes

Update web

Deploy web if it is included in jar

Merge branch 'master' into kris/add_support_for_restore_ui

Add support for showing latest backup on FE

Merge branch 'master' into kris/add_support_for_restore_ui

Update web

Use proper flag for waiting restoring bubble

Use separate bash to avoid continuing within venv

Start restore monitor on instance when needed

Save iptables in packer instance

Merge branch 'master' into kris/add_support_for_restore_ui

Save iptables before corresponding service restart

Merge branch 'master' into kris/add_support_for_restore_ui

# Conflicts:
#	bubble-server/src/main/java/bubble/server/BubbleConfiguration.java
#	bubble-web

Fix iptables entries again

Echo error to stderr

Fix iptable rules creation

Add back needed tags in algo related ansible tasks

Update web

Merge branch 'master' into kris/add_support_for_restore_ui

# Conflicts:
#	bubble-web

Add new labels and update web

Use network state as restore mode tag

Create full jar with web on full patching

Update first_time_marker file with correct value

Run first time listener for restoring instances also

Merge branch 'master' into kris/add_support_for_restore_ui

# Conflicts:
#	bubble-web

Add new lib to utils pom

Co-authored-by: Jonathan Cobb <jonathan@kyuss.org>
Co-authored-by: Kristijan Mitrovic <kmitrovic@itekako.com>
Reviewed-on: #20

bin/bpatchfull

@@ -45,7 +45,7 @@ else
     echo "Files changed, rebuilding bubble jar: "
     find "./src/main" -type f -newer "$(find "./target" -type f -name "bubble*.jar" | head -1)"
   fi
-  mvn -DskipTests=true -Dcheckstyle.skip=true clean package || die "Error packaging jar"
+  BUBBLE_PRODUCTION=1 mvn -DskipTests=true -Dcheckstyle.skip=true clean package || die "Error packaging jar"
   scp ./target/bubble*.jar ${HOST}:/tmp/bubble.jar || die "Error copying file to remote host ${HOST}"
 fi
 
@@ -56,3 +56,8 @@ else
   echo "Patching and restarting..."
   ssh ${HOST} "cat /tmp/bubble.jar > ~bubble/api/bubble.jar && supervisorctl restart bubble"
 fi
+
+if [[ $(jar tf ./target/bubble*.jar | grep "^site/$") ]] ; then
+  echo "Deploying new web..."
+  ssh ${HOST} "cd ~bubble && jar xf /tmp/bubble.jar site && chown -R bubble:bubble site"
+fi

bin/bscript

@@ -7,7 +7,7 @@
 #
 # Usage:
 #
-#     run-script script-file [options] [args]
+#     bscript script-file [options] [args]
 #
 #   script-file    : a JSON API script
 #   options        : script options, see bubble.main.BubbleScriptOptions (and parent classes) for more info

bin/vultr/vcurl

@@ -3,7 +3,7 @@
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
 if [[ -z "${VULTR_API_KEY}" ]] ; then
-  echo "VULTR_API_KEY not defined in environment"
+  echo 1>&2 "VULTR_API_KEY not defined in environment"
   exit 1
 fi
 

bubble-server/src/main/java/bubble/auth/BubbleAuthFilter.java

@@ -37,7 +37,8 @@ public class BubbleAuthFilter extends AuthFilter<Account> {
     public static final Set<String> SKIP_AUTH_PATHS = new SingletonSet<>(AUTH_ENDPOINT);
     public static final Set<String> SKIP_ALL_AUTH = new SingletonSet<>("/");
     public static final Set<String> SKIP_AUTH_RESTORE = new HashSet<>(Arrays.asList(new String[] {
-            AUTH_ENDPOINT, BUBBLE_MAGIC_ENDPOINT, NOTIFY_ENDPOINT, MESSAGES_ENDPOINT
+            AUTH_ENDPOINT + EP_RESTORE + "/", AUTH_ENDPOINT + EP_CONFIGS, AUTH_ENDPOINT + EP_READY,
+            BUBBLE_MAGIC_ENDPOINT, NOTIFY_ENDPOINT, MESSAGES_ENDPOINT
     }));
     public static final Set<String> SKIP_AUTH_TEST = new HashSet<>(Arrays.asList(ArrayUtil.append(SKIP_AUTH_PREFIXES.toArray(new String[0]),
             DEBUG_ENDPOINT

bubble-server/src/main/java/bubble/model/cloud/BubbleBackup.java

@@ -7,6 +7,7 @@ package bubble.model.cloud;
 import bubble.model.account.Account;
 import bubble.model.account.HasAccount;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -75,4 +76,6 @@ public class BubbleBackup extends IdentifiableBase implements HasAccount {
     public boolean hasError () { return !empty(error); }
 
     public boolean canDelete() { return status.isDeletable() || getCtimeAge() > BR_STATE_LOCK_TIMEOUT; }
+
+    @JsonProperty public long getCtime () { return super.getCtime(); }
 }

bubble-server/src/main/java/bubble/resources/account/AuthResource.java

@@ -112,6 +112,7 @@ public class AuthResource {
     @GET @Path(EP_READY)
     public Response getNodeIsReady(@Context ContainerRequest ctx) {
         try {
+            if (configuration.getThisNetwork().getState() == BubbleNetworkState.restoring) return ok();
             if (deviceDAO.findByAccountAndUninitialized(accountDAO.getFirstAdmin().getUuid())
                     .stream()
                     .anyMatch(Device::configsOk)) {

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -13,9 +13,11 @@ import bubble.dao.account.AccountDAO;
 import bubble.dao.cloud.CloudServiceDAO;
 import bubble.model.cloud.AnsibleInstallType;
 import bubble.model.cloud.BubbleNetwork;
+import bubble.model.cloud.BubbleNetworkState;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.device.DeviceSecurityLevel;
 import bubble.server.listener.BubbleFirstTimeListener;
+import bubble.service.backup.RestoreService;
 import bubble.service.boot.ActivationService;
 import bubble.service.boot.StandardSelfNodeService;
 import bubble.service.notify.LocalNotificationStrategy;
@@ -90,6 +92,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration
     public static final String TAG_REQUIRE_SEND_METRICS = "requireSendMetrics";
     public static final String TAG_SUPPORT = "support";
     public static final String TAG_SECURITY_LEVELS = "securityLevels";
+    public static final String TAG_RESTORE_MODE = "awaitingRestore";
 
     public static final String DEFAULT_LOCAL_STORAGE_DIR = HOME_DIR + "/.bubble_local_storage";
 
@@ -290,11 +293,12 @@ public class BubbleConfiguration extends PgRestServerConfiguration
     public Map<String, Object> getPublicSystemConfigs () {
         synchronized (publicSystemConfigs) {
             if (publicSystemConfigs.get() == null) {
-                final BubbleNode thisNode = getThisNode();
                 final BubbleNetwork thisNetwork = getThisNetwork();
                 final AccountDAO accountDAO = getBean(AccountDAO.class);
                 final CloudServiceDAO cloudDAO = getBean(CloudServiceDAO.class);
                 final ActivationService activationService = getBean(ActivationService.class);
+                final RestoreService restoreService = getBean(RestoreService.class);
+
                 publicSystemConfigs.set(MapBuilder.build(new Object[][]{
                         {TAG_ALLOW_REGISTRATION, thisNetwork == null ? null : thisNetwork.getBooleanTag(TAG_ALLOW_REGISTRATION, false)},
                         {TAG_NETWORK_UUID, thisNetwork == null ? null : thisNetwork.getUuid()},
@@ -308,6 +312,10 @@ public class BubbleConfiguration extends PgRestServerConfiguration
                         {TAG_CLOUD_CONFIGS, accountDAO.activated() ? null : activationService.getCloudDefaults()},
                         {TAG_LOCKED, accountDAO.locked()},
                         {TAG_LAUNCH_LOCK, isSageLauncher() || thisNetwork == null ? null : thisNetwork.launchLock()},
+                        {TAG_RESTORE_MODE, thisNetwork == null
+                                           ? false
+                                           : thisNetwork.getState() == BubbleNetworkState.restoring
+                                             && !restoreService.isRestoreStarted(thisNetwork.getUuid())},
                         {TAG_SSL_PORT, getDefaultSslPort()},
                         {TAG_SUPPORT, getSupport()},
                         {TAG_SECURITY_LEVELS, DeviceSecurityLevel.values()}

bubble-server/src/main/java/bubble/server/BubbleServer.java

@@ -54,7 +54,8 @@ public class BubbleServer extends RestServerBase<BubbleConfiguration> {
     });
 
     public static final List<RestServerLifecycleListener> RESTORE_LIFECYCLE_LISTENERS = Arrays.asList(new RestServerLifecycleListener[] {
-            new NodeInitializerListener()
+            new NodeInitializerListener(),
+            new BubbleFirstTimeListener()
     });
 
     public static final String[] DEFAULT_ENV_FILE_PATHS = {

bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java

@@ -7,6 +7,7 @@ package bubble.server.listener;
 import bubble.ApiConstants;
 import bubble.dao.account.AccountDAO;
 import bubble.dao.account.AccountPolicyDAO;
+import bubble.dao.cloud.BubbleNetworkDAO;
 import bubble.model.account.Account;
 import bubble.model.account.AccountPolicy;
 import bubble.model.account.message.AccountAction;
@@ -14,8 +15,10 @@ import bubble.model.account.message.AccountMessage;
 import bubble.model.account.message.AccountMessageType;
 import bubble.model.account.message.ActionTarget;
 import bubble.model.cloud.BubbleNetwork;
+import bubble.model.cloud.BubbleNetworkState;
 import bubble.server.BubbleConfiguration;
 import bubble.service.boot.SageHelloService;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.cobbzilla.wizard.server.RestServer;
@@ -27,6 +30,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import static java.util.concurrent.TimeUnit.HOURS;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
 import static org.cobbzilla.util.io.FileUtil.abs;
+import static org.cobbzilla.util.io.FileUtil.toStringOrDie;
 import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Slf4j
@@ -37,6 +41,8 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase<Bub
     public static final long UNLOCK_EXPIRATION = HOURS.toSeconds(48);
     public static final int UNLOCK_KEY_LEN = 6;
 
+    private static final FirstTimeType FIRST_TIME_TYPE_DEFAULT = FirstTimeType.install;
+
     private static AtomicReference<RedisService> redis = new AtomicReference<>();
     public static String getUnlockKey () {
         final RedisService r = redis.get();
@@ -49,15 +55,24 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase<Bub
         redis.set(configuration.getBean(RedisService.class));
 
         final AccountDAO accountDAO = configuration.getBean(AccountDAO.class);
+        final var network = configuration.getThisNetwork();
         if (FIRST_TIME_FILE.exists()) {
             try {
-                // final FirstTimeType firstTimeType = FirstTimeType.fromString(FileUtil.toStringOrDie(FIRST_TIME_FILE));
+                try {
+                    final var firstTimeType = FirstTimeType.fromString(toStringOrDie(FIRST_TIME_FILE));
+                    updateNetworkState(configuration, network, firstTimeType);
+                } catch (Exception e) {
+                    log.warn("Cannot open and/or read/parse first time file " + FIRST_TIME_FILE.getAbsolutePath());
+                    updateNetworkState(configuration, network, FIRST_TIME_TYPE_DEFAULT);
+                }
+
                 final Account adminAccount = accountDAO.getFirstAdmin();
                 if (adminAccount == null) {
                     log.error("onStart: no admin account found, cannot send first time install message, unlocking now");
                     accountDAO.unlock();
                     return;
                 }
+
                 final AccountPolicy adminPolicy = configuration.getBean(AccountPolicyDAO.class).findSingleByAccount(adminAccount.getUuid());
                 if (adminPolicy == null || !adminPolicy.hasVerifiedNonAuthenticatorAccountContacts()) {
                     log.error("onStart: no AccountPolicy found (or no verified non-authenticator contacts) for admin account (" + adminAccount.getEmail() + "), cannot send first time install message, unlocking now");
@@ -65,9 +80,6 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase<Bub
                     return;
                 }
 
-                final BubbleNetwork network = configuration.getThisNetwork();
-                final SageHelloService helloService = configuration.getBean(SageHelloService.class);
-
                 final AccountMessage readyMessage = new AccountMessage()
                         .setAccount(adminAccount.getUuid())
                         .setNetwork(network.getUuid())
@@ -78,13 +90,13 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase<Bub
                 if (!network.launchLock()) {
                     log.info("onStart: thisNetwork.launchLock was false, unlocking now");
                     accountDAO.unlock();
-                    helloService.setUnlockMessage(readyMessage.setData(null));
-                    return;
+                    readyMessage.setData(null);
+                } else {
+                    final String unlockKey = randomAlphabetic(UNLOCK_KEY_LEN).toUpperCase();
+                    redis.get().set(UNLOCK_KEY, unlockKey, EX, UNLOCK_EXPIRATION);
+                    readyMessage.setData(unlockKey);
                 }
-
-                final String unlockKey = randomAlphabetic(UNLOCK_KEY_LEN).toUpperCase();
-                redis.get().set(UNLOCK_KEY, unlockKey, EX, UNLOCK_EXPIRATION);
-                helloService.setUnlockMessage(readyMessage.setData(unlockKey));
+                configuration.getBean(SageHelloService.class).setUnlockMessage(readyMessage);
 
             } finally {
                 if (!FIRST_TIME_FILE.delete()) {
@@ -95,8 +107,18 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase<Bub
             if (!accountDAO.locked()) {
                 log.info("onStart: system is not locked, ensuring all accounts are unlocked");
                 accountDAO.unlock();
+                updateNetworkState(configuration, network, FIRST_TIME_TYPE_DEFAULT);
             }
         }
     }
 
+    private void updateNetworkState(@NonNull final BubbleConfiguration config, @NonNull final BubbleNetwork network,
+                                    @NonNull final FirstTimeType firstTimeType) {
+        if (network.getState() == BubbleNetworkState.starting) {
+            network.setState(firstTimeType == FirstTimeType.restore ? BubbleNetworkState.restoring
+                                                                    : BubbleNetworkState.running);
+            config.getBean(BubbleNetworkDAO.class).update(network);
+        }
+    }
+
 }

bubble-server/src/main/java/bubble/service/backup/RestoreService.java

@@ -35,6 +35,7 @@ import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE;
 import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
+import static org.cobbzilla.wizard.cache.redis.RedisService.LOCK_SUFFIX;
 
 @Service @Slf4j
 public class RestoreService {
@@ -64,6 +65,8 @@ public class RestoreService {
 
     public boolean isValidRestoreKey(String restoreKey) { return getRestoreKeys().exists(restoreKey); }
 
+    public boolean isRestoreStarted(String networkUuid) { return getRestoreKeys().exists(networkUuid + LOCK_SUFFIX); }
+
     public boolean restore(String restoreKey, BubbleBackup backup) {
         final String thisNodeUuid = configuration.getThisNode().getUuid();
         final String thisNetworkUuid = configuration.getThisNode().getNetwork();

bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java

@@ -26,6 +26,7 @@ import bubble.service.bill.BillingService;
 import bubble.service.bill.StandardRefundService;
 import bubble.service.notify.NotificationService;
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.cache.Refreshable;
 import org.cobbzilla.util.http.HttpSchemes;
@@ -184,11 +185,8 @@ public class StandardSelfNodeService implements SelfNodeService {
         synchronized (thisNode) {
             final BubbleNode self = thisNode.get();
             if (self == null) {
-                final BubbleNode initSelf = initThisNode();
-                if (initSelf == null) {
-                    return die("getThisNode: error initializing selfNode, initThisNode returned null (should never happen)");
-                }
-                log.debug("getThisNode: setting thisNode="+initSelf.id());
+                final var initSelf = initThisNode(); // NonNull
+                log.debug("getThisNode: setting thisNode=" + initSelf.id());
                 thisNode.set(initSelf);
                 if (initSelf == NULL_NODE) {
                     if (!nullWarningPrinted.check()) log.warn("getThisNode: initThisNode returned NULL_NODE");
@@ -246,24 +244,24 @@ public class StandardSelfNodeService implements SelfNodeService {
         }
     }
 
-    private BubbleNode initSageNode(BubbleNode selfNode) {
-        BubbleNode sage = nodeDAO.findByUuid(selfNode.getSageNode());
-        if (sage == null) {
-            // do we have a local file we can fall back on?
-            if (!SAGE_NODE_FILE.exists()) {
-                log.warn("initSageNode: DB contains no entry for selfNode.sage ("+selfNode.getSageNode()+") and "+abs(SAGE_NODE_FILE)+ " does not exist, returning null");
-                return NULL_NODE;
-            }
-            sage = syncSage(selfNode, nodeFromFile(SAGE_NODE_FILE));
+    @NonNull private BubbleNode initSageNode(@NonNull final BubbleNode selfNode) {
+        var sage = nodeDAO.findByUuid(selfNode.getSageNode());
+        final var isSageNodeFilePresent = SAGE_NODE_FILE.exists();
+
+        if (sage == null && !isSageNodeFilePresent) {
+            // local file if required here to fall back on
+            log.warn("initSageNode: DB contains no entry for selfNode.sage (" + selfNode.getSageNode() + ") and "
+                     + abs(SAGE_NODE_FILE) + " does not exist, returning special null node object");
+            return NULL_NODE;
         }
-        sage = syncSage(selfNode, SAGE_NODE_FILE.exists()
-                ? nodeFromFile(SAGE_NODE_FILE)
-                : sage);
+
+        sage = syncSage(selfNode, isSageNodeFilePresent ? nodeFromFile(SAGE_NODE_FILE) : sage);
         initSageKey(sage);
+
         return sage;
     }
 
-    private BubbleNode syncSage(BubbleNode selfNode, BubbleNode sage) {
+    @NonNull private BubbleNode syncSage(@NonNull final BubbleNode selfNode, @NonNull final BubbleNode sage) {
         // if the sage has a local ip4, then selfNode is the sage. should only happen if fork was done incorrectly
         if (sage.localIp4()) {
             if (selfNode.localIp4()) return die("syncSage: selfNode is local: "+selfNode.id());
@@ -285,7 +283,7 @@ public class StandardSelfNodeService implements SelfNodeService {
         return nodeDAO.create(sage);
     }
 
-    private BubbleNode initThisNode() {
+    @NonNull private BubbleNode initThisNode() {
         if (!THIS_NODE_FILE.exists()) {
             log.warn("initThisNode: "+abs(THIS_NODE_FILE)+" does not exist, returning null");
             return NULL_NODE;
@@ -296,13 +294,14 @@ public class StandardSelfNodeService implements SelfNodeService {
         return initSelf(selfNode);
     }
 
-    private BubbleNode initSelf(BubbleNode selfNode) {
+    @NonNull private BubbleNode initSelf(@NonNull final BubbleNode selfNode) {
         log.debug("initSelf: starting with selfNode="+selfNode.id());
         final BubbleNode foundByUuid = nodeDAO.findByUuid(selfNode.getUuid());
         final BubbleNode foundByFqdn = nodeDAO.findByFqdn(selfNode.getFqdn());
         final BubbleNode foundByIp4 = nodeDAO.findByIp4(selfNode.getIp4());
         if (foundByUuid == null && foundByFqdn == null && foundByIp4 == null) {
-            // node exists in JSON but not in DB: write it to DB
+            // node exists in JSON but not in DB: write it to DB - also sage node is required to be in DB:
+            if (nodeDAO.findByUuid(selfNode.getSageNode()) == null) initSageNode(selfNode);
             return ensureRunning(nodeDAO.create(selfNode));
 
         } else if (foundByUuid != null && foundByFqdn != null) {
@@ -332,7 +331,7 @@ public class StandardSelfNodeService implements SelfNodeService {
         }
     }
 
-    private BubbleNode ensureRunning(BubbleNode selfNode) {
+    @NonNull private BubbleNode ensureRunning(@NonNull final BubbleNode selfNode) {
         return selfNode.getState() == BubbleNodeState.running
                 ? selfNode
                 : nodeDAO.update(selfNode.setState(BubbleNodeState.running));

bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterTick.java

@@ -14,6 +14,7 @@ import java.util.regex.Pattern;
 
 import static bubble.ApiConstants.enumFromString;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 
 @Accessors(chain=true)
 public class NodeProgressMeterTick {
@@ -23,6 +24,15 @@ public class NodeProgressMeterTick {
         @JsonCreator public static TickMatchType fromString(String v) { return enumFromString(TickMatchType.class, v); }
     }
 
+    public NodeProgressMeterTick() {
+        this.ctime = now();
+    }
+
+    @Setter private Long ctime;
+    // backward compatibility - the following getter may be removed and default one may be used after some time, while
+    // ctime can be changed to be of simple `long` type
+    public long getCtime() { return ctime == null ? 0 : ctime; }
+
     @Getter @Setter private String account;
     public boolean hasAccount() { return !empty(account); }
 

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -49,6 +49,7 @@ import org.cobbzilla.util.io.TempDir;
 import org.cobbzilla.util.system.Command;
 import org.cobbzilla.util.system.CommandResult;
 import org.cobbzilla.util.system.CommandShell;
+import org.cobbzilla.wizard.api.ApiException;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.cobbzilla.wizard.validation.MultiViolationException;
 import org.cobbzilla.wizard.validation.SimpleViolationException;
@@ -362,6 +363,7 @@ public class StandardNetworkService implements NetworkService {
             if (node.getInstallType() == AnsibleInstallType.node) {
                 final long readyStart = now();
                 boolean ready = false;
+                Exception lastEx = null;
                 final String readyUri = nodeBaseUri(node, configuration) + AUTH_ENDPOINT + EP_READY;
                 int i = 1;
                 while (now() - readyStart < NODE_READY_TIMEOUT) {
@@ -380,9 +382,17 @@ public class StandardNetworkService implements NetworkService {
                         }
                     } catch (Exception e) {
                         log.warn("newNode: node (" + node.id() + ") error checking if ready: " + shortError(e));
+                        lastEx = e;
                     }
                 }
                 if (!ready) {
+                    if (lastEx != null) {
+                        var responseStatus = "";
+                        if (lastEx instanceof ApiException) {
+                            responseStatus = " (HTTP status: " + ((ApiException) lastEx).getResponse().status + ")";
+                        }
+                        log.warn("newNode: the last exception in checking if ready" + responseStatus, lastEx);
+                    }
                     return launchFailureCanRetry(node, "newNode: timeout waiting for node (" + node.id() + ") to be ready");
                 }
             }
@@ -446,6 +456,7 @@ public class StandardNetworkService implements NetworkService {
                 }
                 closeQuietly(progressMeter);
             }
+            unlockNetwork(nn.getNetwork(), lock);
             backgroundJobs.shutdownNow();
         }
         return node;
@@ -660,10 +671,10 @@ public class StandardNetworkService implements NetworkService {
             // sanity checks
             final List<BubbleNode> nodes = nodeDAO.findByNetwork(network.getUuid());
             if (!nodes.isEmpty()) {
-                throw invalidEx("err.network.restore.nodesExist");
+                throw invalidEx("err.networkRestore.nodesExist");
             }
             if (network.getState() != BubbleNetworkState.stopped) {
-                throw invalidEx("err.network.restore.notStopped");
+                throw invalidEx("err.networkRestore.notStopped");
             }
             network.setState(BubbleNetworkState.starting);
             networkDAO.update(network);
@@ -688,8 +699,14 @@ public class StandardNetworkService implements NetworkService {
 
             return newNodeRequest;
 
+        } catch (SimpleViolationException e) {
+            // TODO: should this go here, or just in some specific cases within above try block?
+            //       also, should this go into other method here that are locking network?
+            try { unlockNetwork(network.getUuid(), lock); } catch (Exception e1) { }
+            log.error("startNetwork: original SimpleViolationException: ", e);
+            throw e;
         } catch (Exception e) {
-            return die("startNetwork: "+e, e);
+            return die("startNetwork: " + e, e);
         }
     }
 

bubble-server/src/main/java/bubble/service/dbfilter/FilteredEntityIterator.java

@@ -80,7 +80,8 @@ public class FilteredEntityIterator extends EntityIterator {
             if (!AccountOwnedEntityDAO.class.isAssignableFrom(dao.getClass())) {
                 log.debug("iterate: skipping entity: " + c.getSimpleName());
             } else if (isPostCopyEntity(c)) {
-                log.debug("iterate: skipping " + c.getSimpleName() + ", will copy after other objects are copied");
+                log.debug("iterate: skipping " + c.getSimpleName()
+                          + ", will copy some of these after other objects are copied");
             } else {
                 // copy entities. this is how the re-keying works (decrypt using current spring config,
                 // encrypt using new config)

bubble-server/src/main/resources/ansible/roles/algo/tasks/algo_firewall.yml

@@ -2,54 +2,42 @@
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
 # Insert additional firewall rules to allow required services to function
-- name: Allow HTTP
+# Insert them all on rule_num 5, and insert them in reverse order here:
+- name: Allow SSH
   iptables:
     chain: INPUT
-    action: insert
-    rule_num: 5
     protocol: tcp
-    destination_port: 80
+    destination_port: 22
     ctstate: NEW
     syn: match
     jump: ACCEPT
-    comment: Accept new HTTP connections
+    comment: Accept new SSH connections
   become: yes
 
-- name: Allow HTTPS
+- name: "Allow HTTP on port {{ item }}"
   iptables:
     chain: INPUT
-    action: insert
-    rule_num: 6
     protocol: tcp
-    destination_port: 443
+    destination_port: "{{ item }}"
     ctstate: NEW
     syn: match
     jump: ACCEPT
-    comment: Accept new HTTPS connections
+    comment: "Accept new HTTP ({{ item }}) connections"
+  with_items:
+    - 80
+    - 1080
   become: yes
 
-- name: Allow admin HTTPS on port 1443
+- name: "Allow HTTPS on port {{ item }}"
   iptables:
     chain: INPUT
-    action: insert
-    rule_num: 7
     protocol: tcp
-    destination_port: 1443
+    destination_port: "{{ item }}"
     ctstate: NEW
     syn: match
     jump: ACCEPT
-    comment: Accept new admin SSL connections
-  become: yes
-
-- name: Allow admin HTTP on port 1080
-  iptables:
-    chain: INPUT
-    action: insert
-    rule_num: 8
-    protocol: tcp
-    destination_port: 1080
-    ctstate: NEW
-    syn: match
-    jump: ACCEPT
-    comment: Accept new admin SSL connections
+    comment: "Accept new HTTPS ({{ item }}) connections"
+  with_items:
+    - 443
+    - 1443
   become: yes

bubble-server/src/main/resources/ansible/roles/algo/tasks/main.yml

@@ -9,15 +9,14 @@
     group: root
     mode: 0500
 
+- name: Stop algo monitors just in case
+  shell: bash -c "supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections"
+
 # Don't setup algo when in restore mode, bubble_restore_monitor.sh will set it up after the CA key has been restored
 - name: Run algo playbook to install algo
   shell: bash -c "/root/ansible/roles/algo/algo/install_algo.sh 2>&1 >> /tmp/install_algo.log"
-  when: restore_key is not defined
-
-# Don't start monitors when in restore mode, bubble_restore_monitor.sh will start it after algo is installed
-- name: Stop algo monitors (in restore mode)
-  shell: bash -c "supervisorctl stop algo_refresh_users_monitor && supervisorctl stop wg_monitor_connections"
-  when: restore_key is defined
+  tags: algo_related
 
-# Add bubble rules
+# Algo installation clears out iptable rules. Add needed bubble rules back:
 - include: algo_firewall.yml
+  tags: algo_related

bubble-server/src/main/resources/packer/roles/bubble/files/bubble_restore_monitor.sh → bubble-server/src/main/resources/ansible/roles/bubble/files/bubble_restore_monitor.sh

@@ -99,8 +99,13 @@ log "Removing node keys"
 echo "DELETE FROM bubble_node_key" | bsql.sh
 
 # restore local storage
-log "Restoring bubble LocalStorage"
-rm -rf ${BUBBLE_HOME}/.bubble_local_storage/* && rsync -ac ${RESTORE_BASE}/LocalStorage/* ${BUBBLE_HOME}/.bubble_local_storage/ || die "Error restoring LocalStorage"
+LOCAL_STORAGE_DIR="${RESTORE_BASE}/LocalStorage"
+if [[ -d LOCAL_STORAGE_DIR ]] ; then
+  log "Restoring bubble LocalStorage"
+  rm -rf ${BUBBLE_HOME}/.bubble_local_storage/* \
+    && rsync -ac ${LOCAL_STORAGE_DIR}/* ${BUBBLE_HOME}/.bubble_local_storage/ \
+    || die "Error restoring LocalStorage"
+fi
 
 # flush redis
 log "Flushing redis"
@@ -121,11 +126,9 @@ else
   fi
   cd ${ALGO_BASE} && tar xzf ${CONFIGS_BACKUP} || die "Error restoring algo VPN configs"
 
-  cd "${ANSIBLE_DIR}" && \
-    . ./venv/bin/activate && \
-    bash -c \
-      "ansible-playbook --tags 'algo_related,always' --inventory ./hosts ./playbook.yml 2>&1 >> ${LOG}" \
-  || die "Error running ansible in post-restore. journalctl -xe = $(journalctl -xe | tail -n 50)"
+  ANSIBLE_CMD="ansible-playbook --tags 'algo_related,always' --inventory ./hosts ./playbook.yml"
+  bash -c "cd '${ANSIBLE_DIR}' && . ./venv/bin/activate && ${ANSIBLE_CMD} 2>&1 >> ${LOG}" \
+    || die "Error running ansible in post-restore. journalctl -xe = $(journalctl -xe | tail -n 50)"
 fi
 
 # restart mitm proxy service

bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml

@@ -45,3 +45,7 @@
     - sage_key.json
 
 - import_tasks: postgresql_data.yml
+
+- name: Start monitor for restoring this bubble if applicable
+  include: restore.yml
+  when: restore_key is defined

bubble-server/src/main/resources/ansible/roles/bubble/tasks/restore.yml

@@ -10,8 +10,6 @@
     mode: 0550
   with_items:
     - "bubble_restore_monitor.sh"
-  when: restore_key is defined
 
 - name: Start restore monitor
   shell: bash -c 'nohup /usr/local/bin/bubble_restore_monitor.sh {{ admin_port }} {{ restore_timeout }} > /dev/null &'
-  when: restore_key is defined

bubble-server/src/main/resources/ansible/roles/finalizer/tasks/main.yml

@@ -4,12 +4,12 @@
 - name: Snapshot ansible roles in the background
   command: bash -c "/usr/local/bin/snapshot_ansible.sh &"
 
-- name: Touch first-time setup file
-  shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n install > /home/bubble/first_time_marker ; fi"
+- name: Create first-time setup file
+  shell: su - bubble bash -c "echo -n install > /home/bubble/first_time_marker"
   when: restore_key is not defined
 
-- name: Touch first-time setup file (restore)
-  shell: su - bubble bash -c "if [[ ! -f /home/bubble/first_time_marker ]] ; then echo -n restore > /home/bubble/first_time_marker ; fi"
+- name: Create first-time setup file (restore)
+  shell: su - bubble bash -c "echo -n restore > /home/bubble/first_time_marker"
   when: restore_key is defined
 
 - name: Install mitmproxy CA cert in local CA store
@@ -25,6 +25,19 @@
     src: "supervisor_bubble.conf.j2"
     dest: /etc/supervisor/conf.d/bubble.conf
 
+- name: save iptables v4 rules
+  shell: iptables-save > /etc/iptables/rules.v4
+  become: yes
+
+- name: save iptables v6 rules
+  shell: ip6tables-save > /etc/iptables/rules.v6
+  become: yes
+
+- name: Restart iptables
+  service:
+    name: netfilter-persistent
+    state: restarted
+
 # We cannot receive notifications until nginx is running, so start bubble API as the very last step
 - name: reload supervisord
   shell: supervisorctl reload

bubble-server/src/main/resources/ansible/roles/mitmproxy/tasks/main.yml

@@ -31,11 +31,9 @@
     src: supervisor_mitmdump_monitor.conf
     dest: /etc/supervisor/conf.d/mitmdump_monitor.conf
 
-- name: "Allow MITM private port"
+- name: Allow MITM private port
   iptables:
     chain: INPUT
-    action: insert
-    rule_num: 7
     protocol: tcp
     destination_port: 8888
     ctstate: NEW
@@ -43,6 +41,9 @@
     jump: ACCEPT
     comment: Accept new local connections on mitm port
   become: yes
+  tags: algo_related
+  # ensuring that algo did its work on iptables before, so rule num 5 is ok to use
 
 - name: reload supervisord
   shell: supervisorctl reload
+  tags: always

bubble-server/src/main/resources/bubble/host-prefixes.txt

@@ -1418,7 +1418,6 @@ briny
 brios
 brise
 brisk
-briss
 brith
 brits
 britt
@@ -3496,7 +3495,6 @@ fease
 feast
 feats
 feaze
-fecal
 feces
 fecht
 fecit
@@ -3548,7 +3546,6 @@ fesse
 festa
 fests
 festy
-fetal
 fetas
 fetch
 feted
@@ -3557,7 +3554,6 @@ fetid
 fetor
 fetta
 fetts
-fetus
 fetwa
 feuar
 feuds
@@ -4915,7 +4911,6 @@ horde
 horis
 horme
 horns
-horny
 horse
 horst
 horsy
@@ -8060,9 +8055,6 @@ porge
 porgy
 porks
 porky
-porno
-porns
-porny
 porta
 ports
 porty

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -225,6 +225,7 @@ message_plan_node_apps=Your Bubble will include these apps:
 
 # Network Page - Connect
 message_network_connect=Connect to Bubble
+message_network_restore=Connect to Bubble to actually start restoring process
 
 # Network Page - Restore Keys
 link_network_action_request_keys=Request Bubble Restore Key
@@ -236,6 +237,10 @@ field_network_key_download_code=Download Code
 field_network_key_download_password=Encrypt with password
 button_label_retrieve_keys=Download
 err.retrieveNetworkKeys.notFound=Download Code Not Found
+restore_key_label=Your restore short key is:
+button_label_restore=Restore
+button_description_restore=You will need full network restore key build from this bubble while if was running.
+restore_not_possible_nodes_exist_html=Cannot restore bubbles with existing nodes. Please contact <a href="mailto:support@getbubblenow.com">support@getbubblenow.com</a>
 
 # Network Page - Danger Zone
 title_network_danger_zone=Danger Zone
@@ -243,6 +248,9 @@ link_network_action_stop=Stop
 link_network_action_stop_description=Stop this Bubble. If you have downloaded your restore key, you can later restore it.
 network_action_stop_not_ready=Still loading, try again in a moment.
 confirmation_network_action_stop=Please confirm stop.\nConnected devices will lose Internet access until they are disconnected from the Bubble\nYou can restore this Bubble later.
+label_latest_backup=Latest Backup:
+label_no_latest_backup=No backups available
+link_backup_network=Queue new backup
 link_network_action_delete=Delete
 link_network_action_delete_description=Delete this Bubble and all backups. You will not be able to restore this Bubble. This action cannot be undone.
 confirmation_network_action_delete=Please confirm deletion.\nYou will not be able to restore this Bubble.\nThis action cannot be undone.
@@ -712,8 +720,8 @@ err.networkKeys.invalid=Bubble Restore Key was not valid
 err.networkName.required=Network name is required
 err.network.cannotStartInCurrentState=Cannot proceed: network cannot be started in its current state
 err.network.required=Network is required
-err.network.restore.nodesExist=Cannot restore when active nodes exist
-err.network.restore.notStopped=Cannot restore when network is running
+err.networkRestore.nodesExist=Cannot restore when active nodes exist
+err.networkRestore.notStopped=Cannot restore when network is running
 err.nick.alreadyInUse=Nickname is already in use by another contact
 err.nick.tooLong=Nickname cannot be longer than 100 characters
 err.node.notInitialized=Node is not initialized
@@ -868,4 +876,4 @@ err.addFilter.analyticsFilterRequired=Filter pattern is required
 err.nodemanager.error=Error calling nodemanager
 err.nodemanager.noPasswordSet=No nodemanager password is set
 err.nodemanager.invalidPath=Path is invalid
-err.nodemanager.nodeNotLocal=Target node must be this node
+err.nodemanager.nodeNotLocal=Target node must be this node

bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties

@@ -132,6 +132,7 @@ label_menu_apps=Apps
 label_menu_apps_icon=fa fa-smile
 label_menu_notifications=Notifications
 label_menu_notifications_icon=fa fa-flag
+label_menu_network=My Bubble
 label_menu_networks=Bubbles
 label_menu_networks_icon=fa fa-cloud
 label_menu_bills=Bills
@@ -281,6 +282,13 @@ field_label_password_guidance=Password must be at least 8 characters long.<br/>P
 field_label_confirm_password=Confirm Password
 field_label_unlock_key=Unlock Key
 form_title_register=Register
+form_title_restore=Restore
+message_restore_not_applicable=Restore already started or not applicable.
+message_back_to_root=Back to your Bubble
+field_label_restore_short_key=Short Key (6 letters)
+field_label_restore_long_key=Long Network Key
+err.restoreShortKey.required=Short Key is required
+err.restoreLongNetworkKey.required=Long Network Key is required
 field_label_contactType=Contact Type
 field_label_email=Email
 field_label_promoCode=Beta Invite Code
@@ -302,6 +310,7 @@ button_label_login=Login
 button_label_register=Register
 button_label_forgotPassword=Forgot Password
 button_label_cancel=Cancel
+button_label_restore=Restore
 alert_registration_success=Registration successful
 form_title_forgotPassword=Forgot Password
 button_label_resetPassword=Request Password Reset

bubble-server/src/main/resources/packer/roles/firewall/tasks/rules.yml

@@ -29,33 +29,46 @@
   become: yes
   when: fw_enable_ssh
 
-- name: Allow HTTP
+- name: "Allow HTTP on port {{ item }}"
   iptables:
     chain: INPUT
     protocol: tcp
-    destination_port: 80
+    destination_port: "{{ item }}"
     ctstate: NEW
     syn: match
     jump: ACCEPT
-    comment: Accept new HTTP connections
+    comment: "Accept new HTTP ({{ item }}) connections"
+  with_items:
+    - 80
+    - 1080
   become: yes
   when: fw_enable_http
 
-- name: Allow HTTPS
+- name: "Allow HTTPS on port {{ item }}"
   iptables:
     chain: INPUT
     protocol: tcp
-    destination_port: 443
+    destination_port: "{{ item }}"
     ctstate: NEW
     syn: match
     jump: ACCEPT
-    comment: Accept new HTTPS connections
+    comment: "Accept new HTTPS ({{ item }}) connections"
+  with_items:
+    - 443
+    - 1443
   become: yes
   when: fw_enable_http
 
 - name: Drop everything else
   iptables:
     chain: INPUT
-    jump: DROP
-    comment: Drop anything else
+    policy: DROP
+  become: yes
+
+- name: save iptables v4 rules
+  shell: iptables-save > /etc/iptables/rules.v4
+  become: yes
+
+- name: save iptables v6 rules
+  shell: ip6tables-save > /etc/iptables/rules.v6
   become: yes

bubble-server/src/test/resources/models/include/new_bubble.json

@@ -223,5 +223,11 @@
         {"condition": "json.admin() == true"}
       ]
     }
+  },
+
+  {
+    "comment": "network should be in running state by now",
+    "request": { "uri": "me/networks/<<network>>" },
+    "response": { "check": [{ "condition": "json.getState().name() == 'running'" }] }
   }
 ]

bubble-server/src/test/resources/models/tests/live/backup_and_restore.json

@@ -151,18 +151,29 @@
     "response": {
       "store": "restoreNN",
       "check": [
-        {"condition": "restoreNN.getNetwork() == bubbleNetwork.getNetwork()"}
+        { "condition": "restoreNN.getNetwork() == bubbleNetwork.getNetwork()" },
+        { "condition": "restoreNN.getState().name() == 'starting'" }
       ]
     }
   },
 
   {
-    "comment": "restore node using restoreKey",
-    "before": "await_url .bubble 16m:20m 20s",
+    "comment": "wait for network and then try to login - cannot do that as network is in restoring state",
     "connection": {
       "name": "restoredBubbleConnection",
       "baseUri": "https://{{restoreNN.fqdn}}:{{serverConfig.nginxPort}}/api"
     },
+    "before": "await_url .bubble 16m:20m 20s",
+    "request": {
+      "session": "new",
+      "uri": "auth/login",
+      "entity": { "name": "{{username}}", "password": "password1!" }
+    },
+    "response": { "status": "401" }
+  },
+
+  {
+    "comment": "restore node using restoreKey",
     "request": {
       "uri": "auth/restore/{{restoreNN.restoreKey}}",
       "entity": {
@@ -191,6 +202,12 @@
     }
   },
 
+  {
+    "comment": "check again for bubble's status - should be running",
+    "request": { "uri": "me/networks/{{ restoreNN.getNetwork() }}" },
+    "response": { "check": [{ "condition": "json.getState().name() == 'running'" }] }
+  },
+
   {
     "comment": "verify account we added has been restored",
     "request": {

utils/pom.xml

@@ -26,6 +26,7 @@ This code is available under the GNU Affero General Public License, version 3: h
         <module>cobbzilla-wizard</module>
         <module>restex</module>
         <module>templated-mail-sender</module>
+        <module>abp-parser</module>
     </modules>
 
 </project>