WIP. implementing upgrade jar

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -197,6 +197,7 @@ public class ApiConstants {
     public static final String EP_PROMOTIONS = PROMOTIONS_ENDPOINT;
     public static final String EP_FORK = "/fork";
     public static final String EP_NODE_MANAGER = "/nodeman";
+    public static final String EP_UPGRADE = "/upgrade";
 
     public static final String DETECT_ENDPOINT = "/detect";
     public static final String EP_LOCALE = "/locale";

bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java

@@ -55,6 +55,16 @@ public class VultrDriver extends ComputeServiceDriverBase {
     public static final String VULTR_SUBID = "SUBID";
     public static final String VULTR_V4_IP = "main_ip";
     public static final String VULTR_V6_IP = "v6_main_ip";
+    public static final String VULTR_LABEL = "label";
+
+    public static final String VULTR_STATUS = "status";
+    public static final String VULTR_STATUS_PENDING = "pending";
+    public static final String VULTR_STATUS_ACTIVE = "active";
+
+    public static final String VULTR_SERVER_STATE = "server_state";
+    public static final String VULTR_STATE_NONE = "none";
+    public static final String VULTR_STATE_OK = "ok";
+    public static final String VULTR_STATE_LOCKED = "locked";
 
     public static final String CREATE_SERVER_URL = VULTR_API_BASE + "server/create";
     public static final String DESTROY_SERVER_URL = VULTR_API_BASE + "server/destroy";
@@ -144,7 +154,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
 
         node.setState(BubbleNodeState.booting);
         node.setTag(TAG_INSTANCE_ID, subId);
-        nodeDAO.update(node);
+//        nodeDAO.update(node);
 
         final long start = now();
         boolean startedOk = false;
@@ -161,29 +171,29 @@ public class VultrDriver extends ComputeServiceDriverBase {
             if (serverNode != null) {
                 if (serverNode.has("tag")
                         && serverNode.get("tag").textValue().equals(cloud.getUuid())
-                        && serverNode.has("status")
-                        && serverNode.has("server_state")
+                        && serverNode.has(VULTR_STATUS)
+                        && serverNode.has(VULTR_SERVER_STATE)
                         && serverNode.has(VULTR_V4_IP)) {
 
-                    final String status = serverNode.get("status").textValue();
-                    final String serverState = serverNode.get("server_state").textValue();
+                    final String status = serverNode.get(VULTR_STATUS).textValue();
+                    final String serverState = serverNode.get(VULTR_SERVER_STATE).textValue();
                     final String ip4 = serverNode.get(VULTR_V4_IP).textValue();
                     final String ip6 = serverNode.get(VULTR_V6_IP).textValue();
                     // if (log.isInfoEnabled()) log.info("start: server_state="+serverState+", status="+status, "ip4="+ip4+", ip6="+ip6);
 
                     if (ip4 != null && ip4.length() > 0 && !ip4.equals("0.0.0.0")) {
                         node.setIp4(ip4);
-                        nodeDAO.update(node);
+//                        nodeDAO.update(node);
                     }
                     if (ip6 != null && ip6.length() > 0) {
                         node.setIp6(ip6);
-                        nodeDAO.update(node);
+//                        nodeDAO.update(node);
                     }
-                    if (status.equals("active") && (node.hasIp4() || node.hasIp6())) {
+                    if (status.equals(VULTR_STATUS_ACTIVE) && (node.hasIp4() || node.hasIp6())) {
                         node.setState(BubbleNodeState.booted);
-                        nodeDAO.update(node);
+//                        nodeDAO.update(node);
                     }
-                    if (serverState.equals("ok")) {
+                    if (serverState.equals(VULTR_STATE_OK)) {
                         if (log.isInfoEnabled()) log.info("start: server is ready: "+node.id());
                         startedOk = true;
                         break;
@@ -217,6 +227,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
                 return node;
 
             } catch (Exception e) {
+                if (log.isInfoEnabled()) log.info("stop: _stop failed with: "+shortError(e));
                 lastEx = e;
             }
             sleep(SERVER_STOP_CHECK_INTERVAL, "stop: waiting to try stopping again until node is not found");
@@ -228,30 +239,19 @@ public class VultrDriver extends ComputeServiceDriverBase {
     }
 
     public BubbleNode _stop(BubbleNode node) throws IOException {
-        BubbleNode vultrNode;
-        final String ip4 = node.getIp4();
-        if (!node.hasTag(TAG_INSTANCE_ID)) {
-            if (ip4 == null) {
-                throw notFoundEx(node.id());
-            }
-            if (log.isWarnEnabled()) log.warn("stop: no "+TAG_INSTANCE_ID+" tag found on node ("+node.getFqdn()+"/"+ ip4 +"), searching based in ip4...");
-            vultrNode = findByIp4(node, ip4);
-        } else {
-            // does the node still exist?
-            vultrNode = listNode(node);
-            if (vultrNode == null) {
-                vultrNode = findByIp4(node, ip4);
-            }
-        }
+        // does the node still exist?
+        BubbleNode vultrNode = listNode(node);
         if (vultrNode == null) {
             throw notFoundEx(node.id());
         }
 
         final String subId = vultrNode.getTag(TAG_INSTANCE_ID);
         if (subId == null) {
+            if (log.isErrorEnabled()) log.error("_stop: node "+node.id()+" is missing tag "+TAG_INSTANCE_ID+", cannot stop, throwing invalidEx");
             throw invalidEx("err.node.stop.error", "stop: no " + VULTR_SUBID + " on node, returning");
         }
 
+        if (log.isInfoEnabled()) log.info("_stop: calling stopServer("+subId+") for node "+node.id());
         stopServer(subId);
         return node;
     }
@@ -291,14 +291,17 @@ public class VultrDriver extends ComputeServiceDriverBase {
                     final JsonNode subId = jsonNode.get(VULTR_SUBID);
                     final JsonNode ip4 = jsonNode.get(VULTR_V4_IP);
                     final JsonNode ip6 = jsonNode.get(VULTR_V6_IP);
-                    return (subId != null && node.hasTag(TAG_INSTANCE_ID) && subId.textValue().equals(node.getTag(TAG_INSTANCE_ID)))
-                            || (ip4 != null && node.hasIp4() && ip4.textValue().equals(node.getIp4()))
-                            || (ip6 != null && node.hasIp6() && ip6.textValue().equals(node.getIp6())) ? node : null;
+                    if (log.isTraceEnabled()) log.trace("listNode("+node.id()+") found node: "+json(jsonNode, COMPACT_MAPPER));
+                    if (subId != null) node.setTag(TAG_INSTANCE_ID, subId.textValue());
+                    return node.setIp4(ip4 == null ? null : ip4.textValue()).setIp6(ip6 == null ? null : ip6.textValue());
+
                 } catch (Exception e) {
                     if (log.isErrorEnabled()) log.error("listNode: error finding node "+node.id()+", status="+listResponse.getStatus()+": "+listResponse+": exception="+shortError(e));
                     return null;
                 }
-            case NOT_FOUND: return null;
+            case NOT_FOUND:
+                if (log.isDebugEnabled()) log.debug("listNode("+node.id()+") returned 404 Not Found");
+                return null;
             default:
                 if (log.isErrorEnabled()) log.error("listNode: error finding node "+node.id()+", status="+listResponse.getStatus()+": "+listResponse);
                 return null;
@@ -323,7 +326,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
                     final String subid = iter.next();
                     final ObjectNode server = (ObjectNode) entity.get(subid);
                     if (!filter.apply(server)) {
-                        if (log.isDebugEnabled()) log.debug("Skipping node without cloud tag "+cloud.getUuid()+": "+subid);
+                        if (log.isTraceEnabled()) log.trace("Skipping node without cloud tag "+cloud.getUuid()+": "+subid);
                         continue;
                     }
                     final String subId = server.has(VULTR_SUBID) ? server.get(VULTR_SUBID).textValue() : null;
@@ -339,41 +342,53 @@ public class VultrDriver extends ComputeServiceDriverBase {
     }
 
     @Override public BubbleNode status(BubbleNode node) throws Exception {
-        if (node.hasTag(TAG_INSTANCE_ID)) {
-            final BubbleNode found = listNode(node);
-            if (found == null) return node.setState(BubbleNodeState.stopped);
-            return node;
-
-        } else if (node.hasIp4()) {
-            // find by IPv4
-            final HttpRequestBean listServerRequest = auth(new HttpRequestBean(LIST_SERVERS_URL));
-            final HttpResponseBean listResponse = getResponse(listServerRequest);
-            switch (listResponse.getStatus()) {
-                case OK:
-                    final JsonNode entity = json(listResponse.getEntityString(), JsonNode.class);
-                    for (Iterator<String> iter = entity.fieldNames(); iter.hasNext(); ) {
-                        final String subid = iter.next();
-                        final ObjectNode server = (ObjectNode) entity.get(subid);
-                        final String ip4 = server.has(VULTR_V4_IP) ? server.get(VULTR_V4_IP).textValue() : "";
-                        if (ip4.equals(node.getIp4())) {
-                            if (server.has("power_status") && server.get("power_status").textValue().equals("running")
-                                    && server.has("server_state") && server.get("server_state").textValue().equals("ok")) {
-                                final String ip6 = server.has(VULTR_V6_IP) ? server.get(VULTR_V6_IP).textValue() : null;
-                                return node.setIp4(ip4).setIp6(ip6).setState(BubbleNodeState.running);
+        // find by label
+        final HttpRequestBean listServerRequest = auth(new HttpRequestBean(LIST_SERVERS_URL+"?"+VULTR_LABEL+"="+node.getFqdn()));
+        final HttpResponseBean listResponse = getResponse(listServerRequest);
+        switch (listResponse.getStatus()) {
+            case OK:
+                final JsonNode entity = json(listResponse.getEntityString(), JsonNode.class);
+                for (Iterator<String> iter = entity.fieldNames(); iter.hasNext(); ) {
+                    final String subid = iter.next();
+                    final ObjectNode server = (ObjectNode) entity.get(subid);
+                    final String label = server.has(VULTR_LABEL) ? server.get(VULTR_LABEL).textValue() : "";
+                    if (label.equals(node.getFqdn())) {
+                        if (log.isDebugEnabled()) log.debug("status("+node.id()+"): found json: "+json(server, COMPACT_MAPPER));
+                        if (server.has(VULTR_SERVER_STATE) && server.has(VULTR_STATUS)) {
+                            final String status = server.get(VULTR_STATUS).textValue();
+                            final String serverState = server.get(VULTR_SERVER_STATE).textValue();
+                            final String ip4 = server.has(VULTR_V4_IP) ? server.get(VULTR_V4_IP).textValue() : null;
+                            final String ip6 = server.has(VULTR_V6_IP) ? server.get(VULTR_V6_IP).textValue() : null;
+                            node.setIp4(ip4).setIp6(ip6);
+                            if (status.equals(VULTR_STATUS_PENDING) || serverState.equals(VULTR_STATE_NONE)) {
+                                if (log.isDebugEnabled()) log.debug("status("+node.id()+"): pending/none: returning node status==starting");
+                                return node.setState(BubbleNodeState.starting);
+                            }
+                            if (status.equals(VULTR_STATUS_ACTIVE)) {
+                                if (serverState.equals(VULTR_STATE_OK)) {
+                                    if (log.isDebugEnabled()) log.debug("status(" + node.id() + "): active/ok: returning node status==running");
+                                    return node.setState(BubbleNodeState.running);
+
+                                } else if (serverState.equals(VULTR_STATE_LOCKED)) {
+                                    if (log.isDebugEnabled()) log.debug("status(" + node.id() + "): active/locked: returning node status==starting");
+                                    return node.setState(BubbleNodeState.starting);
+                                }
                             }
+                            if (log.isDebugEnabled()) log.debug("status("+node.id()+"): status/state = "+status+"/"+serverState+": returning node status==unknown_error");
+                            return node.setState(BubbleNodeState.unknown_error);
                         }
                     }
-                case NOT_FOUND: case PRECONDITION_FAILED:
-                    log.error("status: error response from API, returning unknown");
-                    return node.setState(BubbleNodeState.unknown_error);
+                }
+                log.error("status: error finding node "+node.id()+", status="+listResponse.getStatus()+": "+listResponse);
+                return node.setState(BubbleNodeState.unknown_error);
 
-                default:
-                    log.error("status: error finding node "+node.id()+", status="+listResponse.getStatus()+": "+listResponse);
-                    return node.setState(BubbleNodeState.unknown_error);
-            }
-        } else {
-            // Node has no IP4
-            return node.setState(BubbleNodeState.unknown_error);
+            case NOT_FOUND: case PRECONDITION_FAILED:
+                log.error("status: error response from API, returning unknown");
+                return node.setState(BubbleNodeState.unknown_error);
+
+            default:
+                log.error("status: error finding node "+node.id()+", status="+listResponse.getStatus()+": "+listResponse);
+                return node.setState(BubbleNodeState.unknown_error);
         }
     }
 

bubble-server/src/main/java/bubble/model/cloud/BubbleNetwork.java

@@ -34,7 +34,8 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 
-import static bubble.ApiConstants.*;
+import static bubble.ApiConstants.EP_NETWORKS;
+import static bubble.ApiConstants.ROOT_NETWORK_UUID;
 import static bubble.model.cloud.BubbleDomain.DOMAIN_NAME_MAXLEN;
 import static bubble.model.cloud.BubbleNetworkState.created;
 import static bubble.server.BubbleConfiguration.getDEFAULT_LOCALE;
@@ -52,7 +53,7 @@ import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENC_PAD;
         @ECTypeChild(type=BubbleNode.class, backref="network")
 })
 @Entity @NoArgsConstructor @Accessors(chain=true)
-@Slf4j @ToString(of={"name", "domainName", "installType"})
+@Slf4j @ToString(of={"name", "domainName", "installType", "state"})
 @ECIndexes({
         @ECIndex(unique=true, of={"account", "name"}),
         @ECIndex(unique=true, of={"name", "domainName"})

bubble-server/src/main/java/bubble/model/cloud/BubbleNetworkState.java

@@ -16,6 +16,6 @@ public enum BubbleNetworkState {
 
     public boolean canStart() { return this == created || this == stopped; }
 
-    public boolean canStop() { return this != stopping && this != stopped && this != error_stopping; }
+    public boolean canStop() { return this != stopped && this != error_stopping; }
 
 }

bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java

@@ -254,4 +254,8 @@ public class BubbleNode extends IdentifiableBase implements HasNetwork, HasBubbl
         }
         if (!hasIp4() || !hasIp6()) throw new TimeoutException("waitForIpAddresses: timeout");
     }
+
+    @Transient @Getter @Setter private BubbleVersionInfo sageVersion;
+    public boolean hasSageVersion () { return sageVersion != null && sageVersion.valid(); }
+
 }

bubble-server/src/main/java/bubble/model/cloud/BubbleVersionInfo.java

@@ -0,0 +1,17 @@
+package bubble.model.cloud;
+
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+
+@Accessors(chain=true)
+public class BubbleVersionInfo {
+
+    @Getter @Setter private String version;
+    @Getter @Setter private String sha256;
+
+    public boolean valid() { return !empty(version) && !empty(sha256); }
+
+}

bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java

@@ -37,6 +37,10 @@ public enum NotificationType {
     health_check, hello_to_sage, hello_from_sage, peer_hello, sync_password,
     register_backup, retrieve_backup, backup_response, restore_complete, fork,
 
+    // upgrade notifications
+    upgrade_request (String.class),
+    upgrade_response (true),
+
     // driver-level notifications
 
     // delegated dns driver notifications

bubble-server/src/main/java/bubble/notify/NotificationHandler_hello_to_sage.java

@@ -32,7 +32,7 @@ public class NotificationHandler_hello_to_sage extends ReceivedNotificationHandl
             log.info("hello_to_sage: returning peers: "+peers.stream().map(BubbleNode::getFqdn).collect(joining(", ")));
             node.setPeers(peers);
 
-            notificationService.notify(node, hello_from_sage, node);
+            notificationService.notify(node, hello_from_sage, node.setSageVersion(configuration.getVersionInfo()));
         }
     }
 }

bubble-server/src/main/java/bubble/notify/compute/NotificationHandler_compute_driver_stop.java

@@ -19,7 +19,8 @@ public class NotificationHandler_compute_driver_stop extends NotificationHandler
     @Override protected BubbleNode handle(ReceivedNotification n,
                                           ComputeDriverNotification notification,
                                           ComputeServiceDriver compute) throws Exception {
-        return nodeService.stopNode(compute, notification.getNode());
+        nodeService.stopNode(compute, notification.getNode());
+        return notification.getNode();
     }
 
 }

bubble-server/src/main/java/bubble/notify/upgrade/JarUpgradeNotification.java

@@ -0,0 +1,18 @@
+package bubble.notify.upgrade;
+
+import bubble.model.cloud.BubbleVersionInfo;
+import bubble.notify.SynchronousNotification;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@NoArgsConstructor @AllArgsConstructor @Accessors(chain=true)
+public class JarUpgradeNotification extends SynchronousNotification {
+
+    @Getter @Setter private BubbleVersionInfo versionInfo;
+
+    @Override protected String getCacheKey() { return versionInfo.getSha256(); }
+
+}

bubble-server/src/main/java/bubble/notify/upgrade/NotificationHandler_upgrade_request.java

@@ -0,0 +1,28 @@
+package bubble.notify.upgrade;
+
+import bubble.dao.cloud.BubbleNodeDAO;
+import bubble.model.cloud.BubbleNode;
+import bubble.model.cloud.notify.ReceivedNotification;
+import bubble.notify.DelegatedNotificationHandlerBase;
+import bubble.service.boot.BubbleJarUpgradeService;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import static bubble.model.cloud.notify.NotificationType.upgrade_response;
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+
+public class NotificationHandler_upgrade_request extends DelegatedNotificationHandlerBase {
+
+    @Autowired private BubbleNodeDAO nodeDAO;
+    @Autowired private BubbleJarUpgradeService upgradeService;
+
+    @Override public void handleNotification(ReceivedNotification n) {
+        final BubbleNode sender = nodeDAO.findByUuid(n.getFromNode());
+        if (sender == null) {
+            die("sender not found: "+n.getFromNode());
+        } else {
+            final String key = upgradeService.registerNodeUpgrade(sender.getUuid());
+            notifySender(upgrade_response, n.getNotificationId(), sender, key);
+        }
+    }
+
+}

bubble-server/src/main/java/bubble/resources/account/AuthResource.java

@@ -30,6 +30,7 @@ import bubble.service.account.StandardAuthenticatorService;
 import bubble.service.backup.RestoreService;
 import bubble.service.bill.PromotionService;
 import bubble.service.boot.ActivationService;
+import bubble.service.boot.BubbleJarUpgradeService;
 import bubble.service.boot.NodeManagerService;
 import bubble.service.boot.SageHelloService;
 import bubble.service.cloud.DeviceIdService;
@@ -675,4 +676,32 @@ public class AuthResource {
         return send(new FileSendableResource(patch));
     }
 
+    @Autowired private BubbleJarUpgradeService upgradeService;
+
+    @GET @Path(EP_UPGRADE+"/{key}")
+    @Produces(APPLICATION_OCTET_STREAM)
+    public Response getUpgrade(@Context Request req,
+                               @Context ContainerRequest ctx,
+                               @PathParam("key") String key) {
+        final String nodeUuid = upgradeService.getNodeForKey(key);
+        if (nodeUuid == null) {
+            log.warn("getUpgrade: key not found: "+key);
+            return unauthorized();
+        }
+
+        final BubbleNode node = nodeDAO.findByUuid(nodeUuid);
+        if (node == null) {
+            log.warn("getUpgrade: node not found: "+nodeUuid);
+            return unauthorized();
+        }
+
+        final String remoteAddr = req.getRemoteAddr();
+        if (!node.hasSameIp(remoteAddr)) {
+            log.warn("getUpgrade: node has wrong IP (request came from "+remoteAddr+"): "+node.id());
+            return unauthorized();
+        }
+
+        return send(new FileSendableResource(configuration.getBubbleJar()));
+    }
+
 }

bubble-server/src/main/java/bubble/resources/account/MeResource.java

@@ -26,6 +26,7 @@ import bubble.server.BubbleConfiguration;
 import bubble.service.account.StandardAccountMessageService;
 import bubble.service.account.StandardAuthenticatorService;
 import bubble.service.account.download.AccountDownloadService;
+import bubble.service.boot.BubbleJarUpgradeService;
 import bubble.service.boot.BubbleModelSetupService;
 import bubble.service.cloud.NodeLaunchMonitor;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -65,8 +66,7 @@ import java.util.Map;
 import static bubble.ApiConstants.*;
 import static bubble.model.account.Account.validatePassword;
 import static bubble.resources.account.AuthResource.forgotPasswordMessage;
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.daemon.ZillaRuntime.errorString;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.http.HttpContentTypes.*;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.wizard.resources.ResourceUtil.*;
@@ -406,4 +406,13 @@ public class MeResource {
         return ok(modelSetupService.setupModel(api, caller, modelFile));
     }
 
+    @Autowired private BubbleJarUpgradeService jarUpgradeService;
+
+    @POST @Path(EP_UPGRADE)
+    public Response uploadModel(@Context Request req,
+                                @Context ContainerRequest ctx) {
+        background(() -> jarUpgradeService.upgrade());
+        return ok(configuration.getPublicSystemConfigs());
+    }
+
 }

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -11,10 +11,7 @@ import bubble.client.BubbleApiClient;
 import bubble.cloud.CloudServiceDriver;
 import bubble.dao.account.AccountDAO;
 import bubble.dao.cloud.CloudServiceDAO;
-import bubble.model.cloud.AnsibleInstallType;
-import bubble.model.cloud.BubbleNetwork;
-import bubble.model.cloud.BubbleNetworkState;
-import bubble.model.cloud.BubbleNode;
+import bubble.model.cloud.*;
 import bubble.model.device.DeviceSecurityLevel;
 import bubble.server.listener.BubbleFirstTimeListener;
 import bubble.service.backup.RestoreService;
@@ -68,6 +65,7 @@ import static org.cobbzilla.util.io.FileUtil.abs;
 import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 import static org.cobbzilla.util.security.ShaUtil.sha256_file;
+import static org.cobbzilla.wizard.model.SemanticVersion.isNewerVersion;
 
 @Configuration @NoArgsConstructor @Slf4j
 public class BubbleConfiguration extends PgRestServerConfiguration
@@ -93,6 +91,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration
     public static final String TAG_SUPPORT = "support";
     public static final String TAG_SECURITY_LEVELS = "securityLevels";
     public static final String TAG_RESTORE_MODE = "awaitingRestore";
+    public static final String TAG_JAR_VERSION = "jarVersion";
+    public static final String TAG_JAR_UPGRADE_AVAILABLE = "jarUpgradeAvailable";
 
     public static final String DEFAULT_LOCAL_STORAGE_DIR = HOME_DIR + "/.bubble_local_storage";
 
@@ -243,6 +243,24 @@ public class BubbleConfiguration extends PgRestServerConfiguration
         }
         return properties.getProperty(META_PROP_BUBBLE_VERSION);
     }
+    @Getter(lazy=true) private final BubbleVersionInfo versionInfo = initBubbleVersionInfo();
+    private BubbleVersionInfo initBubbleVersionInfo() {
+        return new BubbleVersionInfo()
+                .setVersion(getVersion())
+                .setSha256(getJarSha());
+    }
+
+    @Getter private BubbleVersionInfo sageVersionInfo;
+    public void setSageVersionInfo(BubbleVersionInfo version) {
+        sageVersionInfo = version;
+        final boolean isNewer = isNewerVersion(getVersionInfo().getVersion(), sageVersionInfo.getVersion());
+        if (!jarUpgradeAvailable && isNewer) {
+            jarUpgradeAvailable = true;
+            refreshPublicSystemConfigs();
+        }
+    }
+    public boolean hasSageVersionInfo () { return sageVersionInfo != null; }
+    @Getter private Boolean jarUpgradeAvailable = false;
 
     @JsonIgnore public String getUnlockKey () { return BubbleFirstTimeListener.getUnlockKey(); }
 
@@ -318,7 +336,9 @@ public class BubbleConfiguration extends PgRestServerConfiguration
                                              && !restoreService.isRestoreStarted(thisNetwork.getUuid())},
                         {TAG_SSL_PORT, getDefaultSslPort()},
                         {TAG_SUPPORT, getSupport()},
-                        {TAG_SECURITY_LEVELS, DeviceSecurityLevel.values()}
+                        {TAG_SECURITY_LEVELS, DeviceSecurityLevel.values()},
+                        {TAG_JAR_VERSION, getVersion()},
+                        {TAG_JAR_UPGRADE_AVAILABLE, getJarUpgradeAvailable() ? getSageVersionInfo() : null}
                 }));
             }
             return publicSystemConfigs.get();

bubble-server/src/main/java/bubble/service/boot/BubbleJarUpgradeService.java

@@ -0,0 +1,93 @@
+package bubble.service.boot;
+
+import bubble.dao.cloud.BubbleBackupDAO;
+import bubble.model.cloud.BackupStatus;
+import bubble.model.cloud.BubbleBackup;
+import bubble.model.cloud.BubbleVersionInfo;
+import bubble.notify.upgrade.JarUpgradeNotification;
+import bubble.server.BubbleConfiguration;
+import bubble.service.backup.BackupService;
+import bubble.service.notify.NotificationService;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.http.HttpRequestBean;
+import org.cobbzilla.wizard.cache.redis.RedisService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+
+import static bubble.ApiConstants.AUTH_ENDPOINT;
+import static bubble.ApiConstants.EP_UPGRADE;
+import static bubble.client.BubbleNodeClient.nodeBaseUri;
+import static bubble.model.cloud.notify.NotificationType.upgrade_request;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
+import static org.cobbzilla.util.http.HttpMethods.GET;
+import static org.cobbzilla.util.io.FileUtil.*;
+import static org.cobbzilla.util.system.Sleep.sleep;
+import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_YYYY_MM_DD_HH_mm_ss_SSS;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
+
+@Service @Slf4j
+public class BubbleJarUpgradeService {
+
+    private static final long PRE_UPGRADE_BACKUP_TIMEOUT = MINUTES.toMillis(20);
+
+    @Autowired private BubbleConfiguration configuration;
+    @Autowired private BackupService backupService;
+    @Autowired private BubbleBackupDAO backupDAO;
+    @Autowired private NotificationService notificationService;
+    @Autowired private RedisService redis;
+
+    @Getter(lazy=true) private final RedisService nodeUpgradeRequests = redis.prefixNamespace(getClass().getName());
+
+    public String registerNodeUpgrade(String nodeUuid) {
+        final String key = randomAlphanumeric(10) + "." + now();
+        getNodeUpgradeRequests().set(key, nodeUuid, EX, MINUTES.toMillis(1));
+        return key;
+    }
+
+    public String getNodeForKey(String key) { return getNodeUpgradeRequests().get(key); }
+
+    public void upgrade() {
+        if (!configuration.getJarUpgradeAvailable()) {
+            log.warn("upgrade: No upgrade available, returning");
+            return;
+        }
+        final String currentVersion = configuration.getVersion();
+        final BubbleVersionInfo sageVersion = configuration.getSageVersionInfo();
+        final String newVersion = sageVersion.getVersion();
+        BubbleBackup bubbleBackup = backupService.queueBackup("before_upgrade_" + currentVersion + "_to_" + newVersion + "_on_" + DATE_FORMAT_YYYY_MM_DD_HH_mm_ss_SSS.print(now()));
+
+        // monitor backup, ensure it completes
+        final long start = now();
+        while (bubbleBackup.getStatus() != BackupStatus.backup_completed && now() - start < PRE_UPGRADE_BACKUP_TIMEOUT) {
+            sleep(SECONDS.toMillis(5), "waiting for backup to complete before upgrading");
+            bubbleBackup = backupDAO.findByUuid(bubbleBackup.getUuid());
+        }
+        if (bubbleBackup.getStatus() != BackupStatus.backup_completed) {
+            log.warn("upgrade: timeout waiting for backup to complete, status="+bubbleBackup.getStatus());
+            return;
+        }
+
+        final File upgradeJar = new File(configuration.getBubbleJar().getParentFile(), ".upgrade.jar");
+        if (upgradeJar.exists()) {
+            log.error("upgrade: jar already exists, not upgrading: "+abs(upgradeJar));
+            return;
+        }
+
+        // ask the sage to allow us to download the upgrade
+        final String key = notificationService.notifySync(configuration.getSageNode(), upgrade_request, new JarUpgradeNotification(sageVersion));
+
+        // request the jar from the sage
+        final String uri = nodeBaseUri(configuration.getSageNode(), configuration) + AUTH_ENDPOINT + EP_UPGRADE + "/" + key;
+        final HttpRequestBean requestBean = new HttpRequestBean(GET, uri);
+        final File newJar = temp(".jar");
+
+        // move to upgrade location
+        renameOrDie(newJar, upgradeJar);
+    }
+}

bubble-server/src/main/java/bubble/service/cloud/NodeLaunchMonitor.java

@@ -28,8 +28,7 @@ import java.util.concurrent.ConcurrentHashMap;
 
 import static bubble.service.cloud.NodeProgressMeter.getProgressMeterKey;
 import static bubble.service.cloud.NodeProgressMeter.getProgressMeterPrefix;
-import static java.util.concurrent.TimeUnit.HOURS;
-import static java.util.concurrent.TimeUnit.SECONDS;
+import static java.util.concurrent.TimeUnit.*;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.json.JsonUtil.json;
 
@@ -37,7 +36,7 @@ import static org.cobbzilla.util.json.JsonUtil.json;
 public class NodeLaunchMonitor extends SimpleDaemon {
 
     private static final long LAUNCH_ACTIVITY_TIMEOUT = SECONDS.toMillis(180);
-    private static final long LAUNCH_TERMINATE_TIMEOUT = SECONDS.toMillis(5);
+    private static final long LAUNCH_TERMINATE_TIMEOUT = MINUTES.toMillis(6);
 
     @Getter private final long sleepTime = SECONDS.toMillis(15);
 
@@ -50,7 +49,6 @@ public class NodeLaunchMonitor extends SimpleDaemon {
     @Getter(lazy=true) private final RedisService networkSetupStatus = redis.prefixNamespace(getClass().getSimpleName()+"_status_");
 
     private final Map<String, LauncherEntry> launcherThreads = new ConcurrentHashMap<>();
-    private final Map<String, String> canceledNetworks = new ExpirationMap<>(50, HOURS.toMillis(2));
 
     public void register(String nnUuid, String networkUuid, Thread t) {
         startIfNotRunning();
@@ -63,7 +61,6 @@ public class NodeLaunchMonitor extends SimpleDaemon {
     }
 
     public void cancel(String networkUuid) {
-        canceledNetworks.put(networkUuid, networkUuid);
         final LauncherEntry previousLaunch = launcherThreads.get(networkUuid);
         if (previousLaunch == null || !previousLaunch.isAlive()) {
             log.warn("cancel("+networkUuid+"): entry does not thread exist, or is not alive: "+previousLaunch);

bubble-server/src/main/java/bubble/service/cloud/NodeLauncher.java

@@ -32,12 +32,13 @@ public class NodeLauncher implements Runnable {
         try {
             for (int i=0; i<LAUNCH_MAX_RETRIES; i++) {
                 if (i > 0 && !launchMonitor.isRegistered(networkUuid)) {
-                    throw new IllegalStateException("NodeLauncher.run: no longer registered: "+networkUuid);
+                    log.warn("NodeLauncher.run: no longer registered: "+networkUuid);
+                    return;
                 }
                 if (!lock.get().equals(newNodeRequest.getLock())) {
                     die("NodeLauncher.run: existingLock (" + lock.get() + ") is different than lock in NewNodeNotification: " + newNodeRequest.getLock());
                 }
-                if (!networkService.confirmLock(networkUuid, lock.get())) {
+                if (!networkService.confirmNetLock(networkUuid, lock.get())) {
                     die("NodeLauncher.run: error confirming lock (" + lock.get() + ") for network: " + networkUuid);
                 }
 
@@ -52,12 +53,11 @@ public class NodeLauncher implements Runnable {
                 launchThread.start();
                 do {
                     launchThread.join(SECONDS.toMillis(5));
-                    log.info("NodeLauncher.run: still waiting for thread join..."+newNodeRequest.getFqdn());
+                    if (log.isTraceEnabled()) log.trace("NodeLauncher.run: still waiting for thread join: "+newNodeRequest.getFqdn()+" stack="+stacktrace(launchThread));
                 } while (launchThread.isAlive() && !launchThread.isInterrupted());
 
                 if (launchThread.isInterrupted()) {
                     log.warn("NodeLauncher.run: launch interrupted while waiting for join, exiting early");
-                    if (launchThread.isAlive()) terminate(launchThread, SECONDS.toMillis(1));
                     return;
                 }
 
@@ -81,7 +81,7 @@ public class NodeLauncher implements Runnable {
                                 die("NodeLauncher.run: unknown launch exception (type="+launchException.getType()+"): "+shortError(launchException));
                         }
                     } else {
-                        die("NodeLauncher.run: fatal launch exception: " + shortError(exception));
+                        die("NodeLauncher.run: fatal launch exception: " + shortError(exception), exception);
                     }
                 }
                 if (node != null && node.isRunning()) {

bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java

@@ -59,7 +59,7 @@ public class NodeProgressMeter extends PipedOutputStream implements Runnable {
     public void touch() {
         if (now() > lastTouch + MAX_TOUCH_INTERVAL) {
             launchMonitor.touch(nn.getNetwork());
-            networkService.confirmLock(nn.getNetwork(), nn.getLock());
+            networkService.confirmNetLock(nn.getNetwork(), nn.getLock());
             lastTouch = now();
         }
     }
@@ -99,6 +99,10 @@ public class NodeProgressMeter extends PipedOutputStream implements Runnable {
 
     public void write(String line) throws IOException {
         touch();
+        if (closed.get()) {
+            log.warn("write("+line+"): stream closed, not writing");
+            return;
+        }
         writer.write(line.endsWith("\n") ? line : line+"\n");
         writer.flush();
     }

bubble-server/src/main/java/bubble/service/cloud/NodeService.java

@@ -6,10 +6,10 @@ package bubble.service.cloud;
 
 import bubble.cloud.compute.ComputeServiceDriver;
 import bubble.dao.cloud.BubbleDomainDAO;
-import bubble.dao.cloud.BubbleNetworkDAO;
-import bubble.dao.cloud.BubbleNodeDAO;
 import bubble.dao.cloud.CloudServiceDAO;
-import bubble.model.cloud.*;
+import bubble.model.cloud.BubbleDomain;
+import bubble.model.cloud.BubbleNode;
+import bubble.model.cloud.CloudService;
 import bubble.server.BubbleConfiguration;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.validation.SimpleViolationException;
@@ -18,58 +18,36 @@ import org.springframework.stereotype.Service;
 
 import javax.persistence.EntityNotFoundException;
 
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 
 @Service @Slf4j
 public class NodeService {
 
-    @Autowired private BubbleNodeDAO nodeDAO;
-    @Autowired private BubbleNetworkDAO networkDAO;
     @Autowired private BubbleDomainDAO domainDAO;
     @Autowired private CloudServiceDAO cloudDAO;
     @Autowired private BubbleConfiguration configuration;
 
-    public BubbleNode stopNode(ComputeServiceDriver compute, BubbleNode node) {
+    public void stopNode(ComputeServiceDriver compute, BubbleNode node) {
+        log.info("stopNode: starting for node "+node.id());
 
         final BubbleDomain domain = domainDAO.findByUuid(node.getDomain());
         final CloudService dns = cloudDAO.findByUuid(domain.getPublicDns());
-
-        node.setState(BubbleNodeState.stopping);
-        if (node.hasUuid()) nodeDAO.update(node);
         try {
+            log.debug("stopNode: deleting dns entries for node: "+node.id());
             dns.getDnsDriver(configuration).deleteNode(node);
+            log.debug("stopNode: stopping instance for node: "+node.id());
             node = compute.stop(node);
-            return safeUpdateNodeState(node, BubbleNodeState.stopped);
+            log.debug("stopNode: node stopped: "+node.id());
 
         } catch (EntityNotFoundException e) {
-            log.info("stopNode: node not found by compute service: "+node.id()+": "+e);
-            return safeUpdateNodeState(node, BubbleNodeState.unreachable);
+            log.warn("stopNode: node not found by compute service: "+node.id()+": "+e);
 
         } catch (SimpleViolationException e) {
-            log.info("stopNode: error stopping "+node.id()+": "+e);
+            log.warn("stopNode: error stopping "+node.id()+": "+e);
             throw e;
 
         } catch (Exception e) {
-            log.info("stopNode: error stopping "+node.id());
-            return die("stopNode: "+e, e);
-        }
-    }
-
-    public BubbleNode safeUpdateNodeState(BubbleNode node, BubbleNodeState newState) {
-        // ensure node still exists
-        final BubbleNode existingNode = nodeDAO.findByUuid(node.getUuid());
-        if (existingNode == null) {
-            log.warn("stopNode: node not found, not updating: " + node.id());
-            return node;
-        } else {
-            // ensure network still exists
-            final BubbleNetwork network = networkDAO.findByUuid(node.getNetwork());
-            if (network == null) {
-                log.warn("stopNode: node exists (" + node.id() + ") but network (" + node.getNetwork() + ") does not, deleting node");
-                nodeDAO.delete(node.getUuid());
-                return node;
-            }
-            return nodeDAO.update(node.setState(newState));
+            log.warn("stopNode: error stopping "+node.id()+": "+shortError(e));
         }
     }
 

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -136,7 +136,8 @@ public class StandardNetworkService implements NetworkService {
     @Autowired private NodeLaunchMonitor launchMonitor;
 
     @Autowired private RedisService redisService;
-    @Getter(lazy=true) private final RedisService networkLocks = redisService.prefixNamespace(getClass().getSimpleName()+"_lock_");
+    @Getter(lazy=true) private final RedisService networkLocks = redisService.prefixNamespace(getClass().getSimpleName()+"_net_lock_");
+    @Getter(lazy=true) private final RedisService nodeKillLocks = redisService.prefixNamespace(getClass().getSimpleName()+"_node_kill_lock_");
 
     @NonNull public BubbleNode newNode(@NonNull final NewNodeNotification nn,
                                        NodeLaunchMonitor launchMonitor) {
@@ -153,7 +154,7 @@ public class StandardNetworkService implements NetworkService {
             progressMeter = launchMonitor.getProgressMeter(nn);
             progressMeter.write(METER_TICK_CONFIRMING_NETWORK_LOCK);
 
-            if (!confirmLock(nn.getNetwork(), lock)) {
+            if (!confirmNetLock(nn.getNetwork(), lock)) {
                 progressMeter.error(METER_ERROR_CONFIRMING_NETWORK_LOCK);
                 return launchFailureCanRetry("newNode: Error confirming network lock");
             }
@@ -246,7 +247,7 @@ public class StandardNetworkService implements NetworkService {
             final List<Future<?>> jobFutures = new ArrayList<>();
 
             // Start the cloud compute instance
-            final NodeStartJob startJob = new NodeStartJob(node, nodeDAO, computeDriver);
+            final NodeStartJob startJob = new NodeStartJob(node, computeDriver);
             jobFutures.add(backgroundJobs.submit(startJob));
 
             // Create DNS records for node
@@ -436,10 +437,6 @@ public class StandardNetworkService implements NetworkService {
                 nodeDAO.update(node);
                 if (!progressMeter.hasError()) progressMeter.error(METER_UNKNOWN_ERROR);
                 killNode(node, "error: node not running: "+node.id()+": "+node.getState());
-                if (noNodesActive(network)) {
-                    // if no nodes are running, then the network is stopped
-                    networkDAO.update(network.setState(BubbleNetworkState.stopped));
-                }
             }
 
             if (progressMeter != null) {
@@ -498,25 +495,30 @@ public class StandardNetworkService implements NetworkService {
     }
 
     public BubbleNode killNode(BubbleNode node, String message) {
-        if (node == null) return die("(but node was null?): "+message);
-        node.setState(BubbleNodeState.error_stopping);
-        node.setTag(TAG_ERROR, message);
-        if (node.hasUuid()) nodeDAO.update(node);
+        if (node == null) return die("killNode: node was null (message=" + message + ")");
+        String lock = null;
         try {
-            stopNode(node);  // kill it
-        } catch (Exception e) {
-            log.warn("killNode("+node.id()+"): error stopping: "+e);
-        }
-        node.setState(BubbleNodeState.error_stopped);
-        if (node.hasUuid()) nodeDAO.update(node);
+            lock = lockNode(node.getUuid());
+            if (nodeDAO.findByUuid(node.getUuid()) == null) {
+                log.warn("killNode: node already deleted");
+                return node;
+            }
 
-        final BubbleNetwork network = networkDAO.findByUuid(node.getNetwork());
-        if (noNodesActive(network)) {
-            // if no nodes are running, then the network is stopped
-            networkDAO.update(network.setState(BubbleNetworkState.stopped));
-        }
+            node.setState(BubbleNodeState.error_stopping);
+            node.setTag(TAG_ERROR, message);
+            if (node.hasUuid()) nodeDAO.update(node);
+            try {
+                stopNode(node);  // kill it
+            } catch (Exception e) {
+                log.warn("killNode(" + node.id() + "): error stopping: " + e);
+            }
+            node.setState(BubbleNodeState.error_stopped);
+            nodeDAO.update(node);
+            return node;
 
-        return node;
+        } finally {
+            if (lock != null) unlockNode(node.getUuid(), lock);
+        }
     }
 
     protected String lockNetwork(String network) {
@@ -526,7 +528,7 @@ public class StandardNetworkService implements NetworkService {
         return lock;
     }
 
-    protected boolean confirmLock(String network, String lock) {
+    protected boolean confirmNetLock(String network, String lock) {
         return getNetworkLocks().confirmLock(network, lock);
     }
 
@@ -536,10 +538,27 @@ public class StandardNetworkService implements NetworkService {
         log.info("unlockNetwork: unlocked "+network);
     }
 
-    public BubbleNode stopNode(BubbleNode node) {
+    protected String lockNode(String node) {
+        log.info("lockNode: locking "+node);
+        final String lock = getNodeKillLocks().lock(node, NET_LOCK_TIMEOUT, NET_DEADLOCK_TIMEOUT);
+        log.info("lockNode: locked "+node);
+        return lock;
+    }
+
+    protected boolean confirmNodeLock(String node, String lock) {
+        return getNodeKillLocks().confirmLock(node, lock);
+    }
+
+    protected void unlockNode(String node, String lock) {
+        log.info("unlockNode: unlocking "+node);
+        getNodeKillLocks().unlock(node, lock);
+        log.info("unlockNode: unlocked "+node);
+    }
+
+    public void stopNode(BubbleNode node) {
         log.info("stopNode: stopping "+node.id());
         final CloudService cloud = cloudDAO.findByUuid(node.getCloud());
-        return nodeService.stopNode(cloud.getComputeDriver(configuration), node);
+        nodeService.stopNode(cloud.getComputeDriver(configuration), node);
     }
 
     public boolean isReachable(BubbleNode node) {
@@ -547,6 +566,7 @@ public class StandardNetworkService implements NetworkService {
         try {
             log.info(prefix+"starting");
             final NotificationReceipt receipt = notificationService.notify(node, NotificationType.health_check, null);
+            BubbleNodeState state = null;
             if (receipt == null) {
                 log.info(prefix+"health_check failed, checking via cloud");
                 final CloudService cloud = cloudDAO.findByUuid(node.getCloud());
@@ -556,14 +576,14 @@ public class StandardNetworkService implements NetworkService {
                 }
                 final BubbleNode status = cloud.getComputeDriver(configuration).status(node);
                 if (status != null) {
-                    final BubbleNodeState state = status.getState();
+                    state = status.getState();
                     if (state != null && state.active()) {
                         log.info(prefix + "cloud status was: " + state + ", returning true");
                         return true;
                     }
                 }
             }
-            log.warn(prefix+"no way of reaching node, returning false");
+            log.warn(prefix+"no way of reaching node "+node.id()+" (state="+state+"), returning false");
             return false;
 
         } catch (Exception e) {

bubble-server/src/main/java/bubble/service/cloud/job/NodeStartJob.java

@@ -5,9 +5,7 @@
 package bubble.service.cloud.job;
 
 import bubble.cloud.compute.ComputeServiceDriver;
-import bubble.dao.cloud.BubbleNodeDAO;
 import bubble.model.cloud.BubbleNode;
-import bubble.model.cloud.BubbleNodeState;
 import lombok.extern.slf4j.Slf4j;
 
 import static bubble.service.cloud.NodeProgressMeterConstants.METER_ERROR_NO_IP;
@@ -18,28 +16,28 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 public class NodeStartJob implements Runnable {
 
     private BubbleNode node;
-    private final BubbleNodeDAO nodeDAO;
+//    private final BubbleNodeDAO nodeDAO;
     private final ComputeServiceDriver computeDriver;
 
     public NodeStartJob(BubbleNode node,
-                        BubbleNodeDAO nodeDAO,
+//                        BubbleNodeDAO nodeDAO,
                         ComputeServiceDriver computeDriver) {
         this.node = node;
-        this.nodeDAO = nodeDAO;
+//        this.nodeDAO = nodeDAO;
         this.computeDriver = computeDriver;
     }
 
     @Override public void run() {
         try {
-            node.setState(BubbleNodeState.booting);
-            nodeDAO.update(node);
+//            node.setState(BubbleNodeState.booting);
+//            nodeDAO.update(node);
 
             log.debug("run: calling computeDriver.start("+node.id()+")");
             node = computeDriver.start(node);
             log.debug("run: computeDriver.start("+node.id()+") returned successfully");
 
-            node.setState(BubbleNodeState.booted);
-            nodeDAO.update(node);
+//            node.setState(BubbleNodeState.booted);
+//            nodeDAO.update(node);
 
             if (!node.hasIp4()) {
                 throw new NodeJobException(METER_ERROR_NO_IP, "node booted but has no IP");

bubble-server/src/main/resources/logback.xml

@@ -52,6 +52,10 @@
     <logger name="bubble.service.stream" level="INFO" />
 <!--    <logger name="bubble.service.account.StandardAccountMessageService" level="DEBUG" />-->
 <!--    <logger name="bubble.dao.account.message.AccountMessageDAO" level="DEBUG" />-->
+    <logger name="bubble.service.cloud.job" level="DEBUG" />
+    <logger name="bubble.service.cloud.NodeLauncher" level="DEBUG" />
+    <logger name="bubble.service.cloud.NodeService" level="DEBUG" />
+    <logger name="bubble.cloud.compute.vultr" level="DEBUG" />
     <logger name="bubble.resources.message" level="INFO" />
     <logger name="bubble.app.analytics" level="DEBUG" />
     <logger name="bubble.app.passthru" level="DEBUG" />

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -41,6 +41,14 @@ message_profile_update_success=Profile update was successful
 downloading_notice=File download will start promptly...
 downloading_failed=File download failed. Please retry from the start
 
+# Jar upgrade
+message_jar_upgrade_available=A new version of Bubble is available
+message_jar_upgrade_version=The new Bubble version is
+message_jar_current_version=Your current Bubble version is
+button_label_jar_upgrade=Upgrade Your Bubble
+button_label_jar_upgrading=Upgrading...
+message_jar_upgrading=Your Bubble may be unresponsive for a minute or two while the upgrade occurs
+
 # Account SSH key fields
 form_title_ssh_keys=Account SSH Keys
 form_title_add_ssh_key=Add SSH Key

bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+BUBBLE_HOME="/home/bubble"
+UPGRADE_JAR="${BUBBLE_HOME}/api/.upgrade.jar"
+BUBBLE_JAR="${BUBBLE_HOME}/api/bubble.jar"
+
+LOG=/tmp/bubble.upgrade.log
+
+function die {
+  echo 1>&2 "${1}"
+  log "${1}"
+  exit 1
+}
+
+function log {
+  echo "$(date): ${1}" >> ${LOG}
+}
+
+function verify_api_ok {
+    log "Restarting API..."
+    supervisorctl restart bubble || die "Error restarting bubble"
+
+    OK=255
+    START_VERIFY=$(date +%s)
+    VERIFY_TIMEOUT=180
+    VERIFY_URL="https://$(hostname):1443/api/auth/ready"
+    if [[ ${OK} -ne 0 && $(expr $(date +%s) - ${START_VERIFY} -le ${VERIFY_TIMEOUT}) ]] ; then
+      sleep 10s
+      log "Verifying ${VERIFY_URL} is OK...."
+      curl "${VERIFY_URL}" 2>&1 | tee -a ${LOG}
+      OK=$?
+    fi
+
+    if [[ ${OK} -eq 0 ]] ; then
+      echo "ok"
+    else
+      echo "error"
+    fi
+}
+
+BACKUP_JAR=$(mktemp /tmp/bubble.jar.XXXXXXX)
+log "Backing up to ${BACKUP_JAR} ..."
+cp ${BUBBLE_JAR} ${BACKUP_JAR} || die "Error backing up existing jar before upgrade ${BUBBLE_JAR} ${BACKUP_JAR}"
+
+log "Upgrading..."
+mv ${UPGRADE_JAR} ${BUBBLE_JAR} || die "Error moving ${UPGRADE_JAR} -> ${BUBBLE_JAR}"
+
+log "Verifying upgrade..."
+API_OK=$(verify_api_ok)
+if [[ -z "${API_OK}" || "${API_OK}" != "ok" ]] ; then
+  log "Error starting upgraded API, reverting...."
+  cp ${BACKUP_JAR} ${BUBBLE_JAR} || die "Error restoring API jar from backup!"
+  API_OK=$(verify_api_ok)
+  if [[ -z "${API_OK}" || "${API_OK}" != "ok" ]] ; then
+    log "Error starting API from backup!"
+  fi
+else
+  log "Upgrading web site files..."
+  cd ~bubble && jar xf ${BUBBLE_JAR} site && chown -R bubble:bubble site || die "Error updating web files..."
+fi

bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade_monitor.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+THIS_DIR="$(cd "$(dirname "${0}")" && pwd)"
+LOG=/tmp/bubble.upgrade.log
+
+function log {
+  echo "$(date): ${1}" >> ${LOG}
+}
+
+while : ; do
+  sleep 5
+  if [[ -f "${UPGRADE_JAR}" ]] ; then
+    "${THIS_DIR}/bubble_upgrade.sh"
+    if [[ $? -eq 0 ]] ; then
+      log "Upgrade completed successfully"
+    else
+      log "Upgrade failed"
+    fi
+    rm -f "${UPGRADE_JAR}"
+  fi
+done

bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_bubble_upgrade_monitor.conf

@@ -0,0 +1,5 @@
+
+[program:supervisor_bubble_upgrade_monitor]
+stdout_logfile = /dev/null
+stderr_logfile = /dev/null
+command=/usr/local/sbin/supervisor_bubble_upgrade_monitor.sh

bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml

@@ -101,27 +101,29 @@
   with_items:
     - init_bubble_db.sh
 
-- name: Install refresh_bubble_ssh_keys monitor
+- name: Install sbin monitors and scripts
   copy:
-    src: "refresh_bubble_ssh_keys_monitor.sh"
-    dest: "/usr/local/sbin/refresh_bubble_ssh_keys_monitor.sh"
-    owner: root
-    group: root
-    mode: 0500
-
-- name: Install refresh_bubble_ssh_keys script
-  copy:
-    src: refresh_bubble_ssh_keys.sh
-    dest: /usr/local/sbin/refresh_bubble_ssh_keys.sh
+    src: "{{ item }}"
+    dest: "/usr/local/sbin/{{ item }}"
     owner: root
     group: root
     mode: 0500
+  with_items:
+    - refresh_bubble_ssh_keys_monitor.sh
+    - refresh_bubble_ssh_keys.sh
+    - bubble_upgrade_monitor.sh
+    - bubble_upgrade.sh
 
 - name: Install refresh_bubble_ssh_keys_monitor supervisor conf file
   copy:
     src: supervisor_refresh_bubble_ssh_keys_monitor.conf
     dest: /etc/supervisor/conf.d/refresh_bubble_ssh_keys_monitor.conf
 
+- name: Install bubble_upgrade_monitor supervisor conf file
+  copy:
+    src: supervisor_bubble_upgrade_monitor.conf
+    dest: /etc/supervisor/conf.d/bubble_upgrade_monitor.conf
+
 - name: Install packer for sage node
   shell: su - bubble bash -c install_packer.sh
   when: install_type == 'sage'

bubble-server/src/test/java/bubble/mock/MockNetworkService.java

@@ -28,7 +28,7 @@ public class MockNetworkService extends StandardNetworkService {
     @Autowired private BubbleConfiguration configuration;
 
     @Override protected String lockNetwork(String network) { return "lock"; }
-    @Override protected boolean confirmLock(String network, String lock) { return true; }
+    @Override protected boolean confirmNetLock(String network, String lock) { return true; }
     @Override protected void unlockNetwork(String network, String lock) {}
 
     @Override public BubbleNode newNode(NewNodeNotification nn, NodeLaunchMonitor launchMonitor) {
@@ -70,9 +70,7 @@ public class MockNetworkService extends StandardNetworkService {
         return true;
     }
 
-    @Override public BubbleNode stopNode(BubbleNode node) {
-        return node.setState(BubbleNodeState.stopped);
-    }
+    @Override public void stopNode(BubbleNode node) { node.setState(BubbleNodeState.stopped); }
 
     @Override public BubbleNode killNode(BubbleNode node, String message) {
         return node.setState(BubbleNodeState.stopped);

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 1594a1ade170fd9b690682894f9b5f410659548e
+Subproject commit 549884d63dc1d46f15c33cdcf5d9604deb821992

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 773a75f6cc2659bc330a5b4bdbb61d31affeec10
+Subproject commit 035690f052f72841bfedc5b25e3798fa22f7b2dd