copy flyway table to all launched instances, copy packer keys to launched sages

bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java

@@ -130,6 +130,7 @@ public abstract class ComputeServiceDriverBase
             if (packerImage == null) {
                 return die("getPackerImage: error creating packer image");
             }
+            node.setPackerImageCreation(false);
         }
         return packerImage;
     }

bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java

@@ -54,6 +54,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
     public static final String SNAPSHOT_URL = VULTR_API_BASE + "snapshot/list";
 
     public static final String VULTR_SUBID = "SUBID";
+    public static final String VULTR_TAG = "tag";
     public static final String VULTR_V4_IP = "main_ip";
     public static final String VULTR_V6_IP = "v6_main_ip";
     public static final String VULTR_LABEL = "label";
@@ -185,8 +186,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
             final JsonNode serverNode = json(pollResponse.getEntityString(), JsonNode.class);
             // if (log.isDebugEnabled()) log.debug("start: polled node "+node.id()+" json="+json(serverNode, COMPACT_MAPPER));
             if (serverNode != null) {
-                if (serverNode.has("tag")
-                        && serverNode.get("tag").textValue().equals(cloud.getUuid())
+                if (hasCorrectTag(serverNode)
                         && serverNode.has(VULTR_STATUS)
                         && serverNode.has(VULTR_SERVER_STATE)
                         && serverNode.has(VULTR_POWER_STATUS)
@@ -200,9 +200,11 @@ public class VultrDriver extends ComputeServiceDriverBase {
                     if (log.isDebugEnabled()) log.debug("start("+node.id()+"): found server_state="+serverState+", status="+status+", power_status="+powerStatus+", ip4="+ip4+", ip6="+ip6);
 
                     if (ip4 != null && ip4.length() > 0 && !ip4.equals("0.0.0.0")) {
+                        if (log.isTraceEnabled()) log.trace("start("+node.id()+") setting ip4="+ip4);
                         node.setIp4(ip4);
                     }
                     if (ip6 != null && ip6.length() > 0) {
+                        if (log.isTraceEnabled()) log.trace("start("+node.id()+") setting ip6="+ip6);
                         node.setIp6(ip6);
                     }
                     if (status.equals(VULTR_STATUS_ACTIVE) && (node.hasIp4() || node.hasIp6())) {
@@ -319,10 +321,12 @@ public class VultrDriver extends ComputeServiceDriverBase {
     }
 
     @Override public List<BubbleNode> listNodes() throws IOException {
-        return listNodes(server -> {
-            final String tag = server.has("tag") ? server.get("tag").textValue() : null;
-            return tag != null && tag.contains(cloud.getUuid()) && tag.contains(domainname());
-        });
+        return listNodes(this::hasCorrectTag);
+    }
+
+    private boolean hasCorrectTag(JsonNode server) {
+        final String tag = server.has(VULTR_TAG) ? server.get(VULTR_TAG).textValue() : null;
+        return tag != null && tag.contains(cloud.getUuid()) && tag.contains(domainname());
     }
 
     public List<BubbleNode> listNodes(Function<ObjectNode, Boolean> filter) throws IOException {
@@ -449,7 +453,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
         // find the server(s)
         try {
             servers = listNodes(server -> {
-                final String tag = server.has("tag") ? server.get("tag").textValue() : null;
+                final String tag = server.has(VULTR_TAG) ? server.get(VULTR_TAG).textValue() : null;
                 return tag != null && tag.contains("_"+installType.name()+"_") && tag.contains(keyHash);
             });
         } catch (IOException e) {

bubble-server/src/main/java/bubble/dao/cloud/BubbleNodeDAO.java

@@ -62,16 +62,6 @@ public class BubbleNodeDAO extends AccountOwnedEntityDAO<BubbleNode> {
     public BubbleNode findByFqdn(String fqdn) { return findByUniqueField("fqdn", fqdn); }
 
     @Override public void delete(String uuid) {
-        final BubbleNode node = findByUuid(uuid);
-        if (node == null) return;
-        if (node.isRunning() || networkService.isReachable(node)) {
-            throw invalidEx("err.node.running", "Node must be stopped before deleting");
-        }
-        getConfiguration().deleteDependencies(node);
-        super.delete(uuid);
-    }
-
-    @Override public void forceDelete(String uuid) {
         final BubbleNode node = findByUuid(uuid);
         if (node == null) return;
         try {
@@ -82,7 +72,7 @@ public class BubbleNodeDAO extends AccountOwnedEntityDAO<BubbleNode> {
             log.error("forceDelete: error checking/stopping node: "+node.getUuid()+": "+e);
         }
         getConfiguration().deleteDependencies(node);
-        super.forceDelete(uuid);
+        super.delete(uuid);
     }
 
     public List<BubbleNode> findPeersByNetwork(String network) {

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -80,6 +80,7 @@ import static bubble.server.BubbleConfiguration.ENV_DEBUG_NODE_INSTALL;
 import static bubble.service.boot.StandardSelfNodeService.*;
 import static bubble.service.cloud.NodeLaunchException.*;
 import static bubble.service.cloud.NodeProgressMeterConstants.*;
+import static bubble.service.packer.PackerService.PACKER_KEY_NAME;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
@@ -515,6 +516,15 @@ public class StandardNetworkService implements NetworkService {
         writeFile(bubbleFilesDir, null, SAGE_NODE_JSON, json(BubbleNode.sageMask(sageNode)));
         writeFile(bubbleFilesDir, null, SAGE_KEY_JSON, json(BubbleNodeKey.sageMask(sageKey)));
 
+        // write packer keys if launching sage
+        if (network.getInstallType() == AnsibleInstallType.sage) {
+            final File packerPubKeyFile = new File(bubbleFilesDir, PACKER_KEY_NAME+".pub");
+            copyFile(packerService.getPackerPublicKey(), packerPubKeyFile);
+
+            final File packerPrivateKeyFile = new File(bubbleFilesDir, PACKER_KEY_NAME);
+            copyFile(packerService.getPackerPrivateKey(), packerPrivateKeyFile);
+        }
+
         // write install_local.sh script
         final File installLocalScript = writeFile(automation, ctx, INSTALL_LOCAL_SH, INSTALL_LOCAL_TEMPLATE);
         chmod(installLocalScript, "500");

bubble-server/src/main/java/bubble/service/dbfilter/DatabaseFilterService.java

@@ -51,6 +51,7 @@ public class DatabaseFilterService {
 
     public static final String ENV_OLD_DB_KEY = "OLD_DB_KEY";
     public static final String ENV_NEW_DB_KEY = "NEW_DB_KEY";
+    public static final String[] FLYWAY_DUMP_OPTIONS = {"--table=flyway_schema_history", "--data-only"};
 
     @Autowired private BubbleConfiguration configuration;
 
@@ -148,6 +149,15 @@ public class DatabaseFilterService {
                 return die("copyDatabase: writer exited with an error (dbName="+dbName+"): "+writeResult.get());
             }
 
+            // copy flyway schema table
+            log.debug("copyDatabase: dumping flyway_schema_version data");
+            final CommandResult flywayData = pgExec("pg_dump", dbConfig.getDatabaseName(), null, null, FLYWAY_DUMP_OPTIONS);
+            if (!flywayData.isZeroExitStatus()) return die("copyDatabase: error dumping flyway_schema_version data: "+flywayData);
+
+            log.debug("copyDatabase: inserting flyway_schema_version data");
+            final CommandResult flywayInsert = pgExec("psql", dbName, new ByteArrayInputStream(flywayData.getStdout().getBytes()), null);
+            if (!flywayInsert.isZeroExitStatus()) return die("copyDatabase: error inserting flyway_schema_version data: "+flywayInsert);
+
             // dump new DB
             log.info("copyDatabase: dumping new database: "+dbName);
             try (OutputStream out = new GZIPOutputStream(new FileOutputStream(pgDumpFile))) {
@@ -177,8 +187,14 @@ public class DatabaseFilterService {
     }
 
     public CommandResult pgExec(String command, String dbName, InputStream in, OutputStream out) throws IOException {
-        final Command pgCommand = new Command(new CommandLine(command)
-                .addArguments(configuration.pgOptions(dbName)))
+        return pgExec(command, dbName, in, out, null);
+    }
+
+    public CommandResult pgExec(String command, String dbName, InputStream in, OutputStream out, String[] args) throws IOException {
+        final CommandLine commandLine = new CommandLine(command)
+                .addArguments(configuration.pgOptions(dbName));
+        if (args != null) commandLine.addArguments(args);
+        final Command pgCommand = new Command(commandLine)
                 .setEnv(configuration.pgEnv())
                 .setStdin(in)
                 .setOut(out)

bubble-server/src/main/java/bubble/service/packer/PackerService.java

@@ -84,7 +84,7 @@ public class PackerService {
     public File getPackerPrivateKey () { return initPackerKey(false); }
     public String getPackerPublicKeyHash () { return ShaUtil.sha256_file(getPackerPublicKey()); }
 
-    private synchronized File initPackerKey(boolean pub) {
+    public synchronized File initPackerKey(boolean pub) {
         final File keyDir = new File(System.getProperty("user.home"),".ssh");
         if (!keyDir.exists()) mkdirOrDie(keyDir);
         chmod(keyDir, "700");

bubble-server/src/main/resources/ansible/roles/bubble/tasks/main.yml

@@ -49,3 +49,29 @@
 - name: Start monitor for restoring this bubble if applicable
   shell: bash -c 'nohup /usr/local/bin/bubble_restore_monitor.sh {{ admin_port }} {{ restore_timeout }} > /dev/null &'
   when: restore_key is defined
+
+- name: Create .ssh directory for packer keys
+  file:
+    path: /home/bubble/.ssh
+    owner: bubble
+    group: bubble
+    state: directory
+  when: install_type == 'sage'
+
+- name: Install packer public key for sage
+  copy:
+    src: packer_rsa.pub
+    dest: /home/bubble/.ssh/packer_rsa.pub
+    owner: bubble
+    group: bubble
+    mode: 0400
+  when: install_type == 'sage'
+
+- name: Install packer private key for sage
+  copy:
+    src: packer_rsa
+    dest: /home/bubble/.ssh/packer_rsa
+    owner: bubble
+    group: bubble
+    mode: 0400
+  when: install_type == 'sage'