WIP: packer build for vultr now working

bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java

@@ -8,6 +8,7 @@ import bubble.cloud.CloudServiceDriver;
 import bubble.cloud.CloudServiceType;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.RegionalServiceDriver;
+import org.cobbzilla.util.system.CommandResult;
 
 import java.util.List;
 import java.util.Map;
@@ -34,5 +35,6 @@ public interface ComputeServiceDriver extends CloudServiceDriver, RegionalServic
     @Override default boolean test () { return true; }
 
     List<PackerImage> getPackerImages();
+    default List<PackerImage> finalizeIncompletePackerRun(CommandResult commandResult, String jarSha) { return null; }
 
 }

bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java

@@ -12,8 +12,10 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.collection.SingletonList;
 import org.cobbzilla.util.http.HttpRequestBean;
 import org.cobbzilla.util.http.HttpResponseBean;
+import org.cobbzilla.util.system.CommandResult;
 
 import javax.persistence.EntityNotFoundException;
 import java.io.IOException;
@@ -21,6 +23,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.Function;
 
 import static bubble.model.cloud.BubbleNode.TAG_INSTANCE_ID;
 import static bubble.model.cloud.BubbleNode.TAG_SSH_KEY_ID;
@@ -244,12 +247,16 @@ public class VultrDriver extends ComputeServiceDriverBase {
             throw invalidEx("err.node.stop.error", "stop: no " + VULTR_SUBID + " on node, returning");
         }
 
+        stopServer(subId);
+        return node;
+    }
+
+    private void stopServer(String subId) {
         final HttpRequestBean destroyServerRequest = auth(new HttpRequestBean(POST, DESTROY_SERVER_URL, VULTR_SUBID + "=" + subId));
         final HttpResponseBean destroyResponse = destroyServerRequest.curl();
         if (destroyResponse.getStatus() != OK) {
             throw invalidEx("err.node.stop.error", "stop: error stopping node: "+destroyResponse);
         }
-        return node;
     }
 
     private BubbleNode findByIp4(BubbleNode node, String ip4) throws IOException {
@@ -294,6 +301,13 @@ public class VultrDriver extends ComputeServiceDriverBase {
     }
 
     @Override public List<BubbleNode> listNodes() throws IOException {
+        return listNodes(server -> {
+            final String tag = server.has("tag") ? server.get("tag").textValue() : null;
+            return tag != null && tag.equals(cloud.getUuid());
+        });
+    }
+
+    public List<BubbleNode> listNodes(Function<ObjectNode, Boolean> filter) throws IOException {
         final List<BubbleNode> nodes = new ArrayList<>();
         final HttpRequestBean listServerRequest = auth(new HttpRequestBean(LIST_SERVERS_URL));
         final HttpResponseBean listResponse = getResponse(listServerRequest);
@@ -303,8 +317,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
                 for (Iterator<String> iter = entity.fieldNames(); iter.hasNext(); ) {
                     final String subid = iter.next();
                     final ObjectNode server = (ObjectNode) entity.get(subid);
-                    final String tag = server.has("tag") ? server.get("tag").textValue() : null;
-                    if (tag == null || !tag.equals(cloud.getUuid())) {
+                    if (!filter.apply(server)) {
                         log.debug("Skipping node without cloud tag "+cloud.getUuid()+": "+subid);
                         continue;
                     }
@@ -378,4 +391,59 @@ public class VultrDriver extends ComputeServiceDriverBase {
         return images == null ? Collections.emptyList() : images;
     }
 
+    public static final long SNAPSHOT_TIMEOUT = MINUTES.toMillis(60);
+    @Override public List<PackerImage> finalizeIncompletePackerRun(CommandResult commandResult, String jarSha) {
+        if (!commandResult.getStdout().contains("Waiting 300s for snapshot")
+                || !commandResult.getStdout().contains("Error waiting for snapshot")
+                || !commandResult.getStdout().contains("Unable to destroy server: Unable to remove VM: Server is currently locked")) {
+            return null;
+        }
+
+        // wait longer for the snapshot...
+        final long start = now();
+        PackerImage snapshot = null;
+        while (now() - start < SNAPSHOT_TIMEOUT) {
+            snapshot = getPackerImages().stream()
+                    .filter(i -> i.getName().endsWith(jarSha))
+                    .findFirst()
+                    .orElse(null);
+            if (snapshot != null) break;
+            sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+jarSha);
+        }
+        if (snapshot == null) {
+            log.error("finalizeIncompletePackerRun: timeout waiting for snapshot");
+        }
+
+        // find the server
+        final List<BubbleNode> servers;
+        try {
+            servers = listNodes(server -> {
+                final String tag = server.has("tag") ? server.get("tag").textValue() : null;
+                return tag != null && tag.endsWith(jarSha);
+            });
+        } catch (IOException e) {
+            log.error("finalizeIncompletePackerRun: error listing servers: "+shortError(e), e);
+            return null;
+        }
+        if (servers.isEmpty()) {
+            log.error("finalizeIncompletePackerRun: snapshot server not found");
+            return null;
+        }
+        if (servers.size() != 1) {
+            log.error("finalizeIncompletePackerRun: expected only one server, found: "+servers.size());
+            return null;
+        }
+
+        // now shut down the server
+        try {
+            stopServer(servers.get(0).getTag(TAG_INSTANCE_ID));
+        } catch (Exception e) {
+            log.error("finalizeIncompletePackerRun: error stopping server: "+shortError(e));
+            return null;
+        }
+        if (snapshot == null) return null;
+
+        return new SingletonList<>(snapshot);
+    }
+
 }

bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrPackerImageParser.java

@@ -17,6 +17,8 @@ public class VultrPackerImageParser extends ListResourceParser<PackerImage> {
         if (!item.has("SNAPSHOTID")) return die("parse: SNAPSHOTID not found");
         if (!item.has("OSID")) return die("parse: OSID not found");
         if (!item.has("description")) return die("parse: description not found");
+        if (!item.has("status")) return die("parse: status not found");
+        if (!item.get("status").textValue().equals("complete")) return null;
         final String name = item.get("description").textValue();
         if (!name.startsWith(PACKER_IMAGE_PREFIX)) return null;
         if (!name.endsWith("_"+jarSha)) return null;

bubble-server/src/main/java/bubble/service/packer/PackerJob.java

@@ -155,13 +155,20 @@ public class PackerJob implements Callable<List<PackerImage>> {
             // run packer, return handle to running packer
             log.info("running packer for "+installType+"...");
             final CommandResult commandResult = CommandShell.exec(new Command(new CommandLine("packer")
-                    .addArgument("build").addArgument("-parallel-builds=2").addArgument("packer.json"))
+                    .addArgument("build")
+                    .addArgument("-parallel-builds=2")
+                    .addArgument("-color=false")
+                    .addArgument("packer.json"))
                     .setDir(tempDir)
                     .setEnv(env)
                     .setCopyToStandard(true));
 
             if (!commandResult.isZeroExitStatus()) {
-                return die("Error executing packer: exit status "+commandResult.getExitStatus());
+                images = computeDriver.finalizeIncompletePackerRun(commandResult, jarSha);
+                if (empty(images)) {
+                    return die("Error executing packer: exit status " + commandResult.getExitStatus());
+                }
+                return images;
             }
 
             // read manifest, populate images