From fb08744b429cc3d9866a7945cb45fc71da6527df Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 4 Jun 2020 20:49:45 -0400 Subject: [PATCH] WIP: packer build for vultr now working --- .../cloud/compute/ComputeServiceDriver.java | 2 + .../cloud/compute/vultr/VultrDriver.java | 74 ++++++++++++++++++- .../compute/vultr/VultrPackerImageParser.java | 2 + .../java/bubble/service/packer/PackerJob.java | 11 ++- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java index e9ebf106..a5c508f0 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java @@ -8,6 +8,7 @@ import bubble.cloud.CloudServiceDriver; import bubble.cloud.CloudServiceType; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.RegionalServiceDriver; +import org.cobbzilla.util.system.CommandResult; import java.util.List; import java.util.Map; @@ -34,5 +35,6 @@ public interface ComputeServiceDriver extends CloudServiceDriver, RegionalServic @Override default boolean test () { return true; } List getPackerImages(); + default List finalizeIncompletePackerRun(CommandResult commandResult, String jarSha) { return null; } } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java index 2d937a4b..340d35b0 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java @@ -12,8 +12,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.SingletonList; import org.cobbzilla.util.http.HttpRequestBean; import org.cobbzilla.util.http.HttpResponseBean; +import org.cobbzilla.util.system.CommandResult; import javax.persistence.EntityNotFoundException; import java.io.IOException; @@ -21,6 +23,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.function.Function; import static bubble.model.cloud.BubbleNode.TAG_INSTANCE_ID; import static bubble.model.cloud.BubbleNode.TAG_SSH_KEY_ID; @@ -244,12 +247,16 @@ public class VultrDriver extends ComputeServiceDriverBase { throw invalidEx("err.node.stop.error", "stop: no " + VULTR_SUBID + " on node, returning"); } + stopServer(subId); + return node; + } + + private void stopServer(String subId) { final HttpRequestBean destroyServerRequest = auth(new HttpRequestBean(POST, DESTROY_SERVER_URL, VULTR_SUBID + "=" + subId)); final HttpResponseBean destroyResponse = destroyServerRequest.curl(); if (destroyResponse.getStatus() != OK) { throw invalidEx("err.node.stop.error", "stop: error stopping node: "+destroyResponse); } - return node; } private BubbleNode findByIp4(BubbleNode node, String ip4) throws IOException { @@ -294,6 +301,13 @@ public class VultrDriver extends ComputeServiceDriverBase { } @Override public List listNodes() throws IOException { + return listNodes(server -> { + final String tag = server.has("tag") ? server.get("tag").textValue() : null; + return tag != null && tag.equals(cloud.getUuid()); + }); + } + + public List listNodes(Function filter) throws IOException { final List nodes = new ArrayList<>(); final HttpRequestBean listServerRequest = auth(new HttpRequestBean(LIST_SERVERS_URL)); final HttpResponseBean listResponse = getResponse(listServerRequest); @@ -303,8 +317,7 @@ public class VultrDriver extends ComputeServiceDriverBase { for (Iterator iter = entity.fieldNames(); iter.hasNext(); ) { final String subid = iter.next(); final ObjectNode server = (ObjectNode) entity.get(subid); - final String tag = server.has("tag") ? server.get("tag").textValue() : null; - if (tag == null || !tag.equals(cloud.getUuid())) { + if (!filter.apply(server)) { log.debug("Skipping node without cloud tag "+cloud.getUuid()+": "+subid); continue; } @@ -378,4 +391,59 @@ public class VultrDriver extends ComputeServiceDriverBase { return images == null ? Collections.emptyList() : images; } + public static final long SNAPSHOT_TIMEOUT = MINUTES.toMillis(60); + @Override public List finalizeIncompletePackerRun(CommandResult commandResult, String jarSha) { + if (!commandResult.getStdout().contains("Waiting 300s for snapshot") + || !commandResult.getStdout().contains("Error waiting for snapshot") + || !commandResult.getStdout().contains("Unable to destroy server: Unable to remove VM: Server is currently locked")) { + return null; + } + + // wait longer for the snapshot... + final long start = now(); + PackerImage snapshot = null; + while (now() - start < SNAPSHOT_TIMEOUT) { + snapshot = getPackerImages().stream() + .filter(i -> i.getName().endsWith(jarSha)) + .findFirst() + .orElse(null); + if (snapshot != null) break; + sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+jarSha); + } + if (snapshot == null) { + log.error("finalizeIncompletePackerRun: timeout waiting for snapshot"); + } + + // find the server + final List servers; + try { + servers = listNodes(server -> { + final String tag = server.has("tag") ? server.get("tag").textValue() : null; + return tag != null && tag.endsWith(jarSha); + }); + } catch (IOException e) { + log.error("finalizeIncompletePackerRun: error listing servers: "+shortError(e), e); + return null; + } + if (servers.isEmpty()) { + log.error("finalizeIncompletePackerRun: snapshot server not found"); + return null; + } + if (servers.size() != 1) { + log.error("finalizeIncompletePackerRun: expected only one server, found: "+servers.size()); + return null; + } + + // now shut down the server + try { + stopServer(servers.get(0).getTag(TAG_INSTANCE_ID)); + } catch (Exception e) { + log.error("finalizeIncompletePackerRun: error stopping server: "+shortError(e)); + return null; + } + if (snapshot == null) return null; + + return new SingletonList<>(snapshot); + } + } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrPackerImageParser.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrPackerImageParser.java index 0938a89d..353344e5 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrPackerImageParser.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrPackerImageParser.java @@ -17,6 +17,8 @@ public class VultrPackerImageParser extends ListResourceParser { if (!item.has("SNAPSHOTID")) return die("parse: SNAPSHOTID not found"); if (!item.has("OSID")) return die("parse: OSID not found"); if (!item.has("description")) return die("parse: description not found"); + if (!item.has("status")) return die("parse: status not found"); + if (!item.get("status").textValue().equals("complete")) return null; final String name = item.get("description").textValue(); if (!name.startsWith(PACKER_IMAGE_PREFIX)) return null; if (!name.endsWith("_"+jarSha)) return null; diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index b5c859a7..dafe9f23 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -155,13 +155,20 @@ public class PackerJob implements Callable> { // run packer, return handle to running packer log.info("running packer for "+installType+"..."); final CommandResult commandResult = CommandShell.exec(new Command(new CommandLine("packer") - .addArgument("build").addArgument("-parallel-builds=2").addArgument("packer.json")) + .addArgument("build") + .addArgument("-parallel-builds=2") + .addArgument("-color=false") + .addArgument("packer.json")) .setDir(tempDir) .setEnv(env) .setCopyToStandard(true)); if (!commandResult.isZeroExitStatus()) { - return die("Error executing packer: exit status "+commandResult.getExitStatus()); + images = computeDriver.finalizeIncompletePackerRun(commandResult, jarSha); + if (empty(images)) { + return die("Error executing packer: exit status " + commandResult.getExitStatus()); + } + return images; } // read manifest, populate images