|
|
@@ -12,8 +12,10 @@ import com.fasterxml.jackson.databind.JsonNode; |
|
|
|
import com.fasterxml.jackson.databind.node.ObjectNode; |
|
|
|
import lombok.Getter; |
|
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
|
import org.cobbzilla.util.collection.SingletonList; |
|
|
|
import org.cobbzilla.util.http.HttpRequestBean; |
|
|
|
import org.cobbzilla.util.http.HttpResponseBean; |
|
|
|
import org.cobbzilla.util.system.CommandResult; |
|
|
|
|
|
|
|
import javax.persistence.EntityNotFoundException; |
|
|
|
import java.io.IOException; |
|
|
@@ -21,6 +23,7 @@ import java.util.ArrayList; |
|
|
|
import java.util.Collections; |
|
|
|
import java.util.Iterator; |
|
|
|
import java.util.List; |
|
|
|
import java.util.function.Function; |
|
|
|
|
|
|
|
import static bubble.model.cloud.BubbleNode.TAG_INSTANCE_ID; |
|
|
|
import static bubble.model.cloud.BubbleNode.TAG_SSH_KEY_ID; |
|
|
@@ -244,12 +247,16 @@ public class VultrDriver extends ComputeServiceDriverBase { |
|
|
|
throw invalidEx("err.node.stop.error", "stop: no " + VULTR_SUBID + " on node, returning"); |
|
|
|
} |
|
|
|
|
|
|
|
stopServer(subId); |
|
|
|
return node; |
|
|
|
} |
|
|
|
|
|
|
|
private void stopServer(String subId) { |
|
|
|
final HttpRequestBean destroyServerRequest = auth(new HttpRequestBean(POST, DESTROY_SERVER_URL, VULTR_SUBID + "=" + subId)); |
|
|
|
final HttpResponseBean destroyResponse = destroyServerRequest.curl(); |
|
|
|
if (destroyResponse.getStatus() != OK) { |
|
|
|
throw invalidEx("err.node.stop.error", "stop: error stopping node: "+destroyResponse); |
|
|
|
} |
|
|
|
return node; |
|
|
|
} |
|
|
|
|
|
|
|
private BubbleNode findByIp4(BubbleNode node, String ip4) throws IOException { |
|
|
@@ -294,6 +301,13 @@ public class VultrDriver extends ComputeServiceDriverBase { |
|
|
|
} |
|
|
|
|
|
|
|
@Override public List<BubbleNode> listNodes() throws IOException { |
|
|
|
return listNodes(server -> { |
|
|
|
final String tag = server.has("tag") ? server.get("tag").textValue() : null; |
|
|
|
return tag != null && tag.equals(cloud.getUuid()); |
|
|
|
}); |
|
|
|
} |
|
|
|
|
|
|
|
public List<BubbleNode> listNodes(Function<ObjectNode, Boolean> filter) throws IOException { |
|
|
|
final List<BubbleNode> nodes = new ArrayList<>(); |
|
|
|
final HttpRequestBean listServerRequest = auth(new HttpRequestBean(LIST_SERVERS_URL)); |
|
|
|
final HttpResponseBean listResponse = getResponse(listServerRequest); |
|
|
@@ -303,8 +317,7 @@ public class VultrDriver extends ComputeServiceDriverBase { |
|
|
|
for (Iterator<String> iter = entity.fieldNames(); iter.hasNext(); ) { |
|
|
|
final String subid = iter.next(); |
|
|
|
final ObjectNode server = (ObjectNode) entity.get(subid); |
|
|
|
final String tag = server.has("tag") ? server.get("tag").textValue() : null; |
|
|
|
if (tag == null || !tag.equals(cloud.getUuid())) { |
|
|
|
if (!filter.apply(server)) { |
|
|
|
log.debug("Skipping node without cloud tag "+cloud.getUuid()+": "+subid); |
|
|
|
continue; |
|
|
|
} |
|
|
@@ -378,4 +391,59 @@ public class VultrDriver extends ComputeServiceDriverBase { |
|
|
|
return images == null ? Collections.emptyList() : images; |
|
|
|
} |
|
|
|
|
|
|
|
public static final long SNAPSHOT_TIMEOUT = MINUTES.toMillis(60); |
|
|
|
@Override public List<PackerImage> finalizeIncompletePackerRun(CommandResult commandResult, String jarSha) { |
|
|
|
if (!commandResult.getStdout().contains("Waiting 300s for snapshot") |
|
|
|
|| !commandResult.getStdout().contains("Error waiting for snapshot") |
|
|
|
|| !commandResult.getStdout().contains("Unable to destroy server: Unable to remove VM: Server is currently locked")) { |
|
|
|
return null; |
|
|
|
} |
|
|
|
|
|
|
|
// wait longer for the snapshot... |
|
|
|
final long start = now(); |
|
|
|
PackerImage snapshot = null; |
|
|
|
while (now() - start < SNAPSHOT_TIMEOUT) { |
|
|
|
snapshot = getPackerImages().stream() |
|
|
|
.filter(i -> i.getName().endsWith(jarSha)) |
|
|
|
.findFirst() |
|
|
|
.orElse(null); |
|
|
|
if (snapshot != null) break; |
|
|
|
sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+jarSha); |
|
|
|
} |
|
|
|
if (snapshot == null) { |
|
|
|
log.error("finalizeIncompletePackerRun: timeout waiting for snapshot"); |
|
|
|
} |
|
|
|
|
|
|
|
// find the server |
|
|
|
final List<BubbleNode> servers; |
|
|
|
try { |
|
|
|
servers = listNodes(server -> { |
|
|
|
final String tag = server.has("tag") ? server.get("tag").textValue() : null; |
|
|
|
return tag != null && tag.endsWith(jarSha); |
|
|
|
}); |
|
|
|
} catch (IOException e) { |
|
|
|
log.error("finalizeIncompletePackerRun: error listing servers: "+shortError(e), e); |
|
|
|
return null; |
|
|
|
} |
|
|
|
if (servers.isEmpty()) { |
|
|
|
log.error("finalizeIncompletePackerRun: snapshot server not found"); |
|
|
|
return null; |
|
|
|
} |
|
|
|
if (servers.size() != 1) { |
|
|
|
log.error("finalizeIncompletePackerRun: expected only one server, found: "+servers.size()); |
|
|
|
return null; |
|
|
|
} |
|
|
|
|
|
|
|
// now shut down the server |
|
|
|
try { |
|
|
|
stopServer(servers.get(0).getTag(TAG_INSTANCE_ID)); |
|
|
|
} catch (Exception e) { |
|
|
|
log.error("finalizeIncompletePackerRun: error stopping server: "+shortError(e)); |
|
|
|
return null; |
|
|
|
} |
|
|
|
if (snapshot == null) return null; |
|
|
|
|
|
|
|
return new SingletonList<>(snapshot); |
|
|
|
} |
|
|
|
|
|
|
|
} |