From 07fcafa516ec1101cace4bc06049410b1a2259ec Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 4 Jun 2020 15:09:25 -0400 Subject: [PATCH] WIP: packer image creation working for digitalocean --- .../main/java/bubble/cloud/CloudRegion.java | 1 + .../src/main/java/bubble/cloud/NoopCloud.java | 10 ++-- .../bubble/cloud/compute/ComputeConfig.java | 3 +- .../bubble/cloud/compute/ComputeNodeSize.java | 5 ++ .../cloud/compute/ComputeServiceDriver.java | 8 ++++ .../compute/ComputeServiceDriverBase.java | 48 +++++++++++++++++-- .../cloud/compute/ListResourceParser.java | 12 +++++ .../java/bubble/cloud/compute/OsImage.java | 12 +++++ .../bubble/cloud/compute/PackerConfig.java | 6 +++ .../{digitalocean => }/ResourceParser.java | 2 +- .../delegate/DelegatedComputeDriver.java | 10 ++-- .../DigitalOceanComputeNodeSizeParser.java | 25 ++++++++++ .../digitalocean/DigitalOceanDriver.java | 35 +++++--------- .../DigitalOceanOsImageParser.java | 31 ++++++++++++ ...ava => DigitalOceanPackerImageParser.java} | 10 +++- .../DigitalOceanRegionParser.java | 19 ++++++++ .../digitalocean/ResourceSlugParser.java | 24 ---------- .../cloud/compute/ec2/AmazonEC2Driver.java | 17 +++++++ .../compute/local/LocalComputeDriver.java | 1 + .../cloud/compute/mock/MockComputeDriver.java | 11 ++--- .../vultr/VultrComputeNodeSizeParser.java | 26 ++++++++++ .../cloud/compute/vultr/VultrDriver.java | 37 +++++++------- .../compute/vultr/VultrOsImageParser.java | 19 ++++++++ .../compute/vultr/VultrRegionParser.java | 19 ++++++++ .../model/cloud/notify/NotificationType.java | 2 + ...ficationHandler_compute_driver_get_os.java | 15 ++++++ .../java/bubble/service/packer/PackerJob.java | 16 ++++++- .../models/defaults/cloudService.json | 21 ++++---- .../resources/packer/packer-sage.json.hbs | 5 +- 29 files changed, 350 insertions(+), 100 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/ListResourceParser.java create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/OsImage.java rename bubble-server/src/main/java/bubble/cloud/compute/{digitalocean => }/ResourceParser.java (85%) create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java rename bubble-server/src/main/java/bubble/cloud/compute/digitalocean/{ImageParser.java => DigitalOceanPackerImageParser.java} (80%) create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanRegionParser.java delete mode 100644 bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceSlugParser.java create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrComputeNodeSizeParser.java create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrOsImageParser.java create mode 100644 bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrRegionParser.java create mode 100644 bubble-server/src/main/java/bubble/notify/compute/NotificationHandler_compute_driver_get_os.java diff --git a/bubble-server/src/main/java/bubble/cloud/CloudRegion.java b/bubble-server/src/main/java/bubble/cloud/CloudRegion.java index 15ad8e1d..1a79aa75 100644 --- a/bubble-server/src/main/java/bubble/cloud/CloudRegion.java +++ b/bubble-server/src/main/java/bubble/cloud/CloudRegion.java @@ -20,6 +20,7 @@ public class CloudRegion { @Getter @Setter private String cloud; + @Getter @Setter private Long id; @Getter @Setter private String name; @Setter private String internalName; public String getInternalName () { return internalName != null ? internalName : name; } diff --git a/bubble-server/src/main/java/bubble/cloud/NoopCloud.java b/bubble-server/src/main/java/bubble/cloud/NoopCloud.java index 68416b29..2c4a4b51 100644 --- a/bubble-server/src/main/java/bubble/cloud/NoopCloud.java +++ b/bubble-server/src/main/java/bubble/cloud/NoopCloud.java @@ -6,10 +6,7 @@ package bubble.cloud; import bubble.cloud.auth.AuthenticationDriver; import bubble.cloud.auth.RenderedMessage; -import bubble.cloud.compute.ComputeNodeSize; -import bubble.cloud.compute.ComputeNodeSizeType; -import bubble.cloud.compute.ComputeServiceDriver; -import bubble.cloud.compute.PackerImage; +import bubble.cloud.compute.*; import bubble.cloud.dns.DnsServiceDriver; import bubble.cloud.email.EmailServiceDriver; import bubble.cloud.email.RenderedEmail; @@ -41,6 +38,7 @@ import java.io.InputStream; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; @Slf4j public class NoopCloud implements @@ -254,6 +252,10 @@ public class NoopCloud implements return null; } + @Override public OsImage getOs() { return null; } + + @Override public Map getSizesMap() { return null; } + @Override public List getRegions() { if (log.isDebugEnabled()) log.debug("getRegions()"); return null; diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java index 224e34b2..000defe1 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeConfig.java @@ -13,8 +13,9 @@ import org.cobbzilla.util.collection.NameAndValue; public class ComputeConfig extends RegionalConfig { @Getter @Setter private ComputeNodeSize[] sizes; - @Getter @Setter private NameAndValue[] config; + @Getter @Setter private String os; @Getter @Setter private PackerConfig packer; + @Getter @Setter private NameAndValue[] config; public CloudRegion getRegion (String name) { for (CloudRegion r : getRegions()) { diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java index 1b311ea7..e16b1d34 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java @@ -4,6 +4,7 @@ */ package bubble.cloud.compute; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; import lombok.experimental.Accessors; @@ -12,6 +13,9 @@ import lombok.experimental.Accessors; public class ComputeNodeSize { @Getter @Setter private ComputeNodeSizeType type; + @JsonIgnore public String getTypeName() { return type.name(); } + + @Getter @Setter private Long id; @Getter @Setter private String name; @Setter private String internalName; public String getInternalName () { return internalName != null ? internalName : name; } @@ -22,6 +26,7 @@ public class ComputeNodeSize { @Getter @Setter private int ssdGB; @Getter @Setter private int hddGB; @Getter @Setter private Integer networkMbps; + @Getter @Setter private Integer transferGB; @Setter private Integer costUnits; public Integer getCostUnits () { return costUnits != null ? costUnits : type != null ? type.getCostUnits() : null; } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java index 14c189da..c840fe19 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java @@ -10,6 +10,9 @@ import bubble.model.cloud.BubbleNode; import bubble.model.cloud.RegionalServiceDriver; import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; public interface ComputeServiceDriver extends CloudServiceDriver, RegionalServiceDriver { @@ -17,6 +20,11 @@ public interface ComputeServiceDriver extends CloudServiceDriver, RegionalServic List getSizes(); ComputeNodeSize getSize(ComputeNodeSizeType type); + OsImage getOs(); + + default Map getSizesMap () { + return getSizes().stream().collect(Collectors.toMap(ComputeNodeSize::getTypeName, Function.identity())); + } BubbleNode start(BubbleNode node) throws Exception; BubbleNode cleanupStart(BubbleNode node) throws Exception; diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java index c5d2d57b..e9878be7 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java @@ -8,13 +8,14 @@ import bubble.cloud.CloudRegion; import bubble.cloud.CloudServiceDriverBase; import bubble.dao.cloud.BubbleNodeDAO; import bubble.model.cloud.BubbleNode; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.http.HttpRequestBean; import org.cobbzilla.util.http.HttpResponseBean; import org.springframework.beans.factory.annotation.Autowired; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -64,8 +65,49 @@ public abstract class ComputeServiceDriverBase public abstract List listNodes() throws IOException; - @Override public List getRegions() { return Arrays.asList(config.getRegions()); } - @Override public List getSizes() { return Arrays.asList(config.getSizes()); } + protected abstract List getCloudRegions(); + protected abstract List getCloudSizes(); + protected abstract List getCloudOsImages(); + + @Getter(lazy=true) private final List regions = initRegions(); + private List initRegions() { + final ArrayList cloudRegions = new ArrayList<>(); + for (CloudRegion configRegion : config.getRegions()) { + final CloudRegion region = getCloudRegions().stream().filter(s -> s.getInternalName().equals(configRegion.getInternalName())).findFirst().orElse(null); + if (region == null) { + log.warn("initRegions: config region not found: "+configRegion.getInternalName()); + } else { + cloudRegions.add(configRegion); + } + } + return cloudRegions; + } + + @Getter(lazy=true) private final List sizes = initSizes(); + private List initSizes() { + final ArrayList cloudSizes = new ArrayList<>(); + for (ComputeNodeSize configSize : config.getSizes()) { + final ComputeNodeSize cloudSize = getCloudSizes().stream().filter(sz -> sz.getInternalName().equals(configSize.getInternalName())).findFirst().orElse(null); + if (cloudSize == null) { + log.warn("initSizes: config region not found: "+configSize.getInternalName()); + } else { + cloudSizes.add(cloudSize + .setName(configSize.getName()) + .setType(configSize.getType())); + } + } + return cloudSizes; + } + + @Getter(lazy=true) private final OsImage os = initOs(); + private OsImage initOs() { + final OsImage os = getCloudOsImages().stream() + .filter(s -> s.getName().equals(config.getOs())) + .findFirst() + .orElse(null); + if (os == null) return die("initOs: os not found: "+config.getOs()); + return os; + } @Override public CloudRegion getRegion(String region) { return getRegions().stream() diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ListResourceParser.java b/bubble-server/src/main/java/bubble/cloud/compute/ListResourceParser.java new file mode 100644 index 00000000..0c6ec884 --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/ListResourceParser.java @@ -0,0 +1,12 @@ +package bubble.cloud.compute; + +import bubble.cloud.compute.ResourceParser; + +import java.util.ArrayList; +import java.util.List; + +public abstract class ListResourceParser implements ResourceParser> { + + @Override public List newResults() { return new ArrayList<>(); } + +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/OsImage.java b/bubble-server/src/main/java/bubble/cloud/compute/OsImage.java new file mode 100644 index 00000000..afc9cc70 --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/OsImage.java @@ -0,0 +1,12 @@ +package bubble.cloud.compute; + +import lombok.*; +import lombok.experimental.Accessors; + +@NoArgsConstructor @Accessors(chain=true) +public class OsImage { + + @Getter @Setter private Long id; + @Getter @Setter private String name; + +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java b/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java index cc230da2..54a25c81 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java @@ -5,9 +5,15 @@ import lombok.Getter; import lombok.Setter; import org.cobbzilla.util.collection.NameAndValue; +import static org.cobbzilla.util.daemon.ZillaRuntime.bool; + public class PackerConfig { @Getter @Setter private NameAndValue[] vars; + + @Getter @Setter private Boolean iterateRegions; + public boolean iterateRegions() { return bool(iterateRegions); } + @Getter @Setter private JsonNode builder; } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java b/bubble-server/src/main/java/bubble/cloud/compute/ResourceParser.java similarity index 85% rename from bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java rename to bubble-server/src/main/java/bubble/cloud/compute/ResourceParser.java index 309901ce..f2a488ed 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ResourceParser.java @@ -1,4 +1,4 @@ -package bubble.cloud.compute.digitalocean; +package bubble.cloud.compute; import com.fasterxml.jackson.databind.JsonNode; diff --git a/bubble-server/src/main/java/bubble/cloud/compute/delegate/DelegatedComputeDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/delegate/DelegatedComputeDriver.java index 197569d2..2eb75538 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/delegate/DelegatedComputeDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/delegate/DelegatedComputeDriver.java @@ -6,10 +6,7 @@ package bubble.cloud.compute.delegate; import bubble.cloud.CloudRegion; import bubble.cloud.DelegatedCloudServiceDriverBase; -import bubble.cloud.compute.ComputeNodeSize; -import bubble.cloud.compute.ComputeNodeSizeType; -import bubble.cloud.compute.ComputeServiceDriver; -import bubble.cloud.compute.PackerImage; +import bubble.cloud.compute.*; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.CloudService; import bubble.notify.compute.ComputeDriverNotification; @@ -53,6 +50,11 @@ public class DelegatedComputeDriver extends DelegatedCloudServiceDriverBase impl .orElse(null); } + @Override public OsImage getOs() { + final BubbleNode delegate = getDelegateNode(); + return notificationService.notifySync(delegate, compute_driver_get_os, notification()); + } + @Override public BubbleNode start(BubbleNode node) throws Exception { final BubbleNode delegate = getDelegateNode(); return notificationService.notifySync(delegate, compute_driver_start, notification(node)); diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java new file mode 100644 index 00000000..ddaaed3f --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java @@ -0,0 +1,25 @@ +package bubble.cloud.compute.digitalocean; + +import bubble.cloud.compute.ComputeNodeSize; +import bubble.cloud.compute.ListResourceParser; +import com.fasterxml.jackson.databind.JsonNode; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class DigitalOceanComputeNodeSizeParser extends ListResourceParser { + + @Override public ComputeNodeSize parse(JsonNode item) { + if (!item.has("slug")) return die("parse: slug not found"); + if (!item.has("vcpus")) return die("parse: vcpus not found"); + if (!item.has("memory")) return die("parse: memory not found"); + if (!item.has("disk")) return die("parse: disk not found"); + if (!item.has("transfer")) return die("parse: transfer not found"); + return new ComputeNodeSize() + .setInternalName(item.get("slug").textValue()) + .setVcpu(item.get("vcpus").intValue()) + .setMemoryMB(item.get("memory").intValue()) + .setSsdGB(item.get("disk").intValue()) + .setTransferGB(1024 * item.get("transfer").intValue()); + } + +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java index 88b684ba..f92b725f 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java @@ -5,9 +5,7 @@ package bubble.cloud.compute.digitalocean; import bubble.cloud.CloudRegion; -import bubble.cloud.compute.ComputeNodeSize; -import bubble.cloud.compute.ComputeServiceDriverBase; -import bubble.cloud.compute.PackerImage; +import bubble.cloud.compute.*; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.BubbleNodeState; import com.fasterxml.jackson.databind.JsonNode; @@ -47,11 +45,9 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase { public static final String TAG_PREFIX_NODE = "node_"; public static final String PACKER_IMAGES_URI = "images?private=true"; - @Getter(lazy=true) private final Set regionSlugs = getResourceSlugs("regions"); - @Getter(lazy=true) private final Set sizeSlugs = getResourceSlugs("sizes"); - @Getter(lazy=true) private final Set imageSlugs = getResourceSlugs("images?type=distribution"); - - private Set getResourceSlugs(String uri) { return getResources(uri, new ResourceSlugParser(uri)); } + @Getter(lazy=true) private final List cloudRegions = getResources("regions", new DigitalOceanRegionParser()); + @Getter(lazy=true) private final List cloudSizes = getResources("sizes", new DigitalOceanComputeNodeSizeParser()); + @Getter(lazy=true) private final List cloudOsImages = getResources("images?type=distribution", new DigitalOceanOsImageParser()); private > C getResources(String uri, ResourceParser parser) { final int qPos = uri.indexOf('?'); @@ -175,21 +171,16 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase { @Override public BubbleNode start(BubbleNode node) throws Exception { - final CloudRegion region = config.getRegion(node.getRegion()); - final ComputeNodeSize size = config.getSize(node.getSize()); - // todo: lookup image based on node installType - final String os = config.getConfig("os"); + final CloudRegion region = getRegions().stream() + .filter(r -> r.getInternalName().equals(node.getRegion())) + .findFirst() + .orElse(null); + if (region == null) return die("start: region not found: " + node.getRegion()); - if (!getRegionSlugs().contains(region.getInternalName())) { - return die("start: region not found: " + region.getInternalName()); - } - if (!getSizeSlugs().contains(size.getInternalName())) { - return die("start: region not found: " + region.getInternalName()); - } - if (!getImageSlugs().contains(os)) { - return die("start: region not found: " + region.getInternalName()); - } + final ComputeNodeSize size = config.getSize(node.getSize()); + // todo: lookup image based on node installType and region + final String os = getOs().getName(); final String sshKeyId = registerSshKey(node); final CreateDropletRequest createRequest = new CreateDropletRequest() @@ -267,7 +258,7 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase { } @Override public List getPackerImages() { - final List images = getResources(PACKER_IMAGES_URI, new ImageParser()); + final List images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getJarSha())); return images == null ? Collections.emptyList() : images; } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java new file mode 100644 index 00000000..ae200794 --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java @@ -0,0 +1,31 @@ +package bubble.cloud.compute.digitalocean; + +import bubble.cloud.compute.ListResourceParser; +import bubble.cloud.compute.OsImage; +import com.fasterxml.jackson.databind.JsonNode; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class DigitalOceanOsImageParser extends ListResourceParser { + + @Override public OsImage parse(JsonNode item) { + final OsImage image = new OsImage(); + if (item.has("id")) { + final JsonNode id = item.get("id"); + if (id.isNumber()) { + image.setId(id.numberValue().longValue()); + } else { + return die("parse: id was not numeric"); + } + } else { + return die("parse: id not found"); + } + if (item.has("name")) { + final JsonNode name = item.get("slug"); + image.setName(name.textValue()); + } else { + return die("parse: name not found"); + } + return image; + } +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanPackerImageParser.java similarity index 80% rename from bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java rename to bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanPackerImageParser.java index 8f3aaeb1..a224eedc 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanPackerImageParser.java @@ -1,6 +1,7 @@ package bubble.cloud.compute.digitalocean; import bubble.cloud.CloudRegion; +import bubble.cloud.compute.ListResourceParser; import bubble.cloud.compute.PackerImage; import com.fasterxml.jackson.databind.JsonNode; @@ -9,9 +10,13 @@ import java.util.List; import static bubble.service.packer.PackerJob.PACKER_IMAGE_PREFIX; -public class ImageParser implements ResourceParser> { +public class DigitalOceanPackerImageParser extends ListResourceParser { - @Override public List newResults() { return new ArrayList<>(); } + private String jarSha; + + public DigitalOceanPackerImageParser (String jarSha) { + this.jarSha = jarSha; + } @Override public boolean allowEmpty() { return true; } @@ -23,6 +28,7 @@ public class ImageParser implements ResourceParser { + + @Override public CloudRegion parse(JsonNode item) { + if (!item.has("name")) return die("parse: name not found"); + if (!item.has("slug")) return die("parse: slug not found"); + return new CloudRegion() + .setName(item.get("name").textValue()) + .setInternalName(item.get("slug").textValue()); + } + +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceSlugParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceSlugParser.java deleted file mode 100644 index 7be8fdf1..00000000 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceSlugParser.java +++ /dev/null @@ -1,24 +0,0 @@ -package bubble.cloud.compute.digitalocean; - -import com.fasterxml.jackson.databind.JsonNode; -import lombok.AllArgsConstructor; - -import java.util.HashSet; -import java.util.Set; - -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.json.JsonUtil.json; - -@AllArgsConstructor -class ResourceSlugParser implements ResourceParser> { - - private final String uri; - - @Override public Set newResults() { return new HashSet<>(); } - - @Override public String parse(JsonNode item) { - final JsonNode slug = item.get("slug"); - if (slug == null) return die("getResourceSlugs("+ uri +"): no 'slug' found in item: "+json(item)); - return slug.textValue(); - } -} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java index 294519f0..9feaa15d 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java @@ -4,8 +4,10 @@ */ package bubble.cloud.compute.ec2; +import bubble.cloud.CloudRegion; import bubble.cloud.compute.ComputeNodeSize; import bubble.cloud.compute.ComputeServiceDriverBase; +import bubble.cloud.compute.OsImage; import bubble.cloud.compute.PackerImage; import bubble.cloud.shared.aws.BubbleAwsCredentialsProvider; import bubble.model.cloud.BubbleNode; @@ -51,6 +53,21 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { @Getter(lazy=true) private final AWSCredentialsProvider ec2credentials = new BubbleAwsCredentialsProvider(cloud, getCredentials()); @Getter(lazy=true) private final Map ec2ClientMap = new HashMap<>(); + @Override protected List getCloudRegions() { + // todo + return null; + } + + @Override protected List getCloudSizes() { + // todo + return null; + } + + @Override protected List getCloudOsImages() { + // todo + return null; + } + private static final ExecutorService perRegionExecutor = fixedPool(8); private AmazonEC2 getEC2Client(final String regionName) { diff --git a/bubble-server/src/main/java/bubble/cloud/compute/local/LocalComputeDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/local/LocalComputeDriver.java index 9fd24db5..6b7a025b 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/local/LocalComputeDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/local/LocalComputeDriver.java @@ -17,6 +17,7 @@ public class LocalComputeDriver extends CloudServiceDriverBase im @Override public List getSizes() { return notSupported("getSizes"); } @Override public ComputeNodeSize getSize(ComputeNodeSizeType type) { return notSupported("getSize"); } + @Override public OsImage getOs() { return notSupported("getOs"); } @Override public List getRegions() { return notSupported("getRegions"); } @Override public CloudRegion getRegion(String region) { return notSupported("getRegion"); } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/mock/MockComputeDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/mock/MockComputeDriver.java index 99bc66ff..22271244 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/mock/MockComputeDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/mock/MockComputeDriver.java @@ -5,10 +5,7 @@ package bubble.cloud.compute.mock; import bubble.cloud.CloudRegion; -import bubble.cloud.compute.ComputeNodeSize; -import bubble.cloud.compute.ComputeNodeSizeType; -import bubble.cloud.compute.ComputeServiceDriverBase; -import bubble.cloud.compute.PackerImage; +import bubble.cloud.compute.*; import bubble.cloud.geoLocation.mock.MockGeoLocationDriver; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.BubbleNodeState; @@ -27,15 +24,17 @@ public class MockComputeDriver extends ComputeServiceDriverBase { private Map nodes = new ConcurrentHashMap<>(); - @Getter private final List regions = singletonList(new CloudRegion() + @Getter private final List cloudRegions = singletonList(new CloudRegion() .setDescription("New York City (mock)") .setName("nyc_mock") .setLocation(MockGeoLocationDriver.MOCK_LOCAION)); - @Getter private final List sizes = singletonList(new ComputeNodeSize() + @Getter private final List cloudSizes = singletonList(new ComputeNodeSize() .setName("standard") .setType(ComputeNodeSizeType.small)); + @Getter private final List cloudOsImages = singletonList(new OsImage().setName("dummy operating system")); + @Override protected String readSshKeyId(HttpResponseBean keyResponse) { return "dummy_ssh_key_id_"+now(); } @Override public String registerSshKey(BubbleNode node) { return readSshKeyId(null); } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrComputeNodeSizeParser.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrComputeNodeSizeParser.java new file mode 100644 index 00000000..a2775788 --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrComputeNodeSizeParser.java @@ -0,0 +1,26 @@ +package bubble.cloud.compute.vultr; + +import bubble.cloud.compute.ComputeNodeSize; +import bubble.cloud.compute.ListResourceParser; +import com.fasterxml.jackson.databind.JsonNode; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class VultrComputeNodeSizeParser extends ListResourceParser { + + @Override public ComputeNodeSize parse(JsonNode item) { + if (!item.has("VPSPLANID")) return die("parse: VPSPLANID not found"); + if (!item.has("name")) return die("parse: name not found"); + if (!item.has("vcpu_count")) return die("parse: vcpu_count not found"); + if (!item.has("ram")) return die("parse: ram not found"); + if (!item.has("disk")) return die("parse: disk not found"); + return new ComputeNodeSize() + .setId(item.get("VPSPLANID").asLong()) + .setInternalName(item.get("name").textValue()) + .setVcpu(item.get("vcpu_count").asInt()) + .setMemoryMB(item.get("ram").asInt()) + .setSsdGB(item.get("disk").asInt()) + .setTransferGB(item.get("bandwidth_gb").asInt()); + } + +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java index f2d1aabd..9c42e269 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java @@ -5,9 +5,7 @@ package bubble.cloud.compute.vultr; import bubble.cloud.CloudRegion; -import bubble.cloud.compute.ComputeNodeSize; -import bubble.cloud.compute.ComputeServiceDriverBase; -import bubble.cloud.compute.PackerImage; +import bubble.cloud.compute.*; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.BubbleNodeState; import com.fasterxml.jackson.databind.JsonNode; @@ -59,28 +57,29 @@ public class VultrDriver extends ComputeServiceDriverBase { public static final String LIST_SERVERS_URL = VULTR_API_BASE + "server/list"; public static final String POLL_SERVER_URL = LIST_SERVERS_URL + "?" + VULTR_SUBID + "="; - @Getter(lazy=true) private static final Map regionMap = getResourceMap(REGIONS_URL); - @Getter(lazy=true) private static final Map plansMap = getResourceMap(PLANS_URL); - @Getter(lazy=true) private static final Map osMap = getResourceMap(OS_URL); + @Getter(lazy=true) private final List cloudRegions = loadCloudResources(REGIONS_URL, new VultrRegionParser()); + @Getter(lazy=true) private final List cloudSizes = loadCloudResources(PLANS_URL, new VultrComputeNodeSizeParser()); + @Getter(lazy=true) private final List cloudOsImages = loadCloudResources(OS_URL, new VultrOsImageParser()); public static final long SERVER_START_POLL_INTERVAL = SECONDS.toMillis(5); public static final long SERVER_START_TIMEOUT = MINUTES.toMillis(10); public static final long SERVER_STOP_TIMEOUT = SECONDS.toMillis(60); public static final long SERVER_STOP_CHECK_INTERVAL = SECONDS.toMillis(5); - private static Map getResourceMap(String uri) { + private static List loadCloudResources(String uri, ResourceParser> parser) { try { final HttpResponseBean response = getResponse(uri); - final JsonNode regionsNode = json(response.getEntityString(), JsonNode.class); - final Map map = new HashMap<>(); - for (Iterator fields = regionsNode.fieldNames(); fields.hasNext(); ) { - final String regionId = fields.next(); - final JsonNode region = regionsNode.get(regionId); - map.put(region.get("name").textValue(), Integer.parseInt(regionId)); + final JsonNode node = json(response.getEntityString(), JsonNode.class); + final List resources = parser.newResults(); + for (Iterator fields = node.fieldNames(); fields.hasNext(); ) { + final String id = fields.next(); + final JsonNode item = node.get(id); + final T obj = parser.parse(item); + if (obj != null) resources.add(obj); } - return map; + return resources; } catch (Exception e) { - return die("getResourceMap("+uri+"): "+e, e); + return die("loadCloudResources("+uri+"): "+e, e); } } @@ -97,14 +96,14 @@ public class VultrDriver extends ComputeServiceDriverBase { final CloudRegion region = config.getRegion(node.getRegion()); final ComputeNodeSize size = config.getSize(node.getSize()); - final Integer regionId = getRegionMap().get(region.getInternalName()); + final Long regionId = getRegion(region.getInternalName()).getId(); if (regionId == null) return die("start: region not found: "+region.getInternalName()); - final Integer planId = getPlansMap().get(size.getInternalName()); + final Long planId = getSize(size.getType()).getId(); if (planId == null) return die("start: plan not found: "+size.getInternalName()); - final Integer osId = getOsMap().get(config.getConfig("os")); - if (osId == null) return die("start: OS not found: "+config.getConfig("os")); + // todo: lookup osId based on installType and region + final Integer osId = getOs().getId().intValue(); // register ssh key, check response final String sshKeyId = registerSshKey(node); diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrOsImageParser.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrOsImageParser.java new file mode 100644 index 00000000..4052825d --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrOsImageParser.java @@ -0,0 +1,19 @@ +package bubble.cloud.compute.vultr; + +import bubble.cloud.compute.ListResourceParser; +import bubble.cloud.compute.OsImage; +import com.fasterxml.jackson.databind.JsonNode; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class VultrOsImageParser extends ListResourceParser { + + @Override public OsImage parse(JsonNode item) { + if (!item.has("OSID")) return die("parse: OSID not found"); + if (!item.has("name")) return die("parse: name not found"); + return new OsImage() + .setId(item.get("OSID").asLong()) + .setName(item.get("name").textValue()); + } + +} diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrRegionParser.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrRegionParser.java new file mode 100644 index 00000000..b552e205 --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrRegionParser.java @@ -0,0 +1,19 @@ +package bubble.cloud.compute.vultr; + +import bubble.cloud.CloudRegion; +import bubble.cloud.compute.ListResourceParser; +import com.fasterxml.jackson.databind.JsonNode; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; + +public class VultrRegionParser extends ListResourceParser { + + @Override public CloudRegion parse(JsonNode item) { + if (!item.has("DCID")) return die("parse: DCID not found"); + if (!item.has("name")) return die("parse: name not found"); + return new CloudRegion() + .setId(item.get("DCID").asLong()) + .setInternalName(item.get("name").textValue()); + } + +} diff --git a/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java b/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java index d7fde96d..deed6b07 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java +++ b/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java @@ -6,6 +6,7 @@ package bubble.model.cloud.notify; import bubble.cloud.CloudRegion; import bubble.cloud.compute.ComputeNodeSize; +import bubble.cloud.compute.OsImage; import bubble.cloud.geoCode.GeoCodeResult; import bubble.cloud.geoLocation.GeoLocation; import bubble.cloud.geoTime.GeoTimeZone; @@ -63,6 +64,7 @@ public enum NotificationType { // delegated compute driver notifications compute_driver_get_sizes (ComputeNodeSize[].class), compute_driver_get_regions (CloudRegion[].class), + compute_driver_get_os (OsImage.class), compute_driver_start (BubbleNode.class), compute_driver_cleanup_start (BubbleNode.class), compute_driver_stop (BubbleNode.class), diff --git a/bubble-server/src/main/java/bubble/notify/compute/NotificationHandler_compute_driver_get_os.java b/bubble-server/src/main/java/bubble/notify/compute/NotificationHandler_compute_driver_get_os.java new file mode 100644 index 00000000..d2725223 --- /dev/null +++ b/bubble-server/src/main/java/bubble/notify/compute/NotificationHandler_compute_driver_get_os.java @@ -0,0 +1,15 @@ +package bubble.notify.compute; + +import bubble.cloud.compute.ComputeServiceDriver; +import bubble.cloud.compute.OsImage; +import bubble.model.cloud.notify.ReceivedNotification; + +public class NotificationHandler_compute_driver_get_os extends NotificationHandler_compute_driver { + + @Override protected OsImage handle(ReceivedNotification n, + ComputeDriverNotification notification, + ComputeServiceDriver compute) { + return compute.getOs(); + } + +} diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index 15c21820..8c0187d6 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -56,7 +56,7 @@ public class PackerJob implements Callable> { public static final String VARIABLES_VAR = "packerVariables"; public static final String BUILD_REGION_VAR = "buildRegion"; public static final String IMAGE_REGIONS_VAR = "imageRegions"; - public static final String BUILDER_VAR = "builder"; + public static final String BUILDERS_VAR = "builders"; @Autowired private BubbleConfiguration configuration; @Autowired private GeoService geoService; @@ -95,6 +95,8 @@ public class PackerJob implements Callable> { final Map ctx = new HashMap<>(); ctx.put("credentials", NameAndValue.toMap(cloud.getCredentials().getParams())); ctx.put("compute", computeDriver); + ctx.put("sizes", computeDriver.getSizesMap()); + ctx.put("os", computeDriver.getOs()); // determine lat/lon to find closest cloud region to perform build in final GeoLocation here = geoService.locate(account.getUuid(), getExternalIp()); @@ -134,7 +136,17 @@ public class PackerJob implements Callable> { final String packerTemplatePath = PACKER_TEMPLATE.replace(INSTALL_TYPE_VAR, installType.name()); final String packerConfigTemplate = stream2string(packerTemplatePath); - ctx.put(BUILDER_VAR, generateBuilder(packerConfig, ctx)); + + final List builderJsons = new ArrayList<>(); + if (packerConfig.iterateRegions()) { + for (CloudRegion region : computeDriver.getRegions()) { + ctx.put("region", region); + builderJsons.add(generateBuilder(packerConfig, ctx)); + } + } else { + builderJsons.add(generateBuilder(packerConfig, ctx)); + } + ctx.put(BUILDERS_VAR, builderJsons); // write packer file final String packerJson = HandlebarsUtil.apply(configuration.getHandlebars(), packerConfigTemplate, ctx, '[', ']'); diff --git a/bubble-server/src/main/resources/models/defaults/cloudService.json b/bubble-server/src/main/resources/models/defaults/cloudService.json index 8a304574..e901498b 100644 --- a/bubble-server/src/main/resources/models/defaults/cloudService.json +++ b/bubble-server/src/main/resources/models/defaults/cloudService.json @@ -195,11 +195,11 @@ "location": {"city": "Amsterdam", "country": "NL", "region": "North Holland", "lat": "52.366667", "lon": "4.9"} }], "sizes": [ - {"name": "small", "type": "small", "internalName": "1024 MB RAM,25 GB SSD,1.00 TB BW", "vcpu": 1, "memoryMB": 1024, "ssdGB": 25}, - {"name": "medium", "type": "medium", "internalName": "2048 MB RAM,55 GB SSD,2.00 TB BW", "vcpu": 1, "memoryMB": 2048, "ssdGB": 55}, - {"name": "large", "type": "large", "internalName": "4096 MB RAM,80 GB SSD,3.00 TB BW", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80} + {"name": "small", "type": "small", "internalName": "1024 MB RAM,25 GB SSD,1.00 TB BW"}, + {"name": "medium", "type": "medium", "internalName": "2048 MB RAM,55 GB SSD,2.00 TB BW"}, + {"name": "large", "type": "large", "internalName": "4096 MB RAM,80 GB SSD,3.00 TB BW"} ], - "config": [{"name": "os", "value": "Ubuntu 18.04 x64"}], + "os": "Ubuntu 18.04 x64", "packer": { "vars": [{"name": "VULTR_API_KEY", "value": "[[credentials.apiKey]]"}], "iterateRegions": true, @@ -229,7 +229,8 @@ "type": "compute", "driverClass": "bubble.cloud.compute.digitalocean.DigitalOceanDriver", "driverConfig": { - "regions": [{ + "regions": [ + { "name": "DigitalOcean - New York City 1", "internalName": "nyc1", "location": {"city": "New York", "country": "US", "region": "NY", "lat": "40.661", "lon": "-73.944"} @@ -267,11 +268,11 @@ "location": {"city": "Bangalore", "country": "IN", "region": "Karnataka", "lat": "12.983333", "lon": "77.583333"} }], "sizes": [ - {"name": "small", "type": "small", "internalName": "s-1vcpu-1gb", "vcpu": 1, "memoryMB": 1024, "ssdGB": 25}, - {"name": "medium", "type": "medium", "internalName": "s-1vcpu-2gb", "vcpu": 1, "memoryMB": 2048, "ssdGB": 50}, - {"name": "large", "type": "large", "internalName": "s-2vcpu-4gb", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80} + {"name": "small", "type": "small", "internalName": "s-1vcpu-1gb"}, + {"name": "medium", "type": "medium", "internalName": "s-1vcpu-2gb"}, + {"name": "large", "type": "large", "internalName": "s-2vcpu-4gb"} ], - "config": [{"name": "os", "value": "ubuntu-18-04-x64"}], + "os": "ubuntu-18-04-x64", "packer": { "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "[[credentials.apiKey]]"}], "builder": { @@ -280,7 +281,7 @@ "api_token": "[[user `DIGITALOCEAN_API_KEY`]]", "image": "<>", "region": "<>", - "size": "<>", + "size": "<>", "ipv6": true, "snapshot_name": "<>", "snapshot_regions": ["<<>>"] diff --git a/bubble-server/src/main/resources/packer/packer-sage.json.hbs b/bubble-server/src/main/resources/packer/packer-sage.json.hbs index c6df33fc..c2f30752 100644 --- a/bubble-server/src/main/resources/packer/packer-sage.json.hbs +++ b/bubble-server/src/main/resources/packer/packer-sage.json.hbs @@ -1,10 +1,11 @@ { "variables": { - [[#each packerVariables]]"[[name]]": "{{env `[[name]]`}}"[[#unless @last]], +[[#each packerVariables]] "[[name]]": "{{env `[[name]]`}}"[[#unless @last]], [[/unless]][[/each]] }, "builders": [ - [[[builder]]] +[[#each builders]] [[[this]]][[#unless @last]], +[[/unless]][[/each]] ], "provisioners": [ {