diff --git a/bin/aws/config.template b/bin/aws/config.template
new file mode 100644
index 00000000..720f4f85
--- /dev/null
+++ b/bin/aws/config.template
@@ -0,0 +1,2 @@
+[default]
+region = __REGION__
diff --git a/bin/aws/delete_subnets.sh b/bin/aws/delete_subnets.sh
new file mode 100755
index 00000000..4a9f0fb9
--- /dev/null
+++ b/bin/aws/delete_subnets.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+function die {
+ echo 1>&2 "${1}"
+ exit 1
+}
+
+THISDIR=$(cd $(dirname ${0}) && pwd)
+for region in $(${THISDIR}/list_regions.sh) ; do
+ echo "Deleting subnets in region ${region}"
+ ${THISDIR}/set_aws_region.sh ${region} || die "Error setting aws region ${region}"
+ for subnet in $(aws ec2 describe-subnets --filters "Name=default-for-az,Values=false" | grep SubnetId | cut -d\" -f4) ; do
+ echo "Deleting subnet ${subnet} in region ${region}"
+ aws ec2 delete-subnet --subnet-id ${subnet} || echo "WARNING: Error deleting subnet ${subnet} in region ${region}"
+ done
+done
diff --git a/bin/compute/ec2/delete_test_instances.sh b/bin/aws/delete_test_instances.sh
similarity index 100%
rename from bin/compute/ec2/delete_test_instances.sh
rename to bin/aws/delete_test_instances.sh
diff --git a/bin/aws/init_aws_configs.sh b/bin/aws/init_aws_configs.sh
new file mode 100755
index 00000000..703d4550
--- /dev/null
+++ b/bin/aws/init_aws_configs.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+THISDIR=$(cd $(dirname ${0}) && pwd)
+for region in $(${THISDIR}/list_regions.sh) ; do
+ cat ${THISDIR}/config.template > ~/.aws/config.${region} && echo "created config for region ${region}"
+done
diff --git a/bin/aws/list_regions.sh b/bin/aws/list_regions.sh
new file mode 100755
index 00000000..1826ede9
--- /dev/null
+++ b/bin/aws/list_regions.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+aws ec2 describe-regions --filters "Name=opt-in-status,Values=opt-in-not-required" | grep RegionName | cut -d\" -f4 | sort
diff --git a/bin/compute/ec2/list_test_instances.sh b/bin/aws/list_test_instances.sh
similarity index 100%
rename from bin/compute/ec2/list_test_instances.sh
rename to bin/aws/list_test_instances.sh
diff --git a/bin/aws/set_aws_region.sh b/bin/aws/set_aws_region.sh
new file mode 100755
index 00000000..ad4574a1
--- /dev/null
+++ b/bin/aws/set_aws_region.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+function die {
+ echo 1>&2 "${1}"
+ exit 1
+}
+
+region=${1:?no region specified}
+if [[ ! -f ~/.aws/config.${region} ]] ; then
+ die "Region not found: ${region}"
+fi
+rm -f ~/.aws/config && ln -s ~/.aws/config.${region} ~/.aws/config
diff --git a/bubble-server/pom.xml b/bubble-server/pom.xml
index f711b6a4..387ed25c 100644
--- a/bubble-server/pom.xml
+++ b/bubble-server/pom.xml
@@ -106,11 +106,10 @@
jetty-proxy
${jetty.version}
-
com.amazonaws
aws-java-sdk-ec2
- 1.11.762
+ 1.11.797
diff --git a/bubble-server/src/main/java/bubble/cloud/NoopCloud.java b/bubble-server/src/main/java/bubble/cloud/NoopCloud.java
index 06a15837..f6a7de16 100644
--- a/bubble-server/src/main/java/bubble/cloud/NoopCloud.java
+++ b/bubble-server/src/main/java/bubble/cloud/NoopCloud.java
@@ -36,7 +36,6 @@ import org.cobbzilla.util.dns.DnsRecordMatch;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -69,7 +68,9 @@ public class NoopCloud implements
return false;
}
- @Override public List getPackerImages() { return Collections.emptyList(); }
+ @Override public List getAllPackerImages() { return null; }
+
+ @Override public List getPackerImagesForRegion(String region) { return null; }
@Override public boolean _write(String fromNode, String key, InputStream data, StorageMetadata metadata, String requestId) throws IOException {
if (log.isDebugEnabled()) log.debug("_write(fromNode=" + fromNode + ")");
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeDiskType.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeDiskType.java
new file mode 100644
index 00000000..ed4857ba
--- /dev/null
+++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeDiskType.java
@@ -0,0 +1,7 @@
+package bubble.cloud.compute;
+
+public enum ComputeDiskType {
+
+ ssd, hdd, ebs_gp2, ebs_magnetic;
+
+}
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java
index e16b1d34..75a4ad4e 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeNodeSize.java
@@ -23,8 +23,8 @@ public class ComputeNodeSize {
@Getter @Setter private String description;
@Getter @Setter private int vcpu;
@Getter @Setter private int memoryMB;
- @Getter @Setter private int ssdGB;
- @Getter @Setter private int hddGB;
+ @Getter @Setter private int diskGB;
+ @Getter @Setter private ComputeDiskType diskType;
@Getter @Setter private Integer networkMbps;
@Getter @Setter private Integer transferGB;
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java
index c9eec8dd..a54f988a 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java
@@ -4,6 +4,7 @@
*/
package bubble.cloud.compute;
+import bubble.cloud.CloudRegion;
import bubble.cloud.CloudServiceDriver;
import bubble.cloud.CloudServiceType;
import bubble.model.cloud.AnsibleInstallType;
@@ -35,7 +36,12 @@ public interface ComputeServiceDriver extends CloudServiceDriver, RegionalServic
@Override default boolean test () { return true; }
- List getPackerImages();
- default List finalizeIncompletePackerRun(CommandResult commandResult, AnsibleInstallType installType, String jarSha) { return null; }
+ List getAllPackerImages();
+ List getPackerImagesForRegion(String region);
+ default List finalizeIncompletePackerRun(CommandResult commandResult, AnsibleInstallType installType) { return null; }
+
+ default Map getPackerRegionContext(CloudRegion region) { return null; }
+
+ default int getPackerParallelBuilds() { return 1; }
}
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java
index 3b0e1055..53e0b60a 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java
@@ -7,6 +7,7 @@ package bubble.cloud.compute;
import bubble.cloud.CloudRegion;
import bubble.cloud.CloudServiceDriverBase;
import bubble.dao.cloud.BubbleNodeDAO;
+import bubble.model.cloud.AnsibleInstallType;
import bubble.model.cloud.BubbleNode;
import bubble.service.packer.PackerService;
import lombok.Getter;
@@ -18,13 +19,19 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
+import static org.cobbzilla.util.system.Sleep.sleep;
@Slf4j
public abstract class ComputeServiceDriverBase
extends CloudServiceDriverBase
implements ComputeServiceDriver {
+ public static final long PACKER_TIMEOUT = MINUTES.toMillis(60);
+
private final AtomicReference reaper = new AtomicReference<>();
@Override public void postSetup() {
@@ -83,7 +90,7 @@ public abstract class ComputeServiceDriverBase
}
@Getter(lazy=true) private final OsImage os = initOs();
- private OsImage initOs() {
+ protected OsImage initOs() {
final OsImage os = getCloudOsImages().stream()
.filter(s -> s.getName().equals(config.getOs()))
.findFirst()
@@ -104,4 +111,32 @@ public abstract class ComputeServiceDriverBase
.findAny().orElse(null);
}
+ public PackerImage getPackerImage(BubbleNode node) {
+ PackerImage packerImage = getPackerImage(node.getInstallType(), node.getRegion());
+ if (packerImage == null) {
+ final AtomicReference> imagesRef = new AtomicReference<>();
+ packerService.writePackerImages(cloud, node.getInstallType(), imagesRef);
+ long start = now();
+ while (imagesRef.get() == null && now() - start < PACKER_TIMEOUT) {
+ sleep(SECONDS.toMillis(1), "getPackerImage: waiting for packer image creation");
+ }
+ if (imagesRef.get() == null) {
+ return die("getPackerImage: timeout creating packer image");
+ }
+ packerImage = getPackerImage(node.getInstallType(), node.getRegion());
+ if (packerImage == null) {
+ return die("getPackerImage: error creating packer image");
+ }
+ }
+ return packerImage;
+ }
+
+ public PackerImage getPackerImage(AnsibleInstallType installType, String region) {
+ final List images = getPackerImagesForRegion(region);
+ return images == null ? null : images.stream()
+ .filter(i -> i.getName().contains("_"+installType.name()+"_"))
+ .findFirst()
+ .orElse(null);
+ }
+
}
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/OsImage.java b/bubble-server/src/main/java/bubble/cloud/compute/OsImage.java
index afc9cc70..274e24e1 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/OsImage.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/OsImage.java
@@ -1,12 +1,14 @@
package bubble.cloud.compute;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.*;
import lombok.experimental.Accessors;
@NoArgsConstructor @Accessors(chain=true)
public class OsImage {
- @Getter @Setter private Long id;
+ @Getter @Setter private String id;
@Getter @Setter private String name;
+ @Getter @Setter private String region;
}
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/PackerImage.java b/bubble-server/src/main/java/bubble/cloud/compute/PackerImage.java
index 2c60a9ef..e456e245 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/PackerImage.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/PackerImage.java
@@ -7,7 +7,7 @@ import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.Accessors;
-@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of={"id"})
+@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of={"id", "regions"})
public class PackerImage {
@Getter @Setter private String id;
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java b/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java
index 73f9b568..0e1c41c1 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java
@@ -6,19 +6,16 @@ public abstract class PackerImageParserBase extends ListResourceParser getPackerImages() { return notSupported("getPackerImages"); }
+ @Override public List getAllPackerImages() { return notSupported("getPackerImages"); }
+ @Override public List getPackerImagesForRegion(String region) { return notSupported("getPackerImagesForRegion"); }
+
}
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java
index ddaaed3f..50dcaf7b 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanComputeNodeSizeParser.java
@@ -1,5 +1,6 @@
package bubble.cloud.compute.digitalocean;
+import bubble.cloud.compute.ComputeDiskType;
import bubble.cloud.compute.ComputeNodeSize;
import bubble.cloud.compute.ListResourceParser;
import com.fasterxml.jackson.databind.JsonNode;
@@ -18,7 +19,8 @@ public class DigitalOceanComputeNodeSizeParser extends ListResourceParser getPackerImages() {
- final List images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getVersion(), packerService.getPackerPublicKeyHash(), configuration.getJarSha()));
+ @Override public List getAllPackerImages() { return getPackerImages(); }
+ @Override public List getPackerImagesForRegion(String region) { return getPackerImages(); }
+
+ public List getPackerImages () {
+ final List images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getVersion(), packerService.getPackerPublicKeyHash()));
return images == null ? Collections.emptyList() : images;
}
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java
index ae200794..e87b2c32 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanOsImageParser.java
@@ -13,7 +13,7 @@ public class DigitalOceanOsImageParser extends ListResourceParser {
if (item.has("id")) {
final JsonNode id = item.get("id");
if (id.isNumber()) {
- image.setId(id.numberValue().longValue());
+ image.setId(id.asText());
} else {
return die("parse: id was not numeric");
}
@@ -22,7 +22,7 @@ public class DigitalOceanOsImageParser extends ListResourceParser {
}
if (item.has("name")) {
final JsonNode name = item.get("slug");
- image.setName(name.textValue());
+ image.setName(name.asText());
} else {
return die("parse: name not found");
}
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanPackerImageParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanPackerImageParser.java
index 93c58ee0..d28782d4 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanPackerImageParser.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanPackerImageParser.java
@@ -10,8 +10,8 @@ import java.util.List;
public class DigitalOceanPackerImageParser extends PackerImageParserBase {
- public DigitalOceanPackerImageParser (String bubbleVersion, String keyHash, String jarSha) {
- super(bubbleVersion, keyHash, jarSha);
+ public DigitalOceanPackerImageParser (String bubbleVersion, String keyHash) {
+ super(bubbleVersion, keyHash);
}
@Override public boolean allowEmpty() { return true; }
diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java
index 5d8d2130..e34701cc 100644
--- a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java
+++ b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java
@@ -18,109 +18,391 @@ import com.amazonaws.regions.Regions;
import com.amazonaws.services.ec2.AmazonEC2;
import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
import com.amazonaws.services.ec2.model.*;
+import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.collection.ExpirationMap;
+import org.cobbzilla.util.collection.SingletonList;
import org.cobbzilla.util.daemon.AwaitResult;
+import org.cobbzilla.util.reflect.ReflectionUtil;
+import org.cobbzilla.wizard.cache.redis.RedisService;
+import org.springframework.beans.factory.annotation.Autowired;
import java.io.IOException;
import java.util.*;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import static bubble.model.cloud.BubbleNode.TAG_INSTANCE_ID;
import static bubble.model.cloud.BubbleNode.TAG_TEST;
+import static java.util.Comparator.comparing;
+import static java.util.concurrent.TimeUnit.DAYS;
import static org.cobbzilla.util.daemon.Await.awaitAll;
import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool;
import static org.cobbzilla.util.daemon.ZillaRuntime.*;
import static org.cobbzilla.util.http.HttpStatusCodes.OK;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
@Slf4j
public class AmazonEC2Driver extends ComputeServiceDriverBase {
- public static final long TIMEOUT = TimeUnit.SECONDS.toMillis(10);
+ public static final long PARALLEL_TIMEOUT = TimeUnit.SECONDS.toMillis(20);
public static final String TAG_CLOUD_UUID = "cloudUUID";
public static final String TAG_NODE_UUID = "nodeUUID";
public static final String KEY_NAME_PREFIX = "keyName_";
- public static final int MIN_COUNT = 1;
- public static final int MAX_COUNT = 1;
- @Getter(lazy=true) private final AWSCredentialsProvider ec2credentials = new BubbleAwsCredentialsProvider(cloud, getCredentials());
- @Getter(lazy=true) private final Map ec2ClientMap = new HashMap<>();
+ public static final String VPC_CIDR_BLOCK = "10.0.0.0/16";
+ public static final String VPC_IPV6_CIDR_BLOCK = "fdb0:bb00::/48";
+ public static final String TAG_BUBBLE_CLASS = "BUBBLE_CLASS";
+ public static final String TAG_BUBBLE_CLASS_PACKER_VPC = "BUBBLE_PACKER_VPC";
+ public static final String TAG_BUBBLE_CLASS_PACKER_SUBNET = "BUBBLE_PACKER_SUBNET";
+
+ public static final Filter[] VPC_FILTERS = new Filter[]{
+ new Filter("tag:" + TAG_BUBBLE_CLASS, new SingletonList<>(TAG_BUBBLE_CLASS_PACKER_VPC))
+ };
+ public static final Filter[] SUBNET_FILTERS = new Filter[]{
+ new Filter("tag:" + TAG_BUBBLE_CLASS, new SingletonList<>(TAG_BUBBLE_CLASS_PACKER_SUBNET))
+ };
+ public static final String IP4_CIDR_ALL = "0.0.0.0/0";
+ public static final String IP6_CIDR_ALL = "::/0";
+
+ @Autowired private RedisService redis;
- @Override protected List getCloudRegions() {
- // todo
- return null;
+ @Getter(lazy=true) private final String securityGroup = config.getConfig("securityGroup");
+
+ @Getter(lazy=true) private final AWSCredentialsProvider ec2credentials = new BubbleAwsCredentialsProvider(cloud, getCredentials());
+ @Getter(lazy=true) private final Map ec2ClientMap = initClientMap();
+
+ private Map initClientMap() {
+ final Map clients = new HashMap<>();
+ final AWSCredentialsProvider ec2credentials = getEc2credentials();
+ for (CloudRegion region : getCloudRegions()) {
+ final AmazonEC2 ec2 = AmazonEC2ClientBuilder.standard()
+ .withRegion(Regions.fromName(region.getInternalName()))
+ .withCredentials(ec2credentials).build();
+ clients.put(region.getInternalName(), ec2);
+ }
+ return clients;
}
- @Override protected List getCloudSizes() {
- // todo
- return null;
+ @Getter(lazy=true) private final List cloudRegions = driverConfig("regions");
+ @Getter(lazy=true) private final List cloudSizes = driverConfig("sizes");
+
+ private List driverConfig(String field) { return Arrays.asList((T[]) ReflectionUtil.get(config, field)); }
+
+ @Getter(lazy=true) private final RedisService imageCache = redis.prefixNamespace(getClass().getSimpleName()+".ec2_ubuntu_image");
+ public static final long IMAGE_CACHE_TIME = DAYS.toSeconds(30);
+
+ @Getter(lazy=true) private final ExecutorService perRegionExecutor = fixedPool(getRegions().size());
+
+ @Getter(lazy=true) private final List cloudOsImages = initImages();
+ private List initImages() {
+ final ArrayList filters = new ArrayList<>();
+ filters.add(new Filter("root-device-type", new SingletonList<>("ebs")));
+ filters.add(new Filter("state", new SingletonList<>("available")));
+ filters.add(new Filter("name", new SingletonList<>(config.getOs())));
+ final List> futures = new ArrayList<>();
+ for (CloudRegion region : getCloudRegions()) {
+ futures.add(getPerRegionExecutor().submit(() -> {
+ final String internalName = region.getInternalName();
+ final String cachedJson = getImageCache().get(internalName);
+ if (cachedJson != null) {
+ return json(cachedJson, OsImage.class);
+ }
+ final AmazonEC2 ec2 = getEc2Client(region);
+ final DescribeImagesRequest imageRequest = new DescribeImagesRequest().withFilters(filters);
+ final DescribeImagesResult imagesResult = ec2.describeImages(imageRequest);
+ if (empty(imagesResult.getImages())) die("no images found");
+ final List sorted = new ArrayList<>(imagesResult.getImages());
+ sorted.sort(comparing(Image::getCreationDate));
+ final Image first = sorted.get(0);
+ final OsImage image = new OsImage()
+ .setName(first.getName())
+ .setId(first.getImageId())
+ .setRegion(internalName);
+ getImageCache().set(internalName, json(image), EX, IMAGE_CACHE_TIME);
+ return image;
+ }));
+ }
+ final AwaitResult