From a228f7d9e6ebfaa4831e43ca12dc21fd50197841 Mon Sep 17 00:00:00 2001 From: Svitlana Date: Thu, 30 Apr 2020 19:13:23 +0200 Subject: [PATCH] Fix after PR review --- bin/compute/ec2/delete_test_instances.sh | 20 +++ .../ec2/list_test_instances.sh} | 12 +- .../cloud/compute/ec2/AmazonEC2Driver.java | 124 ++++++++++-------- .../java/bubble/model/cloud/BubbleNode.java | 2 +- .../models/defaults/cloudService.json | 4 +- config/activation.json | 2 +- 6 files changed, 101 insertions(+), 63 deletions(-) create mode 100755 bin/compute/ec2/delete_test_instances.sh rename bin/{delete_test_instances.sh => compute/ec2/list_test_instances.sh} (67%) diff --git a/bin/compute/ec2/delete_test_instances.sh b/bin/compute/ec2/delete_test_instances.sh new file mode 100755 index 00000000..c77d36d0 --- /dev/null +++ b/bin/compute/ec2/delete_test_instances.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# +# Terminate EC2 test compute cloud instances +# + +SCRIPT="${0}" +SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd) +. ${SCRIPT_DIR}/list_test_instances.sh + +if [[ ${INSTANCES_COUNT} -gt 0 ]] ; then + + echo "Terminating ${INSTANCES_COUNT} test instances..." + aws ec2 terminate-instances --instance-ids "$INSTANCES" + +else + echo "No instances contains tag \"${TEST_TAG}\" to delete" +fi diff --git a/bin/delete_test_instances.sh b/bin/compute/ec2/list_test_instances.sh similarity index 67% rename from bin/delete_test_instances.sh rename to bin/compute/ec2/list_test_instances.sh index 5ff54efb..1ff1c035 100755 --- a/bin/delete_test_instances.sh +++ b/bin/compute/ec2/list_test_instances.sh @@ -3,7 +3,7 @@ # Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ # # -# Terminate EC2 test compute cloud instances +# List test Amazon EC2 instances # SCRIPT="${0}" @@ -18,15 +18,15 @@ if [[ -z "${TEST_TAG}" ]] ; then fi fi -INSTANCES=$(aws ec2 describe-instances --filters "Name=tag:Test_instance,Values=${TEST_TAG}" --query "Reservations[].Instances[].InstanceId") +INSTANCES=$(aws ec2 describe-instances --filters "Name=tag:test_instance,Values=${TEST_TAG}" --query "Reservations[].Instances[].InstanceId") -INSTANCES_COUNT="${#ARRAY[@]}" +echo "$INSTANCES" +INSTANCES_COUNT="${#INSTANCES[@]}" if [[ ${INSTANCES_COUNT} -gt 0 ]] ; then - echo "Deleting ${INSTANCES_COUNT} test instances..." - aws ec2 terminate-instances --instance-ids "$INSTANCES" + echo "Found ${INSTANCES_COUNT} test instances contain tag ${TEST_TAG}" else - echo "No instances contains tag \"${TEST_TAG_CLOUD}\" to delete" + echo "No instances contain tag \"${TEST_TAG}\" found" fi diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java index ad40aedb..0dff5cbf 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java @@ -21,12 +21,11 @@ import org.cobbzilla.util.http.HttpRequestBean; import org.cobbzilla.util.http.HttpResponseBean; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; +import java.util.*; import static bubble.model.cloud.BubbleNode.*; import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.http.HttpStatusCodes.OK; import static org.cobbzilla.util.security.RsaKeyPair.newRsaKeyPair; import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; @@ -36,28 +35,35 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { public static final String TAG_CLOUD_UUID = "cloudUUID"; public static final String TAG_NODE_UUID = "nodeUUID"; - public static final String KEY_NAME_PREFIX = "KeyName_"; + public static final String KEY_NAME_PREFIX = "keyName_"; public static final int MIN_COUNT = 1; public static final int MAX_COUNT = 1; - @Getter(lazy = true) private final AWSCredentialsProvider ec2credentials = new BubbleAwsCredentialsProvider(cloud, getCredentials()); + @Getter(lazy=true) private final AWSCredentialsProvider ec2credentials = new BubbleAwsCredentialsProvider(cloud, getCredentials()); - @Getter(lazy = true) private final AmazonEC2 ec2Client = initEC2Client(); + @Getter(lazy=true) private final Map ec2ClientMap = new HashMap<>(); - private AmazonEC2 initEC2Client() { - final Regions region; - final String regionName = config.getRegions()[0].getName(); - try { - region = Regions.valueOf(regionName); - } catch (Exception e) { - return die("initEC2Client: invalid region: " + regionName); + private AmazonEC2 getEC2Client(String regionName) { + Map ec2ClientMap = getEc2ClientMap(); + AmazonEC2 ec2client; + if (!ec2ClientMap.containsKey(regionName)) { + final Regions region; + try { + region = Regions.valueOf(regionName); + } catch (Exception e) { + return die("initEC2Client: invalid region: " + regionName); + } + ec2client = AmazonEC2ClientBuilder.standard() + .withRegion(region) + .withCredentials(getEc2credentials()) + .build(); + ec2ClientMap.put(regionName, ec2client); } - return AmazonEC2ClientBuilder.standard() - .withRegion(region) - .withCredentials(getEc2credentials()) - .build(); + ec2client = ec2ClientMap.get(regionName); + return ec2client; } + @Override protected String readSshKeyId(HttpResponseBean keyResponse) { throw new UnsupportedOperationException(); } @@ -69,37 +75,36 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { @Override public List listNodes() throws IOException { final List nodes = new ArrayList<>(); - final AmazonEC2 ec2Client = getEc2Client(); - - DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest() - .withFilters( - new Filter("instance.group-id").withValues(config.getConfig("group")), - new Filter("tag:" + TAG_CLOUD_UUID).withValues(cloud.getUuid()), - new Filter("instance-state-name").withValues("running", "pending") - ); - - DescribeInstancesResult result = ec2Client.describeInstances(describeInstancesRequest); - - if (result.getSdkHttpMetadata().getHttpStatusCode() == OK) { - for (Reservation reservation : result.getReservations()) { - for (Instance instance : reservation.getInstances()) { - final String instanceId = instance.getInstanceId(); - final String ip4 = instance.getPrivateIpAddress(); - final String ip6 = instance.getPublicIpAddress(); - nodes.add(new BubbleNode().setIp4(ip4).setIp6(ip6).setTag(TAG_INSTANCE_ID, instanceId)); + for (AmazonEC2 ec2Client : getEc2ClientMap().values()) { + DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest() + .withFilters( + new Filter("instance.group-id").withValues(config.getConfig("group")), + new Filter("tag:" + TAG_CLOUD_UUID).withValues(cloud.getUuid()), + new Filter("instance-state-name").withValues("running", "pending") + ); + + DescribeInstancesResult result = ec2Client.describeInstances(describeInstancesRequest); + + if (result.getSdkHttpMetadata().getHttpStatusCode() == OK) { + for (Reservation reservation : result.getReservations()) { + for (Instance instance : reservation.getInstances()) { + final String instanceId = instance.getInstanceId(); + final String ip4 = instance.getPrivateIpAddress(); + final String ip6 = instance.getPublicIpAddress(); + nodes.add(new BubbleNode().setIp4(ip4).setIp6(ip6).setTag(TAG_INSTANCE_ID, instanceId)); + } } + } else { + return die("listNode: error listing nodes, status=" + result.getSdkHttpMetadata().getHttpStatusCode() + + ": " + result.getSdkHttpMetadata().getAllHttpHeaders()); } - } else { - return die("listNode: error listing nodes, status=" + result.getSdkHttpMetadata().getHttpStatusCode() - + ": " + result.getSdkHttpMetadata().getAllHttpHeaders()); } return nodes; } @Override public BubbleNode start(BubbleNode node) throws Exception { - final ComputeNodeSize size = config.getSize(node.getSize()); - final AmazonEC2 ec2Client = getEc2Client(); + final AmazonEC2 ec2Client = getEC2Client(node.getRegion()); DescribeSubnetsRequest describeSubnetsRequest = new DescribeSubnetsRequest() .withFilters(new Filter().withName("state").withValues("available")); @@ -142,7 +147,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { for (Instance i : reservation.getInstances()) { if (i.getInstanceId().equals(instanceId)) { final String ip4 = i.getPrivateIpAddress(); - if (ip4 != null && ip4.length() > 0 && !ip4.equals("0.0.0.0")){ + if (ip4 != null && ip4.length() > 0 && !ip4.equals("0.0.0.0")) { node.setIp4(ip4); nodeDAO.update(node); } @@ -155,21 +160,34 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { } } } + } else { + log.error("start: error describe instance, status: " + result.getSdkHttpMetadata().getHttpStatusCode() + + result.getSdkHttpMetadata().getAllHttpHeaders()); } // Setting up the tags for the instance - CreateTagsRequest createTagsRequest = new CreateTagsRequest() - .withResources(instanceId) - .withTags(new Tag(TAG_NODE_UUID, node.getUuid()), - new Tag(TAG_CLOUD_UUID, cloud.getUuid())); - - // Setting up the tag for the test instance - if (configuration.getEnvironment().containsKey("TEST_TAG_CLOUD")){ - CreateTagsRequest createTestTagRequest = new CreateTagsRequest() + try { + ec2Client.createTags(new CreateTagsRequest() .withResources(instanceId) - .withTags(new Tag(TAG_TEST, configuration.getEnvironment().get("TEST_TAG_CLOUD"))); + .withTags(new Tag(TAG_NODE_UUID, node.getUuid()), + new Tag(TAG_CLOUD_UUID, cloud.getUuid()))); + } catch (AmazonServiceException e) { + log.warn("start: error creating tags: " + e.getErrorMessage() + e.getErrorCode()); } + // Setting up the tag for the test instance + if (configuration.testMode()) { + final String testTag = configuration.getEnvironment().get("TEST_TAG_CLOUD"); + if (empty(testTag)) return die("TEST_TAG_CLOUD env var is not defined or is empty"); + + try { + ec2Client.createTags(new CreateTagsRequest() + .withResources(instanceId) + .withTags(new Tag(TAG_TEST, configuration.getEnvironment().get("TEST_TAG_CLOUD")))); + } catch (AmazonServiceException e) { + log.warn("start: error creating test tag: " + e.getErrorMessage() + e.getErrorCode()); + } + } return node; } @@ -190,7 +208,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { StopInstancesRequest stopInstancesRequest = new StopInstancesRequest() .withInstanceIds(instanceID); - final AmazonEC2 ec2Client = getEc2Client(); + final AmazonEC2 ec2Client = getEC2Client(node.getRegion()); try { ec2Client.stopInstances(stopInstancesRequest); @@ -224,7 +242,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { .withKeyName(KEY_NAME_PREFIX + node.getUuid()); // destroy key, check response - final AmazonEC2 ec2Client = getEc2Client(); + final AmazonEC2 ec2Client = getEC2Client(node.getRegion()); DeleteKeyPairResult response = ec2Client.deleteKeyPair(request); if (response.getSdkHttpMetadata().getHttpStatusCode() != OK) { @@ -237,7 +255,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { if (node.hasSshKey()) return die("registerSshKey: node already has a key: "+node.getUuid()); node.setSshKey(newRsaKeyPair()); - final AmazonEC2 ec2Client = getEc2Client(); + final AmazonEC2 ec2Client = getEC2Client(node.getRegion()); ImportKeyPairRequest importKeyPairRequest = new ImportKeyPairRequest( KEY_NAME_PREFIX + node.getUuid(), node.getSshKey().getSshPublicKey() ); diff --git a/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java b/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java index a2c8c409..b53acf50 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java +++ b/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java @@ -55,7 +55,7 @@ public class BubbleNode extends IdentifiableBase implements HasNetwork, HasBubbl public static final String TAG_INSTANCE_ID = "instance_id"; public static final String TAG_SSH_KEY_ID = "ssh_key_id"; public static final String TAG_ERROR = "X-Bubble-Error"; - public static final String TAG_TEST = "Test_instance"; + public static final String TAG_TEST = "test_instance"; private static final List TAG_NAMES = Arrays.asList(TAG_INSTANCE_ID, TAG_SSH_KEY_ID, TAG_ERROR); diff --git a/bubble-server/src/main/resources/models/defaults/cloudService.json b/bubble-server/src/main/resources/models/defaults/cloudService.json index eb5d7737..ed11e97e 100644 --- a/bubble-server/src/main/resources/models/defaults/cloudService.json +++ b/bubble-server/src/main/resources/models/defaults/cloudService.json @@ -281,8 +281,8 @@ {"name": "large", "type": "large", "internalName": "t2.medium", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80} ], "config": [ - {"name": "imageId", "value": "{{IMAGE_ID}}"}, - {"name": "group", "value": "{{EC2_GROUP}}"} + {"name": "imageId", "value": "{{AWS_EC2_IMAGE_ID}}"}, + {"name": "group", "value": "{{EC2_SECURITY_GROUP}}"} ] }, "credentials": { diff --git a/config/activation.json b/config/activation.json index 05c1a6bd..8c29af19 100644 --- a/config/activation.json +++ b/config/activation.json @@ -31,7 +31,7 @@ "AmazonEC2Driver" : { "config": { // region must be a valid value from the Regions enum: https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/regions/Regions.java - "region": "US_EAST_1" + "region": "your_aws_region" }, "credentials" : { "AWS_ACCESS_KEY_ID": "your_aws_access_key_id",