add packer endpoints, introduce packer support to cloud compute drivers

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -44,6 +44,10 @@ public class ApiConstants {
     public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "bubble_finalizer"};
     public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "bubble_finalizer"};
 
+    public static final String PACKER_IMAGE_TAG = "packer_bubble";
+    public static final String PACKER_IMAGE_TAG_SAGE = "packer_bubble_sage";
+    public static final String PACKER_IMAGE_TAG_NODE = "packer_bubble_node";
+
     private static final AtomicReference<String> bubbleDefaultDomain = new AtomicReference<>();
 
     public static final ObjectMapper DB_JSON_MAPPER = COMPACT_MAPPER;
@@ -166,6 +170,7 @@ public class ApiConstants {
     public static final String EP_RECEIVED_NOTIFICATIONS = "/notifications/inbox";
     public static final String EP_REFERRAL_CODES = "/referralCodes";
     public static final String EP_STORAGE = "/storage";
+    public static final String EP_PACKER = "/packer";
     public static final String EP_DNS = "/dns";
     public static final String EP_BACKUPS = "/backups";
     public static final String EP_FIND_DNS = "/find";

bubble-server/src/main/java/bubble/cloud/CloudRegion.java

@@ -14,6 +14,8 @@ import static java.util.UUID.randomUUID;
 @Accessors(chain=true)
 public class CloudRegion {
 
+    public static final CloudRegion[] EMPTY_REGIONS = new CloudRegion[0];
+
     @Getter @Setter private String uuid = randomUUID().toString();
 
     @Getter @Setter private String cloud;

bubble-server/src/main/java/bubble/cloud/NoopCloud.java

@@ -9,6 +9,7 @@ import bubble.cloud.auth.RenderedMessage;
 import bubble.cloud.compute.ComputeNodeSize;
 import bubble.cloud.compute.ComputeNodeSizeType;
 import bubble.cloud.compute.ComputeServiceDriver;
+import bubble.cloud.compute.PackerImage;
 import bubble.cloud.dns.DnsServiceDriver;
 import bubble.cloud.email.EmailServiceDriver;
 import bubble.cloud.email.RenderedEmail;
@@ -38,6 +39,7 @@ import org.cobbzilla.util.dns.DnsRecordMatch;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
 @Slf4j
@@ -69,6 +71,9 @@ public class NoopCloud implements
         return false;
     }
 
+    @Override public List<PackerImage> getPackerImages() { return Collections.emptyList(); }
+    @Override public List<PackerImage> writePackerImages() { return Collections.emptyList(); }
+
     @Override public boolean _write(String fromNode, String key, InputStream data, StorageMetadata metadata, String requestId) throws IOException {
         if (log.isDebugEnabled()) log.debug("_write(fromNode=" + fromNode + ")");
         return false;

bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java

@@ -25,4 +25,7 @@ public interface ComputeServiceDriver extends CloudServiceDriver, RegionalServic
 
     @Override default boolean test () { return true; }
 
+    List<PackerImage> getPackerImages();
+    List<PackerImage> writePackerImages();
+
 }

bubble-server/src/main/java/bubble/cloud/compute/PackerImage.java

@@ -0,0 +1,17 @@
+package bubble.cloud.compute;
+
+import bubble.cloud.CloudRegion;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@NoArgsConstructor @Accessors(chain=true) @EqualsAndHashCode(of={"id"})
+public class PackerImage {
+
+    @Getter @Setter private String id;
+    @Getter @Setter private String name;
+    @Getter @Setter private CloudRegion[] regions;
+
+}

bubble-server/src/main/java/bubble/cloud/compute/delegate/DelegatedComputeDriver.java

@@ -9,6 +9,7 @@ import bubble.cloud.DelegatedCloudServiceDriverBase;
 import bubble.cloud.compute.ComputeNodeSize;
 import bubble.cloud.compute.ComputeNodeSizeType;
 import bubble.cloud.compute.ComputeServiceDriver;
+import bubble.cloud.compute.PackerImage;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.CloudService;
 import bubble.notify.compute.ComputeDriverNotification;
@@ -17,6 +18,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import static bubble.model.cloud.notify.NotificationType.*;
+import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported;
 
 public class DelegatedComputeDriver extends DelegatedCloudServiceDriverBase implements ComputeServiceDriver {
 
@@ -70,4 +72,8 @@ public class DelegatedComputeDriver extends DelegatedCloudServiceDriverBase impl
         final BubbleNode delegate = getDelegateNode();
         return notificationService.notifySync(delegate, compute_driver_status, notification(node));
     }
+
+    @Override public List<PackerImage> getPackerImages() { return notSupported("getPackerImages"); }
+    @Override public List<PackerImage> writePackerImages() { return notSupported("writePackerImages"); }
+
 }

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java

@@ -4,9 +4,11 @@
  */
 package bubble.cloud.compute.digitalocean;
 
+import bubble.ApiConstants;
 import bubble.cloud.CloudRegion;
 import bubble.cloud.compute.ComputeNodeSize;
 import bubble.cloud.compute.ComputeServiceDriverBase;
+import bubble.cloud.compute.PackerImage;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.BubbleNodeState;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -44,34 +46,33 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase {
 
     public static final String TAG_PREFIX_CLOUD = "cloud_";
     public static final String TAG_PREFIX_NODE = "node_";
+    public static final String PACKER_IMAGES_URI = "images?tag=" + ApiConstants.PACKER_IMAGE_TAG;
 
     @Getter(lazy=true) private final Set<String> regionSlugs = getResourceSlugs("regions");
     @Getter(lazy=true) private final Set<String> sizeSlugs = getResourceSlugs("sizes");
     @Getter(lazy=true) private final Set<String> imageSlugs = getResourceSlugs("images?type=distribution");
 
-    private Set<String> getResourceSlugs(String uri) {
+    private Set<String> getResourceSlugs(String uri) { return getResources(uri, new ResourceSlugParser(uri)); }
+
+    private <E, C extends Collection<E>> C getResources(String uri, ResourceParser<E, C> parser) {
         final int qPos = uri.indexOf('?');
         final String type = qPos == -1 ? uri : uri.substring(0, qPos);
         final JsonNode found = doGet(uri, JsonNode.class);
-        final Set<String> slugs = new HashSet<>();
+        final C results = parser.newResults();
 
         JsonNode page = found;
         do {
             final JsonNode items = page.has(type) ? page.get(type) : null;
             if (empty(items) || !items.isArray()) return die("getResourceSlugs("+uri+"): expected "+type+" property to contain a (non-empty) array");
 
-            for (int i=0; i<items.size(); i++) {
-                final JsonNode slug = items.get(i).get("slug");
-                if (slug == null) return die("getResourceSlugs("+uri+"): no 'slug' found in item: "+json(items.get(i)));
-                slugs.add(slug.textValue());
-            }
+            for (int i=0; i<items.size(); i++) results.add(parser.parse(items.get(i)));
 
             final String next = getNext(page);
             page = next == null ? null : doGet(next, JsonNode.class, false);
 
         } while (page != null);
 
-        return slugs;
+        return results;
     }
 
     private String getNext(JsonNode node) {
@@ -258,4 +259,12 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase {
         return node.setState(found.get(0).getState());
     }
 
+    @Override public List<PackerImage> getPackerImages() {
+        return getResources(PACKER_IMAGES_URI, new ImageParser());
+    }
+
+    @Override public List<PackerImage> writePackerImages() {
+        return null;
+    }
+
 }

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java

@@ -0,0 +1,32 @@
+package bubble.cloud.compute.digitalocean;
+
+import bubble.cloud.CloudRegion;
+import bubble.cloud.compute.PackerImage;
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ImageParser implements ResourceParser<PackerImage, List<PackerImage>> {
+
+    @Override public List<PackerImage> newResults() { return new ArrayList<>(); }
+
+    @Override public PackerImage parse(JsonNode item) {
+        final PackerImage image = new PackerImage();
+        if (item.has("id")) image.setId(item.get("id").textValue());
+        if (item.has("name")) image.setName(item.get("name").textValue());
+        if (item.has("regions")) {
+            final JsonNode regionsNode = item.get("regions");
+            if (regionsNode.isArray()) {
+                final List<CloudRegion> regions = new ArrayList<>();
+                for (int i=0; i<regionsNode.size(); i++) {
+                    final String regionName = regionsNode.get(i).textValue();
+                    regions.add(new CloudRegion().setInternalName(regionName));
+                }
+                image.setRegions(regions.toArray(CloudRegion.EMPTY_REGIONS));
+            }
+        }
+        return image;
+    }
+
+}

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java

@@ -0,0 +1,13 @@
+package bubble.cloud.compute.digitalocean;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import java.util.Collection;
+
+public interface ResourceParser<E, C extends Collection<E>> {
+
+    C newResults();
+
+    E parse(JsonNode item);
+
+}

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceSlugParser.java

@@ -0,0 +1,24 @@
+package bubble.cloud.compute.digitalocean;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.AllArgsConstructor;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.json.JsonUtil.json;
+
+@AllArgsConstructor
+class ResourceSlugParser implements ResourceParser<String, Set<String>> {
+
+    private final String uri;
+
+    @Override public Set<String> newResults() { return new HashSet<>(); }
+
+    @Override public String parse(JsonNode item) {
+        final JsonNode slug = item.get("slug");
+        if (slug == null) return die("getResourceSlugs("+ uri +"): no 'slug' found in item: "+json(item));
+        return slug.textValue();
+    }
+}

bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java

@@ -6,6 +6,7 @@ package bubble.cloud.compute.ec2;
 
 import bubble.cloud.compute.ComputeNodeSize;
 import bubble.cloud.compute.ComputeServiceDriverBase;
+import bubble.cloud.compute.PackerImage;
 import bubble.cloud.shared.aws.BubbleAwsCredentialsProvider;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.BubbleNodeState;
@@ -31,8 +32,8 @@ import java.util.concurrent.TimeUnit;
 import static bubble.model.cloud.BubbleNode.*;
 import static org.cobbzilla.util.daemon.Await.awaitAll;
 import static org.cobbzilla.util.daemon.DaemonThreadFactory.fixedPool;
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported;
 import static org.cobbzilla.util.http.HttpStatusCodes.OK;
 import static org.cobbzilla.util.security.RsaKeyPair.newRsaKeyPair;
 import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
@@ -308,4 +309,9 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase {
         }
         return importKeyPairResult.getKeyFingerprint();
     }
+
+    // todo
+    @Override public List<PackerImage> getPackerImages() { return notSupported("getPackerImages"); }
+    @Override public List<PackerImage> writePackerImages() { return notSupported("writePackerImages"); }
+
 }

bubble-server/src/main/java/bubble/cloud/compute/local/LocalComputeDriver.java

@@ -6,10 +6,7 @@ package bubble.cloud.compute.local;
 
 import bubble.cloud.CloudRegion;
 import bubble.cloud.CloudServiceDriverBase;
-import bubble.cloud.compute.ComputeConfig;
-import bubble.cloud.compute.ComputeNodeSize;
-import bubble.cloud.compute.ComputeNodeSizeType;
-import bubble.cloud.compute.ComputeServiceDriver;
+import bubble.cloud.compute.*;
 import bubble.model.cloud.BubbleNode;
 
 import java.util.List;
@@ -29,4 +26,7 @@ public class LocalComputeDriver extends CloudServiceDriverBase<ComputeConfig> im
     @Override public BubbleNode stop(BubbleNode node) throws Exception { return notSupported("stop"); }
     @Override public BubbleNode status(BubbleNode node) throws Exception { return notSupported("status"); }
 
+    @Override public List<PackerImage> getPackerImages() { return notSupported("getPackerImages"); }
+    @Override public List<PackerImage> writePackerImages() { return notSupported("writePackerImages"); }
+
 }

bubble-server/src/main/java/bubble/cloud/compute/mock/MockComputeDriver.java

@@ -8,6 +8,7 @@ import bubble.cloud.CloudRegion;
 import bubble.cloud.compute.ComputeNodeSize;
 import bubble.cloud.compute.ComputeNodeSizeType;
 import bubble.cloud.compute.ComputeServiceDriverBase;
+import bubble.cloud.compute.PackerImage;
 import bubble.cloud.geoLocation.mock.MockGeoLocationDriver;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.BubbleNodeState;
@@ -68,4 +69,7 @@ public class MockComputeDriver extends ComputeServiceDriverBase {
         return node;
     }
 
+    @Override public List<PackerImage> getPackerImages() { return Collections.emptyList(); }
+    @Override public List<PackerImage> writePackerImages() { return Collections.emptyList(); }
+
 }

bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java

@@ -7,6 +7,7 @@ package bubble.cloud.compute.vultr;
 import bubble.cloud.CloudRegion;
 import bubble.cloud.compute.ComputeNodeSize;
 import bubble.cloud.compute.ComputeServiceDriverBase;
+import bubble.cloud.compute.PackerImage;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.BubbleNodeState;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -366,4 +367,9 @@ public class VultrDriver extends ComputeServiceDriverBase {
             }
         }
     }
+
+    // todo
+    @Override public List<PackerImage> getPackerImages() { return notSupported("getPackerImages"); }
+    @Override public List<PackerImage> writePackerImages() { return notSupported("writePackerImages"); }
+
 }

bubble-server/src/main/java/bubble/resources/cloud/CloudServicesResource.java

@@ -18,8 +18,7 @@ import javax.ws.rs.core.Context;
 import java.util.List;
 import java.util.Map;
 
-import static bubble.ApiConstants.EP_DATA;
-import static bubble.ApiConstants.EP_STORAGE;
+import static bubble.ApiConstants.*;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.http.URIUtil.queryParams;
 import static org.cobbzilla.wizard.resources.ResourceUtil.*;
@@ -61,4 +60,14 @@ public class CloudServicesResource extends AccountOwnedResource<CloudService, Cl
         return configuration.subResource(StorageResource.class, account, cloud);
     }
 
+    @Path("/{id}"+EP_PACKER)
+    public ComputePackerResource getPacker(@Context ContainerRequest ctx,
+                                           @PathParam("id") String id) {
+        final Account caller = userPrincipal(ctx);
+        final CloudService cloud = find(ctx, id);
+        if (cloud == null) throw notFoundEx(id);
+        if (cloud.getType() != CloudServiceType.compute) throw invalidEx("err.cloudServiceType.invalid");
+        return configuration.subResource(ComputePackerResource.class, caller, cloud);
+    }
+
 }

bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java

@@ -0,0 +1,49 @@
+package bubble.resources.cloud;
+
+import bubble.cloud.compute.ComputeServiceDriver;
+import bubble.model.account.Account;
+import bubble.model.cloud.CloudService;
+import bubble.server.BubbleConfiguration;
+import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.jersey.server.ContainerRequest;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+import static org.cobbzilla.wizard.resources.ResourceUtil.ok;
+
+@Consumes(APPLICATION_JSON)
+@Produces(APPLICATION_JSON)
+public class ComputePackerResource {
+
+    @Autowired private BubbleConfiguration configuration;
+
+    private Account account;
+    private CloudService cloud;
+
+    public ComputePackerResource (Account account, CloudService cloud) {
+        this.account = account;
+        this.cloud = cloud;
+    }
+
+    @GET
+    public Response listImages(@Context Request req,
+                               @Context ContainerRequest ctx) {
+        final ComputeServiceDriver driver = cloud.getComputeDriver(configuration);
+        return ok(driver.getPackerImages());
+    }
+
+    @PUT
+    public Response ensureImagesExist(@Context Request req,
+                                      @Context ContainerRequest ctx) {
+        final ComputeServiceDriver driver = cloud.getComputeDriver(configuration);
+        return ok(driver.writePackerImages());
+    }
+
+}

bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java

@@ -147,8 +147,4 @@ public class AnsiblePrepService {
         return ctx;
     }
 
-    public static void main (String[] args) {
-        final TempDir tempDir = copyClasspathDirectory("ansible/roles/common");
-        log.info("tempDir="+abs(tempDir));
-    }
 }