WIP: packer basics working for digitalocean

bin/first_time_ubuntu.sh

@@ -12,9 +12,13 @@ sudo apt update -y || die "Error running apt update"
 sudo apt upgrade -y || die "Error running apt upgrade"
 
 # Install packages
-sudo apt install openjdk-11-jdk maven postgresql-10 redis-server jq python3 python3-pip npm webpack curl -y || die "Error installing apt packages"
+sudo apt install openjdk-11-jdk maven postgresql-10 redis-server jq python3 python3-pip npm webpack curl unzip -y || die "Error installing apt packages"
 sudo pip3 install setuptools psycopg2-binary || die "Error installing pip packages"
 
+# Install packer
+BUBBLE_BIN="$(cd "$(dirname "${0}")" && pwd)"
+"${BUBBLE_BIN}/install_packer.sh" || die "Error installing packer"
+
 # Create DB user for current user, as superuser
 CURRENT_USER="$(whoami)"
 sudo su - postgres bash -c 'createuser -U postgres --createdb --createrole --superuser '"${CURRENT_USER}"'' || die "Error creating ${CURRENT_USER} DB user"

bin/install_packer.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+function die {
+  echo 1>&2 "${1}"
+  exit 1
+}
+
+# Install packer
+if [[ ! -f ${HOME}/packer/packer ]] ; then
+  PACKER_VERSION=1.5.6
+  PACKER_FILE=packer_${PACKER_VERSION}_linux_amd64.zip
+  PACKER_URL=https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_FILE}
+  mkdir -p ${HOME}/packer && cd ${HOME}/packer && wget ${PACKER_URL} && unzip ${PACKER_FILE} || die "Error installing packer"
+else
+  echo "Packer already installed"
+fi
+
+# Install packer Vultr plugin
+if [[ ! -f ${HOME}/.packer.d/plugins/packer-builder-vultr ]] ; then
+  PACKER_VULTR_VERSION=1.0.8
+  PACKER_VULTR_FILE=packer-builder-vultr_${PACKER_VULTR_VERSION}_linux_64-bit.tar.gz
+  PACKER_VULTR_URL=https://github.com/vultr/packer-builder-vultr/releases/download/v${PACKER_VULTR_VERSION}/${PACKER_VULTR_FILE}
+  mkdir -p ${HOME}/.packer.d/plugins && cd ${HOME}/.packer.d/plugins && wget ${PACKER_VULTR_URL} && tar xzf ${PACKER_VULTR_FILE}  || die "Error installing packer vultr plugin"
+else
+  echo "Packer vultr plugin already installed"
+fi

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -17,21 +17,24 @@ import org.glassfish.jersey.server.ContainerRequest;
 
 import javax.ws.rs.core.Context;
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
 import static org.apache.http.HttpHeaders.*;
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-import static org.cobbzilla.util.io.FileUtil.abs;
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.network.NetworkUtil.*;
+import static org.cobbzilla.util.string.StringUtil.splitAndTrim;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 
 @Slf4j
@@ -44,9 +47,9 @@ public class ApiConstants {
     public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "bubble_finalizer"};
     public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "bubble_finalizer"};
 
-    public static final String PACKER_IMAGE_TAG = "packer_bubble";
-    public static final String PACKER_IMAGE_TAG_SAGE = "packer_bubble_sage";
-    public static final String PACKER_IMAGE_TAG_NODE = "packer_bubble_node";
+    public static final String ANSIBLE_DIR = "ansible";
+    public static final List<String> BUBBLE_SCRIPTS = splitAndTrim(stream2string(ANSIBLE_DIR + "/bubble_scripts.txt"), "\n")
+            .stream().filter(s -> !empty(s)).collect(Collectors.toList());
 
     private static final AtomicReference<String> bubbleDefaultDomain = new AtomicReference<>();
 
@@ -291,4 +294,19 @@ public class ApiConstants {
                         invalidEx("err."+e.getSimpleName()+".invalid", "Invalid "+e.getSimpleName()+": "+v, v));
     }
 
+    public static void copyScriptsOrDie(File scriptsParentDir) {
+        try {
+            copyScripts(scriptsParentDir);
+        } catch (IOException e) {
+            die("copyScriptsOrDie: "+shortError(e), e);
+        }
+    }
+
+    public static void copyScripts(File scriptsParentDir) throws IOException {
+        // write scripts
+        final File scriptsDir = mkdirOrDie(new File(scriptsParentDir, "scripts"));
+        for (String script : BUBBLE_SCRIPTS) {
+            toFile(new File(scriptsDir, script), stream2string("scripts/"+script));
+        }
+    }
 }

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java

@@ -4,7 +4,6 @@
  */
 package bubble.cloud.compute.digitalocean;
 
-import bubble.ApiConstants;
 import bubble.cloud.CloudRegion;
 import bubble.cloud.compute.ComputeNodeSize;
 import bubble.cloud.compute.ComputeServiceDriverBase;
@@ -40,13 +39,13 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 @Slf4j
 public class DigitalOceanDriver extends ComputeServiceDriverBase {
 
-    private static final String PARAM_API_KEY = "apiKey";
+    public static final String PARAM_API_KEY = "apiKey";
 
     public static final String DO_API_BASE = "https://api.digitalocean.com/v2/";
 
     public static final String TAG_PREFIX_CLOUD = "cloud_";
     public static final String TAG_PREFIX_NODE = "node_";
-    public static final String PACKER_IMAGES_URI = "images?tag=" + ApiConstants.PACKER_IMAGE_TAG;
+    public static final String PACKER_IMAGES_URI = "images?private=true";
 
     @Getter(lazy=true) private final Set<String> regionSlugs = getResourceSlugs("regions");
     @Getter(lazy=true) private final Set<String> sizeSlugs = getResourceSlugs("sizes");
@@ -63,9 +62,16 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase {
         JsonNode page = found;
         do {
             final JsonNode items = page.has(type) ? page.get(type) : null;
-            if (empty(items) || !items.isArray()) return die("getResourceSlugs("+uri+"): expected "+type+" property to contain a (non-empty) array");
+            if (empty(items)) {
+                if (!parser.allowEmpty()) return die("getResources("+uri+"): expected "+type+" property to contain a (non-empty) array");
+                return null;
+            }
+            if (!items.isArray()) return die("getResources("+uri+"): expected "+type+" property to contain a (non-empty) array");
 
-            for (int i=0; i<items.size(); i++) results.add(parser.parse(items.get(i)));
+            for (int i=0; i<items.size(); i++) {
+                final E item = parser.parse(items.get(i));
+                if (item != null) results.add(item);
+            }
 
             final String next = getNext(page);
             page = next == null ? null : doGet(next, JsonNode.class, false);
@@ -171,6 +177,7 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase {
 
         final CloudRegion region = config.getRegion(node.getRegion());
         final ComputeNodeSize size = config.getSize(node.getSize());
+        // todo: lookup image based on node installType
         final String os = config.getConfig("os");
 
         if (!getRegionSlugs().contains(region.getInternalName())) {
@@ -260,7 +267,8 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase {
     }
 
     @Override public List<PackerImage> getPackerImages() {
-        return getResources(PACKER_IMAGES_URI, new ImageParser());
+        final List<PackerImage> images = getResources(PACKER_IMAGES_URI, new ImageParser());
+        return images == null ? Collections.emptyList() : images;
     }
 
     @Override public List<PackerImage> writePackerImages() {

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java

@@ -7,14 +7,27 @@ import com.fasterxml.jackson.databind.JsonNode;
 import java.util.ArrayList;
 import java.util.List;
 
+import static bubble.service.packer.PackerJob.PACKER_IMAGE_PREFIX;
+
 public class ImageParser implements ResourceParser<PackerImage, List<PackerImage>> {
 
     @Override public List<PackerImage> newResults() { return new ArrayList<>(); }
 
+    @Override public boolean allowEmpty() { return true; }
+
     @Override public PackerImage parse(JsonNode item) {
+
         final PackerImage image = new PackerImage();
+
+        final String name;
+        if (item.has("name")) {
+            name = item.get("name").textValue();
+            if (!name.startsWith(PACKER_IMAGE_PREFIX)) return null;
+            image.setName(name);
+        }
+
         if (item.has("id")) image.setId(item.get("id").textValue());
-        if (item.has("name")) image.setName(item.get("name").textValue());
+
         if (item.has("regions")) {
             final JsonNode regionsNode = item.get("regions");
             if (regionsNode.isArray()) {

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java

@@ -10,4 +10,6 @@ public interface ResourceParser<E, C extends Collection<E>> {
 
     E parse(JsonNode item);
 
+    default boolean allowEmpty () { return false; }
+
 }

bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java

@@ -39,6 +39,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 @Slf4j
 public class VultrDriver extends ComputeServiceDriverBase {
 
+    public static final String PARAM_API_KEY = "apiKey";
     public static final String API_KEY_HEADER = "API-Key";
 
     public static final String VULTR_API_BASE = "https://api.vultr.com/v1/";
@@ -84,9 +85,9 @@ public class VultrDriver extends ComputeServiceDriverBase {
     }
 
     @Override public void postSetup() {
-        if (credentials != null && credentials.hasParam(API_KEY_HEADER) && credentials.getParam(API_KEY_HEADER).contains("{{")) {
-            final String apiKey = configuration.applyHandlebars(credentials.getParam(API_KEY_HEADER));
-            credentials.setParam(API_KEY_HEADER, apiKey);
+        if (credentials != null && credentials.hasParam(PARAM_API_KEY) && credentials.getParam(PARAM_API_KEY).contains("{{")) {
+            final String apiKey = configuration.applyHandlebars(credentials.getParam(PARAM_API_KEY));
+            credentials.setParam(PARAM_API_KEY, apiKey);
         }
         super.postSetup();
     }
@@ -187,7 +188,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
     }
 
     private HttpRequestBean auth(HttpRequestBean req) {
-        return req.setHeader(API_KEY_HEADER, credentials.getParam(API_KEY_HEADER));
+        return req.setHeader(API_KEY_HEADER, credentials.getParam(PARAM_API_KEY));
     }
 
     @Override public BubbleNode stop(BubbleNode node) throws Exception {

bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java

@@ -1,9 +1,11 @@
 package bubble.resources.cloud;
 
 import bubble.cloud.compute.ComputeServiceDriver;
+import bubble.cloud.compute.PackerImage;
 import bubble.model.account.Account;
 import bubble.model.cloud.CloudService;
 import bubble.server.BubbleConfiguration;
+import bubble.service.packer.PackerService;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -14,6 +16,8 @@ import javax.ws.rs.PUT;
 import javax.ws.rs.Produces;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
+import java.util.Collections;
+import java.util.List;
 
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
 import static org.cobbzilla.wizard.resources.ResourceUtil.ok;
@@ -23,6 +27,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.ok;
 public class ComputePackerResource {
 
     @Autowired private BubbleConfiguration configuration;
+    @Autowired private PackerService packer;
 
     private Account account;
     private CloudService cloud;
@@ -40,10 +45,11 @@ public class ComputePackerResource {
     }
 
     @PUT
-    public Response ensureImagesExist(@Context Request req,
-                                      @Context ContainerRequest ctx) {
-        final ComputeServiceDriver driver = cloud.getComputeDriver(configuration);
-        return ok(driver.writePackerImages());
+    public Response writeImages(@Context Request req,
+                                @Context ContainerRequest ctx) {
+        final List<PackerImage> images = packer.writePackerImages(account, cloud);
+        if (images != null) return ok(images);
+        return ok(Collections.emptyList());
     }
 
 }

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -64,6 +64,7 @@ import static org.cobbzilla.util.handlebars.HandlebarsUtil.registerUtilityHelper
 import static org.cobbzilla.util.io.FileUtil.abs;
 import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+import static org.cobbzilla.util.security.ShaUtil.sha256_file;
 
 @Configuration @NoArgsConstructor @Slf4j
 public class BubbleConfiguration extends PgRestServerConfiguration
@@ -160,6 +161,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration
         return bubbleJar;
     }
 
+    @Getter(lazy=true) private final String jarSha = sha256_file(getBubbleJar());
+
     private static final AtomicReference<String> _DEFAULT_LOCALE = new AtomicReference<>();
     public static String getDEFAULT_LOCALE() {
         final String locale = _DEFAULT_LOCALE.get();

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -4,6 +4,7 @@
  */
 package bubble.service.cloud;
 
+import bubble.ApiConstants;
 import bubble.cloud.CloudAndRegion;
 import bubble.cloud.compute.ComputeServiceDriver;
 import bubble.cloud.dns.DnsServiceDriver;
@@ -57,7 +58,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.getRemoteHost;
 import static bubble.ApiConstants.newNodeHostname;
@@ -77,7 +77,6 @@ import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.reflect.ReflectionUtil.closeQuietly;
-import static org.cobbzilla.util.string.StringUtil.splitAndTrim;
 import static org.cobbzilla.util.system.CommandShell.chmod;
 import static org.cobbzilla.util.system.Sleep.sleep;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
@@ -86,16 +85,11 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 @Service @Slf4j
 public class StandardNetworkService implements NetworkService {
 
-    public static final String ANSIBLE_DIR = "ansible";
-
     public static final String PLAYBOOK_YML = "playbook.yml";
-    public static final String PLAYBOOK_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + PLAYBOOK_YML + ".hbs");
+    public static final String PLAYBOOK_TEMPLATE = stream2string(ApiConstants.ANSIBLE_DIR + "/" + PLAYBOOK_YML + ".hbs");
 
     public static final String INSTALL_LOCAL_SH = "install_local.sh";
-    public static final String INSTALL_LOCAL_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH + ".hbs");
-
-    public static final List<String> BUBBLE_SCRIPTS = splitAndTrim(stream2string(ANSIBLE_DIR + "/bubble_scripts.txt"), "\n")
-            .stream().filter(s -> !empty(s)).collect(Collectors.toList());
+    public static final String INSTALL_LOCAL_TEMPLATE = stream2string(ApiConstants.ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH + ".hbs");
 
     public static final int MAX_ANSIBLE_TRIES = 1;
     public static final int RESTORE_KEY_LEN = 6;
@@ -288,12 +282,7 @@ public class StandardNetworkService implements NetworkService {
                 // add a newline after so keys appended later will be OK
                 toFile(sshPubKeyFile, "\n"+sshKey.getSshPublicKey()+"\n");
             }
-
-            // write scripts
-            final File scriptsDir = mkdirOrDie(new File(bubbleFilesDir, "scripts"));
-            for (String script : BUBBLE_SCRIPTS) {
-                toFile(new File(scriptsDir, script), stream2string("scripts/"+script));
-            }
+            ApiConstants.copyScripts(bubbleFilesDir);
 
             // write self_node.json file
             writeFile(bubbleFilesDir, null, SELF_NODE_JSON, json(node
@@ -428,11 +417,11 @@ public class StandardNetworkService implements NetworkService {
 
                 // rsync ansible dir to remote host
                 "echo '" + METER_TICK_COPYING_ANSIBLE + "' && " +
-                "rsync -az -e \"ssh " + sshArgs + "\" . "+sshTarget+ ":" + ANSIBLE_DIR + " && " +
+                "rsync -az -e \"ssh " + sshArgs + "\" . "+sshTarget+ ":" + ApiConstants.ANSIBLE_DIR + " && " +
 
                 // run install_local.sh on remote host, installs ansible locally
                 "echo '" + METER_TICK_RUNNING_ANSIBLE + "' && " +
-                "ssh "+sshArgs+" "+sshTarget+" ~"+nodeUser+ "/" + ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH;
+                "ssh "+sshArgs+" "+sshTarget+" ~"+nodeUser+ "/" + ApiConstants.ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH;
     }
 
     private File writeFile(File dir, Map<String, Object> ctx, String filename, String templateOrData) throws IOException {

bubble-server/src/main/java/bubble/service/packer/PackerBuild.java

@@ -0,0 +1,29 @@
+package bubble.service.packer;
+
+import bubble.cloud.CloudRegion;
+import bubble.cloud.compute.PackerImage;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import lombok.Getter;
+import lombok.Setter;
+
+public class PackerBuild {
+
+    @Getter @Setter private String name;
+    @Getter @Setter private String builder_type;
+    @Getter @Setter private Long build_time;
+    @Getter @Setter private ArrayNode files;
+    @Getter @Setter private String artifact_id;
+    @Getter @Setter private String packer_run_uuid;
+    @Getter @Setter private JsonNode custom_data;
+
+    public PackerImage toPackerImage(String name) {
+        final String[] parts = artifact_id.split(":");
+        final String regionName = parts[0];
+        final String id = parts[1];
+        return new PackerImage()
+                .setId(id)
+                .setName(name)
+                .setRegions(new CloudRegion[]{new CloudRegion().setInternalName(regionName)});
+    }
+}

bubble-server/src/main/java/bubble/service/packer/PackerJob.java

@@ -0,0 +1,187 @@
+package bubble.service.packer;
+
+import bubble.cloud.CloudRegion;
+import bubble.cloud.CloudRegionRelative;
+import bubble.cloud.compute.ComputeConfig;
+import bubble.cloud.compute.ComputeServiceDriver;
+import bubble.cloud.compute.PackerConfig;
+import bubble.cloud.compute.PackerImage;
+import bubble.cloud.geoLocation.GeoLocation;
+import bubble.model.account.Account;
+import bubble.model.cloud.AnsibleInstallType;
+import bubble.model.cloud.CloudService;
+import bubble.server.BubbleConfiguration;
+import bubble.service.cloud.GeoService;
+import lombok.Cleanup;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.exec.CommandLine;
+import org.cobbzilla.util.collection.NameAndValue;
+import org.cobbzilla.util.collection.SingletonList;
+import org.cobbzilla.util.handlebars.HandlebarsUtil;
+import org.cobbzilla.util.io.FileUtil;
+import org.cobbzilla.util.io.TempDir;
+import org.cobbzilla.util.system.Command;
+import org.cobbzilla.util.system.CommandResult;
+import org.cobbzilla.util.system.CommandShell;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.stream.Collectors;
+
+import static bubble.ApiConstants.copyScripts;
+import static bubble.model.cloud.RegionalServiceDriver.findClosestRegions;
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.io.FileUtil.abs;
+import static org.cobbzilla.util.io.FileUtil.toFileOrDie;
+import static org.cobbzilla.util.io.StreamUtil.copyClasspathDirectory;
+import static org.cobbzilla.util.io.StreamUtil.stream2string;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.network.NetworkUtil.getExternalIp;
+
+@Slf4j
+public class PackerJob implements Callable<List<PackerImage>> {
+
+    public static final AnsibleInstallType[] PACKER_TYPES = {AnsibleInstallType.sage, AnsibleInstallType.node};
+    public static final String INSTALL_TYPE_VAR = "@@TYPE@@";
+    public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@";
+    public static final String JAR_SHA_VAR = "@@JAR_SHA256@@";
+    public static final String PACKER_TEMPLATE = "packer/packer-"+INSTALL_TYPE_VAR+".json.hbs";
+    public static final String PACKER_IMAGE_NAME_VAR = "packerImageName";
+    public static final String PACKER_IMAGE_PREFIX = "packer_bubble_";
+    public static final String PACKER_IMAGE_NAME_TEMPLATE = PACKER_IMAGE_PREFIX+INSTALL_TYPE_VAR+"_"+BUBBLE_VERSION_VAR+"_"+JAR_SHA_VAR;
+    public static final String VARIABLES_VAR = "packerVariables";
+    public static final String BUILD_REGION_VAR = "buildRegion";
+    public static final String IMAGE_REGIONS_VAR = "imageRegions";
+    public static final String BUILDER_VAR = "builder";
+
+    @Autowired private BubbleConfiguration configuration;
+    @Autowired private GeoService geoService;
+    @Autowired private PackerService packerService;
+
+    @Getter private Account account;
+    @Getter private CloudService cloud;
+    @Getter private List<PackerImage> images = new ArrayList<>();
+    private CloudRegionRelative buildRegion;
+
+    public PackerJob(Account account, CloudService cloud) {
+        this.account = account;
+        this.cloud = cloud;
+    }
+
+    @Override public List<PackerImage> call() throws Exception {
+        try {
+            final List<PackerImage> images = _call();
+            packerService.recordJobCompleted(this);
+            return images;
+
+        } catch (Exception e) {
+            packerService.recordJobError(this, e);
+            throw e;
+        }
+    }
+
+    public List<PackerImage> _call() throws Exception {
+        final String jarSha = configuration.getJarSha();
+
+        final ComputeConfig computeConfig = json(cloud.getDriverConfigJson(), ComputeConfig.class);
+        final ComputeServiceDriver computeDriver = cloud.getComputeDriver(configuration);
+        final PackerConfig packerConfig = computeConfig.getPacker();
+
+        // create handlebars context
+        final Map<String, Object> ctx = new HashMap<>();
+        ctx.put("credentials", NameAndValue.toMap(cloud.getCredentials().getParams()));
+        ctx.put("compute", computeDriver);
+
+        // determine lat/lon to find closest cloud region to perform build in
+        final GeoLocation here = geoService.locate(account.getUuid(), getExternalIp());
+        final List<CloudRegionRelative> closestRegions = findClosestRegions(new SingletonList<>(cloud), null, here.getLatitude(), here.getLongitude());
+        if (empty(closestRegions)) return die("writePackerImages: no closest region could be determined");
+        buildRegion = closestRegions.get(0);
+        ctx.put(BUILD_REGION_VAR, buildRegion);
+
+        // create list of all regions, without leading/trailing double-quote, which should already be in the template
+        ctx.put(IMAGE_REGIONS_VAR, toInnerStringList(computeDriver.getRegions().stream()
+                .map(CloudRegion::getInternalName)
+                .collect(Collectors.toList())));
+
+        // set environment variables
+        final Map<String, String> env = new HashMap<>();
+        for (NameAndValue variable : packerConfig.getVars()) {
+            env.put(variable.getName(), HandlebarsUtil.apply(configuration.getHandlebars(), variable.getValue(), ctx, '[', ']'));
+        }
+        ctx.put(VARIABLES_VAR, packerConfig.getVars());
+
+        // copy ansible and other packer files to temp dir
+        @Cleanup final TempDir tempDir = copyClasspathDirectory("packer");
+
+        // copy bubble jar and scripts to role dir, calculate shasum for packer image name
+        final File jar = configuration.getBubbleJar();
+        final File bubbleFilesDir = new File(abs(tempDir)+"/roles/bubble/files");
+        FileUtil.copyFile(jar, new File(abs(bubbleFilesDir)+"/bubble.jar"));
+        copyScripts(bubbleFilesDir);
+
+        // write packer config file
+        for (AnsibleInstallType installType : PACKER_TYPES) {
+            final String imageName = PACKER_IMAGE_NAME_TEMPLATE
+                    .replace(INSTALL_TYPE_VAR, installType.name())
+                    .replace(BUBBLE_VERSION_VAR, configuration.getVersion())
+                    .replace(JAR_SHA_VAR, jarSha);
+            ctx.put(PACKER_IMAGE_NAME_VAR, imageName);
+
+            final String packerTemplatePath = PACKER_TEMPLATE.replace(INSTALL_TYPE_VAR, installType.name());
+            final String packerConfigTemplate = stream2string(packerTemplatePath);
+            ctx.put(BUILDER_VAR, generateBuilder(packerConfig, ctx));
+
+            // write packer file
+            final String packerJson = HandlebarsUtil.apply(configuration.getHandlebars(), packerConfigTemplate, ctx, '[', ']');
+            toFileOrDie(abs(tempDir)+"/packer.json", packerJson);
+
+            // run packer, return handle to running packer
+            log.info("running packer for "+installType+"...");
+            final CommandResult commandResult = CommandShell.exec(new Command(new CommandLine("packer")
+                    .addArgument("build").addArgument("packer.json"))
+                    .setDir(tempDir)
+                    .setEnv(env)
+                    .setCopyToStandard(true));
+
+            if (!commandResult.isZeroExitStatus()) {
+                return die("Error executing packer: exit status "+commandResult.getExitStatus());
+            }
+
+            // read manifest, populate images
+            final File packerManifestFile = new File(tempDir, "manifest.json");
+            if (!packerManifestFile.exists()) {
+                return die("Error executing packer: manifest file not found: "+abs(packerManifestFile));
+            }
+            final PackerManifest packerManifest = json(FileUtil.toString(packerManifestFile), PackerManifest.class);
+            final PackerBuild[] builds = packerManifest.getBuilds();
+            if (empty(builds)) {
+                return die("Error executing packer: no builds found");
+            }
+            images.addAll(Arrays.stream(builds).map(b -> b.toPackerImage(imageName)).collect(Collectors.toList()));
+        }
+
+        return images;
+    }
+
+    private String toInnerStringList(List<String> list) {
+        if (empty(list)) return die("toInnerStringList: empty list");
+        final StringBuilder b = new StringBuilder();
+        for (String val : list) {
+            if (b.length() > 0) b.append("\", \"");
+            b.append(val);
+        }
+        return b.toString();
+    }
+
+    public String generateBuilder(PackerConfig packerConfig, Map<String, Object> ctx) {
+        return HandlebarsUtil.apply(configuration.getHandlebars(), json(packerConfig.getBuilder()), ctx, '<', '>')
+                .replace("[[", "{{")
+                .replace("]]", "}}");
+    }
+
+}

bubble-server/src/main/java/bubble/service/packer/PackerManifest.java

@@ -0,0 +1,11 @@
+package bubble.service.packer;
+
+import lombok.Getter;
+import lombok.Setter;
+
+public class PackerManifest {
+
+    @Getter @Setter private PackerBuild[] builds;
+    @Getter @Setter private String last_run_uuid;
+
+}

bubble-server/src/main/java/bubble/service/packer/PackerService.java

@@ -0,0 +1,53 @@
+package bubble.service.packer;
+
+import bubble.cloud.compute.PackerImage;
+import bubble.model.account.Account;
+import bubble.model.cloud.CloudService;
+import bubble.server.BubbleConfiguration;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.daemon.DaemonThreadFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
+
+@Service @Slf4j
+public class PackerService {
+
+    private final Map<String, Future<List<PackerImage>>> activeJobs = new ConcurrentHashMap<>(16);
+    private final Map<String, List<PackerImage>> completedJobs = new ConcurrentHashMap<>(16);
+    private final ExecutorService pool = DaemonThreadFactory.fixedPool(5);
+
+    @Autowired private BubbleConfiguration configuration;
+
+    public List<PackerImage> writePackerImages(Account account, CloudService cloud) {
+        synchronized (activeJobs) {
+            final List<PackerImage> images = completedJobs.get(cloud.getUuid());
+            if (images != null) return images;
+            activeJobs.computeIfAbsent(cloud.getUuid(), k -> {
+                final PackerJob packerJob = configuration.autowire(new PackerJob(account, cloud));
+                return pool.submit(packerJob);
+            });
+            return null;
+        }
+    }
+
+    public void recordJobCompleted(PackerJob job) {
+        synchronized (activeJobs) {
+            activeJobs.remove(job.getCloud().getUuid());
+            completedJobs.put(job.getCloud().getUuid(), job.getImages());
+        }
+    }
+
+    public void recordJobError(PackerJob job, Exception e) {
+        log.error("recordJobError: "+shortError(e), e);
+        activeJobs.remove(job.getCloud().getUuid());
+    }
+
+}

bubble-server/src/main/resources/ansible/bubble_scripts.txt

@@ -15,3 +15,4 @@ bencrypt
 bdecrypt
 list_bubble_databases
 cleanup_bubble_databases
+install_packer.sh

bubble-server/src/main/resources/models/defaults/cloudService.json

@@ -199,11 +199,26 @@
         {"name": "medium", "type": "medium", "internalName": "2048 MB RAM,55 GB SSD,2.00 TB BW", "vcpu": 1, "memoryMB": 2048, "ssdGB": 55},
         {"name": "large", "type": "large", "internalName": "4096 MB RAM,80 GB SSD,3.00 TB BW", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80}
       ],
-      "config": [{"name": "os", "value": "Ubuntu 18.04 x64"}]
+      "config": [{"name": "os", "value": "Ubuntu 18.04 x64"}],
+      "packer": {
+        "vars": [{"name": "VULTR_API_KEY", "value": "[[credentials.apiKey]]"}],
+        "iterateRegions": true,
+        "builder": {
+          "type": "vultr",
+          "ssh_username": "root",
+          "api_key": "[[user `VULTR_API_KEY`]]",
+          "os_id": "<<os.id>>",
+          "region_id": "<<region.id>>",
+          "plan_id": "<<sizes.small.id>>",
+          "instance_label": "<<packerImageName>>",
+          "snapshot_description": "<<packerImageName>>",
+          "tag": "<<packerImageName>>"
+        }
+      }
     },
     "credentials": {
       "params": [
-        {"name": "API-Key", "value": "{{VULTR_API_KEY}}"}
+        {"name": "apiKey", "value": "{{VULTR_API_KEY}}"}
       ]
     },
     "template": true
@@ -256,20 +271,21 @@
         {"name": "medium", "type": "medium", "internalName": "s-1vcpu-2gb", "vcpu": 1, "memoryMB": 2048, "ssdGB": 50},
         {"name": "large", "type": "large", "internalName": "s-2vcpu-4gb", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80}
       ],
+      "config": [{"name": "os", "value": "ubuntu-18-04-x64"}],
       "packer": {
-        "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "credentials.apiKey"}],
+        "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "[[credentials.apiKey]]"}],
         "builder": {
           "type": "digitalocean",
           "ssh_username": "root",
-          "api_token": "<<user `DIGITALOCEAN_API_KEY`>>",
-          "image": "ubuntu-18-04-x64",
-          "region": "[[region]]",
-          "size": "s-1vcpu-1gb",
+          "api_token": "[[user `DIGITALOCEAN_API_KEY`]]",
+          "image": "<<os.name>>",
+          "region": "<<buildRegion.internalName>>",
+          "size": "<<sizes.small.name>>",
           "ipv6": true,
-          "tags": ["packer-bubble"]
+          "snapshot_name": "<<packerImageName>>",
+          "snapshot_regions": ["<<<imageRegions>>>"]
         }
-      },
-      "config": [{"name": "os", "value": "ubuntu-18-04-x64"}]
+      }
     },
     "credentials": {
       "params": [

bubble-server/src/main/resources/packer/packer-sage.json.hbs

@@ -1,12 +1,11 @@
 {
   "variables": {
-    [[#each packer.vars]]"[[name]]": "{{env `[[name]]`}}"[[#unless @last]],
+    [[#each packerVariables]]"[[name]]": "{{env `[[name]]`}}"[[#unless @last]],
 [[/unless]][[/each]]
   },
   "builders": [
-    [[#each packer.builders]][[json this]][[#unless @last]],
-[[/unless]][[/each]]
-],
+    [[[builder]]]
+  ],
   "provisioners": [
     {
       "type": "shell",
@@ -21,7 +20,7 @@
     {
       "type": "ansible-local",
       "playbook_file": "packer-sage-playbook.yml",
-      "role_paths": ["."],
+      "role_paths": ["roles/common", "roles/firewall", "roles/nginx", "roles/bubble", "roles/bubble_finalizer"],
       "inventory_file": "hosts"
     }
   ],

bubble-server/src/test/resources/models/system/cloudService.json

@@ -223,7 +223,20 @@
         {"name": "medium", "type": "medium", "internalName": "s-1vcpu-2gb", "vcpu": 1, "memoryMB": 2048, "ssdGB": 50},
         {"name": "large", "type": "large", "internalName": "s-2vcpu-4gb", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80}
       ],
-      "config": [{"name": "os", "value": "ubuntu-18-04-x64"}]
+      "packer": {
+        "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "[[credentials.apiKey]]"}],
+        "builder": {
+          "type": "digitalocean",
+          "ssh_username": "root",
+          "api_token": "[[user `DIGITALOCEAN_API_KEY`]]",
+          "image": "ubuntu-18-04-x64",
+          "region": "<<buildRegion.internalName>>",
+          "size": "s-1vcpu-1gb",
+          "ipv6": true,
+          "snapshot_name": "<<packerImageName>>",
+          "snapshot_regions": ["<<<imageRegions>>>"]
+        }
+      }
     },
     "credentials": {
       "params": [

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 5e49b6e3e4281c59abd127279e90299f633db53f
+Subproject commit b1273943835c8002c7aae24b880f2038fa71e73c

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 2a086ae6d7887b42b647db33e241cfada5574a2e
+Subproject commit a07578cf1fde1cdaee0abc626fa4761fe8421446