diff --git a/bin/first_time_ubuntu.sh b/bin/first_time_ubuntu.sh index 894c7a63..568690ad 100755 --- a/bin/first_time_ubuntu.sh +++ b/bin/first_time_ubuntu.sh @@ -12,9 +12,13 @@ sudo apt update -y || die "Error running apt update" sudo apt upgrade -y || die "Error running apt upgrade" # Install packages -sudo apt install openjdk-11-jdk maven postgresql-10 redis-server jq python3 python3-pip npm webpack curl -y || die "Error installing apt packages" +sudo apt install openjdk-11-jdk maven postgresql-10 redis-server jq python3 python3-pip npm webpack curl unzip -y || die "Error installing apt packages" sudo pip3 install setuptools psycopg2-binary || die "Error installing pip packages" +# Install packer +BUBBLE_BIN="$(cd "$(dirname "${0}")" && pwd)" +"${BUBBLE_BIN}/install_packer.sh" || die "Error installing packer" + # Create DB user for current user, as superuser CURRENT_USER="$(whoami)" sudo su - postgres bash -c 'createuser -U postgres --createdb --createrole --superuser '"${CURRENT_USER}"'' || die "Error creating ${CURRENT_USER} DB user" diff --git a/bin/install_packer.sh b/bin/install_packer.sh new file mode 100755 index 00000000..e9a1f88a --- /dev/null +++ b/bin/install_packer.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +function die { + echo 1>&2 "${1}" + exit 1 +} + +# Install packer +if [[ ! -f ${HOME}/packer/packer ]] ; then + PACKER_VERSION=1.5.6 + PACKER_FILE=packer_${PACKER_VERSION}_linux_amd64.zip + PACKER_URL=https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_FILE} + mkdir -p ${HOME}/packer && cd ${HOME}/packer && wget ${PACKER_URL} && unzip ${PACKER_FILE} || die "Error installing packer" +else + echo "Packer already installed" +fi + +# Install packer Vultr plugin +if [[ ! -f ${HOME}/.packer.d/plugins/packer-builder-vultr ]] ; then + PACKER_VULTR_VERSION=1.0.8 + PACKER_VULTR_FILE=packer-builder-vultr_${PACKER_VULTR_VERSION}_linux_64-bit.tar.gz + PACKER_VULTR_URL=https://github.com/vultr/packer-builder-vultr/releases/download/v${PACKER_VULTR_VERSION}/${PACKER_VULTR_FILE} + mkdir -p ${HOME}/.packer.d/plugins && cd ${HOME}/.packer.d/plugins && wget ${PACKER_VULTR_URL} && tar xzf ${PACKER_VULTR_FILE} || die "Error installing packer vultr plugin" +else + echo "Packer vultr plugin already installed" +fi diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index f3281e7e..7946548d 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -17,21 +17,24 @@ import org.glassfish.jersey.server.ContainerRequest; import javax.ws.rs.core.Context; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.apache.http.HttpHeaders.*; -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.io.FileUtil.*; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.network.NetworkUtil.*; +import static org.cobbzilla.util.string.StringUtil.splitAndTrim; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @Slf4j @@ -44,9 +47,9 @@ public class ApiConstants { public static final String[] ROLES_SAGE = {"common", "nginx", "bubble", "bubble_finalizer"}; public static final String[] ROLES_NODE = {"common", "nginx", "algo", "mitmproxy", "bubble", "bubble_finalizer"}; - public static final String PACKER_IMAGE_TAG = "packer_bubble"; - public static final String PACKER_IMAGE_TAG_SAGE = "packer_bubble_sage"; - public static final String PACKER_IMAGE_TAG_NODE = "packer_bubble_node"; + public static final String ANSIBLE_DIR = "ansible"; + public static final List BUBBLE_SCRIPTS = splitAndTrim(stream2string(ANSIBLE_DIR + "/bubble_scripts.txt"), "\n") + .stream().filter(s -> !empty(s)).collect(Collectors.toList()); private static final AtomicReference bubbleDefaultDomain = new AtomicReference<>(); @@ -291,4 +294,19 @@ public class ApiConstants { invalidEx("err."+e.getSimpleName()+".invalid", "Invalid "+e.getSimpleName()+": "+v, v)); } + public static void copyScriptsOrDie(File scriptsParentDir) { + try { + copyScripts(scriptsParentDir); + } catch (IOException e) { + die("copyScriptsOrDie: "+shortError(e), e); + } + } + + public static void copyScripts(File scriptsParentDir) throws IOException { + // write scripts + final File scriptsDir = mkdirOrDie(new File(scriptsParentDir, "scripts")); + for (String script : BUBBLE_SCRIPTS) { + toFile(new File(scriptsDir, script), stream2string("scripts/"+script)); + } + } } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java index d8ef69b0..88b684ba 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java @@ -4,7 +4,6 @@ */ package bubble.cloud.compute.digitalocean; -import bubble.ApiConstants; import bubble.cloud.CloudRegion; import bubble.cloud.compute.ComputeNodeSize; import bubble.cloud.compute.ComputeServiceDriverBase; @@ -40,13 +39,13 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @Slf4j public class DigitalOceanDriver extends ComputeServiceDriverBase { - private static final String PARAM_API_KEY = "apiKey"; + public static final String PARAM_API_KEY = "apiKey"; public static final String DO_API_BASE = "https://api.digitalocean.com/v2/"; public static final String TAG_PREFIX_CLOUD = "cloud_"; public static final String TAG_PREFIX_NODE = "node_"; - public static final String PACKER_IMAGES_URI = "images?tag=" + ApiConstants.PACKER_IMAGE_TAG; + public static final String PACKER_IMAGES_URI = "images?private=true"; @Getter(lazy=true) private final Set regionSlugs = getResourceSlugs("regions"); @Getter(lazy=true) private final Set sizeSlugs = getResourceSlugs("sizes"); @@ -63,9 +62,16 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase { JsonNode page = found; do { final JsonNode items = page.has(type) ? page.get(type) : null; - if (empty(items) || !items.isArray()) return die("getResourceSlugs("+uri+"): expected "+type+" property to contain a (non-empty) array"); + if (empty(items)) { + if (!parser.allowEmpty()) return die("getResources("+uri+"): expected "+type+" property to contain a (non-empty) array"); + return null; + } + if (!items.isArray()) return die("getResources("+uri+"): expected "+type+" property to contain a (non-empty) array"); - for (int i=0; i getPackerImages() { - return getResources(PACKER_IMAGES_URI, new ImageParser()); + final List images = getResources(PACKER_IMAGES_URI, new ImageParser()); + return images == null ? Collections.emptyList() : images; } @Override public List writePackerImages() { diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java index 2de8d06e..8f3aaeb1 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ImageParser.java @@ -7,14 +7,27 @@ import com.fasterxml.jackson.databind.JsonNode; import java.util.ArrayList; import java.util.List; +import static bubble.service.packer.PackerJob.PACKER_IMAGE_PREFIX; + public class ImageParser implements ResourceParser> { @Override public List newResults() { return new ArrayList<>(); } + @Override public boolean allowEmpty() { return true; } + @Override public PackerImage parse(JsonNode item) { + final PackerImage image = new PackerImage(); + + final String name; + if (item.has("name")) { + name = item.get("name").textValue(); + if (!name.startsWith(PACKER_IMAGE_PREFIX)) return null; + image.setName(name); + } + if (item.has("id")) image.setId(item.get("id").textValue()); - if (item.has("name")) image.setName(item.get("name").textValue()); + if (item.has("regions")) { final JsonNode regionsNode = item.get("regions"); if (regionsNode.isArray()) { diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java index 12b848cf..309901ce 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/ResourceParser.java @@ -10,4 +10,6 @@ public interface ResourceParser> { E parse(JsonNode item); + default boolean allowEmpty () { return false; } + } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java index 6e02eeff..f2d1aabd 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java @@ -39,6 +39,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; @Slf4j public class VultrDriver extends ComputeServiceDriverBase { + public static final String PARAM_API_KEY = "apiKey"; public static final String API_KEY_HEADER = "API-Key"; public static final String VULTR_API_BASE = "https://api.vultr.com/v1/"; @@ -84,9 +85,9 @@ public class VultrDriver extends ComputeServiceDriverBase { } @Override public void postSetup() { - if (credentials != null && credentials.hasParam(API_KEY_HEADER) && credentials.getParam(API_KEY_HEADER).contains("{{")) { - final String apiKey = configuration.applyHandlebars(credentials.getParam(API_KEY_HEADER)); - credentials.setParam(API_KEY_HEADER, apiKey); + if (credentials != null && credentials.hasParam(PARAM_API_KEY) && credentials.getParam(PARAM_API_KEY).contains("{{")) { + final String apiKey = configuration.applyHandlebars(credentials.getParam(PARAM_API_KEY)); + credentials.setParam(PARAM_API_KEY, apiKey); } super.postSetup(); } @@ -187,7 +188,7 @@ public class VultrDriver extends ComputeServiceDriverBase { } private HttpRequestBean auth(HttpRequestBean req) { - return req.setHeader(API_KEY_HEADER, credentials.getParam(API_KEY_HEADER)); + return req.setHeader(API_KEY_HEADER, credentials.getParam(PARAM_API_KEY)); } @Override public BubbleNode stop(BubbleNode node) throws Exception { diff --git a/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java b/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java index b5c57e0a..001498cd 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java @@ -1,9 +1,11 @@ package bubble.resources.cloud; import bubble.cloud.compute.ComputeServiceDriver; +import bubble.cloud.compute.PackerImage; import bubble.model.account.Account; import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; +import bubble.service.packer.PackerService; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; @@ -14,6 +16,8 @@ import javax.ws.rs.PUT; import javax.ws.rs.Produces; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; +import java.util.Collections; +import java.util.List; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; import static org.cobbzilla.wizard.resources.ResourceUtil.ok; @@ -23,6 +27,7 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.ok; public class ComputePackerResource { @Autowired private BubbleConfiguration configuration; + @Autowired private PackerService packer; private Account account; private CloudService cloud; @@ -40,10 +45,11 @@ public class ComputePackerResource { } @PUT - public Response ensureImagesExist(@Context Request req, - @Context ContainerRequest ctx) { - final ComputeServiceDriver driver = cloud.getComputeDriver(configuration); - return ok(driver.writePackerImages()); + public Response writeImages(@Context Request req, + @Context ContainerRequest ctx) { + final List images = packer.writePackerImages(account, cloud); + if (images != null) return ok(images); + return ok(Collections.emptyList()); } } \ No newline at end of file diff --git a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java index bd0f24e1..89e4a4b0 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java +++ b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java @@ -64,6 +64,7 @@ import static org.cobbzilla.util.handlebars.HandlebarsUtil.registerUtilityHelper import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; +import static org.cobbzilla.util.security.ShaUtil.sha256_file; @Configuration @NoArgsConstructor @Slf4j public class BubbleConfiguration extends PgRestServerConfiguration @@ -160,6 +161,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration return bubbleJar; } + @Getter(lazy=true) private final String jarSha = sha256_file(getBubbleJar()); + private static final AtomicReference _DEFAULT_LOCALE = new AtomicReference<>(); public static String getDEFAULT_LOCALE() { final String locale = _DEFAULT_LOCALE.get(); diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index f420ac54..bf503266 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -4,6 +4,7 @@ */ package bubble.service.cloud; +import bubble.ApiConstants; import bubble.cloud.CloudAndRegion; import bubble.cloud.compute.ComputeServiceDriver; import bubble.cloud.dns.DnsServiceDriver; @@ -57,7 +58,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import static bubble.ApiConstants.getRemoteHost; import static bubble.ApiConstants.newNodeHostname; @@ -77,7 +77,6 @@ import static org.cobbzilla.util.io.FileUtil.*; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.reflect.ReflectionUtil.closeQuietly; -import static org.cobbzilla.util.string.StringUtil.splitAndTrim; import static org.cobbzilla.util.system.CommandShell.chmod; import static org.cobbzilla.util.system.Sleep.sleep; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @@ -86,16 +85,11 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; @Service @Slf4j public class StandardNetworkService implements NetworkService { - public static final String ANSIBLE_DIR = "ansible"; - public static final String PLAYBOOK_YML = "playbook.yml"; - public static final String PLAYBOOK_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + PLAYBOOK_YML + ".hbs"); + public static final String PLAYBOOK_TEMPLATE = stream2string(ApiConstants.ANSIBLE_DIR + "/" + PLAYBOOK_YML + ".hbs"); public static final String INSTALL_LOCAL_SH = "install_local.sh"; - public static final String INSTALL_LOCAL_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH + ".hbs"); - - public static final List BUBBLE_SCRIPTS = splitAndTrim(stream2string(ANSIBLE_DIR + "/bubble_scripts.txt"), "\n") - .stream().filter(s -> !empty(s)).collect(Collectors.toList()); + public static final String INSTALL_LOCAL_TEMPLATE = stream2string(ApiConstants.ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH + ".hbs"); public static final int MAX_ANSIBLE_TRIES = 1; public static final int RESTORE_KEY_LEN = 6; @@ -288,12 +282,7 @@ public class StandardNetworkService implements NetworkService { // add a newline after so keys appended later will be OK toFile(sshPubKeyFile, "\n"+sshKey.getSshPublicKey()+"\n"); } - - // write scripts - final File scriptsDir = mkdirOrDie(new File(bubbleFilesDir, "scripts")); - for (String script : BUBBLE_SCRIPTS) { - toFile(new File(scriptsDir, script), stream2string("scripts/"+script)); - } + ApiConstants.copyScripts(bubbleFilesDir); // write self_node.json file writeFile(bubbleFilesDir, null, SELF_NODE_JSON, json(node @@ -428,11 +417,11 @@ public class StandardNetworkService implements NetworkService { // rsync ansible dir to remote host "echo '" + METER_TICK_COPYING_ANSIBLE + "' && " + - "rsync -az -e \"ssh " + sshArgs + "\" . "+sshTarget+ ":" + ANSIBLE_DIR + " && " + + "rsync -az -e \"ssh " + sshArgs + "\" . "+sshTarget+ ":" + ApiConstants.ANSIBLE_DIR + " && " + // run install_local.sh on remote host, installs ansible locally "echo '" + METER_TICK_RUNNING_ANSIBLE + "' && " + - "ssh "+sshArgs+" "+sshTarget+" ~"+nodeUser+ "/" + ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH; + "ssh "+sshArgs+" "+sshTarget+" ~"+nodeUser+ "/" + ApiConstants.ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH; } private File writeFile(File dir, Map ctx, String filename, String templateOrData) throws IOException { diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerBuild.java b/bubble-server/src/main/java/bubble/service/packer/PackerBuild.java new file mode 100644 index 00000000..85de8153 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/packer/PackerBuild.java @@ -0,0 +1,29 @@ +package bubble.service.packer; + +import bubble.cloud.CloudRegion; +import bubble.cloud.compute.PackerImage; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import lombok.Getter; +import lombok.Setter; + +public class PackerBuild { + + @Getter @Setter private String name; + @Getter @Setter private String builder_type; + @Getter @Setter private Long build_time; + @Getter @Setter private ArrayNode files; + @Getter @Setter private String artifact_id; + @Getter @Setter private String packer_run_uuid; + @Getter @Setter private JsonNode custom_data; + + public PackerImage toPackerImage(String name) { + final String[] parts = artifact_id.split(":"); + final String regionName = parts[0]; + final String id = parts[1]; + return new PackerImage() + .setId(id) + .setName(name) + .setRegions(new CloudRegion[]{new CloudRegion().setInternalName(regionName)}); + } +} diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java new file mode 100644 index 00000000..15c21820 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -0,0 +1,187 @@ +package bubble.service.packer; + +import bubble.cloud.CloudRegion; +import bubble.cloud.CloudRegionRelative; +import bubble.cloud.compute.ComputeConfig; +import bubble.cloud.compute.ComputeServiceDriver; +import bubble.cloud.compute.PackerConfig; +import bubble.cloud.compute.PackerImage; +import bubble.cloud.geoLocation.GeoLocation; +import bubble.model.account.Account; +import bubble.model.cloud.AnsibleInstallType; +import bubble.model.cloud.CloudService; +import bubble.server.BubbleConfiguration; +import bubble.service.cloud.GeoService; +import lombok.Cleanup; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.exec.CommandLine; +import org.cobbzilla.util.collection.NameAndValue; +import org.cobbzilla.util.collection.SingletonList; +import org.cobbzilla.util.handlebars.HandlebarsUtil; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.util.io.TempDir; +import org.cobbzilla.util.system.Command; +import org.cobbzilla.util.system.CommandResult; +import org.cobbzilla.util.system.CommandShell; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.File; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +import static bubble.ApiConstants.copyScripts; +import static bubble.model.cloud.RegionalServiceDriver.findClosestRegions; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.toFileOrDie; +import static org.cobbzilla.util.io.StreamUtil.copyClasspathDirectory; +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.network.NetworkUtil.getExternalIp; + +@Slf4j +public class PackerJob implements Callable> { + + public static final AnsibleInstallType[] PACKER_TYPES = {AnsibleInstallType.sage, AnsibleInstallType.node}; + public static final String INSTALL_TYPE_VAR = "@@TYPE@@"; + public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@"; + public static final String JAR_SHA_VAR = "@@JAR_SHA256@@"; + public static final String PACKER_TEMPLATE = "packer/packer-"+INSTALL_TYPE_VAR+".json.hbs"; + public static final String PACKER_IMAGE_NAME_VAR = "packerImageName"; + public static final String PACKER_IMAGE_PREFIX = "packer_bubble_"; + public static final String PACKER_IMAGE_NAME_TEMPLATE = PACKER_IMAGE_PREFIX+INSTALL_TYPE_VAR+"_"+BUBBLE_VERSION_VAR+"_"+JAR_SHA_VAR; + public static final String VARIABLES_VAR = "packerVariables"; + public static final String BUILD_REGION_VAR = "buildRegion"; + public static final String IMAGE_REGIONS_VAR = "imageRegions"; + public static final String BUILDER_VAR = "builder"; + + @Autowired private BubbleConfiguration configuration; + @Autowired private GeoService geoService; + @Autowired private PackerService packerService; + + @Getter private Account account; + @Getter private CloudService cloud; + @Getter private List images = new ArrayList<>(); + private CloudRegionRelative buildRegion; + + public PackerJob(Account account, CloudService cloud) { + this.account = account; + this.cloud = cloud; + } + + @Override public List call() throws Exception { + try { + final List images = _call(); + packerService.recordJobCompleted(this); + return images; + + } catch (Exception e) { + packerService.recordJobError(this, e); + throw e; + } + } + + public List _call() throws Exception { + final String jarSha = configuration.getJarSha(); + + final ComputeConfig computeConfig = json(cloud.getDriverConfigJson(), ComputeConfig.class); + final ComputeServiceDriver computeDriver = cloud.getComputeDriver(configuration); + final PackerConfig packerConfig = computeConfig.getPacker(); + + // create handlebars context + final Map ctx = new HashMap<>(); + ctx.put("credentials", NameAndValue.toMap(cloud.getCredentials().getParams())); + ctx.put("compute", computeDriver); + + // determine lat/lon to find closest cloud region to perform build in + final GeoLocation here = geoService.locate(account.getUuid(), getExternalIp()); + final List closestRegions = findClosestRegions(new SingletonList<>(cloud), null, here.getLatitude(), here.getLongitude()); + if (empty(closestRegions)) return die("writePackerImages: no closest region could be determined"); + buildRegion = closestRegions.get(0); + ctx.put(BUILD_REGION_VAR, buildRegion); + + // create list of all regions, without leading/trailing double-quote, which should already be in the template + ctx.put(IMAGE_REGIONS_VAR, toInnerStringList(computeDriver.getRegions().stream() + .map(CloudRegion::getInternalName) + .collect(Collectors.toList()))); + + // set environment variables + final Map env = new HashMap<>(); + for (NameAndValue variable : packerConfig.getVars()) { + env.put(variable.getName(), HandlebarsUtil.apply(configuration.getHandlebars(), variable.getValue(), ctx, '[', ']')); + } + ctx.put(VARIABLES_VAR, packerConfig.getVars()); + + // copy ansible and other packer files to temp dir + @Cleanup final TempDir tempDir = copyClasspathDirectory("packer"); + + // copy bubble jar and scripts to role dir, calculate shasum for packer image name + final File jar = configuration.getBubbleJar(); + final File bubbleFilesDir = new File(abs(tempDir)+"/roles/bubble/files"); + FileUtil.copyFile(jar, new File(abs(bubbleFilesDir)+"/bubble.jar")); + copyScripts(bubbleFilesDir); + + // write packer config file + for (AnsibleInstallType installType : PACKER_TYPES) { + final String imageName = PACKER_IMAGE_NAME_TEMPLATE + .replace(INSTALL_TYPE_VAR, installType.name()) + .replace(BUBBLE_VERSION_VAR, configuration.getVersion()) + .replace(JAR_SHA_VAR, jarSha); + ctx.put(PACKER_IMAGE_NAME_VAR, imageName); + + final String packerTemplatePath = PACKER_TEMPLATE.replace(INSTALL_TYPE_VAR, installType.name()); + final String packerConfigTemplate = stream2string(packerTemplatePath); + ctx.put(BUILDER_VAR, generateBuilder(packerConfig, ctx)); + + // write packer file + final String packerJson = HandlebarsUtil.apply(configuration.getHandlebars(), packerConfigTemplate, ctx, '[', ']'); + toFileOrDie(abs(tempDir)+"/packer.json", packerJson); + + // run packer, return handle to running packer + log.info("running packer for "+installType+"..."); + final CommandResult commandResult = CommandShell.exec(new Command(new CommandLine("packer") + .addArgument("build").addArgument("packer.json")) + .setDir(tempDir) + .setEnv(env) + .setCopyToStandard(true)); + + if (!commandResult.isZeroExitStatus()) { + return die("Error executing packer: exit status "+commandResult.getExitStatus()); + } + + // read manifest, populate images + final File packerManifestFile = new File(tempDir, "manifest.json"); + if (!packerManifestFile.exists()) { + return die("Error executing packer: manifest file not found: "+abs(packerManifestFile)); + } + final PackerManifest packerManifest = json(FileUtil.toString(packerManifestFile), PackerManifest.class); + final PackerBuild[] builds = packerManifest.getBuilds(); + if (empty(builds)) { + return die("Error executing packer: no builds found"); + } + images.addAll(Arrays.stream(builds).map(b -> b.toPackerImage(imageName)).collect(Collectors.toList())); + } + + return images; + } + + private String toInnerStringList(List list) { + if (empty(list)) return die("toInnerStringList: empty list"); + final StringBuilder b = new StringBuilder(); + for (String val : list) { + if (b.length() > 0) b.append("\", \""); + b.append(val); + } + return b.toString(); + } + + public String generateBuilder(PackerConfig packerConfig, Map ctx) { + return HandlebarsUtil.apply(configuration.getHandlebars(), json(packerConfig.getBuilder()), ctx, '<', '>') + .replace("[[", "{{") + .replace("]]", "}}"); + } + +} diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerManifest.java b/bubble-server/src/main/java/bubble/service/packer/PackerManifest.java new file mode 100644 index 00000000..6fcdedd6 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/packer/PackerManifest.java @@ -0,0 +1,11 @@ +package bubble.service.packer; + +import lombok.Getter; +import lombok.Setter; + +public class PackerManifest { + + @Getter @Setter private PackerBuild[] builds; + @Getter @Setter private String last_run_uuid; + +} diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerService.java b/bubble-server/src/main/java/bubble/service/packer/PackerService.java new file mode 100644 index 00000000..6d480af1 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/packer/PackerService.java @@ -0,0 +1,53 @@ +package bubble.service.packer; + +import bubble.cloud.compute.PackerImage; +import bubble.model.account.Account; +import bubble.model.cloud.CloudService; +import bubble.server.BubbleConfiguration; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.daemon.DaemonThreadFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; + +@Service @Slf4j +public class PackerService { + + private final Map>> activeJobs = new ConcurrentHashMap<>(16); + private final Map> completedJobs = new ConcurrentHashMap<>(16); + private final ExecutorService pool = DaemonThreadFactory.fixedPool(5); + + @Autowired private BubbleConfiguration configuration; + + public List writePackerImages(Account account, CloudService cloud) { + synchronized (activeJobs) { + final List images = completedJobs.get(cloud.getUuid()); + if (images != null) return images; + activeJobs.computeIfAbsent(cloud.getUuid(), k -> { + final PackerJob packerJob = configuration.autowire(new PackerJob(account, cloud)); + return pool.submit(packerJob); + }); + return null; + } + } + + public void recordJobCompleted(PackerJob job) { + synchronized (activeJobs) { + activeJobs.remove(job.getCloud().getUuid()); + completedJobs.put(job.getCloud().getUuid(), job.getImages()); + } + } + + public void recordJobError(PackerJob job, Exception e) { + log.error("recordJobError: "+shortError(e), e); + activeJobs.remove(job.getCloud().getUuid()); + } + +} diff --git a/bubble-server/src/main/resources/ansible/bubble_scripts.txt b/bubble-server/src/main/resources/ansible/bubble_scripts.txt index 207a3238..7fbd4e33 100644 --- a/bubble-server/src/main/resources/ansible/bubble_scripts.txt +++ b/bubble-server/src/main/resources/ansible/bubble_scripts.txt @@ -15,3 +15,4 @@ bencrypt bdecrypt list_bubble_databases cleanup_bubble_databases +install_packer.sh \ No newline at end of file diff --git a/bubble-server/src/main/resources/models/defaults/cloudService.json b/bubble-server/src/main/resources/models/defaults/cloudService.json index a0cef4b2..8a304574 100644 --- a/bubble-server/src/main/resources/models/defaults/cloudService.json +++ b/bubble-server/src/main/resources/models/defaults/cloudService.json @@ -199,11 +199,26 @@ {"name": "medium", "type": "medium", "internalName": "2048 MB RAM,55 GB SSD,2.00 TB BW", "vcpu": 1, "memoryMB": 2048, "ssdGB": 55}, {"name": "large", "type": "large", "internalName": "4096 MB RAM,80 GB SSD,3.00 TB BW", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80} ], - "config": [{"name": "os", "value": "Ubuntu 18.04 x64"}] + "config": [{"name": "os", "value": "Ubuntu 18.04 x64"}], + "packer": { + "vars": [{"name": "VULTR_API_KEY", "value": "[[credentials.apiKey]]"}], + "iterateRegions": true, + "builder": { + "type": "vultr", + "ssh_username": "root", + "api_key": "[[user `VULTR_API_KEY`]]", + "os_id": "<>", + "region_id": "<>", + "plan_id": "<>", + "instance_label": "<>", + "snapshot_description": "<>", + "tag": "<>" + } + } }, "credentials": { "params": [ - {"name": "API-Key", "value": "{{VULTR_API_KEY}}"} + {"name": "apiKey", "value": "{{VULTR_API_KEY}}"} ] }, "template": true @@ -256,20 +271,21 @@ {"name": "medium", "type": "medium", "internalName": "s-1vcpu-2gb", "vcpu": 1, "memoryMB": 2048, "ssdGB": 50}, {"name": "large", "type": "large", "internalName": "s-2vcpu-4gb", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80} ], + "config": [{"name": "os", "value": "ubuntu-18-04-x64"}], "packer": { - "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "credentials.apiKey"}], + "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "[[credentials.apiKey]]"}], "builder": { "type": "digitalocean", "ssh_username": "root", - "api_token": "<>", - "image": "ubuntu-18-04-x64", - "region": "[[region]]", - "size": "s-1vcpu-1gb", + "api_token": "[[user `DIGITALOCEAN_API_KEY`]]", + "image": "<>", + "region": "<>", + "size": "<>", "ipv6": true, - "tags": ["packer-bubble"] + "snapshot_name": "<>", + "snapshot_regions": ["<<>>"] } - }, - "config": [{"name": "os", "value": "ubuntu-18-04-x64"}] + } }, "credentials": { "params": [ diff --git a/bubble-server/src/main/resources/packer/packer-sage.json.hbs b/bubble-server/src/main/resources/packer/packer-sage.json.hbs index 0efca0cd..c6df33fc 100644 --- a/bubble-server/src/main/resources/packer/packer-sage.json.hbs +++ b/bubble-server/src/main/resources/packer/packer-sage.json.hbs @@ -1,12 +1,11 @@ { "variables": { - [[#each packer.vars]]"[[name]]": "{{env `[[name]]`}}"[[#unless @last]], + [[#each packerVariables]]"[[name]]": "{{env `[[name]]`}}"[[#unless @last]], [[/unless]][[/each]] }, "builders": [ - [[#each packer.builders]][[json this]][[#unless @last]], -[[/unless]][[/each]] -], + [[[builder]]] + ], "provisioners": [ { "type": "shell", @@ -21,7 +20,7 @@ { "type": "ansible-local", "playbook_file": "packer-sage-playbook.yml", - "role_paths": ["."], + "role_paths": ["roles/common", "roles/firewall", "roles/nginx", "roles/bubble", "roles/bubble_finalizer"], "inventory_file": "hosts" } ], diff --git a/bubble-server/src/test/resources/models/system/cloudService.json b/bubble-server/src/test/resources/models/system/cloudService.json index 84683469..8ca43435 100644 --- a/bubble-server/src/test/resources/models/system/cloudService.json +++ b/bubble-server/src/test/resources/models/system/cloudService.json @@ -223,7 +223,20 @@ {"name": "medium", "type": "medium", "internalName": "s-1vcpu-2gb", "vcpu": 1, "memoryMB": 2048, "ssdGB": 50}, {"name": "large", "type": "large", "internalName": "s-2vcpu-4gb", "vcpu": 2, "memoryMB": 4096, "ssdGB": 80} ], - "config": [{"name": "os", "value": "ubuntu-18-04-x64"}] + "packer": { + "vars": [{"name": "DIGITALOCEAN_API_KEY", "value": "[[credentials.apiKey]]"}], + "builder": { + "type": "digitalocean", + "ssh_username": "root", + "api_token": "[[user `DIGITALOCEAN_API_KEY`]]", + "image": "ubuntu-18-04-x64", + "region": "<>", + "size": "s-1vcpu-1gb", + "ipv6": true, + "snapshot_name": "<>", + "snapshot_regions": ["<<>>"] + } + } }, "credentials": { "params": [ diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index 5e49b6e3..b1273943 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit 5e49b6e3e4281c59abd127279e90299f633db53f +Subproject commit b1273943835c8002c7aae24b880f2038fa71e73c diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 2a086ae6..a07578cf 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 2a086ae6d7887b42b647db33e241cfada5574a2e +Subproject commit a07578cf1fde1cdaee0abc626fa4761fe8421446