diff --git a/bubble-server/pom.xml b/bubble-server/pom.xml index f80f3789..1a56e0c4 100644 --- a/bubble-server/pom.xml +++ b/bubble-server/pom.xml @@ -197,6 +197,27 @@ ${aws.sdk.version} + + com.github.docker-java + docker-java-core + 3.2.6 + + + com.github.docker-java + docker-java-api + 3.2.6 + + + com.github.docker-java + docker-java-transport-httpclient5 + 3.2.6 + + + com.github.docker-java + docker-java-transport-zerodep + 3.2.6 + + redis.clients jedis diff --git a/bubble-server/src/main/java/bubble/cloud/CloudRegionRelative.java b/bubble-server/src/main/java/bubble/cloud/CloudRegionRelative.java index e0b976d3..45b7da20 100644 --- a/bubble-server/src/main/java/bubble/cloud/CloudRegionRelative.java +++ b/bubble-server/src/main/java/bubble/cloud/CloudRegionRelative.java @@ -20,7 +20,7 @@ public class CloudRegionRelative extends CloudRegion { @Getter @Setter private double distance; public void setDistance(double latitude, double longitude) { - distance = getLocation().distance(latitude, longitude); + if (getLocation() != null) distance = getLocation().distance(latitude, longitude); } } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java b/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java index d7aae86c..3dfc5bd8 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/PackerConfig.java @@ -20,4 +20,10 @@ public class PackerConfig { @Getter @Setter private JsonNode builder; + @Getter @Setter private JsonNode post; + public boolean hasPost () { return post != null; } + + @Getter @Setter private Boolean sudo; + public boolean sudo () { return sudo == null || sudo; } + } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/docker/DockerComputeDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/docker/DockerComputeDriver.java new file mode 100644 index 00000000..83c3ce8c --- /dev/null +++ b/bubble-server/src/main/java/bubble/cloud/compute/docker/DockerComputeDriver.java @@ -0,0 +1,172 @@ +package bubble.cloud.compute.docker; + +import bubble.cloud.CloudRegion; +import bubble.cloud.compute.*; +import bubble.model.cloud.BubbleNode; +import bubble.model.cloud.BubbleNodeState; +import bubble.model.cloud.CloudCredentials; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.api.model.Container; +import com.github.dockerjava.api.model.Image; +import com.github.dockerjava.core.DefaultDockerClientConfig; +import com.github.dockerjava.core.DockerClientConfig; +import com.github.dockerjava.core.DockerClientImpl; +import com.github.dockerjava.transport.DockerHttpClient; +import com.github.dockerjava.zerodep.ZerodepDockerHttpClient; +import edu.emory.mathcs.backport.java.util.Arrays; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.MapBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static bubble.service.packer.PackerJob.PACKER_IMAGE_PREFIX; +import static java.lang.Boolean.parseBoolean; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.system.Sleep.sleep; + +@Slf4j +public class DockerComputeDriver extends ComputeServiceDriverBase { + + public static final List CLOUD_REGIONS = Arrays.asList(new CloudRegion[]{ + new CloudRegion().setName("local").setInternalName("local") + }); + public static final List CLOUD_SIZES = Arrays.asList(new ComputeNodeSize[]{ + new ComputeNodeSize().setName("local").setInternalName("local").setType(ComputeNodeSizeType.local) + }); + public static final List CLOUD_OS_IMAGES = Arrays.asList(new OsImage[]{ + new OsImage().setName("ubuntu:20.04").setId("ubuntu:20.04").setRegion("local") + }); + + public static final long START_TIMEOUT = SECONDS.toMillis(120); + public static final String DEFAULT_HOST = "unix:///var/run/docker.sock"; + + private static final String LABEL_IMAGE = "bubble_image"; + private static final String LABEL_CLOUD = "bubble_cloud"; + private static final String LABEL_NODE = "bubble_node"; + + @Getter private final List cloudRegions = CLOUD_REGIONS; + @Getter private final List cloudSizes = CLOUD_SIZES; + @Getter private final List cloudOsImages = CLOUD_OS_IMAGES; + + @Getter(lazy=true) private final DockerClient dockerClient = initDockerClient(); + private DockerClient initDockerClient() { + CloudCredentials creds = getCredentials(); + if (creds == null) creds = new CloudCredentials(); + + final String host = creds.hasParam("host") ? creds.getParam("host") : DEFAULT_HOST; + final boolean tlsVerify = creds.hasParam("tlsVerify") && parseBoolean(creds.getParam("tlsVerify")); + final String certPath = creds.hasParam("certPath") ? creds.getParam("certPath") : null; + + final DockerClientConfig dockerConfig = DefaultDockerClientConfig.createDefaultConfigBuilder() + .withDockerHost(host) + .withDockerTlsVerify(tlsVerify) + .withDockerCertPath(certPath) + .withRegistryUsername(creds.getParam("registryUsername")) + .withRegistryPassword(creds.getParam("registryPassword")) + .withRegistryEmail(creds.getParam("registryEmail")) + .withRegistryUrl(creds.getParam("registryUrl")) + .build(); + + final DockerHttpClient client = new ZerodepDockerHttpClient.Builder() + .dockerHost(dockerConfig.getDockerHost()) + .build(); + + return DockerClientImpl.getInstance(dockerConfig, client); + } + + @Override public BubbleNode cleanupStart(BubbleNode node) throws Exception { return node; } + + @Override public BubbleNode start(BubbleNode node) throws Exception { + final DockerClient dc = getDockerClient(); + + final PackerImage packerImage = getOrCreatePackerImage(node); + + final CreateContainerResponse ccr = dc.createContainerCmd(packerImage.getId()) + .withLabels(MapBuilder.build(new String[][] { + {LABEL_CLOUD, cloud.getUuid()}, + {LABEL_NODE, node.getUuid()} + })) + .exec(); + final long start = now(); + while (listNodes().stream().noneMatch(n -> n.isRunning() && n.getUuid().equals(node.getUuid()))) { + if (now() - start > START_TIMEOUT) { + return die("start("+node.id()+"): timeout"); + } + sleep(SECONDS.toMillis(5), "waiting for docker container to be running"); + } + return node; + } + + private String lookupContainer(BubbleNode node) { + final DockerClient dc = getDockerClient(); + final List containers = dc.listContainersCmd() + .withLabelFilter(MapBuilder.build(LABEL_NODE, node.getUuid())) + .exec(); + if (empty(containers)) return die("lookupContainer: node not found: "+node.getUuid()); + if (containers.size() > 1) return die("lookupContainer: multiple containers found for node: "+node.getUuid()); + return containers.get(0).getId(); + } + + @Override public BubbleNode stop(BubbleNode node) throws Exception { + final DockerClient dc = getDockerClient(); + final String containerId = lookupContainer(node); + dc.stopContainerCmd(containerId).exec(); + return node; + } + + @Override public BubbleNode status(BubbleNode node) throws Exception { + final DockerClient dc = getDockerClient(); + final String containerId = lookupContainer(node); + final InspectContainerResponse status = dc.inspectContainerCmd(containerId).exec(); + log.info("status("+node.id()+"): "+json(status)); + + final Boolean dead = status.getState().getDead(); + if (dead != null && dead) return node.setState(BubbleNodeState.stopped); + + final Boolean running = status.getState().getRunning(); + if (running != null && running) return node.setState(BubbleNodeState.running); + + log.warn("status("+node.id()+"): recognized state: "+json(status.getState())); + + return node; + } + + @Override public List getAllPackerImages() { + final DockerClient dc = getDockerClient(); + final List images = dc.listImagesCmd().withImageNameFilter(PACKER_IMAGE_PREFIX).withLabelFilter(MapBuilder.build(LABEL_IMAGE, PACKER_IMAGE_PREFIX)).exec(); + final List packerImages = new ArrayList<>(); + for (Image i : images) { + final PackerImage p = new PackerImage(); + p.setId(i.getId()); + p.setName(empty(i.getLabels()) ? i.getId() : i.getLabels().size() == 1 ? i.getLabels().values().iterator().next() : json(i.getLabels())); + p.setRegions(null); + packerImages.add(p); + } + return packerImages; + } + + @Override public List getPackerImagesForRegion(String region) { return getAllPackerImages(); } + + @Override public List listNodes() throws IOException { + final DockerClient dc = getDockerClient(); + final List nodes = new ArrayList<>(); + final List containers = dc.listContainersCmd() + .withLabelFilter(MapBuilder.build(LABEL_CLOUD, cloud.getUuid())) + .exec(); + for (Container c : containers) { + final BubbleNode n = new BubbleNode().setState(BubbleNodeState.running); + n.setUuid(c.getLabels().get(LABEL_NODE)); + n.setCloud(c.getLabels().get(LABEL_CLOUD)); + nodes.add(n); + } + return nodes; + } + +} diff --git a/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java b/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java index 44272644..23596d32 100644 --- a/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java +++ b/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocation.java @@ -49,10 +49,14 @@ public class GeoLocation { } public double distance(double lat, double lon) { + if (lat < 0 || lon < 0) return -1.0; + double thisLat = big(getLat()).doubleValue(); + double thisLon = big(getLon()).doubleValue(); + if (thisLat < 0 || thisLon < 0) return -1.0; return Haversine.distance( - big(getLat()).doubleValue(), + thisLat, lat, - big(getLon()).doubleValue(), + thisLon, lon); } diff --git a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java index df8952ac..9de7835b 100644 --- a/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java +++ b/bubble-server/src/main/java/bubble/dao/account/AccountDAO.java @@ -41,7 +41,6 @@ import org.glassfish.grizzly.http.server.Request; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; -import javax.annotation.Nullable; import javax.transaction.Transactional; import java.util.Collection; import java.util.HashMap; @@ -359,7 +358,7 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc private final String NETWORK_OWNER_ACCOUNT_UUID_PARAM = "__thisNetworkOwnerAccountUuid__"; @Override public int bulkDeleteWhere(@NonNull final String whereClause, - @Nullable final Map parameters) { + final Map parameters) { final Map enhancedParams = parameters != null ? parameters : new HashMap<>(); enhancedParams.put(NETWORK_OWNER_ACCOUNT_UUID_PARAM, configuration.getThisNetwork().getAccount()); @@ -367,7 +366,7 @@ public class AccountDAO extends AbstractCRUDDAO implements SqlViewSearc enhancedParams); } - @Override public void delete(@Nullable final Collection accounts) { + @Override public void delete(final Collection accounts) { if (empty(accounts)) return; final var networkOwnerUuid = configuration.getThisNetwork().getAccount(); if (accounts.removeIf(a -> a != null && a.getUuid().equals(networkOwnerUuid))) { diff --git a/bubble-server/src/main/java/bubble/dao/cloud/CloudServiceDAO.java b/bubble-server/src/main/java/bubble/dao/cloud/CloudServiceDAO.java index f2defe2d..66aa4589 100644 --- a/bubble-server/src/main/java/bubble/dao/cloud/CloudServiceDAO.java +++ b/bubble-server/src/main/java/bubble/dao/cloud/CloudServiceDAO.java @@ -19,7 +19,6 @@ import org.hibernate.criterion.Order; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; -import javax.annotation.Nullable; import javax.transaction.Transactional; import java.util.ArrayList; import java.util.Collection; @@ -161,7 +160,7 @@ public class CloudServiceDAO extends AccountOwnedTemplateDAO { return !findPublicTemplatesByType(admin.getUuid(), CloudServiceType.payment).isEmpty(); } - @Override public int bulkDeleteWhere(@NonNull String whereClause, @Nullable Map parameters) { + @Override public int bulkDeleteWhere(@NonNull String whereClause, Map parameters) { // TODO for these maybe an outside cron would be better solution. BulkDelete is used here to be fast. // For now this postServiceDelete is called within a single place where this method is used - Account Deletion. log.warn("Not calling postServiceDelete for services deleted in this way"); diff --git a/bubble-server/src/main/java/bubble/model/cloud/RegionalServiceDriver.java b/bubble-server/src/main/java/bubble/model/cloud/RegionalServiceDriver.java index 2999a766..39cef74f 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/RegionalServiceDriver.java +++ b/bubble-server/src/main/java/bubble/model/cloud/RegionalServiceDriver.java @@ -59,10 +59,10 @@ public interface RegionalServiceDriver { } final CloudRegionRelative r = new CloudRegionRelative(region); r.setCloud(c.getUuid()); - if (latLonIsValid) { + if (latLonIsValid && latitude >= 0 && longitude >= 0) { r.setDistance(latitude, longitude); } else { - r.setDistance(-1); + r.setDistance(0); } allRegions.add(r); } diff --git a/bubble-server/src/main/java/bubble/resources/DebugResource.java b/bubble-server/src/main/java/bubble/resources/DebugResource.java index a72258d1..63f86a1b 100644 --- a/bubble-server/src/main/java/bubble/resources/DebugResource.java +++ b/bubble-server/src/main/java/bubble/resources/DebugResource.java @@ -24,7 +24,6 @@ import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Repository; import org.springframework.stereotype.Service; -import javax.annotation.Nullable; import javax.validation.Valid; import javax.ws.rs.*; import javax.ws.rs.core.Context; @@ -153,7 +152,7 @@ public class DebugResource { ) public Response echoJsonInLog(@Context ContainerRequest ctx, @Valid @NonNull final JsonNode input, - @QueryParam("respondWith") @Nullable final String respondWith) throws IOException { + @QueryParam("respondWith") final String respondWith) throws IOException { final var output = "ECHO: \n" + toJsonOrDie(input); log.info(output); diff --git a/bubble-server/src/main/java/bubble/resources/account/AuthResource.java b/bubble-server/src/main/java/bubble/resources/account/AuthResource.java index f5a97b5f..71b90caa 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AuthResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AuthResource.java @@ -58,7 +58,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import javax.annotation.Nullable; import javax.validation.Valid; import javax.ws.rs.*; import javax.ws.rs.core.Context; @@ -238,7 +237,7 @@ public class AuthResource { @Autowired private SageHelloService sageHelloService; @Autowired private RestoreService restoreService; - @NonNull private BubbleNode checkRestoreRequest(@Nullable final String restoreKey) { + @NonNull private BubbleNode checkRestoreRequest(final String restoreKey) { if (restoreKey == null) throw invalidEx("err.restoreKey.required"); // ensure we have been initialized @@ -271,7 +270,7 @@ public class AuthResource { ) public Response restore(@NonNull @Context final Request req, @NonNull @Context final ContainerRequest ctx, - @Nullable @PathParam("restoreKey") final String restoreKey, + @PathParam("restoreKey") final String restoreKey, @NonNull @Valid final NetworkKeys.EncryptedNetworkKeys encryptedKeys) { final var sageNode = checkRestoreRequest(restoreKey); diff --git a/bubble-server/src/main/java/bubble/resources/cloud/LogsResource.java b/bubble-server/src/main/java/bubble/resources/cloud/LogsResource.java index 4bddc94a..95628076 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/LogsResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/LogsResource.java @@ -14,7 +14,6 @@ import lombok.NonNull; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.Nullable; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; @@ -69,7 +68,7 @@ public class LogsResource { responses=@ApiResponse(responseCode=SC_OK, description="empty response indicates success") ) @NonNull public Response startLogging(@NonNull @Context final ContainerRequest ctx, - @Nullable @QueryParam("ttlDays") final Byte ttlDays) { + @QueryParam("ttlDays") final Byte ttlDays) { final Account caller = userPrincipal(ctx); if (!caller.admin()) throw forbiddenEx(); return setLogFlag(true, Optional.ofNullable(ttlDays)); diff --git a/bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java b/bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java index dcce0b0b..a2ca3d83 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java @@ -26,7 +26,6 @@ import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; import org.springframework.beans.factory.annotation.Autowired; -import javax.annotation.Nullable; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; @@ -78,7 +77,7 @@ public class NetworkBackupKeysResource { return ok(); } - @NonNull private String fetchAndCheckEncryptionKey(@Nullable final NameAndValue enc) { + @NonNull private String fetchAndCheckEncryptionKey(final NameAndValue enc) { final String encryptionKey = enc == null ? null : enc.getValue(); final ConstraintViolationBean error = validatePassword(encryptionKey); if (error != null) throw new SimpleViolationException(error); @@ -95,7 +94,7 @@ public class NetworkBackupKeysResource { @NonNull public Response retrieveNetworkKeys(@NonNull @Context final Request req, @NonNull @Context final ContainerRequest ctx, @NonNull @PathParam("keysCode") final String keysCode, - @Nullable final NameAndValue enc) { + final NameAndValue enc) { final var encryptionKey = fetchAndCheckEncryptionKey(enc); final var networkKeys = keysService.retrieveKeys(keysCode); return ok(networkKeys.encrypt(encryptionKey)); @@ -115,7 +114,7 @@ public class NetworkBackupKeysResource { @NonNull public Response backupDownloadStart(@NonNull @Context final ContainerRequest ctx, @NonNull @PathParam("keysCode") final String keysCode, @NonNull @QueryParam("backupId") final String backupId, - @Nullable final NameAndValue enc) { + final NameAndValue enc) { final var passphrase = fetchAndCheckEncryptionKey(enc); keysService.retrieveKeys(keysCode); diff --git a/bubble-server/src/main/java/bubble/service/backup/RestoreService.java b/bubble-server/src/main/java/bubble/service/backup/RestoreService.java index ed3a0f1d..5ded5130 100644 --- a/bubble-server/src/main/java/bubble/service/backup/RestoreService.java +++ b/bubble-server/src/main/java/bubble/service/backup/RestoreService.java @@ -22,7 +22,6 @@ import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -117,7 +116,7 @@ public class RestoreService { } } - @Nullable private String checkAndGetKeyJson(@NonNull final String restoreKey) { + private String checkAndGetKeyJson(@NonNull final String restoreKey) { final String keyJson = getRestoreKeys().get(restoreKey); if (keyJson == null) { log.error("restore: restoreKey not found: " + restoreKey); @@ -126,7 +125,7 @@ public class RestoreService { return keyJson; } - @Nullable private String checkAndGetRestoreDirPath() { + private String checkAndGetRestoreDirPath() { final var existingFiles = RESTORE_DIR.list(); final var restoreDirAbs = abs(RESTORE_DIR); diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index be3176ee..95f14268 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -14,10 +14,12 @@ import bubble.cloud.geoLocation.GeoLocation; import bubble.dao.account.AccountDAO; import bubble.model.account.Account; import bubble.model.cloud.AnsibleInstallType; +import bubble.model.cloud.CloudCredentials; import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; import bubble.server.SoftwareVersions; import bubble.service.cloud.GeoService; +import com.fasterxml.jackson.databind.JsonNode; import lombok.Cleanup; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -79,6 +81,8 @@ public class PackerJob implements Callable> { public static final String BUILD_REGION_VAR = "buildRegion"; public static final String IMAGE_REGIONS_VAR = "imageRegions"; public static final String BUILDERS_VAR = "builders"; + public static final String POST_PROCESSOR_VAR = "postProcessor"; + public static final String SUDO_VAR = "sudo"; public static final String PACKER_PLAYBOOK_TEMPLATE = "packer-playbook.yml.hbs"; public static final String PACKER_PLAYBOOK = "packer-playbook.yml"; public static final String PACKER_BINARY = System.getProperty("user.home")+"/packer/packer"; @@ -146,7 +150,8 @@ public class PackerJob implements Callable> { // create handlebars context final Map ctx = new HashMap<>(); - ctx.put("credentials", NameAndValue.toMap(cloud.getCredentials().getParams())); + final CloudCredentials creds = cloud.getCredentials(); + ctx.put("credentials", creds == null ? Collections.emptyMap() : NameAndValue.toMap(creds.getParams())); ctx.put("compute", computeDriver); ctx.put("sizes", computeDriver.getSizesMap()); ctx.put("os", computeDriver.getOs()); @@ -167,6 +172,15 @@ public class PackerJob implements Callable> { env.put(variable.getName(), HandlebarsUtil.apply(configuration.getHandlebars(), variable.getValue(), ctx, '[', ']')); } if (!env.containsKey("HOME")) env.put("HOME", HOME_DIR); + + // Docker builder requires "docker" command to be on our path + // It is usually in /usr/local/bin + // May need to make this more flexible if docker is elsewhere, or other tools/paths are needed + if (env.containsKey("PATH")) { + env.put("PATH", "${PATH}:/usr/local/bin"); + } else { + env.put("PATH", "/usr/local/bin"); + } ctx.put(VARIABLES_VAR, packerConfig.getVars()); // copy ansible and other packer files to temp dir @@ -254,6 +268,9 @@ public class PackerJob implements Callable> { builderJsons.add(generateBuilder(packerConfig, ctx)); } ctx.put(BUILDERS_VAR, builderJsons); + ctx.put(SUDO_VAR, packerConfig.sudo()); + + if (packerConfig.hasPost()) ctx.put(POST_PROCESSOR_VAR, generatePostProcessor(packerConfig, ctx)); // write playbook file final String playbookTemplate = FileUtil.toString(abs(tempDir)+ "/" + PACKER_PLAYBOOK_TEMPLATE); @@ -333,7 +350,15 @@ public class PackerJob implements Callable> { } public String generateBuilder(PackerConfig packerConfig, Map ctx) { - return HandlebarsUtil.apply(configuration.getHandlebars(), json(packerConfig.getBuilder()), ctx, '<', '>') + return appyHandlebars(ctx, packerConfig.getBuilder()); + } + + public String generatePostProcessor(PackerConfig packerConfig, Map ctx) { + return packerConfig.hasPost() ? appyHandlebars(ctx, packerConfig.getPost()) : null; + } + + private String appyHandlebars(Map ctx, JsonNode thing) { + return HandlebarsUtil.apply(configuration.getHandlebars(), json(thing), ctx, '<', '>') .replace("[[", "{{") .replace("]]", "}}"); } diff --git a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java index 25024647..d93d5915 100644 --- a/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java +++ b/bubble-server/src/main/java/bubble/service/stream/StandardAppPrimerService.java @@ -22,7 +22,6 @@ import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import javax.annotation.Nullable; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -115,7 +114,7 @@ public class StandardAppPrimerService implements AppPrimerService { getPrimerThread().submit(() -> _prime(account, singleApp)); } - private synchronized void _prime(@NonNull final Account account, @Nullable final BubbleApp singleApp) { + private synchronized void _prime(@NonNull final Account account, final BubbleApp singleApp) { try { final List devices = deviceDAO.findByAccount(account.getUuid()); if (devices.isEmpty()) return; diff --git a/bubble-server/src/main/resources/models/defaults/cloudService.json b/bubble-server/src/main/resources/models/defaults/cloudService.json index d9a6d083..71239578 100644 --- a/bubble-server/src/main/resources/models/defaults/cloudService.json +++ b/bubble-server/src/main/resources/models/defaults/cloudService.json @@ -151,80 +151,117 @@ "template": true }, + { + "name": "DockerCompute", + "type": "compute", + "driverClass": "bubble.cloud.compute.docker.DockerComputeDriver", + "driverConfig": { + "regions": [{"name": "local", "internalName": "local"}], + "sizes": [{"name": "local", "type": "local", "internalName": "local"}], + "os": "ubuntu:20.04", + "packer": { + "vars": [], + "sudo": false, + "builder": { + "type": "docker", + "image": "<>", + "export_path": "<>.tar", + "changes": [ + "LABEL bubble_image=<>", + "EXPOSE 80 443 1202" + ] + }, + "post": { + "type": "docker-import", + "repository": "local/bubble", + "tag": "<>" + } + } + }, + "credentials": { + "params": [ + {"name": "host", "value": "unix:///var/run/docker.sock"} + ] + }, + "template": false + }, + { "name": "VultrCompute", "type": "compute", "driverClass": "bubble.cloud.compute.vultr.VultrDriver", "driverConfig": { - "regions": [{ - "name": "Vultr - Dallas", - "internalName": "Dallas", - "location": {"city": "Dallas", "country": "US", "region": "TX", "lat": "32.779167", "lon": "-96.808889"} - }, { - "name": "Vultr - Los Angeles", - "internalName": "Los Angeles", - "location": {"city": "Los Angeles", "country": "US", "region": "CA", "lat": "34.05", "lon": "-118.25"} - }, { - "name": "Vultr - Miami", - "internalName": "Miami", - "location": {"city": "Miami", "country": "US", "region": "FL", "lat": "25.775278", "lon": "-80.208889"} - }, { - "name": "Vultr - Seattle", - "internalName": "Seattle", - "location": {"city": "Seattle", "country": "US", "region": "WA", "lat": "47.609722", "lon": "-122.333056"} - }, { - "name": "Vultr - New Jersey", - "internalName": "New Jersey", - "location": {"city": "Newark", "country": "US", "region": "NJ", "lat": "40.72", "lon": "-74.17"} - }, { - "name": "Vultr - Atlanta", - "internalName": "Atlanta", - "location": {"city": "Atlanta", "country": "US", "region": "GA", "lat": "33.755", "lon": "-84.39"} - }, { - "name": "Vultr - Chicago", - "internalName": "Chicago", - "location": {"city": "Chicago", "country": "US", "region": "IL", "lat": "41.881944", "lon": "-87.627778"} - }, { - "name": "Vultr - San Jose", - "internalName": "Silicon Valley", - "location": {"city": "San Jose", "country": "US", "region": "CA", "lat": "37.333333", "lon": "-121.9"} - }, { - "name": "Vultr - Toronto", - "internalName": "Toronto", - "location": {"city": "Toronto", "country": "CA", "region": "ON", "lat": "43.741667", "lon": "-79.373333"} - }, { - "name": "Vultr - London", - "internalName": "London", - "location": {"city": "London", "country": "GB", "region": "London", "lat": "51.507222", "lon": "-0.1275"} - }, { - "name": "Vultr - Paris", - "internalName": "Paris", - "location": {"city": "Paris", "country": "FR", "region": "Ile-de-Paris", "lat": "48.8567", "lon": "2.3508"} - }, { - "name": "Vultr - Frankfurt", - "internalName": "Frankfurt", - "location": {"city": "Frankfurt", "country": "DE", "region": "Hesse", "lat": "50.116667", "lon": "8.683333"} - }, { - "name": "Vultr - Singapore", - "internalName": "Singapore", - "location": {"city": "Singapore", "country": "SG", "region": "Singapore", "lat": "1.283333", "lon": "103.833333"} - }, { - "name": "Vultr - Tokyo", - "internalName": "Tokyo", - "location": {"city": "Tokyo", "country": "JP", "region": "Kantō", "lat": "35.689722", "lon": "139.692222"} - }, { - "name": "Vultr - Seoul", - "internalName": "Seoul", - "location": {"city": "Seoul", "country": "KR", "region": "Sudogwon", "lat": "37.566667", "lon": "126.966667"} - }, { - "name": "Vultr - Sydney", - "internalName": "Sydney", - "location": {"city": "Sydney", "country": "AU", "region": "NSW", "lat": "-33.865", "lon": "151.209444"} - }, { - "name": "Vultr - Amsterdam", - "internalName": "ams3", - "location": {"city": "Amsterdam", "country": "NL", "region": "North Holland", "lat": "52.366667", "lon": "4.9"} - }], + "regions": [ + { + "name": "Vultr - Dallas", + "internalName": "Dallas", + "location": {"city": "Dallas", "country": "US", "region": "TX", "lat": "32.779167", "lon": "-96.808889"} + }, { + "name": "Vultr - Los Angeles", + "internalName": "Los Angeles", + "location": {"city": "Los Angeles", "country": "US", "region": "CA", "lat": "34.05", "lon": "-118.25"} + }, { + "name": "Vultr - Miami", + "internalName": "Miami", + "location": {"city": "Miami", "country": "US", "region": "FL", "lat": "25.775278", "lon": "-80.208889"} + }, { + "name": "Vultr - Seattle", + "internalName": "Seattle", + "location": {"city": "Seattle", "country": "US", "region": "WA", "lat": "47.609722", "lon": "-122.333056"} + }, { + "name": "Vultr - New Jersey", + "internalName": "New Jersey", + "location": {"city": "Newark", "country": "US", "region": "NJ", "lat": "40.72", "lon": "-74.17"} + }, { + "name": "Vultr - Atlanta", + "internalName": "Atlanta", + "location": {"city": "Atlanta", "country": "US", "region": "GA", "lat": "33.755", "lon": "-84.39"} + }, { + "name": "Vultr - Chicago", + "internalName": "Chicago", + "location": {"city": "Chicago", "country": "US", "region": "IL", "lat": "41.881944", "lon": "-87.627778"} + }, { + "name": "Vultr - San Jose", + "internalName": "Silicon Valley", + "location": {"city": "San Jose", "country": "US", "region": "CA", "lat": "37.333333", "lon": "-121.9"} + }, { + "name": "Vultr - Toronto", + "internalName": "Toronto", + "location": {"city": "Toronto", "country": "CA", "region": "ON", "lat": "43.741667", "lon": "-79.373333"} + }, { + "name": "Vultr - London", + "internalName": "London", + "location": {"city": "London", "country": "GB", "region": "London", "lat": "51.507222", "lon": "-0.1275"} + }, { + "name": "Vultr - Paris", + "internalName": "Paris", + "location": {"city": "Paris", "country": "FR", "region": "Ile-de-Paris", "lat": "48.8567", "lon": "2.3508"} + }, { + "name": "Vultr - Frankfurt", + "internalName": "Frankfurt", + "location": {"city": "Frankfurt", "country": "DE", "region": "Hesse", "lat": "50.116667", "lon": "8.683333"} + }, { + "name": "Vultr - Singapore", + "internalName": "Singapore", + "location": {"city": "Singapore", "country": "SG", "region": "Singapore", "lat": "1.283333", "lon": "103.833333"} + }, { + "name": "Vultr - Tokyo", + "internalName": "Tokyo", + "location": {"city": "Tokyo", "country": "JP", "region": "Kantō", "lat": "35.689722", "lon": "139.692222"} + }, { + "name": "Vultr - Seoul", + "internalName": "Seoul", + "location": {"city": "Seoul", "country": "KR", "region": "Sudogwon", "lat": "37.566667", "lon": "126.966667"} + }, { + "name": "Vultr - Sydney", + "internalName": "Sydney", + "location": {"city": "Sydney", "country": "AU", "region": "NSW", "lat": "-33.865", "lon": "151.209444"} + }, { + "name": "Vultr - Amsterdam", + "internalName": "ams3", + "location": {"city": "Amsterdam", "country": "NL", "region": "North Holland", "lat": "52.366667", "lon": "4.9"} + } + ], "sizes": [ {"name": "small", "type": "small", "internalName": "1024 MB RAM,25 GB SSD,1.00 TB BW", "vcpu": 1, "memoryMB": 1024, "diskGB": 25}, {"name": "medium", "type": "medium", "internalName": "2048 MB RAM,55 GB SSD,2.00 TB BW", "vcpu": 1, "memoryMB": 2048, "diskGB": 55}, @@ -331,55 +368,57 @@ "type": "compute", "driverClass": "bubble.cloud.compute.ec2.AmazonEC2Driver", "driverConfig": { - "regions": [{ - "name": "Amazon - N. Virginia", "description": "US East 1 (N. Virginia)", "internalName": "us-east-1", - "location": {"city": "Arlington", "region": "VA", "country": "US", "lat": "38.880278", "lon": "-77.108333"} - }, { - "name": "Amazon - Ohio", "description": "US East 2 (Ohio)", "internalName": "us-east-2", - "location": {"region": "OH", "country": "US", "lat": "40.3416167", "lon": "-84.9180579"} - }, { - "name": "Amazon - N. California", "description": "US West 1 (N. California)", "internalName": "us-west-1", - "location": {"city": "San Jose", "country": "US", "region": "CA", "lat": "37.333333", "lon": "-121.9"} - }, { - "name": "Amazon - Oregon", "description": "US West 2 (Oregon)", "internalName": "us-west-2", - "location": {"city": "Hermiston", "region": "OR", "country": "US", "lat": "45.841111", "lon": "-119.291667"} - }, { - "name": "Amazon - Canada", "description": "Canada (Central)", "internalName": "ca-central-1", - "location": {"region": "QC", "country": "CA", "lat": "46.813889", "lon": "-71.208056"} - }, { - "name": "Amazon - Stockholm", "description": "EU (Stockholm)", "internalName": "eu-north-1", - "location": {"city": "Stockholm", "region": "Södermanland", "country": "SE", "lat": "59.329444", "lon": "18.068611"} - }, { - "name": "Amazon - Ireland", "description": "EU (Ireland)", "internalName": "eu-west-1", - "location": {"city": "Dublin", "region": "Leinster", "country": "IE", "lat": "53.35", "lon": "-6.266667"} - }, { - "name": "Amazon - London", "description": "EU (London)", "internalName": "eu-west-2", - "location": {"city": "London", "country": "GB", "region": "London", "lat": "51.507222", "lon": "-0.1275"} - }, { - "name": "Amazon - Milan", "description": "EU (Milan)", "internalName": "eu-west-3", - "location": {"city": "Milan", "country": "IT", "region": "Lombardy", "lat": "45.466944", "lon": "9.19"} - }, { - "name": "Amazon - Frankfurt", "description": "EU (Frankfurt)", "internalName": "eu-central-1", - "location": {"city": "Frankfurt", "country": "DE", "region": "Hesse", "lat": "50.116667", "lon": "8.683333"} - }, { - "name": "Amazon - Tokyo", "description": "Asia Pacific (Tokyo)", "internalName": "ap-northeast-1", - "location": {"city": "Tokyo", "country": "JP", "region": "Kantō", "lat": "35.689722", "lon": "139.692222"} - }, { - "name": "Amazon - Seoul", "description": "Asia Pacific (Seoul)", "internalName": "ap-northeast-2", - "location": {"city": "Seoul", "country": "KR", "region": "Sudogwon", "lat": "37.566667", "lon": "126.966667"} - }, { - "name": "Amazon - Singapore", "description": "Asia Pacific (Singapore)", "internalName": "ap-southeast-1", - "location": {"city": "Singapore", "country": "SG", "region": "Singapore", "lat": "1.283333", "lon": "103.833333"} - }, { - "name": "Amazon - Sydney", "description": "Asia Pacific (Sydney)", "internalName": "ap-southeast-2", - "location": {"city": "Sydney", "country": "AU", "region": "NSW", "lat": "-33.865", "lon": "151.209444"} - }, { - "name": "Amazon - Mumbai", "description": "Asia Pacific (Mumbai)", "internalName": "ap-south-1", - "location": {"city": "Mumbai", "country": "IN", "region": "Konkan", "lat": "18.975", "lon": "72.825833"} - }, { - "name": "Amazon - São Paulo", "description": "South America (São Paulo)", "internalName": "sa-east-1", - "location": {"city": "São Paulo", "country": "BR", "region": "São Paulo", "lat": "-23.55", "lon": "-46.633333"} - }], + "regions": [ + { + "name": "Amazon - N. Virginia", "description": "US East 1 (N. Virginia)", "internalName": "us-east-1", + "location": {"city": "Arlington", "region": "VA", "country": "US", "lat": "38.880278", "lon": "-77.108333"} + }, { + "name": "Amazon - Ohio", "description": "US East 2 (Ohio)", "internalName": "us-east-2", + "location": {"region": "OH", "country": "US", "lat": "40.3416167", "lon": "-84.9180579"} + }, { + "name": "Amazon - N. California", "description": "US West 1 (N. California)", "internalName": "us-west-1", + "location": {"city": "San Jose", "country": "US", "region": "CA", "lat": "37.333333", "lon": "-121.9"} + }, { + "name": "Amazon - Oregon", "description": "US West 2 (Oregon)", "internalName": "us-west-2", + "location": {"city": "Hermiston", "region": "OR", "country": "US", "lat": "45.841111", "lon": "-119.291667"} + }, { + "name": "Amazon - Canada", "description": "Canada (Central)", "internalName": "ca-central-1", + "location": {"region": "QC", "country": "CA", "lat": "46.813889", "lon": "-71.208056"} + }, { + "name": "Amazon - Stockholm", "description": "EU (Stockholm)", "internalName": "eu-north-1", + "location": {"city": "Stockholm", "region": "Södermanland", "country": "SE", "lat": "59.329444", "lon": "18.068611"} + }, { + "name": "Amazon - Ireland", "description": "EU (Ireland)", "internalName": "eu-west-1", + "location": {"city": "Dublin", "region": "Leinster", "country": "IE", "lat": "53.35", "lon": "-6.266667"} + }, { + "name": "Amazon - London", "description": "EU (London)", "internalName": "eu-west-2", + "location": {"city": "London", "country": "GB", "region": "London", "lat": "51.507222", "lon": "-0.1275"} + }, { + "name": "Amazon - Milan", "description": "EU (Milan)", "internalName": "eu-west-3", + "location": {"city": "Milan", "country": "IT", "region": "Lombardy", "lat": "45.466944", "lon": "9.19"} + }, { + "name": "Amazon - Frankfurt", "description": "EU (Frankfurt)", "internalName": "eu-central-1", + "location": {"city": "Frankfurt", "country": "DE", "region": "Hesse", "lat": "50.116667", "lon": "8.683333"} + }, { + "name": "Amazon - Tokyo", "description": "Asia Pacific (Tokyo)", "internalName": "ap-northeast-1", + "location": {"city": "Tokyo", "country": "JP", "region": "Kantō", "lat": "35.689722", "lon": "139.692222"} + }, { + "name": "Amazon - Seoul", "description": "Asia Pacific (Seoul)", "internalName": "ap-northeast-2", + "location": {"city": "Seoul", "country": "KR", "region": "Sudogwon", "lat": "37.566667", "lon": "126.966667"} + }, { + "name": "Amazon - Singapore", "description": "Asia Pacific (Singapore)", "internalName": "ap-southeast-1", + "location": {"city": "Singapore", "country": "SG", "region": "Singapore", "lat": "1.283333", "lon": "103.833333"} + }, { + "name": "Amazon - Sydney", "description": "Asia Pacific (Sydney)", "internalName": "ap-southeast-2", + "location": {"city": "Sydney", "country": "AU", "region": "NSW", "lat": "-33.865", "lon": "151.209444"} + }, { + "name": "Amazon - Mumbai", "description": "Asia Pacific (Mumbai)", "internalName": "ap-south-1", + "location": {"city": "Mumbai", "country": "IN", "region": "Konkan", "lat": "18.975", "lon": "72.825833"} + }, { + "name": "Amazon - São Paulo", "description": "South America (São Paulo)", "internalName": "sa-east-1", + "location": {"city": "São Paulo", "country": "BR", "region": "São Paulo", "lat": "-23.55", "lon": "-46.633333"} + } + ], "sizes": [ {"name": "small", "type": "small", "internalName": "t2.micro", "vcpu": 1, "memoryMB": 1024, "diskGB": 10, "diskType": "ebs_magnetic"}, {"name": "medium", "type": "medium", "internalName": "t2.small", "vcpu": 1, "memoryMB": 2048, "diskGB": 20, "diskType": "ebs_magnetic"}, diff --git a/bubble-server/src/main/resources/packer/packer.json.hbs b/bubble-server/src/main/resources/packer/packer.json.hbs index bf9b66a5..278f6bef 100644 --- a/bubble-server/src/main/resources/packer/packer.json.hbs +++ b/bubble-server/src/main/resources/packer/packer.json.hbs @@ -12,10 +12,10 @@ "type": "shell", "inline": [ "sleep 30", - "sudo bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y update'", - "sudo bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y upgrade'", - "sudo bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip virtualenv'", - "sudo pip3 install setuptools psycopg2-binary ansible" + "[[#if sudo]]sudo [[/if]]bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y update'", + "[[#if sudo]]sudo [[/if]]bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y upgrade'", + "[[#if sudo]]sudo [[/if]]bash -c 'DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip virtualenv'", + "[[#if sudo]]sudo [[/if]]pip3 install setuptools psycopg2-binary ansible" ] }, { @@ -26,6 +26,7 @@ } ], "post-processors": [ +[[#if postProcessor]][[[postProcessor]]],[[/if]] { "type": "manifest", "output": "manifest.json" diff --git a/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml index 756f6dcd..81fdab10 100644 --- a/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml @@ -27,6 +27,14 @@ src: dot-screenrc dest: /root/.screenrc +- name: Ensure /root/.ssh exists + file: + path: /root/.ssh + owner: root + group: root + mode: 0700 + state: directory + - name: Install packer key as only authorized key copy: src: packer_rsa @@ -51,13 +59,6 @@ group: root mode: 0500 -- name: Start common services - service: - name: '{{ item }}' - state: restarted - with_items: - - fail2ban - - name: Create bubble-log group group: name: bubble-log diff --git a/bubble-server/src/test/resources/models/system/cloudService.json b/bubble-server/src/test/resources/models/system/cloudService.json index c3ab4fc4..9779eb12 100644 --- a/bubble-server/src/test/resources/models/system/cloudService.json +++ b/bubble-server/src/test/resources/models/system/cloudService.json @@ -118,6 +118,41 @@ "template": true }, + { + "name": "DockerCompute", + "type": "compute", + "driverClass": "bubble.cloud.compute.docker.DockerComputeDriver", + "driverConfig": { + "regions": [{"name": "local", "internalName": "local"}], + "sizes": [{"name": "local", "type": "local", "internalName": "local"}], + "os": "ubuntu:20.04", + "packer": { + "vars": [], + "sudo": false, + "builder": { + "type": "docker", + "image": "<>", + "export_path": "<>.tar", + "changes": [ + "LABEL bubble_image=<>", + "EXPOSE 80 443 1202" + ] + }, + "post": { + "type": "docker-import", + "repository": "local/bubble", + "tag": "<>" + } + } + }, + "credentials": { + "params": [ + {"name": "host", "value": "unix:///var/run/docker.sock"} + ] + }, + "template": false + }, + { "_subst": true, "name": "VultrCompute", @@ -196,7 +231,7 @@ ], "config": [{"name": "os", "value": "Ubuntu 18.04 x64"}] }, - "credentials": { + "credentials": { "params": [ {"name": "API-Key", "value": "{{VULTR_API_KEY}}"} ] diff --git a/bubble-web b/bubble-web index 23b2a5c0..e0248d40 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit 23b2a5c0188618909feea52730e27e1c0685617e +Subproject commit e0248d40efe8394a13da7c5ba0a422827e623611 diff --git a/config/activation.json b/config/activation.json index 3544b924..0f1db8d4 100644 --- a/config/activation.json +++ b/config/activation.json @@ -70,6 +70,21 @@ } }, + // Docker can be used for testing or for advanced use cases + "DockerCompute": { + "config": {}, + "credentials": { + // these are the default settings, change as needed + "host": "unix:///var/run/docker.sock", + "tlsVerify": "false", // if tlsVerify is "true" then certPath must be set + "certPath": null, + "registryUrl": null, + "registryUsername": null, + "registryEmail": null, + "registryPassword": null + } + }, + /////////////////////// // Storage ///////////////////////