Browse Source

simplify activation

tags/v0.1.6
Jonathan Cobb 5 years ago
parent
commit
979193d137
37 changed files with 609 additions and 250 deletions
  1. +1
    -0
      .gitignore
  2. +0
    -84
      bin/activate
  3. +25
    -0
      bin/bactivate
  4. +3
    -0
      bin/build_dist
  5. +0
    -2
      bubble-server/src/main/java/bubble/cloud/CloudServiceDriver.java
  6. +1
    -1
      bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java
  7. +1
    -0
      bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java
  8. +1
    -1
      bubble-server/src/main/java/bubble/cloud/dns/DnsServiceDriver.java
  9. +10
    -2
      bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsDriver.java
  10. +12
    -7
      bubble-server/src/main/java/bubble/cloud/dns/route53/Route53DnsDriver.java
  11. +1
    -1
      bubble-server/src/main/java/bubble/cloud/geoTime/GeoTimeServiceDriver.java
  12. +1
    -1
      bubble-server/src/main/java/bubble/cloud/sms/twilio/TwilioSmsDriver.java
  13. +1
    -1
      bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriverBase.java
  14. +30
    -12
      bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java
  15. +13
    -0
      bubble-server/src/main/java/bubble/dao/cloud/CloudServiceDAO.java
  16. +4
    -0
      bubble-server/src/main/java/bubble/main/BubbleMain.java
  17. +17
    -1
      bubble-server/src/main/java/bubble/main/http/BubbleHttpEntityOptions.java
  18. +1
    -0
      bubble-server/src/main/java/bubble/model/account/Account.java
  19. +15
    -8
      bubble-server/src/main/java/bubble/model/boot/ActivationRequest.java
  20. +34
    -0
      bubble-server/src/main/java/bubble/model/boot/CloudServiceConfig.java
  21. +67
    -7
      bubble-server/src/main/java/bubble/model/cloud/CloudService.java
  22. +11
    -3
      bubble-server/src/main/java/bubble/resources/account/AuthResource.java
  23. +5
    -3
      bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java
  24. +9
    -2
      bubble-server/src/main/java/bubble/server/BubbleServer.java
  25. +112
    -34
      bubble-server/src/main/java/bubble/service/boot/ActivationService.java
  26. +1
    -1
      bubble-server/src/main/resources/bubble-config.yml
  27. +1
    -0
      bubble-server/src/main/resources/logback.xml
  28. +23
    -1
      bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties
  29. +80
    -18
      bubble-server/src/main/resources/models/dist/cloudService.json
  30. +4
    -3
      bubble-server/src/test/java/bubble/test/ActivatedBubbleModelTestBase.java
  31. +2
    -2
      bubble-server/src/test/resources/models/system/cloudService.json
  32. +1
    -46
      bubble-server/src/test/resources/models/system/cloudService_live.json
  33. +1
    -1
      bubble-web
  34. +102
    -0
      config/activation.json
  35. +17
    -6
      dist-README.md
  36. +1
    -1
      utils/cobbzilla-utils
  37. +1
    -1
      utils/cobbzilla-wizard

+ 1
- 0
.gitignore View File

@@ -14,3 +14,4 @@ target
.DS_Store

.BUBBLE*
configs

+ 0
- 84
bin/activate View File

@@ -1,84 +0,0 @@
#!/bin/bash
#
# Initial activation of a bubble server
#
# Usage: activate [domain] [dns] [storage]
#
# domain : a domain name. Must be listed in bubble-server/src/test/resources/models/system/bubbleDomain.json
# default value is bubblev.org
# dns : name of a CloudService of type 'dns'. Must be listed in bubble-server/src/test/resources/models/system/cloudService.json
# default is GoDaddyDns
# storage : name of a CloudService of type 'storage'. Must be listed in cloudService.json
# default is S3_US_Standard
#
# Environment variables
#
# BUBBLE_API : which API to use. Default is local (http://127.0.0.1:PORT, where PORT is found in .bubble.env)
# BUBBLE_USER : account to use. Default is root
# BUBBLE_PASS : password for account. Default is root
#
SCRIPT="${0}"
SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd)
. ${SCRIPT_DIR}/bubble_common

if [[ -z "${BUBBLE_JAR}" ]] ; then
die "BUBBLE_JAR env var not set and no jar file found"
fi

MODELS_DIR="${SCRIPT_DIR}/../bubble-server/src/test/resources/models/system"
if [[ ! -d ${MODELS_DIR} ]] ; then
die "Models directory not found: ${MODELS_DIR}"
fi

ENV_FILE="${HOME}/.bubble.env"
if [[ ! -f ${ENV_FILE} ]] ; then
die "env file not found: ${ENV_FILE}"
fi

# Source env vars
. ${ENV_FILE}

DOMAIN=${1:-bubblev.org}
DOMAINS_FILE="${MODELS_DIR}/bubbleDomain.json"
DOMAIN_JSON=$(cat ${DOMAINS_FILE} | sed -e 's,// .*,,g' | grep -v "_subst" | java -cp ${BUBBLE_JAR} bubble.main.BubbleMain handlebars | jq ".[] | select(.name==\"${DOMAIN}\")")
if [[ -z "${DOMAIN_JSON}" ]] ; then
die "Domain ${DOMAIN} not found in ${DOMAINS_FILE}"
fi

DNS_CLOUD=${2:-GoDaddyDns}
CLOUDS_FILE="${MODELS_DIR}/cloudService.json"
DNS_JSON=$(cat ${CLOUDS_FILE} | sed -e 's,// .*,,g' | grep -v "_subst" | java -cp ${BUBBLE_JAR} bubble.main.BubbleMain handlebars | jq ".[] | select(.name==\"${DNS_CLOUD}\")")
if [[ -z "${DNS_JSON}" ]] ; then
die "DNS CloudService ${DNS_CLOUD} not found in ${CLOUDS_FILE}"
fi
CLOUD_TYPE="$(echo ${DNS_JSON} | jq -r .type)"
if [[ -z "${CLOUD_TYPE}" ]] ; then
die "DNS service ${DNS_CLOUD} has no type"
fi
if [[ "${CLOUD_TYPE}" != 'dns' ]] ; then
die "DNS service ${DNS_CLOUD} has wrong type (${CLOUD_TYPE}), expected: dns"
fi

STORAGE_CLOUD=${2:-S3_US_Standard}
STORAGE_JSON=$(cat ${CLOUDS_FILE} | sed -e 's,// .*,,g' | grep -v "_subst" | java -cp ${BUBBLE_JAR} bubble.main.BubbleMain handlebars | jq ".[] | select(.name==\"${STORAGE_CLOUD}\")")
if [[ -z "${STORAGE_JSON}" ]] ; then
die "Storage CloudService ${STORAGE_CLOUD} not found in ${CLOUDS_FILE}"
fi
CLOUD_TYPE="$(echo ${STORAGE_JSON} | jq -r .type)"
if [[ -z "${CLOUD_TYPE}" ]] ; then
die "Storage service ${DNS_CLOUD} has no type"
fi
if [[ "${CLOUD_TYPE}" != 'storage' ]] ; then
die "Storage service ${DNS_CLOUD} has wrong type (${CLOUD_TYPE}), expected: storage"
fi

exec ${SCRIPT_DIR}/bput auth/activate - --no-login <<EOF
{
"name": "${BUBBLE_USER}",
"password": "${BUBBLE_PASS}",
"networkName": "$(hostname -s)",
"domain": ${DOMAIN_JSON},
"dns": ${DNS_JSON},
"storage": ${STORAGE_JSON}
}
EOF

+ 25
- 0
bin/bactivate View File

@@ -0,0 +1,25 @@
#!/bin/bash
#
# Initial activation of a bubble server
#
# Usage: activate activation.json
#
# activation.json : a JSON file containing the activation information.
# see config/activation.json for an example
#
# Environment variables
#
# BUBBLE_API : which API to use. Default is local (http://127.0.0.1:PORT, where PORT is found in .bubble.env)
# BUBBLE_USER : account to use. Default is root
# BUBBLE_PASS : password for account. Default is root
#
SCRIPT="${0}"
SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd)
. ${SCRIPT_DIR}/bubble_common

if [[ -z "${BUBBLE_JAR}" ]] ; then
die "BUBBLE_JAR env var not set and no jar file found"
fi

ACTIVATION_JSON="${1:?no activation json file provided}"
${SCRIPT_DIR}/bput <"${ACTIVATION_JSON}" auth/activate - --no-login

+ 3
- 0
bin/build_dist View File

@@ -47,3 +47,6 @@ fi
DIST="${DIST_BASE}/bubble-${VERSION}"
cp "${JAR}" "${DIST}/bubble.jar" || die "Error copying ${JAR} to ${DIST}/bubble.jar"
cp "${BASE}/dist-README.md" "${DIST}/README.md" || die "Error copying dist-README.md to ${DIST}/README.md"
cp -R "${BASE}/bin" "${DIST}" || die "Error copying bin directory to ${DIST}"
cp -R "${BASE}/scripts" "${DIST}" || die "Error copying scripts directory to ${DIST}"
cp -R "${BASE}/config" "${DIST}" || die "Error copying config directory to ${DIST}"

+ 0
- 2
bubble-server/src/main/java/bubble/cloud/CloudServiceDriver.java View File

@@ -24,8 +24,6 @@ public interface CloudServiceDriver {

default boolean disableDelegation () { return false; }

default void startDriver() {}

void setConfig(JsonNode json, CloudService cloudService);

static <T extends CloudServiceDriver> T setupDriver(BubbleConfiguration configuration, T driver) {


+ 1
- 1
bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriverBase.java View File

@@ -25,7 +25,7 @@ public abstract class ComputeServiceDriverBase

private final AtomicReference<NodeReaper> reaper = new AtomicReference<>();

@Override public void startDriver() {
@Override public void postSetup() {
if (configuration.isSelfSage()) {
synchronized (reaper) {
if (reaper.get() == null) {


+ 1
- 0
bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java View File

@@ -78,6 +78,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
final String apiKey = HandlebarsUtil.apply(getHandlebars(), credentials.getParam(API_KEY_HEADER), configuration.getEnvCtx());
credentials.setParam(API_KEY_HEADER, apiKey);
}
super.postSetup();
}

@Override public BubbleNode start(BubbleNode node) throws Exception {


+ 1
- 1
bubble-server/src/main/java/bubble/cloud/dns/DnsServiceDriver.java View File

@@ -48,7 +48,7 @@ public interface DnsServiceDriver extends CloudServiceDriver {

default Collection<DnsRecord> list() { return list(null); }

@Override default boolean test(Object arg) { return !empty(list((DnsRecordMatch) arg)); }
@Override default boolean test(Object arg) { return arg == null ? test() : !empty(list((DnsRecordMatch) arg)); }
@Override default boolean test() { return true; }

default Collection<DnsRecord> listNew(Long lastMod) { return list(); }


+ 10
- 2
bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsDriver.java View File

@@ -19,8 +19,7 @@ import java.util.stream.Collectors;
import static java.util.Collections.emptyList;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.http.HttpHeaders.CONTENT_TYPE;
import static org.cobbzilla.util.daemon.ZillaRuntime.die;
import static org.cobbzilla.util.daemon.ZillaRuntime.retry;
import static org.cobbzilla.util.daemon.ZillaRuntime.*;
import static org.cobbzilla.util.dns.DnsType.NS;
import static org.cobbzilla.util.dns.DnsType.SOA;
import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
@@ -143,6 +142,15 @@ public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
}
}

@Override public boolean test() {
final String url = config.getBaseUri();
try {
return HttpUtil.getResponse(auth(url)).isOk();
} catch (Exception e) {
return die("test: "+shortError(e));
}
}

@Override public Collection<DnsRecord> list(DnsRecordMatch matcher) {

final BubbleDomain domain = getDomain(matcher);


+ 12
- 7
bubble-server/src/main/java/bubble/cloud/dns/route53/Route53DnsDriver.java View File

@@ -45,10 +45,10 @@ public class Route53DnsDriver extends DnsDriverBase<Route53DnsConfig> {
}

@Getter(lazy=true) private final Map<String, HostedZone> cachedZoneLookups = new ExpirationMap<>();
private HostedZone getHostedZone(AmazonRoute53 client, BubbleDomain domain) {
private HostedZone getHostedZone(BubbleDomain domain) {
return getCachedZoneLookups().computeIfAbsent(domain.getName(), key -> {
try {
final ListHostedZonesResult zones = client.listHostedZones(new ListHostedZonesRequest().withMaxItems("100"));
final ListHostedZonesResult zones = getRoute53client().listHostedZones(new ListHostedZonesRequest().withMaxItems("100"));
for (HostedZone z : zones.getHostedZones()) {
if (z.getName().equalsIgnoreCase(key + ".")) return z;
}
@@ -59,10 +59,15 @@ public class Route53DnsDriver extends DnsDriverBase<Route53DnsConfig> {
});
}

@Override public boolean test() {
getRoute53client().listHostedZones(new ListHostedZonesRequest().withMaxItems("10"));
return true;
}

@Override public Collection<DnsRecord> create(BubbleDomain domain) {

final AmazonRoute53 client = getRoute53client();
final HostedZone hostedZone = getHostedZone(client, domain);
final HostedZone hostedZone = getHostedZone(domain);

final ListResourceRecordSetsResult soaRecords = client.listResourceRecordSets(new ListResourceRecordSetsRequest()
.withHostedZoneId(hostedZone.getId())
@@ -144,7 +149,7 @@ public class Route53DnsDriver extends DnsDriverBase<Route53DnsConfig> {
@Override public DnsRecord update(DnsRecord record) {
final AmazonRoute53 client = getRoute53client();
final BubbleDomain domain = getDomain(record.getFqdn());
final HostedZone hostedZone = getHostedZone(client, domain);
final HostedZone hostedZone = getHostedZone(domain);
final ChangeResourceRecordSetsResult changeResult = client.changeResourceRecordSets(new ChangeResourceRecordSetsRequest()
.withHostedZoneId(hostedZone.getId())
.withChangeBatch(new ChangeBatch().withChanges(new Change()
@@ -159,7 +164,7 @@ public class Route53DnsDriver extends DnsDriverBase<Route53DnsConfig> {
@Override public DnsRecord remove(DnsRecord record) {
final AmazonRoute53 client = getRoute53client();
final BubbleDomain domain = getDomain(record.getFqdn());
final HostedZone hostedZone = getHostedZone(client, domain);
final HostedZone hostedZone = getHostedZone(domain);

final ListResourceRecordSetsRequest listRequest = new ListResourceRecordSetsRequest()
.withHostedZoneId(hostedZone.getId())
@@ -186,11 +191,11 @@ public class Route53DnsDriver extends DnsDriverBase<Route53DnsConfig> {
}

@Override public Collection<DnsRecord> list(DnsRecordMatch matcher) {
final BubbleDomain domain = getDomain(matcher.getFqdn());
final BubbleDomain domain = getDomain(matcher);
if (domain == null) return emptyList();

final AmazonRoute53 client = getRoute53client();
final HostedZone hostedZone = getHostedZone(client, domain);
final HostedZone hostedZone = getHostedZone(domain);
final ListResourceRecordSetsRequest listRequest = new ListResourceRecordSetsRequest()
.withHostedZoneId(hostedZone.getId())
.withStartRecordName(matcher.hasFqdn() ? matcher.getFqdn() : domain.getName())


+ 1
- 1
bubble-server/src/main/java/bubble/cloud/geoTime/GeoTimeServiceDriver.java View File

@@ -9,7 +9,7 @@ public interface GeoTimeServiceDriver extends CloudServiceDriver {
String TEST_LONGITUDE = "-73.944";
String TEST_LATITUDE = "40.661";
String TEST_STANDARD_NAME = "Eastern Standard Time";
String TEST_TIMEZONE_ID = "America/New York";
String TEST_TIMEZONE_ID = "America/New_York";

@Override default CloudServiceType getType() { return CloudServiceType.geoTime; }



+ 1
- 1
bubble-server/src/main/java/bubble/cloud/sms/twilio/TwilioSmsDriver.java View File

@@ -25,7 +25,7 @@ public class TwilioSmsDriver extends SmsServiceDriverBase<TwilioSmsConfig> {

@Autowired @Getter protected BubbleConfiguration configuration;

@Override public void startDriver() {
@Override public void postSetup() {
synchronized (twilioInitDone) {
if (!twilioInitDone.get()) {
sid = getCredentials().getParam(PARAM_ACCOUNT_SID);


+ 1
- 1
bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriverBase.java View File

@@ -19,7 +19,7 @@ public abstract class StorageServiceDriverBase<T> extends CloudServiceDriverBase

private static final Map<String, WriteRequestCleaner> cleaners = new ConcurrentHashMap<>();

@Override public void startDriver() {
@Override public void postSetup() {
final String key = sha256_hex(json(getConfig())+":"+json(getCredentials()));
synchronized (cleaners) {
WriteRequestCleaner cleaner = cleaners.get(key);


+ 30
- 12
bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java View File

@@ -10,6 +10,7 @@ import bubble.model.cloud.CloudService;
import bubble.model.cloud.StorageMetadata;
import bubble.notify.storage.StorageListing;
import lombok.Cleanup;
import lombok.Getter;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.cobbzilla.util.io.FileUtil;
@@ -23,8 +24,7 @@ import java.util.List;

import static bubble.ApiConstants.ROOT_NETWORK_UUID;
import static bubble.dao.cloud.AnsibleRoleDAO.ROLE_PATH;
import static org.cobbzilla.util.daemon.ZillaRuntime.die;
import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported;
import static org.cobbzilla.util.daemon.ZillaRuntime.*;
import static org.cobbzilla.util.io.FileUtil.*;
import static org.cobbzilla.util.json.JsonUtil.json;
import static org.cobbzilla.util.security.ShaUtil.sha256_file;
@@ -38,12 +38,30 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
@Autowired private AccountDAO accountDAO;

public static final String LOCAL_STORAGE = "LocalStorage";
public static final String LOCAL_STORAGE_STANDARD_BASE_DIR = "/home/bubble/.bubble_local_storage";
public static final String BUBBLE_LOCAL_STORAGE_DIR = ".bubble_local_storage";
public static final String LOCAL_STORAGE_STANDARD_BASE_DIR = "/home/bubble/" + BUBBLE_LOCAL_STORAGE_DIR;

public static final Class[] FATAL_EX_CLASSES = {FileNotFoundException.class};

@Override public Class[] getFatalExceptionClasses() { return FATAL_EX_CLASSES; }

@Getter(lazy=true) private final String baseDir = initBaseDir();
public String initBaseDir() {
if (!empty(config.getBaseDir())) return config.getBaseDir();

final File standardBaseDir = new File(LOCAL_STORAGE_STANDARD_BASE_DIR);
if ((standardBaseDir.exists() || standardBaseDir.mkdirs()) && standardBaseDir.canRead() && standardBaseDir.canWrite()) {
return abs(standardBaseDir);
}

final File userBaseDir = new File(System.getProperty("user.home")+"/"+BUBBLE_LOCAL_STORAGE_DIR);
if ((userBaseDir.exists() || userBaseDir.mkdirs()) && userBaseDir.canRead() && userBaseDir.canWrite()) {
return abs(userBaseDir);
}

return die("getBaseDir: not set and no defaults exist");
}

@Override public boolean _exists(String fromNode, String key) throws IOException {
final BubbleNode from = getFromNode(fromNode);
if (from != null) {
@@ -55,7 +73,7 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
if (activated() || !key.startsWith(ROLE_PATH)) return false;

// check root network filesystem
final File file = keyFileForNetwork(ROOT_NETWORK_UUID, config.getBaseDir(), key);
final File file = keyFileForNetwork(ROOT_NETWORK_UUID, getBaseDir(), key);
if (file.exists()) return true;

// check classpath
@@ -85,7 +103,7 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
if (activated() || !key.startsWith(ROLE_PATH)) return null;

// check root network filesystem
final File rootNetFile = keyFileForNetwork(ROOT_NETWORK_UUID, config.getBaseDir(), key);
final File rootNetFile = keyFileForNetwork(ROOT_NETWORK_UUID, getBaseDir(), key);
if (rootNetFile.exists()) return new FileInputStream(rootNetFile);

// check classpath
@@ -93,7 +111,7 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
if (in == null) return null;

// copy file to root network storage, so we can find it after activation
final File file = keyFileForNetwork(ROOT_NETWORK_UUID, config.getBaseDir(), key);
final File file = keyFileForNetwork(ROOT_NETWORK_UUID, getBaseDir(), key);
@Cleanup OutputStream out = new FileOutputStream(file);
IOUtils.copyLarge(in, out);
return new FileInputStream(file);
@@ -172,14 +190,14 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
return node != null ? node : !activated() ? null : die("fromNode not found: "+fromNode);
}

private File keyFile(BubbleNode from, String key) { return keyFile(from, config.getBaseDir(), key); }
private File keyFileNoNetwork(String key) { return keyFileNoNetwork(config.getBaseDir(), key); }
private File keyFile(BubbleNode from, String key) { return keyFile(from, getBaseDir(), key); }
private File keyFileNoNetwork(String key) { return keyFileNoNetwork(getBaseDir(), key); }

public static File keyFile(BubbleNode from, String baseDir, String key) {
return keyFileForNetwork(from.getNetwork(), baseDir, key);
}
public File keyFileForEntireNetwork(String network) {
return keyFileForNetwork(network, config.getBaseDir(), "");
return keyFileForNetwork(network, getBaseDir(), "");
}
public static File keyFileForNetwork(String network, String baseDir, String key) {
return new File(baseDir + "/" + network + "/" + key);
@@ -189,14 +207,14 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
}

public void migrateInitialData(BubbleNetwork network) {
final File base = new File(config.getBaseDir());
final File base = new File(getBaseDir());
final File[] matched = base.listFiles((file, s) -> new File(file, s).isDirectory() && !UUID_PATTERN.matcher(s).find());
if (matched != null) {
Arrays.stream(matched)
.forEach(f -> {
final String path = abs(f);
final String key = path.substring(config.getBaseDir().length());
final File dest = new File(config.getBaseDir() + "/" + network.getUuid() + key);
final String key = path.substring(getBaseDir().length());
final File dest = new File(getBaseDir() + "/" + network.getUuid() + key);
try {
FileUtils.copyDirectory(f, dest);
} catch (IOException e) {


+ 13
- 0
bubble-server/src/main/java/bubble/dao/cloud/CloudServiceDAO.java View File

@@ -1,5 +1,6 @@
package bubble.dao.cloud;

import bubble.cloud.storage.local.LocalStorageDriver;
import bubble.dao.account.AccountOwnedTemplateDAO;
import bubble.model.cloud.CloudService;
import bubble.cloud.CloudServiceType;
@@ -8,11 +9,23 @@ import org.springframework.stereotype.Repository;

import java.util.List;

import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE;
import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;

@Repository
public class CloudServiceDAO extends AccountOwnedTemplateDAO<CloudService> {

@Override public Order getDefaultSortOrder() { return Order.desc("priority"); }

@Override public Object preCreate(CloudService cloud) {
if (cloud.getType() == CloudServiceType.storage
&& cloud.getName().equals(LOCAL_STORAGE)
&& !cloud.getDriver().getClass().equals(LocalStorageDriver.class)) {
throw invalidEx("err.cloud.localStorageIsReservedName");
}
return super.preCreate(cloud);
}

@Override public CloudService postUpdate(CloudService cloud, Object context) {
CloudService.clearDriverCache(cloud.getUuid());
return super.postUpdate(cloud, context);


+ 4
- 0
bubble-server/src/main/java/bubble/main/BubbleMain.java View File

@@ -8,6 +8,7 @@ import bubble.server.BubbleServer;
import org.cobbzilla.util.collection.MapBuilder;
import org.cobbzilla.util.string.StringUtil;
import org.cobbzilla.wizard.main.MainBase;
import org.slf4j.bridge.SLF4JBridgeHandler;

import java.util.Map;
import java.util.TreeSet;
@@ -34,6 +35,9 @@ public class BubbleMain {

if (args.length == 0) die(noCommandProvided());

SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();

// extract command
final String command = args[0];
final Class mainClass = mainClasses.get(command.toLowerCase());


+ 17
- 1
bubble-server/src/main/java/bubble/main/http/BubbleHttpEntityOptions.java View File

@@ -1,16 +1,32 @@
package bubble.main.http;

import com.fasterxml.jackson.databind.JsonNode;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.entity.ContentType;
import org.kohsuke.args4j.Option;

import static org.cobbzilla.util.daemon.ZillaRuntime.readStdin;
import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
import static org.cobbzilla.util.json.JsonUtil.*;

@Slf4j
public class BubbleHttpEntityOptions extends BubbleHttpOptions {

public String getRequestJson() { return readStdin(); }
public String getRequestJson() {
final String data = readStdin();
// does the JSON contain any comments? scrub them before sending...
if (data.contains("//") || data.contains("/*")) {
try {
return json(json(data, JsonNode.class, FULL_MAPPER_ALLOW_COMMENTS), COMPACT_MAPPER);
} catch (Exception e) {
log.warn("getRequestJson: error scrubbing comments from JSON, sending as-is: "+shortError(e));
}
}
return data;
}

public static final String USAGE_CONTENT_TYPE = "Content-Type to send. Default is application/json";
public static final String OPT_CONTENT_TYPE = "-C";


+ 1
- 0
bubble-server/src/main/java/bubble/model/account/Account.java View File

@@ -3,6 +3,7 @@ package bubble.model.account;
import bubble.dao.account.AccountInitializer;
import bubble.model.app.AppData;
import bubble.model.app.BubbleApp;
import bubble.model.boot.ActivationRequest;
import bubble.model.cloud.*;
import bubble.model.cloud.notify.SentNotification;
import bubble.model.device.Device;


bubble-server/src/main/java/bubble/model/account/ActivationRequest.java → bubble-server/src/main/java/bubble/model/boot/ActivationRequest.java View File

@@ -1,4 +1,4 @@
package bubble.model.account;
package bubble.model.boot;

import bubble.model.cloud.AnsibleRole;
import bubble.model.cloud.BubbleDomain;
@@ -9,6 +9,9 @@ import lombok.Setter;
import lombok.experimental.Accessors;
import org.cobbzilla.wizard.validation.HasValue;

import java.util.LinkedHashMap;
import java.util.Map;

import static org.cobbzilla.util.daemon.ZillaRuntime.empty;

@NoArgsConstructor @Accessors(chain=true)
@@ -26,18 +29,22 @@ public class ActivationRequest {
public boolean hasDescription() { return !empty(description); }

@HasValue(message="err.networkName.required")
@Getter @Setter private String networkName;
@Getter @Setter private String networkName = "boot-network";

@Getter @Setter private AnsibleRole[] roles;
public boolean hasRoles () { return roles != null && roles.length > 0; }
public boolean hasRoles () { return !empty(roles); }

@Getter @Setter private Map<String, CloudServiceConfig> cloudConfigs = new LinkedHashMap<>();
public boolean hasCloudConfigs () { return !empty(cloudConfigs); }
public ActivationRequest addCloudConfig(CloudService cloud) {
cloudConfigs.put(cloud.getName(), cloud.toCloudConfig());
return this;
}

@HasValue(message="err.domain.required")
@Getter @Setter private BubbleDomain domain;

@HasValue(message="err.dns.required")
@Getter @Setter private CloudService dns;

@HasValue(message="err.storage.required")
@Getter @Setter private CloudService storage;
@Getter @Setter private Boolean createDefaultObjects = true;
public boolean createDefaultObjects () { return createDefaultObjects != null && createDefaultObjects; };

}

+ 34
- 0
bubble-server/src/main/java/bubble/model/boot/CloudServiceConfig.java View File

@@ -0,0 +1,34 @@
package bubble.model.boot;

import bubble.model.cloud.CloudCredentials;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.Accessors;
import org.cobbzilla.util.collection.NameAndValue;

import java.util.LinkedHashMap;
import java.util.Map;

import static org.cobbzilla.util.daemon.ZillaRuntime.empty;

@NoArgsConstructor @Accessors(chain=true)
public class CloudServiceConfig {

@Getter @Setter private Map<String, String> config = new LinkedHashMap<>();
public boolean hasConfig() { return !empty(config); }

public CloudServiceConfig addConfig(String name, String value) {
getConfig().put(name, value);
return this;
}

@Getter @Setter private Map<String, String> credentials = new LinkedHashMap<>();
public boolean hasCredentials() { return !empty(credentials); }

@JsonIgnore public CloudCredentials getCredentialsObject() {
return new CloudCredentials(NameAndValue.map2list(getCredentials()).toArray(NameAndValue.EMPTY_ARRAY));
}

}

+ 67
- 7
bubble-server/src/main/java/bubble/model/cloud/CloudService.java View File

@@ -13,9 +13,11 @@ import bubble.cloud.storage.StorageServiceDriver;
import bubble.dao.cloud.CloudServiceDAO;
import bubble.model.account.Account;
import bubble.model.account.AccountTemplate;
import bubble.model.boot.CloudServiceConfig;
import bubble.server.BubbleConfiguration;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
@@ -23,22 +25,26 @@ import lombok.experimental.Accessors;
import org.cobbzilla.util.collection.ArrayUtil;
import org.cobbzilla.util.collection.ExpirationMap;
import org.cobbzilla.util.collection.HasPriority;
import org.cobbzilla.util.collection.NameAndValue;
import org.cobbzilla.wizard.filters.Scrubbable;
import org.cobbzilla.wizard.filters.ScrubbableField;
import org.cobbzilla.wizard.model.Identifiable;
import org.cobbzilla.wizard.model.entityconfig.IdentifiableBaseParentEntity;
import org.cobbzilla.wizard.model.entityconfig.annotations.*;
import org.cobbzilla.wizard.validation.HasValue;
import org.cobbzilla.wizard.validation.ValidationResult;
import org.hibernate.annotations.Type;

import javax.persistence.*;
import javax.validation.constraints.Size;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import static bubble.ApiConstants.EP_CLOUDS;
import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE;
import static org.cobbzilla.util.daemon.ZillaRuntime.die;
import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
import static org.cobbzilla.util.json.JsonUtil.json;
import static org.cobbzilla.util.reflect.ReflectionUtil.*;
import static org.cobbzilla.wizard.model.crypto.EncryptedTypes.ENCRYPTED_STRING;
@@ -75,6 +81,8 @@ public class CloudService extends IdentifiableBaseParentEntity implements Accoun
@HasValue(message="err.name.required")
@ECIndex @Column(nullable=false, updatable=false, length=200)
@Getter @Setter private String name;
@JsonIgnore public boolean isLocalStorage () { return name != null && name.equals(LOCAL_STORAGE); }
@JsonIgnore public boolean isNotLocalStorage () { return !isLocalStorage(); }

@ECSearchable @ECField(index=20)
@ECForeignKey(entity=Account.class)
@@ -143,6 +151,10 @@ public class CloudService extends IdentifiableBaseParentEntity implements Accoun

@Transient public CloudCredentials getCredentials () { return credentialsJson == null ? null : json(credentialsJson, CloudCredentials.class); }
public CloudService setCredentials (CloudCredentials credentials) { return setCredentialsJson(credentials == null ? null : json(credentials)); }
public boolean hasCredentials () {
final CloudCredentials creds = getCredentials();
return creds != null && !empty(creds.getParams());
}

@Transient @JsonIgnore @Getter(lazy=true) private final CloudServiceDriver driver = initDriver();
private CloudServiceDriver initDriver () {
@@ -152,17 +164,31 @@ public class CloudService extends IdentifiableBaseParentEntity implements Accoun
return d;
}

@Transient @JsonIgnore public CloudServiceDriver getConfiguredDriver (BubbleConfiguration configuration) {
switch (getType()) {
case compute: return getComputeDriver(configuration);
case email: case sms:
case authenticator: return getAuthenticationDriver(configuration);
case geoLocation: return getGeoLocateDriver(configuration);
case geoCode: return getGeoCodeDriver(configuration);
case geoTime: return getGeoTimeDriver(configuration);
case dns: return getDnsDriver(configuration);
case storage: return getStorageDriver(configuration);
case payment: return getPaymentDriver(configuration);
default:
log.warn("getConfiguredDriver: unrecognized type: "+getType());
case local:
return wireAndSetup(configuration);
}
}

@Transient @JsonIgnore public ComputeServiceDriver getComputeDriver (BubbleConfiguration configuration) {
final ComputeServiceDriver compute = wireAndSetup(configuration);
compute.startDriver();
return compute;
return wireAndSetup(configuration);
}

@Transient @JsonIgnore public AuthenticationDriver getAuthenticationDriver(BubbleConfiguration configuration) {
if (!getType().isAuthenticationType()) return die("getAuthenticationDriver: not an authentication type: "+getType());
final AuthenticationDriver driver = wireAndSetup(configuration);
driver.startDriver();
return driver;
return wireAndSetup(configuration);
}

@Transient @JsonIgnore public RegionalServiceDriver getRegionalDriver () { return (RegionalServiceDriver) getDriver(); }
@@ -180,7 +206,7 @@ public class CloudService extends IdentifiableBaseParentEntity implements Accoun
return (DnsServiceDriver) wireAndSetup(configuration);
}
@Transient @JsonIgnore public StorageServiceDriver getStorageDriver(BubbleConfiguration configuration) {
if (!getName().equals(LOCAL_STORAGE)) {
if (isNotLocalStorage()) {
final CloudCredentials credentials = getCredentials();
final BubbleNetwork thisNetwork = configuration.getThisNetwork();
if (thisNetwork != null && credentials.needsNewNetworkKey(thisNetwork.getUuid())) {
@@ -217,4 +243,38 @@ public class CloudService extends IdentifiableBaseParentEntity implements Accoun
});
}

public CloudService configure(CloudServiceConfig config, ValidationResult errors) {
if (config.hasConfig()) {
final JsonNode driverConfig = getDriverConfig();
for (Map.Entry<String, String> cfg : config.getConfig().entrySet()) {
final String name = cfg.getKey();
if (driverConfig == null || !driverConfig.has(name)) {
errors.addViolation("err.cloud.noSuchField", "driver config field does not exist: "+name, name);
} else if (errors.isValid()) {
((ObjectNode) driverConfig).put(name, cfg.getValue());
}
}
}
if (config.hasCredentials()) {
setCredentials(config.getCredentialsObject());
} else {
setCredentials(null);
}
return this;
}

public CloudServiceConfig toCloudConfig() {
final CloudServiceConfig config = new CloudServiceConfig();
final JsonNode driverConfig = getDriverConfig();
if (driverConfig != null) {
for (Iterator<String> iter = driverConfig.fieldNames(); iter.hasNext(); ) {
final String field = iter.next();
config.addConfig(field, driverConfig.get(field).textValue());
}
}
if (hasCredentials()) {
config.setCredentials(NameAndValue.toMap(getCredentials().getParams()));
}
return config;
}
}

+ 11
- 3
bubble-server/src/main/java/bubble/resources/account/AuthResource.java View File

@@ -8,6 +8,7 @@ import bubble.dao.cloud.BubbleNodeDAO;
import bubble.model.CertType;
import bubble.model.account.*;
import bubble.model.account.message.*;
import bubble.model.boot.ActivationRequest;
import bubble.model.cloud.BubbleNetwork;
import bubble.model.cloud.BubbleNode;
import bubble.model.cloud.NetworkKeys;
@@ -78,15 +79,22 @@ public class AuthResource {
@GET @Path(EP_ACTIVATE)
public Response isActivated(@Context ContainerRequest ctx) { return ok(accountDAO.activated()); }

@GET @Path(EP_ACTIVATE+EP_CONFIGS)
public Response getActivationConfigs(@Context ContainerRequest ctx) {
final Account caller = optionalUserPrincipal(ctx);
if (accountDAO.activated() && (caller == null || !caller.admin())) return ok();
return ok(activationService.getCloudDefaults());
}

@Transactional
@PUT @Path(EP_ACTIVATE)
public Response activate(@Context Request req,
@Context ContainerRequest ctx,
@Valid ActivationRequest request) {
if (request == null) return invalid("err.activation.request.required");
final Account found = optionalUserPrincipal(ctx);
if (found != null) {
if (!found.admin()) return forbidden();
final Account caller = optionalUserPrincipal(ctx);
if (caller != null) {
if (!caller.admin()) return forbidden();
return invalid("err.activation.alreadyDone", "activation has already been done");
}
if (accountDAO.activated()) {


+ 5
- 3
bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java View File

@@ -13,7 +13,10 @@ import bubble.model.account.Account;
import bubble.model.bill.AccountPaymentMethod;
import bubble.model.bill.AccountPlan;
import bubble.model.bill.BubblePlan;
import bubble.model.cloud.*;
import bubble.model.cloud.BubbleDomain;
import bubble.model.cloud.BubbleFootprint;
import bubble.model.cloud.BubbleNetwork;
import bubble.model.cloud.CloudService;
import bubble.resources.account.AccountOwnedResource;
import bubble.server.BubbleConfiguration;
import lombok.extern.slf4j.Slf4j;
@@ -31,7 +34,6 @@ import java.util.List;
import java.util.stream.Collectors;

import static bubble.ApiConstants.*;
import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE;
import static org.cobbzilla.wizard.resources.ResourceUtil.*;

@Slf4j
@@ -150,7 +152,7 @@ public class AccountPlansResource extends AccountOwnedResource<AccountPlan, Acco
final List<CloudService> storageClouds = cloudDAO.findByAccountAndType(caller.getUuid(), CloudServiceType.storage);

// find the first one that is not LocalStorage
final List<CloudService> remoteStorage = storageClouds.stream().filter(c -> !c.getName().equals(LOCAL_STORAGE)).collect(Collectors.toList());
final List<CloudService> remoteStorage = storageClouds.stream().filter(CloudService::isNotLocalStorage).collect(Collectors.toList());

if (!remoteStorage.isEmpty()) {
// todo: storage should know what region it is in.


+ 9
- 2
bubble-server/src/main/java/bubble/server/BubbleServer.java View File

@@ -13,8 +13,10 @@ import org.cobbzilla.wizard.server.RestServerBase;
import org.cobbzilla.wizard.server.RestServerLifecycleListener;
import org.cobbzilla.wizard.server.RestServerLifecycleListenerBase;
import org.cobbzilla.wizard.server.config.factory.ConfigurationSource;
import org.cobbzilla.wizard.server.listener.BrowserLauncherListener;
import org.cobbzilla.wizard.server.listener.FlywayMigrationListener;
import org.cobbzilla.wizard.server.listener.SystemInitializerListener;
import org.slf4j.bridge.SLF4JBridgeHandler;

import java.io.File;
import java.util.Arrays;
@@ -43,7 +45,8 @@ public class BubbleServer extends RestServerBase<BubbleConfiguration> {
new FlywayMigrationListener<BubbleConfiguration>(),
new NodeInitializerListener(),
new DeviceInitializerListener(),
new BubbleFirstTimeListener()
new BubbleFirstTimeListener(),
new BrowserLauncherListener()
});

public static final List<RestServerLifecycleListener> RESTORE_LIFECYCLE_LISTENERS = Arrays.asList(new RestServerLifecycleListener[] {
@@ -52,7 +55,8 @@ public class BubbleServer extends RestServerBase<BubbleConfiguration> {

public static final String[] DEFAULT_ENV_FILE_PATHS = {
HOME_DIR + ".bubble.env",
HOME_DIR + "/current/bubble.env"
HOME_DIR + "/current/bubble.env",
System.getProperty("user.dir") + "/bubble.env"
};

private static AtomicReference<String> restoreKey = new AtomicReference<>();
@@ -67,6 +71,9 @@ public class BubbleServer extends RestServerBase<BubbleConfiguration> {

// config is loaded from the classpath
public static void main(String[] args) throws Exception {
SLF4JBridgeHandler.removeHandlersForRootLogger();
SLF4JBridgeHandler.install();

final Map<String, String> env = loadEnvironment(args);
final ConfigurationSource configSource = getConfigurationSource();



+ 112
- 34
bubble-server/src/main/java/bubble/service/boot/ActivationService.java View File

@@ -7,20 +7,29 @@ import bubble.cloud.compute.local.LocalComputeDriver;
import bubble.cloud.dns.DnsServiceDriver;
import bubble.cloud.storage.local.LocalStorageConfig;
import bubble.cloud.storage.local.LocalStorageDriver;
import bubble.dao.account.AccountDAO;
import bubble.dao.cloud.*;
import bubble.model.account.Account;
import bubble.model.account.ActivationRequest;
import bubble.model.boot.ActivationRequest;
import bubble.model.boot.CloudServiceConfig;
import bubble.model.cloud.*;
import bubble.server.BubbleConfiguration;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.cobbzilla.util.dns.DnsRecordMatch;
import org.cobbzilla.util.dns.DnsType;
import org.cobbzilla.util.handlebars.HandlebarsUtil;
import org.cobbzilla.wizard.api.CrudOperation;
import org.cobbzilla.wizard.client.ApiClientBase;
import org.cobbzilla.wizard.model.Identifiable;
import org.cobbzilla.wizard.model.ModelSetupService;
import org.cobbzilla.wizard.validation.SimpleViolationException;
import org.cobbzilla.wizard.validation.ValidationResult;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.File;
import java.util.Arrays;
import java.util.*;

import static bubble.ApiConstants.ROOT_NETWORK_UUID;
import static bubble.cloud.storage.StorageServiceDriver.STORAGE_PREFIX;
@@ -30,14 +39,16 @@ import static bubble.model.cloud.BubbleFootprint.DEFAULT_FOOTPRINT_OBJECT;
import static bubble.model.cloud.BubbleNetwork.TAG_ALLOW_REGISTRATION;
import static bubble.model.cloud.BubbleNetwork.TAG_PARENT_ACCOUNT;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.cobbzilla.util.daemon.ZillaRuntime.die;
import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
import static java.util.function.Function.identity;
import static java.util.stream.Collectors.toMap;
import static org.cobbzilla.util.daemon.ZillaRuntime.*;
import static org.cobbzilla.util.io.FileUtil.toStringOrDie;
import static org.cobbzilla.util.io.StreamUtil.stream2string;
import static org.cobbzilla.util.json.JsonUtil.json;
import static org.cobbzilla.util.network.NetworkUtil.getFirstPublicIpv4;
import static org.cobbzilla.util.network.NetworkUtil.getLocalhostIpv4;
import static org.cobbzilla.util.system.CommandShell.execScript;
import static org.cobbzilla.util.system.Sleep.sleep;
import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;

@Service @Slf4j
@@ -45,7 +56,7 @@ public class ActivationService {

public static final String DEFAULT_ROLES = "ansible/default_roles.json";

public static final long ROOT_CREATE_TIMEOUT = SECONDS.toMillis(10);
public static final long ACTIVATION_TIMEOUT = SECONDS.toMillis(10);

@Autowired private AnsibleRoleDAO roleDAO;
@Autowired private CloudServiceDAO cloudDAO;
@@ -56,6 +67,7 @@ public class ActivationService {
@Autowired private BubbleNodeKeyDAO nodeKeyDAO;
@Autowired private StandardSelfNodeService selfNodeService;
@Autowired private BubbleConfiguration configuration;
@Autowired private ModelSetupService modelSetupService;

public BubbleNode bootstrapThisNode(Account account, ActivationRequest request) {
String ip = getFirstPublicIpv4();
@@ -65,35 +77,75 @@ public class ActivationService {
}
if (ip == null) die("bootstrapThisNode: no IP could be found, not even a localhost address");

final CloudService publicDns = cloudDAO.create(new CloudService(request.getDns())
.setType(CloudServiceType.dns)
.setTemplate(true)
.setAccount(account.getUuid()));
final DnsServiceDriver dnsDriver = publicDns.getDnsDriver(configuration);
final DnsRecordMatch nsMatcher = (DnsRecordMatch) new DnsRecordMatch()
.setType(DnsType.NS)
.setFqdn(request.getDomain().getName());
checkDriver(dnsDriver, nsMatcher, "err.dns.testFailed", "err.dns.unknownError");

final CloudService localStorage = cloudDAO.create(new CloudService()
.setAccount(account.getUuid())
.setType(CloudServiceType.storage)
.setDriverClass(LocalStorageDriver.class.getName())
.setDriverConfigJson(json(new LocalStorageConfig().setBaseDir(configuration.getLocalStorageDir())))
.setName(LOCAL_STORAGE)
.setTemplate(true));
final Map<String, CloudServiceConfig> requestConfigs = request.getCloudConfigs();
final Map<String, CloudService> defaultConfigs = getCloudDefaultsMap();
final ValidationResult errors = new ValidationResult();
final List<CloudService> toCreate = new ArrayList<>();
CloudService publicDns = null;
CloudService localStorage = null;
CloudService storage = null;
CloudService compute = null;
for (Map.Entry<String, CloudServiceConfig> requestedCloud : requestConfigs.entrySet()) {
final String name = requestedCloud.getKey();
final CloudServiceConfig config = requestedCloud.getValue();
final CloudService defaultCloud = defaultConfigs.get(name);
if (defaultCloud == null) {
errors.addViolation("err.cloud.notFound", "No cloud template found with name: "+name, name);

final CloudService storage = request.getStorage();
final CloudService networkStorage;
if (storage.getName().equals(LOCAL_STORAGE)) {
networkStorage = localStorage;
} else {
if (storage.getCredentials().needsNewNetworkKey(ROOT_NETWORK_UUID)) {
storage.setCredentials(storage.getCredentials().initNetworkKey(ROOT_NETWORK_UUID));
} else if (errors.isValid()) {
final CloudService cloud = new CloudService(defaultCloud).configure(config, errors);
toCreate.add(cloud);
if (defaultCloud.getType() == CloudServiceType.dns && defaultCloud.getName().equals(request.getDomain().getPublicDns()) && publicDns == null) publicDns = cloud;
if (defaultCloud.getType() == CloudServiceType.storage && localStorage == null && defaultCloud.isLocalStorage()) localStorage = cloud;
if (defaultCloud.getType() == CloudServiceType.storage && storage == null) storage = cloud;
if (defaultCloud.getType() == CloudServiceType.compute && compute == null) compute = cloud;
}
}
if (publicDns == null) errors.addViolation("err.dns.noneSpecified");
if (storage == null) errors.addViolation("err.storage.noneSpecified");
if (compute == null && !configuration.testMode()) errors.addViolation("err.compute.noneSpecified");
if (errors.isInvalid()) throw invalidEx(errors);

// create local storage if it was not provided
if (localStorage == null) {
localStorage = cloudDAO.create(new CloudService()
.setAccount(account.getUuid())
.setType(CloudServiceType.storage)
.setDriverClass(LocalStorageDriver.class.getName())
.setDriverConfigJson(json(new LocalStorageConfig().setBaseDir(configuration.getLocalStorageDir())))
.setName(LOCAL_STORAGE)
.setTemplate(true));
}

// create all clouds
CloudService networkStorage = localStorage;
for (CloudService cloud : toCreate) {
final CloudService c = cloudDAO.create(cloud
.setTemplate(true)
.setEnabled(true)
.setAccount(account.getUuid()));
if (cloud == publicDns) {
final DnsServiceDriver dnsDriver = publicDns.getDnsDriver(configuration);
if (cloud.getName().equals(request.getDomain().getPublicDns())) {
final DnsRecordMatch nsMatcher = (DnsRecordMatch) new DnsRecordMatch()
.setType(DnsType.NS)
.setFqdn(request.getDomain().getName());
checkDriver(dnsDriver, nsMatcher, "err.dns.testFailed", "err.dns.unknownError");
} else {
checkDriver(dnsDriver, null, "err.dns.testFailed", "err.dns.unknownError");
}

} else if (cloud == storage && cloud.isNotLocalStorage()) {
networkStorage = cloud; // prefer non-local storage for network
if (storage.getCredentials().needsNewNetworkKey(ROOT_NETWORK_UUID)) {
storage.setCredentials(storage.getCredentials().initNetworkKey(ROOT_NETWORK_UUID));
}
final CloudServiceDriver storageDriver = networkStorage.getStorageDriver(configuration);
checkDriver(storageDriver, null, "err.storage.testFailed", "err.storage.unknownError");

} else {
checkDriver(cloud.getConfiguredDriver(configuration), null, "err."+cloud.getType()+".testFailed", "err."+cloud.getType()+".unknownError");
}
networkStorage = cloudDAO.create(new CloudService(storage).setAccount(account.getUuid()));
final CloudServiceDriver storageDriver = networkStorage.getStorageDriver(configuration);
checkDriver(storageDriver, null, "err.storage.testFailed", "err.storage.unknownError");
}

final AnsibleRole[] roles = request.hasRoles() ? request.getRoles() : json(loadDefaultRoles(), AnsibleRole[].class);
@@ -162,16 +214,33 @@ public class ActivationService {
selfNodeService.initThisNode(node);
configuration.refreshPublicSystemConfigs();

if (request.createDefaultObjects()) {
background(() -> {
// wait for activation to complete
final AccountDAO accountDAO = configuration.getBean(AccountDAO.class);
final long start = now();
while (!accountDAO.activated() && now() - start < ACTIVATION_TIMEOUT) {
sleep(SECONDS.toMillis(1), "waiting for activation to complete before creating default objects");
}
if (!accountDAO.activated()) die("bootstrapThisNode: timeout waiting for activation to complete, default objects not created");

final ApiClientBase api = configuration.newApiClient().setToken(account.getToken());
final Map<CrudOperation, Collection<Identifiable>> objects
= modelSetupService.setupModel(api, account, "manifest-dist");
log.info("bootstrapThisNode: created default objects\n"+json(objects));
});
}

return node;
}

public void checkDriver(CloudServiceDriver driver, Object arg, String errTestFailed, String errException) {
try {
if (!driver.test(arg)) throw invalidEx(errTestFailed);
if (!driver.test(arg)) throw invalidEx(errTestFailed, "test failed for driver: "+driver.getClass().getName()+(arg != null ? " with arg="+arg : ""), (arg == null ? null : arg.toString()));
} catch (SimpleViolationException e) {
throw e;
} catch (Exception e) {
throw invalidEx(errException, shortError(e));
throw invalidEx(errException, "test failed for driver: "+driver.getClass().getName()+(arg != null ? " with arg="+arg : "")+": "+shortError(e), (arg == null ? null : arg.toString()));
}
}

@@ -191,4 +260,13 @@ public class ActivationService {
return networkDAO.create(network);
}

@Getter(lazy=true) private final CloudService[] cloudDefaults = initCloudDefaults();
private CloudService[] initCloudDefaults() {
return json(HandlebarsUtil.apply(configuration.getHandlebars(), stream2string("models/dist/cloudService.json"), configuration.getEnvCtx()), CloudService[].class);
}

@Getter(lazy=true) private final Map<String, CloudService> cloudDefaultsMap = initCloudDefaultsMap();
private Map<String, CloudService> initCloudDefaultsMap() {
return Arrays.stream(getCloudDefaults()).collect(toMap(CloudService::getName, identity()));
}
}

+ 1
- 1
bubble-server/src/main/resources/bubble-config.yml View File

@@ -3,7 +3,6 @@

serverName: bubble-api

#publicUriBase: https://127.0.0.1
publicUriBase: {{PUBLIC_BASE_URI}}

defaultLocale: {{#exists BUBBLE_DEFAULT_LOCALE}}{{BUBBLE_DEFAULT_LOCALE}}{{else}}en_US{{/exists}}
@@ -35,6 +34,7 @@ staticAssets:
baseUri: /
assetRoot: site/
localOverride: {{BUBBLE_ASSETS_DIR}}
singlePageApp: /index.html
utilPaths:
INDEX_PATH: /index.html
INDEX_ALIASES: /:/index.php


+ 1
- 0
bubble-server/src/main/resources/logback.xml View File

@@ -36,6 +36,7 @@
<logger name="com.moparisthebest.dns.listen.intercept" level="ERROR" />
<logger name="bubble.cloud.email.SmtpEmailDriver" level="DEBUG" />
<logger name="org.cobbzilla.wizard.dao.AbstractCRUDDAO" level="DEBUG" />
<logger name="org.cobbzilla.wizard.server.listener.BrowserLauncherListener" level="INFO" />
<logger name="bubble" level="INFO" />

<root level="INFO">


+ 23
- 1
bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties View File

@@ -75,7 +75,23 @@ form_section_title_credentials=Credentials
form_section_title_config=Configuration
button_label_activate=Activate

# Activation errors
# Activation cloud test errors
err.compute.testFailed=Error testing connection to Compute service
err.compute.unknownError=Error connecting to Compute service
err.geoCode.testFailed=Error testing connection to GeoCode service
err.geoCode.unknownError=Error connecting to GeoCode service
err.payment.testFailed=Error testing connection to Payment service
err.payment.unknownError=Error connecting to Payment service
err.geoLocation.testFailed=Error testing connection to GeoLocation service
err.geoLocation.unknownError=Error connecting to GeoLocation service
err.geoTime.testFailed=Error testing connection to GeoTime service
err.geoTime.unknownError=Error connecting to GeoTime service
err.email.testFailed=Error testing connection to Email service
err.email.unknownError=Error connecting to Email service
err.sms.testFailed=Error testing connection to SMS service
err.sms.unknownError=Error connecting to SMS service
err.authenticator.testFailed=Error testing connection to Authenticator service
err.authenticator.unknownError=Error connecting to Authenticator service
err.dns.testFailed=Error testing connection to DNS service
err.dns.unknownError=Error connecting to DNS service
err.storage.testFailed=Error testing connection to Storage service
@@ -113,6 +129,12 @@ err.token.invalid=Code is incorrect

# Low-level errors
err.driverConfig.initFailure=Cloud driver failed to initialize properlyu
err.cloud.noSuchField=A cloud driver config field name is invalid
err.cloud.notFound=No cloud exists with this name
err.dns.noneSpecified=No DNS service was configured
err.storage.noneSpecified=No Storage service was configured
err.compute.noneSpecified=No Compute service was configured
err.cloud.localStorageIsReservedName=LocalStorage is a reserved name

# Entity config errors
err.ec.param.invalid=Parameter is invalid


+ 80
- 18
bubble-server/src/main/resources/models/dist/cloudService.json View File

@@ -1,15 +1,68 @@
[
{
"_subst": true,
"name": "Route53Dns",
"type": "dns",
"driverClass": "bubble.cloud.dns.route53.Route53DnsDriver",
"driverConfig": {},
"credentials": {
"params": [
{"name": "AWS_ACCESS_KEY_ID", "value": "{{AWS_ACCESS_KEY_ID}}"},
{"name": "AWS_SECRET_KEY", "value": "{{AWS_SECRET_KEY}}"}
]
},
"template": true
},

{
"name": "GoDaddyDns",
"type": "dns",
"driverClass": "bubble.cloud.dns.godaddy.GoDaddyDnsDriver",
"driverConfig": {},
"credentials": {
"params": [
{"name": "GODADDY_API_KEY", "value": "{{GODADDY_API_KEY}}"},
{"name": "GODADDY_API_SECRET", "value": "{{GODADDY_API_SECRET}}"}
]
},
"template": true
},

{
"name": "LocalStorage",
"type": "storage",
"driverClass": "bubble.cloud.storage.local.LocalStorageDriver",
"driverConfig": { "baseDir": "{{LOCALSTORAGE_BASE_DIR}}" },
"template": false
},

{
"name": "S3_US_Standard",
"type": "storage",
"driverClass": "bubble.cloud.storage.s3.S3StorageDriver",
"driverConfig": {
"region": "US_EAST_1",
"bucket": "{{BUBBLE_S3_BUCKET}}",
"prefix": "{{BUBBLE_S3_BUCKET_PREFIX}}",
"listFetchSize": 100
},
"credentials": {
"params": [
{"name": "AWS_ACCESS_KEY_ID", "value": "{{AWS_ACCESS_KEY_ID}}"},
{"name": "AWS_SECRET_KEY", "value": "{{AWS_SECRET_KEY}}"}
]
},
"template": true
},

{
"name": "GoogleGeoCoder",
"type": "geoCode",
"driverClass": "bubble.cloud.geoCode.google.GoogleGeoCodeDriver",
"credentials": { "params": [ {"name": "apiKey", "value": "{{required GOOGLE_API_KEY}}"} ] },
"credentials": { "params": [ {"name": "apiKey", "value": "{{GOOGLE_API_KEY}}"} ] },
"template": true
},

{
"_subst": true,
"name": "TOTPAuthenticator",
"type": "authenticator",
"driverClass": "bubble.cloud.authenticator.TOTPAuthenticatorDriver",
@@ -17,7 +70,6 @@
},

{
"_subst": true,
"name": "SmtpServer",
"type": "email",
"driverClass": "{{BUBBLE_SMTP_DRIVER}}",
@@ -26,42 +78,53 @@
},
"credentials": {
"params": [
{"name": "user", "value": "{{required BUBBLE_SMTP_USER}}"},
{"name": "password", "value": "{{required BUBBLE_SMTP_PASS}}"},
{"name": "host", "value": "{{required BUBBLE_SMTP_SERVER}}"},
{"name": "port", "value": "{{required BUBBLE_SMTP_PORT}}"}
{"name": "user", "value": "{{BUBBLE_SMTP_USER}}"},
{"name": "password", "value": "{{BUBBLE_SMTP_PASS}}"},
{"name": "host", "value": "{{BUBBLE_SMTP_SERVER}}"},
{"name": "port", "value": "{{BUBBLE_SMTP_PORT}}"}
]
},
"template": true
},

{
"_subst": true,
"name": "TwilioSms",
"type": "sms",
"driverClass": "{{BUBBLE_SMS_DRIVER}}",
"driverConfig": {},
"credentials": {
"params": [
{"name": "accountSID", "value": "{{required TWILIO_ACCOUNT_SID}}"},
{"name": "authToken", "value": "{{required TWILIO_AUTH_TOKEN}}"},
{"name": "fromPhoneNumber", "value": "{{required TWILIO_FROM_PHONE_NUMBER}}"}
{"name": "accountSID", "value": "{{TWILIO_ACCOUNT_SID}}"},
{"name": "authToken", "value": "{{TWILIO_AUTH_TOKEN}}"},
{"name": "fromPhoneNumber", "value": "{{TWILIO_FROM_PHONE_NUMBER}}"}
]
},
"template": true
},

{
"_subst": true,
"name": "MaxMind",
"type": "geoLocation",
"driverClass": "bubble.cloud.geoLocation.maxmind.MaxMindDriver",
"driverConfig": {
"url": "{{{MAXMIND_URL}}}",
"file": "{{{MAXMIND_FILE_REGEX}}}"
},
"credentials": {
"params": [ {"name": "apiKey", "value": "{{{MAXMIND_API_KEY}}}"} ]
},
"template": true
},

{
"name": "GoogleGeoTime",
"type": "geoTime",
"driverClass": "bubble.cloud.geoTime.google.GoogleGeoTimeDriver",
"credentials": { "params": [ {"name": "apiKey", "value": "{{required GOOGLE_API_KEY}}"} ] },
"credentials": { "params": [ {"name": "apiKey", "value": "{{GOOGLE_API_KEY}}"} ] },
"template": true
},

{
"_subst": true,
"name": "VultrCompute",
"type": "compute",
"driverClass": "bubble.cloud.compute.vultr.VultrDriver",
@@ -140,14 +203,13 @@
},
"credentials": {
"params": [
{"name": "API-Key", "value": "{{required VULTR_API_KEY}}"}
{"name": "API-Key", "value": "{{VULTR_API_KEY}}"}
]
},
"template": true
},

{
"_subst": true,
"name": "DigitalOceanCompute",
"type": "compute",
"driverClass": "bubble.cloud.compute.digitalocean.DigitalOceanDriver",
@@ -198,7 +260,7 @@
},
"credentials": {
"params": [
{"name": "apiKey", "value": "{{required DIGITALOCEAN_API_KEY}}"}
{"name": "apiKey", "value": "{{DIGITALOCEAN_API_KEY}}"}
]
},
"template": true


+ 4
- 3
bubble-server/src/test/java/bubble/test/ActivatedBubbleModelTestBase.java View File

@@ -5,7 +5,7 @@ import bubble.cloud.CloudServiceType;
import bubble.cloud.dns.godaddy.GoDaddyDnsDriver;
import bubble.cloud.storage.local.LocalStorageDriver;
import bubble.model.account.Account;
import bubble.model.account.ActivationRequest;
import bubble.model.boot.ActivationRequest;
import bubble.model.cloud.BubbleDomain;
import bubble.model.cloud.CloudService;
import bubble.server.BubbleConfiguration;
@@ -109,8 +109,9 @@ public abstract class ActivatedBubbleModelTestBase extends BubbleModelTestBase {
.setName(ROOT_USERNAME)
.setPassword(ROOT_PASSWORD)
.setNetworkName(hostname_short())
.setDns(dns)
.setStorage(storage)
.addCloudConfig(dns)
.addCloudConfig(storage)
.setCreateDefaultObjects(false)
.setDomain(domain), Account.class);
} catch (ValidationException e) {
if (e.hasViolations() && e.getViolations().containsKey("err.activation.alreadyDone")) {


+ 2
- 2
bubble-server/src/test/resources/models/system/cloudService.json View File

@@ -6,8 +6,8 @@
"driverClass": "bubble.cloud.storage.s3.S3StorageDriver",
"driverConfig": {
"region": "US_EAST_1",
"bucket": "dev.bubblev.org",
"prefix": "2020-01",
"bucket": "{{BUBBLE_S3_BUCKET}}",
"prefix": "{{BUBBLE_S3_PREFIX}}",
"listFetchSize": 100
},
"credentials": {


+ 1
- 46
bubble-server/src/test/resources/models/system/cloudService_live.json View File

@@ -27,51 +27,6 @@
"params": [ {"name": "apiKey", "value": "{{{MAXMIND_API_KEY}}}"} ]
},
"template": true
},

{
"_subst": true,
"name": "VultrCompute",
"type": "compute",
"driverClass": "bubble.cloud.compute.vultr.VultrDriver",
"driverConfig": {
"regions": [{
"name": "Vultr - New Jersey",
"internalName": "New Jersey",
"location": {"city": "Newark", "country": "US", "region": "NJ", "lat": "40.72", "lon": "-74.17"}
}, {
"name": "Vultr - Atlanta",
"internalName": "Atlanta",
"location": {"city": "Atlanta", "country": "US", "region": "GA", "lat": "33.755", "lon": "-84.39"}
}, {
"name": "Vultr - Chicago",
"internalName": "Chicago",
"location": {"city": "Chicago", "country": "US", "region": "IL", "lat": "41.881944", "lon": "-87.627778"}
}, {
"name": "Vultr - San Jose",
"internalName": "Silicon Valley",
"location": {"city": "San Jose", "country": "US", "region": "CA", "lat": "37.333333", "lon": "-121.9"}
}, {
"name": "Vultr - London",
"internalName": "London",
"location": {"city": "London", "country": "GB", "region": "London", "lat": "51.507222", "lon": "-0.1275"}
}, {
"name": "Vultr - Paris",
"internalName": "Paris",
"location": {"city": "Paris", "country": "FR", "region": "Ile-de-Paris", "lat": "48.8567", "lon": "2.3508"}
}],
"sizes": [
{"name": "small", "type": "small", "internalName": "1024 MB RAM,25 GB SSD,1.00 TB BW"},
{"name": "medium", "type": "medium", "internalName": "4096 MB RAM,80 GB SSD,3.00 TB BW"},
{"name": "large", "type": "large", "internalName": "8192 MB RAM,160 GB SSD,4.00 TB BW"}
],
"config": [{"name": "os", "value": "Ubuntu 18.04 x64"}]
},
"credentials": {
"params": [
{"name": "API-Key", "value": "{{VULTR_API_KEY}}"}
]
},
"template": true
}

]

+ 1
- 1
bubble-web

@@ -1 +1 @@
Subproject commit 2b405967c0ea92f619ab43071e32f876cfb66b53
Subproject commit 89a819baba806857daab90eb7539cbc9f1f665c1

+ 102
- 0
config/activation.json View File

@@ -0,0 +1,102 @@
{
// name, password and description of the initial admin user
"name": "root",
"password": "REPLACE WITH YOUR ROOT PASSWORD",
"description": "root user",

"cloudConfigs" : {
// You must configure at least one of these DNS services
// Comment the other one out if you're not going to use it
"Route53Dns" : {
"credentials" : {
"AWS_ACCESS_KEY_ID": "your_aws_access_key_id",
"AWS_SECRET_KEY": "your_aws_secret_key"
}
},
"GoDaddyDns" : {
"credentials" : {
"GODADDY_API_KEY": "your_godaddy_api_key",
"GODADDY_API_SECRET": "your_godaddy_api_secret"
}
},

// You must configure at least one of these Compute services
// Comment the other one out if you're not going to use it
"VultrCompute": {
"credentials": {"API-Key": "your_vultr_api_key"}
},
"DigitalOceanCompute": {
"credentials": {"apiKey": "your_digitalocean_api_key"}
},

// You must configure the AWS S3 Storage service in order to launch new Bubbles
"S3_US_Standard" : {
"config": {
// region must be a valid value from the Regions enum: https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/regions/Regions.java
"region": "US_EAST_1",
"bucket": "must be a valid bucket name that the credentials can read/write/list/delete",
"prefix": "", // optional: all paths for S3 operations will be prefixed with this path
"listFetchSize": 100
},
"credentials" : {
"AWS_ACCESS_KEY_ID": "your_aws_access_key_id",
"AWS_SECRET_KEY": "your_aws_secret_key"
}
},

// Required for TOTP-based authentication. Highly recommended.
"TOTPAuthenticator": {},

// OPTIONAL SERVICES
// Comment out any services you have not configured

// Required for sending emails
"SmtpServer": {
"config": {
"tlsEnabled": true
},
"credentials": {
"user": "your_smtp_username",
"password": "your_smtp_password",
"host": "your_smtp_server_hostname",
"port": "your_smtp_server_port"
}
},

// Required for sending SMS messages
"TwilioSms": {
"credentials": {
"accountSID": "your_twilio_account_SID",
"authToken": "your_twilio_auth_token",
// text messages sent by Bubble will come "from" this phone number, must be in Twilio-compatible format
"fromPhoneNumber": "your_twilio_from_number"
}
},

// Required for locale and "nearest compute region" auto-detection
"MaxMind": {
"config": {
// these values work for the free GeoLite database. you still have to specify an apiKey
"url": "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=[[apiKey]]&suffix=tar.gz",
"file": "GeoLite2-City_20[\\\\d]{6}/GeoLite2-City\\\\.mmdb"
},
"credentials": {"apiKey": "your_maxmind_api_key"}
},

// Required for "nearest compute region" auto-detection
"GoogleGeoCoder": {
"credentials": {"apiKey": "your_google_api_key"}
},

// Required for timezone auto-detection
"GoogleGeoTime": {
"credentials": {"apiKey": "your_google_api_key"}
}
},

// the domain that new Bubbles will be launched within
"domain": {
"publicDns": "Route53Dns or GoDaddyDns", // name of a DNS provider configured above
"name": "example.com" // a domain name that you own, that is managed by the DNS provider named in `publicDns`
}
}

+ 17
- 6
dist-README.md View File

@@ -18,8 +18,6 @@ will also be "yours only" -- all Bubbles disconnect from their launcher during c

## Getting Started

### Download a Bubble Distribution

### Install PostgreSQL and Redis
Install [PostgreSQL](https://www.postgresql.org/download/) if it is not installed on your system.
It will probably be easier to install using an OS package, for example `sudo apt install postgresql`
@@ -38,13 +36,26 @@ Otherwise, please either:
* Create a PostgreSQL database named `bubble` and a database user named `bubble`. Set a password for the `bubble` user,
and set the environment variable `BUBBLE_PG_PASSWORD` to this password when starting the Bubble launcher.

### Download a Bubble Distribution
Download and unzip the latest [Bubble Distribution ZIP](https://bubblev.com/download).

### Start the Bubble launcher
Running a Bubble locally
Run the `./bin/run.sh` script to start the Bubble launcher. Once the server is running, it will try to open a browser window
to continue configuration. It will also print out the URL, so if the browser doesn't start correctly, you can paste this
into your browser's location bar.

### Activate your local Bubble
Your Bubble is running locally in a "blank" mode. It needs an initial "root" account and some basic services configured.

#### Activate via Web UI
The browser-based admin UI should be displaying an "Activate" page. Complete the information on this page and submit the
data. The Bubble Launcher will create an initial "root" account and other basic system configurations.

#### Activate using the Web UI
#### Activate via command line
Copy the file in `config/activation.json`, then edit the file. There are comments in the file to guide you.
After saving the updated file, run this command:

#### Activate using the command line
`./bin/bactivate /path/to/activation.json`

### Configure Cloud Services
### Launch a new Bubble!
Using the web UI, click "Bubbles", select "New Bubble". Fill out and submit the New Bubble form, and your Bubble will be created!

+ 1
- 1
utils/cobbzilla-utils

@@ -1 +1 @@
Subproject commit 94b18c1772002c013c1cc2a856126378aebc7460
Subproject commit 4f62fa3e49dd90282d48860cf60a46e51a3f6aa0

+ 1
- 1
utils/cobbzilla-wizard

@@ -1 +1 @@
Subproject commit 76727a2b02643af97d03535e474c483804664bce
Subproject commit 9b6ec9b885c1d42bee7a1e19c4e8e15c2f900941

Loading…
Cancel
Save