diff --git a/bubble-server/src/main/java/bubble/ApiConstants.java b/bubble-server/src/main/java/bubble/ApiConstants.java index 62190b18..f4c4582b 100644 --- a/bubble-server/src/main/java/bubble/ApiConstants.java +++ b/bubble-server/src/main/java/bubble/ApiConstants.java @@ -147,6 +147,7 @@ public class ApiConstants { public static final String EP_STOP = "/stop"; public static final String EP_RESTORE = "/restore"; public static final String EP_KEYS = "/keys"; + public static final String EP_STATUS = "/status"; public static final String EP_FORK = "/fork"; public static final String DEBUG_ENDPOINT = "/debug"; diff --git a/bubble-server/src/main/java/bubble/cloud/geoCode/GeoCodeDriverBase.java b/bubble-server/src/main/java/bubble/cloud/geoCode/GeoCodeDriverBase.java index b1aa82f7..bd0da0ea 100644 --- a/bubble-server/src/main/java/bubble/cloud/geoCode/GeoCodeDriverBase.java +++ b/bubble-server/src/main/java/bubble/cloud/geoCode/GeoCodeDriverBase.java @@ -11,6 +11,7 @@ import org.springframework.beans.factory.annotation.Autowired; import java.util.concurrent.TimeUnit; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; @Slf4j public abstract class GeoCodeDriverBase extends CloudServiceDriverBase implements GeoCodeServiceDriver { @@ -46,7 +47,7 @@ public abstract class GeoCodeDriverBase extends CloudServiceDriverBase imp ttl = ERROR_TTL; } val = json(r); - getCache().set(key, val, "EX", ttl); + getCache().set(key, val, EX, ttl); } return valOrError(val, GeoCodeResult.class); } diff --git a/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocateServiceDriverBase.java b/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocateServiceDriverBase.java index ba153f97..43905bbf 100644 --- a/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocateServiceDriverBase.java +++ b/bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocateServiceDriverBase.java @@ -25,6 +25,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf; import static org.cobbzilla.util.io.FileUtil.*; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.security.ShaUtil.sha256_hex; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; @Slf4j public abstract class GeoLocateServiceDriverBase extends CloudServiceDriverBase implements GeoLocateServiceDriver { @@ -59,7 +60,7 @@ public abstract class GeoLocateServiceDriverBase extends CloudServiceDriverBa ttl = ERROR_TTL; } val = json(loc); - getCache().set(ip, val, "EX", ttl); + getCache().set(ip, val, EX, ttl); } return valOrError(val, GeoLocation.class); } diff --git a/bubble-server/src/main/java/bubble/cloud/geoTime/GeoTimeServiceDriverBase.java b/bubble-server/src/main/java/bubble/cloud/geoTime/GeoTimeServiceDriverBase.java index eae9a8e8..8a36461c 100644 --- a/bubble-server/src/main/java/bubble/cloud/geoTime/GeoTimeServiceDriverBase.java +++ b/bubble-server/src/main/java/bubble/cloud/geoTime/GeoTimeServiceDriverBase.java @@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Autowired; import java.util.concurrent.TimeUnit; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; @Slf4j public abstract class GeoTimeServiceDriverBase extends CloudServiceDriverBase implements GeoTimeServiceDriver { @@ -45,7 +46,7 @@ public abstract class GeoTimeServiceDriverBase extends CloudServiceDriverBase ttl = ERROR_TTL; } val = json(tz); - getCache().set(key, val, "EX", ttl); + getCache().set(key, val, EX, ttl); } return valOrError(val, GeoTimeZone.class); } diff --git a/bubble-server/src/main/java/bubble/cloud/payment/stripe/StripePaymentDriver.java b/bubble-server/src/main/java/bubble/cloud/payment/stripe/StripePaymentDriver.java index d345daee..7f8b35b6 100644 --- a/bubble-server/src/main/java/bubble/cloud/payment/stripe/StripePaymentDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/payment/stripe/StripePaymentDriver.java @@ -29,6 +29,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @Slf4j @@ -162,7 +163,7 @@ public class StripePaymentDriver extends PaymentDriverBase { @@ -254,7 +255,7 @@ public class S3StorageDriver extends StorageServiceDriverBase { listing.getObjectSummaries().forEach(o -> keys.add(o.getKey().substring(rootPrefix.length()))); final ListingRequest listingRequest = new ListingRequest(key, listing); - getActiveListings().set(listRequestId, json(listingRequest), "EX", LISTING_TIMEOUT); + getActiveListings().set(listRequestId, json(listingRequest), EX, LISTING_TIMEOUT); return new StorageListing() .setListingId(listRequestId) @@ -274,7 +275,7 @@ public class S3StorageDriver extends StorageServiceDriverBase { listingRequest.objectListing = s3client.listNextBatchOfObjects(listingRequest.objectListing); listingRequest.objectListing.getObjectSummaries().forEach(o -> keys.add(o.getKey().substring(rootPrefix.length()))); - activeListings.set(listingId, json(listingRequest), "EX", LISTING_TIMEOUT); + activeListings.set(listingId, json(listingRequest), EX, LISTING_TIMEOUT); return new StorageListing() .setListingId(listingId) diff --git a/bubble-server/src/main/java/bubble/notify/NewNodeNotification.java b/bubble-server/src/main/java/bubble/notify/NewNodeNotification.java index 7fe458a7..9a84c0a3 100644 --- a/bubble-server/src/main/java/bubble/notify/NewNodeNotification.java +++ b/bubble-server/src/main/java/bubble/notify/NewNodeNotification.java @@ -5,11 +5,13 @@ import lombok.NoArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; +import static java.util.UUID.randomUUID; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @NoArgsConstructor @Accessors(chain=true) public class NewNodeNotification { + @Getter @Setter private String uuid = randomUUID().toString(); @Getter @Setter private String account; @Getter @Setter private String host; @Getter @Setter private String network; diff --git a/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java b/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java index cf4e86d2..4aebc074 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java @@ -16,6 +16,7 @@ import bubble.model.bill.AccountPlan; import bubble.model.cloud.*; import bubble.server.BubbleConfiguration; import bubble.service.backup.NetworkKeysService; +import bubble.service.cloud.NodeProgressMeterTick; import bubble.service.cloud.StandardNetworkService; import lombok.extern.slf4j.Slf4j; import org.glassfish.grizzly.http.server.Request; @@ -73,6 +74,15 @@ public class NetworkActionsResource { return _startNetwork(network, cloud, region, req); } + @GET @Path(EP_STATUS+"/{uuid}") + public Response requestLaunchStatus(@Context Request req, + @Context ContainerRequest ctx, + @PathParam("uuid") String uuid) { + final Account caller = userPrincipal(ctx); + final NodeProgressMeterTick tick = networkService.getLaunchStatus(caller, uuid); + return tick == null ? notFound(uuid) : ok(tick); + } + @GET @Path(EP_KEYS) public Response requestNetworkKeys(@Context Request req, @Context ContainerRequest ctx) { diff --git a/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java b/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java index 6b8f14c7..e3d84c6c 100644 --- a/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java +++ b/bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java @@ -2,7 +2,6 @@ package bubble.server.listener; import bubble.ApiConstants; import bubble.dao.account.AccountDAO; -import bubble.dao.account.message.AccountMessageDAO; import bubble.model.account.Account; import bubble.model.account.message.AccountAction; import bubble.model.account.message.AccountMessage; @@ -22,6 +21,7 @@ import java.util.concurrent.atomic.AtomicReference; import static java.util.concurrent.TimeUnit.HOURS; import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; @Slf4j public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase { @@ -53,7 +53,7 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBaser.getName().startsWith("bubble-"))) { diff --git a/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java new file mode 100644 index 00000000..481c86ce --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java @@ -0,0 +1,151 @@ +package bubble.service.cloud; + +import bubble.notify.NewNodeNotification; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.io.FileUtil; +import org.cobbzilla.wizard.cache.redis.RedisService; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static bubble.service.cloud.NodeProgressMeterConstants.*; +import static java.util.Arrays.asList; +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.daemon.ZillaRuntime.terminate; +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.reflect.ReflectionUtil.closeQuietly; +import static org.cobbzilla.util.system.Bytes.KB; +import static org.cobbzilla.util.system.Sleep.sleep; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; + +@Slf4j @Accessors(chain=true) +public class NodeProgressMeter extends PipedOutputStream implements Runnable { + + public static final int PIPE_SIZE = (int) (16*KB); + + public static final long TICK_REDIS_EXPIRATION = DAYS.toSeconds(1); + + private final List standardTicks; + + private BufferedReader reader; + private BufferedWriter writer; + private List ticks; + private int tickPos = 0; + private AtomicBoolean error = new AtomicBoolean(false); + private AtomicBoolean closed = new AtomicBoolean(false); + private final Thread thread; + + private RedisService redis; + private NewNodeNotification nn; + private String key; + private final NodeProgressMeterTick lastStandardTick; + + public NodeProgressMeter(NewNodeNotification nn, RedisService redis) throws IOException { + + this.nn = nn; + this.redis = redis; + + standardTicks = getStandardTicks(); + lastStandardTick = standardTicks.get(standardTicks.size()-1); + + ticks = new ArrayList<>(standardTicks); + final NodeProgressMeterTick[] installTicks = json(stream2string(TICKS_JSON), NodeProgressMeterTick[].class); + for (NodeProgressMeterTick tick : installTicks) { + tick.setAccount(nn.getAccount()).relativizePercent(lastStandardTick.getPercent()); + } + ticks.addAll(asList(installTicks)); + + key = nn.getUuid(); + + final PipedInputStream pipeIn = new PipedInputStream(PIPE_SIZE); + connect(pipeIn); + + reader = new BufferedReader(new InputStreamReader(pipeIn)); + writer = new BufferedWriter(new OutputStreamWriter(this)); + + thread = new Thread(this); + thread.setDaemon(true); + thread.start(); + } + + public void write(String line) throws IOException { + writer.write(line.endsWith("\n") ? line : line+"\n"); + writer.flush(); + } + + public void error(String line) { + if (error.get()) { + log.warn("error("+line+") ignored, error already set"); + return; + } + error.set(true); + close(); + setCurrentTick(errorTick(getErrorMessageKey(line), line)); + } + + private NodeProgressMeterTick errorTick(String messageKey, String line) { + return new NodeProgressMeterTick() + .setAccount(nn.getAccount()) + .setMessageKey(messageKey) + .setDetails(line); + } + + public void reset() { + error.set(false); + tickPos = standardTicks.size(); + setCurrentTick(lastStandardTick); + } + + @Override public void run() { + String line; + try { + final File file = new File("/tmp/node_launch_progress.txt"); + while ((line = reader.readLine()) != null && !closed.get()) { + FileUtil.appendFile(file, now()+" : "+line+"\n"); + for (int i=tickPos; i STANDARD_TICKS = MapBuilder.build(new Object[][] { + {METER_TICK_CONFIRMING_NETWORK_LOCK, 1}, + {METER_TICK_VALIDATING_NODE_NETWORK_AND_PLAN, 2}, + {METER_TICK_CREATING_NODE, 3}, + {METER_TICK_LAUNCHING_NODE, 4}, + {METER_TICK_PREPARING_ROLES, 9}, + {METER_TICK_WRITING_DNS_RECORDS, 10}, + {METER_TICK_PREPARING_INSTALL, 12}, + {METER_TICK_AWAITING_DNS, 13}, + {METER_TICK_STARTING_INSTALL, 19}, + {METER_TICK_COPYING_ANSIBLE, 20}, + {METER_TICK_RUNNING_ANSIBLE, 24} + }); + + public static List getStandardTicks() { + final List ticks = new ArrayList<>(); + for (Field f : getAllFields(NodeProgressMeterConstants.class)) { + if (isStaticFinalString(f, TICK_PREFIX)) { + final String value = constValue(f); + final Integer percent = STANDARD_TICKS.get(value); + if (percent == null) return die("getStandardTicks: "+f.getName()+" entry missing from STANDARD_TICKS"); + ticks.add(new NodeProgressMeterTick() + .setPattern(value) + .setExact(true) + .setStandard(true) + .setMessageKey(f.getName().toLowerCase()) + .setPercent(percent)); + } + } + return ticks; + } + + public static final Map ERRORS = initErrors(); + private static Map initErrors() { + final Map errors = new HashMap<>(); + for (Field f : getAllFields(NodeProgressMeterConstants.class)) { + if (isStaticFinalString(f, ERROR_PREFIX)) { + errors.put(constValue(f), f.getName().toLowerCase()); + } + } + return errors; + } + + public static String getErrorMessageKey (String error) { + final String messageKey = ERRORS.get(error); + return messageKey != null ? messageKey : METER_UNKNOWN_ERROR; + } + +} diff --git a/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterTick.java b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterTick.java new file mode 100644 index 00000000..c4be0f46 --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterTick.java @@ -0,0 +1,40 @@ +package bubble.service.cloud; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.util.regex.Pattern; + +@Accessors(chain=true) +public class NodeProgressMeterTick { + + @Getter @Setter private String account; + + @Getter @Setter private String pattern; + @JsonIgnore @Getter(lazy=true) private final Pattern _pattern = Pattern.compile(getPattern()); + + @Getter @Setter private Boolean exact; + public boolean exact() { return exact != null && exact; } + + @Getter @Setter private Boolean standard; + public boolean standard() { return standard != null && standard; } + + @Getter @Setter private Integer percent; + + public NodeProgressMeterTick relativizePercent(int lastStandardPercent) { + setPercent(Math.round(((float) lastStandardPercent) + (100f - lastStandardPercent) * getPercent() / 100f)); + return this; + } + + @Getter @Setter private String messageKey; + @Getter @Setter private String details; + + public boolean matches(String line) { + return exact() + ? line.trim().equals(getPattern().trim()) + : get_pattern().matcher(line).matches(); + } + +} diff --git a/bubble-server/src/main/java/bubble/service/cloud/RequestCoordinationService.java b/bubble-server/src/main/java/bubble/service/cloud/RequestCoordinationService.java index b32d4073..1b8b17cf 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/RequestCoordinationService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/RequestCoordinationService.java @@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; @Service public class RequestCoordinationService { @@ -15,11 +16,11 @@ public class RequestCoordinationService { @Getter(lazy=true) private final RedisService requests = redisService.prefixNamespace(getClass().getSimpleName()+"_"); public void set(String prefix, String id, JsonNode thing) { - getRequests().set(prefix+":"+id, json(thing), "EX", 600); + getRequests().set(prefix+":"+id, json(thing), EX, 600); } public void set(String prefix, String id, String thing) { - getRequests().set(prefix+":"+id, thing, "EX", 600); + getRequests().set(prefix+":"+id, thing, EX, 600); } public String get(String prefix, String id) { diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index c0a11e20..6ffc22cc 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -15,8 +15,8 @@ import bubble.model.cloud.notify.NotificationReceipt; import bubble.model.cloud.notify.NotificationType; import bubble.notify.NewNodeNotification; import bubble.server.BubbleConfiguration; -import bubble.service.notify.NotificationService; import bubble.service.backup.RestoreService; +import bubble.service.notify.NotificationService; import com.github.jknack.handlebars.Handlebars; import lombok.Cleanup; import lombok.Getter; @@ -37,6 +37,7 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.IOException; +import java.io.OutputStream; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -45,6 +46,7 @@ import static bubble.ApiConstants.getRemoteHost; import static bubble.ApiConstants.newNodeHostname; import static bubble.model.cloud.BubbleNode.TAG_ERROR; import static bubble.service.boot.StandardSelfNodeService.*; +import static bubble.service.cloud.NodeProgressMeterConstants.*; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; @@ -53,6 +55,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.io.FileUtil.*; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.reflect.ReflectionUtil.closeQuietly; import static org.cobbzilla.util.system.CommandShell.chmod; import static org.cobbzilla.util.system.Sleep.sleep; import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @@ -62,8 +65,12 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; public class StandardNetworkService implements NetworkService { public static final String ANSIBLE_DIR = "ansible"; - public static final String PLAYBOOK_TEMPLATE = stream2string(ANSIBLE_DIR + "/playbook.yml.hbs"); - public static final String INSTALL_LOCAL_SH = stream2string(ANSIBLE_DIR + "/install_local.sh.hbs"); + + public static final String PLAYBOOK_YML = "playbook.yml"; + public static final String PLAYBOOK_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + PLAYBOOK_YML + ".hbs"); + + public static final String INSTALL_LOCAL_SH = "install_local.sh"; + public static final String INSTALL_LOCAL_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH + ".hbs"); public static final String[] BUBBLE_SCRIPTS = { "run.sh", "bubble_common", "bubble", @@ -97,24 +104,33 @@ public class StandardNetworkService implements NetworkService { @Autowired private RestoreService restoreService; @Autowired private RedisService redisService; - @Getter(lazy=true) private final RedisService networkLocks = redisService.prefixNamespace(getClass().getSimpleName()+"_netlock_"); + @Getter(lazy=true) private final RedisService networkLocks = redisService.prefixNamespace(getClass().getSimpleName()+"_lock_"); + @Getter(lazy=true) private final RedisService networkSetupStatus = redisService.prefixNamespace(getClass().getSimpleName()+"_status_"); public BubbleNode newNode(NewNodeNotification nn) { log.info("newNode starting:\n"+json(nn)); ComputeServiceDriver computeDriver = null; BubbleNode node = null; String lock = nn.getLock(); + NodeProgressMeter progressMeter = null; try { + progressMeter = new NodeProgressMeter(nn, getNetworkSetupStatus()); + progressMeter.write(METER_TICK_CONFIRMING_NETWORK_LOCK); + if (!confirmLock(nn.getNetwork(), lock)) { + progressMeter.error(METER_ERROR_CONFIRMING_NETWORK_LOCK); return die("newNode: Error confirming network lock"); } + progressMeter.write(METER_TICK_VALIDATING_NODE_NETWORK_AND_PLAN); final BubbleNetwork network = networkDAO.findByUuid(nn.getNetwork()); if (network.getState() != BubbleNetworkState.setup) { + progressMeter.error(METER_ERROR_NETWORK_NOT_READY_FOR_SETUP); return die("newNode: network is not in 'setup' state: "+network.getState()); } final BubbleNode thisNode = configuration.getThisNode(); if (thisNode == null || !thisNode.hasUuid() || thisNode.getNetwork() == null) { + progressMeter.error(METER_ERROR_NO_CURRENT_NODE_OR_NETWORK); return die("newNode: thisNode not set or has no network"); } @@ -130,6 +146,7 @@ public class StandardNetworkService implements NetworkService { // ensure AccountPlan has been paid for if (!accountPlan.enabled()) { + progressMeter.error(METER_ERROR_PLAN_NOT_ENABLED); return die("newNode: accountPlan is not enabled: "+accountPlan.getUuid()); } @@ -138,6 +155,7 @@ public class StandardNetworkService implements NetworkService { final List peers = nodeDAO.findByAccountAndNetwork(account.getUuid(), network.getUuid()); if (peers.size() >= plan.getNodesIncluded() && nn.automated()) { // automated requests to go past network limit are not honored + progressMeter.error(METER_ERROR_PEER_LIMIT_REACHED); return die("newNode: peer limit reached ("+plan.getNodesIncluded()+")"); } @@ -145,8 +163,12 @@ public class StandardNetworkService implements NetworkService { computeDriver = cloud.getComputeDriver(configuration); final CloudService nodeCloud = cloudDAO.findByAccountAndName(network.getAccount(), cloud.getName()); - if (nodeCloud == null) return die("newNode: node cloud not found: "+cloud.getName()+" for account "+network.getAccount()); + if (nodeCloud == null) { + progressMeter.error(METER_ERROR_NODE_CLOUD_NOT_FOUND); + return die("newNode: node cloud not found: "+cloud.getName()+" for account "+network.getAccount()); + } + progressMeter.write(METER_TICK_CREATING_NODE); node = nodeDAO.create(new BubbleNode() .setHost(nn.getHost()) .setState(BubbleNodeState.created) @@ -170,21 +192,25 @@ public class StandardNetworkService implements NetworkService { } final File bubbleJar = configuration.getBubbleJar(); - if (!bubbleJar.exists()) return die("newNode: bubble.jar not found"); + if (!bubbleJar.exists()) { + progressMeter.error(METER_ERROR_BUBBLE_JAR_NOT_FOUND); + return die("newNode: bubble.jar not found"); + } @Cleanup("delete") final TempDir automation = new TempDir(); final File bubbleFilesDir = mkdirOrDie(new File(abs(automation) + "/roles/bubble/files")); final List roles = roleDAO.findByAccountAndNames(account, domain.getRoles()); if (roles.size() != domain.getRoles().length) { - return die("newNode: error finding roles"); + progressMeter.error(METER_ERROR_ROLES_NOT_FOUND); + return die("newNode: error finding ansible roles"); } // build automation directory for this run final ValidationResult errors = new ValidationResult(); final File roleTgzDir = mkdirOrDie(new File(abs(bubbleFilesDir), "role_tgz")); - // Someone needs to create a new cloud compute instance... + progressMeter.write(METER_TICK_LAUNCHING_NODE); node.setState(BubbleNodeState.starting); nodeDAO.update(node); @@ -197,6 +223,7 @@ public class StandardNetworkService implements NetworkService { // Sanity check that it came up OK if (!node.hasIp4() || !node.hasSshKey()) { + progressMeter.error(METER_ERROR_NO_IP_OR_SSH_KEY); final String message = "newNode: node booted but has no IP or SSH key"; killNode(node, message); return die(message); @@ -204,19 +231,25 @@ public class StandardNetworkService implements NetworkService { // Prepare ansible roles // We must wait until after server is started, because some roles require ip4 in vars + progressMeter.write(METER_TICK_PREPARING_ROLES); final Map ctx = ansiblePrep.prepAnsible(automation, bubbleFilesDir, account, network, node, roles, errors, roleTgzDir, nn.fork(), nn.getRestoreKey()); - if (errors.isInvalid()) throw new MultiViolationException(errors.getViolationBeans()); + if (errors.isInvalid()) { + progressMeter.error(METER_ERROR_ROLE_VALIDATION_ERRORS); + throw new MultiViolationException(errors.getViolationBeans()); + } // Create DNS A and AAAA records for node + progressMeter.write(METER_TICK_WRITING_DNS_RECORDS); final CloudService dnsService = cloudDAO.findByUuid(domain.getPublicDns()); dnsService.getDnsDriver(configuration).setNode(node); + progressMeter.write(METER_TICK_PREPARING_INSTALL); node.setState(BubbleNodeState.preparing_install); nodeDAO.update(node); // This node is on our network, or is the very first server. We must run ansible on it ourselves. // write playbook file - writeFile(automation, ctx, "playbook.yml", PLAYBOOK_TEMPLATE); + writeFile(automation, ctx, PLAYBOOK_YML, PLAYBOOK_TEMPLATE); // write inventory file final File inventory = new File(automation, "hosts"); @@ -244,11 +277,12 @@ public class StandardNetworkService implements NetworkService { writeFile(bubbleFilesDir, null, SAGE_KEY_JSON, json(BubbleNodeKey.sageMask(sageKey))); // write install_local.sh script - final File file = writeFile(automation, ctx, "install_local.sh", INSTALL_LOCAL_SH); - chmod(file, "500"); + final File installLocalScript = writeFile(automation, ctx, INSTALL_LOCAL_SH, INSTALL_LOCAL_TEMPLATE); + chmod(installLocalScript, "500"); // ensure this hostname is visible in our DNS and in public DNS, // or else node can't create its own letsencrypt SSL cert + progressMeter.write(METER_TICK_AWAITING_DNS); node.setState(BubbleNodeState.awaiting_dns); nodeDAO.update(node); @@ -257,6 +291,7 @@ public class StandardNetworkService implements NetworkService { final DnsServiceDriver dnsDriver = dnsService.getDnsDriver(configuration); dnsDriver.ensureResolvable(domain, node, DNS_TIMEOUT); + progressMeter.write(METER_TICK_STARTING_INSTALL); node.setState(BubbleNodeState.installing); nodeDAO.update(node); @@ -274,7 +309,7 @@ public class StandardNetworkService implements NetworkService { for (int i=0; i ctx, String filename, String templateOrData) throws IOException { @@ -583,4 +627,13 @@ public class StandardNetworkService implements NetworkService { } return cloud; } + + public NodeProgressMeterTick getLaunchStatus(Account caller, String uuid) { + final String json = getNetworkSetupStatus().get(uuid); + if (json == null) return null; + final NodeProgressMeterTick tick = json(json, NodeProgressMeterTick.class); + if (!tick.getAccount().equals(caller.getUuid())) return null; + return tick.setPattern(null); + } + } diff --git a/bubble-server/src/main/java/bubble/service/cloud/StorageStreamService.java b/bubble-server/src/main/java/bubble/service/cloud/StorageStreamService.java index 10e6fb26..f66a192d 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StorageStreamService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StorageStreamService.java @@ -21,6 +21,7 @@ import static java.util.UUID.randomUUID; import static java.util.concurrent.TimeUnit.SECONDS; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.cache.redis.RedisService.EX; import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; @Service @Slf4j @@ -39,13 +40,13 @@ public class StorageStreamService { public String registerRead(StorageStreamRequest request) { final String token = randomUUID().toString(); request.setToken(token); - getReadRequests().set(token, json(request.setToken(token)), "EX", TOKEN_TTL); + getReadRequests().set(token, json(request.setToken(token)), EX, TOKEN_TTL); return token; } public String registerRead(StorageStreamRequest request, WriteRequest writeRequest) { final String token = WR_PREFIX + writeRequest.requestId; - getReadRequests().set(token, json(request.setToken(token)), "EX", TOKEN_TTL); + getReadRequests().set(token, json(request.setToken(token)), EX, TOKEN_TTL); return token; } diff --git a/bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json b/bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json new file mode 100644 index 00000000..dace6061 --- /dev/null +++ b/bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json @@ -0,0 +1,5 @@ +[ + { "percent": 10, "messageKey": "meter_tick_apt_install", "pattern": "some pattern installing apt packages" }, + { "percent": 30, "messageKey": "meter_tick_db_setup", "pattern": "some pattern setting up database" }, + { "percent": 99, "messageKey": "meter_tick_starting_bubble_api_server", "pattern": "some pattern starting api" } +] \ No newline at end of file diff --git a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties index 251aeb47..670c300c 100644 --- a/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties +++ b/bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties @@ -193,6 +193,32 @@ field_payment_card_number=Credit or Debit Card button_label_submit_card=Verify Card message_verified_card=Card Successfully Verified +# Launch progress meter +meter_tick_confirming_network_lock=Confirming network lock +meter_tick_verifying_node_network_and_plan=Verifying settings for Bubble +meter_tick_creating_node=Creating Bubble node +meter_tick_launching_node=Launching Bubble node +meter_tick_preparing_roles=Preparing installation parameters +meter_tick_writing_dns_records=Writing DNS records +meter_tick_preparing_install=Creating installation package +meter_tick_awaiting_dns=Awaiting DNS visibility +meter_tick_starting_install=Connecting to node to install Bubble +meter_tick_copying_ansible=Copying files required to install Bubble +meter_tick_running_ansible=Starting Bubble installation + +meter_completed=Bubble installation completed successfully! On to your Bubble! + +meter_error_confirming_network_lock=Error confirming network lock +meter_error_network_not_ready_for_setup=Cannot launch Bubble when network is not in 'setup' state +meter_error_no_current_node_or_network=Current API does not have a node or network, cannot launch Bubble +meter_error_plan_not_enabled=Account plan is not enabled, cannot launch Bubble +meter_error_node_cloud_not_found=Compute cloud was not found, cannot launch Bubble +meter_error_bubble_jar_not_found=Bubble jar file was not found, cannot launch Bubble +meter_error_roles_not_found=Ansible roles were not found, cannot launch Bubble +meter_error_no_ip_or_ssh_key=Bubble node started, but does not have an IP address or SSH key, cannot install Bubble +meter_error_role_validation_errors=Validation of ansible roles failed, cannot install Bubble +meter_unknown_error=An unknown error occurred + # Error messages from API server err.accountContactsJson.length=Account contacts length violation err.accountOperationTimeout.required=Account operation timeout is required diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index c7c15206..34b069c3 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit c7c15206e20cbfbb3fdc543a53fa4b376e8a08f5 +Subproject commit 34b069c3453fd2b9dbb9c7af10306346e1bc3d8a diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 9a6cb168..e3256afb 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 9a6cb168d03c7cf3f765425607c9477bb0c870b1 +Subproject commit e3256afba721949ef4993ce7e13ec5d5ef8c2368