WIP. adding progress meter. use redis constants.

bubble-server/src/main/java/bubble/ApiConstants.java

@@ -147,6 +147,7 @@ public class ApiConstants {
     public static final String EP_STOP = "/stop";
     public static final String EP_RESTORE = "/restore";
     public static final String EP_KEYS = "/keys";
+    public static final String EP_STATUS = "/status";
     public static final String EP_FORK = "/fork";
 
     public static final String DEBUG_ENDPOINT = "/debug";

bubble-server/src/main/java/bubble/cloud/geoCode/GeoCodeDriverBase.java

@@ -11,6 +11,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import java.util.concurrent.TimeUnit;
 
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Slf4j
 public abstract class GeoCodeDriverBase<T> extends CloudServiceDriverBase<T> implements GeoCodeServiceDriver {
@@ -46,7 +47,7 @@ public abstract class GeoCodeDriverBase<T> extends CloudServiceDriverBase<T> imp
                 ttl = ERROR_TTL;
             }
             val = json(r);
-            getCache().set(key, val, "EX", ttl);
+            getCache().set(key, val, EX, ttl);
         }
         return valOrError(val, GeoCodeResult.class);
     }

bubble-server/src/main/java/bubble/cloud/geoLocation/GeoLocateServiceDriverBase.java

@@ -25,6 +25,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf;
 import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Slf4j
 public abstract class GeoLocateServiceDriverBase<T> extends CloudServiceDriverBase<T> implements GeoLocateServiceDriver {
@@ -59,7 +60,7 @@ public abstract class GeoLocateServiceDriverBase<T> extends CloudServiceDriverBa
                 ttl = ERROR_TTL;
             }
             val = json(loc);
-            getCache().set(ip, val, "EX", ttl);
+            getCache().set(ip, val, EX, ttl);
         }
         return valOrError(val, GeoLocation.class);
     }

bubble-server/src/main/java/bubble/cloud/geoTime/GeoTimeServiceDriverBase.java

@@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import java.util.concurrent.TimeUnit;
 
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Slf4j
 public abstract class GeoTimeServiceDriverBase<T> extends CloudServiceDriverBase<T> implements GeoTimeServiceDriver {
@@ -45,7 +46,7 @@ public abstract class GeoTimeServiceDriverBase<T> extends CloudServiceDriverBase
                 ttl = ERROR_TTL;
             }
             val = json(tz);
-            getCache().set(key, val, "EX", ttl);
+            getCache().set(key, val, EX, ttl);
         }
         return valOrError(val, GeoTimeZone.class);
     }

bubble-server/src/main/java/bubble/cloud/payment/stripe/StripePaymentDriver.java

@@ -29,6 +29,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 
 @Slf4j
@@ -162,7 +163,7 @@ public class StripePaymentDriver extends PaymentDriverBase<StripePaymentDriverCo
                         } else {
                             log.info("authorize: successful: charge=" + chargeJson);
                         }
-                        authCache.set(authCacheKey, charge.getId(), "EX", AUTH_CACHE_DURATION);
+                        authCache.set(authCacheKey, charge.getId(), EX, AUTH_CACHE_DURATION);
                         return true;
 
                     case "pending":
@@ -244,7 +245,7 @@ public class StripePaymentDriver extends PaymentDriverBase<StripePaymentDriverCo
                 switch (captured.getStatus()) {
                     case "succeeded":
                         log.info("charge: charge successful: "+authCacheKey);
-                        chargeCache.set(billUuid, captured.getId(), "EX", CHARGE_CACHE_DURATION);
+                        chargeCache.set(billUuid, captured.getId(), EX, CHARGE_CACHE_DURATION);
                         authCache.del(authCacheKey);
                         return captured.getId();
 
@@ -314,7 +315,7 @@ public class StripePaymentDriver extends PaymentDriverBase<StripePaymentDriverCo
                 switch (refund.getStatus()) {
                     case "succeeded":
                         log.info("refund: refund of "+refundAmount+" successful for bill: "+billUuid);
-                        refundCache.set(billUuid, refund.getId(), "EX", REFUND_CACHE_DURATION);
+                        refundCache.set(billUuid, refund.getId(), EX, REFUND_CACHE_DURATION);
                         return refund.getId();
 
                     case "pending":

bubble-server/src/main/java/bubble/cloud/storage/s3/S3StorageDriver.java

@@ -40,6 +40,7 @@ import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE;
 import static org.cobbzilla.util.security.ShaUtil.sha256_file;
 import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Slf4j
 public class S3StorageDriver extends StorageServiceDriverBase<S3StorageConfig> {
@@ -254,7 +255,7 @@ public class S3StorageDriver extends StorageServiceDriverBase<S3StorageConfig> {
         listing.getObjectSummaries().forEach(o -> keys.add(o.getKey().substring(rootPrefix.length())));
 
         final ListingRequest listingRequest = new ListingRequest(key, listing);
-        getActiveListings().set(listRequestId, json(listingRequest), "EX", LISTING_TIMEOUT);
+        getActiveListings().set(listRequestId, json(listingRequest), EX, LISTING_TIMEOUT);
 
         return new StorageListing()
                 .setListingId(listRequestId)
@@ -274,7 +275,7 @@ public class S3StorageDriver extends StorageServiceDriverBase<S3StorageConfig> {
 
         listingRequest.objectListing = s3client.listNextBatchOfObjects(listingRequest.objectListing);
         listingRequest.objectListing.getObjectSummaries().forEach(o -> keys.add(o.getKey().substring(rootPrefix.length())));
-        activeListings.set(listingId, json(listingRequest), "EX", LISTING_TIMEOUT);
+        activeListings.set(listingId, json(listingRequest), EX, LISTING_TIMEOUT);
 
         return new StorageListing()
                 .setListingId(listingId)

bubble-server/src/main/java/bubble/notify/NewNodeNotification.java

@@ -5,11 +5,13 @@ import lombok.NoArgsConstructor;
 import lombok.Setter;
 import lombok.experimental.Accessors;
 
+import static java.util.UUID.randomUUID;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
 @NoArgsConstructor @Accessors(chain=true)
 public class NewNodeNotification {
 
+    @Getter @Setter private String uuid = randomUUID().toString();
     @Getter @Setter private String account;
     @Getter @Setter private String host;
     @Getter @Setter private String network;

bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java

@@ -16,6 +16,7 @@ import bubble.model.bill.AccountPlan;
 import bubble.model.cloud.*;
 import bubble.server.BubbleConfiguration;
 import bubble.service.backup.NetworkKeysService;
+import bubble.service.cloud.NodeProgressMeterTick;
 import bubble.service.cloud.StandardNetworkService;
 import lombok.extern.slf4j.Slf4j;
 import org.glassfish.grizzly.http.server.Request;
@@ -73,6 +74,15 @@ public class NetworkActionsResource {
         return _startNetwork(network, cloud, region, req);
     }
 
+    @GET @Path(EP_STATUS+"/{uuid}")
+    public Response requestLaunchStatus(@Context Request req,
+                                        @Context ContainerRequest ctx,
+                                        @PathParam("uuid") String uuid) {
+        final Account caller = userPrincipal(ctx);
+        final NodeProgressMeterTick tick = networkService.getLaunchStatus(caller, uuid);
+        return tick == null ? notFound(uuid) : ok(tick);
+    }
+
     @GET @Path(EP_KEYS)
     public Response requestNetworkKeys(@Context Request req,
                                        @Context ContainerRequest ctx) {

bubble-server/src/main/java/bubble/server/listener/BubbleFirstTimeListener.java

@@ -2,7 +2,6 @@ package bubble.server.listener;
 
 import bubble.ApiConstants;
 import bubble.dao.account.AccountDAO;
-import bubble.dao.account.message.AccountMessageDAO;
 import bubble.model.account.Account;
 import bubble.model.account.message.AccountAction;
 import bubble.model.account.message.AccountMessage;
@@ -22,6 +21,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import static java.util.concurrent.TimeUnit.HOURS;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic;
 import static org.cobbzilla.util.io.FileUtil.abs;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Slf4j
 public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase<BubbleConfiguration> {
@@ -53,7 +53,7 @@ public class BubbleFirstTimeListener extends RestServerLifecycleListenerBase<Bub
             final BubbleNetwork network = configuration.getThisNetwork();
 
             final String unlockKey = randomAlphabetic(UNLOCK_KEY_LEN).toUpperCase();
-            redis.get().set(UNLOCK_KEY, unlockKey, "EX", UNLOCK_EXPIRATION);
+            redis.get().set(UNLOCK_KEY, unlockKey, EX, UNLOCK_EXPIRATION);
 
             final SageHelloService helloService = configuration.getBean(SageHelloService.class);
             helloService.setUnlockMessage(new AccountMessage()

bubble-server/src/main/java/bubble/service/account/StandardAccountMessageService.java

@@ -31,6 +31,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.string.ValidationRegexes.NUMERIC_PATTERN;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
 
 @Service @Slf4j
@@ -107,8 +108,8 @@ public class StandardAccountMessageService implements AccountMessageService {
             token = randomNumeric(6);
             final long tokenTimeout = message.tokenTimeoutSeconds(policy);
             if (tokenTimeout == -1) return null;
-            getConfirmationTokens().set(key, token, "EX", tokenTimeout);
-            getConfirmationTokens().set(token, json(amc), "EX", tokenTimeout);
+            getConfirmationTokens().set(key, token, EX, tokenTimeout);
+            getConfirmationTokens().set(token, json(amc), EX, tokenTimeout);
             log.debug("confirmationToken: action="+message.getAction()+", token="+token+", key="+key);
         }
         return token;

bubble-server/src/main/java/bubble/service/account/download/AccountDownloadMonitor.java

@@ -16,6 +16,7 @@ import static bubble.service.account.download.AccountDownloadService.*;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.terminate;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @AllArgsConstructor @Slf4j
 public class AccountDownloadMonitor implements Runnable {
@@ -40,7 +41,7 @@ public class AccountDownloadMonitor implements Runnable {
                         .setAction(AccountAction.download)
                         .setTarget(ActionTarget.account)
                         .setRemoteHost(remoteHost));
-                downloadService.getAccountData().set(message.getRequestId(), json(data), "EX", ACCOUNT_DOWNLOAD_EXPIRATION);
+                downloadService.getAccountData().set(message.getRequestId(), json(data), EX, ACCOUNT_DOWNLOAD_EXPIRATION);
             }
         } catch (Exception e) {
             die("error: "+e, e);

bubble-server/src/main/java/bubble/service/account/download/AccountDownloadService.java

@@ -21,6 +21,7 @@ import static bubble.service.account.download.AccountDownloadMonitor.waitForData
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Service @Slf4j
 public class AccountDownloadService {
@@ -47,7 +48,7 @@ public class AccountDownloadService {
     public void approve(String uuid) {
         final String data = getAccountData().get(uuid);
         if (data != null) {
-            getApprovedAccountData().set(uuid, data, "EX", ACCOUNT_DOWNLOAD_EXPIRATION);
+            getApprovedAccountData().set(uuid, data, EX, ACCOUNT_DOWNLOAD_EXPIRATION);
         }
     }
 

bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java

@@ -14,6 +14,7 @@ import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE;
 import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE_CREDENTIALS;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Service @Slf4j
 public class NetworkKeysService {
@@ -35,7 +36,7 @@ public class NetworkKeysService {
             keys.addKey(PARAM_STORAGE, json(storage));
             keys.addKey(PARAM_STORAGE_CREDENTIALS, json(storage.getCredentials()));
         }
-        getNetworkPasswordTokens().set(uuid, json(keys), "EX", KEY_EXPIRATION);
+        getNetworkPasswordTokens().set(uuid, json(keys), EX, KEY_EXPIRATION);
     }
 
     public NetworkKeys retrieveKeys(String uuid) {

bubble-server/src/main/java/bubble/service/backup/RestoreService.java

@@ -30,6 +30,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Service @Slf4j
 public class RestoreService {
@@ -54,7 +55,7 @@ public class RestoreService {
     @Autowired private BubbleConfiguration configuration;
 
     public void registerRestore(String restoreKey, NetworkKeys keys) {
-        getRestoreKeys().set(restoreKey, json(keys), "EX", RESTORE_WINDOW_SECONDS);
+        getRestoreKeys().set(restoreKey, json(keys), EX, RESTORE_WINDOW_SECONDS);
     }
 
     public boolean isValidRestoreKey(String restoreKey) { return getRestoreKeys().exists(restoreKey); }

bubble-server/src/main/java/bubble/service/cloud/AnsiblePrepService.java

@@ -71,7 +71,7 @@ public class AnsiblePrepService {
         ctx.put("network", network);
         ctx.put("node", node);
         ctx.put("roles", installRoles.stream().map(AnsibleRole::getRoleName).collect(Collectors.toList()));
-        ctx.put("testMode", Boolean.FALSE.toString());
+        ctx.put("testMode", !fork && configuration.testMode());
 
         // Copy database with new encryption key
         if (installRoles.stream().anyMatch(r->r.getName().startsWith("bubble-"))) {

bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeter.java

@@ -0,0 +1,151 @@
+package bubble.service.cloud;
+
+import bubble.notify.NewNodeNotification;
+import lombok.experimental.Accessors;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.io.FileUtil;
+import org.cobbzilla.wizard.cache.redis.RedisService;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static bubble.service.cloud.NodeProgressMeterConstants.*;
+import static java.util.Arrays.asList;
+import static java.util.concurrent.TimeUnit.DAYS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
+import static org.cobbzilla.util.daemon.ZillaRuntime.terminate;
+import static org.cobbzilla.util.io.StreamUtil.stream2string;
+import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.reflect.ReflectionUtil.closeQuietly;
+import static org.cobbzilla.util.system.Bytes.KB;
+import static org.cobbzilla.util.system.Sleep.sleep;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
+
+@Slf4j @Accessors(chain=true)
+public class NodeProgressMeter extends PipedOutputStream implements Runnable {
+
+    public static final int PIPE_SIZE = (int) (16*KB);
+
+    public static final long TICK_REDIS_EXPIRATION = DAYS.toSeconds(1);
+
+    private final List<NodeProgressMeterTick> standardTicks;
+
+    private BufferedReader reader;
+    private BufferedWriter writer;
+    private List<NodeProgressMeterTick> ticks;
+    private int tickPos = 0;
+    private AtomicBoolean error = new AtomicBoolean(false);
+    private AtomicBoolean closed = new AtomicBoolean(false);
+    private final Thread thread;
+
+    private RedisService redis;
+    private NewNodeNotification nn;
+    private String key;
+    private final NodeProgressMeterTick lastStandardTick;
+
+    public NodeProgressMeter(NewNodeNotification nn, RedisService redis) throws IOException {
+
+        this.nn = nn;
+        this.redis = redis;
+
+        standardTicks = getStandardTicks();
+        lastStandardTick = standardTicks.get(standardTicks.size()-1);
+
+        ticks = new ArrayList<>(standardTicks);
+        final NodeProgressMeterTick[] installTicks = json(stream2string(TICKS_JSON), NodeProgressMeterTick[].class);
+        for (NodeProgressMeterTick tick : installTicks) {
+            tick.setAccount(nn.getAccount()).relativizePercent(lastStandardTick.getPercent());
+        }
+        ticks.addAll(asList(installTicks));
+
+        key = nn.getUuid();
+
+        final PipedInputStream pipeIn = new PipedInputStream(PIPE_SIZE);
+        connect(pipeIn);
+
+        reader = new BufferedReader(new InputStreamReader(pipeIn));
+        writer = new BufferedWriter(new OutputStreamWriter(this));
+
+        thread = new Thread(this);
+        thread.setDaemon(true);
+        thread.start();
+    }
+
+    public void write(String line) throws IOException {
+        writer.write(line.endsWith("\n") ? line : line+"\n");
+        writer.flush();
+    }
+
+    public void error(String line) {
+        if (error.get()) {
+            log.warn("error("+line+") ignored, error already set");
+            return;
+        }
+        error.set(true);
+        close();
+        setCurrentTick(errorTick(getErrorMessageKey(line), line));
+    }
+
+    private NodeProgressMeterTick errorTick(String messageKey, String line) {
+        return new NodeProgressMeterTick()
+                .setAccount(nn.getAccount())
+                .setMessageKey(messageKey)
+                .setDetails(line);
+    }
+
+    public void reset() {
+        error.set(false);
+        tickPos = standardTicks.size();
+        setCurrentTick(lastStandardTick);
+    }
+
+    @Override public void run() {
+        String line;
+        try {
+            final File file = new File("/tmp/node_launch_progress.txt");
+            while ((line = reader.readLine()) != null && !closed.get()) {
+                FileUtil.appendFile(file, now()+" : "+line+"\n");
+                for (int i=tickPos; i<ticks.size(); i++) {
+                    if (ticks.get(i).matches(line)) {
+                        if (!error.get() && !closed.get()) setCurrentTick(ticks.get(i));
+                        tickPos = i+1;
+                        break;
+                    }
+                }
+                sleep(50, "checking for interrupt in between reads");
+            }
+        } catch (Exception e) {
+            log.info("run: "+e);
+        }
+    }
+
+    public void setCurrentTick(NodeProgressMeterTick tick) {
+        redis.set(key, json(tick), EX, TICK_REDIS_EXPIRATION);
+    }
+
+    public static final long THREAD_KILL_TIMEOUT = SECONDS.toMillis(5);
+
+    @Override public void close() {
+        closed.set(true);
+        try {
+            super.close();
+        } catch (IOException e) {
+            log.warn("close: "+e);
+        }
+        terminate(thread, THREAD_KILL_TIMEOUT);
+        closeQuietly(reader);
+        closeQuietly(writer);
+    }
+
+    public void completed() {
+        close();
+        setCurrentTick(new NodeProgressMeterTick()
+                .setAccount(nn.getAccount())
+                .setMessageKey(METER_COMPLETED)
+                .setPercent(100));
+    }
+
+}

bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterConstants.java

@@ -0,0 +1,98 @@
+package bubble.service.cloud;
+
+import org.cobbzilla.util.collection.MapBuilder;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.commons.lang3.reflect.FieldUtils.getAllFields;
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.reflect.ReflectionUtil.constValue;
+import static org.cobbzilla.util.reflect.ReflectionUtil.isStaticFinalString;
+
+public class NodeProgressMeterConstants {
+
+    public static final String TICKS_JSON = "bubble/node_progress_meter_ticks.json";
+
+    public static final String TICK_PREFIX = "METER_TICK_";
+
+    public static final String METER_TICK_CONFIRMING_NETWORK_LOCK = "BUBBLE: CONFIRMING NETWORK LOCK...";
+    public static final String METER_TICK_VALIDATING_NODE_NETWORK_AND_PLAN = "BUBBLE: VALIDATING NODE, NETWORK, AND PLAN...";
+    public static final String METER_TICK_CREATING_NODE = "BUBBLE: CREATING NODE...";
+    public static final String METER_TICK_LAUNCHING_NODE = "BUBBLE: LAUNCHING NODE...";
+    public static final String METER_TICK_PREPARING_ROLES = "BUBBLE: PREPARING ANSIBLE ROLES...";
+    public static final String METER_TICK_WRITING_DNS_RECORDS = "BUBBLE: WRITING DNS RECORDS...";
+    public static final String METER_TICK_PREPARING_INSTALL = "BUBBLE: PREPARING INSTALL FILES...";
+    public static final String METER_TICK_AWAITING_DNS = "BUBBLE: AWAITING DNS RECORDS...";
+    public static final String METER_TICK_STARTING_INSTALL = "BUBBLE: STARTING INSTALLATION...";
+    public static final String METER_TICK_COPYING_ANSIBLE = "BUBBLE: COPYING ANSIBLE FILES...";
+    public static final String METER_TICK_RUNNING_ANSIBLE = "BUBBLE: RUNNING ANSIBLE PLAYBOOK...";
+
+    public static final String ERROR_PREFIX = "METER_ERROR_";
+
+    public static final String METER_ERROR_CONFIRMING_NETWORK_LOCK = "BUBBLE-ERROR: ERROR CONFIRMING NETWORK LOCK";
+    public static final String METER_ERROR_NETWORK_NOT_READY_FOR_SETUP = "BUBBLE-ERROR: NETWORK NOT READY FOR SETUP";
+    public static final String METER_ERROR_NO_CURRENT_NODE_OR_NETWORK = "BUBBLE-ERROR: NO CURRENT NODE OR NETWORK";
+    public static final String METER_ERROR_PLAN_NOT_ENABLED = "BUBBLE-ERROR: PLAN NOT ENABLED";
+    public static final String METER_ERROR_PEER_LIMIT_REACHED = "BUBBLE-ERROR: PEER LIMIT REACHED";
+    public static final String METER_ERROR_NODE_CLOUD_NOT_FOUND = "BUBBLE-ERROR: NODE CLOUD NOT FOUND";
+    public static final String METER_ERROR_BUBBLE_JAR_NOT_FOUND = "BUBBLE-ERROR: BUBBLE JAR NOT FOUND";
+    public static final String METER_ERROR_ROLES_NOT_FOUND = "BUBBLE-ERROR: ANSIBLE ROLES NOT FOUND";
+    public static final String METER_ERROR_NO_IP_OR_SSH_KEY = "BUBBLE-ERROR: NODE STARTED BUT HAS NO IP ADDRESS OR SSH KEY";
+    public static final String METER_ERROR_ROLE_VALIDATION_ERRORS = "BUBBLE-ERROR: ROLE VALIDATION FAILED";
+
+    public static final String METER_COMPLETED = "meter_completed";
+    public static final String METER_UNKNOWN_ERROR = "meter_unknown_error";
+
+    private static final Map<String, Integer> STANDARD_TICKS = MapBuilder.build(new Object[][] {
+            {METER_TICK_CONFIRMING_NETWORK_LOCK, 1},
+            {METER_TICK_VALIDATING_NODE_NETWORK_AND_PLAN, 2},
+            {METER_TICK_CREATING_NODE, 3},
+            {METER_TICK_LAUNCHING_NODE, 4},
+            {METER_TICK_PREPARING_ROLES, 9},
+            {METER_TICK_WRITING_DNS_RECORDS, 10},
+            {METER_TICK_PREPARING_INSTALL, 12},
+            {METER_TICK_AWAITING_DNS, 13},
+            {METER_TICK_STARTING_INSTALL, 19},
+            {METER_TICK_COPYING_ANSIBLE, 20},
+            {METER_TICK_RUNNING_ANSIBLE, 24}
+    });
+
+    public static List<NodeProgressMeterTick> getStandardTicks() {
+        final List<NodeProgressMeterTick> ticks = new ArrayList<>();
+        for (Field f : getAllFields(NodeProgressMeterConstants.class)) {
+            if (isStaticFinalString(f, TICK_PREFIX)) {
+                final String value = constValue(f);
+                final Integer percent = STANDARD_TICKS.get(value);
+                if (percent == null) return die("getStandardTicks: "+f.getName()+" entry missing from STANDARD_TICKS");
+                ticks.add(new NodeProgressMeterTick()
+                        .setPattern(value)
+                        .setExact(true)
+                        .setStandard(true)
+                        .setMessageKey(f.getName().toLowerCase())
+                        .setPercent(percent));
+            }
+        }
+        return ticks;
+    }
+
+    public static final Map<String, String> ERRORS = initErrors();
+    private static Map<String, String> initErrors() {
+        final Map<String, String> errors = new HashMap<>();
+        for (Field f : getAllFields(NodeProgressMeterConstants.class)) {
+            if (isStaticFinalString(f, ERROR_PREFIX)) {
+                errors.put(constValue(f), f.getName().toLowerCase());
+            }
+        }
+        return errors;
+    }
+
+    public static String getErrorMessageKey (String error) {
+        final String messageKey = ERRORS.get(error);
+        return messageKey != null ? messageKey : METER_UNKNOWN_ERROR;
+    }
+
+}

bubble-server/src/main/java/bubble/service/cloud/NodeProgressMeterTick.java

@@ -0,0 +1,40 @@
+package bubble.service.cloud;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+import java.util.regex.Pattern;
+
+@Accessors(chain=true)
+public class NodeProgressMeterTick {
+
+    @Getter @Setter private String account;
+
+    @Getter @Setter private String pattern;
+    @JsonIgnore @Getter(lazy=true) private final Pattern _pattern = Pattern.compile(getPattern());
+
+    @Getter @Setter private Boolean exact;
+    public boolean exact() { return exact != null && exact; }
+
+    @Getter @Setter private Boolean standard;
+    public boolean standard() { return standard != null && standard; }
+
+    @Getter @Setter private Integer percent;
+
+    public NodeProgressMeterTick relativizePercent(int lastStandardPercent) {
+        setPercent(Math.round(((float) lastStandardPercent) + (100f - lastStandardPercent) * getPercent() / 100f));
+        return this;
+    }
+
+    @Getter @Setter private String messageKey;
+    @Getter @Setter private String details;
+
+    public boolean matches(String line) {
+        return exact()
+                ? line.trim().equals(getPattern().trim())
+                : get_pattern().matcher(line).matches();
+    }
+
+}

bubble-server/src/main/java/bubble/service/cloud/RequestCoordinationService.java

@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Service
 public class RequestCoordinationService {
@@ -15,11 +16,11 @@ public class RequestCoordinationService {
     @Getter(lazy=true) private final RedisService requests = redisService.prefixNamespace(getClass().getSimpleName()+"_");
 
     public void set(String prefix, String id, JsonNode thing) {
-        getRequests().set(prefix+":"+id, json(thing), "EX", 600);
+        getRequests().set(prefix+":"+id, json(thing), EX, 600);
     }
 
     public void set(String prefix, String id, String thing) {
-        getRequests().set(prefix+":"+id, thing, "EX", 600);
+        getRequests().set(prefix+":"+id, thing, EX, 600);
     }
 
     public String get(String prefix, String id) {

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -15,8 +15,8 @@ import bubble.model.cloud.notify.NotificationReceipt;
 import bubble.model.cloud.notify.NotificationType;
 import bubble.notify.NewNodeNotification;
 import bubble.server.BubbleConfiguration;
-import bubble.service.notify.NotificationService;
 import bubble.service.backup.RestoreService;
+import bubble.service.notify.NotificationService;
 import com.github.jknack.handlebars.Handlebars;
 import lombok.Cleanup;
 import lombok.Getter;
@@ -37,6 +37,7 @@ import org.springframework.stereotype.Service;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
@@ -45,6 +46,7 @@ import static bubble.ApiConstants.getRemoteHost;
 import static bubble.ApiConstants.newNodeHostname;
 import static bubble.model.cloud.BubbleNode.TAG_ERROR;
 import static bubble.service.boot.StandardSelfNodeService.*;
+import static bubble.service.cloud.NodeProgressMeterConstants.*;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
@@ -53,6 +55,7 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.reflect.ReflectionUtil.closeQuietly;
 import static org.cobbzilla.util.system.CommandShell.chmod;
 import static org.cobbzilla.util.system.Sleep.sleep;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
@@ -62,8 +65,12 @@ import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 public class StandardNetworkService implements NetworkService {
 
     public static final String ANSIBLE_DIR = "ansible";
-    public static final String PLAYBOOK_TEMPLATE = stream2string(ANSIBLE_DIR + "/playbook.yml.hbs");
-    public static final String INSTALL_LOCAL_SH = stream2string(ANSIBLE_DIR + "/install_local.sh.hbs");
+
+    public static final String PLAYBOOK_YML = "playbook.yml";
+    public static final String PLAYBOOK_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + PLAYBOOK_YML + ".hbs");
+
+    public static final String INSTALL_LOCAL_SH = "install_local.sh";
+    public static final String INSTALL_LOCAL_TEMPLATE = stream2string(ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH + ".hbs");
 
     public static final String[] BUBBLE_SCRIPTS = {
             "run.sh", "bubble_common", "bubble",
@@ -97,24 +104,33 @@ public class StandardNetworkService implements NetworkService {
     @Autowired private RestoreService restoreService;
 
     @Autowired private RedisService redisService;
-    @Getter(lazy=true) private final RedisService networkLocks = redisService.prefixNamespace(getClass().getSimpleName()+"_netlock_");
+    @Getter(lazy=true) private final RedisService networkLocks = redisService.prefixNamespace(getClass().getSimpleName()+"_lock_");
+    @Getter(lazy=true) private final RedisService networkSetupStatus = redisService.prefixNamespace(getClass().getSimpleName()+"_status_");
 
     public BubbleNode newNode(NewNodeNotification nn) {
         log.info("newNode starting:\n"+json(nn));
         ComputeServiceDriver computeDriver = null;
         BubbleNode node = null;
         String lock = nn.getLock();
+        NodeProgressMeter progressMeter = null;
         try {
+            progressMeter = new NodeProgressMeter(nn, getNetworkSetupStatus());
+            progressMeter.write(METER_TICK_CONFIRMING_NETWORK_LOCK);
+
             if (!confirmLock(nn.getNetwork(), lock)) {
+                progressMeter.error(METER_ERROR_CONFIRMING_NETWORK_LOCK);
                 return die("newNode: Error confirming network lock");
             }
 
+            progressMeter.write(METER_TICK_VALIDATING_NODE_NETWORK_AND_PLAN);
             final BubbleNetwork network = networkDAO.findByUuid(nn.getNetwork());
             if (network.getState() != BubbleNetworkState.setup) {
+                progressMeter.error(METER_ERROR_NETWORK_NOT_READY_FOR_SETUP);
                 return die("newNode: network is not in 'setup' state: "+network.getState());
             }
             final BubbleNode thisNode = configuration.getThisNode();
             if (thisNode == null || !thisNode.hasUuid() || thisNode.getNetwork() == null) {
+                progressMeter.error(METER_ERROR_NO_CURRENT_NODE_OR_NETWORK);
                 return die("newNode: thisNode not set or has no network");
             }
 
@@ -130,6 +146,7 @@ public class StandardNetworkService implements NetworkService {
 
             // ensure AccountPlan has been paid for
             if (!accountPlan.enabled()) {
+                progressMeter.error(METER_ERROR_PLAN_NOT_ENABLED);
                 return die("newNode: accountPlan is not enabled: "+accountPlan.getUuid());
             }
 
@@ -138,6 +155,7 @@ public class StandardNetworkService implements NetworkService {
             final List<BubbleNode> peers = nodeDAO.findByAccountAndNetwork(account.getUuid(), network.getUuid());
             if (peers.size() >= plan.getNodesIncluded() && nn.automated()) {
                 // automated requests to go past network limit are not honored
+                progressMeter.error(METER_ERROR_PEER_LIMIT_REACHED);
                 return die("newNode: peer limit reached ("+plan.getNodesIncluded()+")");
             }
 
@@ -145,8 +163,12 @@ public class StandardNetworkService implements NetworkService {
             computeDriver = cloud.getComputeDriver(configuration);
 
             final CloudService nodeCloud = cloudDAO.findByAccountAndName(network.getAccount(), cloud.getName());
-            if (nodeCloud == null) return die("newNode: node cloud not found: "+cloud.getName()+" for account "+network.getAccount());
+            if (nodeCloud == null) {
+                progressMeter.error(METER_ERROR_NODE_CLOUD_NOT_FOUND);
+                return die("newNode: node cloud not found: "+cloud.getName()+" for account "+network.getAccount());
+            }
 
+            progressMeter.write(METER_TICK_CREATING_NODE);
             node = nodeDAO.create(new BubbleNode()
                     .setHost(nn.getHost())
                     .setState(BubbleNodeState.created)
@@ -170,21 +192,25 @@ public class StandardNetworkService implements NetworkService {
             }
 
             final File bubbleJar = configuration.getBubbleJar();
-            if (!bubbleJar.exists()) return die("newNode: bubble.jar not found");
+            if (!bubbleJar.exists()) {
+                progressMeter.error(METER_ERROR_BUBBLE_JAR_NOT_FOUND);
+                return die("newNode: bubble.jar not found");
+            }
 
             @Cleanup("delete") final TempDir automation = new TempDir();
             final File bubbleFilesDir = mkdirOrDie(new File(abs(automation) + "/roles/bubble/files"));
 
             final List<AnsibleRole> roles = roleDAO.findByAccountAndNames(account, domain.getRoles());
             if (roles.size() != domain.getRoles().length) {
-                return die("newNode: error finding roles");
+                progressMeter.error(METER_ERROR_ROLES_NOT_FOUND);
+                return die("newNode: error finding ansible roles");
             }
 
             // build automation directory for this run
             final ValidationResult errors = new ValidationResult();
             final File roleTgzDir = mkdirOrDie(new File(abs(bubbleFilesDir), "role_tgz"));
 
-            // Someone needs to create a new cloud compute instance...
+            progressMeter.write(METER_TICK_LAUNCHING_NODE);
             node.setState(BubbleNodeState.starting);
             nodeDAO.update(node);
 
@@ -197,6 +223,7 @@ public class StandardNetworkService implements NetworkService {
 
             // Sanity check that it came up OK
             if (!node.hasIp4() || !node.hasSshKey()) {
+                progressMeter.error(METER_ERROR_NO_IP_OR_SSH_KEY);
                 final String message = "newNode: node booted but has no IP or SSH key";
                 killNode(node, message);
                 return die(message);
@@ -204,19 +231,25 @@ public class StandardNetworkService implements NetworkService {
 
             // Prepare ansible roles
             // We must wait until after server is started, because some roles require ip4 in vars
+            progressMeter.write(METER_TICK_PREPARING_ROLES);
             final Map<String, Object> ctx = ansiblePrep.prepAnsible(automation, bubbleFilesDir, account, network, node, roles, errors, roleTgzDir, nn.fork(), nn.getRestoreKey());
-            if (errors.isInvalid()) throw new MultiViolationException(errors.getViolationBeans());
+            if (errors.isInvalid()) {
+                progressMeter.error(METER_ERROR_ROLE_VALIDATION_ERRORS);
+                throw new MultiViolationException(errors.getViolationBeans());
+            }
 
             // Create DNS A and AAAA records for node
+            progressMeter.write(METER_TICK_WRITING_DNS_RECORDS);
             final CloudService dnsService = cloudDAO.findByUuid(domain.getPublicDns());
             dnsService.getDnsDriver(configuration).setNode(node);
 
+            progressMeter.write(METER_TICK_PREPARING_INSTALL);
             node.setState(BubbleNodeState.preparing_install);
             nodeDAO.update(node);
 
             // This node is on our network, or is the very first server. We must run ansible on it ourselves.
             // write playbook file
-            writeFile(automation, ctx, "playbook.yml", PLAYBOOK_TEMPLATE);
+            writeFile(automation, ctx, PLAYBOOK_YML, PLAYBOOK_TEMPLATE);
 
             // write inventory file
             final File inventory = new File(automation, "hosts");
@@ -244,11 +277,12 @@ public class StandardNetworkService implements NetworkService {
             writeFile(bubbleFilesDir, null, SAGE_KEY_JSON, json(BubbleNodeKey.sageMask(sageKey)));
 
             // write install_local.sh script
-            final File file = writeFile(automation, ctx, "install_local.sh", INSTALL_LOCAL_SH);
-            chmod(file, "500");
+            final File installLocalScript = writeFile(automation, ctx, INSTALL_LOCAL_SH, INSTALL_LOCAL_TEMPLATE);
+            chmod(installLocalScript, "500");
 
             // ensure this hostname is visible in our DNS and in public DNS,
             // or else node can't create its own letsencrypt SSL cert
+            progressMeter.write(METER_TICK_AWAITING_DNS);
             node.setState(BubbleNodeState.awaiting_dns);
             nodeDAO.update(node);
 
@@ -257,6 +291,7 @@ public class StandardNetworkService implements NetworkService {
             final DnsServiceDriver dnsDriver = dnsService.getDnsDriver(configuration);
             dnsDriver.ensureResolvable(domain, node, DNS_TIMEOUT);
 
+            progressMeter.write(METER_TICK_STARTING_INSTALL);
             node.setState(BubbleNodeState.installing);
             nodeDAO.update(node);
 
@@ -274,7 +309,7 @@ public class StandardNetworkService implements NetworkService {
             for (int i=0; i<MAX_ANSIBLE_TRIES; i++) {
                 sleep((i+1) * SECONDS.toMillis(5), "waiting to try ansible setup");
                 try {
-                    final CommandResult result = ansibleSetup(script);
+                    final CommandResult result = ansibleSetup(script, progressMeter);
                     // .... wait for ansible ...
                     if (!result.isZeroExitStatus()) {
                         return die("newNode: error in setup:\nstdout=" + result.getStdout() + "\nstderr=" + result.getStderr());
@@ -283,6 +318,7 @@ public class StandardNetworkService implements NetworkService {
                     break;
                 } catch (Exception e) {
                     log.error("newNode: error running ansible: "+e);
+                    progressMeter.reset();
                 }
             }
             if (!setupOk) return die("newNode: error setting up, all retries failed for node: "+node.getUuid());
@@ -295,12 +331,14 @@ public class StandardNetworkService implements NetworkService {
             }
             node.setState(BubbleNodeState.running);
             nodeDAO.update(node);
+            progressMeter.completed();
 
         } catch (Exception e) {
             log.error("newNode: "+e, e);
             if (node != null) {
                 node.setState(BubbleNodeState.unknown_error);
                 nodeDAO.update(node);
+                progressMeter.error(METER_UNKNOWN_ERROR);
                 killNode(node, "error: "+e);
             }
             return die("newNode: "+e, e);
@@ -313,24 +351,30 @@ public class StandardNetworkService implements NetworkService {
                     log.warn("newNode: compute.cleanupStart error: "+e, e);
                 }
             }
+            if (progressMeter != null) closeQuietly(progressMeter);
             unlockNetwork(nn.getNetwork(), lock);
         }
         return node;
     }
 
-    public CommandResult ansibleSetup(String script) throws IOException {
+    public CommandResult ansibleSetup(String script, OutputStream progressMeter) throws IOException {
         return CommandShell.exec(new Command(new CommandLine("/bin/bash")
                 .addArgument("-c")
                 .addArgument(script, false))
-                .setCopyToStandard(true));
+                .setOut(progressMeter)
+                .setCopyToStandard(configuration.testMode()));
     }
 
     protected String getAnsibleSetupScript(TempDir automation, String sshArgs, String nodeUser, String sshTarget) {
         return "cd " + abs(automation) + " && " +
+
                 // rsync ansible dir to remote host
+                "echo '" + METER_TICK_COPYING_ANSIBLE + "' && " +
                 "rsync -az -e \"ssh " + sshArgs + "\" . "+sshTarget+ ":" + ANSIBLE_DIR + " && " +
+
                 // run install_local.sh on remote host, installs ansible locally
-                "ssh "+sshArgs+" "+sshTarget+" ~"+nodeUser+ "/" + ANSIBLE_DIR + "/install_local.sh";
+                "echo '" + METER_TICK_RUNNING_ANSIBLE + "' && " +
+                "ssh "+sshArgs+" "+sshTarget+" ~"+nodeUser+ "/" + ANSIBLE_DIR + "/" + INSTALL_LOCAL_SH;
     }
 
     private File writeFile(File dir, Map<String, Object> ctx, String filename, String templateOrData) throws IOException {
@@ -583,4 +627,13 @@ public class StandardNetworkService implements NetworkService {
         }
         return cloud;
     }
+
+    public NodeProgressMeterTick getLaunchStatus(Account caller, String uuid) {
+        final String json = getNetworkSetupStatus().get(uuid);
+        if (json == null) return null;
+        final NodeProgressMeterTick tick = json(json, NodeProgressMeterTick.class);
+        if (!tick.getAccount().equals(caller.getUuid())) return null;
+        return tick.setPattern(null);
+    }
+
 }

bubble-server/src/main/java/bubble/service/cloud/StorageStreamService.java

@@ -21,6 +21,7 @@ import static java.util.UUID.randomUUID;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 
 @Service @Slf4j
@@ -39,13 +40,13 @@ public class StorageStreamService {
     public String registerRead(StorageStreamRequest request) {
         final String token = randomUUID().toString();
         request.setToken(token);
-        getReadRequests().set(token, json(request.setToken(token)), "EX", TOKEN_TTL);
+        getReadRequests().set(token, json(request.setToken(token)), EX, TOKEN_TTL);
         return token;
     }
 
     public String registerRead(StorageStreamRequest request, WriteRequest writeRequest) {
         final String token = WR_PREFIX + writeRequest.requestId;
-        getReadRequests().set(token, json(request.setToken(token)), "EX", TOKEN_TTL);
+        getReadRequests().set(token, json(request.setToken(token)), EX, TOKEN_TTL);
         return token;
     }
 

bubble-server/src/main/resources/bubble/node_progress_meter_ticks.json

@@ -0,0 +1,5 @@
+[
+  { "percent": 10, "messageKey": "meter_tick_apt_install", "pattern": "some pattern installing apt packages" },
+  { "percent": 30, "messageKey": "meter_tick_db_setup", "pattern": "some pattern setting up database" },
+  { "percent": 99, "messageKey": "meter_tick_starting_bubble_api_server", "pattern": "some pattern starting api" }
+]

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -193,6 +193,32 @@ field_payment_card_number=Credit or Debit Card
 button_label_submit_card=Verify Card
 message_verified_card=Card Successfully Verified
 
+# Launch progress meter
+meter_tick_confirming_network_lock=Confirming network lock
+meter_tick_verifying_node_network_and_plan=Verifying settings for Bubble
+meter_tick_creating_node=Creating Bubble node
+meter_tick_launching_node=Launching Bubble node
+meter_tick_preparing_roles=Preparing installation parameters
+meter_tick_writing_dns_records=Writing DNS records
+meter_tick_preparing_install=Creating installation package
+meter_tick_awaiting_dns=Awaiting DNS visibility
+meter_tick_starting_install=Connecting to node to install Bubble
+meter_tick_copying_ansible=Copying files required to install Bubble
+meter_tick_running_ansible=Starting Bubble installation
+
+meter_completed=Bubble installation completed successfully! On to your Bubble!
+
+meter_error_confirming_network_lock=Error confirming network lock
+meter_error_network_not_ready_for_setup=Cannot launch Bubble when network is not in 'setup' state
+meter_error_no_current_node_or_network=Current API does not have a node or network, cannot launch Bubble
+meter_error_plan_not_enabled=Account plan is not enabled, cannot launch Bubble
+meter_error_node_cloud_not_found=Compute cloud was not found, cannot launch Bubble
+meter_error_bubble_jar_not_found=Bubble jar file was not found, cannot launch Bubble
+meter_error_roles_not_found=Ansible roles were not found, cannot launch Bubble
+meter_error_no_ip_or_ssh_key=Bubble node started, but does not have an IP address or SSH key, cannot install Bubble
+meter_error_role_validation_errors=Validation of ansible roles failed, cannot install Bubble
+meter_unknown_error=An unknown error occurred
+
 # Error messages from API server
 err.accountContactsJson.length=Account contacts length violation
 err.accountOperationTimeout.required=Account operation timeout is required

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit c7c15206e20cbfbb3fdc543a53fa4b376e8a08f5
+Subproject commit 34b069c3453fd2b9dbb9c7af10306346e1bc3d8a

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 9a6cb168d03c7cf3f765425607c9477bb0c870b1
+Subproject commit e3256afba721949ef4993ce7e13ec5d5ef8c2368