upgrade jar now works

bin/bpatchfull

@@ -14,8 +14,6 @@
 # Patch the bubble.jar on a remote node.
 # This script updates the entire jar file, and takes a lot longer than bpatch
 #
-# You install the JDK on the remote node first: apt install openjdk-11-jdk-headless
-#
 SCRIPT="${0}"
 SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd)
 . ${SCRIPT_DIR}/bubble_common
@@ -57,7 +55,7 @@ else
   ssh ${HOST} "cat /tmp/bubble.jar > ~bubble/api/bubble.jar && supervisorctl restart bubble"
 fi
 
-if [[ $(jar tf ./target/bubble*.jar | grep "^site/$") ]] ; then
+if unzip -Z -1 ./target/bubble*.jar | grep -q "^site/$" ; then
   echo "Deploying new web..."
-  ssh ${HOST} "cd ~bubble && jar xf /tmp/bubble.jar site && chown -R bubble:bubble site"
+  ssh ${HOST} "cd ~bubble && unzip -o /tmp/bubble.jar 'site/*' && chown -R bubble:bubble site"
 fi

bubble-server/src/main/java/bubble/model/cloud/BubbleVersionInfo.java

@@ -1,12 +1,15 @@
 package bubble.model.cloud;
 
+import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.ToString;
 import lombok.experimental.Accessors;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.wizard.model.SemanticVersion.isNewerVersion;
 
-@Accessors(chain=true)
+@Accessors(chain=true) @EqualsAndHashCode(of={"version", "sha256"}) @ToString(of={"version", "sha256"})
 public class BubbleVersionInfo {
 
     @Getter @Setter private String version;
@@ -14,4 +17,8 @@ public class BubbleVersionInfo {
 
     public boolean valid() { return !empty(version) && !empty(sha256); }
 
+    public boolean newerThan(String otherVersion) { return isNewerVersion(otherVersion, version); }
+
+    public boolean newerThan(BubbleVersionInfo info) { return newerThan(info.getVersion()); }
+
 }

bubble-server/src/main/java/bubble/notify/NotificationHandler_hello_from_sage.java

@@ -43,6 +43,13 @@ public class NotificationHandler_hello_from_sage extends ReceivedNotificationHan
     @Override public void handleNotification(ReceivedNotification n) {
         // Upstream is telling us about our peers
         final BubbleNode payloadNode = n.getNode();
+
+        // First check to see if the sage reported a new jar version available
+        if (payloadNode.hasSageVersion()) {
+            log.info("handleNotification: payload node has sage version: "+payloadNode.getSageVersion());
+            configuration.setSageVersion(payloadNode.getSageVersion());
+        }
+
         final BubbleNode thisNode = configuration.getThisNode();
         final List<BubbleNode> peers = payloadNode.getPeers();
         int peerCount = 0;

bubble-server/src/main/java/bubble/notify/upgrade/NotificationHandler_upgrade_response.java

@@ -0,0 +1,9 @@
+/**
+ * Copyright (c) 2020 Bubble, Inc.  All rights reserved.
+ * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+ */
+package bubble.notify.upgrade;
+
+import bubble.notify.ReceivedNotificationHandlerBase;
+
+public class NotificationHandler_upgrade_response extends ReceivedNotificationHandlerBase {}

bubble-server/src/main/java/bubble/resources/account/AuthResource.java

@@ -678,26 +678,25 @@ public class AuthResource {
 
     @Autowired private BubbleJarUpgradeService upgradeService;
 
-    @GET @Path(EP_UPGRADE+"/{key}")
+    @GET @Path(EP_UPGRADE+"/{node}/{key}")
     @Produces(APPLICATION_OCTET_STREAM)
     public Response getUpgrade(@Context Request req,
                                @Context ContainerRequest ctx,
+                               @PathParam("node") String nodeUuid,
                                @PathParam("key") String key) {
-        final String nodeUuid = upgradeService.getNodeForKey(key);
-        if (nodeUuid == null) {
+        final String nodeForKey = upgradeService.getNodeForKey(key);
+        if (nodeForKey == null) {
             log.warn("getUpgrade: key not found: "+key);
             return unauthorized();
         }
-
-        final BubbleNode node = nodeDAO.findByUuid(nodeUuid);
-        if (node == null) {
-            log.warn("getUpgrade: node not found: "+nodeUuid);
+        if (!nodeForKey.equals(nodeUuid)) {
+            log.warn("getUpgrade: key not for provided node");
             return unauthorized();
         }
 
-        final String remoteAddr = req.getRemoteAddr();
-        if (!node.hasSameIp(remoteAddr)) {
-            log.warn("getUpgrade: node has wrong IP (request came from "+remoteAddr+"): "+node.id());
+        final BubbleNode node = nodeDAO.findByUuid(nodeForKey);
+        if (node == null) {
+            log.warn("getUpgrade: node not found: "+nodeForKey);
             return unauthorized();
         }
 

bubble-server/src/main/java/bubble/resources/account/MeResource.java

@@ -28,6 +28,7 @@ import bubble.service.account.StandardAuthenticatorService;
 import bubble.service.account.download.AccountDownloadService;
 import bubble.service.boot.BubbleJarUpgradeService;
 import bubble.service.boot.BubbleModelSetupService;
+import bubble.service.boot.SageHelloService;
 import bubble.service.cloud.NodeLaunchMonitor;
 import com.fasterxml.jackson.databind.JsonNode;
 import lombok.Cleanup;
@@ -62,10 +63,12 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
 
 import static bubble.ApiConstants.*;
 import static bubble.model.account.Account.validatePassword;
 import static bubble.resources.account.AuthResource.forgotPasswordMessage;
+import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.http.HttpContentTypes.*;
 import static org.cobbzilla.util.json.JsonUtil.json;
@@ -406,11 +409,35 @@ public class MeResource {
         return ok(modelSetupService.setupModel(api, caller, modelFile));
     }
 
+    @Autowired private SageHelloService sageHelloService;
     @Autowired private BubbleJarUpgradeService jarUpgradeService;
 
+    private final AtomicLong lastUpgradeCheck = new AtomicLong(0);
+    private static final long UPGRADE_CHECK_INTERVAL = MINUTES.toMillis(5);
+
+    @GET @Path(EP_UPGRADE)
+    public Response checkForUpgrade(@Context Request req,
+                                    @Context ContainerRequest ctx) {
+        final Account caller = userPrincipal(ctx);
+        if (!caller.admin()) return forbidden();
+        authenticatorService.ensureAuthenticated(ctx);
+
+        synchronized (lastUpgradeCheck) {
+            if (now() - lastUpgradeCheck.get() > UPGRADE_CHECK_INTERVAL) {
+                lastUpgradeCheck.set(now());
+                sageHelloService.interrupt();
+            }
+        }
+        return ok_empty();
+    }
+
     @POST @Path(EP_UPGRADE)
-    public Response uploadModel(@Context Request req,
-                                @Context ContainerRequest ctx) {
+    public Response upgrade(@Context Request req,
+                            @Context ContainerRequest ctx) {
+        final Account caller = userPrincipal(ctx);
+        if (!caller.admin()) return forbidden();
+        authenticatorService.ensureAuthenticated(ctx);
+
         background(() -> jarUpgradeService.upgrade());
         return ok(configuration.getPublicSystemConfigs());
     }

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -251,16 +251,18 @@ public class BubbleConfiguration extends PgRestServerConfiguration
                 .setSha256(getJarSha());
     }
 
-    @Getter private BubbleVersionInfo sageVersionInfo;
-    public void setSageVersionInfo(BubbleVersionInfo version) {
-        sageVersionInfo = version;
-        final boolean isNewer = isNewerVersion(getVersionInfo().getVersion(), sageVersionInfo.getVersion());
+    @Getter private BubbleVersionInfo sageVersion;
+    public void setSageVersion(BubbleVersionInfo version) {
+        sageVersion = version;
+        final boolean isNewer = version == null ? false : isNewerVersion(getVersionInfo().getVersion(), sageVersion.getVersion());
         if (!jarUpgradeAvailable && isNewer) {
             jarUpgradeAvailable = true;
-            refreshPublicSystemConfigs();
+        } else {
+            jarUpgradeAvailable = false;
         }
+        refreshPublicSystemConfigs();
     }
-    public boolean hasSageVersionInfo () { return sageVersionInfo != null; }
+    public boolean hasSageVersion () { return sageVersion != null; }
     @Getter private Boolean jarUpgradeAvailable = false;
 
     @JsonIgnore public String getUnlockKey () { return BubbleFirstTimeListener.getUnlockKey(); }
@@ -339,7 +341,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration
                         {TAG_SUPPORT, getSupport()},
                         {TAG_SECURITY_LEVELS, DeviceSecurityLevel.values()},
                         {TAG_JAR_VERSION, getVersion()},
-                        {TAG_JAR_UPGRADE_AVAILABLE, getJarUpgradeAvailable() ? getSageVersionInfo() : null}
+                        {TAG_JAR_UPGRADE_AVAILABLE, getJarUpgradeAvailable() ? getSageVersion() : null}
                 }));
             } else {
                 // some things has to be refreshed all the time in some cases:

bubble-server/src/main/java/bubble/service/bill/StandardRefundService.java

@@ -32,13 +32,17 @@ public class StandardRefundService extends SimpleDaemon implements RefundService
     @Autowired private AccountPaymentMethodDAO paymentMethodDAO;
     @Autowired private BubbleConfiguration configuration;
 
-    @Override public void processRefunds () { interrupt(); }
+    @Override public void processRefunds () {
+        log.info("processRefunds: interrupting thread...");
+        interrupt();
+    }
 
     @Override protected long getSleepTime() { return REFUND_CHECK_INTERVAL; }
 
     @Override protected boolean canInterruptSleep() { return true; }
 
     @Override protected void process() {
+        log.info("process: handling refunds...");
         // iterate over all account plans that have been deleted but not yet closed
         final List<AccountPlan> pendingPlans = accountPlanDAO.findByDeletedAndNotClosedAndNoRefundIssued();
         for (AccountPlan accountPlan : pendingPlans) {

bubble-server/src/main/java/bubble/service/boot/BubbleJarUpgradeService.java

@@ -8,24 +8,26 @@ import bubble.notify.upgrade.JarUpgradeNotification;
 import bubble.server.BubbleConfiguration;
 import bubble.service.backup.BackupService;
 import bubble.service.notify.NotificationService;
+import lombok.Cleanup;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.http.HttpRequestBean;
+import org.cobbzilla.util.http.HttpUtil;
+import org.cobbzilla.util.io.FileUtil;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
+import java.io.InputStream;
 
-import static bubble.ApiConstants.AUTH_ENDPOINT;
-import static bubble.ApiConstants.EP_UPGRADE;
+import static bubble.ApiConstants.*;
 import static bubble.client.BubbleNodeClient.nodeBaseUri;
 import static bubble.model.cloud.notify.NotificationType.upgrade_request;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
 import static org.cobbzilla.util.daemon.ZillaRuntime.now;
-import static org.cobbzilla.util.http.HttpMethods.GET;
+import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
 import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.system.Sleep.sleep;
 import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_YYYY_MM_DD_HH_mm_ss_SSS;
@@ -52,28 +54,35 @@ public class BubbleJarUpgradeService {
 
     public String getNodeForKey(String key) { return getNodeUpgradeRequests().get(key); }
 
+    // set to 'false' for faster debugging of upgrade process
+    private static final boolean BACKUP_BEFORE_UPGRADE = true;
+
     public void upgrade() {
         if (!configuration.getJarUpgradeAvailable()) {
             log.warn("upgrade: No upgrade available, returning");
             return;
         }
-        final String currentVersion = configuration.getVersion();
-        final BubbleVersionInfo sageVersion = configuration.getSageVersionInfo();
-        final String newVersion = sageVersion.getVersion();
-        BubbleBackup bubbleBackup = backupService.queueBackup("before_upgrade_" + currentVersion + "_to_" + newVersion + "_on_" + DATE_FORMAT_YYYY_MM_DD_HH_mm_ss_SSS.print(now()));
-
-        // monitor backup, ensure it completes
-        final long start = now();
-        while (bubbleBackup.getStatus() != BackupStatus.backup_completed && now() - start < PRE_UPGRADE_BACKUP_TIMEOUT) {
-            sleep(SECONDS.toMillis(5), "waiting for backup to complete before upgrading");
-            bubbleBackup = backupDAO.findByUuid(bubbleBackup.getUuid());
-        }
-        if (bubbleBackup.getStatus() != BackupStatus.backup_completed) {
-            log.warn("upgrade: timeout waiting for backup to complete, status="+bubbleBackup.getStatus());
-            return;
+
+        final BubbleVersionInfo sageVersion = configuration.getSageVersion();
+
+        if (BACKUP_BEFORE_UPGRADE) {
+            final String currentVersion = configuration.getVersion();
+            final String newVersion = sageVersion.getVersion();
+            BubbleBackup bubbleBackup = backupService.queueBackup("before_upgrade_" + currentVersion + "_to_" + newVersion + "_on_" + DATE_FORMAT_YYYY_MM_DD_HH_mm_ss_SSS.print(now()));
+
+            // monitor backup, ensure it completes
+            final long start = now();
+            while (bubbleBackup.getStatus() != BackupStatus.backup_completed && now() - start < PRE_UPGRADE_BACKUP_TIMEOUT) {
+                sleep(SECONDS.toMillis(5), "waiting for backup to complete before upgrading");
+                bubbleBackup = backupDAO.findByUuid(bubbleBackup.getUuid());
+            }
+            if (bubbleBackup.getStatus() != BackupStatus.backup_completed) {
+                log.warn("upgrade: timeout waiting for backup to complete, status=" + bubbleBackup.getStatus());
+                return;
+            }
         }
 
-        final File upgradeJar = new File(configuration.getBubbleJar().getParentFile(), ".upgrade.jar");
+        final File upgradeJar = new File(HOME_DIR, "upgrade.jar");
         if (upgradeJar.exists()) {
             log.error("upgrade: jar already exists, not upgrading: "+abs(upgradeJar));
             return;
@@ -81,13 +90,23 @@ public class BubbleJarUpgradeService {
 
         // ask the sage to allow us to download the upgrade
         final String key = notificationService.notifySync(configuration.getSageNode(), upgrade_request, new JarUpgradeNotification(sageVersion));
+        log.info("upgrade: received upgrade key from sage: "+key);
 
         // request the jar from the sage
-        final String uri = nodeBaseUri(configuration.getSageNode(), configuration) + AUTH_ENDPOINT + EP_UPGRADE + "/" + key;
-        final HttpRequestBean requestBean = new HttpRequestBean(GET, uri);
-        final File newJar = temp(".jar");
+        final String uri = AUTH_ENDPOINT + EP_UPGRADE + "/" + configuration.getThisNode().getUuid() + "/" + key;
+        final String url = nodeBaseUri(configuration.getSageNode(), configuration) + uri;
+        final File newJar;
+        try {
+            newJar = temp(".jar");
+            @Cleanup final InputStream in = HttpUtil.getUrlInputStream(url);
+            FileUtil.toFile(newJar, in);
+        } catch (Exception e) {
+            log.error("upgrade: error downloading jar: "+shortError(e));
+            return;
+        }
 
-        // move to upgrade location
+        // move to upgrade location, should trigger upgrade monitor
+        log.info("upgrade: writing upgradeJar: "+abs(upgradeJar));
         renameOrDie(newJar, upgradeJar);
     }
 }

bubble-server/src/main/java/bubble/service/boot/SageHelloService.java

@@ -34,6 +34,7 @@ public class SageHelloService extends SimpleDaemon {
 
     @Override protected long getStartupDelay() { return HELLO_SAGE_START_DELAY; }
     @Override protected long getSleepTime() { return HELLO_SAGE_INTERVAL; }
+    @Override protected boolean canInterruptSleep() { return true; }
 
     @Autowired private BubbleNodeDAO nodeDAO;
     @Autowired private BubbleConfiguration configuration;

bubble-server/src/main/resources/message_templates/en_US/server/post_auth/ResourceMessages.properties

@@ -47,6 +47,7 @@ message_jar_upgrade_version=The new Bubble version is
 message_jar_current_version=Your current Bubble version is
 button_label_jar_upgrade=Upgrade Your Bubble
 button_label_jar_upgrading=Upgrading...
+message_jar_checking_for_upgrade=Checking for Bubble upgrade...
 message_jar_upgrading=Your Bubble may be unresponsive for a minute or two while the upgrade occurs
 
 # Account SSH key fields

bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
 
 BUBBLE_HOME="/home/bubble"
-UPGRADE_JAR="${BUBBLE_HOME}/api/.upgrade.jar"
+UPGRADE_JAR="${BUBBLE_HOME}/upgrade.jar"
 BUBBLE_JAR="${BUBBLE_HOME}/api/bubble.jar"
-
 LOG=/tmp/bubble.upgrade.log
 
 function die {
@@ -17,44 +16,57 @@ function log {
 }
 
 function verify_api_ok {
-    log "Restarting API..."
-    supervisorctl restart bubble || die "Error restarting bubble"
+    log "verify_api_ok: Restarting API..."
+    if supervisorctl restart bubble > /dev/null 2>> ${LOG} ; then
+      log "verify_api_ok: Restarted API"
+    else
+      log "verify_api_ok: Error restarting API"
+      echo "error"
+      return
+    fi
 
-    OK=255
+    sleep 20s
+    CURL_STATUS=255
     START_VERIFY=$(date +%s)
     VERIFY_TIMEOUT=180
     VERIFY_URL="https://$(hostname):1443/api/auth/ready"
-    if [[ ${OK} -ne 0 && $(expr $(date +%s) - ${START_VERIFY} -le ${VERIFY_TIMEOUT}) ]] ; then
-      sleep 10s
-      log "Verifying ${VERIFY_URL} is OK...."
-      curl "${VERIFY_URL}" 2>&1 | tee -a ${LOG}
-      OK=$?
-    fi
+    while [[ $(expr $(date +%s) - ${START_VERIFY}) -le ${VERIFY_TIMEOUT} ]] ; do
+      log "verify_api_ok: Verifying ${VERIFY_URL} is OK...."
+      CURL_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "${VERIFY_URL}")
+      if [[ -z "${CURL_STATUS}" || ${CURL_STATUS} -ne 200 ]] ; then
+        log "verify_api_ok: curl ${VERIFY_URL} returned not-ok HTTP status: ${CURL_STATUS}"
+        sleep 4s
+        continue
+      else
+        break
+      fi
+    done
 
-    if [[ ${OK} -eq 0 ]] ; then
+    log "verify_api_ok: while loop ended, CURL_STATUS=${CURL_STATUS}, (date - start)=$(expr $(date +%s) - ${START_VERIFY}), VERIFY_TIMEOUT=${VERIFY_TIMEOUT}"
+    if [[ ! -z "${CURL_STATUS}" && ${CURL_STATUS} -eq 200 ]] ; then
       echo "ok"
     else
       echo "error"
     fi
 }
 
-BACKUP_JAR=$(mktemp /tmp/bubble.jar.XXXXXXX)
+BACKUP_JAR="$(mktemp /tmp/bubble.jar.XXXXXXX)"
 log "Backing up to ${BACKUP_JAR} ..."
-cp ${BUBBLE_JAR} ${BACKUP_JAR} || die "Error backing up existing jar before upgrade ${BUBBLE_JAR} ${BACKUP_JAR}"
+cp "${BUBBLE_JAR}" "${BACKUP_JAR}" || die "Error backing up existing jar before upgrade ${BUBBLE_JAR} ${BACKUP_JAR}"
 
 log "Upgrading..."
-mv ${UPGRADE_JAR} ${BUBBLE_JAR} || die "Error moving ${UPGRADE_JAR} -> ${BUBBLE_JAR}"
+mv "${UPGRADE_JAR}" "${BUBBLE_JAR}" || die "Error moving ${UPGRADE_JAR} -> ${BUBBLE_JAR}"
 
 log "Verifying upgrade..."
 API_OK=$(verify_api_ok)
 if [[ -z "${API_OK}" || "${API_OK}" != "ok" ]] ; then
-  log "Error starting upgraded API, reverting...."
-  cp ${BACKUP_JAR} ${BUBBLE_JAR} || die "Error restoring API jar from backup!"
+  log "Error starting upgraded API (API_OK=${API_OK}), reverting...."
+  cp "${BACKUP_JAR}" "${BUBBLE_JAR}" || die "Error restoring API jar from backup!"
   API_OK=$(verify_api_ok)
   if [[ -z "${API_OK}" || "${API_OK}" != "ok" ]] ; then
-    log "Error starting API from backup!"
+    die "Error starting API from backup (API_OK=${API_OK})"
   fi
 else
   log "Upgrading web site files..."
-  cd ~bubble && jar xf ${BUBBLE_JAR} site && chown -R bubble:bubble site || die "Error updating web files..."
+  cd ~bubble && unzip -o "${BUBBLE_JAR}" 'site/*' && chown -R bubble:bubble site || die "Error updating web files..."
 fi

bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade_monitor.sh

@@ -3,15 +3,20 @@
 # Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
 #
 THIS_DIR="$(cd "$(dirname "${0}")" && pwd)"
+
+BUBBLE_HOME="/home/bubble"
+UPGRADE_JAR="${BUBBLE_HOME}/upgrade.jar"
 LOG=/tmp/bubble.upgrade.log
 
 function log {
   echo "$(date): ${1}" >> ${LOG}
 }
 
+log "Watching ${UPGRADE_JAR} for upgrades"
 while : ; do
   sleep 5
   if [[ -f "${UPGRADE_JAR}" ]] ; then
+    log "${UPGRADE_JAR} exists, upgrading..."
     "${THIS_DIR}/bubble_upgrade.sh"
     if [[ $? -eq 0 ]] ; then
       log "Upgrade completed successfully"

bubble-server/src/main/resources/packer/roles/bubble/files/supervisor_bubble_upgrade_monitor.conf

@@ -1,5 +1,5 @@
 
-[program:supervisor_bubble_upgrade_monitor]
+[program:bubble_upgrade_monitor]
 stdout_logfile = /dev/null
 stderr_logfile = /dev/null
-command=/usr/local/sbin/supervisor_bubble_upgrade_monitor.sh
+command=/usr/local/sbin/bubble_upgrade_monitor.sh

bubble-server/src/test/resources/models/tests/live/backup_and_restore.json

@@ -138,7 +138,7 @@
 
   {
     "comment": "wait for network to stop",
-    "before": "await_url me/networks/{{bubbleNetwork.network}} 5m 10s await_json.getState().name() == 'stopped'",
+    "before": "await_url me/networks/{{bubbleNetwork.network}} 5m 10s await_json.getState().name() === 'stopped'",
     "request": { "uri": "me" }
   },
 

bubble-server/src/test/resources/models/tests/payment/pay_credit_refund_and_restart.json

@@ -184,7 +184,7 @@
   },
 
   {
-    "before": "sleep 15s",
+    "before": "await_url me/payments 1m 2s await_json.length === 2",
     "comment": "verify refund payment has been processed",
     "request": { "uri": "me/payments" },
     "response": {

bubble-web

@@ -1 +1 @@
-Subproject commit 33c83fdbfebcbaa7349bf78538855d5878cd5a75
+Subproject commit d9d5146ff2229942ec0637b43ebd927d2f7ebb1e

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 549884d63dc1d46f15c33cdcf5d9604deb821992
+Subproject commit 946d62be17b43672695c780026d8a742ca03ce0e