recurring billing works

bubble-server/src/main/java/bubble/cloud/payment/stripe/StripePaymentDriver.java

@@ -245,6 +245,7 @@ public class StripePaymentDriver extends PaymentDriverBase<StripePaymentDriverCo
                     case "succeeded":
                         log.info("charge: charge successful: "+authCacheKey);
                         chargeCache.set(billUuid, captured.getId(), "EX", CHARGE_CACHE_DURATION);
+                        authCache.del(authCacheKey);
                         return captured.getId();
 
                     case "pending":

bubble-server/src/main/java/bubble/dao/bill/AccountPlanDAO.java

@@ -4,7 +4,9 @@ import bubble.cloud.payment.PaymentServiceDriver;
 import bubble.dao.account.AccountOwnedEntityDAO;
 import bubble.dao.cloud.BubbleNetworkDAO;
 import bubble.dao.cloud.CloudServiceDAO;
-import bubble.model.bill.*;
+import bubble.model.bill.AccountPlan;
+import bubble.model.bill.Bill;
+import bubble.model.bill.BubblePlan;
 import bubble.model.cloud.BubbleNetwork;
 import bubble.model.cloud.BubbleNetworkState;
 import bubble.model.cloud.CloudService;
@@ -52,6 +54,14 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO<AccountPlan> {
         )));
     }
 
+    public List<AccountPlan> findBillableAccountPlans(long time) {
+        return list(criteria().add(and(
+                isNull("deleted"),
+                eq("closed", false),
+                le("nextBill", time)
+        )));
+    }
+
     @Override public Object preCreate(AccountPlan accountPlan) {
         if (configuration.paymentsEnabled()) {
             if (!accountPlan.hasPaymentMethodObject()) throw invalidEx("err.paymentMethod.required");
@@ -75,6 +85,10 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO<AccountPlan> {
                 paymentDriver.authorize(plan, accountPlan.getUuid(), accountPlan.getPaymentMethodObject());
             }
             accountPlan.setPaymentMethod(accountPlan.getPaymentMethodObject().getUuid());
+            accountPlan.setNextBill(0L); // bill and payment occurs in postCreate, will update this
+            accountPlan.setNextBillDate();
+        } else {
+            accountPlan.setNextBill(Long.MAX_VALUE);
         }
         return super.preCreate(accountPlan);
     }
@@ -84,20 +98,14 @@ public class AccountPlanDAO extends AccountOwnedEntityDAO<AccountPlan> {
             final String accountPlanUuid = accountPlan.getUuid();
             final String paymentMethodUuid = accountPlan.getPaymentMethodObject().getUuid();
             final BubblePlan plan = planDAO.findByUuid(accountPlan.getPlan());
-            final Bill bill = billDAO.create(new Bill()
-                    .setAccount(accountPlan.getAccount())
-                    .setPlan(plan.getUuid())
-                    .setAccountPlan(accountPlanUuid)
-                    .setPrice(plan.getPrice())
-                    .setCurrency(plan.getCurrency())
-                    .setPeriod(plan.getPeriod().currentPeriod())
-                    .setPeriodStart(plan.getPeriod().getFirstPeriodStart())
-                    .setPeriodEnd(plan.getPeriod().getFirstPeriodEnd())
-                    .setQuantity(1L)
-                    .setType(BillItemType.compute)
-                    .setStatus(BillStatus.unpaid));
+            final Bill bill = billDAO.createFirstBill(plan, accountPlan);
             final String billUuid = bill.getUuid();
 
+            // set nextBill to be just after the current bill period ends
+            accountPlan.setNextBill(plan.getPeriod().periodMillis(bill.getPeriodEnd()));
+            accountPlan.setNextBillDate();
+            update(accountPlan);
+
             final CloudService paymentService = cloudDAO.findByUuid(accountPlan.getPaymentMethodObject().getCloud());
             if (paymentService == null) throw invalidEx("err.paymentService.notFound");
 

bubble-server/src/main/java/bubble/dao/bill/BillDAO.java

@@ -1,8 +1,7 @@
 package bubble.dao.bill;
 
 import bubble.dao.account.AccountOwnedEntityDAO;
-import bubble.model.bill.Bill;
-import bubble.model.bill.BillStatus;
+import bubble.model.bill.*;
 import org.hibernate.criterion.Order;
 import org.springframework.stereotype.Repository;
 
@@ -27,4 +26,24 @@ public class BillDAO extends AccountOwnedEntityDAO<Bill> {
         return findByFields("accountPlan", accountPlanUuid, "status", BillStatus.unpaid);
     }
 
+    public Bill createFirstBill(BubblePlan plan, AccountPlan accountPlan) {
+        return create(newBill(plan, accountPlan, accountPlan.getCtime()));
+    }
+
+    public Bill newBill(BubblePlan plan, AccountPlan accountPlan, long periodStartMillis) {
+        final BillPeriod period = plan.getPeriod();
+        return new Bill()
+                    .setAccount(accountPlan.getAccount())
+                    .setPlan(plan.getUuid())
+                    .setAccountPlan(accountPlan.getUuid())
+                    .setPrice(plan.getPrice())
+                    .setCurrency(plan.getCurrency())
+                    .setPeriodLabel(period.periodLabel(periodStartMillis))
+                    .setPeriodStart(period.periodStart(periodStartMillis))
+                    .setPeriodEnd(period.periodEnd(periodStartMillis))
+                    .setQuantity(1L)
+                    .setType(BillItemType.compute)
+                    .setStatus(BillStatus.unpaid);
+    }
+
 }

bubble-server/src/main/java/bubble/dao/cloud/BubbleNodeKeyDAO.java

@@ -1,9 +1,12 @@
 package bubble.dao.cloud;
 
 import bubble.dao.account.AccountOwnedEntityDAO;
+import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.BubbleNodeKey;
+import bubble.service.boot.SelfNodeService;
 import lombok.extern.slf4j.Slf4j;
 import org.hibernate.criterion.Order;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
 import java.util.List;
@@ -14,6 +17,8 @@ import static bubble.model.cloud.BubbleNodeKey.defaultExpiration;
 @Repository @Slf4j
 public class BubbleNodeKeyDAO extends AccountOwnedEntityDAO<BubbleNodeKey> {
 
+    @Autowired private SelfNodeService selfNodeService;
+
     @Override public Order getDefaultSortOrder() { return Order.desc("expiration"); }
 
     @Override public Object preCreate(BubbleNodeKey key) {
@@ -28,9 +33,19 @@ public class BubbleNodeKeyDAO extends AccountOwnedEntityDAO<BubbleNodeKey> {
     public BubbleNodeKey filterValid(BubbleNodeKey token) { return token != null && token.valid() ? token : null; }
 
     public List<BubbleNodeKey> findByNode(String uuid) {
-        final List<BubbleNodeKey> tokens = findByField("node", uuid);
-        tokens.forEach(t -> { if (!t.valid()) delete(t.getUuid()); });
-        return filterValid(tokens);
+        final List<BubbleNodeKey> keys = findByField("node", uuid);
+        keys.forEach(t -> { if (!t.valid()) delete(t.getUuid()); });
+        final List<BubbleNodeKey> validKeys = filterValid(keys);
+
+        if (validKeys.isEmpty()) {
+            final BubbleNode thisNode = selfNodeService.getThisNode();
+            if (thisNode != null && thisNode.getUuid().equals(uuid)) {
+                // we just deleted the last key for ourselves. create a new one.
+                validKeys.add(create(new BubbleNodeKey(thisNode)));
+            }
+        }
+
+        return validKeys;
     }
 
     @Override public BubbleNodeKey findByUuid(String uuid) { return filterValid(super.findByUuid(uuid)); }

bubble-server/src/main/java/bubble/model/bill/AccountPlan.java

@@ -17,6 +17,7 @@ import javax.persistence.Entity;
 import javax.persistence.Transient;
 import javax.validation.constraints.Size;
 
+import static bubble.model.bill.BillPeriod.BILL_START_END_FORMAT;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 
 @ECType(root=true)
@@ -69,6 +70,13 @@ public class AccountPlan extends IdentifiableBase implements HasAccount {
     @Getter @Setter private Boolean enabled = false;
     public boolean enabled() { return enabled != null && enabled; }
 
+    @Column(nullable=false)
+    @ECIndex @Getter @Setter private Long nextBill;
+
+    @Column(nullable=false, length=20)
+    @Getter @Setter private String nextBillDate;
+    public AccountPlan setNextBillDate() { return setNextBillDate(BILL_START_END_FORMAT.print(getNextBill())); }
+
     @ECIndex @Getter @Setter private Long deleted;
     public boolean deleted() { return deleted != null; }
     public boolean notDeleted() { return !deleted(); }

bubble-server/src/main/java/bubble/model/bill/Bill.java

@@ -52,7 +52,7 @@ public class Bill extends IdentifiableBase implements HasAccountNoName {
     @Getter @Setter private BillItemType type;
 
     @Column(nullable=false, updatable=false, length=20)
-    @ECIndex @Getter @Setter private String period;
+    @ECIndex @Getter @Setter private String periodLabel;
 
     @Column(nullable=false, updatable=false, length=20)
     @Getter @Setter private String periodStart;

bubble-server/src/main/java/bubble/model/bill/BillPeriod.java

@@ -11,7 +11,6 @@ import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 
 import static bubble.ApiConstants.enumFromString;
-import static org.cobbzilla.util.daemon.ZillaRuntime.now;
 import static org.cobbzilla.util.reflect.ReflectionUtil.instantiate;
 import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_YYYY_MM;
 
@@ -33,17 +32,20 @@ public enum BillPeriod {
         return Days.daysBetween(start.withTimeAtStartOfDay(), end.withTimeAtStartOfDay()).getDays();
     }
 
-    public String currentPeriod() { return formatter.print(now()); }
-
     public long calculateRefund(Bill bill, AccountPlan accountPlan) {
         return getDriver().calculateRefund(bill, accountPlan);
     }
 
-    public String getFirstPeriodStart() { return BILL_START_END_FORMAT.print(now()); }
+    public String periodLabel(long t) { return getFormatter().print(t); }
+
+    public String periodStart(long time) { return BILL_START_END_FORMAT.print(new DateTime(time).withTimeAtStartOfDay()); }
 
-    public String getFirstPeriodEnd() {
+    public String periodEnd(long time) {
         final DurationFieldType fieldType = getDriver().getDurationFieldType();
-        return BILL_START_END_FORMAT.print(new DateTime(now()).withTimeAtStartOfDay().withFieldAdded(fieldType, 1));
+        return BILL_START_END_FORMAT.print(new DateTime(time).withTimeAtStartOfDay().withFieldAdded(fieldType, 1));
     }
 
+    public long periodMillis(String period) {
+        return new DateTime(BILL_START_END_FORMAT.parseMillis(period)).withTimeAtStartOfDay().getMillis();
+    }
 }

bubble-server/src/main/java/bubble/server/listener/NodeInitializerListener.java

@@ -1,26 +1,15 @@
 package bubble.server.listener;
 
-import bubble.cloud.CloudServiceType;
-import bubble.cloud.storage.local.LocalStorageDriver;
 import bubble.dao.account.AccountDAO;
-import bubble.dao.cloud.BubbleNodeDAO;
-import bubble.dao.cloud.BubbleNodeKeyDAO;
-import bubble.dao.cloud.CloudServiceDAO;
 import bubble.model.cloud.BubbleNode;
-import bubble.model.cloud.BubbleNodeKey;
 import bubble.server.BubbleConfiguration;
-import bubble.service.bill.RefundService;
-import bubble.service.boot.SageHelloService;
+import bubble.service.boot.SelfNodeService;
 import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.string.StringUtil;
 import org.cobbzilla.wizard.server.RestServer;
 import org.cobbzilla.wizard.server.RestServerLifecycleListenerBase;
 
 import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
 
-import static bubble.server.BubbleServer.isRestoreMode;
 import static bubble.service.boot.StandardSelfNodeService.SELF_NODE_JSON;
 import static bubble.service.boot.StandardSelfNodeService.THIS_NODE_FILE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
@@ -32,7 +21,6 @@ import static org.cobbzilla.util.time.TimeUtil.DATE_FORMAT_YYYY_MM_DD_HH_mm_ss;
 public class NodeInitializerListener extends RestServerLifecycleListenerBase<BubbleConfiguration> {
 
     @Override public void onStart(RestServer server) {
-
         final BubbleConfiguration c = (BubbleConfiguration) server.getConfiguration();
         if (!c.getBean(AccountDAO.class).activated()) {
             final File nodeFile = THIS_NODE_FILE;
@@ -47,75 +35,10 @@ public class NodeInitializerListener extends RestServerLifecycleListenerBase<Bub
 
         final BubbleNode thisNode = c.getThisNode();
         if (thisNode != null) {
-            initThisNode(c, thisNode);
+            c.getBean(SelfNodeService.class).initThisNode(thisNode);
         } else {
             log.warn("onStart: thisNode was null, not doing standard initializations");
         }
     }
 
-    private boolean initThisNode(BubbleConfiguration c, BubbleNode thisNode) {
-        final BubbleNodeDAO nodeDAO = c.getBean(BubbleNodeDAO.class);
-
-        final BubbleNode dbThis = nodeDAO.findByUuid(thisNode.getUuid());
-        if (dbThis == null) return die("initThisNode: self_node not found in database: "+thisNode.getUuid());
-
-        // check database, ip4/ip6 may not have been set for ourselves. let's set them now
-        if (!dbThis.hasIp4()) {
-            log.info("initThisNode: updating ip4 for self_node in database: "+thisNode.id());
-            dbThis.setIp4(thisNode.getIp4());
-        } else if (thisNode.hasIp4() && !dbThis.getIp4().equals(thisNode.getIp4())) {
-            log.warn("initThisNode: self_node ("+thisNode.getIp4()+") and database row ("+dbThis.getIp4()+") have differing ip4 addresses for node "+thisNode.getUuid());
-            dbThis.setIp4(thisNode.getIp4());
-        }
-
-        if (!dbThis.hasIp6()) {
-            log.info("initThisNode: updating ip6 for self_node in database: "+thisNode.id());
-            dbThis.setIp6(thisNode.getIp6());
-        } else if (thisNode.hasIp6() && !dbThis.getIp6().equals(thisNode.getIp6())) {
-            log.warn("initThisNode: self_node ("+thisNode.getIp6()+") and database row ("+dbThis.getIp6()+") have differing ip6 addresses for node "+thisNode.getUuid());
-            dbThis.setIp6(thisNode.getIp6());
-        }
-        nodeDAO.update(dbThis);
-
-        // ensure a token exists so we can call ourselves
-        final BubbleNodeKeyDAO keyDAO = c.getBean(BubbleNodeKeyDAO.class);
-        final List<BubbleNodeKey> keys = keyDAO.findByNode(thisNode.getUuid());
-        if (BubbleNodeKey.shouldGenerateNewKey(keys)) {
-            keyDAO.create(new BubbleNodeKey(thisNode));
-        }
-
-        if (!isRestoreMode()) {
-            final CloudServiceDAO cloudDAO = c.getBean(CloudServiceDAO.class);
-            final String network = thisNode.getNetwork();
-
-            // ensure storage delegates use a network-specific key
-            final List<String> updatedClouds = new ArrayList<>();
-            cloudDAO.findByType(CloudServiceType.storage).stream()
-                    .filter(cloud -> cloud.getCredentials() != null
-                            && cloud.getCredentials().needsNewNetworkKey(network)
-                            && !cloud.getDriverClass().equals(LocalStorageDriver.class.getName()))
-                    .forEach(cloud -> {
-                            cloudDAO.update(cloud.setCredentials(cloud.getCredentials().initNetworkKey(network)));
-                            log.info("onStart: set network-specific key for storage: " + cloud.getName());
-                            updatedClouds.add(cloud.getName() + "/" + cloud.getUuid());
-                    });
-            if (!updatedClouds.isEmpty()) {
-                log.info("onStart: updated network-specific keys for storage clouds: " + StringUtil.toString(updatedClouds));
-            }
-        }
-
-        // start hello sage service, if we have a sage that is not ourselves
-        if (c.hasSageNode() && !c.isSelfSage()) {
-            log.info("onStart: starting SageHelloService");
-            c.getBean(SageHelloService.class).start();
-        }
-
-        // start RefundService if payments are enabled and this is a SageLauncher
-        if (c.paymentsEnabled() && c.isSageLauncher()) {
-            log.info("onStart: starting RefundService");
-            c.getBean(RefundService.class).start();
-        }
-
-        return true;
-    }
 }

bubble-server/src/main/java/bubble/service/bill/BillingService.java

@@ -0,0 +1,121 @@
+package bubble.service.bill;
+
+import bubble.cloud.payment.PaymentServiceDriver;
+import bubble.dao.bill.AccountPaymentMethodDAO;
+import bubble.dao.bill.AccountPlanDAO;
+import bubble.dao.bill.BillDAO;
+import bubble.dao.bill.BubblePlanDAO;
+import bubble.dao.cloud.CloudServiceDAO;
+import bubble.model.bill.*;
+import bubble.model.cloud.CloudService;
+import bubble.server.BubbleConfiguration;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.daemon.SimpleDaemon;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.concurrent.TimeUnit.HOURS;
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
+
+@Service @Slf4j
+public class BillingService extends SimpleDaemon {
+
+    private static final long BILLING_CHECK_INTERVAL = HOURS.toMillis(6);
+
+    @Autowired private AccountPlanDAO accountPlanDAO;
+    @Autowired private BubblePlanDAO planDAO;
+    @Autowired private BillDAO billDAO;
+    @Autowired private CloudServiceDAO cloudDAO;
+    @Autowired private AccountPaymentMethodDAO paymentMethodDAO;
+    @Autowired private BubbleConfiguration configuration;
+
+    public void processBilling () { interrupt(); }
+
+    @Override protected long getSleepTime() { return BILLING_CHECK_INTERVAL; }
+
+    @Override protected boolean canInterruptSleep() { return true; }
+
+    @Override protected void process() {
+        final List<AccountPlan> plansToBill = accountPlanDAO.findBillableAccountPlans(now());
+        for (AccountPlan accountPlan : plansToBill) {
+
+            final BubblePlan plan = planDAO.findByUuid(accountPlan.getPlan());
+            if (plan == null) {
+                // todo: this is really bad -- notify admin
+                log.error("billPlan: plan not found ("+accountPlan.getPlan()+") for accountPlan: "+accountPlan.getUuid());
+                continue;
+            }
+
+            final List<Bill> bills;
+            try {
+                bills = billPlan(plan, accountPlan);
+            } catch (Exception e) {
+                log.error("process: error creating bill(s) for accountPlan "+accountPlan.getUuid()+": "+e);
+                continue;
+            }
+
+            try {
+                if (!payBills(plan, accountPlan, bills)) {
+                    log.error("process: payBills returned false for "+bills.size()+" bill(s) for accountPlan "+accountPlan.getUuid());
+                }
+            } catch (Exception e) {
+                log.error("process: error paying "+bills.size()+" bill(s) for accountPlan "+accountPlan.getUuid()+": "+e);
+            }
+        }
+    }
+
+    private List<Bill> billPlan(BubblePlan plan, AccountPlan accountPlan) {
+        final Bill recentBill = billDAO.findMostRecentBillForAccountPlan(accountPlan.getUuid());
+        if (recentBill == null) return die("billPlan: no recent bill found for accountPlan: "+accountPlan.getUuid());
+
+        final BillPeriod period = plan.getPeriod();
+        final List<Bill> bills = new ArrayList<>();
+
+        // create bills for the past, until a bill has a periodEnd beyond the AccountPlan.nextBill date
+        Bill bill = recentBill;
+        while (true) {
+            final long nextBillMillis = period.periodMillis(bill.getPeriodEnd());
+            final Bill nextBill = billDAO.newBill(plan, accountPlan, nextBillMillis);
+            if (nextBillMillis <= now()) {
+                bill = billDAO.create(nextBill);
+                bills.add(bill);
+            } else {
+                accountPlan.setNextBill(nextBillMillis);
+                accountPlan.setNextBillDate();
+                accountPlanDAO.update(accountPlan);
+                break;
+            }
+        }
+
+        if (bills.size() > 1) {
+            log.warn("billPlan: "+bills.size()+" bills created for accountPlan: "+accountPlan.getUuid());
+        }
+        return bills;
+    }
+
+    private boolean payBills(BubblePlan plan, AccountPlan accountPlan, List<Bill> bills) {
+        final AccountPaymentMethod paymentMethod = paymentMethodDAO.findByUuid(accountPlan.getPaymentMethod());
+        if (paymentMethod == null) return die("payBills: paymentMethod "+accountPlan.getPaymentMethod()+" not found for accountPlan: "+accountPlan.getUuid());
+
+        final CloudService paymentService = cloudDAO.findByUuid(paymentMethod.getCloud());
+        if (paymentService == null) return die("payBills: payment cloud "+paymentMethod.getCloud()+" not found for paymentMethod: "+paymentMethod.getUuid()+", accountPlan: "+accountPlan.getUuid());
+
+        final PaymentServiceDriver paymentDriver = paymentService.getPaymentDriver(configuration);
+        for (Bill bill : bills) {
+            if (paymentDriver.getPaymentMethodType().requiresAuth()) {
+                if (!paymentDriver.authorize(plan, accountPlan.getUuid(), paymentMethod)) {
+                    return die("payBills: paymentDriver.authorized returned false for accountPlan="+accountPlan.getUuid()+", paymentMethod="+paymentMethod.getUuid()+", bill="+bill.getUuid());
+                }
+            }
+            if (!paymentDriver.purchase(accountPlan.getUuid(), paymentMethod.getUuid(), bill.getUuid())) {
+                return die("payBills: paymentDriver.purchase returned false for accountPlan="+accountPlan.getUuid()+", paymentMethod="+paymentMethod.getUuid()+", bill="+bill.getUuid());
+            }
+        }
+        return true;
+    }
+
+}

bubble-server/src/main/java/bubble/service/boot/ActivationService.java

@@ -143,6 +143,8 @@ public class ActivationService {
         }
         domainDAO.update(domain.setRoles(domainRoles));
 
+        selfNodeService.initThisNode(node);
+
         return node;
     }
 

bubble-server/src/main/java/bubble/service/boot/SelfNodeService.java

@@ -1,9 +1,14 @@
 package bubble.service.boot;
 
 import bubble.model.cloud.BubbleNetwork;
+import bubble.model.cloud.BubbleNode;
 
 public interface SelfNodeService {
 
+    boolean initThisNode(BubbleNode thisNode);
+
+    BubbleNode getThisNode ();
+
     BubbleNetwork getThisNetwork();
 
     void refreshThisNetwork ();

bubble-server/src/main/java/bubble/service/boot/StandardSelfNodeService.java

@@ -1,22 +1,31 @@
 package bubble.service.boot;
 
+import bubble.cloud.CloudServiceType;
+import bubble.cloud.storage.local.LocalStorageDriver;
 import bubble.dao.cloud.BubbleNetworkDAO;
 import bubble.dao.cloud.BubbleNodeDAO;
 import bubble.dao.cloud.BubbleNodeKeyDAO;
+import bubble.dao.cloud.CloudServiceDAO;
 import bubble.model.cloud.BubbleNetwork;
 import bubble.model.cloud.BubbleNode;
 import bubble.model.cloud.BubbleNodeKey;
 import bubble.model.cloud.BubbleNodeState;
 import bubble.model.cloud.notify.NotificationReceipt;
 import bubble.model.cloud.notify.NotificationType;
+import bubble.server.BubbleConfiguration;
+import bubble.service.bill.BillingService;
+import bubble.service.bill.RefundService;
 import bubble.service.notify.NotificationService;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.cache.AutoRefreshingReference;
+import org.cobbzilla.util.string.StringUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -25,6 +34,7 @@ import static bubble.ApiConstants.NULL_NODE;
 import static bubble.model.cloud.BubbleNode.nodeFromFile;
 import static bubble.model.cloud.BubbleNodeKey.nodeKeyFromFile;
 import static bubble.server.BubbleServer.disableRestoreMode;
+import static bubble.server.BubbleServer.isRestoreMode;
 import static java.util.concurrent.TimeUnit.DAYS;
 import static java.util.concurrent.TimeUnit.MINUTES;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
@@ -47,12 +57,80 @@ public class StandardSelfNodeService implements SelfNodeService {
     @Autowired private BubbleNodeDAO nodeDAO;
     @Autowired private BubbleNodeKeyDAO nodeKeyDAO;
     @Autowired private BubbleNetworkDAO networkDAO;
+    @Autowired private CloudServiceDAO cloudDAO;
     @Autowired private NotificationService notificationService;
+    @Autowired private BubbleConfiguration configuration;
 
     private static final AtomicReference<BubbleNode> thisNode = new AtomicReference<>();
     private static final AtomicReference<BubbleNode> sageNode = new AtomicReference<>();
     private static final AtomicBoolean wasRestored = new AtomicBoolean(false);
 
+    @Override public boolean initThisNode(BubbleNode thisNode) {
+        log.info("initThisNode: initializing with thisNode="+thisNode.id());
+        final BubbleConfiguration c = configuration;
+
+        final BubbleNode dbThis = nodeDAO.findByUuid(thisNode.getUuid());
+        if (dbThis == null) return die("initThisNode: self_node not found in database: "+thisNode.getUuid());
+
+        // check database, ip4/ip6 may not have been set for ourselves. let's set them now
+        if (!dbThis.hasIp4()) {
+            log.info("initThisNode: updating ip4 for self_node in database: "+thisNode.id());
+            dbThis.setIp4(thisNode.getIp4());
+        } else if (thisNode.hasIp4() && !dbThis.getIp4().equals(thisNode.getIp4())) {
+            log.warn("initThisNode: self_node ("+thisNode.getIp4()+") and database row ("+dbThis.getIp4()+") have differing ip4 addresses for node "+thisNode.getUuid());
+            dbThis.setIp4(thisNode.getIp4());
+        }
+
+        if (!dbThis.hasIp6()) {
+            log.info("initThisNode: updating ip6 for self_node in database: "+thisNode.id());
+            dbThis.setIp6(thisNode.getIp6());
+        } else if (thisNode.hasIp6() && !dbThis.getIp6().equals(thisNode.getIp6())) {
+            log.warn("initThisNode: self_node ("+thisNode.getIp6()+") and database row ("+dbThis.getIp6()+") have differing ip6 addresses for node "+thisNode.getUuid());
+            dbThis.setIp6(thisNode.getIp6());
+        }
+        nodeDAO.update(dbThis);
+
+        // ensure a token exists so we can call ourselves
+        final List<BubbleNodeKey> keys = nodeKeyDAO.findByNode(thisNode.getUuid());
+        if (BubbleNodeKey.shouldGenerateNewKey(keys)) {
+            nodeKeyDAO.create(new BubbleNodeKey(thisNode));
+        }
+
+        if (!isRestoreMode()) {
+            final String network = thisNode.getNetwork();
+
+            // ensure storage delegates use a network-specific key
+            final List<String> updatedClouds = new ArrayList<>();
+            cloudDAO.findByType(CloudServiceType.storage).stream()
+                    .filter(cloud -> cloud.getCredentials() != null
+                            && cloud.getCredentials().needsNewNetworkKey(network)
+                            && !cloud.getDriverClass().equals(LocalStorageDriver.class.getName()))
+                    .forEach(cloud -> {
+                        cloudDAO.update(cloud.setCredentials(cloud.getCredentials().initNetworkKey(network)));
+                        log.info("onStart: set network-specific key for storage: " + cloud.getName());
+                        updatedClouds.add(cloud.getName() + "/" + cloud.getUuid());
+                    });
+            if (!updatedClouds.isEmpty()) {
+                log.info("onStart: updated network-specific keys for storage clouds: " + StringUtil.toString(updatedClouds));
+            }
+        }
+
+        // start hello sage service, if we have a sage that is not ourselves
+        if (c.hasSageNode() && !c.isSelfSage()) {
+            log.info("onStart: starting SageHelloService");
+            c.getBean(SageHelloService.class).start();
+        }
+
+        // start RefundService if payments are enabled and this is a SageLauncher
+        if (c.paymentsEnabled() && c.isSageLauncher()) {
+            log.info("onStart: starting RefundService");
+            c.getBean(RefundService.class).start();
+            c.getBean(BillingService.class).start();
+        }
+
+        return true;
+    }
+
     public void setActivated(BubbleNode node) {
         // only called by ActivationService on a brand-new instance
         log.info("setActivated: setting thisNode="+node.id());

bubble-server/src/main/java/bubble/service_dbfilter/DbFilterSelfNodeService.java

@@ -1,6 +1,7 @@
 package bubble.service_dbfilter;
 
 import bubble.model.cloud.BubbleNetwork;
+import bubble.model.cloud.BubbleNode;
 import bubble.service.boot.SelfNodeService;
 import org.springframework.stereotype.Service;
 
@@ -9,6 +10,10 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported;
 @Service
 public class DbFilterSelfNodeService implements SelfNodeService {
 
+    @Override public boolean initThisNode(BubbleNode thisNode) { return notSupported("initThisNode"); }
+
+    @Override public BubbleNode getThisNode() { return notSupported("getThisNode"); }
+
     @Override public BubbleNetwork getThisNetwork() { return notSupported("getThisNetwork"); }
 
     @Override public void refreshThisNetwork() { notSupported("refreshThisNetwork"); }

bubble-server/src/test/java/bubble/mock/MockStripePaymentDriver.java

@@ -0,0 +1,45 @@
+package bubble.mock;
+
+import bubble.cloud.payment.stripe.StripePaymentDriver;
+import bubble.model.bill.AccountPaymentMethod;
+import bubble.model.bill.AccountPlan;
+import bubble.model.bill.Bill;
+import bubble.model.bill.BubblePlan;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
+
+public class MockStripePaymentDriver extends StripePaymentDriver {
+
+    public static final AtomicReference<String> error = new AtomicReference<>(null);
+    public static void setError(String err) { error.set(err); }
+
+    @Override public boolean authorize(BubblePlan plan, String accountPlanUuid, AccountPaymentMethod paymentMethod) {
+        final String err = error.get();
+        if (err != null && (err.equals("authorize") || err.equals("all"))) {
+            throw invalidEx("err.purchase.authNotFound", "mock: error flag="+err);
+        } else {
+            return super.authorize(plan, accountPlanUuid, paymentMethod);
+        }
+    }
+
+    @Override protected String charge(BubblePlan plan, AccountPlan accountPlan, AccountPaymentMethod paymentMethod, Bill bill) {
+        final String err = error.get();
+        if (err != null && (err.equals("charge") || err.equals("all"))) {
+            throw invalidEx("err.purchase.declined", "mock: error flag="+err);
+        } else {
+            return super.charge(plan, accountPlan, paymentMethod, bill);
+        }
+    }
+
+    @Override public boolean refund(String accountPlanUuid) {
+        final String err = error.get();
+        if (err != null && (err.equals("refund") || err.equals("all"))) {
+            throw invalidEx("err.refund.unknownError", "mock: error flag="+err);
+        } else {
+            return super.refund(accountPlanUuid);
+        }
+    }
+
+}

bubble-server/src/test/java/bubble/mock/StripePostHandler.java

@@ -1,31 +0,0 @@
-package bubble.mock;
-
-import lombok.extern.slf4j.Slf4j;
-import org.glassfish.jersey.server.ContainerRequest;
-import org.springframework.stereotype.Service;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.Response;
-
-import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_FORM_URL_ENCODED;
-import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
-import static org.cobbzilla.wizard.resources.ResourceUtil.ok_empty;
-
-@Service @Slf4j
-@Path("/stripe")
-public class StripePostHandler {
-
-    @Consumes(APPLICATION_FORM_URL_ENCODED)
-    @Produces(APPLICATION_JSON)
-    @POST
-    public Response receiveStripeToken(@Context ContainerRequest ctx) {
-//        log.info("bubble_uuid="+bubble_uuid+", stripeToken="+stripeToken);
-        log.info("wtf");
-        return ok_empty();
-    }
-
-}

bubble-server/src/test/java/bubble/test/BubbleApiRunnerListener.java

@@ -5,21 +5,33 @@ import bubble.cloud.CloudServiceType;
 import bubble.cloud.payment.stripe.StripePaymentDriver;
 import bubble.dao.account.AccountDAO;
 import bubble.dao.cloud.CloudServiceDAO;
+import bubble.mock.MockStripePaymentDriver;
 import bubble.model.account.Account;
 import bubble.model.cloud.CloudService;
 import bubble.server.BubbleConfiguration;
+import bubble.service.bill.BillingService;
 import com.github.jknack.handlebars.Handlebars;
 import com.stripe.model.Token;
+import org.cobbzilla.util.string.StringUtil;
 import org.cobbzilla.wizard.client.script.SimpleApiRunnerListener;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static bubble.ApiConstants.getBubbleDefaultDomain;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
+import static org.cobbzilla.util.daemon.ZillaRuntime.incrementSystemTimeOffset;
+import static org.cobbzilla.util.reflect.ReflectionUtil.forName;
+import static org.cobbzilla.util.system.Sleep.sleep;
+import static org.cobbzilla.util.time.TimeUtil.parseDuration;
 
 public class BubbleApiRunnerListener extends SimpleApiRunnerListener {
 
+    public static final String FAST_FORWARD_AND_BILL = "fast_forward_and_bill";
+    public static final String SET_STRIPE_ERROR = "set_stripe_error";
+    public static final String UNSET_STRIPE_ERROR = "unset_stripe_error";
     public static final String STRIPE_TOKENIZE_CARD = "stripe_tokenize_card";
     public static final String CTX_STRIPE_TOKEN = "stripeToken";
 
@@ -32,16 +44,38 @@ public class BubbleApiRunnerListener extends SimpleApiRunnerListener {
         this.configuration = configuration;
     }
 
+    @Override public void beforeScript(String before, Map<String, Object> ctx) throws Exception {
+        if (before == null) return;
+        if (before.startsWith(FAST_FORWARD_AND_BILL)) {
+            final List<String> parts = StringUtil.splitAndTrim(before.substring(FAST_FORWARD_AND_BILL.length()), " ");
+            final long delta = parseDuration(parts.get(0));
+            final long sleepTime = parts.size() > 1 ? parseDuration(parts.get(1)) : SECONDS.toMillis(20);
+            incrementSystemTimeOffset(delta);
+            configuration.getBean(BillingService.class).processBilling();
+            sleep(sleepTime, "waiting for BillingService to complete");
+
+        } else if (before.equals(SET_STRIPE_ERROR)) {
+            MockStripePaymentDriver.setError(before.substring(SET_STRIPE_ERROR.length()).trim());
+
+        } else if (before.equals(UNSET_STRIPE_ERROR)) {
+            MockStripePaymentDriver.setError(null);
+
+        } else {
+            super.beforeScript(before, ctx);
+        }
+    }
+
     @Override public void afterScript(String after, Map<String, Object> ctx) throws Exception {
-        if (after != null && after.equals(STRIPE_TOKENIZE_CARD)) {
+        if (after == null) return;
+        if (after.equals(STRIPE_TOKENIZE_CARD)) {
             // ensure stripe API token is initialized
             final Account admin = configuration.getBean(AccountDAO.class).findFirstAdmin();
             final CloudService stripe = configuration.getBean(CloudServiceDAO.class)
                     .findByAccountAndType(admin.getUuid(), CloudServiceType.payment)
-                    .stream().filter(c -> c.getDriverClass().equals(StripePaymentDriver.class.getName()))
+                    .stream().filter(c -> StripePaymentDriver.class.isAssignableFrom(forName(c.getDriverClass())))
                     .findFirst().orElse(null);
             if (stripe == null) {
-                die("afterScript: no cloud found with driverClass="+StripePaymentDriver.class.getName());
+                die("afterScript: no cloud found with driverClass=" + StripePaymentDriver.class.getName());
                 return;
             }
             stripe.getPaymentDriver(configuration);
@@ -57,9 +91,15 @@ public class BubbleApiRunnerListener extends SimpleApiRunnerListener {
                 final Token token = Token.create(tokenParams);
                 ctx.put(CTX_STRIPE_TOKEN, token.getId());
             } catch (Exception e) {
-                die("afterScript: error creating Stripe token: "+e);
+                die("afterScript: error creating Stripe token: " + e);
             }
 
+        } else if (after.equals(SET_STRIPE_ERROR)) {
+            MockStripePaymentDriver.setError(after.substring(SET_STRIPE_ERROR.length()).trim());
+
+        } else if (after.equals(UNSET_STRIPE_ERROR)) {
+            MockStripePaymentDriver.setError(null);
+
         } else {
             super.afterScript(after, ctx);
         }

bubble-server/src/test/java/bubble/test/PaymentTest.java

@@ -1,6 +1,7 @@
 package bubble.test;
 
 import bubble.server.BubbleConfiguration;
+import bubble.service.bill.BillingService;
 import bubble.service.bill.RefundService;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.server.RestServer;
@@ -22,6 +23,7 @@ public class PaymentTest extends ActivatedBubbleModelTestBase {
     @Override public void onStart(RestServer<BubbleConfiguration> server) {
         final BubbleConfiguration configuration = server.getConfiguration();
         configuration.getBean(RefundService.class).start(); // ensure RefundService is always started
+        configuration.getBean(BillingService.class).start(); // ensure BillingService is always started
         super.onStart(server);
     }
 
@@ -29,9 +31,10 @@ public class PaymentTest extends ActivatedBubbleModelTestBase {
     @Test public void testCodePayment () throws Exception { modelTest("payment/pay_code"); }
     @Test public void testCreditPayment () throws Exception { modelTest("payment/pay_credit"); }
 
-    @Test public void testCreditPaymentMultipleStartStop () throws Exception {
-        modelTest("payment/pay_credit_multi_start_stop");
+    @Test public void testCreditPaymentWithRefundAndRestart() throws Exception {
+        modelTest("payment/pay_credit_refund_and_restart");
     }
 
+    @Test public void testRecurringBilling () throws Exception { modelTest("payment/recurring_billing"); }
 
 }

bubble-server/src/test/resources/models/system/cloudService.json

@@ -159,20 +159,6 @@
     "template": true
   },
 
-  {
-    "_subst": true,
-    "name": "StripePayments",
-    "type": "payment",
-    "driverClass": "bubble.cloud.payment.stripe.StripePaymentDriver",
-    "driverConfig": {
-      "publicApiKey": "{{STRIPE_PUBLIC_API_KEY}}"
-    },
-    "credentials": {
-      "params": [ {"name": "secretApiKey", "value": "{{STRIPE_SECRET_API_KEY}}"} ]
-    },
-    "template": true
-  },
-
   // keep this as the last entry. ActivatedBubbleModelTestBase uses the last entry in this file as the DNS
   // for the initial domain
   {

bubble-server/src/test/resources/models/system/cloudService_live.json

@@ -1,4 +1,18 @@
 [
+  {
+    "_subst": true,
+    "name": "StripePayments",
+    "type": "payment",
+    "driverClass": "bubble.cloud.payment.stripe.StripePaymentDriver",
+    "driverConfig": {
+      "publicApiKey": "{{STRIPE_PUBLIC_API_KEY}}"
+    },
+    "credentials": {
+      "params": [ {"name": "secretApiKey", "value": "{{STRIPE_SECRET_API_KEY}}"} ]
+    },
+    "template": true
+  },
+
   {
     "_subst": true,
     "name": "VultrCompute",

bubble-server/src/test/resources/models/system/cloudService_test.json

@@ -53,6 +53,19 @@
         }
       ]
     }
-  }
+  },
 
+  {
+    "_subst": true,
+    "name": "StripePayments",
+    "type": "payment",
+    "driverClass": "bubble.mock.MockStripePaymentDriver",
+    "driverConfig": {
+      "publicApiKey": "{{STRIPE_PUBLIC_API_KEY}}"
+    },
+    "credentials": {
+      "params": [ {"name": "secretApiKey", "value": "{{STRIPE_SECRET_API_KEY}}"} ]
+    },
+    "template": true
+  }
 ]

bubble-server/src/test/resources/models/tests/payment/pay_credit_multi_start_stop.json → bubble-server/src/test/resources/models/tests/payment/pay_credit_refund_and_restart.json

@@ -354,35 +354,4 @@
       ]
     }
   }
-
-  // todo: fast-forward 32 days, trigger BillGenerator
-
-  // todo: verify a new Bill exists for accountPlan2, and payment has been made successfully
-
-  // todo: set mock such that charging the card fails
-
-  // todo: fast-forward 32 days, trigger BillGenerator
-
-  // todo: verify a new Bill exists for accountPlan2 and Bill remains 'unpaid', and AccountPayment failed
-
-  // todo: verify payment reminder messages have been sent
-
-  // todo: fast-forward 1 day, trigger BillGenerator
-
-  // todo: verify a new Bill exists for accountPlan2 and Bill remains 'unpaid', and AccountPayment failed
-
-  // todo: verify payment reminder messages have been sent
-
-  // todo: fast-forward 3 days, trigger BillGenerator.
-
-  // todo: verify a new Bill exists for accountPlan2 and Bill remains 'unpaid', and AccountPayment failed
-
-  // todo: verify network associated with plan has been stopped
-
-  // todo: try to start network, fails due to non-payment
-
-  // todo: submit payment
-
-  // todo: start network, succeeds
-
 ]

bubble-server/src/test/resources/models/tests/payment/recurring_billing.json

@@ -0,0 +1,332 @@
+[
+  {
+    "comment": "create a user account",
+    "request": {
+      "uri": "users",
+      "method": "put",
+      "entity": {
+        "name": "test_user",
+        "password": "password",
+        "contact": {"type": "email", "info": "test-user@example.com"}
+      }
+    }
+  },
+
+  {
+    "before": "sleep 22s",  // wait for account objects to be created
+    "comment": "login as new user",
+    "request": {
+      "session": "new",
+      "uri": "auth/login",
+      "entity": {
+        "name": "test_user",
+        "password": "password"
+      }
+    },
+    "response": {
+      "store": "testAccount",
+      "sessionName": "userSession",
+      "session": "token"
+    }
+  },
+
+  {
+    "comment": "get payment methods, tokenize a credit card",
+    "request": { "uri": "paymentMethods" },
+    "response": {
+      "store": "paymentMethods"
+    },
+    "after": "stripe_tokenize_card"
+  },
+
+  {
+    "before": "sleep 1s",
+    "comment": "as root, check email inbox for verification message",
+    "request": {
+      "session": "rootSession",
+      "uri": "debug/inbox/email/test-user@example.com?type=request&action=verify&target=account"
+    },
+    "response": {
+      "store": "emailInbox",
+      "check": [
+        {"condition": "'{{json.[0].ctx.message.messageType}}' === 'request'"},
+        {"condition": "'{{json.[0].ctx.message.action}}' === 'verify'"},
+        {"condition": "'{{json.[0].ctx.message.target}}' === 'account'"}
+      ]
+    }
+  },
+
+  {
+    "comment": "approve email verification request",
+    "request": {
+      "session": "userSession",
+      "uri": "auth/approve/{{emailInbox.[0].ctx.confirmationToken}}",
+      "method": "post"
+    }
+  },
+
+  {
+    "comment": "get plans",
+    "request": { "uri": "plans" },
+    "response": {
+      "store": "plans",
+      "check": [{"condition": "json.length >= 1"}]
+    }
+  },
+
+  {
+    "comment": "add plan, using 'credit' payment method with a valid card token, creates a stripe customer",
+    "request": {
+      "uri": "me/plans",
+      "method": "put",
+      "entity": {
+        "name": "test-net-{{rand 5}}",
+        "domain": "{{defaultDomain}}",
+        "locale": "en_US",
+        "timezone": "EST",
+        "plan": "{{plans.[0].name}}",
+        "footprint": "US",
+        "paymentMethodObject": {
+          "paymentMethodType": "credit",
+          "paymentInfo": "{{stripeToken}}"
+        }
+      }
+    },
+    "response": {
+      "store": "accountPlan"
+    }
+  },
+
+  {
+    "before": "sleep 15s",
+    "comment": "verify account payment methods, should be one",
+    "request": { "uri": "me/paymentMethods" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 1"},
+        {"condition": "json[0].getPaymentMethodType().name() === 'credit'"}
+      ]
+    }
+  },
+
+  {
+    "comment": "verify account plans, should be one",
+    "request": { "uri": "me/plans" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 1"},
+        {"condition": "json[0].getName() === accountPlan.getName()"},
+        {"condition": "json[0].enabled()"}
+      ]
+    }
+  },
+
+  {
+    "comment": "verify account plan payment info",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/paymentMethod" },
+    "response": {
+      "store": "savedPaymentMethod",
+      "check": [
+        {"condition": "json.getPaymentMethodType().name() === 'credit'"},
+        {"condition": "json.getMaskedPaymentInfo() === 'XXXX-XXXX-XXXX-4242'"}
+      ]
+    }
+  },
+
+  {
+    "comment": "verify bill exists for new service with correct price and has been paid",
+    "request": { "uri": "me/bills" },
+    "response": {
+      "store": "bills",
+      "check": [
+        {"condition": "json.length === 1"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getQuantity() === 1"},
+        {"condition": "json[0].getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[0].getTotal() === {{plans.[0].price}}"},
+        {"condition": "json[0].getStatus().name() === 'paid'"}
+      ]
+    }
+  },
+
+  {
+    "comment": "verify successful payment exists for new service",
+    "request": { "uri": "me/payments" },
+    "response": {
+      "store": "payments",
+      "check": [
+        {"condition": "json.length === 1"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getAmount() === {{plans.[0].price}}"},
+        {"condition": "json[0].getType().name() === 'payment'"},
+        {"condition": "json[0].getStatus().name() === 'success'"}
+      ]
+    }
+  },
+
+  {
+    "comment": "verify successful payment has paid for the bill above",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/bills" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 1"},
+        {"condition": "json[0].getUuid() === '{{bills.[0].uuid}}'"},
+        {"condition": "json[0].getStatus().name() === 'paid'"},
+        {"condition": "json[0].getRefundedAmount() === 0"}
+      ]
+    }
+  },
+
+  {
+    "before": "fast_forward_and_bill 33d",
+    "comment": "1st fast-forward: +33 days, verify a new bill exists for accountPlan",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/bills" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 2"},
+        {"condition": "json[0].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getQuantity() === 1"},
+        {"condition": "json[0].getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[0].getTotal() === {{plans.[0].price}}"}
+      ]
+    }
+  },
+
+  {
+    "comment": "1st fast-forward: verify a successful payment has been made for the new bill for accountPlan",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/payments" },
+    "response": {
+      "store": "plan2payments",
+      "check": [
+        {"condition": "json.length === 2"},
+        {"condition": "json[0].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getAmount() === {{plans.[0].price}}"},
+        {"condition": "json[0].getStatus().name() === 'success'"},
+        {"condition": "json[0].getBillObject().getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getBillObject().getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getBillObject().getQuantity() === 1"},
+        {"condition": "json[0].getBillObject().getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[0].getBillObject().getTotal() === {{plans.[0].price}}"},
+        {"condition": "json[0].getBillObject().getStatus().name() === 'paid'"}
+      ]
+    }
+  },
+
+  {
+    "before": "fast_forward_and_bill 33d",
+    "comment": "2nd fast-forward: fast-forward another +33 days, verify a new bill exists for accountPlan",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/bills" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 3"},
+        {"condition": "json[0].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getQuantity() === 1"},
+        {"condition": "json[0].getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[0].getTotal() === {{plans.[0].price}}"}
+      ]
+    }
+  },
+
+  {
+    "comment": "2nd fast-forward: verify a successful payment has been made for the new bill for accountPlan",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/payments" },
+    "response": {
+      "store": "plan2payments",
+      "check": [
+        {"condition": "json.length === 3"},
+        {"condition": "json[0].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getAmount() === {{plans.[0].price}}"},
+        {"condition": "json[0].getStatus().name() === 'success'"},
+        {"condition": "json[0].getBillObject().getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getBillObject().getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getBillObject().getQuantity() === 1"},
+        {"condition": "json[0].getBillObject().getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[0].getBillObject().getTotal() === {{plans.[0].price}}"},
+        {"condition": "json[0].getBillObject().getStatus().name() === 'paid'"}
+      ]
+    }
+  },
+
+  {
+    "before": "fast_forward_and_bill 66d",
+    "comment": "3rd fast-forward: fast-forward even more, +66 days, we have missed a billing cycle, so two new bills should be created",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/bills" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 5"},
+        {"condition": "json[0].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getQuantity() === 1"},
+        {"condition": "json[0].getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[0].getTotal() === {{plans.[0].price}}"},
+        {"condition": "json[1].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[1].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[1].getQuantity() === 1"},
+        {"condition": "json[1].getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[1].getTotal() === {{plans.[0].price}}"}
+      ]
+    }
+  },
+
+  {
+    "comment": "3rd fast-forward: verify a successful payment has been made for all new bills for accountPlan",
+    "request": { "uri": "me/plans/{{accountPlan.uuid}}/payments" },
+    "response": {
+      "store": "plan2payments",
+      "check": [
+        {"condition": "json.length === 5"},
+        {"condition": "json[0].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getAmount() === {{plans.[0].price}}"},
+        {"condition": "json[0].getStatus().name() === 'success'"},
+        {"condition": "json[0].getBillObject().getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[0].getBillObject().getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[0].getBillObject().getQuantity() === 1"},
+        {"condition": "json[0].getBillObject().getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[0].getBillObject().getTotal() === {{plans.[0].price}}"},
+        {"condition": "json[0].getBillObject().getStatus().name() === 'paid'"},
+        {"condition": "json[1].getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[1].getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[1].getAmount() === {{plans.[0].price}}"},
+        {"condition": "json[1].getStatus().name() === 'success'"},
+        {"condition": "json[1].getBillObject().getPlan() === '{{plans.[0].uuid}}'"},
+        {"condition": "json[1].getBillObject().getAccountPlan() === '{{accountPlan.uuid}}'"},
+        {"condition": "json[1].getBillObject().getQuantity() === 1"},
+        {"condition": "json[1].getBillObject().getPrice() === {{plans.[0].price}}"},
+        {"condition": "json[1].getBillObject().getTotal() === {{plans.[0].price}}"},
+        {"condition": "json[1].getBillObject().getStatus().name() === 'paid'"}
+      ]
+    },
+    "after": "set_stripe_error charge"  // set mock so charging the card fails
+  }
+
+  // todo: set mock such that charging the card fails
+
+  // todo: fast-forward 32 days, trigger BillingService
+
+  // todo: verify a new Bill exists for accountPlan and Bill remains 'unpaid', and AccountPayment failed
+
+  // todo: verify payment reminder messages have been sent
+
+  // todo: fast-forward 1 day, trigger BillingService
+
+  // todo: verify a new Bill exists for accountPlan and Bill remains 'unpaid', and AccountPayment failed
+
+  // todo: verify payment reminder messages have been sent
+
+  // todo: fast-forward 3 days, trigger BillingService.
+
+  // todo: verify a new Bill exists for accountPlan and Bill remains 'unpaid', and AccountPayment failed
+
+  // todo: verify network associated with plan has been stopped
+
+  // todo: try to start network, fails due to non-payment
+
+  // todo: submit payment
+
+  // todo: start network, succeeds
+]

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 5a1ad1ac22c1d6624089944ea18a3a1326ca5b06
+Subproject commit 862c6282b7ecdf9b1146f4ecb005cf38be1c4e86