refactor handlebars/include file handling. add support for removing godaddy dns records

bubble-server/src/main/java/bubble/BubbleHandlebars.java

@@ -4,7 +4,8 @@ import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
 import org.cobbzilla.util.handlebars.HandlebarsUtil;
 import org.cobbzilla.util.handlebars.HasHandlebars;
-import org.cobbzilla.util.javascript.StandardJsEngine;
+
+import static org.cobbzilla.wizard.client.script.ApiRunner.standardHandlebars;
 
 public class BubbleHandlebars implements HasHandlebars {
 
@@ -12,11 +13,7 @@ public class BubbleHandlebars implements HasHandlebars {
 
     @Getter(lazy=true) private final Handlebars handlebars = initHandlebars();
     private Handlebars initHandlebars() {
-        final Handlebars hbs = new Handlebars(new HandlebarsUtil(ApiConstants.class.getSimpleName()));
-        HandlebarsUtil.registerUtilityHelpers(hbs);
-        HandlebarsUtil.registerDateHelpers(hbs);
-        HandlebarsUtil.registerJavaScriptHelper(hbs, StandardJsEngine::new);
-        return hbs;
+        return standardHandlebars(new Handlebars(new HandlebarsUtil(ApiConstants.class.getSimpleName())));
     }
 
 }

bubble-server/src/main/java/bubble/cloud/CloudServiceDriver.java

@@ -17,6 +17,8 @@ import static org.cobbzilla.util.json.JsonUtil.json;
 
 public interface CloudServiceDriver {
 
+    String[] CLOUD_DRIVER_PACKAGE = new String[]{"bubble.cloud"};
+
     String CTX_API_KEY = "apiKey";
     String CTX_PARAMS = "params";
 

bubble-server/src/main/java/bubble/cloud/dns/DnsServiceDriver.java

@@ -78,6 +78,7 @@ public interface DnsServiceDriver extends CloudServiceDriver {
 
     static DnsRecord recordFromLine(DnsType type, String name, String line) {
         line = line.trim();
+        if (line.startsWith("\"") && line.endsWith("\"")) line = line.substring(1, line.length()-1).trim();
         if (line.startsWith("/")) line = line.substring(1);
         if (line.endsWith(".")) line = line.substring(0, line.length()-1);
         line = line.trim();
@@ -105,7 +106,7 @@ public interface DnsServiceDriver extends CloudServiceDriver {
                         .setFqdn(name)
                         .setValue(IPv4_ALL_ADDRS);
 
-            case A: case AAAA: case CNAME: default:
+            case A: case AAAA: case CNAME: case TXT: default:
                 return (DnsRecord) new DnsRecord()
                         .setType(type)
                         .setFqdn(name)

bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsDriver.java

@@ -3,6 +3,7 @@ package bubble.cloud.dns.godaddy;
 import bubble.cloud.dns.DnsDriverBase;
 import bubble.model.cloud.BubbleDomain;
 import org.apache.http.HttpHeaders;
+import org.cobbzilla.util.collection.ExpirationMap;
 import org.cobbzilla.util.dns.DnsRecord;
 import org.cobbzilla.util.dns.DnsRecordMatch;
 import org.cobbzilla.util.dns.DnsType;
@@ -10,17 +11,22 @@ import org.cobbzilla.util.http.HttpRequestBean;
 import org.cobbzilla.util.http.HttpResponseBean;
 import org.cobbzilla.util.http.HttpUtil;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyList;
+import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.http.HttpHeaders.CONTENT_TYPE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.retry;
+import static org.cobbzilla.util.dns.DnsType.NS;
+import static org.cobbzilla.util.dns.DnsType.SOA;
 import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
 import static org.cobbzilla.util.http.HttpMethods.PATCH;
-import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.http.HttpMethods.PUT;
+import static org.cobbzilla.util.json.JsonUtil.*;
 
 public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
 
@@ -30,7 +36,7 @@ public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
 
     @Override public Collection<DnsRecord> create(BubbleDomain domain) {
         // lookup SOA and NS records for domain, they must already exist
-        final Collection<DnsRecord> soaRecords = readRecords(domain, urlForType(domain, "SOA"), matchSOA(domain));
+        final Collection<DnsRecord> soaRecords = readRecords(domain, urlForType(domain, SOA), matchSOA(domain));
         final List<DnsRecord> records = new ArrayList<>();
         if (soaRecords.isEmpty()) {
             log.warn("create: no SOA found for "+domain.getName());
@@ -40,20 +46,26 @@ public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
             records.add(soaRecords.iterator().next());
         }
 
-        records.addAll(readRecords(domain, urlForType(domain, "NS"), matchNS(domain)));
+        records.addAll(readRecords(domain, urlForType(domain, NS), matchNS(domain)));
         return records;
     }
 
-    public String urlForType(BubbleDomain domain, final String type) {
-        return config.getBaseUri()+domain.getName()+ "/records/" + type;
+    public String urlForDomain(BubbleDomain domain) { return config.getBaseUri() + domain.getName() + "/records/"; }
+
+    public String urlForType(BubbleDomain domain, final DnsType type) {
+        return urlForDomain(domain) + type;
+    }
+
+    public String urlForTypeAndName(BubbleDomain domain, final DnsType type, String name) {
+        return urlForType(domain, type)+"/"+domain.dropDomainSuffix(name);
     }
 
     public DnsRecordMatch matchSOA(BubbleDomain domain) {
-        return (DnsRecordMatch) new DnsRecordMatch().setType(DnsType.SOA).setFqdn(domain.getName());
+        return (DnsRecordMatch) new DnsRecordMatch().setType(SOA).setFqdn(domain.getName());
     }
 
     public DnsRecordMatch matchNS(BubbleDomain domain) {
-        return (DnsRecordMatch) new DnsRecordMatch().setType(DnsType.NS).setFqdn(domain.getName());
+        return (DnsRecordMatch) new DnsRecordMatch().setType(NS).setFqdn(domain.getName());
     }
 
     @Override public DnsRecord update(DnsRecord record) {
@@ -65,30 +77,40 @@ public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
             if (domain == null) return die("update: domain not found for record: "+record.getFqdn());
             lock = lockDomain(domain.getUuid());
 
-            final String name = dropDomainSuffix(domain, record.getFqdn());
-            final String url = urlForType(domain, record.getType().name()) + "/" + name;
+//            final String name = dropDomainSuffix(domain, record.getFqdn());
+            final String name = domain.ensureDomainSuffix(record.getFqdn());
+            final String url = urlForTypeAndName(domain, record.getType(), name);
             final Collection<DnsRecord> found = readRecords(domain, url, null);
-            if (record.getType() == DnsType.SOA || record.getType() == DnsType.NS) {
+            if (record.getType() == SOA || record.getType() == NS) {
                 // can't do this!
                 log.warn("update: declining to call API to add SOA or NS record: " + record);
                 return record;
             }
+            final String method;
+            final String updateUrl;
             if (found.isEmpty()) {
-                final HttpRequestBean update = auth(config.getBaseUri() + domain.getName() + "/records")
-                        .setMethod(PATCH)
-                        .setHeader(CONTENT_TYPE, APPLICATION_JSON)
-                        .setEntity(json(new GoDaddyDnsRecord[] {
-                                new GoDaddyDnsRecord()
-                                        .setName(name)
-                                        .setType(record.getType())
-                                        .setTtl(record.getTtl())
-                                        .setData(record.getValue())
-                        }));
-                retry(() -> {
-                    final HttpResponseBean response = HttpUtil.getResponse(update);
-                    return response.isOk() ? response : die("update: " + response);
-                }, MAX_GODADDY_RETRIES);
+                method = PATCH;
+                updateUrl = urlForDomain(domain);
+            } else if (found.size() == 1) {
+                method = PUT;
+                updateUrl = url;
+            } else {
+                return die("update("+json(record, COMPACT_MAPPER)+"): "+found.size()+" matching records found, cannot update");
             }
+            final HttpRequestBean update = auth(updateUrl)
+                    .setMethod(method)
+                    .setHeader(CONTENT_TYPE, APPLICATION_JSON)
+                    .setEntity(json(new GoDaddyDnsRecord[] {
+                            new GoDaddyDnsRecord()
+                                    .setName(domain.dropDomainSuffix(name))
+                                    .setType(record.getType())
+                                    .setTtl(record.getTtl())
+                                    .setData(record.getValue())
+                    }));
+            retry(() -> {
+                final HttpResponseBean response = HttpUtil.getResponse(update);
+                return response.isOk() ? response : die("update: " + response);
+            }, MAX_GODADDY_RETRIES);
             return record;
 
         } finally {
@@ -97,9 +119,36 @@ public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
     }
 
     @Override public DnsRecord remove(DnsRecord record) {
-        // lookup record, does it exist? if so remove it.
-        return record;  // removal of names is not supported.
-        // if we kept track of ALL names in our db, we could do a "replace all" after removing from here....
+        String lock = null;
+        final AtomicReference<BubbleDomain> domain = new AtomicReference<>();
+        try {
+            domain.set(getDomain(record.getFqdn()));
+
+            if (domain.get() == null) return die("remove: domain not found for record: "+record.getFqdn());
+            lock = lockDomain(domain.get().getUuid());
+
+            final DnsRecordMatch nonMatcher = record.getNonMatcher();
+            final String url = urlForDomain(domain.get());
+            final GoDaddyDnsRecord[] gdRecords = listGoDaddyDnsRecords(url);
+            final Collection<GoDaddyDnsRecord> retained = Arrays.stream(gdRecords)
+                    .filter(r ->  nonMatcher.matches(r.toDnsRecord(domain.get())))
+                    .collect(Collectors.toList());
+            final HttpRequestBean remove = auth(url)
+                    .setMethod(PUT)
+                    .setHeader(CONTENT_TYPE, APPLICATION_JSON)
+                    .setEntity(json(retained));
+            retry(() -> {
+                final HttpResponseBean response = HttpUtil.getResponse(remove);
+                return response.isOk() ? response : die("remove: " + response);
+            }, MAX_GODADDY_RETRIES);
+            return record;
+
+        } catch (IOException e) {
+            return die("remove: "+e);
+
+        } finally {
+            if (lock != null && domain.get() != null) unlockDomain(domain.get().getUuid(), lock);
+        }
     }
 
     @Override public Collection<DnsRecord> list(DnsRecordMatch matcher) {
@@ -115,30 +164,16 @@ public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
             }
             if (matcher.hasFqdn()) {
                 String fqdn = matcher.getFqdn();
-                fqdn = dropDomainSuffix(domain, fqdn);
+                fqdn = domain.dropDomainSuffix(fqdn);
                 url += "/" + fqdn;
             }
         }
-
         return readRecords(domain, url, matcher);
     }
 
-    public String dropDomainSuffix(BubbleDomain domain, String fqdn) {
-        if (fqdn.endsWith("." + domain.getName())) {
-            fqdn = fqdn.substring(0, fqdn.length() - domain.getName().length() - 1);
-        }
-        return fqdn;
-    }
-
     public Collection<DnsRecord> readRecords(BubbleDomain domain, String url, DnsRecordMatch matcher) {
         return retry(() -> {
-            final HttpRequestBean request = auth(url);
-            final HttpResponseBean response = HttpUtil.getResponse(request);
-            if (!response.isOk()) {
-                return die("readRecords: "+response);
-            }
-
-            final GoDaddyDnsRecord[] records = json(response.getEntityString(), GoDaddyDnsRecord[].class);
+            final GoDaddyDnsRecord[] records = listGoDaddyDnsRecords(url);
             final List<DnsRecord> out = new ArrayList<>();
             for (GoDaddyDnsRecord r : records) {
                 final DnsRecord outRecord = r.toDnsRecord(domain);
@@ -150,6 +185,23 @@ public class GoDaddyDnsDriver extends DnsDriverBase<GoDaddyDnsConfig> {
         }, MAX_GODADDY_RETRIES);
     }
 
+    private final Map<String, GoDaddyDnsRecord[]> listCache = new ExpirationMap<>(SECONDS.toMillis(10));
+
+    public GoDaddyDnsRecord[] listGoDaddyDnsRecords(String url) throws IOException {
+        final HttpRequestBean request = auth(url);
+        return listCache.computeIfAbsent(url, k -> {
+            final HttpResponseBean response;
+            try {
+                response = HttpUtil.getResponse(request);
+            } catch (Exception e) {
+                log.error("listGoDaddyDnsRecords("+url+"): "+e);
+                return GoDaddyDnsRecord.EMPTY_ARRAY;
+            }
+            if (!response.isOk()) throw new IllegalStateException("readRecords: "+response);
+            return json(response.getEntityString(), GoDaddyDnsRecord[].class);
+        });
+    }
+
     public HttpRequestBean auth(String url) { return new HttpRequestBean(url).setHeader(HttpHeaders.AUTHORIZATION, authValue()); }
 
     public String authValue() {

bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsRecord.java

@@ -13,6 +13,8 @@ import static org.cobbzilla.util.dns.DnsRecord.OPT_NS_NAME;
 @NoArgsConstructor @Accessors(chain=true)
 public class GoDaddyDnsRecord {
 
+    public static final GoDaddyDnsRecord[] EMPTY_ARRAY = new GoDaddyDnsRecord[0];
+
     @Getter @Setter private String data;
     @Getter @Setter private String name;
     @Getter @Setter private Integer ttl;
@@ -23,7 +25,7 @@ public class GoDaddyDnsRecord {
         return (DnsRecord) new DnsRecord()
                 .setOption(OPT_NS_NAME, type == DnsType.NS ? data : null)
                 .setType(type)
-                .setFqdn((name.equals("@") ? "" : name+".")+domain.getName())
+                .setFqdn(domain.ensureDomainSuffix(name.equals("@") ? "" : name))
                 .setValue(data);
     }
 }

bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDomain.java

@@ -0,0 +1,21 @@
+package bubble.cloud.dns.godaddy;
+
+import bubble.model.cloud.BubbleDomain;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.experimental.Accessors;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class GoDaddyDomain {
+
+    @JsonIgnore @Getter @Setter private String domain;
+    @Getter @Setter private GoDaddyDnsRecord[] records;
+
+    public GoDaddyDomain(BubbleDomain domain, GoDaddyDnsRecord[] records) {
+        this.domain = domain.getName();
+        this.records = records;
+    }
+
+}

bubble-server/src/main/java/bubble/model/cloud/BubbleDomain.java

@@ -63,6 +63,13 @@ public class BubbleDomain extends IdentifiableBase implements AccountTemplate {
     @ECIndex @Column(nullable=false, updatable=false, length=DOMAIN_NAME_MAXLEN)
     @Getter @Setter private String name;
 
+    public String ensureDomainSuffix(String fqdn) { return fqdn.endsWith("." + getName()) ? fqdn : fqdn + "." + getName(); }
+
+    public String dropDomainSuffix(String fqdn) {
+        return !fqdn.endsWith("." + getName()) ? fqdn
+                : fqdn.substring(0, fqdn.length() - getName().length() - 1);
+    }
+
     @ECSearchable(filter=true)
     @Size(max=10000, message="err.description.length")
     @Type(type=ENCRYPTED_STRING) @Column(columnDefinition="varchar("+(10000+ENC_PAD)+")")

bubble-server/src/main/java/bubble/resources/SearchResource.java

@@ -57,7 +57,10 @@ public class SearchResource {
         return search(req, ctx, type, meta, filter, page, size, sort, null);
     }
 
-    private Map<String, Object> _searchCache = new ExpirationMap<>(MINUTES.toMillis(2), MINUTES.toMillis(5));
+    private ExpirationMap<String, Object> _searchCache = new ExpirationMap<String, Object>()
+            .setExpiration(MINUTES.toMillis(2))
+            .setMaxExpiration(MINUTES.toMillis(2))
+            .setCleanInterval(MINUTES.toMillis(5));
 
     @POST @Path("/{type}")
     public Response search(@Context Request req,

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -3,6 +3,7 @@ package bubble.server;
 import bubble.ApiConstants;
 import bubble.BubbleHandlebars;
 import bubble.client.BubbleApiClient;
+import bubble.cloud.CloudServiceDriver;
 import bubble.model.cloud.BubbleNetwork;
 import bubble.model.cloud.BubbleNode;
 import bubble.server.listener.BubbleFirstTimeListener;
@@ -30,6 +31,7 @@ import org.cobbzilla.wizard.server.config.HasDatabaseConfiguration;
 import org.cobbzilla.wizard.server.config.LegalInfo;
 import org.cobbzilla.wizard.server.config.PgRestServerConfiguration;
 import org.cobbzilla.wizard.server.config.RecaptchaConfig;
+import org.cobbzilla.wizard.util.ClasspathScanner;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.io.Resource;
@@ -37,9 +39,7 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
 
 import java.beans.Transient;
 import java.io.File;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.*;
@@ -62,6 +62,7 @@ public class BubbleConfiguration extends PgRestServerConfiguration
     public static final String TAG_SAGE_LAUNCHER = "sageLauncher";
     public static final String TAG_SAGE_UUID = "sageUuid";
     public static final String TAG_PAYMENTS_ENABLED = "paymentsEnabled";
+    public static final String TAG_CLOUD_DRIVERS = "cloudDrivers";
 
     public static final String DEFAULT_LOCALE = "en_US";
 
@@ -218,10 +219,16 @@ public class BubbleConfiguration extends PgRestServerConfiguration
         return harness;
     }
 
+    @Getter(lazy=true) private final List<String> cloudDriverClasses
+            = ClasspathScanner.scan(CloudServiceDriver.class, CloudServiceDriver.CLOUD_DRIVER_PACKAGE).stream()
+                .map(c -> c.getClass().getName())
+                .collect(Collectors.toList());
+
     @Getter(lazy=true) private final Map<String, Object> publicSystemConfigs = MapBuilder.build(new Object[][] {
             { TAG_ALLOW_REGISTRATION, getThisNetwork().getBooleanTag(TAG_ALLOW_REGISTRATION, false) },
             { TAG_SAGE_LAUNCHER, isSageLauncher() },
-            { TAG_PAYMENTS_ENABLED, paymentsEnabled() }
+            { TAG_PAYMENTS_ENABLED, paymentsEnabled() },
+            { TAG_CLOUD_DRIVERS, getCloudDriverClasses() }
     });
 
     @Getter @Setter private String[] disallowedCountries;

bubble-server/src/main/resources/message_templates/en_US/server/pre_auth/ResourceMessages.properties

@@ -69,4 +69,47 @@ err.driverConfig.initFailure=Cloud driver failed to initialize properlyu
 
 # Entity config errors
 err.ec.param.invalid=Parameter is invalid
-err.ec.fieldType.none_set=Field type was not set
+err.ec.fieldType.none_set=Field type was not set
+
+# Driver names
+driver_bubble.cloud.authenticator.TOTPAuthenticatorDriver=TOTP Service
+description_bubble.cloud.authenticator.TOTPAuthenticatorDriver=A TOTP authentication service that is compatible with common authenticators, like Google Authenticator. This service runs locally and does not use any cloud APIs.
+
+driver_bubble.cloud.compute.vultr.VultrDriver=Vultr Compute Cloud
+description_bubble.cloud.compute.vultr.VultrDriver=Use <a href="https://www.vultr.com/">Vultr</a> to launch new Bubbles
+
+driver_bubble.cloud.compute.digitalocean.DigitalOceanDriver=DigitalOcean Compute Cloud
+description_bubble.cloud.compute.digitalocean.DigitalOceanDriver=Use <a href="https://www.digitalocean.com/">DigitalOcean</a> to launch new Bubbles
+
+driver_bubble.cloud.dns.godaddy.GoDaddyDnsDriver=GoDaddy DNS
+description_bubble.cloud.dns.godaddy.GoDaddyDnsDriver=Use <a href="https://www.godaddy.com/">GoDaddy</a> to manage DNS records for Bubbles
+
+driver_bubble.cloud.email.SmtpEmailDriver=SMTP Email
+description_bubble.cloud.email.SmtpEmailDriver=Connect to any standard SMTP service to deliver email.
+
+driver_bubble.cloud.geoCode.google.GoogleGeoCodeDriver=Google Geocoding API
+description_bubble.cloud.geoCode.google.GoogleGeoCodeDriver=Use the <a href="https://developers.google.com/maps/documentation/javascript/geocoding">Google Geocoding API</a> to convert place names to latitude/longitude.
+
+driver_bubble.cloud.geoLocation.maxmind.MaxMindDriver=MaxMind GeoIP Database
+description_bubble.cloud.geoLocation.maxmind.MaxMindDriver=Use the <a href="https://maxmind.com/">MaxMind</a> GeoIP database to resolve IP addresses to city/region/country. This services runs locally using a database file and does not use any cloud APIs, except to download the database file.
+
+driver_bubble.cloud.geoTime.google.GoogleGeoTimeDriver=Google Time Zone API
+description_bubble.cloud.geoTime.google.GoogleGeoTimeDriver=Use the <a href="https://developers.google.com/maps/documentation/timezone">Google Time Zone API</a> to resolve IP addresses to time zones.
+
+driver_bubble.cloud.payment.free.FreePaymentDriver=Free
+description_bubble.cloud.payment.free.FreePaymentDriver=Allows users to add Account Plans without actually paying for anything.
+
+driver_bubble.cloud.payment.code.CodePaymentDriver=Invite Codes
+description_bubble.cloud.payment.code.CodePaymentDriver=Supports invitation codes that can be used once to add an Account Plan.
+
+driver_bubble.cloud.payment.stripe.StripePaymentDriver=Stripe Payments
+description_bubble.cloud.payment.stripe.StripePaymentDriver=Allows payment for Account Plans using credit and debit cards via the <a href="https://stripe.com/">Stripe</a> payments service. 
+
+driver_bubble.cloud.sms.twilio.TwilioSmsDriver=Twilio SMS
+description_bubble.cloud.sms.twilio.TwilioSmsDriver=Deliver SMS messages via <a href="https://twilio.com/">Twilio</a>.
+
+driver_bubble.cloud.storage.s3.S3StorageDriver=Amazon S3
+description_bubble.cloud.storage.s3.S3StorageDriver=Supports storage for <a href="https://aws.amazon.com/s3/">Amazon S3</a>. Data is encrypted locally, S3 never sees unencrypted data.
+
+driver_bubble.cloud.storage.local.LocalStorageDriver=Local Storage
+description_bubble.cloud.storage.local.LocalStorageDriver=Supports local filesystem storage.

bubble-server/src/test/java/bubble/test/ActivatedBubbleModelTestBase.java

@@ -1,6 +1,9 @@
 package bubble.test;
 
+import bubble.cloud.CloudServiceDriver;
 import bubble.cloud.CloudServiceType;
+import bubble.cloud.dns.godaddy.GoDaddyDnsDriver;
+import bubble.cloud.storage.local.LocalStorageDriver;
 import bubble.model.account.Account;
 import bubble.model.account.ActivationRequest;
 import bubble.model.cloud.BubbleDomain;
@@ -23,7 +26,6 @@ import java.util.Map;
 import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.*;
-import static bubble.cloud.storage.local.LocalStorageDriver.LOCAL_STORAGE;
 import static bubble.model.account.Account.ROOT_USERNAME;
 import static bubble.service.boot.StandardSelfNodeService.THIS_NODE_FILE;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
@@ -41,6 +43,7 @@ public abstract class ActivatedBubbleModelTestBase extends BubbleModelTestBase {
 
     public static final String ROOT_PASSWORD = "password";
     public static final String ROOT_SESSION = "rootSession";
+    public static final String ROOT_USER_VAR = "rootUser";
 
     protected Account admin;
 
@@ -88,18 +91,14 @@ public abstract class ActivatedBubbleModelTestBase extends BubbleModelTestBase {
 
             final CloudService[] clouds = scrubSpecial(json(stream2string("models/system/cloudService.json"), JsonNode.class, FULL_MAPPER_ALLOW_COMMENTS), CloudService.class);
 
-            // expect public dns to be the LAST DNS cloud service listed in cloudService.json
-            final List<CloudService> dnsServices = Arrays.stream(clouds)
-                    .filter(c -> c.getType() == CloudServiceType.dns)
-                    .collect(Collectors.toList());
-            if (dnsServices.isEmpty()) die("onStart: no public DNS service found");
-            final CloudService dns = applyReflectively(handlebars, dnsServices.get(dnsServices.size()-1), ctx);
+            // find public DNS service
+            final CloudService dns = getPublicDns(ctx, clouds);
 
             // find storage service
             final CloudService storage = getNetworkStorage(ctx, clouds);
 
             // sanity check
-            if (!dns.getName().equals(domain.getPublicDns())) die("onStart: DNS service mismatch");
+            if (!dns.getName().equals(domain.getPublicDns())) die("onStart: DNS service mismatch: domain references "+domain.getPublicDns()+" but DNS service selected has name "+dns.getName());
 
             @Cleanup final ApiClientBase client = configuration.newApiClient();
 
@@ -122,6 +121,7 @@ public abstract class ActivatedBubbleModelTestBase extends BubbleModelTestBase {
             }
             getApi().setConnectionInfo(client.getConnectionInfo());
             getApi().pushToken(admin.getApiToken());
+            getApiRunner().getContext().put(ROOT_USER_VAR, admin);
             getApiRunner().addNamedSession(ROOT_SESSION, admin.getApiToken());
 
         } catch (Exception e) {
@@ -130,16 +130,26 @@ public abstract class ActivatedBubbleModelTestBase extends BubbleModelTestBase {
         if (!hasExistingDb) super.onStart(server);
     }
 
-    protected CloudService getNetworkStorage(Map<String, Object> ctx, CloudService[] clouds) {
+    private CloudService findByTypeAndDriver(Map<String, Object> ctx,
+                                             CloudService[] clouds,
+                                             CloudServiceType type,
+                                             Class<? extends CloudServiceDriver> driverClass) {
         final Handlebars handlebars = getConfiguration().getHandlebars();
-        final List<CloudService> storageServices = Arrays.stream(clouds)
-                .filter(c -> c.getType() == CloudServiceType.storage && c.getName().equals(getNetworkStorageName()))
+        final List<CloudService> dnsServices = Arrays.stream(clouds)
+                .filter(c -> c.getType() == type && c.getDriverClass().equals(driverClass.getName()))
                 .collect(Collectors.toList());
-        if (storageServices.size() != 1) die("onStart: expected exactly one network storage service");
-        return applyReflectively(handlebars, storageServices.get(0), ctx);
+        if (dnsServices.size() != 1) die("onStart: expected exactly one public dns service");
+        return applyReflectively(handlebars, dnsServices.get(0), ctx);
+    }
+    private CloudService getPublicDns(Map<String, Object> ctx, CloudService[] clouds) {
+        return findByTypeAndDriver(ctx, clouds, CloudServiceType.dns, getPublicDnsDriver());
     }
+    protected Class<? extends CloudServiceDriver> getPublicDnsDriver() { return GoDaddyDnsDriver.class; }
 
-    protected String getNetworkStorageName() { return LOCAL_STORAGE; }
+    protected CloudService getNetworkStorage(Map<String, Object> ctx, CloudService[] clouds) {
+        return findByTypeAndDriver(ctx, clouds, CloudServiceType.storage, getNetworkStorageDriver());
+    }
+    protected Class<? extends CloudServiceDriver> getNetworkStorageDriver() { return LocalStorageDriver.class; }
 
     @Override protected Class<? extends ModelSetupListener> getModelSetupListenerClass() { return BubbleModelSetupListener.class; }
 

bubble-server/src/test/java/bubble/test/AuthTest.java

@@ -8,7 +8,6 @@ public class AuthTest extends ActivatedBubbleModelTestBase {
 
     private static final String MANIFEST_ALL = "manifest-all";
 
-    @Override protected String getModelPrefix() { return "models/"; }
     @Override protected String getManifest() { return MANIFEST_ALL; }
 
     @Test public void testBasicAuth () throws Exception { modelTest("auth/basic_auth"); }

bubble-server/src/test/java/bubble/test/BubbleModelTestBase.java

@@ -5,27 +5,24 @@ import bubble.cloud.sms.mock.MockSmsDriver;
 import bubble.server.BubbleConfiguration;
 import bubble.server.BubbleServer;
 import bubble.server.listener.NodeInitializerListener;
-import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.io.FileUtil;
 import org.cobbzilla.wizard.client.ApiClientBase;
 import org.cobbzilla.wizard.client.script.ApiRunner;
 import org.cobbzilla.wizard.client.script.ApiRunnerListener;
-import org.cobbzilla.wizard.client.script.ApiScriptIncludeHandler;
 import org.cobbzilla.wizard.server.RestServer;
 import org.cobbzilla.wizard.server.RestServerLifecycleListener;
 import org.cobbzilla.wizard.server.config.factory.StreamConfigurationSource;
 import org.cobbzilla.wizardtest.resources.ApiModelTestBase;
 
 import java.io.File;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 
-import static bubble.ApiConstants.ENTITY_CONFIGS_ENDPOINT;
 import static bubble.test.BubbleTestBase.ENV_EXPORT_FILE;
 import static java.util.Arrays.asList;
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.system.CommandShell.loadShellExportsOrDie;
 
 @Slf4j
@@ -52,27 +49,7 @@ public abstract class BubbleModelTestBase extends ApiModelTestBase<BubbleConfigu
 
     @Override protected Collection<RestServerLifecycleListener> getLifecycleListeners() { return TEST_LIFECYCLE_LISTENERS; }
 
-    @Getter(lazy=true) private final ApiRunner apiRunner = new ApiRunner(getApi(), (ApiRunnerListener) getListener()) {
-        @Override public ApiScriptIncludeHandler getIncludeHandler() {
-            return path -> {
-                for (String prefix : getIncludePaths()) {
-                    if (!prefix.endsWith("/") && !path.startsWith("/")) prefix = prefix + "/";
-                    final String data = FileUtil.toStringOrDie(prefix + path + ".json");
-                    if (!empty(data)) return data;
-                }
-                return die("include("+path+"): not found among: "+getIncludePaths());
-            };
-        }
-
-        @Override protected Handlebars initHandlebars() {
-            ctx.putAll((Map) getConfiguration().getEnvironment());
-            ctx.put("configuration", getConfiguration());
-            final Handlebars hb = super.initHandlebars();
-            return HandlebarsTestHelpers.registerHelpers(hb);
-        }
-    };
-
-    @Override protected String getEntityConfigsEndpoint() { return ENTITY_CONFIGS_ENDPOINT; }
+    @Getter(lazy=true) private final ApiRunner apiRunner = new ApiRunner(getApi(), (ApiRunnerListener) getListener());
 
     @Getter private StreamConfigurationSource configurationSource
             = new StreamConfigurationSource("test-bubble-config.yml");
@@ -91,7 +68,7 @@ public abstract class BubbleModelTestBase extends ApiModelTestBase<BubbleConfigu
 
     protected boolean useMocks() { return true; }
 
-    @Override protected List<String> getIncludePaths() {
+    @Override public List<String> getIncludePaths() {
         final List<String> include = new ArrayList<>();
         include.add("models/include");
         include.add("src/test/resources/models/include");

bubble-server/src/test/java/bubble/test/DbInit.java

@@ -12,7 +12,6 @@ import static org.junit.Assert.assertEquals;
 @Slf4j
 public class DbInit extends BubbleModelTestBase {
 
-    @Override protected String getModelPrefix() { return "models/"; }
     @Override protected String getManifest() { return "manifest-empty"; }
 
     @Override protected boolean createSqlIndexes() { return true; }

bubble-server/src/test/java/bubble/test/DriverTest.java

@@ -4,7 +4,6 @@ import org.junit.Test;
 
 public class DriverTest extends ActivatedBubbleModelTestBase {
 
-    @Override protected String getModelPrefix() { return "models/"; }
     @Override protected String getManifest() { return "manifest-proxy"; }
 
     @Test public void testListDrivers () throws Exception { modelTest("list_drivers"); }

bubble-server/src/test/java/bubble/test/GoDaddyDnsTest.java

@@ -0,0 +1,9 @@
+package bubble.test;
+
+import org.junit.Test;
+
+public class GoDaddyDnsTest extends NetworkTestBase {
+
+    @Test public void testGoDaddyDns () throws Exception { modelTest("network/dns_crud"); }
+
+}

bubble-server/src/test/java/bubble/test/NetworkTest.java

@@ -1,12 +1,14 @@
 package bubble.test;
 
+import bubble.cloud.CloudServiceDriver;
+import bubble.cloud.storage.s3.S3StorageDriver;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
 
 @Slf4j
 public class NetworkTest extends NetworkTestBase {
 
-    @Override protected String getNetworkStorageName() { return "S3_US_Standard"; }
+    @Override protected Class<? extends CloudServiceDriver> getNetworkStorageDriver() { return S3StorageDriver.class; }
 
     @Test public void testRegions       () throws Exception { modelTest("network/network_regions"); }
     @Test public void testSimpleNetwork () throws Exception { modelTest("network/simple_network"); }

bubble-server/src/test/java/bubble/test/NetworkTestBase.java

@@ -4,7 +4,6 @@ public class NetworkTestBase extends ActivatedBubbleModelTestBase {
 
     public static final String MANIFEST_NETWORK = "manifest-network";
 
-    @Override protected String getModelPrefix() { return "models/"; }
     @Override protected String getManifest() { return MANIFEST_NETWORK; }
 
 }

bubble-server/src/test/java/bubble/test/PaymentTest.java

@@ -10,7 +10,6 @@ import org.junit.Test;
 @Slf4j
 public class PaymentTest extends ActivatedBubbleModelTestBase {
 
-    @Override protected String getModelPrefix() { return "models/"; }
     @Override protected String getManifest() { return "manifest-payment"; }
 
     @Override public void beforeStart(RestServer<BubbleConfiguration> server) {

bubble-server/src/test/java/bubble/test/ProxyTest.java

@@ -21,7 +21,6 @@ public class ProxyTest extends ActivatedBubbleModelTestBase {
         super.beforeStart(server);
     }
 
-    @Override protected String getModelPrefix() { return "models/"; }
     @Override protected String getManifest() { return MANIFEST_PROXY; }
 
     @Test public void testSimple () throws Exception { modelTest("proxy"); }

bubble-server/src/test/java/bubble/test/dev/BlankDevServerTest.java

@@ -0,0 +1,30 @@
+package bubble.test.dev;
+
+import bubble.resources.EntityConfigsResource;
+import bubble.server.BubbleConfiguration;
+import bubble.test.BubbleModelTestBase;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.wizard.server.RestServer;
+import org.junit.Test;
+
+import static java.util.concurrent.TimeUnit.DAYS;
+import static org.cobbzilla.util.system.Sleep.sleep;
+
+@Slf4j
+public class BlankDevServerTest extends BubbleModelTestBase {
+
+    @Override protected String getManifest() { return "manifest-empty"; }
+
+    @Override protected boolean useMocks() { return false; }
+
+    @Override public void onStart(RestServer<BubbleConfiguration> server) {
+        getConfiguration().getBean(EntityConfigsResource.class).getAllowPublic().set(true);
+        super.onStart(server);
+    }
+
+    @Test public void runDevServer () throws Exception {
+        log.info("runDevServer: Bubble API server started and model initialized. You may now begin testing.");
+        sleep(DAYS.toMillis(30), "running dev server");
+    }
+
+}

bubble-server/src/test/java/bubble/test/dev/DevServerTest.java

@@ -13,7 +13,6 @@ import static org.cobbzilla.util.system.Sleep.sleep;
 @Slf4j
 public class DevServerTest extends ActivatedBubbleModelTestBase {
 
-    @Override protected String getModelPrefix() { return "models/"; }
     @Override protected String getManifest() { return "manifest-dev"; }
 
     @Override protected boolean useMocks() { return false; }

bubble-server/src/test/resources/models/tests/network/dns_crud.json

@@ -0,0 +1,149 @@
+[
+  {
+    "comment": "list networks, should be one",
+    "request": { "uri": "me/networks" },
+    "response": {
+      "store": "networks",
+      "check": [ {"condition": "json.length === 1"} ]
+    }
+  },
+
+  {
+    "comment": "list matching records, should be none",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/find?name={{urlEncode 'test_\\w+_'}}{{rootUser.uuid}}" },
+    "response": { "check": [ {"condition": "json.length === 0"} ] }
+  },
+  {
+    "comment": "dig TXT record, should be none",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/dig?type=txt&name=test_TXT_{{rootUser.uuid}}" },
+    "response": { "check": [ {"condition": "json.length === 0"} ] }
+  },
+  {
+    "comment": "dig A record, should be none",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/dig?type=a&name=test_A_{{rootUser.uuid}}" },
+    "response": { "check": [ {"condition": "json.length === 0"} ] }
+  },
+
+  {
+    "comment": "create a TXT record",
+    "request": {
+      "uri": "me/networks/{{networks.[0].uuid}}/dns/update",
+      "entity": {
+        "type": "TXT",
+        "fqdn": "test_TXT_{{rootUser.uuid}}.{{networks.[0].networkDomain}}",
+        "value": "first-{{rand 15}}"
+      }
+    },
+    "response": {
+      "delay": "15s",
+      "store": "txtRecord"
+    }
+  },
+  {
+    "comment": "create an A record",
+    "request": {
+      "uri": "me/networks/{{networks.[0].uuid}}/dns/update",
+      "entity": {
+        "type": "A",
+        "fqdn": "test_A_{{rootUser.uuid}}.{{networks.[0].networkDomain}}",
+        "value": "127.0.0.1"
+      }
+    },
+    "response": {
+      "store": "aRecord"
+    }
+  },
+
+  {
+    "before": "await_url me/networks/{{networks.[0].uuid}}/dns/dig?type=a&name=test_A_{{rootUser.uuid}} 5m 10s await_json.length > 0",
+    "comment": "dig A record, should be found",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/dig?type=a&name=test_A_{{rootUser.uuid}}" },
+    "response": { "check": [ {"condition": "json.length === 1"} ] }
+  },
+
+  {
+    "before": "await_url me/networks/{{networks.[0].uuid}}/dns/dig?type=txt&name=test_TXT_{{rootUser.uuid}} 5m 10s await_json.length > 0",
+    "comment": "dig TXT record, should be found",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/dig?type=txt&name=test_TXT_{{rootUser.uuid}}" },
+    "response": { "check": [ {"condition": "json.length === 1"} ] }
+  },
+
+  {
+    "before": "await_url me/networks/{{networks.[0].uuid}}/dns/find?name={{urlEncode 'test_\\w+_'}}{{rootUser.uuid}} 5m 10s await_json.length > 0",
+    "comment": "list matching records, should be two",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/find?name={{urlEncode 'test_\\w+_'}}{{rootUser.uuid}}" },
+    "response": { "check": [ {"condition": "json.length === 2"} ] }
+  },
+
+  {
+    "comment": "update TXT record",
+    "request": {
+      "uri": "me/networks/{{networks.[0].uuid}}/dns/update",
+      "entity": {
+        "type": "TXT",
+        "fqdn": "test_TXT_{{rootUser.uuid}}.{{networks.[0].networkDomain}}",
+        "value": "different-{{rand 15}}"
+      }
+    },
+    "response": {
+      "store": "txtRecordAfterUpdate"
+    }
+  },
+  {
+    "before": "await_url me/networks/{{networks.[0].uuid}}/dns/dig?type=txt&name=test_TXT_{{rootUser.uuid}} 5m 10s await_json.length > 0 && await_json[0].getValue() === '{{txtRecordAfterUpdate.value}}'",
+    "comment": "re-dig updated TXT record, should be one, verify new value",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/dig?type=txt&name=test_TXT_{{rootUser.uuid}}" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 1"},
+        {"condition": "json[0].getFqdn().toUpperCase() === 'test_TXT_{{rootUser.uuid}}.{{networks.[0].networkDomain}}'.toUpperCase()"},
+        {"condition": "json[0].getValue() === '{{txtRecordAfterUpdate.value}}'"},
+        {"condition": "json[0].getValue() !== '{{txtRecord.value}}'"}
+      ]
+    }
+  },
+  {
+    "before": "await_url me/networks/{{networks.[0].uuid}}/dns/find?type=TXT&name=test_TXT_{{rootUser.uuid}} 5m 10s await_json.length > 0 && await_json[0].getValue() === '{{txtRecordAfterUpdate.value}}'",
+    "comment": "re-list updated TXT record, should be one, verify new value",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/find?type=txt&name=test_TXT_{{rootUser.uuid}}" },
+    "response": {
+      "check": [
+        {"condition": "json.length === 1"},
+        {"condition": "json[0].getFqdn().toUpperCase() === 'test_TXT_{{rootUser.uuid}}.{{networks.[0].networkDomain}}'.toUpperCase()"},
+        {"condition": "json[0].getValue() === '{{txtRecordAfterUpdate.value}}'"},
+        {"condition": "json[0].getValue() !== '{{txtRecord.value}}'"}
+      ]
+    }
+  },
+
+  {
+    "comment": "delete TXT record",
+    "request": {
+      "uri": "me/networks/{{networks.[0].uuid}}/dns/remove",
+      "entity": {
+        "type": "TXT",
+        "fqdn": "test_TXT_{{rootUser.uuid}}.{{networks.[0].networkDomain}}",
+        "value": "{{txtRecord.value}}"
+      }
+    }
+  },
+
+  {
+    "comment": "delete A record",
+    "request": {
+      "uri": "me/networks/{{networks.[0].uuid}}/dns/remove",
+      "entity": {
+        "type": "A",
+        "fqdn": "test_A_{{rootUser.uuid}}.{{networks.[0].networkDomain}}",
+        "value": "127.0.0.1"
+      }
+    }
+  },
+
+  {
+    "before": "await_url me/networks/{{networks.[0].uuid}}/dns/find?name={{urlEncode 'test_\\w+_'}}{{rootUser.uuid}} 5m 10s await_json.length === 0",
+    "comment": "after deletion, list matching records, should be none",
+    "request": { "uri": "me/networks/{{networks.[0].uuid}}/dns/find?name={{urlEncode 'test_\\w+_'}}{{rootUser.uuid}}" },
+    "response": { "check": [ {"condition": "json.length === 0"} ] }
+  }
+]

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit f804680fd96b25582d9a269cfcd82f173c7da4d4
+Subproject commit 17f98db5704e7cdcaf943120563e130281a18f93

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit 6c17620340f77bb163e3a2089838d580181d9fbf
+Subproject commit fe442f58ebe4cef0443bfdc85f5c1bab623baf26