From ebd3ec1d6ae0db278160560293369a435395fc14 Mon Sep 17 00:00:00 2001 From: Kristijan Mitrovic Date: Wed, 3 Jun 2020 14:18:45 +0000 Subject: [PATCH 1/2] Fix new bubble script (#14) Merge branch 'master' into kris/fix_backup_restore_test_cont # Conflicts: # utils/cobbzilla-utils # utils/cobbzilla-wizard Add activation key for new bubble's first login in test Call new bubble's API before DNS list in test update libs Beautify code Fix DNS list API request URL in test Better DNS listing APIs usage Co-authored-by: Kristijan Mitrovic Reviewed-on: https://git.bubblev.org/bubblev/bubble/pulls/14 --- .../cloud/dns/godaddy/GoDaddyDnsDriver.java | 39 +++++++------ .../cloud/dns/route53/Route53DnsDriver.java | 22 +++---- .../resources/models/include/new_bubble.json | 57 ++++++------------- .../models/tests/live/backup_and_restore.json | 3 +- 4 files changed, 50 insertions(+), 71 deletions(-) diff --git a/bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsDriver.java b/bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsDriver.java index ba0e8abd..e1211ee6 100644 --- a/bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/dns/godaddy/GoDaddyDnsDriver.java @@ -15,7 +15,6 @@ import org.cobbzilla.util.http.HttpRequestBean; import org.cobbzilla.util.http.HttpResponseBean; import org.cobbzilla.util.http.HttpUtil; -import java.io.IOException; import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -32,6 +31,7 @@ import static org.cobbzilla.util.http.HttpMethods.PATCH; import static org.cobbzilla.util.http.HttpMethods.PUT; import static org.cobbzilla.util.json.JsonUtil.COMPACT_MAPPER; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; public class GoDaddyDnsDriver extends DnsDriverBase { @@ -141,10 +141,6 @@ public class GoDaddyDnsDriver extends DnsDriverBase { return response.isOk() ? response : die("remove: " + response); }, MAX_GODADDY_RETRIES); return record; - - } catch (IOException e) { - return die("remove: "+e); - } finally { if (lock != null && domain.get() != null) unlockDomain(domain.get().getUuid(), lock); } @@ -165,18 +161,21 @@ public class GoDaddyDnsDriver extends DnsDriverBase { if (domain == null) return emptyList(); // iterate over all records, return matches - String url = config.getBaseUri()+domain.getName()+"/records"; - if (matcher != null) { - if (matcher.hasType()) { - url += "/" + matcher.getType().name(); - } - if (matcher.hasFqdn()) { - String fqdn = matcher.getFqdn(); - fqdn = domain.dropDomainSuffix(fqdn); - url += "/" + fqdn; + final var url = new StringBuilder(config.getBaseUri()).append(domain.getName()).append("/records"); + if (matcher != null && (matcher.hasType() || matcher.hasFqdn())) { + if (!matcher.hasType() || !matcher.hasPattern()) { + // as per GoDaddy's docs both type and fqdn must be set here + // https://developer.godaddy.com/doc/endpoint/domains#/v1/recordGet + throw invalidEx("err.request.invalid", "Both type and pattern are required"); } + + url.append("/").append(matcher.getType().name()); + + var fqdn = matcher.getPattern(); + fqdn = domain.dropDomainSuffix(fqdn); + url.append("/").append(fqdn); } - return readRecords(domain, url, matcher); + return readRecords(domain, url.toString(), matcher); } public Collection readRecords(BubbleDomain domain, String url, DnsRecordMatch matcher) { @@ -195,17 +194,17 @@ public class GoDaddyDnsDriver extends DnsDriverBase { private final Map listCache = new ExpirationMap<>(SECONDS.toMillis(10)); - public GoDaddyDnsRecord[] listGoDaddyDnsRecords(String url) throws IOException { - final HttpRequestBean request = auth(url); - return listCache.computeIfAbsent(url, k -> { + public GoDaddyDnsRecord[] listGoDaddyDnsRecords(final String goDaddyApiUrl) { + return listCache.computeIfAbsent(goDaddyApiUrl, url -> { + final var request = auth(url); final HttpResponseBean response; try { response = HttpUtil.getResponse(request); } catch (Exception e) { - log.error("listGoDaddyDnsRecords("+url+"): "+e); + log.error("listGoDaddyDnsRecords(" + url + "): " + e, e); return GoDaddyDnsRecord.EMPTY_ARRAY; } - if (!response.isOk()) throw new IllegalStateException("readRecords: "+response); + if (!response.isOk()) throw new IllegalStateException("listGoDaddyDnsRecords: " + response); return json(response.getEntityString(), GoDaddyDnsRecord[].class); }); } diff --git a/bubble-server/src/main/java/bubble/cloud/dns/route53/Route53DnsDriver.java b/bubble-server/src/main/java/bubble/cloud/dns/route53/Route53DnsDriver.java index 2ff760ad..6b989d13 100644 --- a/bubble-server/src/main/java/bubble/cloud/dns/route53/Route53DnsDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/dns/route53/Route53DnsDriver.java @@ -18,10 +18,7 @@ import org.cobbzilla.util.dns.DnsRecord; import org.cobbzilla.util.dns.DnsRecordMatch; import org.cobbzilla.util.dns.DnsType; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -57,15 +54,20 @@ public class Route53DnsDriver extends DnsDriverBase { @Getter(lazy=true) private final Map cachedZoneLookups = new ExpirationMap<>(); private HostedZone getHostedZone(BubbleDomain domain) { return getCachedZoneLookups().computeIfAbsent(domain.getName(), key -> { + final var keyDot = key + "."; + + final Optional found; try { - final ListHostedZonesResult zones = getRoute53client().listHostedZones(new ListHostedZonesRequest().withMaxItems(MAX_ITEMS)); - for (HostedZone z : zones.getHostedZones()) { - if (z.getName().equalsIgnoreCase(key + ".")) return z; - } - return die("HostedZone with name '"+key+".' not found"); + found = getRoute53client().listHostedZones(new ListHostedZonesRequest().withMaxItems(MAX_ITEMS)) + .getHostedZones() + .stream() + .filter(z -> z.getName().equalsIgnoreCase(keyDot)) + .findFirst(); } catch (Exception e) { - return die("getHostedZone: "+e); + return die("getHostedZone: " + e); } + + return found.isPresent() ? found.get() : die("getHostedZone: HostedZone not found with name: " + keyDot); }); } diff --git a/bubble-server/src/test/resources/models/include/new_bubble.json b/bubble-server/src/test/resources/models/include/new_bubble.json index 2b121284..342eb6ce 100644 --- a/bubble-server/src/test/resources/models/include/new_bubble.json +++ b/bubble-server/src/test/resources/models/include/new_bubble.json @@ -165,57 +165,36 @@ }, { - "comment": "list DNS for the network, should now see a DNS A record for new instance", - "before": "await_url me/networks/<>/dns/find?type=A&name=.<>.<> 40m 10s await_json.length > 0", - "request": { - "uri": "me/networks/<>/dns/find?type=A&name=.<>.<>" - }, - "response": { - "store": "dnsRecords", - "check": [ - {"condition": "json.length == 1"} - ] - } - }, - - { - "comment": "call API of deployed node, ensure it is running", - "before": "await_url .bubble 40m 20s", + "comment": "call API of deployed node after some grace period, ensure it is running", + "before": "await_url .bubble 20m:40m 20s", "connection": { "name": "<>", "baseUri": "https://{{<>.host}}.<>.<>:{{serverConfig.nginxPort}}/api" }, "request": { "uri" : ".bubble" }, - "response": { - "raw": true, - "check": [ - {"condition": "response.json == 'you are ok. the magic is ok too.'"} - ] - } + "response": { "raw": true, "check": [{ "condition": "response.json == 'you are ok. the magic is ok too.'" }] } }, -// { -// "comment": "verify new node has said hello, and requested another node", -// "connection": { "name": "<>_connection" }, -// "request": { -// "session": "rootSession", -// "uri": "me/notifications/inbox" -// }, -// "response": { -// "check": [ -// {"condition": "json.length >= 2"}, -// {"condition": "_find(json, function (n) { n.getType().name() == 'upstream_hello' }) != null"}, -// {"condition": "_find(json, function (n) { n.getType().name() == 'new_node' }) != null"} -// ] -// } -// }, + { + "comment": "now list DNS for the network, should now see a DNS A record for new instance", + "connection": { "name": "<>" }, + "request": { "uri": "me/networks/<>/dns/find?type=A&name={{<>.host}}.<>.<>" }, + "response": { "store": "dnsRecords", "check": [{ "condition": "json.length == 1" }] } + }, { - "comment": "login to deployed node", + "comment": "check unauthorized access to debug mailbox required for this test (BUBBLE_TEST_MODE has to be true)", "connection": { "name": "<>" }, + "request": { "uri": "debug/inbox/email/<>?type=request&action=verify&target=network" }, + "response": { "status": 200 }, // confirming status is not 401 here + "after": "await_url debug/inbox/email/<>?type=request&action=verify&target=network 10m 10s len(await_json) > 0" + }, + + { + "comment": "activate and login to deployed node", "request": { "session": "new", - "uri" : "auth/login", + "uri" : "auth/login?k={{await_json.[0].ctx.message.data}}", "entity": { "name": "<>", "password": "<>" diff --git a/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json b/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json index 92cef50a..0712e849 100644 --- a/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json +++ b/bubble-server/src/test/resources/models/tests/live/backup_and_restore.json @@ -54,8 +54,7 @@ }, { - "comment": "add file to storage", - "before": "await_url .bubble 40m 20s", + "comment": "add test file to storage", "connection": { "name": "bubbleConnection" }, "request": { "session": "bubbleUserSession", From fd2c8a0a903bd57f810e9c81ee0617ac8a55bd49 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Wed, 3 Jun 2020 10:20:54 -0400 Subject: [PATCH 2/2] update lib --- utils/cobbzilla-wizard | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index b59f076d..a07578cf 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit b59f076d870bf8d66721cb77a36cc3ce29558b20 +Subproject commit a07578cf1fde1cdaee0abc626fa4761fe8421446