From df286c3e74dee33236d593d08a6677576957cc4d Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 7 Dec 2020 08:50:46 -0500 Subject: [PATCH] WIP. moving docker deploy along. add version to utils instead of snapshot --- bubble-server/pom.xml | 18 ++++--- .../src/main/java/bubble/cloud/NoopCloud.java | 5 -- .../cloud/compute/ComputeServiceDriver.java | 22 ++++++++ .../compute/docker/DockerComputeDriver.java | 53 ++++++++++++++++--- .../java/bubble/cloud/dns/DnsDriverBase.java | 17 ------ .../bubble/cloud/dns/DnsServiceDriver.java | 3 -- .../dns/delegate/DelegatedDnsDriver.java | 7 --- .../model/cloud/notify/NotificationType.java | 1 - ...icationHandler_dns_driver_set_network.java | 27 ---------- .../resources/account/AccountsResource.java | 3 ++ .../bubble/resources/account/MeResource.java | 3 ++ .../resources/bill/AccountPlansResource.java | 3 ++ .../bubble/resources/bill/BillsResource.java | 2 + .../bubble/server/BubbleConfiguration.java | 5 ++ .../service/cloud/StandardNetworkService.java | 18 ++----- .../bubble/service/packer/PackerBuild.java | 16 ++---- .../java/bubble/service/packer/PackerJob.java | 13 +++-- .../META-INF/bubble/bubble.properties | 4 +- .../resources/ansible/install_local.sh.hbs | 2 +- .../models/defaults/cloudService.json | 3 +- .../roles/algo/files/algo_refresh_users.sh | 2 +- .../algo/files/wg_monitor_connections.sh | 22 ++++---- .../roles/bubble/files/bubble_upgrade.sh | 2 +- .../roles/bubble/files/init_bubble_db.sh | 2 +- .../bubble/files/refresh_bubble_ssh_keys.sh | 2 +- .../packer/roles/common/tasks/docker.yml | 5 ++ .../packer/roles/common/tasks/main.yml | 3 ++ .../packer/roles/firewall/tasks/main.yml | 7 +-- .../roles/mitmproxy/files/mitm_monitor.sh | 2 +- .../packer/roles/mitmproxy/files/run_mitm.sh | 2 +- .../packer/roles/mitmproxy/tasks/main.yml | 26 ++++++--- .../resources/models/system/cloudService.json | 3 +- docker/bubble.sh | 4 +- pom.xml | 5 +- utils/abp-parser | 2 +- utils/cobbzilla-parent | 2 +- utils/cobbzilla-utils | 2 +- utils/cobbzilla-wizard | 2 +- utils/templated-mail-sender | 2 +- 39 files changed, 178 insertions(+), 144 deletions(-) delete mode 100644 bubble-server/src/main/java/bubble/notify/dns/NotificationHandler_dns_driver_set_network.java create mode 100644 bubble-server/src/main/resources/packer/roles/common/tasks/docker.yml diff --git a/bubble-server/pom.xml b/bubble-server/pom.xml index 1a56e0c4..9633c75c 100644 --- a/bubble-server/pom.xml +++ b/bubble-server/pom.xml @@ -10,11 +10,13 @@ bubble bubble - 1.0.0-SNAPSHOT + + 1.5.1 bubble-server - 1.0.0-SNAPSHOT + + 1.5.1 @@ -57,7 +59,7 @@ org.cobbzilla cobbzilla-utils - 1.0.0-SNAPSHOT + 2.0.1 @@ -115,7 +117,7 @@ org.cobbzilla wizard-server - 1.0.0-SNAPSHOT + 2.0.1 org.eclipse.jetty @@ -144,7 +146,7 @@ bubble abp-parser - 1.0.0-SNAPSHOT + 2.0.1 @@ -172,7 +174,7 @@ org.cobbzilla templated-mail-sender - 1.0.0-SNAPSHOT + 2.0.1 @@ -375,7 +377,7 @@ org.cobbzilla wizard-server-test - 1.0.0-SNAPSHOT + 2.0.1 test @@ -687,7 +689,7 @@ The exclusions below were generated with: package shade - ${project.basedir}/target/bubble-server-1.0.0-SNAPSHOT-full.jar + ${project.basedir}/target/bubble-server-${project.version}-full.jar bubble.server.BubbleServer diff --git a/bubble-server/src/main/java/bubble/cloud/NoopCloud.java b/bubble-server/src/main/java/bubble/cloud/NoopCloud.java index b11894ea..b8161845 100644 --- a/bubble-server/src/main/java/bubble/cloud/NoopCloud.java +++ b/bubble-server/src/main/java/bubble/cloud/NoopCloud.java @@ -197,11 +197,6 @@ public class NoopCloud implements return null; } - @Override public Collection setNetwork(BubbleNetwork network) { - if (log.isDebugEnabled()) log.debug("setNetwork(network=" + network + ")"); - return null; - } - @Override public Collection setNode(BubbleNode node) { if (log.isDebugEnabled()) log.debug("setNode(node=" + node + ")"); return null; diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java index a54f988a..e9bc6228 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ComputeServiceDriver.java @@ -10,6 +10,7 @@ import bubble.cloud.CloudServiceType; import bubble.model.cloud.AnsibleInstallType; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.RegionalServiceDriver; +import bubble.service.packer.PackerBuild; import org.cobbzilla.util.system.CommandResult; import java.util.List; @@ -44,4 +45,25 @@ public interface ComputeServiceDriver extends CloudServiceDriver, RegionalServic default int getPackerParallelBuilds() { return 1; } + default boolean supportsPacker(AnsibleInstallType installType) { return true; } + + default CloudRegion[] getRegions(PackerBuild packerBuild) { + final String[] parts = packerBuild.getArtifact_id().split(":"); + final String[] regionNames = parts[0].split(","); + final CloudRegion[] regions = new CloudRegion[regionNames.length]; + for (int i=0; i CLOUD_REGIONS = Arrays.asList(new CloudRegion[]{ + public static final CloudRegion[] CLOUD_REGIONS_ARRAY = new CloudRegion[]{ new CloudRegion().setName("local").setInternalName("local") - }); + }; + public static final List CLOUD_REGIONS = Arrays.asList(CLOUD_REGIONS_ARRAY); + public static final List CLOUD_SIZES = Arrays.asList(new ComputeNodeSize[]{ new ComputeNodeSize().setName("local").setInternalName("local").setType(ComputeNodeSizeType.local) }); @@ -55,6 +67,24 @@ public class DockerComputeDriver extends ComputeServiceDriverBase { @Getter private final List cloudSizes = CLOUD_SIZES; @Getter private final List cloudOsImages = CLOUD_OS_IMAGES; + @Override public boolean supportsPacker(AnsibleInstallType installType) { + boolean supported = installType == AnsibleInstallType.sage || CURRENT_OS == linux; + if (!supported) log.warn("supportsPacker: installType "+installType+" not supported (no images will be created) for platform: "+CURRENT_OS); + return supported; + } + + @Override public boolean supportsDns() { return false; } + + @Override public CloudRegion[] getRegions(PackerBuild packerBuild) { return CLOUD_REGIONS_ARRAY; } + + @Override public String getPackerImageId(String name, PackerBuild packerBuild) { return name; } + + private final Map> portMappings = new ConcurrentHashMap(); + + @Override public int getSshPort(BubbleNode node) { + return portMappings.get(node.getUuid()).get(1202); + } + @Getter(lazy=true) private final DockerClient dockerClient = initDockerClient(); private DockerClient initDockerClient() { CloudCredentials creds = getCredentials(); @@ -88,20 +118,31 @@ public class DockerComputeDriver extends ComputeServiceDriverBase { final PackerImage packerImage = getOrCreatePackerImage(node); - final CreateContainerResponse ccr = dc.createContainerCmd(packerImage.getId()) + final CreateContainerCmd ccr = dc.createContainerCmd(packerImage.getId()) .withLabels(MapBuilder.build(new String[][] { {LABEL_CLOUD, cloud.getUuid()}, {LABEL_NODE, node.getUuid()} })) - .exec(); + .withHostConfig(HostConfig.newHostConfig() + .withCapAdd(Capability.NET_ADMIN) + .withCapAdd(Capability.SYS_ADMIN)); + final CreateContainerResponse response = ccr.exec(); final long start = now(); - while (listNodes().stream().noneMatch(n -> n.isRunning() && n.getUuid().equals(node.getUuid()))) { + final Predicate nodeFilter = filterForNode(node); + while (listNodes().stream().noneMatch(nodeFilter)) { if (now() - start > START_TIMEOUT) { return die("start("+node.id()+"): timeout"); } sleep(SECONDS.toMillis(5), "waiting for docker container to be running"); } - return node; + final String containerId = lookupContainer(node); + final InspectContainerResponse status = dc.inspectContainerCmd(containerId).exec(); + + return node.setIp4("127.0.0.1").setIp6("fd00::1"); + } + + private Predicate filterForNode(BubbleNode node) { + return n -> n.isRunning() && n.getUuid().equals(node.getUuid()); } private String lookupContainer(BubbleNode node) { diff --git a/bubble-server/src/main/java/bubble/cloud/dns/DnsDriverBase.java b/bubble-server/src/main/java/bubble/cloud/dns/DnsDriverBase.java index 93833013..1160d803 100644 --- a/bubble-server/src/main/java/bubble/cloud/dns/DnsDriverBase.java +++ b/bubble-server/src/main/java/bubble/cloud/dns/DnsDriverBase.java @@ -23,7 +23,6 @@ import java.util.List; import static java.util.concurrent.TimeUnit.MINUTES; import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.network.NetworkUtil.IPv4_ALL_ADDRS; public abstract class DnsDriverBase extends CloudServiceDriverBase implements DnsServiceDriver { @@ -79,22 +78,6 @@ public abstract class DnsDriverBase extends CloudServiceDriverBase impleme return null; } - @Override public Collection setNetwork(BubbleNetwork network) { - final DnsServiceDriver dns = cloud.getDnsDriver(configuration); - final Collection records = new ArrayList<>(); - if (dns.requireSubnetNS()) { - final BubbleDomain domain = domainDAO.findByUuid(network.getDomain()); - for (String ns : dns.resolveNS(domain)) { - records.add(dns.update((DnsRecord) new DnsRecord() - .setOption(DnsRecord.OPT_NS_NAME, ns) - .setType(DnsType.NS) - .setValue(IPv4_ALL_ADDRS) - .setFqdn(network.getNetworkDomain()))); - } - } - return records; - } - @Override public Collection setNode(BubbleNode node) { final DnsServiceDriver dns = cloud.getDnsDriver(configuration); final BubbleNetwork network = networkDAO.findByUuid(node.getNetwork()); diff --git a/bubble-server/src/main/java/bubble/cloud/dns/DnsServiceDriver.java b/bubble-server/src/main/java/bubble/cloud/dns/DnsServiceDriver.java index 2055ca92..0ce47773 100644 --- a/bubble-server/src/main/java/bubble/cloud/dns/DnsServiceDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/dns/DnsServiceDriver.java @@ -7,7 +7,6 @@ package bubble.cloud.dns; import bubble.cloud.CloudServiceDriver; import bubble.cloud.CloudServiceType; import bubble.model.cloud.BubbleDomain; -import bubble.model.cloud.BubbleNetwork; import bubble.model.cloud.BubbleNode; import org.cobbzilla.util.dns.DnsRecord; import org.cobbzilla.util.dns.DnsRecordBase; @@ -42,7 +41,6 @@ public interface DnsServiceDriver extends CloudServiceDriver { Collection create(BubbleDomain domain); - Collection setNetwork(BubbleNetwork network); Collection setNode(BubbleNode node); Collection deleteNode(BubbleNode node); @@ -213,5 +211,4 @@ public interface DnsServiceDriver extends CloudServiceDriver { return true; } - default boolean requireSubnetNS() { return false; } } diff --git a/bubble-server/src/main/java/bubble/cloud/dns/delegate/DelegatedDnsDriver.java b/bubble-server/src/main/java/bubble/cloud/dns/delegate/DelegatedDnsDriver.java index 12d0fac4..c22899ca 100644 --- a/bubble-server/src/main/java/bubble/cloud/dns/delegate/DelegatedDnsDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/dns/delegate/DelegatedDnsDriver.java @@ -7,7 +7,6 @@ package bubble.cloud.dns.delegate; import bubble.cloud.DelegatedCloudServiceDriverBase; import bubble.cloud.dns.DnsServiceDriver; import bubble.model.cloud.BubbleDomain; -import bubble.model.cloud.BubbleNetwork; import bubble.model.cloud.BubbleNode; import bubble.model.cloud.CloudService; import bubble.notify.dns.DnsDriverNotification; @@ -29,12 +28,6 @@ public class DelegatedDnsDriver extends DelegatedCloudServiceDriverBase implemen return Arrays.asList(records); } - @Override public Collection setNetwork(BubbleNetwork network) { - final BubbleNode delegate = getDelegateNode(); - final DnsRecord[] records = notificationService.notifySync(delegate, dns_driver_set_network, notification(new DnsDriverNotification(network))); - return Arrays.asList(records); - } - @Override public Collection setNode(BubbleNode node) { final BubbleNode delegate = getDelegateNode(); final DnsRecord[] records = notificationService.notifySync(delegate, dns_driver_set_node, notification(new DnsDriverNotification(node))); diff --git a/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java b/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java index 55718f6a..efdb8d01 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java +++ b/bubble-server/src/main/java/bubble/model/cloud/notify/NotificationType.java @@ -47,7 +47,6 @@ public enum NotificationType { // driver-level notifications // delegated dns driver notifications - dns_driver_set_network (DnsRecord[].class), dns_driver_set_node (DnsRecord[].class), dns_driver_delete_node (DnsRecord[].class), dns_driver_create (DnsRecord[].class), diff --git a/bubble-server/src/main/java/bubble/notify/dns/NotificationHandler_dns_driver_set_network.java b/bubble-server/src/main/java/bubble/notify/dns/NotificationHandler_dns_driver_set_network.java deleted file mode 100644 index b12aa633..00000000 --- a/bubble-server/src/main/java/bubble/notify/dns/NotificationHandler_dns_driver_set_network.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright (c) 2020 Bubble, Inc. All rights reserved. - * For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ - */ -package bubble.notify.dns; - -import bubble.cloud.dns.DnsServiceDriver; -import bubble.model.cloud.BubbleDomain; -import bubble.model.cloud.BubbleNetwork; -import bubble.model.cloud.notify.ReceivedNotification; -import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.dns.DnsRecord; - -import java.util.Collection; - -@Slf4j -public class NotificationHandler_dns_driver_set_network extends NotificationHandler_dns_driver> { - - @Override protected Collection handle(ReceivedNotification n, - DnsDriverNotification dnsNotification, - BubbleDomain domain, - BubbleNetwork network, - DnsServiceDriver dns) { - return dns.setNetwork(network); - } - -} diff --git a/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java b/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java index a9cf4467..2e92bdd9 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AccountsResource.java @@ -795,6 +795,7 @@ public class AccountsResource { @Path("/{id}"+EP_PAYMENT_METHODS) public AccountPaymentMethodsResource getAccountPaymentMethods(@Context ContainerRequest ctx, @PathParam("id") String id) { + configuration.requiresPaymentsEnabled(); final AccountContext c = new AccountContext(ctx, id); return configuration.subResource(AccountPaymentMethodsResource.class, c.account); } @@ -802,6 +803,7 @@ public class AccountsResource { @Path("/{id}"+EP_BILLS) public BillsResource getBills(@Context ContainerRequest ctx, @PathParam("id") String id) { + configuration.requiresPaymentsEnabled(); final AccountContext c = new AccountContext(ctx, id); return configuration.subResource(BillsResource.class, c.account); } @@ -809,6 +811,7 @@ public class AccountsResource { @Path("/{id}"+EP_PAYMENTS) public AccountPaymentsResource getPayments(@Context ContainerRequest ctx, @PathParam("id") String id) { + configuration.requiresPaymentsEnabled(); final AccountContext c = new AccountContext(ctx, id); return configuration.subResource(AccountPaymentsResource.class, c.account); } diff --git a/bubble-server/src/main/java/bubble/resources/account/MeResource.java b/bubble-server/src/main/java/bubble/resources/account/MeResource.java index fc8ecbf1..0041f684 100644 --- a/bubble-server/src/main/java/bubble/resources/account/MeResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/MeResource.java @@ -439,18 +439,21 @@ public class MeResource { @Path(EP_PAYMENT_METHODS) public AccountPaymentMethodsResource getAccountPaymentMethods(@Context ContainerRequest ctx) { + configuration.requiresPaymentsEnabled(); final Account caller = userPrincipal(ctx); return configuration.subResource(AccountPaymentMethodsResource.class, caller); } @Path(EP_BILLS) public BillsResource getBills(@Context ContainerRequest ctx) { + configuration.requiresPaymentsEnabled(); final Account caller = userPrincipal(ctx); return configuration.subResource(BillsResource.class, caller); } @Path(EP_PAYMENTS) public AccountPaymentsResource getPayments(@Context ContainerRequest ctx) { + configuration.requiresPaymentsEnabled(); final Account caller = userPrincipal(ctx); return configuration.subResource(AccountPaymentsResource.class, caller); } diff --git a/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java b/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java index 296c408f..99a56c79 100644 --- a/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java +++ b/bubble-server/src/main/java/bubble/resources/bill/AccountPlansResource.java @@ -361,6 +361,7 @@ public class AccountPlansResource extends AccountOwnedResource { @Path("/{id}"+EP_PAYMENTS) public AccountPaymentsResource getPayments(@Context ContainerRequest ctx, @PathParam("id") String id) { + configuration.requiresPaymentsEnabled(); final Bill bill = super.find(ctx, id); if (bill == null) throw notFoundEx(id); return configuration.subResource(AccountPaymentsResource.class, account, bill); @@ -118,6 +119,7 @@ public class BillsResource extends ReadOnlyAccountOwnedResource { public Response payBill(@Context ContainerRequest ctx, @PathParam("id") String id, AccountPaymentMethod paymentMethod) { + configuration.requiresPaymentsEnabled(); final Bill bill = super.find(ctx, id); if (bill == null) return notFound(id); if (bill.paid()) return invalid("err.bill.alreadyPaid"); diff --git a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java index 45b5ed97..57da5ed6 100644 --- a/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java +++ b/bubble-server/src/main/java/bubble/server/BubbleConfiguration.java @@ -77,6 +77,7 @@ import static org.cobbzilla.util.security.ShaUtil.sha256_file; import static org.cobbzilla.util.security.ShaUtil.sha256_hex; import static org.cobbzilla.util.system.CommandShell.totalSystemMemory; import static org.cobbzilla.wizard.model.SemanticVersion.isNewerVersion; +import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; @Configuration @NoArgsConstructor @Slf4j public class BubbleConfiguration extends PgRestServerConfiguration @@ -420,6 +421,10 @@ public class BubbleConfiguration extends PgRestServerConfiguration return peValue != null && Boolean.parseBoolean(peValue.toString()); } + public void requiresPaymentsEnabled () { + if (!paymentsEnabled()) throw invalidEx("err_noPaymentMethods"); + } + @Getter @Setter private Boolean requireSendMetrics; public boolean requireSendMetrics () { return bool(requireSendMetrics); } diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index ccf7c751..443d81ff 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -252,8 +252,10 @@ public class StandardNetworkService implements NetworkService { jobFutures.add(backgroundJobs.submit(startJob)); // Create DNS records for node - final NodeDnsJob dnsJob = new NodeDnsJob(cloudDAO, domain, network, node, configuration); - jobFutures.add(backgroundJobs.submit(dnsJob)); + if (computeDriver.supportsDns()) { + final NodeDnsJob dnsJob = new NodeDnsJob(cloudDAO, domain, network, node, configuration); + jobFutures.add(backgroundJobs.submit(dnsJob)); + } // Prepare ansible roles // We must wait until after server is started, because some roles require ip4 in vars @@ -291,7 +293,7 @@ public class StandardNetworkService implements NetworkService { // run ansible final String sshArgs - = "-p 1202 " + = "-p " + computeDriver.getSshPort(node) + " " + "-o UserKnownHostsFile=/dev/null " + "-o StrictHostKeyChecking=no " + "-o PreferredAuthentications=publickey " @@ -722,11 +724,6 @@ public class StandardNetworkService implements NetworkService { network.setState(BubbleNetworkState.starting); networkDAO.update(network); - // ensure NS records for network are in DNS - final BubbleDomain domain = domainDAO.findByUuid(network.getDomain()); - final CloudService dns = cloudDAO.findByUuid(domain.getPublicDns()); - dns.getDnsDriver(configuration).setNetwork(network); - final NewNodeNotification newNodeRequest = new NewNodeNotification() .setFork(network.fork()) .setLaunchType(network.getLaunchType()) @@ -787,11 +784,6 @@ public class StandardNetworkService implements NetworkService { network.setState(BubbleNetworkState.starting); networkDAO.update(network); - // ensure NS records for network are in DNS - final BubbleDomain domain = domainDAO.findByUuid(network.getDomain()); - final CloudService dns = cloudDAO.findByUuid(domain.getPublicDns()); - dns.getDnsDriver(configuration).setNetwork(network); - final CloudAndRegion cloudAndRegion = geoService.selectCloudAndRegion(network, netLocation); final String restoreKey = randomAlphanumeric(RESTORE_KEY_LEN).toUpperCase(); restoreService.registerRestore(restoreKey, new NetworkKeys()); diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerBuild.java b/bubble-server/src/main/java/bubble/service/packer/PackerBuild.java index b7839d45..460e0776 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerBuild.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerBuild.java @@ -4,7 +4,7 @@ */ package bubble.service.packer; -import bubble.cloud.CloudRegion; +import bubble.cloud.compute.ComputeServiceDriver; import bubble.cloud.compute.PackerImage; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -21,17 +21,11 @@ public class PackerBuild { @Getter @Setter private String packer_run_uuid; @Getter @Setter private JsonNode custom_data; - public PackerImage toPackerImage(String name) { - final String[] parts = artifact_id.split(":"); - final String[] regionNames = parts[0].split(","); - final CloudRegion[] regions = new CloudRegion[regionNames.length]; - for (int i=0; i> { final ComputeServiceDriver computeDriver = cloud.getComputeDriver(configuration); final PackerConfig packerConfig = computeConfig.getPacker(); + if (!computeDriver.supportsPacker(installType)) { + setImagesRefs(); + return Collections.emptyList(); + } + // create handlebars context final Map ctx = new HashMap<>(); final CloudCredentials creds = cloud.getCredentials(); @@ -174,12 +179,12 @@ public class PackerJob implements Callable> { if (!env.containsKey("HOME")) env.put("HOME", HOME_DIR); // Docker builder requires "docker" command to be on our path - // It is usually in /usr/local/bin + // It is usually /usr/local/bin on macosx and /usr/bin on linux // May need to make this more flexible if docker is elsewhere, or other tools/paths are needed if (env.containsKey("PATH")) { - env.put("PATH", "${PATH}:/usr/local/bin"); + env.put("PATH", "${PATH}:/usr/local/bin:/usr/bin"); } else { - env.put("PATH", "/usr/local/bin"); + env.put("PATH", "/usr/local/bin:/usr/bin"); } ctx.put(VARIABLES_VAR, packerConfig.getVars()); @@ -304,7 +309,7 @@ public class PackerJob implements Callable> { if (empty(builds)) { return die("Error executing packer: no builds found"); } - images.addAll(Arrays.stream(builds).map(b -> b.toPackerImage(imageName)).collect(Collectors.toList())); + images.addAll(Arrays.stream(builds).map(b -> b.toPackerImage(imageName, computeDriver)).collect(Collectors.toList())); } else { final List finalizedImages = computeDriver.finalizeIncompletePackerRun(commandResult, installType); diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index 28bdddf0..89063c70 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1,3 @@ -bubble.version=Adventure 1.5.2 +# Do not edit this file directly +# Use _set_version to update the Bubble version in all files +bubble.version=Adventure 1.5.1 diff --git a/bubble-server/src/main/resources/ansible/install_local.sh.hbs b/bubble-server/src/main/resources/ansible/install_local.sh.hbs index f13e8d5a..279dddba 100644 --- a/bubble-server/src/main/resources/ansible/install_local.sh.hbs +++ b/bubble-server/src/main/resources/ansible/install_local.sh.hbs @@ -28,7 +28,7 @@ function kill_bg_jobs { kill %${j} done {{#if isNode}} - if [[ ! -z "${ALGO_LOG}" && -f "${ALGO_LOG}" ]] ; then + if [[ -n "${ALGO_LOG}" && -f "${ALGO_LOG}" ]] ; then rm -f ${ALGO_LOG} fi {{/if}} diff --git a/bubble-server/src/main/resources/models/defaults/cloudService.json b/bubble-server/src/main/resources/models/defaults/cloudService.json index 71239578..71229901 100644 --- a/bubble-server/src/main/resources/models/defaults/cloudService.json +++ b/bubble-server/src/main/resources/models/defaults/cloudService.json @@ -166,9 +166,10 @@ "type": "docker", "image": "<>", "export_path": "<>.tar", + "cap_add": ["NET_ADMIN", "SYS_ADMIN", "SYS_MODULE"], "changes": [ "LABEL bubble_image=<>", - "EXPOSE 80 443 1202" + "EXPOSE 22 80 443 1080 1202 1443 8888 9999 53/udp 500/udp 4500/udp 51820/udp" ] }, "post": { diff --git a/bubble-server/src/main/resources/packer/roles/algo/files/algo_refresh_users.sh b/bubble-server/src/main/resources/packer/roles/algo/files/algo_refresh_users.sh index 790f21a4..446bd910 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/files/algo_refresh_users.sh +++ b/bubble-server/src/main/resources/packer/roles/algo/files/algo_refresh_users.sh @@ -55,7 +55,7 @@ log "Regenerating algo config..." java -cp /home/bubble/api/bubble.jar bubble.main.BubbleMain generate-algo-conf --algo-config ${ALGO_CONFIG}.hbs || die "Error writing algo config.cfg" NEW_ALGO_CONFIG_SHA="$(sha256sum ${ALGO_CONFIG} | cut -f1 -d' ')" -if [[ ! -z "${ALGO_CONFIG_SHA}" && "${ALGO_CONFIG_SHA}" == "${NEW_ALGO_CONFIG_SHA}" ]] ; then +if [[ -n "${ALGO_CONFIG_SHA}" && "${ALGO_CONFIG_SHA}" == "${NEW_ALGO_CONFIG_SHA}" ]] ; then log "Algo configuration is unchanged, not refreshing: ${ALGO_CONFIG}" else diff --git a/bubble-server/src/main/resources/packer/roles/algo/files/wg_monitor_connections.sh b/bubble-server/src/main/resources/packer/roles/algo/files/wg_monitor_connections.sh index 5a29b1bf..269d3896 100644 --- a/bubble-server/src/main/resources/packer/roles/algo/files/wg_monitor_connections.sh +++ b/bubble-server/src/main/resources/packer/roles/algo/files/wg_monitor_connections.sh @@ -28,7 +28,7 @@ while : ; do transfer="" IFS=$'\n' for line in $(wg show all) ; do - if [[ ! -z "${peer}" ]] ; then + if [[ -n "${peer}" ]] ; then if [[ $(echo "${line}" | tr -d ' ') == endpoint* ]] ; then endpoint="$(echo "${line}" | cut -d: -f2- | awk '{$1=$1};1')" @@ -71,16 +71,16 @@ while : ; do fi if [[ ${line} == peer* ]] ; then - if [[ ! -z "${peer}" ]] ; then - if [[ ! -z "${device}" ]] ; then + if [[ -n "${peer}" ]] ; then + if [[ -n "${device}" ]] ; then echo "in-loop, setting stats for peer ${peer} device ${device}" - if [[ ! -z "${endpoint}" ]] ; then + if [[ -n "${endpoint}" ]] ; then echo "set wg_device_status_${device}_endpoint \"${endpoint}\"" | redis-cli fi - if [[ ! -z "${latest_handshake}" ]] ; then + if [[ -n "${latest_handshake}" ]] ; then echo "set wg_device_status_${device}_latestHandshake \"${latest_handshake}\"" | redis-cli fi - if [[ ! -z "${transfer}" ]] ; then + if [[ -n "${transfer}" ]] ; then echo "set wg_device_status_${device}_transfer \"${transfer}\"" | redis-cli fi fi @@ -93,16 +93,16 @@ while : ; do echo "in-loop, set peer: ${peer}" fi done - if [[ ! -z "${peer}" ]] ; then + if [[ -n "${peer}" ]] ; then echo "end-of-loop, setting stats for peer ${peer} device ${device}" - if [[ ! -z "${device}" ]] ; then - if [[ ! -z "${endpoint}" ]] ; then + if [[ -n "${device}" ]] ; then + if [[ -n "${endpoint}" ]] ; then echo "set wg_device_status_${device}_endpoint \"${endpoint}\"" | redis-cli fi - if [[ ! -z "${latest_handshake}" ]] ; then + if [[ -n "${latest_handshake}" ]] ; then echo "set wg_device_status_${device}_latestHandshake \"${latest_handshake}\"" | redis-cli fi - if [[ ! -z "${transfer}" ]] ; then + if [[ -n "${transfer}" ]] ; then echo "set wg_device_status_${device}_transfer \"${transfer}\"" | redis-cli fi fi diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade.sh index ee7cb407..e2cc6b32 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade.sh +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/bubble_upgrade.sh @@ -45,7 +45,7 @@ function verify_api_ok { done log "verify_api_ok: while loop ended, CURL_STATUS=${CURL_STATUS}, (date - start)=$(expr $(date +%s) - ${START_VERIFY}), VERIFY_TIMEOUT=${VERIFY_TIMEOUT}" - if [[ ! -z "${CURL_STATUS}" && ${CURL_STATUS} -eq 200 ]] ; then + if [[ -n "${CURL_STATUS}" && ${CURL_STATUS} -eq 200 ]] ; then echo "ok" else echo "error" diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh index b79bf8dc..bfe692bd 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/init_bubble_db.sh @@ -87,7 +87,7 @@ if [[ "${3}" == "INIT" ]] ; then exit 0 fi -if [[ ! -z "${DROP_AND_RECREATE}" && "${DROP_AND_RECREATE}" == "drop" ]] ; then +if [[ -n "${DROP_AND_RECREATE}" && "${DROP_AND_RECREATE}" == "drop" ]] ; then dropdb ${DB_NAME} || echo "error dropping DB ${DB_NAME} (will continue)" dropuser ${DB_USER} || echo "error dropping DB user ${DB_USER} (will continue)" uuid > ${BUBBLE_HOME}/.BUBBLE_PG_PASSWORD diff --git a/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh index 770c465c..2d93a6a3 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh +++ b/bubble-server/src/main/resources/packer/roles/bubble/files/refresh_bubble_ssh_keys.sh @@ -34,7 +34,7 @@ for key in $(echo "${CURRENT_KEYS_SQL}" | PGPASSWORD="$(cat /home/bubble/.BUBBLE continue fi KEY="$(bdecrypt "${key}" 2> /dev/null)" - if [[ ! -z "${KEY}" && "${KEY}" == ssh-rsa* ]] ; then + if [[ -n "${KEY}" && "${KEY}" == ssh-rsa* ]] ; then log "Adding authorized key: $(echo "${KEY}" | tr -d '\n')" echo "${KEY}" >> ${NEW_KEYS} KEY_COUNT=$(expr ${KEY_COUNT} + 1) diff --git a/bubble-server/src/main/resources/packer/roles/common/tasks/docker.yml b/bubble-server/src/main/resources/packer/roles/common/tasks/docker.yml new file mode 100644 index 00000000..5911028e --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/common/tasks/docker.yml @@ -0,0 +1,5 @@ +- name: Install packages missing on docker ubuntu + apt: + name: [ 'curl', 'cron', 'iptables', 'openssh-server' ] + state: present + update_cache: yes diff --git a/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml index 81fdab10..edb9c3b4 100644 --- a/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/common/tasks/main.yml @@ -94,3 +94,6 @@ system: yes home: /home/bubble-flex when: install_type == 'node' + +- include: docker.yml + when: packer_builder_type == 'docker' diff --git a/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml index c6d4ff79..6f493809 100644 --- a/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/firewall/tasks/main.yml @@ -109,6 +109,7 @@ owner: root group: root mode: 0400 + when: fw_enable_ssh - name: Install SSH fail2ban settings copy: @@ -117,10 +118,6 @@ owner: root group: root mode: 0400 + when: fw_enable_ssh - include: rules.yml - -- supervisorctl: - name: bubble_peer_manager - state: restarted - when: fw_enable_admin diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh index 06e6f08a..9922eac1 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/mitm_monitor.sh @@ -86,7 +86,7 @@ function healthCheck { while [[ $(expr $(date +%s) - ${START}) -le ${HEALTH_CHECK_TIMEOUT} ]] ; do # log "Performing health check on mitm${MITM_PORT} via ${HC_URL} ..." CURL_OUT="$(curl --silent --connect-timeout 2 --max-time 2 ${HC_URL} 2>> ${LOG})" - if [[ ! -z ${CURL_OUT} && ${CURL_OUT} == "OK" ]] ; then + if [[ -n ${CURL_OUT} && ${CURL_OUT} == "OK" ]] ; then # log "Health check on mitm${MITM_PORT} via ${HC_URL} : OK" echo -n "OK" return diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh index 36747c2a..c8009206 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/files/run_mitm.sh @@ -24,7 +24,7 @@ if [[ -f ${MITM_PORT_FILE} ]] ; then done if [[ -s ${MITM_PORT_FILE} ]] ; then MITM_PORT="$(cat ${MITM_PORT_FILE})" - if [[ ! -z "${MITM_PORT}" && ${MITM_PORT} -ne ${PORT} ]] ; then + if [[ -n "${MITM_PORT}" && ${MITM_PORT} -ne ${PORT} ]] ; then log "Our port (${PORT}) is not the primary mitm port (${MITM_PORT}), delaying startup by 30 seconds" sleep 30s fi diff --git a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml index 7e74193c..ca876c69 100644 --- a/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/mitmproxy/tasks/main.yml @@ -7,18 +7,30 @@ state: present update_cache: yes -- sysctl: - name: net.ipv4.ip_forward +- name: Set sysctl networking settings to allow IP forwarding + sysctl: + name: "{{ item }}" value: 1 sysctl_set: yes -- sysctl: - name: net.ipv6.conf.all.forwarding - value: 1 - sysctl_set: yes -- sysctl: + with_items: [ 'net.ipv4.ip_forward', 'net.ipv4.conf.all.forwarding', 'net.ipv6.conf.all.forwarding' ] + when: packer_builder_type != 'docker' + +- name: Verify sysctl networking settings are enabled in host to allow IP forwarding + shell: "if [[ $(sysctl -n {{ item }}) -ne 1 ]] ; then echo 'sysctl validation failed: {{ item }} = 1'; exit 1 ; fi" + with_items: [ 'net.ipv4.ip_forward', 'net.ipv4.conf.all.forwarding', 'net.ipv6.conf.all.forwarding' ] + when: packer_builder_type == 'docker' + +- name: Set sysctl setting net.ipv4.conf.all.send_redirects = 0 + sysctl: name: net.ipv4.conf.all.send_redirects value: 0 sysctl_set: yes + when: packer_builder_type != 'docker' + +- name: Verify sysctl networking settings are disabled in host to allow IP forwarding + shell: "if [[ $(sysctl -n {{ item }}) -ne 0 ]] ; then echo 'sysctl validation failed: {{ item }} = 0'; exit 1 ; fi" + with_items: [ 'net.ipv4.conf.all.send_redirects' ] + when: packer_builder_type == 'docker' - name: Create mitm user user: diff --git a/bubble-server/src/test/resources/models/system/cloudService.json b/bubble-server/src/test/resources/models/system/cloudService.json index 9779eb12..915211c4 100644 --- a/bubble-server/src/test/resources/models/system/cloudService.json +++ b/bubble-server/src/test/resources/models/system/cloudService.json @@ -133,9 +133,10 @@ "type": "docker", "image": "<>", "export_path": "<>.tar", + "cap_add": ["NET_ADMIN", "SYS_ADMIN", "SYS_MODULE"], "changes": [ "LABEL bubble_image=<>", - "EXPOSE 80 443 1202" + "EXPOSE 22 80 443 1080 1202 1443 8888 9999 53/udp 500/udp 4500/udp 51820/udp" ] }, "post": { diff --git a/docker/bubble.sh b/docker/bubble.sh index 053b8e70..70aea9c9 100755 --- a/docker/bubble.sh +++ b/docker/bubble.sh @@ -48,7 +48,7 @@ if [[ -z "${VERSION}" ]] ; then die "Error determining version from: ${META_FILE}" fi DOCKER_REPO="getbubble" -if [[ ! -z "${BUBBLE_DOCKER_REPO}" ]] ; then +if [[ -n "${BUBBLE_DOCKER_REPO}" ]] ; then DOCKER_REPO="${BUBBLE_DOCKER_REPO}" fi BUBBLE_TAG="${DOCKER_REPO}/launcher:${VERSION}" @@ -73,7 +73,7 @@ elif [[ "${MODE}" == "run" ]] ; then export LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL} " >> "${BUBBLE_ENV}" fi - if [[ ! -z "${BUBBLE_RUN_SLIM}" && "${BUBBLE_RUN_SLIM}" == "true" ]] ; then + if [[ -n "${BUBBLE_RUN_SLIM}" && "${BUBBLE_RUN_SLIM}" == "true" ]] ; then RUN_TAG="${BUBBLE_SLIM_TAG}" else RUN_TAG="${BUBBLE_TAG}" diff --git a/pom.xml b/pom.xml index 6cbd5cb4..30ace633 100644 --- a/pom.xml +++ b/pom.xml @@ -8,12 +8,13 @@ org.cobbzilla cobbzilla-parent - 1.0.0-SNAPSHOT + 2.0.1 bubble bubble - 1.0.0-SNAPSHOT + + 1.5.1 pom diff --git a/utils/abp-parser b/utils/abp-parser index 072a11de..7b5ff588 160000 --- a/utils/abp-parser +++ b/utils/abp-parser @@ -1 +1 @@ -Subproject commit 072a11decff65461f12f47e5dae763b56a5a3247 +Subproject commit 7b5ff5887a00ffd89a96f22329315e4867ed09c6 diff --git a/utils/cobbzilla-parent b/utils/cobbzilla-parent index 52e7a99a..8e7003de 160000 --- a/utils/cobbzilla-parent +++ b/utils/cobbzilla-parent @@ -1 +1 @@ -Subproject commit 52e7a99a6625f7e1db998179fa5a3cff90acb1a1 +Subproject commit 8e7003de03983a4a683702436b89469dac0ffe0a diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index e8914281..8647e5f4 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit e8914281714dd6d2ec2caee9dadc5fc500e16a26 +Subproject commit 8647e5f4429377e033b4605ae37c55717fef3e44 diff --git a/utils/cobbzilla-wizard b/utils/cobbzilla-wizard index 0bce26d2..50dbd434 160000 --- a/utils/cobbzilla-wizard +++ b/utils/cobbzilla-wizard @@ -1 +1 @@ -Subproject commit 0bce26d2cc10c4325a0a48d253463205609febe0 +Subproject commit 50dbd4340e4444916023e63d2d5e97469cc17de3 diff --git a/utils/templated-mail-sender b/utils/templated-mail-sender index f6755a30..19ea8b1d 160000 --- a/utils/templated-mail-sender +++ b/utils/templated-mail-sender @@ -1 +1 @@ -Subproject commit f6755a30ad8a2097a5bc439aa798fc8ef6dc6507 +Subproject commit 19ea8b1da6108ace0cde98a9202ad6301976ebc3