diff --git a/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java b/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java index 3230efa4..0340d73e 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java @@ -8,18 +8,18 @@ import static bubble.service.packer.PackerJob.PACKER_IMAGE_PREFIX; public abstract class PackerImageParserBase extends ListResourceParser { - private String bubbleVersion; - private String keyHash; + private final String bubbleVersion; + private final String versionHash; - public PackerImageParserBase(String bubbleVersion, String keyHash) { + public PackerImageParserBase(String bubbleVersion, String versionHash) { this.bubbleVersion = bubbleVersion; - this.keyHash = keyHash; + this.versionHash = versionHash; } public boolean isValidPackerImage(String name) { if (!name.startsWith(PACKER_IMAGE_PREFIX)) return false; if (!name.contains("_"+bubbleVersion+"_")) return false; - if (!name.contains("_"+keyHash+"_")) return false; + if (!name.contains("_"+versionHash+"_")) return false; return true; } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java index 5b950d2f..b312bf61 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java @@ -235,7 +235,7 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase { @Override public List getPackerImagesForRegion(String region) { return getPackerImages(); } public List getPackerImages () { - final List images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getShortVersion(), packerService.getPackerPublicKeyHash())); + final List images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getShortVersion(), packerService.getPackerVersionHash())); return images == null ? Collections.emptyList() : images; } diff --git a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java index 3e13f559..b2b64c78 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java @@ -183,7 +183,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase { final ArrayList filters = new ArrayList<>(); filters.add(new Filter("root-device-type", new SingletonList<>("ebs"))); filters.add(new Filter("state", new SingletonList<>("available"))); - filters.add(new Filter("name", new SingletonList<>("packer_*_"+packerService.getPackerPublicKeyHash()+"_"+configuration.getShortVersion()+"_*"))); + filters.add(new Filter("name", new SingletonList<>("packer_*_"+packerService.getPackerVersionHash()+"_"+configuration.getShortVersion()+"_*"))); final AmazonEC2 ec2 = getEc2Client(region); final DescribeImagesRequest imageRequest = new DescribeImagesRequest().withFilters(filters); final DescribeImagesResult imagesResult = ec2.describeImages(imageRequest); diff --git a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java index 2f498783..5165d39a 100644 --- a/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java @@ -410,7 +410,7 @@ public class VultrDriver extends ComputeServiceDriverBase { @Override public List getPackerImagesForRegion(String region) { return getPackerImages(); } public List getPackerImages () { - final List images = loadCloudResources(SNAPSHOT_URL, new VultrPackerImageParser(configuration.getShortVersion(), packerService.getPackerPublicKeyHash())); + final List images = loadCloudResources(SNAPSHOT_URL, new VultrPackerImageParser(configuration.getShortVersion(), packerService.getPackerVersionHash())); return images == null ? Collections.emptyList() : images; } @@ -424,16 +424,16 @@ public class VultrDriver extends ComputeServiceDriverBase { } // wait longer for the snapshot... - final String keyHash = packerService.getPackerPublicKeyHash(); + final String versionHash = packerService.getPackerVersionHash(); final long start = now(); PackerImage snapshot = null; while (now() - start < SNAPSHOT_TIMEOUT) { snapshot = getPackerImages().stream() - .filter(i -> i.getName().contains("_"+installType.name()+"_") && i.getName().contains(keyHash)) + .filter(i -> i.getName().contains("_"+installType.name()+"_") && i.getName().contains(versionHash)) .findFirst() .orElse(null); if (snapshot != null) break; - sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+keyHash); + sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+versionHash); } if (snapshot == null) { log.error("finalizeIncompletePackerRun: timeout waiting for snapshot"); @@ -447,14 +447,14 @@ public class VultrDriver extends ComputeServiceDriverBase { public boolean stopImageServer(AnsibleInstallType installType) { - final String keyHash = packerService.getPackerPublicKeyHash(); + final String versionHash = packerService.getPackerVersionHash(); final List servers; // find the server(s) try { servers = listNodes(server -> { final String tag = server.has(VULTR_TAG) ? server.get(VULTR_TAG).textValue() : null; - return tag != null && tag.contains("_"+installType.name()+"_") && tag.contains(keyHash); + return tag != null && tag.contains("_"+installType.name()+"_") && tag.contains(versionHash); }); } catch (IOException e) { log.error("stopImageServer: error listing servers: "+shortError(e), e); diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index 41516c2f..b137da18 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -33,6 +33,7 @@ import org.cobbzilla.util.time.TimeUtil; import org.springframework.beans.factory.annotation.Autowired; import java.io.File; +import java.io.IOException; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicReference; @@ -62,12 +63,12 @@ public class PackerJob implements Callable> { public static final String INSTALL_TYPE_VAR = "@@TYPE@@"; public static final String SAGE_NET_VAR = "@@SAGE_NET@@"; - public static final String PACKER_KEY_VAR = "@@PACKER_KEY_HASH@@"; + public static final String PACKER_VERSION_HASH_VAR = "@@PACKER_VERSION_HASH@@"; public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@"; public static final String TIMESTAMP_VAR = "@@TIMESTAMP@@"; public static final String PACKER_IMAGE_NAME_TEMPLATE = PACKER_IMAGE_PREFIX + INSTALL_TYPE_VAR + "_" + SAGE_NET_VAR - + "_" + PACKER_KEY_VAR + + "_" + PACKER_VERSION_HASH_VAR + "_" + BUBBLE_VERSION_VAR + "_" + TIMESTAMP_VAR; @@ -158,18 +159,16 @@ public class PackerJob implements Callable> { // copy ansible and other packer files to temp dir @Cleanup final TempDir tempDir = copyClasspathDirectory("packer"); - final String releaseUrlBase = configuration.getReleaseUrlBase(); - // create var for algo_sha256 - final String algoVarsDir = abs(tempDir) + "/roles/algo/vars"; - mkdirOrDie(algoVarsDir); - final String algoHash = url2string(releaseUrlBase+"/algo/latest/algo.zip.sha256"); - FileUtil.toFileOrDie(new File(algoVarsDir, "main.yml"), "algo_sha256 : '"+algoHash+"'"); - - // create var for mitmproxy_sha256 - final String mitmproxyVarsDir = abs(tempDir) + "/roles/mitmproxy/vars"; - mkdirOrDie(mitmproxyVarsDir); - final String mitmproxyHash = url2string(releaseUrlBase+"/mitmproxy/latest/mitmproxy.zip.sha256"); - FileUtil.toFileOrDie(new File(mitmproxyVarsDir, "main.yml"), "mitmproxy_sha256 : '"+mitmproxyHash+"'"); + // for nodes, record versions of algo, mitmproxy and dnscrypt_proxy + if (installType == AnsibleInstallType.node) { + // ensure we use the latest algo and mitmproxy versions + final Map versions = new HashMap<>(); + versions.putAll(useLatestVersion(ROLE_ALGO, tempDir)); + versions.putAll(useLatestVersion(ROLE_MITMPROXY, tempDir)); + + // write versions to bubble vars + writeBubbleVersions(tempDir, versions); + } // copy packer ssh key copyFile(packerService.getPackerPublicKey(), new File(abs(tempDir)+"/roles/common/files/"+PACKER_KEY_NAME)); @@ -226,7 +225,7 @@ public class PackerJob implements Callable> { final String imageName = PACKER_IMAGE_NAME_TEMPLATE .replace(INSTALL_TYPE_VAR, installType.name()) .replace(SAGE_NET_VAR, truncate(domainname(), 19)) - .replace(PACKER_KEY_VAR, packerService.getPackerPublicKeyHash()) + .replace(PACKER_VERSION_HASH_VAR, packerService.getPackerVersionHash()) .replace(BUBBLE_VERSION_VAR, configuration.getShortVersion()) .replace(TIMESTAMP_VAR, TimeUtil.format(now(), DATE_FORMAT_YYYYMMDDHHMMSS)); if (imageName.length() > 128) return die("imageName.length > 128: "+imageName); // sanity check @@ -296,6 +295,38 @@ public class PackerJob implements Callable> { return images; } + private void writeBubbleVersions(TempDir tempDir, Map versions) { + final File varsDir = mkdirOrDie(abs(tempDir) + "/roles/"+ROLE_BUBBLE+"/vars"); + final StringBuilder b = new StringBuilder(); + for (Map.Entry var : versions.entrySet()) { + b.append(var.getKey()).append(" : '").append(var.getValue()).append("'\n"); + } + FileUtil.toFileOrDie(new File(varsDir, "main.yml"), b.toString()); + } + + private Map useLatestVersion(String roleName, TempDir tempDir) throws IOException { + final Map vars = new HashMap<>(); + final String releaseUrlBase = configuration.getReleaseUrlBase(); + final File varsDir = mkdirOrDie(abs(tempDir) + "/roles/"+roleName+"/vars"); + + // determine latest version + final String version = packerService.getSoftwareVersion(roleName); + vars.put(roleName, version); + + final String hash = url2string(releaseUrlBase+"/"+version+"/"+roleName+".zip.sha256"); + String varsData = roleName+"_sha256 : '"+hash+"'\n" + + roleName+"_version : '" + version + "'\n"; + + if (roleName.equals(ROLE_ALGO)) { + // capture dnscrypt_proxy version for algo + final String dnscryptVersion = url2string(releaseUrlBase+"/"+version+"/dnscrypt-proxy_version.txt"); + varsData += "dnscrypt_version : '"+dnscryptVersion+"'"; + vars.put(ROLE_DNSCRYPT, dnscryptVersion); + } + FileUtil.toFileOrDie(new File(varsDir, "main.yml"), varsData); + return vars; + } + private List getRolesForInstallType(AnsibleInstallType installType) { switch (installType) { case sage: return SAGE_ROLES; diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerService.java b/bubble-server/src/main/java/bubble/service/packer/PackerService.java index 4272ce19..539898ea 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerService.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerService.java @@ -10,11 +10,12 @@ import bubble.model.cloud.CloudService; import bubble.server.BubbleConfiguration; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.util.daemon.DaemonThreadFactory; -import org.cobbzilla.util.security.ShaUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.File; +import java.io.IOException; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -23,9 +24,11 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.http.HttpUtil.url2string; import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.io.FileUtil.mkdirOrDie; import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.security.ShaUtil.*; import static org.cobbzilla.util.string.StringUtil.splitAndTrim; import static org.cobbzilla.util.system.CommandShell.chmod; import static org.cobbzilla.util.system.CommandShell.execScript; @@ -40,6 +43,12 @@ public class PackerService { public static final List NODE_ROLES = splitAndTrim(stream2string(PACKER_DIR + "/node-roles.txt"), "\n") .stream().filter(s -> !empty(s)).collect(Collectors.toList()); + + public static final String ROLE_ALGO = "algo"; + public static final String ROLE_MITMPROXY = "mitmproxy"; + public static final String ROLE_DNSCRYPT = "dnscrypt"; + public static final String ROLE_BUBBLE = "bubble"; + public static final String PACKER_KEY_NAME = "packer_rsa"; private final Map activeJobs = new ConcurrentHashMap<>(16); @@ -82,7 +91,17 @@ public class PackerService { public File getPackerPublicKey () { return initPackerKey(true); } public File getPackerPrivateKey () { return initPackerKey(false); } - public String getPackerPublicKeyHash () { return ShaUtil.sha256_file(getPackerPublicKey()); } + public String getPackerPublicKeyHash () { return sha256_file(getPackerPublicKey()); } + + public String getPackerVersionHash () { + final String keyHash = getPackerPublicKeyHash(); + final String versions = "" + +"_d"+getSoftwareVersion(ROLE_DNSCRYPT) + +"_a"+getSoftwareVersion(ROLE_ALGO) + +"_m"+getSoftwareVersion(ROLE_MITMPROXY); + if (versions.length() > 48) return die("getPackerVersionHash: software versions are too long (versions.length == "+versions.length()+" > 48): "+versions); + return keyHash.substring(64 - versions.length())+versions; + } public synchronized File initPackerKey(boolean pub) { final File keyDir = new File(System.getProperty("user.home"),".ssh"); @@ -98,4 +117,16 @@ public class PackerService { return pub ? pubKeyFile : privateKeyFile; } + private final Map softwareVersions = new HashMap<>(); + public String getSoftwareVersion(String roleName) { + final String releaseUrlBase = configuration.getReleaseUrlBase(); + return softwareVersions.computeIfAbsent(roleName, r -> { + try { + return url2string(releaseUrlBase+"/"+r+"/latest.txt"); + } catch (IOException e) { + return die("getSoftwareVersion("+r+"): "+shortError(e), e); + } + }); + } + } diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index 443625cc..15362a1d 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 1.1.3 +bubble.version=Adventure 1.1.4 diff --git a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml index efffb14d..9b22654a 100644 --- a/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml +++ b/bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml @@ -132,3 +132,12 @@ special_time: "hourly" user: "root" job: "find /tmp ~bubble/tmp -mtime +1 -type f -delete && find /tmp ~bubble/tmp -mtime +1 -type d -empty -delete" + +- name: Record software versions + template: + src: bubble_versions.properties.j2 + dest: /home/bubble/bubble_versions.properties + owner: bubble + group: bubble + mode: 0400 + when: install_type == 'node' diff --git a/bubble-server/src/main/resources/packer/roles/bubble/templates/bubble_versions.properties.j2 b/bubble-server/src/main/resources/packer/roles/bubble/templates/bubble_versions.properties.j2 new file mode 100644 index 00000000..16acdfe9 --- /dev/null +++ b/bubble-server/src/main/resources/packer/roles/bubble/templates/bubble_versions.properties.j2 @@ -0,0 +1,3 @@ +algo_version={{ algo_version }} +dnscrypt_version={{ dnscrypt_version }} +mitmproxy_version={{ mitmproxy_version }}