store software versions in packer label

bubble-server/src/main/java/bubble/cloud/compute/PackerImageParserBase.java

@@ -8,18 +8,18 @@ import static bubble.service.packer.PackerJob.PACKER_IMAGE_PREFIX;
 
 public abstract class PackerImageParserBase extends ListResourceParser<PackerImage> {
 
-    private String bubbleVersion;
-    private String keyHash;
+    private final String bubbleVersion;
+    private final String versionHash;
 
-    public PackerImageParserBase(String bubbleVersion, String keyHash) {
+    public PackerImageParserBase(String bubbleVersion, String versionHash) {
         this.bubbleVersion = bubbleVersion;
-        this.keyHash = keyHash;
+        this.versionHash = versionHash;
     }
 
     public boolean isValidPackerImage(String name) {
         if (!name.startsWith(PACKER_IMAGE_PREFIX)) return false;
         if (!name.contains("_"+bubbleVersion+"_")) return false;
-        if (!name.contains("_"+keyHash+"_")) return false;
+        if (!name.contains("_"+versionHash+"_")) return false;
         return true;
     }
 

bubble-server/src/main/java/bubble/cloud/compute/digitalocean/DigitalOceanDriver.java

@@ -235,7 +235,7 @@ public class DigitalOceanDriver extends ComputeServiceDriverBase {
     @Override public List<PackerImage> getPackerImagesForRegion(String region) { return getPackerImages(); }
 
     public List<PackerImage> getPackerImages () {
-        final List<PackerImage> images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getShortVersion(), packerService.getPackerPublicKeyHash()));
+        final List<PackerImage> images = getResources(PACKER_IMAGES_URI, new DigitalOceanPackerImageParser(configuration.getShortVersion(), packerService.getPackerVersionHash()));
         return images == null ? Collections.emptyList() : images;
     }
 

bubble-server/src/main/java/bubble/cloud/compute/ec2/AmazonEC2Driver.java

@@ -183,7 +183,7 @@ public class AmazonEC2Driver extends ComputeServiceDriverBase {
         final ArrayList<Filter> filters = new ArrayList<>();
         filters.add(new Filter("root-device-type", new SingletonList<>("ebs")));
         filters.add(new Filter("state", new SingletonList<>("available")));
-        filters.add(new Filter("name", new SingletonList<>("packer_*_"+packerService.getPackerPublicKeyHash()+"_"+configuration.getShortVersion()+"_*")));
+        filters.add(new Filter("name", new SingletonList<>("packer_*_"+packerService.getPackerVersionHash()+"_"+configuration.getShortVersion()+"_*")));
         final AmazonEC2 ec2 = getEc2Client(region);
         final DescribeImagesRequest imageRequest = new DescribeImagesRequest().withFilters(filters);
         final DescribeImagesResult imagesResult = ec2.describeImages(imageRequest);

bubble-server/src/main/java/bubble/cloud/compute/vultr/VultrDriver.java

@@ -410,7 +410,7 @@ public class VultrDriver extends ComputeServiceDriverBase {
     @Override public List<PackerImage> getPackerImagesForRegion(String region) { return getPackerImages(); }
 
     public List<PackerImage> getPackerImages () {
-        final List<PackerImage> images = loadCloudResources(SNAPSHOT_URL, new VultrPackerImageParser(configuration.getShortVersion(), packerService.getPackerPublicKeyHash()));
+        final List<PackerImage> images = loadCloudResources(SNAPSHOT_URL, new VultrPackerImageParser(configuration.getShortVersion(), packerService.getPackerVersionHash()));
         return images == null ? Collections.emptyList() : images;
     }
 
@@ -424,16 +424,16 @@ public class VultrDriver extends ComputeServiceDriverBase {
         }
 
         // wait longer for the snapshot...
-        final String keyHash = packerService.getPackerPublicKeyHash();
+        final String versionHash = packerService.getPackerVersionHash();
         final long start = now();
         PackerImage snapshot = null;
         while (now() - start < SNAPSHOT_TIMEOUT) {
             snapshot = getPackerImages().stream()
-                    .filter(i -> i.getName().contains("_"+installType.name()+"_") && i.getName().contains(keyHash))
+                    .filter(i -> i.getName().contains("_"+installType.name()+"_") && i.getName().contains(versionHash))
                     .findFirst()
                     .orElse(null);
             if (snapshot != null) break;
-            sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+keyHash);
+            sleep(SECONDS.toMillis(20), "finalizeIncompletePackerRun: waiting for snapshot: "+versionHash);
         }
         if (snapshot == null) {
             log.error("finalizeIncompletePackerRun: timeout waiting for snapshot");
@@ -447,14 +447,14 @@ public class VultrDriver extends ComputeServiceDriverBase {
 
     public boolean stopImageServer(AnsibleInstallType installType) {
 
-        final String keyHash = packerService.getPackerPublicKeyHash();
+        final String versionHash = packerService.getPackerVersionHash();
         final List<BubbleNode> servers;
 
         // find the server(s)
         try {
             servers = listNodes(server -> {
                 final String tag = server.has(VULTR_TAG) ? server.get(VULTR_TAG).textValue() : null;
-                return tag != null && tag.contains("_"+installType.name()+"_") && tag.contains(keyHash);
+                return tag != null && tag.contains("_"+installType.name()+"_") && tag.contains(versionHash);
             });
         } catch (IOException e) {
             log.error("stopImageServer: error listing servers: "+shortError(e), e);

bubble-server/src/main/java/bubble/service/packer/PackerJob.java

@@ -33,6 +33,7 @@ import org.cobbzilla.util.time.TimeUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicReference;
@@ -62,12 +63,12 @@ public class PackerJob implements Callable<List<PackerImage>> {
 
     public static final String INSTALL_TYPE_VAR = "@@TYPE@@";
     public static final String SAGE_NET_VAR = "@@SAGE_NET@@";
-    public static final String PACKER_KEY_VAR = "@@PACKER_KEY_HASH@@";
+    public static final String PACKER_VERSION_HASH_VAR = "@@PACKER_VERSION_HASH@@";
     public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@";
     public static final String TIMESTAMP_VAR = "@@TIMESTAMP@@";
     public static final String PACKER_IMAGE_NAME_TEMPLATE = PACKER_IMAGE_PREFIX + INSTALL_TYPE_VAR
             + "_" + SAGE_NET_VAR
-            + "_" + PACKER_KEY_VAR
+            + "_" + PACKER_VERSION_HASH_VAR
             + "_" + BUBBLE_VERSION_VAR
             + "_" + TIMESTAMP_VAR;
 
@@ -158,18 +159,16 @@ public class PackerJob implements Callable<List<PackerImage>> {
         // copy ansible and other packer files to temp dir
         @Cleanup final TempDir tempDir = copyClasspathDirectory("packer");
 
-        final String releaseUrlBase = configuration.getReleaseUrlBase();
-        // create var for algo_sha256
-        final String algoVarsDir = abs(tempDir) + "/roles/algo/vars";
-        mkdirOrDie(algoVarsDir);
-        final String algoHash = url2string(releaseUrlBase+"/algo/latest/algo.zip.sha256");
-        FileUtil.toFileOrDie(new File(algoVarsDir, "main.yml"), "algo_sha256 : '"+algoHash+"'");
-
-        // create var for mitmproxy_sha256
-        final String mitmproxyVarsDir = abs(tempDir) + "/roles/mitmproxy/vars";
-        mkdirOrDie(mitmproxyVarsDir);
-        final String mitmproxyHash = url2string(releaseUrlBase+"/mitmproxy/latest/mitmproxy.zip.sha256");
-        FileUtil.toFileOrDie(new File(mitmproxyVarsDir, "main.yml"), "mitmproxy_sha256 : '"+mitmproxyHash+"'");
+        // for nodes, record versions of algo, mitmproxy and dnscrypt_proxy
+        if (installType == AnsibleInstallType.node) {
+            // ensure we use the latest algo and mitmproxy versions
+            final Map<String, String> versions = new HashMap<>();
+            versions.putAll(useLatestVersion(ROLE_ALGO, tempDir));
+            versions.putAll(useLatestVersion(ROLE_MITMPROXY, tempDir));
+
+            // write versions to bubble vars
+            writeBubbleVersions(tempDir, versions);
+        }
 
         // copy packer ssh key
         copyFile(packerService.getPackerPublicKey(), new File(abs(tempDir)+"/roles/common/files/"+PACKER_KEY_NAME));
@@ -226,7 +225,7 @@ public class PackerJob implements Callable<List<PackerImage>> {
         final String imageName = PACKER_IMAGE_NAME_TEMPLATE
                 .replace(INSTALL_TYPE_VAR, installType.name())
                 .replace(SAGE_NET_VAR, truncate(domainname(), 19))
-                .replace(PACKER_KEY_VAR, packerService.getPackerPublicKeyHash())
+                .replace(PACKER_VERSION_HASH_VAR, packerService.getPackerVersionHash())
                 .replace(BUBBLE_VERSION_VAR, configuration.getShortVersion())
                 .replace(TIMESTAMP_VAR, TimeUtil.format(now(), DATE_FORMAT_YYYYMMDDHHMMSS));
         if (imageName.length() > 128) return die("imageName.length > 128: "+imageName); // sanity check
@@ -296,6 +295,38 @@ public class PackerJob implements Callable<List<PackerImage>> {
         return images;
     }
 
+    private void writeBubbleVersions(TempDir tempDir, Map<String, String> versions) {
+        final File varsDir = mkdirOrDie(abs(tempDir) + "/roles/"+ROLE_BUBBLE+"/vars");
+        final StringBuilder b = new StringBuilder();
+        for (Map.Entry<String, String> var : versions.entrySet()) {
+            b.append(var.getKey()).append(" : '").append(var.getValue()).append("'\n");
+        }
+        FileUtil.toFileOrDie(new File(varsDir, "main.yml"), b.toString());
+    }
+
+    private Map<String, String> useLatestVersion(String roleName, TempDir tempDir) throws IOException {
+        final Map<String, String> vars = new HashMap<>();
+        final String releaseUrlBase = configuration.getReleaseUrlBase();
+        final File varsDir = mkdirOrDie(abs(tempDir) + "/roles/"+roleName+"/vars");
+
+        // determine latest version
+        final String version = packerService.getSoftwareVersion(roleName);
+        vars.put(roleName, version);
+
+        final String hash = url2string(releaseUrlBase+"/"+version+"/"+roleName+".zip.sha256");
+        String varsData = roleName+"_sha256 : '"+hash+"'\n"
+                + roleName+"_version : '" + version + "'\n";
+
+        if (roleName.equals(ROLE_ALGO)) {
+            // capture dnscrypt_proxy version for algo
+            final String dnscryptVersion = url2string(releaseUrlBase+"/"+version+"/dnscrypt-proxy_version.txt");
+            varsData += "dnscrypt_version : '"+dnscryptVersion+"'";
+            vars.put(ROLE_DNSCRYPT, dnscryptVersion);
+        }
+        FileUtil.toFileOrDie(new File(varsDir, "main.yml"), varsData);
+        return vars;
+    }
+
     private List<String> getRolesForInstallType(AnsibleInstallType installType) {
         switch (installType) {
             case sage: return SAGE_ROLES;

bubble-server/src/main/java/bubble/service/packer/PackerService.java

@@ -10,11 +10,12 @@ import bubble.model.cloud.CloudService;
 import bubble.server.BubbleConfiguration;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.daemon.DaemonThreadFactory;
-import org.cobbzilla.util.security.ShaUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -23,9 +24,11 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.http.HttpUtil.url2string;
 import static org.cobbzilla.util.io.FileUtil.abs;
 import static org.cobbzilla.util.io.FileUtil.mkdirOrDie;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
+import static org.cobbzilla.util.security.ShaUtil.*;
 import static org.cobbzilla.util.string.StringUtil.splitAndTrim;
 import static org.cobbzilla.util.system.CommandShell.chmod;
 import static org.cobbzilla.util.system.CommandShell.execScript;
@@ -40,6 +43,12 @@ public class PackerService {
 
     public static final List<String> NODE_ROLES = splitAndTrim(stream2string(PACKER_DIR + "/node-roles.txt"), "\n")
             .stream().filter(s -> !empty(s)).collect(Collectors.toList());
+
+    public static final String ROLE_ALGO = "algo";
+    public static final String ROLE_MITMPROXY = "mitmproxy";
+    public static final String ROLE_DNSCRYPT = "dnscrypt";
+    public static final String ROLE_BUBBLE = "bubble";
+
     public static final String PACKER_KEY_NAME = "packer_rsa";
 
     private final Map<String, PackerJob> activeJobs = new ConcurrentHashMap<>(16);
@@ -82,7 +91,17 @@ public class PackerService {
 
     public File getPackerPublicKey () { return initPackerKey(true); }
     public File getPackerPrivateKey () { return initPackerKey(false); }
-    public String getPackerPublicKeyHash () { return ShaUtil.sha256_file(getPackerPublicKey()); }
+    public String getPackerPublicKeyHash () { return sha256_file(getPackerPublicKey()); }
+
+    public String getPackerVersionHash () {
+        final String keyHash = getPackerPublicKeyHash();
+        final String versions = ""
+                +"_d"+getSoftwareVersion(ROLE_DNSCRYPT)
+                +"_a"+getSoftwareVersion(ROLE_ALGO)
+                +"_m"+getSoftwareVersion(ROLE_MITMPROXY);
+        if (versions.length() > 48) return die("getPackerVersionHash: software versions are too long (versions.length == "+versions.length()+" > 48): "+versions);
+        return keyHash.substring(64 - versions.length())+versions;
+    }
 
     public synchronized File initPackerKey(boolean pub) {
         final File keyDir = new File(System.getProperty("user.home"),".ssh");
@@ -98,4 +117,16 @@ public class PackerService {
         return pub ? pubKeyFile : privateKeyFile;
     }
 
+    private final Map<String, String> softwareVersions = new HashMap<>();
+    public String getSoftwareVersion(String roleName) {
+        final String releaseUrlBase = configuration.getReleaseUrlBase();
+        return softwareVersions.computeIfAbsent(roleName, r -> {
+            try {
+                return url2string(releaseUrlBase+"/"+r+"/latest.txt");
+            } catch (IOException e) {
+                return die("getSoftwareVersion("+r+"): "+shortError(e), e);
+            }
+        });
+    }
+
 }

bubble-server/src/main/resources/META-INF/bubble/bubble.properties

@@ -1 +1 @@
-bubble.version=Adventure 1.1.3
+bubble.version=Adventure 1.1.4

bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml

@@ -132,3 +132,12 @@
     special_time: "hourly"
     user: "root"
     job: "find /tmp ~bubble/tmp -mtime +1 -type f -delete && find /tmp ~bubble/tmp -mtime +1 -type d -empty -delete"
+
+- name: Record software versions
+  template:
+    src: bubble_versions.properties.j2
+    dest: /home/bubble/bubble_versions.properties
+    owner: bubble
+    group: bubble
+    mode: 0400
+  when: install_type == 'node'

bubble-server/src/main/resources/packer/roles/bubble/templates/bubble_versions.properties.j2

@@ -0,0 +1,3 @@
+algo_version={{ algo_version }}
+dnscrypt_version={{ dnscrypt_version }}
+mitmproxy_version={{ mitmproxy_version }}