templatize packer file and playbook, use same template for sage and node

bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java

@@ -59,8 +59,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static bubble.ApiConstants.getRemoteHost;
-import static bubble.ApiConstants.newNodeHostname;
+import static bubble.ApiConstants.*;
 import static bubble.dao.bill.AccountPlanDAO.PURCHASE_DELAY;
 import static bubble.model.cloud.BubbleNode.TAG_ERROR;
 import static bubble.server.BubbleConfiguration.DEBUG_NODE_INSTALL_FILE;
@@ -282,7 +281,7 @@ public class StandardNetworkService implements NetworkService {
                 // add a newline after so keys appended later will be OK
                 toFile(sshPubKeyFile, "\n"+sshKey.getSshPublicKey()+"\n");
             }
-            ApiConstants.copyScripts(bubbleFilesDir);
+            copyScripts(bubbleFilesDir);
 
             // write self_node.json file
             writeFile(bubbleFilesDir, null, SELF_NODE_JSON, json(node

bubble-server/src/main/java/bubble/service/packer/PackerJob.java

@@ -33,10 +33,10 @@ import java.util.stream.Collectors;
 
 import static bubble.ApiConstants.copyScripts;
 import static bubble.model.cloud.RegionalServiceDriver.findClosestRegions;
+import static bubble.service.packer.PackerService.*;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
-import static org.cobbzilla.util.io.FileUtil.abs;
-import static org.cobbzilla.util.io.FileUtil.toFileOrDie;
+import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.io.StreamUtil.copyClasspathDirectory;
 import static org.cobbzilla.util.io.StreamUtil.stream2string;
 import static org.cobbzilla.util.json.JsonUtil.json;
@@ -47,16 +47,18 @@ public class PackerJob implements Callable<List<PackerImage>> {
 
     public static final AnsibleInstallType[] PACKER_TYPES = {AnsibleInstallType.sage, AnsibleInstallType.node};
     public static final String INSTALL_TYPE_VAR = "@@TYPE@@";
-    public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@";
-    public static final String JAR_SHA_VAR = "@@JAR_SHA256@@";
-    public static final String PACKER_TEMPLATE = "packer/packer-"+INSTALL_TYPE_VAR+".json.hbs";
+    public static final String PACKER_TEMPLATE = PACKER_DIR+"/packer.json.hbs";
     public static final String PACKER_IMAGE_NAME_VAR = "packerImageName";
     public static final String PACKER_IMAGE_PREFIX = "packer_bubble_";
+    public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@";
+    public static final String JAR_SHA_VAR = "@@JAR_SHA256@@";
     public static final String PACKER_IMAGE_NAME_TEMPLATE = PACKER_IMAGE_PREFIX+INSTALL_TYPE_VAR+"_"+BUBBLE_VERSION_VAR+"_"+JAR_SHA_VAR;
     public static final String VARIABLES_VAR = "packerVariables";
     public static final String BUILD_REGION_VAR = "buildRegion";
     public static final String IMAGE_REGIONS_VAR = "imageRegions";
     public static final String BUILDERS_VAR = "builders";
+    public static final String PACKER_PLAYBOOK_TEMPLATE = "packer-playbook.yml.hbs";
+    public static final String PACKER_PLAYBOOK = "packer-playbook.yml";
 
     @Autowired private BubbleConfiguration configuration;
     @Autowired private GeoService geoService;
@@ -167,8 +169,9 @@ public class PackerJob implements Callable<List<PackerImage>> {
                     .replace(JAR_SHA_VAR, jarSha);
             ctx.put(PACKER_IMAGE_NAME_VAR, imageName);
 
-            final String packerTemplatePath = PACKER_TEMPLATE.replace(INSTALL_TYPE_VAR, installType.name());
-            final String packerConfigTemplate = stream2string(packerTemplatePath);
+            final String packerConfigTemplate = stream2string(PACKER_TEMPLATE);
+            ctx.put("installType", installType.name());
+            ctx.put("roles", getRolesForInstallType(installType));
 
             final List<String> builderJsons = new ArrayList<>();
             if (packerConfig.iterateRegions()) {
@@ -181,6 +184,10 @@ public class PackerJob implements Callable<List<PackerImage>> {
             }
             ctx.put(BUILDERS_VAR, builderJsons);
 
+            // write playbook file
+            final String playbookTemplate = FileUtil.toString(abs(tempDir)+ "/" + PACKER_PLAYBOOK_TEMPLATE);
+            FileUtil.toFile(new File(tempDir, PACKER_PLAYBOOK), HandlebarsUtil.apply(configuration.getHandlebars(), playbookTemplate, ctx, '[', ']'));
+
             // write packer file
             final String packerJson = HandlebarsUtil.apply(configuration.getHandlebars(), packerConfigTemplate, ctx, '[', ']');
             toFileOrDie(abs(tempDir) + "/packer.json", packerJson);
@@ -220,6 +227,14 @@ public class PackerJob implements Callable<List<PackerImage>> {
         return images;
     }
 
+    private List<String> getRolesForInstallType(AnsibleInstallType installType) {
+        switch (installType) {
+            case sage: return SAGE_ROLES;
+            case node: return NODE_ROLES;
+            default: return die("getRolesForInstallType: invalid installType: "+installType);
+        }
+    }
+
     public void addAllRegions(ComputeServiceDriver computeDriver, Map<String, Object> ctx) {
         ctx.put(IMAGE_REGIONS_VAR, toInnerStringList(computeDriver.getRegions().stream()
                 .map(CloudRegion::getInternalName)

bubble-server/src/main/java/bubble/service/packer/PackerService.java

@@ -14,12 +14,27 @@ import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.stream.Collectors;
 
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
+import static org.cobbzilla.util.io.StreamUtil.stream2string;
+import static org.cobbzilla.util.string.StringUtil.splitAndTrim;
 
 @Service @Slf4j
 public class PackerService {
 
+    public static final String PACKER_DIR = "packer";
+
+    public static final List<String> SAGE_ROLES = splitAndTrim(stream2string(PACKER_DIR + "/sage-roles.txt"), "\n")
+            .stream().filter(s -> !empty(s)).collect(Collectors.toList());
+
+    public static final List<String> NODE_ROLES = splitAndTrim(stream2string(PACKER_DIR + "/sage-roles.txt"), "\n")
+            .stream().filter(s -> !empty(s)).collect(Collectors.toList());
+
+    // todo: change packer-sage.json.hbs to packer.json.hbs, insert required roles via handlebars
+    // todo: change packer-playbook.yml to packer-playbook.yml.hbs, insert required roles via handlebars
+
     private final Map<String, Future<List<PackerImage>>> activeJobs = new ConcurrentHashMap<>(16);
     private final Map<String, List<PackerImage>> completedJobs = new ConcurrentHashMap<>(16);
     private final ExecutorService pool = DaemonThreadFactory.fixedPool(5);

bubble-server/src/main/resources/packer/node-roles.txt

@@ -0,0 +1,7 @@
+common
+firewall
+nginx
+algo
+mitmproxy
+bubble
+bubble_finalizer

bubble-server/src/main/resources/packer/packer-sage-playbook.yml → bubble-server/src/main/resources/packer/packer-playbook.yml.hbs

@@ -18,8 +18,5 @@
       setup:
 
   roles:
-    - common
-    - firewall
-    - nginx
-    - bubble
-    - bubble_finalizer
+[[#each roles]]    - [[this]]
+[[/each]]

bubble-server/src/main/resources/packer/packer-sage.json.hbs → bubble-server/src/main/resources/packer/packer.json.hbs

@@ -20,8 +20,8 @@
     },
     {
       "type": "ansible-local",
-      "playbook_file": "packer-sage-playbook.yml",
-      "role_paths": ["roles/common", "roles/firewall", "roles/nginx", "roles/bubble", "roles/bubble_finalizer"],
+      "playbook_file": "packer-playbook.yml",
+      "role_paths": [ [[#each roles]]"roles/[[this]]"[[#unless @last]], [[/unless]][[/each]] ],
       "inventory_file": "hosts"
     }
   ],

bubble-server/src/main/resources/packer/sage-roles.txt

@@ -0,0 +1,5 @@
+common
+firewall
+nginx
+bubble
+bubble_finalizer