diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index bf503266..ef371114 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -59,8 +59,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; -import static bubble.ApiConstants.getRemoteHost; -import static bubble.ApiConstants.newNodeHostname; +import static bubble.ApiConstants.*; import static bubble.dao.bill.AccountPlanDAO.PURCHASE_DELAY; import static bubble.model.cloud.BubbleNode.TAG_ERROR; import static bubble.server.BubbleConfiguration.DEBUG_NODE_INSTALL_FILE; @@ -282,7 +281,7 @@ public class StandardNetworkService implements NetworkService { // add a newline after so keys appended later will be OK toFile(sshPubKeyFile, "\n"+sshKey.getSshPublicKey()+"\n"); } - ApiConstants.copyScripts(bubbleFilesDir); + copyScripts(bubbleFilesDir); // write self_node.json file writeFile(bubbleFilesDir, null, SELF_NODE_JSON, json(node diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index 6595e651..4882f924 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -33,10 +33,10 @@ import java.util.stream.Collectors; import static bubble.ApiConstants.copyScripts; import static bubble.model.cloud.RegionalServiceDriver.findClosestRegions; +import static bubble.service.packer.PackerService.*; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.io.FileUtil.abs; -import static org.cobbzilla.util.io.FileUtil.toFileOrDie; +import static org.cobbzilla.util.io.FileUtil.*; import static org.cobbzilla.util.io.StreamUtil.copyClasspathDirectory; import static org.cobbzilla.util.io.StreamUtil.stream2string; import static org.cobbzilla.util.json.JsonUtil.json; @@ -47,16 +47,18 @@ public class PackerJob implements Callable> { public static final AnsibleInstallType[] PACKER_TYPES = {AnsibleInstallType.sage, AnsibleInstallType.node}; public static final String INSTALL_TYPE_VAR = "@@TYPE@@"; - public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@"; - public static final String JAR_SHA_VAR = "@@JAR_SHA256@@"; - public static final String PACKER_TEMPLATE = "packer/packer-"+INSTALL_TYPE_VAR+".json.hbs"; + public static final String PACKER_TEMPLATE = PACKER_DIR+"/packer.json.hbs"; public static final String PACKER_IMAGE_NAME_VAR = "packerImageName"; public static final String PACKER_IMAGE_PREFIX = "packer_bubble_"; + public static final String BUBBLE_VERSION_VAR = "@@BUBBLE_VERSION@@"; + public static final String JAR_SHA_VAR = "@@JAR_SHA256@@"; public static final String PACKER_IMAGE_NAME_TEMPLATE = PACKER_IMAGE_PREFIX+INSTALL_TYPE_VAR+"_"+BUBBLE_VERSION_VAR+"_"+JAR_SHA_VAR; public static final String VARIABLES_VAR = "packerVariables"; public static final String BUILD_REGION_VAR = "buildRegion"; public static final String IMAGE_REGIONS_VAR = "imageRegions"; public static final String BUILDERS_VAR = "builders"; + public static final String PACKER_PLAYBOOK_TEMPLATE = "packer-playbook.yml.hbs"; + public static final String PACKER_PLAYBOOK = "packer-playbook.yml"; @Autowired private BubbleConfiguration configuration; @Autowired private GeoService geoService; @@ -167,8 +169,9 @@ public class PackerJob implements Callable> { .replace(JAR_SHA_VAR, jarSha); ctx.put(PACKER_IMAGE_NAME_VAR, imageName); - final String packerTemplatePath = PACKER_TEMPLATE.replace(INSTALL_TYPE_VAR, installType.name()); - final String packerConfigTemplate = stream2string(packerTemplatePath); + final String packerConfigTemplate = stream2string(PACKER_TEMPLATE); + ctx.put("installType", installType.name()); + ctx.put("roles", getRolesForInstallType(installType)); final List builderJsons = new ArrayList<>(); if (packerConfig.iterateRegions()) { @@ -181,6 +184,10 @@ public class PackerJob implements Callable> { } ctx.put(BUILDERS_VAR, builderJsons); + // write playbook file + final String playbookTemplate = FileUtil.toString(abs(tempDir)+ "/" + PACKER_PLAYBOOK_TEMPLATE); + FileUtil.toFile(new File(tempDir, PACKER_PLAYBOOK), HandlebarsUtil.apply(configuration.getHandlebars(), playbookTemplate, ctx, '[', ']')); + // write packer file final String packerJson = HandlebarsUtil.apply(configuration.getHandlebars(), packerConfigTemplate, ctx, '[', ']'); toFileOrDie(abs(tempDir) + "/packer.json", packerJson); @@ -220,6 +227,14 @@ public class PackerJob implements Callable> { return images; } + private List getRolesForInstallType(AnsibleInstallType installType) { + switch (installType) { + case sage: return SAGE_ROLES; + case node: return NODE_ROLES; + default: return die("getRolesForInstallType: invalid installType: "+installType); + } + } + public void addAllRegions(ComputeServiceDriver computeDriver, Map ctx) { ctx.put(IMAGE_REGIONS_VAR, toInnerStringList(computeDriver.getRegions().stream() .map(CloudRegion::getInternalName) diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerService.java b/bubble-server/src/main/java/bubble/service/packer/PackerService.java index 6d480af1..6802ad9a 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerService.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerService.java @@ -14,12 +14,27 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.stream.Collectors; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; import static org.cobbzilla.util.daemon.ZillaRuntime.shortError; +import static org.cobbzilla.util.io.StreamUtil.stream2string; +import static org.cobbzilla.util.string.StringUtil.splitAndTrim; @Service @Slf4j public class PackerService { + public static final String PACKER_DIR = "packer"; + + public static final List SAGE_ROLES = splitAndTrim(stream2string(PACKER_DIR + "/sage-roles.txt"), "\n") + .stream().filter(s -> !empty(s)).collect(Collectors.toList()); + + public static final List NODE_ROLES = splitAndTrim(stream2string(PACKER_DIR + "/sage-roles.txt"), "\n") + .stream().filter(s -> !empty(s)).collect(Collectors.toList()); + + // todo: change packer-sage.json.hbs to packer.json.hbs, insert required roles via handlebars + // todo: change packer-playbook.yml to packer-playbook.yml.hbs, insert required roles via handlebars + private final Map>> activeJobs = new ConcurrentHashMap<>(16); private final Map> completedJobs = new ConcurrentHashMap<>(16); private final ExecutorService pool = DaemonThreadFactory.fixedPool(5); diff --git a/bubble-server/src/main/resources/packer/node-roles.txt b/bubble-server/src/main/resources/packer/node-roles.txt new file mode 100644 index 00000000..53d3f934 --- /dev/null +++ b/bubble-server/src/main/resources/packer/node-roles.txt @@ -0,0 +1,7 @@ +common +firewall +nginx +algo +mitmproxy +bubble +bubble_finalizer \ No newline at end of file diff --git a/bubble-server/src/main/resources/packer/packer-sage-playbook.yml b/bubble-server/src/main/resources/packer/packer-playbook.yml.hbs similarity index 84% rename from bubble-server/src/main/resources/packer/packer-sage-playbook.yml rename to bubble-server/src/main/resources/packer/packer-playbook.yml.hbs index 14807d9c..bc13b18c 100644 --- a/bubble-server/src/main/resources/packer/packer-sage-playbook.yml +++ b/bubble-server/src/main/resources/packer/packer-playbook.yml.hbs @@ -18,8 +18,5 @@ setup: roles: - - common - - firewall - - nginx - - bubble - - bubble_finalizer +[[#each roles]] - [[this]] +[[/each]] diff --git a/bubble-server/src/main/resources/packer/packer-sage.json.hbs b/bubble-server/src/main/resources/packer/packer.json.hbs similarity index 80% rename from bubble-server/src/main/resources/packer/packer-sage.json.hbs rename to bubble-server/src/main/resources/packer/packer.json.hbs index c2f30752..b42ecf07 100644 --- a/bubble-server/src/main/resources/packer/packer-sage.json.hbs +++ b/bubble-server/src/main/resources/packer/packer.json.hbs @@ -20,8 +20,8 @@ }, { "type": "ansible-local", - "playbook_file": "packer-sage-playbook.yml", - "role_paths": ["roles/common", "roles/firewall", "roles/nginx", "roles/bubble", "roles/bubble_finalizer"], + "playbook_file": "packer-playbook.yml", + "role_paths": [ [[#each roles]]"roles/[[this]]"[[#unless @last]], [[/unless]][[/each]] ], "inventory_file": "hosts" } ], diff --git a/bubble-server/src/main/resources/packer/sage-roles.txt b/bubble-server/src/main/resources/packer/sage-roles.txt new file mode 100644 index 00000000..97011583 --- /dev/null +++ b/bubble-server/src/main/resources/packer/sage-roles.txt @@ -0,0 +1,5 @@ +common +firewall +nginx +bubble +bubble_finalizer \ No newline at end of file