From 9c9e024893671be67e02af5bd4ae758cbf6eb8a7 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Thu, 4 Jun 2020 21:08:00 -0400 Subject: [PATCH] WIP: do not re-create identical images --- .../java/bubble/service/packer/PackerJob.java | 76 +++++++++++++------ 1 file changed, 54 insertions(+), 22 deletions(-) diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index dafe9f23..d9d26322 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -105,11 +105,6 @@ public class PackerJob implements Callable> { buildRegion = closestRegions.get(0); ctx.put(BUILD_REGION_VAR, buildRegion); - // create list of all regions, without leading/trailing double-quote, which should already be in the template - ctx.put(IMAGE_REGIONS_VAR, toInnerStringList(computeDriver.getRegions().stream() - .map(CloudRegion::getInternalName) - .collect(Collectors.toList()))); - // set environment variables final Map env = new HashMap<>(); for (NameAndValue variable : packerConfig.getVars()) { @@ -128,6 +123,43 @@ public class PackerJob implements Callable> { // write packer config file for (AnsibleInstallType installType : PACKER_TYPES) { + // check to see if we have packer images for all regions + final List existingImages = computeDriver.getPackerImages(); + if (!empty(existingImages)) { + final List existingForInstallType = existingImages.stream() + .filter(i -> i.getName().startsWith(PACKER_IMAGE_PREFIX+installType.name())) + .collect(Collectors.toList()); + if (!empty(existingForInstallType)) { + if (existingForInstallType.size() == 1 && existingForInstallType.get(0).getRegions() == null) { + // this image is for all regions + log.info("packer image already exists for "+installType+" for all regions"); + continue; + } else { + final List existingRegions = new ArrayList<>(); + for (PackerImage image : existingForInstallType) { + existingRegions.addAll(Arrays.asList(image.getRegions())); + } + log.info("packer images already exist for "+installType+" for regions: "+existingRegions.stream().map(CloudRegion::getInternalName).collect(Collectors.joining(", "))); + final List existingRegionNames = existingRegions.stream().map(CloudRegion::getInternalName).collect(Collectors.toList());; + // only create packer images for regions that are missing + final List imagesToCreate = computeDriver.getRegions().stream() + .filter(r -> !existingRegionNames.contains(r.getInternalName())) + .map(CloudRegion::getInternalName) + .collect(Collectors.toList()); + if (empty(imagesToCreate)) { + log.info("packer image already exists for "+installType+" for all regions"); + continue; + } + ctx.put(IMAGE_REGIONS_VAR, toInnerStringList(imagesToCreate)); + } + } + } else { + // create list of all regions, without leading/trailing double-quote, which should already be in the template + ctx.put(IMAGE_REGIONS_VAR, toInnerStringList(computeDriver.getRegions().stream() + .map(CloudRegion::getInternalName) + .collect(Collectors.toList()))); + } + final String imageName = PACKER_IMAGE_NAME_TEMPLATE .replace(INSTALL_TYPE_VAR, installType.name()) .replace(BUBBLE_VERSION_VAR, configuration.getVersion()) @@ -150,10 +182,10 @@ public class PackerJob implements Callable> { // write packer file final String packerJson = HandlebarsUtil.apply(configuration.getHandlebars(), packerConfigTemplate, ctx, '[', ']'); - toFileOrDie(abs(tempDir)+"/packer.json", packerJson); + toFileOrDie(abs(tempDir) + "/packer.json", packerJson); // run packer, return handle to running packer - log.info("running packer for "+installType+"..."); + log.info("running packer for " + installType + "..."); final CommandResult commandResult = CommandShell.exec(new Command(new CommandLine("packer") .addArgument("build") .addArgument("-parallel-builds=2") @@ -163,25 +195,25 @@ public class PackerJob implements Callable> { .setEnv(env) .setCopyToStandard(true)); - if (!commandResult.isZeroExitStatus()) { - images = computeDriver.finalizeIncompletePackerRun(commandResult, jarSha); + if (commandResult.isZeroExitStatus()) { + // read manifest, populate images + final File packerManifestFile = new File(tempDir, "manifest.json"); + if (!packerManifestFile.exists()) { + return die("Error executing packer: manifest file not found: " + abs(packerManifestFile)); + } + final PackerManifest packerManifest = json(FileUtil.toString(packerManifestFile), PackerManifest.class); + final PackerBuild[] builds = packerManifest.getBuilds(); + if (empty(builds)) { + return die("Error executing packer: no builds found"); + } + images.addAll(Arrays.stream(builds).map(b -> b.toPackerImage(imageName)).collect(Collectors.toList())); + + } else { + images.addAll(computeDriver.finalizeIncompletePackerRun(commandResult, jarSha)); if (empty(images)) { return die("Error executing packer: exit status " + commandResult.getExitStatus()); } - return images; - } - - // read manifest, populate images - final File packerManifestFile = new File(tempDir, "manifest.json"); - if (!packerManifestFile.exists()) { - return die("Error executing packer: manifest file not found: "+abs(packerManifestFile)); - } - final PackerManifest packerManifest = json(FileUtil.toString(packerManifestFile), PackerManifest.class); - final PackerBuild[] builds = packerManifest.getBuilds(); - if (empty(builds)) { - return die("Error executing packer: no builds found"); } - images.addAll(Arrays.stream(builds).map(b -> b.toPackerImage(imageName)).collect(Collectors.toList())); } return images;