From e18ac3f768b1f911c109d0d882e9c4b23c36b2e6 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Tue, 17 Nov 2020 09:31:16 -0500 Subject: [PATCH] Add monitoring tools for PackerService to show status of running and completed image builds --- bin/pack_bubble | 16 +++-- bin/pack_status | 51 +++++++++++++++ .../account/AccountPromotionsResource.java | 2 +- .../bubble/resources/account/MeResource.java | 8 ++- .../cloud/ComputePackerResource.java | 5 +- .../resources/cloud/PackerResource.java | 65 +++++++++++++++++++ .../java/bubble/service/packer/PackerJob.java | 11 +++- .../service/packer/PackerJobSummary.java | 35 ++++++++++ .../bubble/service/packer/PackerService.java | 21 +++++- .../META-INF/bubble/bubble.properties | 2 +- .../main/resources/ansible/bubble_scripts.txt | 4 +- 11 files changed, 206 insertions(+), 14 deletions(-) create mode 100755 bin/pack_status create mode 100644 bubble-server/src/main/java/bubble/resources/cloud/PackerResource.java create mode 100644 bubble-server/src/main/java/bubble/service/packer/PackerJobSummary.java diff --git a/bin/pack_bubble b/bin/pack_bubble index 5254b46a..135e167d 100755 --- a/bin/pack_bubble +++ b/bin/pack_bubble @@ -4,11 +4,13 @@ # # Create packer images for sage and/or node # -# Usage: pack_bubble [-node|-sage] [-cloud CloudName] +# Usage: # -# -node : only pack the node image, do not pack the sage -# -sage : only pack the sage image, do not pack the node -# -cloud CloudName : only pack for CloudName compute cloud, do not pack for all clouds +# pack_bubble [node|sage] [cloud CloudName] +# +# node : only pack the node image, do not pack the sage +# sage : only pack the sage image, do not pack the node +# cloud CloudName : only pack for CloudName compute cloud, do not pack for all clouds # SCRIPT="${0}" SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd) @@ -16,10 +18,10 @@ SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd) if [[ -z "${1}" ]] ; then IMAGES="node sage" -elif [[ "${1}" == "-node" ]] ; then +elif [[ "${1}" == "node" ]] ; then IMAGES="node" shift -elif [[ "${1}" == "-sage" ]] ; then +elif [[ "${1}" == "sage" ]] ; then IMAGES="sage" shift fi @@ -31,7 +33,7 @@ if [[ -z "${1}" ]] ; then die "Error reading compute cloud names from ${CLOUDS_URL}" fi -elif [[ "${1}" == "-cloud" ]] ; then +elif [[ "${1}" == "cloud" ]] ; then CLOUDS="${2}" if [[ -z "${CLOUDS}" ]] ; then die "No cloud name specified after -cloud" diff --git a/bin/pack_status b/bin/pack_status new file mode 100755 index 00000000..835324c4 --- /dev/null +++ b/bin/pack_status @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Copyright (c) 2020 Bubble, Inc. All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/ +# +# Display active and completed packer jobs +# +# Usage: +# +# pack_status [running|completed] +# +# If the first argument is 'running' then only the status of running jobs will be shown +# If the first argument is 'completed' then only the status of completed jobs will be shown +# +# Based on your BUBBLE_USER, BUBBLE_PASS and BUBBLE_API environment variables, this command will +# use the bubble API to display the current status of the PackerService +# +# It returns a JSON object in the form: +# { +# "running": [ +# {...job1...}, +# {...job2...}, +# ... +# ], +# "completed": { +# "cloud_key1": [ {...image1...}, {...image2...}, ... ], +# "cloud_key2": [ {...image1...}, {...image2...}, ... ], +# ... +# } +# } +# +# In the above, "running" is an array of job summary objects +# and "completed" is a key/value map where they key indicates a cloud, +# and the value is an array of packer images that have completed +# +# If you pass the 'running' argument, only the array of running jobs will be printed +# If you pass the 'completed' argument, only the map of cloud->image[] will be printed +# +SCRIPT="${0}" +SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd) +. ${SCRIPT_DIR}/bubble_common + +if [[ -z "${1}" ]] ; then + bget me/packer +elif [[ "${1}" == "running" ]] ; then + bget me/packer/running +elif [[ "${1}" == "completed" ]] ; then + bget me/packer/completed +else + echo "Unrecognized argument ${1}, expected 'running' or 'completed' (or nothing)" + exit 1 +fi diff --git a/bubble-server/src/main/java/bubble/resources/account/AccountPromotionsResource.java b/bubble-server/src/main/java/bubble/resources/account/AccountPromotionsResource.java index d28c792a..f1a56e51 100644 --- a/bubble-server/src/main/java/bubble/resources/account/AccountPromotionsResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/AccountPromotionsResource.java @@ -24,7 +24,7 @@ public class AccountPromotionsResource { @Autowired private PromotionService promoService; - private Account account; + private final Account account; public AccountPromotionsResource (Account account) { this.account = account; } diff --git a/bubble-server/src/main/java/bubble/resources/account/MeResource.java b/bubble-server/src/main/java/bubble/resources/account/MeResource.java index 2d52c1f9..f9ccdd9b 100644 --- a/bubble-server/src/main/java/bubble/resources/account/MeResource.java +++ b/bubble-server/src/main/java/bubble/resources/account/MeResource.java @@ -29,7 +29,6 @@ import bubble.service.account.StandardAccountMessageService; import bubble.service.account.StandardAuthenticatorService; import bubble.service.account.download.AccountDownloadService; import bubble.service.boot.BubbleModelSetupService; -import bubble.service.boot.SageHelloService; import bubble.service.boot.StandardSelfNodeService; import bubble.service.cloud.NodeLaunchMonitor; import bubble.service.upgrade.BubbleJarUpgradeService; @@ -398,6 +397,13 @@ public class MeResource { return ok(launchMonitor.listLaunchStatuses(caller.getUuid())); } + @Path(EP_PACKER) + public PackerResource getPackerResource(@Context Request req, + @Context ContainerRequest ctx) { + final Account caller = userPrincipal(ctx); + return configuration.subResource(PackerResource.class, caller); + } + @Path(EP_PROMOTIONS) public AccountPromotionsResource getPromotionsResource(@Context Request req, @Context ContainerRequest ctx) { diff --git a/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java b/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java index 286adf2c..b287afce 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java @@ -18,8 +18,9 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; +import static bubble.resources.cloud.PackerResource.packerNotAllowedForUser; import static javax.ws.rs.core.MediaType.APPLICATION_JSON; -import static org.cobbzilla.wizard.resources.ResourceUtil.ok; +import static org.cobbzilla.wizard.resources.ResourceUtil.*; @Consumes(APPLICATION_JSON) @Produces(APPLICATION_JSON) @@ -39,6 +40,7 @@ public class ComputePackerResource { @GET public Response listImages(@Context Request req, @Context ContainerRequest ctx) { + if (packerNotAllowedForUser(ctx)) return forbidden(); final ComputeServiceDriver driver = cloud.getComputeDriver(configuration); return ok(driver.getAllPackerImages()); } @@ -47,6 +49,7 @@ public class ComputePackerResource { public Response writeImages(@Context Request req, @Context ContainerRequest ctx, @PathParam("type") AnsibleInstallType installType) { + if (packerNotAllowedForUser(ctx)) return forbidden(); packer.writePackerImages(cloud, installType, null); return ok(); } diff --git a/bubble-server/src/main/java/bubble/resources/cloud/PackerResource.java b/bubble-server/src/main/java/bubble/resources/cloud/PackerResource.java new file mode 100644 index 00000000..553d1dc3 --- /dev/null +++ b/bubble-server/src/main/java/bubble/resources/cloud/PackerResource.java @@ -0,0 +1,65 @@ +package bubble.resources.cloud; + +import bubble.model.account.Account; +import bubble.service.packer.PackerService; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.collection.MapBuilder; +import org.glassfish.grizzly.http.server.Request; +import org.glassfish.jersey.server.ContainerRequest; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static org.cobbzilla.wizard.resources.ResourceUtil.*; + +@Produces(APPLICATION_JSON) +@Consumes(APPLICATION_JSON) +@Slf4j +public class PackerResource { + + public static final String STATUS_RUNNING = "running"; + public static final String STATUS_COMPLETED = "completed"; + + private final Account account; + + public PackerResource(Account account) { this.account = account; } + + @Autowired private PackerService packerService; + + @GET + public Response listAllStatus(@Context Request req, + @Context ContainerRequest ctx) { + if (packerNotAllowedForUser(ctx)) return forbidden(); + return ok(MapBuilder.build(new Object[][] { + {STATUS_RUNNING, packerService.getActiveSummary(account.getUuid()) }, + {STATUS_COMPLETED, packerService.getCompletedSummary(account.getUuid()) } + })); + } + + @GET @Path(STATUS_RUNNING) + public Response listRunningBuilds(@Context Request req, + @Context ContainerRequest ctx) { + if (packerNotAllowedForUser(ctx)) return forbidden(); + return ok(packerService.getActiveSummary(account.getUuid())); + } + + @GET @Path(STATUS_COMPLETED) + public Response listCompletedBuilds(@Context Request req, + @Context ContainerRequest ctx) { + if (packerNotAllowedForUser(ctx)) return forbidden(); + return ok(packerService.getCompletedSummary(account.getUuid())); + } + + public static boolean packerNotAllowedForUser(@Context ContainerRequest ctx) { + final Account caller = userPrincipal(ctx); + if (!caller.admin()) return true; + return false; + } + +} diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java index eaa0053b..be3176ee 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerJob.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJob.java @@ -4,7 +4,6 @@ */ package bubble.service.packer; -import bubble.ApiConstants; import bubble.cloud.CloudRegion; import bubble.cloud.CloudRegionRelative; import bubble.cloud.compute.ComputeConfig; @@ -38,6 +37,7 @@ import java.io.File; import java.io.IOException; import java.util.*; import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -92,6 +92,13 @@ public class PackerJob implements Callable> { @Getter private final AnsibleInstallType installType; @Getter private final List>> imagesRefList = new ArrayList<>(); @Getter private List images = new ArrayList<>(); + @Getter private final long ctime = now(); + + private final AtomicLong completedAt = new AtomicLong(0); + public Long getCompletedAt() { + long t = completedAt.get(); + return t == 0 ? null : t; + } public PackerJob(CloudService cloud, AnsibleInstallType installType) { this.cloud = cloud; @@ -121,10 +128,12 @@ public class PackerJob implements Callable> { @Override public List call() throws Exception { try { final List images = _call(); + completedAt.set(now()); packerService.recordJobCompleted(this); return images; } catch (Exception e) { + completedAt.set(now()); packerService.recordJobError(this, e); throw e; } diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerJobSummary.java b/bubble-server/src/main/java/bubble/service/packer/PackerJobSummary.java new file mode 100644 index 00000000..28b929ec --- /dev/null +++ b/bubble-server/src/main/java/bubble/service/packer/PackerJobSummary.java @@ -0,0 +1,35 @@ +package bubble.service.packer; + +import bubble.model.cloud.AnsibleInstallType; +import bubble.model.cloud.CloudService; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; + +import static org.cobbzilla.util.daemon.ZillaRuntime.now; +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; +import static org.cobbzilla.util.time.TimeUtil.formatDuration; + +@NoArgsConstructor @Accessors(chain=true) +public class PackerJobSummary { + + @Getter private CloudService cloud; + @Getter private AnsibleInstallType installType; + @Getter private long ctime; + + private static final String[] CLOUD_SUMMARY_FIELDS = {"uuid", "name", "account"}; + + public PackerJobSummary (PackerJob job) { + this.cloud = new CloudService(); + copy(this.cloud, job.getCloud(), CLOUD_SUMMARY_FIELDS); + + this.installType = job.getInstallType(); + this.ctime = job.getCtime(); + } + + // derived properties, useful when displaying PackerJobSummary as JSON via `pack_status` + public String getDuration () { return formatDuration(now() - getCtime()); } + public PackerJobSummary setDuration (String d) { return this; } // noop + + +} diff --git a/bubble-server/src/main/java/bubble/service/packer/PackerService.java b/bubble-server/src/main/java/bubble/service/packer/PackerService.java index 5962aa03..0c274186 100644 --- a/bubble-server/src/main/java/bubble/service/packer/PackerService.java +++ b/bubble-server/src/main/java/bubble/service/packer/PackerService.java @@ -68,7 +68,10 @@ public class PackerService { } public static String cacheKey(CloudService cloud, AnsibleInstallType installType) { - return cloud.getUuid()+"_"+installType; + // note: only cloud.uuid and installType are needed for uniqueness in the cache key + // we add the account uuid so we can filter completed jobs based on account + // we add the cloud name so to make the key human-readable + return cloud.getAccount()+"_"+cloud.getUuid()+"_"+cloud.getName()+"_"+installType; } public void recordJobCompleted(PackerJob job) { @@ -83,6 +86,22 @@ public class PackerService { activeJobs.remove(job.cacheKey()); } + public List getActiveSummary(String accountUuid) { + synchronized (activeJobs) { + return activeJobs.values().stream() + .filter(j -> j.getCloud().getAccount().equals(accountUuid)) + .map(PackerJobSummary::new) + .collect(Collectors.toList()); + } + } + + public Map> getCompletedSummary(String accountUuid) { + return completedJobs.entrySet() + .stream() + .filter(entry -> entry.getKey().contains(accountUuid)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + public File getPackerPublicKey () { return initPackerKey(true); } public File getPackerPrivateKey () { return initPackerKey(false); } public String getPackerPublicKeyHash () { return sha256_file(getPackerPublicKey()); } diff --git a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties index bffe823d..d854dead 100644 --- a/bubble-server/src/main/resources/META-INF/bubble/bubble.properties +++ b/bubble-server/src/main/resources/META-INF/bubble/bubble.properties @@ -1 +1 @@ -bubble.version=Adventure 1.4.18 +bubble.version=Adventure 1.4.19 diff --git a/bubble-server/src/main/resources/ansible/bubble_scripts.txt b/bubble-server/src/main/resources/ansible/bubble_scripts.txt index 1a354f8b..297a78ac 100644 --- a/bubble-server/src/main/resources/ansible/bubble_scripts.txt +++ b/bubble-server/src/main/resources/ansible/bubble_scripts.txt @@ -20,4 +20,6 @@ rkeys rmembers rdelkeys mitm_pid -reset_bubble_logs \ No newline at end of file +reset_bubble_logs +pack_bubble +pack_status \ No newline at end of file