Add monitoring tools for PackerService to show status of running and completed image builds

bin/pack_bubble

@@ -4,11 +4,13 @@
 #
 # Create packer images for sage and/or node
 #
-# Usage:  pack_bubble [-node|-sage] [-cloud CloudName]
+# Usage:
 #
-#   -node             : only pack the node image, do not pack the sage
-#   -sage             : only pack the sage image, do not pack the node
-#   -cloud CloudName  : only pack for CloudName compute cloud, do not pack for all clouds
+#   pack_bubble [node|sage] [cloud CloudName]
+#
+#   node             : only pack the node image, do not pack the sage
+#   sage             : only pack the sage image, do not pack the node
+#   cloud CloudName  : only pack for CloudName compute cloud, do not pack for all clouds
 #
 SCRIPT="${0}"
 SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd)
@@ -16,10 +18,10 @@ SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd)
 
 if [[ -z "${1}" ]] ; then
   IMAGES="node sage"
-elif [[ "${1}" == "-node" ]] ; then
+elif [[ "${1}" == "node" ]] ; then
   IMAGES="node"
   shift
-elif [[ "${1}" == "-sage" ]] ; then
+elif [[ "${1}" == "sage" ]] ; then
   IMAGES="sage"
   shift
 fi
@@ -31,7 +33,7 @@ if [[ -z "${1}" ]] ; then
     die "Error reading compute cloud names from ${CLOUDS_URL}"
   fi
 
-elif [[ "${1}" == "-cloud" ]] ; then
+elif [[ "${1}" == "cloud" ]] ; then
   CLOUDS="${2}"
   if [[ -z "${CLOUDS}" ]] ; then
     die "No cloud name specified after -cloud"

bin/pack_status

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Bubble, Inc.  All rights reserved. For personal (non-commercial) use, see license: https://getbubblenow.com/bubble-license/
+#
+# Display active and completed packer jobs
+#
+# Usage:
+#
+#   pack_status [running|completed]
+#
+# If the first argument is 'running' then only the status of running jobs will be shown
+# If the first argument is 'completed' then only the status of completed jobs will be shown
+#
+# Based on your BUBBLE_USER, BUBBLE_PASS and BUBBLE_API environment variables, this command will
+# use the bubble API to display the current status of the PackerService
+#
+# It returns a JSON object in the form:
+# {
+#   "running": [
+#     {...job1...},
+#     {...job2...},
+#     ...
+#   ],
+#   "completed": {
+#     "cloud_key1": [ {...image1...}, {...image2...}, ... ],
+#     "cloud_key2": [ {...image1...}, {...image2...}, ... ],
+#     ...
+#   }
+# }
+#
+# In the above, "running" is an array of job summary objects
+# and "completed" is a key/value map where they key indicates a cloud,
+# and the value is an array of packer images that have completed
+#
+# If you pass the 'running' argument, only the array of running jobs will be printed
+# If you pass the 'completed' argument, only the map of cloud->image[] will be printed
+#
+SCRIPT="${0}"
+SCRIPT_DIR=$(cd $(dirname ${SCRIPT}) && pwd)
+. ${SCRIPT_DIR}/bubble_common
+
+if [[ -z "${1}" ]] ; then
+  bget me/packer
+elif [[ "${1}" == "running" ]] ; then
+  bget me/packer/running
+elif [[ "${1}" == "completed" ]] ; then
+  bget me/packer/completed
+else
+  echo "Unrecognized argument ${1}, expected 'running' or 'completed' (or nothing)"
+  exit 1
+fi

bubble-server/src/main/java/bubble/resources/account/AccountPromotionsResource.java

@@ -24,7 +24,7 @@ public class AccountPromotionsResource {
 
     @Autowired private PromotionService promoService;
 
-    private Account account;
+    private final Account account;
 
     public AccountPromotionsResource (Account account) { this.account = account; }
 

bubble-server/src/main/java/bubble/resources/account/MeResource.java

@@ -29,7 +29,6 @@ import bubble.service.account.StandardAccountMessageService;
 import bubble.service.account.StandardAuthenticatorService;
 import bubble.service.account.download.AccountDownloadService;
 import bubble.service.boot.BubbleModelSetupService;
-import bubble.service.boot.SageHelloService;
 import bubble.service.boot.StandardSelfNodeService;
 import bubble.service.cloud.NodeLaunchMonitor;
 import bubble.service.upgrade.BubbleJarUpgradeService;
@@ -398,6 +397,13 @@ public class MeResource {
         return ok(launchMonitor.listLaunchStatuses(caller.getUuid()));
     }
 
+    @Path(EP_PACKER)
+    public PackerResource getPackerResource(@Context Request req,
+                                            @Context ContainerRequest ctx) {
+        final Account caller = userPrincipal(ctx);
+        return configuration.subResource(PackerResource.class, caller);
+    }
+
     @Path(EP_PROMOTIONS)
     public AccountPromotionsResource getPromotionsResource(@Context Request req,
                                                            @Context ContainerRequest ctx) {

bubble-server/src/main/java/bubble/resources/cloud/ComputePackerResource.java

@@ -18,8 +18,9 @@ import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 
+import static bubble.resources.cloud.PackerResource.packerNotAllowedForUser;
 import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
-import static org.cobbzilla.wizard.resources.ResourceUtil.ok;
+import static org.cobbzilla.wizard.resources.ResourceUtil.*;
 
 @Consumes(APPLICATION_JSON)
 @Produces(APPLICATION_JSON)
@@ -39,6 +40,7 @@ public class ComputePackerResource {
     @GET
     public Response listImages(@Context Request req,
                                @Context ContainerRequest ctx) {
+        if (packerNotAllowedForUser(ctx)) return forbidden();
         final ComputeServiceDriver driver = cloud.getComputeDriver(configuration);
         return ok(driver.getAllPackerImages());
     }
@@ -47,6 +49,7 @@ public class ComputePackerResource {
     public Response writeImages(@Context Request req,
                                 @Context ContainerRequest ctx,
                                 @PathParam("type") AnsibleInstallType installType) {
+        if (packerNotAllowedForUser(ctx)) return forbidden();
         packer.writePackerImages(cloud, installType, null);
         return ok();
     }

bubble-server/src/main/java/bubble/resources/cloud/PackerResource.java

@@ -0,0 +1,65 @@
+package bubble.resources.cloud;
+
+import bubble.model.account.Account;
+import bubble.service.packer.PackerService;
+import lombok.extern.slf4j.Slf4j;
+import org.cobbzilla.util.collection.MapBuilder;
+import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.jersey.server.ContainerRequest;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+import static org.cobbzilla.wizard.resources.ResourceUtil.*;
+
+@Produces(APPLICATION_JSON)
+@Consumes(APPLICATION_JSON)
+@Slf4j
+public class PackerResource {
+
+    public static final String STATUS_RUNNING = "running";
+    public static final String STATUS_COMPLETED = "completed";
+
+    private final Account account;
+
+    public PackerResource(Account account) { this.account = account; }
+
+    @Autowired private PackerService packerService;
+
+    @GET
+    public Response listAllStatus(@Context Request req,
+                                  @Context ContainerRequest ctx) {
+        if (packerNotAllowedForUser(ctx)) return forbidden();
+        return ok(MapBuilder.build(new Object[][] {
+                {STATUS_RUNNING, packerService.getActiveSummary(account.getUuid()) },
+                {STATUS_COMPLETED, packerService.getCompletedSummary(account.getUuid()) }
+        }));
+    }
+
+    @GET @Path(STATUS_RUNNING)
+    public Response listRunningBuilds(@Context Request req,
+                                      @Context ContainerRequest ctx) {
+        if (packerNotAllowedForUser(ctx)) return forbidden();
+        return ok(packerService.getActiveSummary(account.getUuid()));
+    }
+
+    @GET @Path(STATUS_COMPLETED)
+    public Response listCompletedBuilds(@Context Request req,
+                                        @Context ContainerRequest ctx) {
+        if (packerNotAllowedForUser(ctx)) return forbidden();
+        return ok(packerService.getCompletedSummary(account.getUuid()));
+    }
+
+    public static boolean packerNotAllowedForUser(@Context ContainerRequest ctx) {
+        final Account caller = userPrincipal(ctx);
+        if (!caller.admin()) return true;
+        return false;
+    }
+
+}

bubble-server/src/main/java/bubble/service/packer/PackerJob.java

@@ -4,7 +4,6 @@
  */
 package bubble.service.packer;
 
-import bubble.ApiConstants;
 import bubble.cloud.CloudRegion;
 import bubble.cloud.CloudRegionRelative;
 import bubble.cloud.compute.ComputeConfig;
@@ -38,6 +37,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
@@ -92,6 +92,13 @@ public class PackerJob implements Callable<List<PackerImage>> {
     @Getter private final AnsibleInstallType installType;
     @Getter private final List<AtomicReference<List<PackerImage>>> imagesRefList = new ArrayList<>();
     @Getter private List<PackerImage> images = new ArrayList<>();
+    @Getter private final long ctime = now();
+
+    private final AtomicLong completedAt = new AtomicLong(0);
+    public Long getCompletedAt() {
+        long t = completedAt.get();
+        return t == 0 ? null : t;
+    }
 
     public PackerJob(CloudService cloud, AnsibleInstallType installType) {
         this.cloud = cloud;
@@ -121,10 +128,12 @@ public class PackerJob implements Callable<List<PackerImage>> {
     @Override public List<PackerImage> call() throws Exception {
         try {
             final List<PackerImage> images = _call();
+            completedAt.set(now());
             packerService.recordJobCompleted(this);
             return images;
 
         } catch (Exception e) {
+            completedAt.set(now());
             packerService.recordJobError(this, e);
             throw e;
         }

bubble-server/src/main/java/bubble/service/packer/PackerJobSummary.java

@@ -0,0 +1,35 @@
+package bubble.service.packer;
+
+import bubble.model.cloud.AnsibleInstallType;
+import bubble.model.cloud.CloudService;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.experimental.Accessors;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.now;
+import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+import static org.cobbzilla.util.time.TimeUtil.formatDuration;
+
+@NoArgsConstructor @Accessors(chain=true)
+public class PackerJobSummary {
+
+    @Getter private CloudService cloud;
+    @Getter private AnsibleInstallType installType;
+    @Getter private long ctime;
+
+    private static final String[] CLOUD_SUMMARY_FIELDS = {"uuid", "name", "account"};
+
+    public PackerJobSummary (PackerJob job) {
+        this.cloud = new CloudService();
+        copy(this.cloud, job.getCloud(), CLOUD_SUMMARY_FIELDS);
+
+        this.installType = job.getInstallType();
+        this.ctime = job.getCtime();
+    }
+
+    // derived properties, useful when displaying PackerJobSummary as JSON via `pack_status`
+    public String getDuration () { return formatDuration(now() - getCtime()); }
+    public PackerJobSummary setDuration (String d) { return this; } // noop
+
+
+}

bubble-server/src/main/java/bubble/service/packer/PackerService.java

@@ -68,7 +68,10 @@ public class PackerService {
     }
 
     public static String cacheKey(CloudService cloud, AnsibleInstallType installType) {
-        return cloud.getUuid()+"_"+installType;
+        // note: only cloud.uuid and installType are needed for uniqueness in the cache key
+        // we add the account uuid so we can filter completed jobs based on account
+        // we add the cloud name so to make the key human-readable
+        return cloud.getAccount()+"_"+cloud.getUuid()+"_"+cloud.getName()+"_"+installType;
     }
 
     public void recordJobCompleted(PackerJob job) {
@@ -83,6 +86,22 @@ public class PackerService {
         activeJobs.remove(job.cacheKey());
     }
 
+    public List<PackerJobSummary> getActiveSummary(String accountUuid) {
+        synchronized (activeJobs) {
+            return activeJobs.values().stream()
+                    .filter(j -> j.getCloud().getAccount().equals(accountUuid))
+                    .map(PackerJobSummary::new)
+                    .collect(Collectors.toList());
+        }
+    }
+
+    public Map<String, List<PackerImage>> getCompletedSummary(String accountUuid) {
+        return completedJobs.entrySet()
+                .stream()
+                .filter(entry -> entry.getKey().contains(accountUuid))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+    }
+
     public File getPackerPublicKey () { return initPackerKey(true); }
     public File getPackerPrivateKey () { return initPackerKey(false); }
     public String getPackerPublicKeyHash () { return sha256_file(getPackerPublicKey()); }

bubble-server/src/main/resources/META-INF/bubble/bubble.properties

@@ -1 +1 @@
-bubble.version=Adventure 1.4.18
+bubble.version=Adventure 1.4.19

bubble-server/src/main/resources/ansible/bubble_scripts.txt

@@ -20,4 +20,6 @@ rkeys
 rmembers
 rdelkeys
 mitm_pid
-reset_bubble_logs
+reset_bubble_logs
+pack_bubble
+pack_status