multipart upload support with chunked encoding. enforce upload size limit.

bubble-server/src/main/java/bubble/main/http/BubbleHttpEntityOptions.java

@@ -11,8 +11,9 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.http.entity.ContentType;
 import org.kohsuke.args4j.Option;
 
-import static org.cobbzilla.util.daemon.ZillaRuntime.readStdin;
-import static org.cobbzilla.util.daemon.ZillaRuntime.shortError;
+import java.io.InputStream;
+
+import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
 import static org.cobbzilla.util.json.JsonUtil.*;
 
@@ -26,12 +27,22 @@ public class BubbleHttpEntityOptions extends BubbleHttpOptions {
             try {
                 return json(json(data, JsonNode.class, FULL_MAPPER_ALLOW_COMMENTS), COMPACT_MAPPER);
             } catch (Exception e) {
-                log.warn("getRequestJson: error scrubbing comments from JSON, sending as-is: "+shortError(e));
+                log.warn("getRequestJson: error scrubbing comments from JSON, sending as-is: " + shortError(e));
             }
         }
         return data;
     }
 
+    public InputStream getRequestStream() { return System.in; }
+
+    public static final String USAGE_MULTIPART = "Send PUT or POST as a multipart-encoded file upload with this file name";
+    public static final String OPT_MULTIPART = "-M";
+    public static final String LONGOPT_MULTIPART= "--multipart";
+    @Option(name=OPT_MULTIPART, aliases=LONGOPT_MULTIPART, usage=USAGE_MULTIPART)
+    @Getter @Setter private String multipartFileName = null;
+    public boolean hasMultipartFileName() { return !empty(multipartFileName); }
+
+
     public static final String USAGE_CONTENT_TYPE = "Content-Type to send. Default is application/json";
     public static final String OPT_CONTENT_TYPE = "-C";
     public static final String LONGOPT_CONTENT_TYPE= "--content-type";

bubble-server/src/main/java/bubble/main/http/BubbleHttpMain.java

@@ -6,6 +6,7 @@ package bubble.main.http;
 
 import bubble.main.BubbleApiMain;
 import org.apache.commons.io.IOUtils;
+import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.http.HttpRequestBean;
 import org.cobbzilla.util.string.Base64;
 import org.cobbzilla.wizard.api.ApiException;
@@ -43,14 +44,25 @@ public abstract class BubbleHttpMain<OPT extends BubbleHttpOptions> extends Bubb
         final String requestUrl = url.startsWith("/") ? url : "/" + url;
 
         if (options.isRaw()) {
-            final String entity = options instanceof BubbleHttpEntityOptions
-                    ? ((BubbleHttpEntityOptions) options).getRequestJson()
-                    : null;
-            final HttpRequestBean request = new HttpRequestBean(getMethod(), requestUrl, entity);
+            final HttpRequestBean request;
+            final BubbleHttpEntityOptions entityOptions = (options instanceof BubbleHttpEntityOptions) ? (BubbleHttpEntityOptions) options : null;
+            if (entityOptions != null) {
+                if (entityOptions.hasMultipartFileName()) {
+                    request = new HttpRequestBean(getMethod(), requestUrl, entityOptions.getRequestStream(), entityOptions.getMultipartFileName(), NameAndValue.EMPTY_ARRAY);
+                } else {
+                    request = new HttpRequestBean(getMethod(), requestUrl, entityOptions.getRequestJson());
+                }
+            } else {
+                request = new HttpRequestBean(getMethod(), requestUrl);
+            }
+
             if (options.hasHttpBasicUser()) request.setAuthUsername(options.getHttpBasicUser());
             if (options.hasHttpBasicPassword()) request.setAuthPassword(options.getHttpBasicPassword());
-
-            IOUtils.copyLarge(getApiClient().getStream(request), System.out);
+            if (entityOptions != null && entityOptions.hasMultipartFileName()) {
+                IOUtils.copyLarge(getApiClient().uploadMultipartStream(request, entityOptions.getMultipartFileName()), System.out);
+            } else {
+                IOUtils.copyLarge(getApiClient().getStream(request), System.out);
+            }
         } else {
             RestResponse response = null;
             try {

bubble-server/src/main/java/bubble/main/http/BubbleHttpOptions.java

@@ -9,6 +9,7 @@ import lombok.Getter;
 import lombok.Setter;
 import org.kohsuke.args4j.Option;
 
+import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
 public class BubbleHttpOptions extends BubbleApiOptionsBase {
@@ -26,11 +27,21 @@ public class BubbleHttpOptions extends BubbleApiOptionsBase {
     @Getter @Setter private String httpBasicUser;
     public boolean hasHttpBasicUser () { return !empty(httpBasicUser); }
 
-    public static final String USAGE_HTTP_PASS = "HTTP Basic Auth username";
+    public static final String USAGE_HTTP_PASS = "HTTP Basic Auth username. Use @ENV_VAR_NAME to read an env var";
     public static final String OPT_HTTP_PASS = "-W";
     public static final String LONGOPT_HTTP_PASS= "--password";
     @Option(name=OPT_HTTP_PASS, aliases=LONGOPT_HTTP_PASS, usage=USAGE_HTTP_PASS)
-    @Getter @Setter private String httpBasicPassword;
+    @Setter private String httpBasicPassword;
+    public String getHttpBasicPassword () {
+        if (!hasHttpBasicPassword()) return null;
+        if (httpBasicPassword.startsWith("@")) {
+            final String envVarName = httpBasicPassword.substring(1);
+            final String pass = System.getenv(envVarName);
+            if (empty(pass)) return die("getHttpBasicPassword: env var not defined: "+ envVarName);
+            return pass;
+        }
+        return httpBasicPassword;
+    }
     public boolean hasHttpBasicPassword () { return !empty(httpBasicPassword); }
 
     public static final String USAGE_RAW = "Raw response: do not parse as JSON";

bubble-server/src/main/java/bubble/resources/cloud/NodeManagerResource.java

@@ -12,9 +12,11 @@ import lombok.extern.slf4j.Slf4j;
 import net.lingala.zip4j.ZipFile;
 import net.lingala.zip4j.exception.ZipException;
 import org.cobbzilla.util.http.*;
+import org.cobbzilla.util.io.ByteLimitedInputStream;
 import org.cobbzilla.util.io.FileUtil;
 import org.cobbzilla.util.io.TempDir;
 import org.cobbzilla.util.string.Base64;
+import org.cobbzilla.util.system.Bytes;
 import org.cobbzilla.wizard.auth.LoginRequest;
 import org.glassfish.jersey.media.multipart.FormDataParam;
 import org.glassfish.jersey.server.ContainerRequest;
@@ -47,7 +49,10 @@ public class NodeManagerResource {
     public static final String ROOT_DIR_PREFIX = "root_dir/";
     public static final String COMPONENT_ROOT = "root";
     public static final Set<String> PATCH_COMPONENTS = new HashSet<>(Arrays.asList(new String[]{COMPONENT_ROOT, "bubble", "mitmproxy"}));
+
+    // other constants
     public static final String AUTH_BASIC_PREFIX = "Basic ";
+    public static final long MAX_PATCH_SIZE = 200 * Bytes.MB;
 
     private BubbleNode node;
 
@@ -170,7 +175,7 @@ public class NodeManagerResource {
                 .setMethod(HttpMethods.POST);
 
         // create a zipfile containing the file at the proper path
-        final File zipFile = buildPatchZip(component, path, in);
+        final File zipFile = buildPatchZip(component, path, new ByteLimitedInputStream(in, MAX_PATCH_SIZE));
 
         // register patch with NodeManagerService, receive URL
         final String url = nodeManagerService.registerPatch(zipFile);
@@ -186,10 +191,12 @@ public class NodeManagerResource {
         }
         if (path.startsWith("/")) path = path.substring(1);
         if (empty(path)) throw invalidEx("err.nodemanager.invalidPath");
-        final String fullPath = abs(tempDir)+"/"+path;
-        final File parentDir = mkdirOrDie(dirname(fullPath));
-        FileUtil.toFileOrDie(new File(parentDir, basename(path)), in);
+        final File dest = new File(abs(tempDir)+"/"+path);
+        mkdirOrDie(dirname(abs(dest)));
+        FileUtil.toFileOrDie(dest, in);
+        log.info("buildPatchZip: wrote temp file: "+abs(dest));
         final File zipFile = FileUtil.temp(".zip");
+        log.info("buildPatchZip: zipping into zipFile: "+abs(zipFile));
         try {
             new ZipFile(zipFile).addFolder(tempDir);
         } catch (ZipException e) {

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit 2b3bd69e5b3abf9644c975539beb8ba64c0c414c
+Subproject commit a168473a8855ea2b05e4afffdce84ae7f54f7dbe

utils/cobbzilla-wizard

@@ -1 +1 @@
-Subproject commit eb4581a2a654748b28ebf37f34e45b77967bc753
+Subproject commit 073dbbc3f8401e946b731e2269f6e253b04bee68