Merge branch 'master' of git.bubblev.org:bubblev/bubble into feature/flex_routing

bubble-server/src/main/java/bubble/resources/account/AuthResource.java

@@ -37,6 +37,7 @@ import bubble.service.cloud.GeoService;
 import bubble.service.notify.NotificationService;
 import bubble.service.upgrade.BubbleJarUpgradeService;
 import lombok.Cleanup;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.NameAndValue;
 import org.cobbzilla.util.security.RsaMessage;
@@ -45,16 +46,20 @@ import org.cobbzilla.wizard.validation.ConstraintViolationBean;
 import org.cobbzilla.wizard.validation.SimpleViolationException;
 import org.cobbzilla.wizard.validation.ValidationResult;
 import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.jersey.media.multipart.FormDataParam;
 import org.glassfish.jersey.server.ContainerRequest;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
+import javax.annotation.Nullable;
 import javax.validation.Valid;
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
@@ -177,29 +182,34 @@ public class AuthResource {
     @Autowired private SageHelloService sageHelloService;
     @Autowired private RestoreService restoreService;
 
-    @PUT @Path(EP_RESTORE+"/{restoreKey}")
-    public Response restore(@Context Request req,
-                            @Context ContainerRequest ctx,
-                            @PathParam("restoreKey") String restoreKey,
-                            @Valid NetworkKeys.EncryptedNetworkKeys encryptedKeys) {
+    @NonNull private BubbleNode checkRestoreRequest(@Nullable final String restoreKey) {
+        if (restoreKey == null) throw invalidEx("err.restoreKey.required");
 
         // ensure we have been initialized
         long start = now();
         while (!sageHelloService.sageHelloSuccessful() && (now() - start < NODE_INIT_TIMEOUT)) {
             sleep(SECONDS.toMillis(1), "restore: waiting for node initialization");
         }
-        if (!sageHelloService.sageHelloSuccessful()) {
-            return invalid("err.node.notInitialized");
-        }
+        if (!sageHelloService.sageHelloSuccessful()) throw invalidEx("err.node.notInitialized");
 
-        if (restoreKey == null) return invalid("err.restoreKey.required");
-        if (!restoreKey.equalsIgnoreCase(getRestoreKey())) return invalid("err.restoreKey.invalid");
+        if (!restoreKey.equalsIgnoreCase(getRestoreKey())) throw invalidEx("err.restoreKey.invalid");
 
         final BubbleNode thisNode = configuration.getThisNode();
-        if (!thisNode.hasSageNode()) return invalid("err.sageNode.required");
+        if (!thisNode.hasSageNode()) throw invalidEx("err.sageNode.required");
 
         final BubbleNode sageNode = nodeDAO.findByUuid(thisNode.getSageNode());
-        if (sageNode == null) return invalid("err.sageNode.notFound");
+        if (sageNode == null) throw invalidEx("err.sageNode.notFound");
+
+        return sageNode;
+    }
+
+    @POST @Path(EP_RESTORE+"/{restoreKey}")
+    public Response restore(@NonNull @Context final Request req,
+                            @NonNull @Context final ContainerRequest ctx,
+                            @Nullable @PathParam("restoreKey") final String restoreKey,
+                            @NonNull @Valid final NetworkKeys.EncryptedNetworkKeys encryptedKeys) {
+
+        final var sageNode = checkRestoreRequest(restoreKey);
 
         final NetworkKeys keys;
         try {
@@ -210,11 +220,33 @@ public class AuthResource {
         }
 
         restoreService.registerRestore(restoreKey, keys);
-        final NotificationReceipt receipt = notificationService.notify(sageNode, retrieve_backup, thisNode.setRestoreKey(getRestoreKey()));
+        final var receipt = notificationService.notify(sageNode, retrieve_backup,
+                                                       configuration.getThisNode().setRestoreKey(getRestoreKey()));
 
         return ok(receipt);
     }
 
+    @POST @Path(EP_RESTORE + EP_APPLY + "/{restoreKey}")
+    @Consumes(MULTIPART_FORM_DATA)
+    @NonNull public Response restoreFromPackage(@NonNull @Context final Request req,
+                                                @NonNull @Context final ContainerRequest ctx,
+                                                @NonNull @PathParam("restoreKey") final String restoreKey,
+                                                @NonNull @FormDataParam("file") final InputStream in,
+                                                @NonNull @FormDataParam("password") final String password) {
+        if (empty(password)) return invalid("err.password.required");
+
+        checkRestoreRequest(restoreKey);
+
+        restoreService.registerRestore(restoreKey, new NetworkKeys());
+
+        try {
+            if (restoreService.restoreFromPackage(restoreKey, in, password)) return ok();
+        } catch (IOException e) {
+            log.error("Exception while restoring from package", e);
+        }
+        return invalid("err.restore.failed", "Restore failed");
+    }
+
     @POST @Path(EP_REGISTER)
     public Response register(@Context Request req,
                              @Context ContainerRequest ctx,

bubble-server/src/main/java/bubble/server/BubbleConfiguration.java

@@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.github.jknack.handlebars.Handlebars;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
+import lombok.NonNull;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.map.DefaultedMap;
@@ -68,6 +69,7 @@ import static org.cobbzilla.util.io.FileUtil.abs;
 import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
 import static org.cobbzilla.util.security.ShaUtil.sha256_file;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
 import static org.cobbzilla.util.system.CommandShell.totalSystemMemory;
 import static org.cobbzilla.wizard.model.SemanticVersion.isNewerVersion;
 
@@ -419,4 +421,8 @@ public class BubbleConfiguration extends PgRestServerConfiguration
 
     @JsonIgnore @Getter @Setter private List<String> testCloudModels = Collections.emptyList();
 
+    public String opensslCmd(boolean encrypt, @NonNull final String passphrase) {
+        return "openssl enc " + (encrypt ? "" : "-d")
+               + " -aes-256-cbc -pbkdf2 -iter 10000 -pass pass:" + sha256_hex(passphrase);
+    }
 }

bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java

@@ -32,7 +32,6 @@ import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 import static org.cobbzilla.util.io.FileUtil.abs;
 import static org.cobbzilla.util.io.FileUtil.createTempDir;
 import static org.cobbzilla.util.json.JsonUtil.json;
-import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
 import static org.cobbzilla.util.system.CommandShell.execScript;
 import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
@@ -92,8 +91,7 @@ public class NetworkKeysService {
                 final var backupPackageAbs = abs(createTempFile("backup-", ".tgz.enc"));
                 execScript("cd " + backupDirAbs
                            + " && tar -cz *"
-                           + " | openssl enc -aes-256-cbc -pbkdf2 -iter 10000 -pass pass:" + sha256_hex(passphrase)
-                           + " > " + backupPackageAbs);
+                           + " | " + configuration.opensslCmd(true, passphrase) + " > " + backupPackageAbs);
                 status.ok(backupPackageAbs);
             } catch (Exception e) {
                 status.fail(e.getMessage());

bubble-server/src/main/java/bubble/service/backup/RestoreService.java

@@ -10,25 +10,32 @@ import bubble.model.cloud.CloudCredentials;
 import bubble.model.cloud.CloudService;
 import bubble.model.cloud.NetworkKeys;
 import bubble.server.BubbleConfiguration;
+import lombok.Cleanup;
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.util.collection.NameAndValue;
+import org.cobbzilla.util.io.FileUtil;
+import org.cobbzilla.util.io.TempDir;
 import org.cobbzilla.util.system.Bytes;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.annotation.Nullable;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 
 import static bubble.ApiConstants.HOME_DIR;
 import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE;
 import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE_CREDENTIALS;
 import static java.util.concurrent.TimeUnit.HOURS;
 import static java.util.concurrent.TimeUnit.MINUTES;
-import static org.cobbzilla.util.io.FileUtil.abs;
-import static org.cobbzilla.util.io.FileUtil.touch;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.io.FileUtil.*;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.system.CommandShell.execScript;
 import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Service @Slf4j
@@ -69,18 +76,19 @@ public class RestoreService {
         return getRestoreKeys().keys("*").size() > 0 || RESTORE_MARKER_FILE.exists();
     }
 
-    public boolean restore(String restoreKey, BubbleBackup backup) {
+    public boolean restore(@NonNull final String restoreKey, @NonNull final BubbleBackup backup) {
         final String thisNodeUuid = configuration.getThisNode().getUuid();
         final String thisNetworkUuid = configuration.getThisNode().getNetwork();
         String lock = null;
         try {
             lock = getRestoreKeys().lock(thisNetworkUuid, RESTORE_LOCK_TIMEOUT, RESTORE_DEADLOCK_TIMEOUT);
 
-            final String keyJson = getRestoreKeys().get(restoreKey);
-            if (keyJson == null) {
-                log.error("restore: restoreKey not found: " + restoreKey);
-                return false;
-            }
+            final var restoreDirAbs = checkAndGetRestoreDirPath();
+            if (restoreDirAbs == null) return false;
+
+            final var keyJson = checkAndGetKeyJson(restoreKey);
+            if (keyJson == null) return false;
+
             final NetworkKeys networkKeys = json(keyJson, NetworkKeys.class);
             final String storageJson = NameAndValue.find(networkKeys.getKeys(), PARAM_STORAGE);
             final String credentialsJson = NameAndValue.find(networkKeys.getKeys(), PARAM_STORAGE_CREDENTIALS);
@@ -89,25 +97,73 @@ public class RestoreService {
                 return false;
             }
 
-            final String[] existingFiles = RESTORE_DIR.list();
-            final var restoreDirAbs = abs(RESTORE_DIR);
-            if (existingFiles != null && existingFiles.length > 0) {
-                log.error("restore: files already exist in " + restoreDirAbs + ", cannot restore");
-                return false;
-            }
-
             final var storageDriver = json(storageJson, CloudService.class)
                     .setCredentials(json(credentialsJson, CloudCredentials.class))
                     .getStorageDriver(configuration);
             try {
                 storageDriver.fetchFiles(thisNodeUuid, backup.getPath(), restoreDirAbs);
-                log.info("restore: notifying system to restore from backup at: " + restoreDirAbs);
-                touch(RESTORE_MARKER_FILE);
-                return true;
             } catch (IOException e) {
                 log.error("restore: error downloading backup ", e);
                 return false;
             }
+
+            log.info("restore: notifying system to restore from backup at: " + restoreDirAbs);
+            touch(RESTORE_MARKER_FILE);
+            return true;
+        } finally {
+            if (lock != null) {
+                getRestoreKeys().unlock(thisNetworkUuid, lock);
+            }
+        }
+    }
+
+    @Nullable private String checkAndGetKeyJson(@NonNull final String restoreKey) {
+        final String keyJson = getRestoreKeys().get(restoreKey);
+        if (keyJson == null) {
+            log.error("restore: restoreKey not found: " + restoreKey);
+            return null;
+        }
+        return keyJson;
+    }
+
+    @Nullable private String checkAndGetRestoreDirPath() {
+        final var existingFiles = RESTORE_DIR.list();
+        final var restoreDirAbs = abs(RESTORE_DIR);
+
+        if (!empty(existingFiles)) {
+            log.error("restore: files already exist in " + restoreDirAbs + ", cannot restore");
+            return null;
+        }
+
+        mkdirOrDie(RESTORE_DIR);
+
+        return restoreDirAbs;
+    }
+
+    public boolean restoreFromPackage(@NonNull final String restoreKey,
+                                      @NonNull final InputStream backupPackageIS,
+                                      @NonNull final String passphrase)  throws IOException {
+        final var thisNetworkUuid = configuration.getThisNode().getNetwork();
+        String lock = null;
+        try {
+            lock = getRestoreKeys().lock(thisNetworkUuid, RESTORE_LOCK_TIMEOUT, RESTORE_DEADLOCK_TIMEOUT);
+
+            final var restoreDirAbs = checkAndGetRestoreDirPath();
+            if (restoreDirAbs == null) return false;
+
+            if (checkAndGetKeyJson(restoreKey) == null) return false;
+
+            @Cleanup final var tempDir = new TempDir();
+            final var uploadedFile = new File(tempDir, "uploaded.tgz.enc");
+            FileUtil.toFileOrDie(uploadedFile, backupPackageIS);
+
+            execScript("cd " + abs(restoreDirAbs)
+                       + " && " + configuration.opensslCmd(false, passphrase) + " < " + abs(uploadedFile)
+                       + " | tar xzv");
+
+            log.info("restore: notifying system to restore from backup at: " + restoreDirAbs);
+            touch(RESTORE_MARKER_FILE);
+            return true;
         } finally {
             if (lock != null) {
                 getRestoreKeys().unlock(thisNetworkUuid, lock);

bubble-server/src/main/resources/ansible/roles/mitmproxy/templates/supervisor_mitmproxy.conf.j2

@@ -1,7 +1,7 @@
 
 [program:mitm{{ port }}]
-stdout_logfile = /home/mitmproxy/mitm{{ port }}-out.log
-stderr_logfile = /home/mitmproxy/mitm{{ port }}-err.log
+stdout_logfile = /var/log/bubble/mitm{{ port }}-out.log
+stderr_logfile = /var/log/bubble/mitm{{ port }}-err.log
 command=sudo -H -u mitmproxy bash -c "/home/mitmproxy/mitmproxy/run_mitm.sh {{ port }}"
 stopasgroup=true
 stopsignal=QUIT

bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node.conf.j2

@@ -51,4 +51,7 @@ server {
     if ($scheme != "https") {
         return 301 https://$host:{{ ssl_port }}$request_uri;
     }
+
+    error_log /var/log/bubble/nginx-error.log;
+    access_log /var/log/bubble/nginx-access.log;
 }

bubble-server/src/main/resources/ansible/roles/nginx/templates/site_node_alias.conf.j2

@@ -51,4 +51,7 @@ server {
     if ($scheme != "https") {
         return 301 https://$host:{{ ssl_port }}$request_uri;
     }
+
+    error_log /var/log/bubble/nginx-error.log;
+    access_log /var/log/bubble/nginx-access.log;
 }

bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage.conf.j2

@@ -49,4 +49,7 @@ server {
     if ($scheme != "https") {
         return 301 https://$host$request_uri;
     }
+
+    error_log /var/log/bubble/nginx-error.log;
+    access_log /var/log/bubble/nginx-access.log;
 }

bubble-server/src/main/resources/ansible/roles/nginx/templates/site_sage_alias.conf.j2

@@ -49,4 +49,7 @@ server {
     if ($scheme != "https") {
         return 301 https://$host$request_uri;
     }
+
+    error_log /var/log/bubble/nginx-error.log;
+    access_log /var/log/bubble/nginx-access.log;
 }

bubble-server/src/main/resources/messages

@@ -1 +1 @@
-Subproject commit e19810a2f1afe057824b4976183fa7dc7253fd0d
+Subproject commit d97340e367e1b15ab71dc1ccd564af3a0dc7a361

bubble-server/src/main/resources/packer/roles/bubble/tasks/main.yml

@@ -125,3 +125,10 @@
 - name: Install packer for sage node
   shell: su - bubble bash -c install_packer.sh
   when: install_type == 'sage'
+
+- name: Install tmp folders' cleaner cron
+  cron:
+    name: "Cleaning tmp folders"
+    special_time: "hourly"
+    user: "root"
+    job: "find /tmp ~bubble/tmp -mtime +1 -type f -delete && find /tmp ~bubble/tmp -mtime +1 -type d -empty -delete"

bubble-server/src/main/resources/packer/roles/bubble/tasks/postgresql.yml

@@ -29,3 +29,10 @@
     group: postgres
     mode: 0400
   when: install_type == 'sage'
+
+- name: Install notifications tables' cleaner cron
+  cron:
+    name: "Cleaning notifications tables"
+    special_time: "hourly"
+    user: "postgres"
+    job: "HOUR_AGO=$(date -d '1 month ago' +\"%s000\") && psql -d bubble -c \"DELETE FROM sent_notification WHERE mtime < ${HOUR_AGO}\" -c \"DELETE FROM received_notification WHERE mtime < ${HOUR_AGO}\""

bubble-web

@@ -1 +1 @@
-Subproject commit 58137b47c8488c172508eb74ee93b1529646513c
+Subproject commit 631284006bb691d3d6d88aa2c3544ebc6cfa2893