update

bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriver.java

@@ -10,22 +10,23 @@ import bubble.model.cloud.CloudService;
 import bubble.model.cloud.StorageMetadata;
 import bubble.notify.storage.StorageListing;
 import lombok.Cleanup;
+import lombok.NonNull;
 import org.apache.commons.io.IOUtils;
 import org.cobbzilla.util.error.ExceptionHandler;
 import org.cobbzilla.util.string.Base64;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
+import java.util.Arrays;
 
 import static bubble.ApiConstants.ROOT_NETWORK_UUID;
 import static java.util.UUID.randomUUID;
 import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
+import static org.cobbzilla.util.io.FileUtil.mkdirOrDie;
 import static org.cobbzilla.util.io.StreamUtil.toStringOrDie;
+import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE;
 import static org.cobbzilla.util.system.Sleep.sleep;
 
 public interface StorageServiceDriver extends CloudServiceDriver {
@@ -196,4 +197,37 @@ public interface StorageServiceDriver extends CloudServiceDriver {
     StorageListing list(String fromNode, String prefix) throws IOException;
     StorageListing listNext(String fromNode, String listingId) throws IOException;
 
+    @NonNull default void fetchFiles(@NonNull final String fromNodeUuid,
+                                     @NonNull final String fromPath,
+                                     @NonNull final String toDir)
+            throws IOException {
+        final var path = getPath(fromPath);
+        log.info("fetchFiles: downloading from path=" + path);
+
+        var listing = list(fromNodeUuid, path);
+        while (true) {
+            Arrays.stream(listing.getKeys())
+                  .parallel()
+                  .forEach(k -> {
+                      final var logMsgPrefix = "fetchFiles [" + k + "]: ";
+                      log.info(logMsgPrefix + "downloading file");
+                      final var file = new File(toDir, k);
+                      mkdirOrDie(file.getParentFile());
+                      try {
+                          @Cleanup final var in = read(fromNodeUuid, k);
+                          try (var out = new BufferedOutputStream(new FileOutputStream(file), BUFFER_SIZE)) {
+                              IOUtils.copyLarge(in, out);
+                          }
+                          log.info(logMsgPrefix + "successfully downloaded file");
+                      } catch (Exception e) {
+                          die(logMsgPrefix + "error downloading file", e);
+                      }
+                  });
+
+            if (!listing.isTruncated()) break;
+            listing = listNext(fromNodeUuid, listing.getListingId());
+        }
+        log.info("fetchFiles: full download successful");
+    }
+
 }

bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java

@@ -15,6 +15,7 @@ import bubble.model.cloud.StorageMetadata;
 import bubble.notify.storage.StorageListing;
 import lombok.Cleanup;
 import lombok.Getter;
+import lombok.NonNull;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.cobbzilla.util.io.FileUtil;
@@ -22,9 +23,7 @@ import org.cobbzilla.util.system.OneWayFlag;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import java.io.*;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.List;
 
 import static bubble.ApiConstants.HOME_DIR;
 import static bubble.ApiConstants.ROOT_NETWORK_UUID;
@@ -200,16 +199,17 @@ public class LocalStorageDriver extends CloudServiceDriverBase<LocalStorageConfi
 
     @Override public boolean rekey(String fromNode, CloudService newCloud) throws IOException { return notSupported("rekey"); }
 
-    @Override public StorageListing list(String fromNode, String prefix) throws IOException {
-        final BubbleNode from = getFromNode(fromNode);
-        final File file = keyFile(from, prefix);
-        final File rootFile = keyFile(from, "");
-        final List<String> keys = new ArrayList<>();
-        listFilesRecursively(file).forEach(f -> keys.add(abs(f).substring((int) rootFile.length())));
-        return new StorageListing()
-                .setListingId(null)
-                .setTruncated(false)
-                .setKeys(keys.toArray(new String[0]));
+    @Override @NonNull public StorageListing list(@NonNull final String fromNodeUuid, @NonNull final String prefix) {
+        final var from = getFromNode(fromNodeUuid);
+        final var rootPathLength = abs(keyFile(from, "")).length();
+        final var keys = listFilesRecursively(keyFile(from, prefix)).stream()
+                                                                    .filter(File::isFile)
+                                                                    .map(FileUtil::abs)
+                                                                    .map(path -> path.substring(rootPathLength))
+                                                                    .toArray(String[]::new);
+        return new StorageListing().setListingId(null)
+                                   .setTruncated(false)
+                                   .setKeys(keys);
     }
 
     @Override public StorageListing listNext(String fromNode, String listingId) throws IOException {

bubble-server/src/main/java/bubble/resources/cloud/BackupsResource.java

@@ -9,6 +9,7 @@ import bubble.model.account.Account;
 import bubble.model.cloud.BackupStatus;
 import bubble.model.cloud.BubbleBackup;
 import bubble.model.cloud.BubbleNetwork;
+import bubble.server.BubbleConfiguration;
 import bubble.service.backup.BackupCleanerService;
 import bubble.service.backup.BackupService;
 import org.glassfish.jersey.server.ContainerRequest;
@@ -36,6 +37,7 @@ public class BackupsResource {
         this.network = network;
     }
 
+    @Autowired private BubbleConfiguration configuration;
     @Autowired private BubbleBackupDAO backupDAO;
     @Autowired private BackupService backupService;
     @Autowired private BackupCleanerService backupCleanerService;

bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java

@@ -5,28 +5,25 @@
 package bubble.resources.cloud;
 
 import bubble.dao.account.AccountPolicyDAO;
-import bubble.dao.account.message.AccountMessageDAO;
 import bubble.dao.bill.AccountPlanDAO;
 import bubble.dao.cloud.BubbleDomainDAO;
 import bubble.dao.cloud.BubbleNetworkDAO;
 import bubble.dao.cloud.BubbleNodeDAO;
 import bubble.model.account.Account;
 import bubble.model.account.AccountPolicy;
-import bubble.model.account.message.AccountAction;
-import bubble.model.account.message.AccountMessage;
-import bubble.model.account.message.AccountMessageType;
 import bubble.model.account.message.ActionTarget;
 import bubble.model.bill.AccountPlan;
-import bubble.model.cloud.*;
+import bubble.model.cloud.BubbleDomain;
+import bubble.model.cloud.BubbleNetwork;
+import bubble.model.cloud.BubbleNode;
+import bubble.model.cloud.NetLocation;
 import bubble.server.BubbleConfiguration;
 import bubble.service.account.StandardAuthenticatorService;
-import bubble.service.backup.NetworkKeysService;
 import bubble.service.cloud.NodeLaunchMonitor;
 import bubble.service.cloud.NodeProgressMeterTick;
 import bubble.service.cloud.StandardNetworkService;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.collection.NameAndValue;
-import org.cobbzilla.wizard.validation.ConstraintViolationBean;
 import org.cobbzilla.wizard.validation.ValidationResult;
 import org.glassfish.grizzly.http.server.Request;
 import org.glassfish.jersey.server.ContainerRequest;
@@ -38,7 +35,6 @@ import javax.ws.rs.core.Response;
 import java.util.List;
 
 import static bubble.ApiConstants.*;
-import static bubble.model.account.Account.validatePassword;
 import static bubble.model.cloud.BubbleNetwork.TAG_ALLOW_REGISTRATION;
 import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
 import static org.cobbzilla.wizard.resources.ResourceUtil.*;
@@ -51,9 +47,7 @@ public class NetworkActionsResource {
     @Autowired private BubbleNodeDAO nodeDAO;
     @Autowired private StandardNetworkService networkService;
     @Autowired private NodeLaunchMonitor launchMonitor;
-    @Autowired private AccountMessageDAO messageDAO;
     @Autowired private AccountPolicyDAO policyDAO;
-    @Autowired private NetworkKeysService keysService;
     @Autowired private BubbleDomainDAO domainDAO;
     @Autowired private BubbleNetworkDAO networkDAO;
     @Autowired private AccountPlanDAO accountPlanDAO;
@@ -110,50 +104,24 @@ public class NetworkActionsResource {
         return tick == null ? notFound(uuid) : ok(tick);
     }
 
-    @GET @Path(EP_KEYS)
-    public Response requestNetworkKeys(@Context Request req,
-                                       @Context ContainerRequest ctx) {
+    @Path(EP_KEYS)
+    @NonNull public NetworkBackupKeysResource getBackupKeys(@NonNull @Context final ContainerRequest ctx) {
         final Account caller = userPrincipal(ctx);
-        if (!caller.admin()) return forbidden();
+        if (!caller.admin()) throw forbiddenEx();
 
         // must request from the network you are on
         if (!network.getUuid().equals(configuration.getThisNetwork().getUuid())) {
-            return invalid("err.networkKeys.mustRequestFromSameNetwork");
+            throw invalidEx("err.networkKeys.mustRequestFromSameNetwork");
         }
 
         final AccountPolicy policy = policyDAO.findSingleByAccount(caller.getUuid());
         if (policy == null || !policy.hasVerifiedAccountContacts()) {
-            return invalid("err.networkKeys.noVerifiedContacts");
+            throw invalidEx("err.networkKeys.noVerifiedContacts");
         }
-        messageDAO.create(new AccountMessage()
-                .setMessageType(AccountMessageType.request)
-                .setAction(AccountAction.password)
-                .setTarget(ActionTarget.network)
-                .setAccount(caller.getUuid())
-                .setNetwork(configuration.getThisNetwork().getUuid())
-                .setName(network.getUuid())
-                .setRemoteHost(getRemoteHost(req)));
-        return ok();
-    }
-
-    @POST @Path(EP_KEYS+"/{uuid}")
-    public Response retrieveNetworkKeys(@Context Request req,
-                                        @Context ContainerRequest ctx,
-                                        @PathParam("uuid") String uuid,
-                                        NameAndValue enc) {
-        final Account caller = userPrincipal(ctx);
-        if (!caller.admin()) return forbidden();
 
         authenticatorService.ensureAuthenticated(ctx, ActionTarget.network);
 
-        final String encryptionKey = enc == null ? null : enc.getValue();
-        final ConstraintViolationBean error = validatePassword(encryptionKey);
-        if (error != null) return invalid(error);
-
-        final NetworkKeys keys = keysService.retrieveKeys(uuid);
-        return keys == null
-                ? invalid("err.retrieveNetworkKeys.notFound")
-                : ok(keys.encrypt(encryptionKey));
+        return configuration.subResource(NetworkBackupKeysResource.class, caller, network);
     }
 
     @POST @Path(EP_STOP)

bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java

@@ -0,0 +1,121 @@
+package bubble.resources.cloud;
+
+import bubble.dao.account.message.AccountMessageDAO;
+import bubble.dao.cloud.BubbleBackupDAO;
+import bubble.model.account.Account;
+import bubble.model.account.message.AccountAction;
+import bubble.model.account.message.AccountMessage;
+import bubble.model.account.message.AccountMessageType;
+import bubble.model.account.message.ActionTarget;
+import bubble.model.cloud.BubbleNetwork;
+import bubble.service.backup.NetworkKeysService;
+import lombok.NonNull;
+import org.cobbzilla.util.collection.NameAndValue;
+import org.cobbzilla.wizard.stream.FileSendableResource;
+import org.cobbzilla.wizard.validation.ConstraintViolationBean;
+import org.cobbzilla.wizard.validation.SimpleViolationException;
+import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.jersey.server.ContainerRequest;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.Nullable;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.Response;
+import java.io.File;
+
+import static bubble.ApiConstants.*;
+import static bubble.model.account.Account.validatePassword;
+import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
+import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_OCTET_STREAM;
+import static org.cobbzilla.wizard.resources.ResourceUtil.*;
+
+/**
+ * Ensure that only network admin can access these calls, and only for current network. Such admin should have verified
+ * account contact, and should be authenticated as in `authenticatorService.ensureAuthenticated`.
+ */
+@Consumes(APPLICATION_JSON)
+@Produces(APPLICATION_JSON)
+public class NetworkBackupKeysResource {
+    @Autowired private AccountMessageDAO messageDAO;
+    @Autowired private BubbleBackupDAO backupDAO;
+    @Autowired private NetworkKeysService keysService;
+
+    private final Account adminCaller;
+    private final BubbleNetwork thisNetwork;
+
+    public NetworkBackupKeysResource(@NonNull final Account adminCaller, @NonNull final BubbleNetwork thisNetwork) {
+        this.adminCaller = adminCaller;
+        this.thisNetwork = thisNetwork;
+    }
+
+    @GET
+    @NonNull public Response requestNetworkKeys(@NonNull @Context final Request req,
+                                                @NonNull @Context final ContainerRequest ctx) {
+        messageDAO.create(new AccountMessage().setMessageType(AccountMessageType.request)
+                                              .setAction(AccountAction.password)
+                                              .setTarget(ActionTarget.network)
+                                              .setAccount(adminCaller.getUuid())
+                                              .setNetwork(thisNetwork.getUuid())
+                                              .setName(thisNetwork.getUuid())
+                                              .setRemoteHost(getRemoteHost(req)));
+        return ok();
+    }
+
+    @NonNull private String fetchAndCheckEncryptionKey(@Nullable final NameAndValue enc) {
+        final String encryptionKey = enc == null ? null : enc.getValue();
+        final ConstraintViolationBean error = validatePassword(encryptionKey);
+        if (error != null) throw new SimpleViolationException(error);
+        return encryptionKey;
+    }
+
+    @POST @Path("/{keysCode}")
+    @NonNull public Response retrieveNetworkKeys(@NonNull @Context final Request req,
+                                                 @NonNull @Context final ContainerRequest ctx,
+                                                 @NonNull @PathParam("keysCode") final String keysCode,
+                                                 @Nullable final NameAndValue enc) {
+        final var encryptionKey = fetchAndCheckEncryptionKey(enc);
+        final var networkKeys = keysService.retrieveKeys(keysCode);
+        return ok(networkKeys.encrypt(encryptionKey));
+    }
+
+    @POST @Path("/{keysCode}" + EP_BACKUPS + EP_START)
+    @NonNull public Response backupDownloadStart(@NonNull @Context final ContainerRequest ctx,
+                                                 @NonNull @PathParam("keysCode") final String keysCode,
+                                                 @NonNull @QueryParam("backupId") final String backupId,
+                                                 @Nullable final NameAndValue enc) {
+        final var passphrase = fetchAndCheckEncryptionKey(enc);
+        keysService.retrieveKeys(keysCode);
+
+        final var backup = backupDAO.findByNetworkAndId(thisNetwork.getUuid(), backupId);
+        if (backup == null) throw notFoundEx(backupId);
+
+        keysService.startBackupDownload(thisNetwork.getUuid(), backup, keysCode, passphrase);
+        keysService.backupDownloadStatus(keysCode);
+        return ok();
+    }
+
+    @GET @Path("/{keysCode}" + EP_BACKUPS + EP_STATUS)
+    @NonNull public Response backupDownloadStatus(@NonNull @Context final ContainerRequest ctx,
+                                                  @NonNull @PathParam("keysCode") final String keysCode) {
+        // not checking keys code now here. However, such key will be required in preparing/prepared backup downloads'
+        // mapping within restoreService.
+        return ok(keysService.backupDownloadStatus(keysCode));
+    }
+
+    @GET @Path("/{keysCode}" + EP_BACKUPS + EP_DOWNLOAD)
+    @Produces(APPLICATION_OCTET_STREAM)
+    @NonNull public Response backupDownload(@NonNull @Context final ContainerRequest ctx,
+                                            @NonNull @PathParam("keysCode") final String keysCode) {
+        final var status = keysService.backupDownloadStatus(keysCode);
+        if (!status.isDone()) return accepted();
+
+        keysService.clearBackupDownloadKey(keysCode);
+        final var outFileName = "backup-" + thisNetwork.getNickname() + ".tgz.enc";
+        final var backupArchiveFile = new File(status.getPackagePath());
+        return send(new FileSendableResource(backupArchiveFile).setContentType(APPLICATION_OCTET_STREAM)
+                                                               .setContentLength(backupArchiveFile.length())
+                                                               .setForceDownload(true)
+                                                               .setName(outFileName));
+    }
+}

bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java

@@ -5,20 +5,38 @@
 package bubble.service.backup;
 
 import bubble.dao.cloud.CloudServiceDAO;
+import bubble.model.cloud.BubbleBackup;
 import bubble.model.cloud.CloudService;
 import bubble.model.cloud.NetworkKeys;
 import bubble.server.BubbleConfiguration;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
 import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE;
 import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE_CREDENTIALS;
+import static java.io.File.createTempFile;
 import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.cobbzilla.util.daemon.ZillaRuntime.background;
+import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.io.FileUtil.abs;
+import static org.cobbzilla.util.io.FileUtil.createTempDir;
 import static org.cobbzilla.util.json.JsonUtil.json;
+import static org.cobbzilla.util.security.ShaUtil.sha256_hex;
+import static org.cobbzilla.util.system.CommandShell.execScript;
 import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
+import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx;
+import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx;
 
 @Service @Slf4j
 public class NetworkKeysService {
@@ -31,6 +49,9 @@ public class NetworkKeysService {
     @Autowired private RedisService redis;
     @Getter(lazy=true) private final RedisService networkPasswordTokens = redis.prefixNamespace(getClass().getSimpleName());
 
+    // keysCode -> status object
+    private static final Map<String, BackupPackagingStatus> activeBackupDownloads = new ConcurrentHashMap<>();
+
     public void registerView(String uuid) {
         final NetworkKeys keys = new NetworkKeys();
         final CloudService storage = cloudDAO.findByUuid(configuration.getThisNetwork().getStorage());
@@ -43,11 +64,87 @@ public class NetworkKeysService {
         getNetworkPasswordTokens().set(uuid, json(keys), EX, KEY_EXPIRATION);
     }
 
-    public NetworkKeys retrieveKeys(String uuid) {
+    @NonNull public NetworkKeys retrieveKeys(@NonNull final String uuid) {
         final String json = getNetworkPasswordTokens().get(uuid);
-        if (json == null) return null;
+        if (empty(json)) throw invalidEx("err.retrieveNetworkKeys.notFound");
         getNetworkPasswordTokens().del(uuid);
         return json(json, NetworkKeys.class);
     }
 
+    public void startBackupDownload(@NonNull final String nodeUuid, @NonNull final BubbleBackup backup,
+                                    @NonNull final String keysCode, @NonNull final String passphrase) {
+        final var storageServiceUuid = configuration.getThisNetwork().getStorage();
+        final var storageService = cloudDAO.findByUuid(storageServiceUuid);
+        if (storageService == null) throw notFoundEx(storageServiceUuid);
+        final var storageDriver = storageService.getStorageDriver(configuration);
+
+        final var status = new BackupPackagingStatus(backup.getUuid());
+        if (activeBackupDownloads.putIfAbsent(keysCode, status) != null) {
+            throw invalidEx("err.download.error", "Already started");
+        }
+
+        background(() -> {
+            File backupDir = null;
+            try {
+                backupDir = createTempDir("backup-");
+                final var backupDirAbs = abs(backupDir);
+                storageDriver.fetchFiles(nodeUuid, backup.getPath(), backupDirAbs);
+                final var backupPackageAbs = abs(createTempFile("backup-", ".tgz.enc"));
+                execScript("cd " + backupDirAbs
+                           + " && tar -cz *"
+                           + " | openssl enc -aes-256-cbc -pbkdf2 -iter 10000 -pass pass:" + sha256_hex(passphrase)
+                           + " > " + backupPackageAbs);
+                status.ok(backupPackageAbs);
+            } catch (Exception e) {
+                status.fail(e.getMessage());
+            } finally {
+                try {
+                    if (backupDir != null) deleteDirectory(backupDir);
+                } catch (IOException e) {
+                    log.error("Cannot delete tmp backup folder " + backupDir, e);
+                }
+            }
+        });
+    }
+
+    @NonNull public BackupPackagingStatus backupDownloadStatus(@NonNull final String keysCode) {
+        final var status = activeBackupDownloads.get(keysCode);
+        if (status == null) throw notFoundEx(keysCode);
+        if (status.hasError()) throw invalidEx("err.download.error", status.getError());
+        return status;
+    }
+
+    public void clearBackupDownloadKey(@NonNull final String keysCode) {
+        activeBackupDownloads.remove(keysCode);
+    }
+
+    public static class BackupPackagingStatus {
+        @Getter private boolean done;
+        @Getter private final String backupUuid;
+        @JsonIgnore @Getter private String packagePath;
+        @Getter private String error;
+        public boolean hasError() { return !empty(error); }
+
+        private BackupPackagingStatus(@NonNull final String backupUuid) {
+            this.done = false;
+            this.backupUuid = backupUuid;
+            this.packagePath = null;
+            this.error = null;
+        }
+
+        private BackupPackagingStatus ok(@NonNull final String packagePath) {
+            this.done = true;
+            this.packagePath = packagePath;
+            this.error = null;
+            return this;
+        }
+
+        private BackupPackagingStatus fail(@NonNull final String error) {
+            this.done = true;
+            this.packagePath = null;
+            this.error = error;
+            return this;
+        }
+    }
+
 }

bubble-server/src/main/java/bubble/service/backup/RestoreService.java

@@ -4,36 +4,31 @@
  */
 package bubble.service.backup;
 
-import bubble.cloud.storage.StorageServiceDriver;
+import bubble.dao.cloud.CloudServiceDAO;
 import bubble.model.cloud.BubbleBackup;
 import bubble.model.cloud.CloudCredentials;
 import bubble.model.cloud.CloudService;
 import bubble.model.cloud.NetworkKeys;
-import bubble.notify.storage.StorageListing;
 import bubble.server.BubbleConfiguration;
-import lombok.Cleanup;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.io.IOUtils;
 import org.cobbzilla.util.collection.NameAndValue;
-import org.cobbzilla.util.io.TempDir;
+import org.cobbzilla.util.system.Bytes;
 import org.cobbzilla.wizard.cache.redis.RedisService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
-import java.io.*;
-import java.util.Arrays;
+import java.io.File;
+import java.io.IOException;
 
 import static bubble.ApiConstants.HOME_DIR;
 import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE;
 import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE_CREDENTIALS;
 import static java.util.concurrent.TimeUnit.HOURS;
 import static java.util.concurrent.TimeUnit.MINUTES;
-import static org.apache.commons.io.FileUtils.copyDirectory;
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-import static org.cobbzilla.util.io.FileUtil.*;
+import static org.cobbzilla.util.io.FileUtil.abs;
+import static org.cobbzilla.util.io.FileUtil.touch;
 import static org.cobbzilla.util.json.JsonUtil.json;
-import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE;
 import static org.cobbzilla.wizard.cache.redis.RedisService.EX;
 
 @Service @Slf4j
@@ -54,6 +49,10 @@ public class RestoreService {
     private static final long RESTORE_DEADLOCK_TIMEOUT = MINUTES.toMillis(30);
     private static final File RESTORE_MARKER_FILE = new File(HOME_DIR, ".restore");
 
+    private static final int BACKUP_ARCHIVE_MANAGEMENT_BUFFER_SIZE = (int) (8 * Bytes.KB);
+
+    @Autowired private CloudServiceDAO cloudDAO;
+
     @Autowired private RedisService redis;
     @Getter(lazy=true) private final RedisService restoreKeys = redis.prefixNamespace(getClass().getSimpleName());
 
@@ -89,49 +88,24 @@ public class RestoreService {
                 log.error("restore: storage/credentials not found in NetworkKeys");
                 return false;
             }
-            final CloudService storageService = json(storageJson, CloudService.class)
-                    .setCredentials(json(credentialsJson, CloudCredentials.class));
-
-            final StorageServiceDriver storageDriver = storageService.getStorageDriver(configuration);
-            final String path = StorageServiceDriver.getPath(backup.getPath());
 
             final String[] existingFiles = RESTORE_DIR.list();
+            final var restoreDirAbs = abs(RESTORE_DIR);
             if (existingFiles != null && existingFiles.length > 0) {
-                log.error("restore: files already exist in " + abs(RESTORE_DIR) + ", cannot restore");
+                log.error("restore: files already exist in " + restoreDirAbs + ", cannot restore");
                 return false;
             }
 
-            log.info("restore: downloading backup from path=" + path);
+            final var storageDriver = json(storageJson, CloudService.class)
+                    .setCredentials(json(credentialsJson, CloudCredentials.class))
+                    .getStorageDriver(configuration);
             try {
-                @Cleanup TempDir temp = new TempDir();
-                StorageListing listing = storageDriver.list(thisNodeUuid, path);
-                while (true) {
-                    Arrays.stream(listing.getKeys()).forEach(k -> {
-                        log.info("restore: downloading file: " + k);
-                        final File file = new File(abs(temp) + "/" + k);
-                        mkdirOrDie(file.getParentFile());
-                        try {
-                            @Cleanup final InputStream in = storageDriver.read(thisNodeUuid, k);
-                            try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file), BUFFER_SIZE)) {
-                                IOUtils.copyLarge(in, out);
-                            }
-                            log.info("restore: successfully downloaded file: " + k);
-                        } catch (Exception e) {
-                            die("restore: error downloading file: " + k + ": " + e);
-                        }
-                    });
-                    if (!listing.isTruncated()) break;
-                    listing = storageDriver.listNext(thisNodeUuid, listing.getListingId());
-                }
-
-                // all successful, copy directory to a safe place
-                copyDirectory(temp, RESTORE_DIR);
-                log.info("restore: full download successful, notifying system to restore from backup at: "+abs(RESTORE_DIR));
+                storageDriver.fetchFiles(thisNodeUuid, backup.getPath(), restoreDirAbs);
+                log.info("restore: notifying system to restore from backup at: " + restoreDirAbs);
                 touch(RESTORE_MARKER_FILE);
                 return true;
-
             } catch (IOException e) {
-                log.error("restore: error downloading backup: " + e);
+                log.error("restore: error downloading backup ", e);
                 return false;
             }
         } finally {

bubble-server/src/test/resources/models/tests/network/simple_backup.json

@@ -68,4 +68,4 @@
       ]
     }
   }
-]
+]