From a60644e1b8165b891c31d89080862c07c2225d9d Mon Sep 17 00:00:00 2001 From: Kristijan Mitrovic Date: Mon, 31 Aug 2020 17:41:55 +0000 Subject: [PATCH] Add API support for downloading backup archive (#42) Merge branch 'master' into kris/download_backup Merge branch 'master' into kris/download_backup # Conflicts: # bubble-server/src/main/resources/messages # bubble-web Fix order of password check and kezs fetching Update web Add mime type of produced data for backup download API call Update Web Refactor backup download API calls Fix backup archive file's extension Update web Use executeScript for creating backup archive Update messages and web Remove download backup test Add content length for downloading file Merge branch 'master' into kris/download_backup # Conflicts: # bubble-server/src/main/resources/messages Use tgz instead of zip for backup package Update web and messages Merge branch 'master' into kris/download_backup # Conflicts: # bubble-server/src/main/resources/messages # bubble-web Update messages and web Merge branch 'master' into kris/download_backup Move download backup to Network Actions Resource Add simple API call for downloading backup archive Fix storage listing method Co-authored-by: jonathan Co-authored-by: Kristijan Mitrovic Reviewed-on: https://git.bubblev.org/bubblev/bubble/pulls/42 --- .../cloud/storage/StorageServiceDriver.java | 42 +++++- .../storage/local/LocalStorageDriver.java | 24 ++-- .../resources/cloud/BackupsResource.java | 2 + .../cloud/NetworkActionsResource.java | 54 ++------ .../cloud/NetworkBackupKeysResource.java | 121 ++++++++++++++++++ .../service/backup/NetworkKeysService.java | 101 ++++++++++++++- .../bubble/service/backup/RestoreService.java | 62 +++------ bubble-server/src/main/resources/messages | 2 +- .../models/tests/network/simple_backup.json | 2 +- bubble-web | 2 +- 10 files changed, 304 insertions(+), 108 deletions(-) create mode 100644 bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java diff --git a/bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriver.java b/bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriver.java index c9486fff..765102c3 100644 --- a/bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/storage/StorageServiceDriver.java @@ -10,22 +10,23 @@ import bubble.model.cloud.CloudService; import bubble.model.cloud.StorageMetadata; import bubble.notify.storage.StorageListing; import lombok.Cleanup; +import lombok.NonNull; import org.apache.commons.io.IOUtils; import org.cobbzilla.util.error.ExceptionHandler; import org.cobbzilla.util.string.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; +import java.util.Arrays; import static bubble.ApiConstants.ROOT_NETWORK_UUID; import static java.util.UUID.randomUUID; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.io.FileUtil.mkdirOrDie; import static org.cobbzilla.util.io.StreamUtil.toStringOrDie; +import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE; import static org.cobbzilla.util.system.Sleep.sleep; public interface StorageServiceDriver extends CloudServiceDriver { @@ -196,4 +197,37 @@ public interface StorageServiceDriver extends CloudServiceDriver { StorageListing list(String fromNode, String prefix) throws IOException; StorageListing listNext(String fromNode, String listingId) throws IOException; + @NonNull default void fetchFiles(@NonNull final String fromNodeUuid, + @NonNull final String fromPath, + @NonNull final String toDir) + throws IOException { + final var path = getPath(fromPath); + log.info("fetchFiles: downloading from path=" + path); + + var listing = list(fromNodeUuid, path); + while (true) { + Arrays.stream(listing.getKeys()) + .parallel() + .forEach(k -> { + final var logMsgPrefix = "fetchFiles [" + k + "]: "; + log.info(logMsgPrefix + "downloading file"); + final var file = new File(toDir, k); + mkdirOrDie(file.getParentFile()); + try { + @Cleanup final var in = read(fromNodeUuid, k); + try (var out = new BufferedOutputStream(new FileOutputStream(file), BUFFER_SIZE)) { + IOUtils.copyLarge(in, out); + } + log.info(logMsgPrefix + "successfully downloaded file"); + } catch (Exception e) { + die(logMsgPrefix + "error downloading file", e); + } + }); + + if (!listing.isTruncated()) break; + listing = listNext(fromNodeUuid, listing.getListingId()); + } + log.info("fetchFiles: full download successful"); + } + } diff --git a/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java b/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java index 4ce80c64..085ec678 100644 --- a/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java +++ b/bubble-server/src/main/java/bubble/cloud/storage/local/LocalStorageDriver.java @@ -15,6 +15,7 @@ import bubble.model.cloud.StorageMetadata; import bubble.notify.storage.StorageListing; import lombok.Cleanup; import lombok.Getter; +import lombok.NonNull; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.cobbzilla.util.io.FileUtil; @@ -22,9 +23,7 @@ import org.cobbzilla.util.system.OneWayFlag; import org.springframework.beans.factory.annotation.Autowired; import java.io.*; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import static bubble.ApiConstants.HOME_DIR; import static bubble.ApiConstants.ROOT_NETWORK_UUID; @@ -200,16 +199,17 @@ public class LocalStorageDriver extends CloudServiceDriverBase keys = new ArrayList<>(); - listFilesRecursively(file).forEach(f -> keys.add(abs(f).substring((int) rootFile.length()))); - return new StorageListing() - .setListingId(null) - .setTruncated(false) - .setKeys(keys.toArray(new String[0])); + @Override @NonNull public StorageListing list(@NonNull final String fromNodeUuid, @NonNull final String prefix) { + final var from = getFromNode(fromNodeUuid); + final var rootPathLength = abs(keyFile(from, "")).length(); + final var keys = listFilesRecursively(keyFile(from, prefix)).stream() + .filter(File::isFile) + .map(FileUtil::abs) + .map(path -> path.substring(rootPathLength)) + .toArray(String[]::new); + return new StorageListing().setListingId(null) + .setTruncated(false) + .setKeys(keys); } @Override public StorageListing listNext(String fromNode, String listingId) throws IOException { diff --git a/bubble-server/src/main/java/bubble/resources/cloud/BackupsResource.java b/bubble-server/src/main/java/bubble/resources/cloud/BackupsResource.java index 66150c17..e3ff4221 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/BackupsResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/BackupsResource.java @@ -9,6 +9,7 @@ import bubble.model.account.Account; import bubble.model.cloud.BackupStatus; import bubble.model.cloud.BubbleBackup; import bubble.model.cloud.BubbleNetwork; +import bubble.server.BubbleConfiguration; import bubble.service.backup.BackupCleanerService; import bubble.service.backup.BackupService; import org.glassfish.jersey.server.ContainerRequest; @@ -36,6 +37,7 @@ public class BackupsResource { this.network = network; } + @Autowired private BubbleConfiguration configuration; @Autowired private BubbleBackupDAO backupDAO; @Autowired private BackupService backupService; @Autowired private BackupCleanerService backupCleanerService; diff --git a/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java b/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java index b4956844..f4b09e32 100644 --- a/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java +++ b/bubble-server/src/main/java/bubble/resources/cloud/NetworkActionsResource.java @@ -5,28 +5,25 @@ package bubble.resources.cloud; import bubble.dao.account.AccountPolicyDAO; -import bubble.dao.account.message.AccountMessageDAO; import bubble.dao.bill.AccountPlanDAO; import bubble.dao.cloud.BubbleDomainDAO; import bubble.dao.cloud.BubbleNetworkDAO; import bubble.dao.cloud.BubbleNodeDAO; import bubble.model.account.Account; import bubble.model.account.AccountPolicy; -import bubble.model.account.message.AccountAction; -import bubble.model.account.message.AccountMessage; -import bubble.model.account.message.AccountMessageType; import bubble.model.account.message.ActionTarget; import bubble.model.bill.AccountPlan; -import bubble.model.cloud.*; +import bubble.model.cloud.BubbleDomain; +import bubble.model.cloud.BubbleNetwork; +import bubble.model.cloud.BubbleNode; +import bubble.model.cloud.NetLocation; import bubble.server.BubbleConfiguration; import bubble.service.account.StandardAuthenticatorService; -import bubble.service.backup.NetworkKeysService; import bubble.service.cloud.NodeLaunchMonitor; import bubble.service.cloud.NodeProgressMeterTick; import bubble.service.cloud.StandardNetworkService; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.collection.NameAndValue; -import org.cobbzilla.wizard.validation.ConstraintViolationBean; import org.cobbzilla.wizard.validation.ValidationResult; import org.glassfish.grizzly.http.server.Request; import org.glassfish.jersey.server.ContainerRequest; @@ -38,7 +35,6 @@ import javax.ws.rs.core.Response; import java.util.List; import static bubble.ApiConstants.*; -import static bubble.model.account.Account.validatePassword; import static bubble.model.cloud.BubbleNetwork.TAG_ALLOW_REGISTRATION; import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; import static org.cobbzilla.wizard.resources.ResourceUtil.*; @@ -51,9 +47,7 @@ public class NetworkActionsResource { @Autowired private BubbleNodeDAO nodeDAO; @Autowired private StandardNetworkService networkService; @Autowired private NodeLaunchMonitor launchMonitor; - @Autowired private AccountMessageDAO messageDAO; @Autowired private AccountPolicyDAO policyDAO; - @Autowired private NetworkKeysService keysService; @Autowired private BubbleDomainDAO domainDAO; @Autowired private BubbleNetworkDAO networkDAO; @Autowired private AccountPlanDAO accountPlanDAO; @@ -110,50 +104,24 @@ public class NetworkActionsResource { return tick == null ? notFound(uuid) : ok(tick); } - @GET @Path(EP_KEYS) - public Response requestNetworkKeys(@Context Request req, - @Context ContainerRequest ctx) { + @Path(EP_KEYS) + @NonNull public NetworkBackupKeysResource getBackupKeys(@NonNull @Context final ContainerRequest ctx) { final Account caller = userPrincipal(ctx); - if (!caller.admin()) return forbidden(); + if (!caller.admin()) throw forbiddenEx(); // must request from the network you are on if (!network.getUuid().equals(configuration.getThisNetwork().getUuid())) { - return invalid("err.networkKeys.mustRequestFromSameNetwork"); + throw invalidEx("err.networkKeys.mustRequestFromSameNetwork"); } final AccountPolicy policy = policyDAO.findSingleByAccount(caller.getUuid()); if (policy == null || !policy.hasVerifiedAccountContacts()) { - return invalid("err.networkKeys.noVerifiedContacts"); + throw invalidEx("err.networkKeys.noVerifiedContacts"); } - messageDAO.create(new AccountMessage() - .setMessageType(AccountMessageType.request) - .setAction(AccountAction.password) - .setTarget(ActionTarget.network) - .setAccount(caller.getUuid()) - .setNetwork(configuration.getThisNetwork().getUuid()) - .setName(network.getUuid()) - .setRemoteHost(getRemoteHost(req))); - return ok(); - } - - @POST @Path(EP_KEYS+"/{uuid}") - public Response retrieveNetworkKeys(@Context Request req, - @Context ContainerRequest ctx, - @PathParam("uuid") String uuid, - NameAndValue enc) { - final Account caller = userPrincipal(ctx); - if (!caller.admin()) return forbidden(); authenticatorService.ensureAuthenticated(ctx, ActionTarget.network); - final String encryptionKey = enc == null ? null : enc.getValue(); - final ConstraintViolationBean error = validatePassword(encryptionKey); - if (error != null) return invalid(error); - - final NetworkKeys keys = keysService.retrieveKeys(uuid); - return keys == null - ? invalid("err.retrieveNetworkKeys.notFound") - : ok(keys.encrypt(encryptionKey)); + return configuration.subResource(NetworkBackupKeysResource.class, caller, network); } @POST @Path(EP_STOP) diff --git a/bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java b/bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java new file mode 100644 index 00000000..db032024 --- /dev/null +++ b/bubble-server/src/main/java/bubble/resources/cloud/NetworkBackupKeysResource.java @@ -0,0 +1,121 @@ +package bubble.resources.cloud; + +import bubble.dao.account.message.AccountMessageDAO; +import bubble.dao.cloud.BubbleBackupDAO; +import bubble.model.account.Account; +import bubble.model.account.message.AccountAction; +import bubble.model.account.message.AccountMessage; +import bubble.model.account.message.AccountMessageType; +import bubble.model.account.message.ActionTarget; +import bubble.model.cloud.BubbleNetwork; +import bubble.service.backup.NetworkKeysService; +import lombok.NonNull; +import org.cobbzilla.util.collection.NameAndValue; +import org.cobbzilla.wizard.stream.FileSendableResource; +import org.cobbzilla.wizard.validation.ConstraintViolationBean; +import org.cobbzilla.wizard.validation.SimpleViolationException; +import org.glassfish.grizzly.http.server.Request; +import org.glassfish.jersey.server.ContainerRequest; +import org.springframework.beans.factory.annotation.Autowired; + +import javax.annotation.Nullable; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import java.io.File; + +import static bubble.ApiConstants.*; +import static bubble.model.account.Account.validatePassword; +import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; +import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_OCTET_STREAM; +import static org.cobbzilla.wizard.resources.ResourceUtil.*; + +/** + * Ensure that only network admin can access these calls, and only for current network. Such admin should have verified + * account contact, and should be authenticated as in `authenticatorService.ensureAuthenticated`. + */ +@Consumes(APPLICATION_JSON) +@Produces(APPLICATION_JSON) +public class NetworkBackupKeysResource { + @Autowired private AccountMessageDAO messageDAO; + @Autowired private BubbleBackupDAO backupDAO; + @Autowired private NetworkKeysService keysService; + + private final Account adminCaller; + private final BubbleNetwork thisNetwork; + + public NetworkBackupKeysResource(@NonNull final Account adminCaller, @NonNull final BubbleNetwork thisNetwork) { + this.adminCaller = adminCaller; + this.thisNetwork = thisNetwork; + } + + @GET + @NonNull public Response requestNetworkKeys(@NonNull @Context final Request req, + @NonNull @Context final ContainerRequest ctx) { + messageDAO.create(new AccountMessage().setMessageType(AccountMessageType.request) + .setAction(AccountAction.password) + .setTarget(ActionTarget.network) + .setAccount(adminCaller.getUuid()) + .setNetwork(thisNetwork.getUuid()) + .setName(thisNetwork.getUuid()) + .setRemoteHost(getRemoteHost(req))); + return ok(); + } + + @NonNull private String fetchAndCheckEncryptionKey(@Nullable final NameAndValue enc) { + final String encryptionKey = enc == null ? null : enc.getValue(); + final ConstraintViolationBean error = validatePassword(encryptionKey); + if (error != null) throw new SimpleViolationException(error); + return encryptionKey; + } + + @POST @Path("/{keysCode}") + @NonNull public Response retrieveNetworkKeys(@NonNull @Context final Request req, + @NonNull @Context final ContainerRequest ctx, + @NonNull @PathParam("keysCode") final String keysCode, + @Nullable final NameAndValue enc) { + final var encryptionKey = fetchAndCheckEncryptionKey(enc); + final var networkKeys = keysService.retrieveKeys(keysCode); + return ok(networkKeys.encrypt(encryptionKey)); + } + + @POST @Path("/{keysCode}" + EP_BACKUPS + EP_START) + @NonNull public Response backupDownloadStart(@NonNull @Context final ContainerRequest ctx, + @NonNull @PathParam("keysCode") final String keysCode, + @NonNull @QueryParam("backupId") final String backupId, + @Nullable final NameAndValue enc) { + final var passphrase = fetchAndCheckEncryptionKey(enc); + keysService.retrieveKeys(keysCode); + + final var backup = backupDAO.findByNetworkAndId(thisNetwork.getUuid(), backupId); + if (backup == null) throw notFoundEx(backupId); + + keysService.startBackupDownload(thisNetwork.getUuid(), backup, keysCode, passphrase); + keysService.backupDownloadStatus(keysCode); + return ok(); + } + + @GET @Path("/{keysCode}" + EP_BACKUPS + EP_STATUS) + @NonNull public Response backupDownloadStatus(@NonNull @Context final ContainerRequest ctx, + @NonNull @PathParam("keysCode") final String keysCode) { + // not checking keys code now here. However, such key will be required in preparing/prepared backup downloads' + // mapping within restoreService. + return ok(keysService.backupDownloadStatus(keysCode)); + } + + @GET @Path("/{keysCode}" + EP_BACKUPS + EP_DOWNLOAD) + @Produces(APPLICATION_OCTET_STREAM) + @NonNull public Response backupDownload(@NonNull @Context final ContainerRequest ctx, + @NonNull @PathParam("keysCode") final String keysCode) { + final var status = keysService.backupDownloadStatus(keysCode); + if (!status.isDone()) return accepted(); + + keysService.clearBackupDownloadKey(keysCode); + final var outFileName = "backup-" + thisNetwork.getNickname() + ".tgz.enc"; + final var backupArchiveFile = new File(status.getPackagePath()); + return send(new FileSendableResource(backupArchiveFile).setContentType(APPLICATION_OCTET_STREAM) + .setContentLength(backupArchiveFile.length()) + .setForceDownload(true) + .setName(outFileName)); + } +} diff --git a/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java b/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java index 4fc27f75..633c2952 100644 --- a/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java +++ b/bubble-server/src/main/java/bubble/service/backup/NetworkKeysService.java @@ -5,20 +5,38 @@ package bubble.service.backup; import bubble.dao.cloud.CloudServiceDAO; +import bubble.model.cloud.BubbleBackup; import bubble.model.cloud.CloudService; import bubble.model.cloud.NetworkKeys; import bubble.server.BubbleConfiguration; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE; import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE_CREDENTIALS; +import static java.io.File.createTempFile; import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.commons.io.FileUtils.deleteDirectory; +import static org.cobbzilla.util.daemon.ZillaRuntime.background; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.createTempDir; import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.security.ShaUtil.sha256_hex; +import static org.cobbzilla.util.system.CommandShell.execScript; import static org.cobbzilla.wizard.cache.redis.RedisService.EX; +import static org.cobbzilla.wizard.resources.ResourceUtil.invalidEx; +import static org.cobbzilla.wizard.resources.ResourceUtil.notFoundEx; @Service @Slf4j public class NetworkKeysService { @@ -31,6 +49,9 @@ public class NetworkKeysService { @Autowired private RedisService redis; @Getter(lazy=true) private final RedisService networkPasswordTokens = redis.prefixNamespace(getClass().getSimpleName()); + // keysCode -> status object + private static final Map activeBackupDownloads = new ConcurrentHashMap<>(); + public void registerView(String uuid) { final NetworkKeys keys = new NetworkKeys(); final CloudService storage = cloudDAO.findByUuid(configuration.getThisNetwork().getStorage()); @@ -43,11 +64,87 @@ public class NetworkKeysService { getNetworkPasswordTokens().set(uuid, json(keys), EX, KEY_EXPIRATION); } - public NetworkKeys retrieveKeys(String uuid) { + @NonNull public NetworkKeys retrieveKeys(@NonNull final String uuid) { final String json = getNetworkPasswordTokens().get(uuid); - if (json == null) return null; + if (empty(json)) throw invalidEx("err.retrieveNetworkKeys.notFound"); getNetworkPasswordTokens().del(uuid); return json(json, NetworkKeys.class); } + public void startBackupDownload(@NonNull final String nodeUuid, @NonNull final BubbleBackup backup, + @NonNull final String keysCode, @NonNull final String passphrase) { + final var storageServiceUuid = configuration.getThisNetwork().getStorage(); + final var storageService = cloudDAO.findByUuid(storageServiceUuid); + if (storageService == null) throw notFoundEx(storageServiceUuid); + final var storageDriver = storageService.getStorageDriver(configuration); + + final var status = new BackupPackagingStatus(backup.getUuid()); + if (activeBackupDownloads.putIfAbsent(keysCode, status) != null) { + throw invalidEx("err.download.error", "Already started"); + } + + background(() -> { + File backupDir = null; + try { + backupDir = createTempDir("backup-"); + final var backupDirAbs = abs(backupDir); + storageDriver.fetchFiles(nodeUuid, backup.getPath(), backupDirAbs); + final var backupPackageAbs = abs(createTempFile("backup-", ".tgz.enc")); + execScript("cd " + backupDirAbs + + " && tar -cz *" + + " | openssl enc -aes-256-cbc -pbkdf2 -iter 10000 -pass pass:" + sha256_hex(passphrase) + + " > " + backupPackageAbs); + status.ok(backupPackageAbs); + } catch (Exception e) { + status.fail(e.getMessage()); + } finally { + try { + if (backupDir != null) deleteDirectory(backupDir); + } catch (IOException e) { + log.error("Cannot delete tmp backup folder " + backupDir, e); + } + } + }); + } + + @NonNull public BackupPackagingStatus backupDownloadStatus(@NonNull final String keysCode) { + final var status = activeBackupDownloads.get(keysCode); + if (status == null) throw notFoundEx(keysCode); + if (status.hasError()) throw invalidEx("err.download.error", status.getError()); + return status; + } + + public void clearBackupDownloadKey(@NonNull final String keysCode) { + activeBackupDownloads.remove(keysCode); + } + + public static class BackupPackagingStatus { + @Getter private boolean done; + @Getter private final String backupUuid; + @JsonIgnore @Getter private String packagePath; + @Getter private String error; + public boolean hasError() { return !empty(error); } + + private BackupPackagingStatus(@NonNull final String backupUuid) { + this.done = false; + this.backupUuid = backupUuid; + this.packagePath = null; + this.error = null; + } + + private BackupPackagingStatus ok(@NonNull final String packagePath) { + this.done = true; + this.packagePath = packagePath; + this.error = null; + return this; + } + + private BackupPackagingStatus fail(@NonNull final String error) { + this.done = true; + this.packagePath = null; + this.error = error; + return this; + } + } + } diff --git a/bubble-server/src/main/java/bubble/service/backup/RestoreService.java b/bubble-server/src/main/java/bubble/service/backup/RestoreService.java index 04fc42a7..3ad39988 100644 --- a/bubble-server/src/main/java/bubble/service/backup/RestoreService.java +++ b/bubble-server/src/main/java/bubble/service/backup/RestoreService.java @@ -4,36 +4,31 @@ */ package bubble.service.backup; -import bubble.cloud.storage.StorageServiceDriver; +import bubble.dao.cloud.CloudServiceDAO; import bubble.model.cloud.BubbleBackup; import bubble.model.cloud.CloudCredentials; import bubble.model.cloud.CloudService; import bubble.model.cloud.NetworkKeys; -import bubble.notify.storage.StorageListing; import bubble.server.BubbleConfiguration; -import lombok.Cleanup; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.IOUtils; import org.cobbzilla.util.collection.NameAndValue; -import org.cobbzilla.util.io.TempDir; +import org.cobbzilla.util.system.Bytes; import org.cobbzilla.wizard.cache.redis.RedisService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.io.*; -import java.util.Arrays; +import java.io.File; +import java.io.IOException; import static bubble.ApiConstants.HOME_DIR; import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE; import static bubble.model.cloud.NetworkKeys.PARAM_STORAGE_CREDENTIALS; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; -import static org.apache.commons.io.FileUtils.copyDirectory; -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.io.FileUtil.*; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.touch; import static org.cobbzilla.util.json.JsonUtil.json; -import static org.cobbzilla.util.security.CryptStream.BUFFER_SIZE; import static org.cobbzilla.wizard.cache.redis.RedisService.EX; @Service @Slf4j @@ -54,6 +49,10 @@ public class RestoreService { private static final long RESTORE_DEADLOCK_TIMEOUT = MINUTES.toMillis(30); private static final File RESTORE_MARKER_FILE = new File(HOME_DIR, ".restore"); + private static final int BACKUP_ARCHIVE_MANAGEMENT_BUFFER_SIZE = (int) (8 * Bytes.KB); + + @Autowired private CloudServiceDAO cloudDAO; + @Autowired private RedisService redis; @Getter(lazy=true) private final RedisService restoreKeys = redis.prefixNamespace(getClass().getSimpleName()); @@ -89,49 +88,24 @@ public class RestoreService { log.error("restore: storage/credentials not found in NetworkKeys"); return false; } - final CloudService storageService = json(storageJson, CloudService.class) - .setCredentials(json(credentialsJson, CloudCredentials.class)); - - final StorageServiceDriver storageDriver = storageService.getStorageDriver(configuration); - final String path = StorageServiceDriver.getPath(backup.getPath()); final String[] existingFiles = RESTORE_DIR.list(); + final var restoreDirAbs = abs(RESTORE_DIR); if (existingFiles != null && existingFiles.length > 0) { - log.error("restore: files already exist in " + abs(RESTORE_DIR) + ", cannot restore"); + log.error("restore: files already exist in " + restoreDirAbs + ", cannot restore"); return false; } - log.info("restore: downloading backup from path=" + path); + final var storageDriver = json(storageJson, CloudService.class) + .setCredentials(json(credentialsJson, CloudCredentials.class)) + .getStorageDriver(configuration); try { - @Cleanup TempDir temp = new TempDir(); - StorageListing listing = storageDriver.list(thisNodeUuid, path); - while (true) { - Arrays.stream(listing.getKeys()).forEach(k -> { - log.info("restore: downloading file: " + k); - final File file = new File(abs(temp) + "/" + k); - mkdirOrDie(file.getParentFile()); - try { - @Cleanup final InputStream in = storageDriver.read(thisNodeUuid, k); - try (OutputStream out = new BufferedOutputStream(new FileOutputStream(file), BUFFER_SIZE)) { - IOUtils.copyLarge(in, out); - } - log.info("restore: successfully downloaded file: " + k); - } catch (Exception e) { - die("restore: error downloading file: " + k + ": " + e); - } - }); - if (!listing.isTruncated()) break; - listing = storageDriver.listNext(thisNodeUuid, listing.getListingId()); - } - - // all successful, copy directory to a safe place - copyDirectory(temp, RESTORE_DIR); - log.info("restore: full download successful, notifying system to restore from backup at: "+abs(RESTORE_DIR)); + storageDriver.fetchFiles(thisNodeUuid, backup.getPath(), restoreDirAbs); + log.info("restore: notifying system to restore from backup at: " + restoreDirAbs); touch(RESTORE_MARKER_FILE); return true; - } catch (IOException e) { - log.error("restore: error downloading backup: " + e); + log.error("restore: error downloading backup ", e); return false; } } finally { diff --git a/bubble-server/src/main/resources/messages b/bubble-server/src/main/resources/messages index 86d499ff..2c86f545 160000 --- a/bubble-server/src/main/resources/messages +++ b/bubble-server/src/main/resources/messages @@ -1 +1 @@ -Subproject commit 86d499ff016f84badb10e3a3c355fe775233b16a +Subproject commit 2c86f5458d71453fa9d487577d6729e08f66945d diff --git a/bubble-server/src/test/resources/models/tests/network/simple_backup.json b/bubble-server/src/test/resources/models/tests/network/simple_backup.json index ddf79ff2..0f79d444 100644 --- a/bubble-server/src/test/resources/models/tests/network/simple_backup.json +++ b/bubble-server/src/test/resources/models/tests/network/simple_backup.json @@ -68,4 +68,4 @@ ] } } -] \ No newline at end of file +] diff --git a/bubble-web b/bubble-web index bf4d8d45..8bf42eb5 160000 --- a/bubble-web +++ b/bubble-web @@ -1 +1 @@ -Subproject commit bf4d8d45b49f01a1c5dd2d63c3b8ae488c750ed6 +Subproject commit 8bf42eb51f93bcdf5be55d39485654c7b95ac0c4