From a8adaaf97508cbe4e39d8bb028a304ee16303648 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Fri, 13 Dec 2019 16:42:37 -0500 Subject: [PATCH] remove obsolete asset storage classes --- .../wizard/asset/AssetStorageService.java | 48 ---- .../cobbzilla/wizard/asset/AssetStream.java | 38 ---- .../asset/LocalAssetStorageService.java | 92 -------- .../wizard/asset/ResourceStorageService.java | 43 ---- .../wizard/asset/S3AssetStorageService.java | 213 ------------------ 5 files changed, 434 deletions(-) delete mode 100644 wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStorageService.java delete mode 100644 wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStream.java delete mode 100644 wizard-server/src/main/java/org/cobbzilla/wizard/asset/LocalAssetStorageService.java delete mode 100644 wizard-server/src/main/java/org/cobbzilla/wizard/asset/ResourceStorageService.java delete mode 100644 wizard-server/src/main/java/org/cobbzilla/wizard/asset/S3AssetStorageService.java diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStorageService.java b/wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStorageService.java deleted file mode 100644 index 4782d18..0000000 --- a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStorageService.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.cobbzilla.wizard.asset; - -import org.cobbzilla.util.http.HttpContentTypes; -import org.cobbzilla.util.io.FileUtil; - -import java.io.File; -import java.io.InputStream; - -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON; -import static org.cobbzilla.util.security.ShaUtil.sha256_file; - -public abstract class AssetStorageService { - - public static AssetStorageService build(AssetStorageConfiguration configuration) { - AssetStorageType type = configuration.getType(); - if (type == null) type = AssetStorageType.local; - switch (type) { - case local: return new LocalAssetStorageService(configuration.getConfig()); - case s3: return new S3AssetStorageService(configuration.getConfig()); - case resource: return new ResourceStorageService(configuration.getConfig()); - default: return die("AssetStorageService.build: invalid type: "+ type); - } - } - - public abstract AssetStream load(String uri); - - public abstract boolean exists(String uri); - - public String store(InputStream fileStream, String filename) { return store(fileStream, filename, null); } - - public abstract String store(InputStream fileStream, String fileName, String uri); - - public abstract void copy(String from, String to); - - public abstract boolean delete(String uri); - - public String getUri(File file, String filename) { return getUri(sha256_file(file), filename); } - - public String getUri(String sha256, String filename) { - return sha256.substring(0, 2) + "/" + sha256.substring(2, 4) + "/" + sha256.substring(4, 6) + "/" + sha256.substring(6, 8) + "/" + sha256 + FileUtil.extension(filename); - } - - public String getMimeType(String filename) { - return filename.endsWith(".json") ? APPLICATION_JSON : HttpContentTypes.contentType(filename); - } - -} diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStream.java b/wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStream.java deleted file mode 100644 index 4039872..0000000 --- a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStream.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.cobbzilla.wizard.asset; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; - -@NoArgsConstructor @AllArgsConstructor @Slf4j -public class AssetStream implements Closeable { - - @Getter @Setter private String uri; - @Getter @Setter private InputStream stream; - @Getter @Setter private String contentType; - - public final static String[][] FORMAT_MAP = { - {"png", "png"}, - {"jpg", "jpg"}, - {"jpeg", "jpg"}, - {"gif", "gif"}, - {"pdf", "pdf"} - }; - @JsonIgnore public String getFormatName() { - for (String[] format : FORMAT_MAP) { - if (contentType.contains("/"+format[0])) return format[1]; - } - return null; - } - - @Override public void close() throws IOException { - if (stream != null) try { stream.close(); } catch (Exception e) { log.warn("close: "+e); } - } -} diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/LocalAssetStorageService.java b/wizard-server/src/main/java/org/cobbzilla/wizard/asset/LocalAssetStorageService.java deleted file mode 100644 index c0cfaad..0000000 --- a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/LocalAssetStorageService.java +++ /dev/null @@ -1,92 +0,0 @@ -package org.cobbzilla.wizard.asset; - -import lombok.Cleanup; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.cobbzilla.util.io.FileUtil; - -import java.io.*; -import java.util.Map; - -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.io.FileUtil.*; - -@Slf4j -public class LocalAssetStorageService extends AssetStorageService { - - public static final String PROP_BASE = "baseDir"; - - @Getter @Setter private File baseDir; - @Getter @Setter private String contentType; - - public LocalAssetStorageService(Map config) { - String basePath = (config == null) ? null : config.get(PROP_BASE); - if (empty(basePath)) basePath = System.getProperty("java.io.tmpdir"); - this.baseDir = FileUtil.mkdirOrDie(basePath); - } - - @Override public AssetStream load(String uri) { - try { - final String path = abs(baseDir) + "/" + uri; - final File f = new File(path); - return f.exists() ? new AssetStream(uri, new FileInputStream(f), getContentType(path)) : null; - - } catch (FileNotFoundException e) { - log.warn("load: "+e); - return null; - } - } - - public String getContentType(String path) { - return contentType != null ? contentType : toStringOrDie(abs(path)+".contentType"); - } - - public File uri2file(String uri) { return new File(abs(baseDir) + "/" + uri); } - - @Override public boolean exists(String uri) { return uri2file(uri).exists(); } - - @Override public String store(InputStream fileStream, String filename, String path) { - final String mimeType = getMimeType(filename); - try { - final File temp = File.createTempFile("localAsset", ".tmp", getDefaultTempDir()); - try (FileOutputStream out = new FileOutputStream(temp)) { - IOUtils.copyLarge(fileStream, out); - } - if (path == null) path = getUri(temp, filename); - final File stored = uri2file(path); - mkdirOrDie(stored.getParentFile()); - - // rename, or copy, or die - if (!temp.renameTo(stored)) { - FileUtils.copyFile(temp, stored); - FileUtils.deleteQuietly(temp); - } - if (contentType == null) FileUtil.toFile(abs(stored)+".contentType", mimeType); - - return path; - - } catch (Exception e) { - return die("store: "+e, e); - } - } - - @Override public void copy(String from, String to) { - try { - @Cleanup InputStream in = load(from).getStream(); - store(in, to, to); - } catch (Exception e) { - die("copy: "+e, e); - } - } - - @Override public boolean delete(String uri) { - if (!exists(uri)) return false; - final File f = uri2file(uri); - return f.exists() && f.delete(); - } - -} diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/ResourceStorageService.java b/wizard-server/src/main/java/org/cobbzilla/wizard/asset/ResourceStorageService.java deleted file mode 100644 index dab098a..0000000 --- a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/ResourceStorageService.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.cobbzilla.wizard.asset; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.experimental.Accessors; -import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.io.StreamUtil; - -import java.io.InputStream; -import java.util.Map; - -import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported; - -@NoArgsConstructor @Accessors(chain=true) @Slf4j -public class ResourceStorageService extends AssetStorageService { - - public static final String PROP_BASE = "base"; - - @Getter @Setter private String base; - - public ResourceStorageService(String base) { this.base = base; } - - public ResourceStorageService(Map config) { this(config.get(PROP_BASE)); } - - @Override public AssetStream load(String uri) { - try { - return new AssetStream(uri, StreamUtil.loadResourceAsStream(base + "/" + uri), getMimeType(uri)); - } catch (Exception e) { - log.error("load: "+e); - } - return null; - } - - @Override public boolean exists(String uri) { return load(uri) != null; } - - @Override public String store(InputStream fileStream, String fileName, String uri) { return notSupported(); } - - @Override public void copy(String from, String to) { notSupported(); } - - @Override public boolean delete(String uri) { return notSupported(); } - -} diff --git a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/S3AssetStorageService.java b/wizard-server/src/main/java/org/cobbzilla/wizard/asset/S3AssetStorageService.java deleted file mode 100644 index be951ea..0000000 --- a/wizard-server/src/main/java/org/cobbzilla/wizard/asset/S3AssetStorageService.java +++ /dev/null @@ -1,213 +0,0 @@ -package org.cobbzilla.wizard.asset; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.internal.Mimetypes; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.io.FileUtil; -import org.cobbzilla.util.system.Sleep; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.daemon.ZillaRuntime.empty; -import static org.cobbzilla.util.io.FileUtil.*; - -@Slf4j -public class S3AssetStorageService extends AssetStorageService { - - public static final int MAX_RETRIES = 10; - public static final long RETRY_SLEEP = TimeUnit.SECONDS.toMillis(5); - - public static final String PROP_ACCESS_KEY = "accessKey"; - public static final String PROP_SECRET_KEY = "secretKey"; - public static final String PROP_BUCKET = "bucket"; - public static final String PROP_PREFIX = "prefix"; - public static final String PROP_LOCAL_CACHE = "localCache"; - public static final String CACHE_DISABLED = "disabled"; - - @Getter @Setter private String accessKey; - @Getter @Setter private String secretKey; - @Getter @Setter private String bucket; - @Getter @Setter private String prefix; - @Getter @Setter private File localCache; - - private final AmazonS3Client s3Client; - - public S3AssetStorageService(Map config) { - setAccessKey(config.get(PROP_ACCESS_KEY)); - setSecretKey(config.get(PROP_SECRET_KEY)); - setBucket(config.get(PROP_BUCKET)); - setPrefix(config.get(PROP_PREFIX)); - - String local = config.get(PROP_LOCAL_CACHE); - if (empty(local)) local = System.getProperty("java.io.tmpdir"); - - if (local.equals(CACHE_DISABLED)) { - localCache = null; - } else { - setLocalCache(mkdirOrDie(local)); - } - - s3Client = new AmazonS3Client(new AWSCredentials() { - @Override public String getAWSAccessKeyId() { return getAccessKey(); } - @Override public String getAWSSecretKey() { return getSecretKey(); } - }); - } - - @Override public AssetStream load(String uri) { - if (localCache != null) { - final File cachefile = new File(abs(localCache) + "/" + uri); - if (cachefile.exists()) { - try { - return new AssetStream(uri, new FileInputStream(cachefile), toStringOrDie(abs(cachefile) + ".contentType")); - } catch (IOException e) { - die("load: " + e, e); - } - } - } - Exception lastException = null; - for (int tries = 0; tries < MAX_RETRIES; tries++) { - try { - synchronized (s3Client) { - final S3Object s3Object = s3Client.getObject(bucket, prefix + "/" + uri); - return new AssetStream(uri, s3Object.getObjectContent(), s3Object.getObjectMetadata().getContentType()); - } - } catch (Exception e) { - log.warn("load("+uri+"): "+e); - lastException = e; - Sleep.sleep(RETRY_SLEEP); - } - } - return die("load("+uri+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException); - } - - @Override public boolean exists(String uri) { - if (localCache != null && new File(abs(localCache) + "/" + uri).exists()) return true; - Exception lastException = null; - try { - for (int tries = 0; tries < MAX_RETRIES; tries++) { - try { - synchronized (s3Client) { - s3Client.getObject(bucket, prefix + "/" + uri); - return true; - } - } catch (Exception e) { - log.warn("exists("+uri+"): "+e); - lastException = e; - Sleep.sleep(RETRY_SLEEP); - } - } - return die("exists("+uri+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException); - - } catch (Exception e) { - return false; - } - } - - @Override public synchronized String store(InputStream fileStream, String filename, String path) { - - final String mimeType = filename.endsWith(".json") ? "application/json" : Mimetypes.getInstance().getMimetype(filename); - - File temp = null; - try { - temp = File.createTempFile("s3file-", ".tmp", getDefaultTempDir()); - FileUtil.toFile(temp, fileStream); - if (path == null) path = getUri(temp, filename); - - final File stored = (localCache == null) ? temp : new File(abs(localCache) + "/" + path); - - if (localCache != null) { - mkdirOrDie(stored.getParentFile()); - FileUtil.toFile(abs(stored) + ".contentType", mimeType); - if (!temp.renameTo(stored)) { - log.warn("store: error renaming file, copying instead: " + abs(temp) + " -> " + abs(stored)); - FileUtil.copyFile(temp, stored); - if (!temp.delete()) log.warn("store: error deleting temp file after copy: "+abs(temp)); - } - } - - final ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentType(mimeType); - metadata.setContentLength(stored.length()); - put(path, stored, metadata); - - return path; - - } catch (Exception e) { - return die("store: "+e, e); - - } finally { - if (localCache == null) { - if (temp != null && temp.exists()) { - if (!temp.delete()) log.warn("store: error deleting temp file: "+abs(temp)); - } - } - } - } - - public void put(String path, File stored, ObjectMetadata metadata) throws IOException { - if (metadata == null) { - put(path, stored); - } else { - // ok we have to use an input stream to do the metadata in the same call - Exception lastException = null; - try (InputStream in = new FileInputStream(stored)) { - for (int tries = 0; tries < MAX_RETRIES; tries++) { - try { - synchronized (s3Client) { - s3Client.putObject(bucket, prefix + "/" + path, in, metadata); - return; - } - } catch (Exception e) { - log.warn("put("+path+"): "+e); - lastException = e; - Sleep.sleep(RETRY_SLEEP); - } - } - die("put("+path+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException); - } - } - } - - public void put(String path, File stored) throws IOException { - Exception lastException = null; - for (int tries = 0; tries < MAX_RETRIES; tries++) { - try { - synchronized (s3Client) { - s3Client.putObject(bucket, prefix + "/" + path, stored); - return; - } - } catch (Exception e) { - log.warn("put("+path+"): "+e); - lastException = e; - Sleep.sleep(RETRY_SLEEP); - } - } - die("put("+path+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException); - } - - @Override public void copy(String from, String to) { - synchronized (s3Client) { - s3Client.copyObject(bucket, prefix + "/" + from, bucket, prefix + "/" + to); - } - } - - @Override public boolean delete(String uri) { - if (!exists(uri)) return false; - synchronized (s3Client) { - s3Client.deleteObject(bucket, prefix + "/" + uri); - } - return true; - } - -}