bubblev
/
cobbzilla-wizard
6
0
포크 0
코드 이슈 0 풀 리퀘스트 0 릴리즈 1 위키 활동
소스 검색

remove obsolete asset storage classes

tags/2.0.1
Jonathan Cobb 4 년 전
부모
fa1150cb5d
커밋
a8adaaf975
5개의 변경된 파일과 0개의 추가작업 그리고 434개의 파일을 삭제
분할 보기
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -48
      wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStorageService.java
  2. +0
    -38
      wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStream.java
  3. +0
    -92
      wizard-server/src/main/java/org/cobbzilla/wizard/asset/LocalAssetStorageService.java
  4. +0
    -43
      wizard-server/src/main/java/org/cobbzilla/wizard/asset/ResourceStorageService.java
  5. +0
    -213
      wizard-server/src/main/java/org/cobbzilla/wizard/asset/S3AssetStorageService.java

+ 0
- 48
wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStorageService.java 파일 보기

@@ -1,48 +0,0 @@
package org.cobbzilla.wizard.asset;

import org.cobbzilla.util.http.HttpContentTypes;
import org.cobbzilla.util.io.FileUtil;

import java.io.File;
import java.io.InputStream;

import static org.cobbzilla.util.daemon.ZillaRuntime.die;
import static org.cobbzilla.util.http.HttpContentTypes.APPLICATION_JSON;
import static org.cobbzilla.util.security.ShaUtil.sha256_file;

public abstract class AssetStorageService {

public static AssetStorageService build(AssetStorageConfiguration configuration) {
AssetStorageType type = configuration.getType();
if (type == null) type = AssetStorageType.local;
switch (type) {
case local: return new LocalAssetStorageService(configuration.getConfig());
case s3: return new S3AssetStorageService(configuration.getConfig());
case resource: return new ResourceStorageService(configuration.getConfig());
default: return die("AssetStorageService.build: invalid type: "+ type);
}
}

public abstract AssetStream load(String uri);

public abstract boolean exists(String uri);

public String store(InputStream fileStream, String filename) { return store(fileStream, filename, null); }

public abstract String store(InputStream fileStream, String fileName, String uri);

public abstract void copy(String from, String to);

public abstract boolean delete(String uri);

public String getUri(File file, String filename) { return getUri(sha256_file(file), filename); }

public String getUri(String sha256, String filename) {
return sha256.substring(0, 2) + "/" + sha256.substring(2, 4) + "/" + sha256.substring(4, 6) + "/" + sha256.substring(6, 8) + "/" + sha256 + FileUtil.extension(filename);
}

public String getMimeType(String filename) {
return filename.endsWith(".json") ? APPLICATION_JSON : HttpContentTypes.contentType(filename);
}

}

+ 0
- 38
wizard-server/src/main/java/org/cobbzilla/wizard/asset/AssetStream.java 파일 보기

@@ -1,38 +0,0 @@
package org.cobbzilla.wizard.asset;

import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;

import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;

@NoArgsConstructor @AllArgsConstructor @Slf4j
public class AssetStream implements Closeable {

@Getter @Setter private String uri;
@Getter @Setter private InputStream stream;
@Getter @Setter private String contentType;

public final static String[][] FORMAT_MAP = {
{"png", "png"},
{"jpg", "jpg"},
{"jpeg", "jpg"},
{"gif", "gif"},
{"pdf", "pdf"}
};
@JsonIgnore public String getFormatName() {
for (String[] format : FORMAT_MAP) {
if (contentType.contains("/"+format[0])) return format[1];
}
return null;
}

@Override public void close() throws IOException {
if (stream != null) try { stream.close(); } catch (Exception e) { log.warn("close: "+e); }
}
}

+ 0
- 92
wizard-server/src/main/java/org/cobbzilla/wizard/asset/LocalAssetStorageService.java 파일 보기

@@ -1,92 +0,0 @@
package org.cobbzilla.wizard.asset;

import lombok.Cleanup;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.cobbzilla.util.io.FileUtil;

import java.io.*;
import java.util.Map;

import static org.cobbzilla.util.daemon.ZillaRuntime.die;
import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
import static org.cobbzilla.util.io.FileUtil.*;

@Slf4j
public class LocalAssetStorageService extends AssetStorageService {

public static final String PROP_BASE = "baseDir";

@Getter @Setter private File baseDir;
@Getter @Setter private String contentType;

public LocalAssetStorageService(Map<String, String> config) {
String basePath = (config == null) ? null : config.get(PROP_BASE);
if (empty(basePath)) basePath = System.getProperty("java.io.tmpdir");
this.baseDir = FileUtil.mkdirOrDie(basePath);
}

@Override public AssetStream load(String uri) {
try {
final String path = abs(baseDir) + "/" + uri;
final File f = new File(path);
return f.exists() ? new AssetStream(uri, new FileInputStream(f), getContentType(path)) : null;

} catch (FileNotFoundException e) {
log.warn("load: "+e);
return null;
}
}

public String getContentType(String path) {
return contentType != null ? contentType : toStringOrDie(abs(path)+".contentType");
}

public File uri2file(String uri) { return new File(abs(baseDir) + "/" + uri); }

@Override public boolean exists(String uri) { return uri2file(uri).exists(); }

@Override public String store(InputStream fileStream, String filename, String path) {
final String mimeType = getMimeType(filename);
try {
final File temp = File.createTempFile("localAsset", ".tmp", getDefaultTempDir());
try (FileOutputStream out = new FileOutputStream(temp)) {
IOUtils.copyLarge(fileStream, out);
}
if (path == null) path = getUri(temp, filename);
final File stored = uri2file(path);
mkdirOrDie(stored.getParentFile());

// rename, or copy, or die
if (!temp.renameTo(stored)) {
FileUtils.copyFile(temp, stored);
FileUtils.deleteQuietly(temp);
}
if (contentType == null) FileUtil.toFile(abs(stored)+".contentType", mimeType);

return path;

} catch (Exception e) {
return die("store: "+e, e);
}
}

@Override public void copy(String from, String to) {
try {
@Cleanup InputStream in = load(from).getStream();
store(in, to, to);
} catch (Exception e) {
die("copy: "+e, e);
}
}

@Override public boolean delete(String uri) {
if (!exists(uri)) return false;
final File f = uri2file(uri);
return f.exists() && f.delete();
}

}

+ 0
- 43
wizard-server/src/main/java/org/cobbzilla/wizard/asset/ResourceStorageService.java 파일 보기

@@ -1,43 +0,0 @@
package org.cobbzilla.wizard.asset;

import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.cobbzilla.util.io.StreamUtil;

import java.io.InputStream;
import java.util.Map;

import static org.cobbzilla.util.daemon.ZillaRuntime.notSupported;

@NoArgsConstructor @Accessors(chain=true) @Slf4j
public class ResourceStorageService extends AssetStorageService {

public static final String PROP_BASE = "base";

@Getter @Setter private String base;

public ResourceStorageService(String base) { this.base = base; }

public ResourceStorageService(Map<String, String> config) { this(config.get(PROP_BASE)); }

@Override public AssetStream load(String uri) {
try {
return new AssetStream(uri, StreamUtil.loadResourceAsStream(base + "/" + uri), getMimeType(uri));
} catch (Exception e) {
log.error("load: "+e);
}
return null;
}

@Override public boolean exists(String uri) { return load(uri) != null; }

@Override public String store(InputStream fileStream, String fileName, String uri) { return notSupported(); }

@Override public void copy(String from, String to) { notSupported(); }

@Override public boolean delete(String uri) { return notSupported(); }

}

+ 0
- 213
wizard-server/src/main/java/org/cobbzilla/wizard/asset/S3AssetStorageService.java 파일 보기

@@ -1,213 +0,0 @@
package org.cobbzilla.wizard.asset;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.internal.Mimetypes;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.cobbzilla.util.io.FileUtil;
import org.cobbzilla.util.system.Sleep;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import static org.cobbzilla.util.daemon.ZillaRuntime.die;
import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
import static org.cobbzilla.util.io.FileUtil.*;

@Slf4j
public class S3AssetStorageService extends AssetStorageService {

public static final int MAX_RETRIES = 10;
public static final long RETRY_SLEEP = TimeUnit.SECONDS.toMillis(5);

public static final String PROP_ACCESS_KEY = "accessKey";
public static final String PROP_SECRET_KEY = "secretKey";
public static final String PROP_BUCKET = "bucket";
public static final String PROP_PREFIX = "prefix";
public static final String PROP_LOCAL_CACHE = "localCache";
public static final String CACHE_DISABLED = "disabled";

@Getter @Setter private String accessKey;
@Getter @Setter private String secretKey;
@Getter @Setter private String bucket;
@Getter @Setter private String prefix;
@Getter @Setter private File localCache;

private final AmazonS3Client s3Client;

public S3AssetStorageService(Map<String, String> config) {
setAccessKey(config.get(PROP_ACCESS_KEY));
setSecretKey(config.get(PROP_SECRET_KEY));
setBucket(config.get(PROP_BUCKET));
setPrefix(config.get(PROP_PREFIX));

String local = config.get(PROP_LOCAL_CACHE);
if (empty(local)) local = System.getProperty("java.io.tmpdir");

if (local.equals(CACHE_DISABLED)) {
localCache = null;
} else {
setLocalCache(mkdirOrDie(local));
}

s3Client = new AmazonS3Client(new AWSCredentials() {
@Override public String getAWSAccessKeyId() { return getAccessKey(); }
@Override public String getAWSSecretKey() { return getSecretKey(); }
});
}

@Override public AssetStream load(String uri) {
if (localCache != null) {
final File cachefile = new File(abs(localCache) + "/" + uri);
if (cachefile.exists()) {
try {
return new AssetStream(uri, new FileInputStream(cachefile), toStringOrDie(abs(cachefile) + ".contentType"));
} catch (IOException e) {
die("load: " + e, e);
}
}
}
Exception lastException = null;
for (int tries = 0; tries < MAX_RETRIES; tries++) {
try {
synchronized (s3Client) {
final S3Object s3Object = s3Client.getObject(bucket, prefix + "/" + uri);
return new AssetStream(uri, s3Object.getObjectContent(), s3Object.getObjectMetadata().getContentType());
}
} catch (Exception e) {
log.warn("load("+uri+"): "+e);
lastException = e;
Sleep.sleep(RETRY_SLEEP);
}
}
return die("load("+uri+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException);
}

@Override public boolean exists(String uri) {
if (localCache != null && new File(abs(localCache) + "/" + uri).exists()) return true;
Exception lastException = null;
try {
for (int tries = 0; tries < MAX_RETRIES; tries++) {
try {
synchronized (s3Client) {
s3Client.getObject(bucket, prefix + "/" + uri);
return true;
}
} catch (Exception e) {
log.warn("exists("+uri+"): "+e);
lastException = e;
Sleep.sleep(RETRY_SLEEP);
}
}
return die("exists("+uri+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException);

} catch (Exception e) {
return false;
}
}

@Override public synchronized String store(InputStream fileStream, String filename, String path) {

final String mimeType = filename.endsWith(".json") ? "application/json" : Mimetypes.getInstance().getMimetype(filename);

File temp = null;
try {
temp = File.createTempFile("s3file-", ".tmp", getDefaultTempDir());
FileUtil.toFile(temp, fileStream);
if (path == null) path = getUri(temp, filename);

final File stored = (localCache == null) ? temp : new File(abs(localCache) + "/" + path);

if (localCache != null) {
mkdirOrDie(stored.getParentFile());
FileUtil.toFile(abs(stored) + ".contentType", mimeType);
if (!temp.renameTo(stored)) {
log.warn("store: error renaming file, copying instead: " + abs(temp) + " -> " + abs(stored));
FileUtil.copyFile(temp, stored);
if (!temp.delete()) log.warn("store: error deleting temp file after copy: "+abs(temp));
}
}

final ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType(mimeType);
metadata.setContentLength(stored.length());
put(path, stored, metadata);

return path;

} catch (Exception e) {
return die("store: "+e, e);

} finally {
if (localCache == null) {
if (temp != null && temp.exists()) {
if (!temp.delete()) log.warn("store: error deleting temp file: "+abs(temp));
}
}
}
}

public void put(String path, File stored, ObjectMetadata metadata) throws IOException {
if (metadata == null) {
put(path, stored);
} else {
// ok we have to use an input stream to do the metadata in the same call
Exception lastException = null;
try (InputStream in = new FileInputStream(stored)) {
for (int tries = 0; tries < MAX_RETRIES; tries++) {
try {
synchronized (s3Client) {
s3Client.putObject(bucket, prefix + "/" + path, in, metadata);
return;
}
} catch (Exception e) {
log.warn("put("+path+"): "+e);
lastException = e;
Sleep.sleep(RETRY_SLEEP);
}
}
die("put("+path+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException);
}
}
}

public void put(String path, File stored) throws IOException {
Exception lastException = null;
for (int tries = 0; tries < MAX_RETRIES; tries++) {
try {
synchronized (s3Client) {
s3Client.putObject(bucket, prefix + "/" + path, stored);
return;
}
} catch (Exception e) {
log.warn("put("+path+"): "+e);
lastException = e;
Sleep.sleep(RETRY_SLEEP);
}
}
die("put("+path+"), failed "+MAX_RETRIES+" times, giving up. lastException: "+lastException, lastException);
}

@Override public void copy(String from, String to) {
synchronized (s3Client) {
s3Client.copyObject(bucket, prefix + "/" + from, bucket, prefix + "/" + to);
}
}

@Override public boolean delete(String uri) {
if (!exists(uri)) return false;
synchronized (s3Client) {
s3Client.deleteObject(bucket, prefix + "/" + uri);
}
return true;
}

}

쓰기 미리보기
불러오는 중...
취소
저장