diff --git a/src/main/java/jvcl/model/JAsset.java b/src/main/java/jvcl/model/JAsset.java index 62fe528..51bf395 100644 --- a/src/main/java/jvcl/model/JAsset.java +++ b/src/main/java/jvcl/model/JAsset.java @@ -1,5 +1,6 @@ package jvcl.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import jvcl.model.info.JMediaInfo; import jvcl.service.AssetManager; @@ -16,7 +17,11 @@ import java.io.FileOutputStream; import java.io.InputStream; import java.io.OutputStream; import java.math.BigDecimal; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import static java.util.Comparator.comparing; import static org.cobbzilla.util.daemon.ZillaRuntime.*; import static org.cobbzilla.util.http.HttpSchemes.isHttpOrHttps; import static org.cobbzilla.util.io.FileUtil.abs; @@ -24,6 +29,7 @@ import static org.cobbzilla.util.io.FileUtil.mkdirOrDie; import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; import static org.cobbzilla.util.json.JsonUtil.json; import static org.cobbzilla.util.reflect.ReflectionUtil.copy; +import static org.cobbzilla.util.system.CommandShell.execScript; @NoArgsConstructor @AllArgsConstructor @Accessors(chain=true) @Slf4j public class JAsset { @@ -41,6 +47,25 @@ public class JAsset { // an asset can specify where its file should live // if the file already exists, it is used and not overwritten @Getter @Setter private String dest; + + public static List flattenAssetList(JAsset[] assets) { + final List list = new ArrayList<>(); + return _flatten(assets, list); + } + + private static List _flatten(JAsset[] assets, final List list) { + if (assets != null) { + for (final JAsset a : assets) { + if (a.hasList()) { + _flatten(a.getList(), list); + } else { + list.add(a); + } + } + } + return list; + } + public boolean hasDest() { return !empty(dest); } public boolean destExists() { return new File(dest).exists(); } public boolean destIsDirectory() { return new File(dest).isDirectory(); } @@ -54,7 +79,9 @@ public class JAsset { @Getter @Setter private JAsset[] list; public boolean hasList () { return list != null; } + public boolean hasListAssets () { return !empty(getList()); } public void addAsset(JAsset slice) { list = ArrayUtil.append(list, slice); } + @JsonIgnore public Integer getLength () { return hasList() ? list.length : null; } @Getter @Setter private JFormat format; public boolean hasFormat() { return format != null; } @@ -67,6 +94,19 @@ public class JAsset { } public boolean hasInfo() { return info != null; } + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final JAsset a = (JAsset) o; + return Objects.equals(path, a.path) && Arrays.equals(list, a.list); + } + + @Override public int hashCode() { + int result = Objects.hash(path); + result = 31 * result + Arrays.hashCode(list); + return result; + } + public static JAsset json2asset(JsonNode node) { if (node.isObject()) return json(node, JAsset.class); if (node.isTextual()) return new JAsset().setName(node.textValue()); @@ -88,8 +128,13 @@ public class JAsset { public BigDecimal duration() { return getInfo().duration(); } public JAsset init(AssetManager assetManager, Toolbox toolbox) { - initPath(assetManager); - setInfo(toolbox.getInfo(this)); + final JAsset asset = initPath(assetManager); + if (!asset.hasListAssets()) { + setInfo(toolbox.getInfo(this)); + } else { + setInfo(toolbox.getInfo(list[0])); + for (JAsset a : asset.getList()) a.setInfo(getInfo()); + } return this; } @@ -132,9 +177,32 @@ public class JAsset { } else { // must be a file - final File f = new File(path); - if (!f.exists()) return die("initPath: file path does not exist: "+path); - if (!f.canRead()) return die("initPath: file path is not readable: "+path); + // does it contain a wildcard? + if (path.contains("*")) { + // find matching files + final int starPos = path.indexOf("*"); + final int lastSlash = path.substring(0, starPos).lastIndexOf("/"); + final String[] parts = path.split("\\*"); + final StringBuilder b = new StringBuilder(); + Arrays.stream(parts).forEach(p -> { + if (b.length() > 0) b.append(".+"); + b.append(Pattern.quote(p)); + }); + final Pattern regex = Pattern.compile(b.toString()); + final File dir = new File(path.substring(0, lastSlash)); + final String filesInDir = execScript("find " + dir + " -type f"); + final Set matches = Arrays.stream(filesInDir.split("\n")) + .filter(f -> regex.matcher(f).matches()) + .collect(Collectors.toCollection(() -> new TreeSet<>(comparing(String::toString)))); + for (String f : matches) { + addAsset(new JAsset(this).setPath(f)); + } + + } else { + final File f = new File(path); + if (!f.exists()) return die("initPath: file path does not exist: "+path); + if (!f.canRead()) return die("initPath: file path is not readable: "+path); + } } return this; } diff --git a/src/main/java/jvcl/model/JOperation.java b/src/main/java/jvcl/model/JOperation.java index 95508e2..3063ffb 100644 --- a/src/main/java/jvcl/model/JOperation.java +++ b/src/main/java/jvcl/model/JOperation.java @@ -19,7 +19,7 @@ public class JOperation { public String hash(JAsset[] sources) { return hash(sources, null); } public String hash(JAsset[] sources, Object[] args) { - return hashOf(getOperation(), json(creates), json(perform), json(sources), args); + return hashOf(getOperation(), json(creates), json(perform), sources, args); } } diff --git a/src/main/java/jvcl/op/ConcatOperation.java b/src/main/java/jvcl/op/ConcatOperation.java index 5c082a1..a4ac718 100644 --- a/src/main/java/jvcl/op/ConcatOperation.java +++ b/src/main/java/jvcl/op/ConcatOperation.java @@ -14,8 +14,10 @@ import org.cobbzilla.util.handlebars.HandlebarsUtil; import java.io.File; import java.util.HashMap; +import java.util.List; import java.util.Map; +import static jvcl.model.JAsset.flattenAssetList; import static jvcl.model.JAsset.json2asset; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; @@ -26,13 +28,13 @@ import static org.cobbzilla.util.system.CommandShell.execScript; @Slf4j public class ConcatOperation implements JOperator { - public static final String CONCAT_RECODE_TEMPLATE_1 - // list inputs - = "{{ffmpeg}} {{#each sources}} -i {{{this.path}}}{{/each}} " - // concat them together - + "-f concat -safe 0 -c copy {{{output.path}}}"; + public static final String CONCAT_RECODE_TEMPLATE_OLD + // concat inputs + = "{{ffmpeg}} -f concat{{#each sources}} -i {{{this.path}}}{{/each}} " + // safely with copy codec + + "-safe 0 -c copy {{{output.path}}}"; - public static final String CONCAT_RECODE_TEMPLATE_2 + public static final String CONCAT_RECODE_TEMPLATE_1 // list inputs = "{{ffmpeg}} {{#each sources}} -i {{{this.path}}}{{/each}} " @@ -50,22 +52,30 @@ public class ConcatOperation implements JOperator { final ConcatConfig config = json(json(op.getPerform()), ConcatConfig.class); // validate sources - final JAsset[] sources = assetManager.resolve(config.getConcat()); + final List sources = flattenAssetList(assetManager.resolve(config.getConcat())); if (empty(sources)) die("operate: no sources"); // create output object final JAsset output = json2asset(op.getCreates()); + if (output.hasDest() && output.destExists()) { + log.info("operate: dest exists, not re-creating: "+output.getDest()); + return; + } // if any format settings are missing, use settings from first source - output.mergeFormat(sources[0].getFormat()); + output.mergeFormat(sources.get(0).getFormat()); // set the path, check if output asset already exists final JFileExtension formatType = output.getFormat().getFileExtension(); - final File outfile = assetManager.assetPath(op, sources, formatType); + final File outfile = output.hasDest() + ? new File(output.getDest()) + : assetManager.assetPath(op, sources, formatType); if (outfile.exists()) { log.info("operate: outfile exists, not re-creating: "+abs(outfile)); return; } + if (!outfile.getParentFile().canWrite()) die("operate: cannot write file (parent directory not writeable): "+abs(outfile)); + output.setPath(abs(outfile)); final Map ctx = new HashMap<>(); diff --git a/src/main/java/jvcl/op/SplitOperation.java b/src/main/java/jvcl/op/SplitOperation.java index e0a8ecf..49ad313 100644 --- a/src/main/java/jvcl/op/SplitOperation.java +++ b/src/main/java/jvcl/op/SplitOperation.java @@ -30,7 +30,7 @@ import static org.cobbzilla.util.system.CommandShell.execScript; public class SplitOperation implements JOperator { public static final String SPLIT_TEMPLATE - = "{{ffmpeg}} -i {{{source.path}}} -ss {{startSeconds}} -t {{endSeconds}} {{{output.path}}}"; + = "{{ffmpeg}} -i {{{source.path}}} -ss {{startSeconds}} -t {{interval}} {{{output.path}}}"; @Override public void operate(JOperation op, Toolbox toolbox, AssetManager assetManager) { final SplitConfig config = json(json(op.getPerform()), SplitConfig.class); @@ -83,7 +83,7 @@ public class SplitOperation implements JOperator { ctx.put("output", slice); ctx.put("startSeconds", i); - ctx.put("endSeconds", i.add(incr)); + ctx.put("interval", incr); final String script = HandlebarsUtil.apply(toolbox.getHandlebars(), SPLIT_TEMPLATE, ctx); log.debug("operate: running script: "+script); diff --git a/src/main/java/jvcl/service/AssetManager.java b/src/main/java/jvcl/service/AssetManager.java index d17f128..e0065a7 100644 --- a/src/main/java/jvcl/service/AssetManager.java +++ b/src/main/java/jvcl/service/AssetManager.java @@ -3,19 +3,24 @@ package jvcl.service; import jvcl.model.JAsset; import jvcl.model.JFileExtension; import jvcl.model.JOperation; +import org.cobbzilla.util.handlebars.HandlebarsUtil; import java.io.File; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import static java.lang.Integer.parseInt; import static jvcl.model.JAsset.NULL_ASSET; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.string.StringUtil.isOnlyDigits; public class AssetManager { public static final String OUTFILE_PREFIX = AssetManager.class.getSimpleName() + ".output."; + public static final String RANGE_SEP = ".."; private final Toolbox toolbox; private final File scratchDir; @@ -40,6 +45,10 @@ public class AssetManager { return assetPath(op, sources, formatType, null); } + public File assetPath(JOperation op, List sources, JFileExtension formatType) { + return assetPath(op, sources.toArray(JAsset[]::new), formatType, null); + } + public File assetPath(JOperation op, JAsset[] sources, JFileExtension formatType, Object[] args) { return new File(scratchDir, OUTFILE_PREFIX + op.hash(sources, args) @@ -93,8 +102,72 @@ public class AssetManager { } public JAsset resolve(String name) { - final JAsset asset = assets.get(name); + final JAsset asset; + if (name.contains("[") && name.contains("]")) { + final int openPos = name.indexOf("["); + final int closePos = name.indexOf("]"); + final String assetName = name.substring(0, openPos); + asset = assets.get(assetName); + if (asset == null) return die("resolve("+name+"): asset not found: "+assetName); + if (!asset.hasList()) return die("resolve("+name+"): "+assetName+" is not a list asset"); + final JAsset[] list = asset.getList(); + + final String indexExpr = name.substring(openPos+1, closePos); + if (indexExpr.contains(RANGE_SEP)) { + final int dotPos = indexExpr.indexOf(RANGE_SEP); + final String from = indexExpr.substring(0, dotPos); + final String to = indexExpr.substring(dotPos+RANGE_SEP.length()); + final int fromIndex = parseIndexExpression(from, list, JIndexType.from); + final int toIndex = parseIndexExpression(to, list, JIndexType.to); + if (toIndex < fromIndex) return die("parseIndexExpression("+indexExpr+"): 'to' index ("+toIndex+") < 'from' index ("+fromIndex+")"); + final int len = toIndex - fromIndex; + final JAsset[] subList = new JAsset[len]; + System.arraycopy(list, fromIndex, subList, 0, len); + return new JAsset(asset).setList(subList); + + } else { + // single element + final int index = parseIndexExpression(indexExpr, list, JIndexType.single); + if (index < 0 || index >= list.length) return die("parseIndexExpression("+indexExpr+"): index out of range: "+index); + return list[index]; + } + + } else { + asset = assets.get(name); + } return asset == null ? die("resolve("+name+")") : asset; } + private int parseIndexExpression(String indexExpr, JAsset[] list, JIndexType type) { + if (empty(indexExpr)) { + switch (type) { + case from: return 0; + case to: return list.length; + case single: return die("parseIndexExpression(): no expression provided!"); + default: return die("parseIndexExpression(): invalid type: "+type); + } + } + final int index; + if (isOnlyDigits(indexExpr)) { + try { + index = parseInt(indexExpr); + } catch (Exception e) { + return die("parseIndexExpression("+indexExpr+"): not an integer: "+indexExpr); + } + + } else { + // not all digits, evaluate as handlebars expression + final Map ctx = new HashMap<>(assets); + final String indexString = HandlebarsUtil.apply(toolbox.getHandlebars(), "{{{" + indexExpr + "}}}", ctx); + if (empty(indexString)) return die("parseIndexExpression("+indexExpr+"): index expression resolved to empty string: "+indexExpr); + try { + index = parseInt(indexString); + } catch (Exception e) { + return die("parseIndexExpression("+indexExpr+"): index expression did not evaluate to an integer: "+indexString); + } + } + if (index < 0 || index > list.length) return die("parseIndexExpression("+indexExpr+"): '"+type+"' index out of range: "+index); + return index; + } + } diff --git a/src/main/java/jvcl/service/JIndexType.java b/src/main/java/jvcl/service/JIndexType.java new file mode 100644 index 0000000..cf60368 --- /dev/null +++ b/src/main/java/jvcl/service/JIndexType.java @@ -0,0 +1,11 @@ +package jvcl.service; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public enum JIndexType { + + single, from, to; + + @JsonCreator public static JIndexType fromString (String v) { return valueOf(v.toLowerCase()); } + +} diff --git a/src/main/java/jvcl/service/Toolbox.java b/src/main/java/jvcl/service/Toolbox.java index 5f5677f..39bd33b 100644 --- a/src/main/java/jvcl/service/Toolbox.java +++ b/src/main/java/jvcl/service/Toolbox.java @@ -70,7 +70,12 @@ public class Toolbox { if (!infoFile.exists() || infoFile.length() == 0) { return die("getInfo: info file was not created or was empty: "+infoPath); } - return infoCache.computeIfAbsent(infoPath, p -> json(FileUtil.toStringOrDie(infoFile), JMediaInfo.class, FULL_MAPPER_ALLOW_UNKNOWN_FIELDS)); + return infoCache.computeIfAbsent(infoPath, p -> { + try { + return json(FileUtil.toStringOrDie(infoFile), JMediaInfo.class, FULL_MAPPER_ALLOW_UNKNOWN_FIELDS); + } catch (Exception e) { + return die("getInfo: "+shortError(e), e); + } + }); } - } diff --git a/src/test/java/javicle/test/BasicTest.java b/src/test/java/javicle/test/BasicTest.java index 6654e7d..a3b41f2 100644 --- a/src/test/java/javicle/test/BasicTest.java +++ b/src/test/java/javicle/test/BasicTest.java @@ -1,21 +1,25 @@ package javicle.test; import jvcl.main.Jvcl; -import jvcl.main.JvclOptions; import lombok.Cleanup; import org.junit.Test; import java.io.File; +import static jvcl.main.JvclOptions.LONGOPT_SPEC; import static org.cobbzilla.util.io.FileUtil.abs; import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; import static org.cobbzilla.util.io.StreamUtil.stream2file; public class BasicTest { - @Test public void testSplitAndConcat () throws Exception { - @Cleanup("delete") final File specFile = stream2file(loadResourceAsStream("tests/test_split.json")); - Jvcl.main(new String[]{JvclOptions.LONGOPT_SPEC, abs(specFile)}); + @Test public void testSplit() { runSpec("tests/test_split.json"); } + + @Test public void testConcat() { runSpec("tests/test_concat.json"); } + + private void runSpec(String specPath) { + @Cleanup("delete") final File specFile = stream2file(loadResourceAsStream(specPath)); + Jvcl.main(new String[]{LONGOPT_SPEC, abs(specFile)}); } } diff --git a/src/test/resources/tests/test_concat.json b/src/test/resources/tests/test_concat.json index 874587b..a0fd053 100644 --- a/src/test/resources/tests/test_concat.json +++ b/src/test/resources/tests/test_concat.json @@ -1,13 +1,16 @@ { "assets": [ - { "name": "vid1_splits" } + { "name": "vid1_splits", "path": "src/test/resources/outputs/vid1_splits_*.mp4" } ], "operations": [ { "operation": "concat", - "creates": "combined_vid", + "creates": { + "name": "combined_vid", + "dest": "src/test/resources/outputs/combined.mp4" + }, "perform": { - "concat": ["vid1_splits[1..{{vid1_splits.length}}"] + "concat": ["vid1_splits[1..]"] } } ] diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils index bcd403f..9f3bc57 160000 --- a/utils/cobbzilla-utils +++ b/utils/cobbzilla-utils @@ -1 +1 @@ -Subproject commit bcd403f6fdf14985cd99dcbeee257683abd47aaf +Subproject commit 9f3bc574b0e9fc99b41b381116ff37a9b8844eaf