split and concat basics working

src/main/java/jvcl/model/JAsset.java

@@ -1,5 +1,6 @@
 package jvcl.model;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.databind.JsonNode;
 import jvcl.model.info.JMediaInfo;
 import jvcl.service.AssetManager;
@@ -16,7 +17,11 @@ import java.io.FileOutputStream;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.math.BigDecimal;
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
+import static java.util.Comparator.comparing;
 import static org.cobbzilla.util.daemon.ZillaRuntime.*;
 import static org.cobbzilla.util.http.HttpSchemes.isHttpOrHttps;
 import static org.cobbzilla.util.io.FileUtil.abs;
@@ -24,6 +29,7 @@ import static org.cobbzilla.util.io.FileUtil.mkdirOrDie;
 import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream;
 import static org.cobbzilla.util.json.JsonUtil.json;
 import static org.cobbzilla.util.reflect.ReflectionUtil.copy;
+import static org.cobbzilla.util.system.CommandShell.execScript;
 
 @NoArgsConstructor @AllArgsConstructor @Accessors(chain=true) @Slf4j
 public class JAsset {
@@ -41,6 +47,25 @@ public class JAsset {
     // an asset can specify where its file should live
     // if the file already exists, it is used and not overwritten
     @Getter @Setter private String dest;
+
+    public static List<JAsset> flattenAssetList(JAsset[] assets) {
+        final List<JAsset> list = new ArrayList<>();
+        return _flatten(assets, list);
+    }
+
+    private static List<JAsset> _flatten(JAsset[] assets, final List<JAsset> list) {
+        if (assets != null) {
+            for (final JAsset a : assets) {
+                if (a.hasList()) {
+                    _flatten(a.getList(), list);
+                } else {
+                    list.add(a);
+                }
+            }
+        }
+        return list;
+    }
+
     public boolean hasDest() { return !empty(dest); }
     public boolean destExists() { return new File(dest).exists(); }
     public boolean destIsDirectory() { return new File(dest).isDirectory(); }
@@ -54,7 +79,9 @@ public class JAsset {
 
     @Getter @Setter private JAsset[] list;
     public boolean hasList () { return list != null; }
+    public boolean hasListAssets () { return !empty(getList()); }
     public void addAsset(JAsset slice) { list = ArrayUtil.append(list, slice); }
+    @JsonIgnore public Integer getLength () { return hasList() ? list.length : null; }
 
     @Getter @Setter private JFormat format;
     public boolean hasFormat() { return format != null; }
@@ -67,6 +94,19 @@ public class JAsset {
     }
     public boolean hasInfo() { return info != null; }
 
+    @Override public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final JAsset a = (JAsset) o;
+        return Objects.equals(path, a.path) && Arrays.equals(list, a.list);
+    }
+
+    @Override public int hashCode() {
+        int result = Objects.hash(path);
+        result = 31 * result + Arrays.hashCode(list);
+        return result;
+    }
+
     public static JAsset json2asset(JsonNode node) {
         if (node.isObject()) return json(node, JAsset.class);
         if (node.isTextual()) return new JAsset().setName(node.textValue());
@@ -88,8 +128,13 @@ public class JAsset {
     public BigDecimal duration() { return getInfo().duration(); }
 
     public JAsset init(AssetManager assetManager, Toolbox toolbox) {
-        initPath(assetManager);
-        setInfo(toolbox.getInfo(this));
+        final JAsset asset = initPath(assetManager);
+        if (!asset.hasListAssets()) {
+            setInfo(toolbox.getInfo(this));
+        } else {
+            setInfo(toolbox.getInfo(list[0]));
+            for (JAsset a : asset.getList()) a.setInfo(getInfo());
+        }
         return this;
     }
 
@@ -132,9 +177,32 @@ public class JAsset {
 
         } else {
             // must be a file
-            final File f = new File(path);
-            if (!f.exists()) return die("initPath: file path does not exist: "+path);
-            if (!f.canRead()) return die("initPath: file path is not readable: "+path);
+            // does it contain a wildcard?
+            if (path.contains("*")) {
+                // find matching files
+                final int starPos = path.indexOf("*");
+                final int lastSlash = path.substring(0, starPos).lastIndexOf("/");
+                final String[] parts = path.split("\\*");
+                final StringBuilder b = new StringBuilder();
+                Arrays.stream(parts).forEach(p -> {
+                    if (b.length() > 0) b.append(".+");
+                    b.append(Pattern.quote(p));
+                });
+                final Pattern regex = Pattern.compile(b.toString());
+                final File dir = new File(path.substring(0, lastSlash));
+                final String filesInDir = execScript("find " + dir + " -type f");
+                final Set<String> matches = Arrays.stream(filesInDir.split("\n"))
+                        .filter(f -> regex.matcher(f).matches())
+                        .collect(Collectors.toCollection(() -> new TreeSet<>(comparing(String::toString))));
+                for (String f : matches) {
+                    addAsset(new JAsset(this).setPath(f));
+                }
+
+            } else {
+                final File f = new File(path);
+                if (!f.exists()) return die("initPath: file path does not exist: "+path);
+                if (!f.canRead()) return die("initPath: file path is not readable: "+path);
+            }
         }
         return this;
     }

src/main/java/jvcl/model/JOperation.java

@@ -19,7 +19,7 @@ public class JOperation {
     public String hash(JAsset[] sources) { return hash(sources, null); }
 
     public String hash(JAsset[] sources, Object[] args) {
-        return hashOf(getOperation(), json(creates), json(perform), json(sources), args);
+        return hashOf(getOperation(), json(creates), json(perform), sources, args);
     }
 
 }

src/main/java/jvcl/op/ConcatOperation.java

@@ -14,8 +14,10 @@ import org.cobbzilla.util.handlebars.HandlebarsUtil;
 
 import java.io.File;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import static jvcl.model.JAsset.flattenAssetList;
 import static jvcl.model.JAsset.json2asset;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
@@ -26,13 +28,13 @@ import static org.cobbzilla.util.system.CommandShell.execScript;
 @Slf4j
 public class ConcatOperation implements JOperator {
 
-    public static final String CONCAT_RECODE_TEMPLATE_1
-            // list inputs
-            = "{{ffmpeg}} {{#each sources}} -i {{{this.path}}}{{/each}} "
-            // concat them together
-            + "-f concat -safe 0 -c copy {{{output.path}}}";
+    public static final String CONCAT_RECODE_TEMPLATE_OLD
+            // concat inputs
+            = "{{ffmpeg}} -f concat{{#each sources}} -i {{{this.path}}}{{/each}} "
+            // safely with copy codec
+            + "-safe 0 -c copy {{{output.path}}}";
 
-    public static final String CONCAT_RECODE_TEMPLATE_2
+    public static final String CONCAT_RECODE_TEMPLATE_1
             // list inputs
             = "{{ffmpeg}} {{#each sources}} -i {{{this.path}}}{{/each}} "
 
@@ -50,22 +52,30 @@ public class ConcatOperation implements JOperator {
         final ConcatConfig config = json(json(op.getPerform()), ConcatConfig.class);
 
         // validate sources
-        final JAsset[] sources = assetManager.resolve(config.getConcat());
+        final List<JAsset> sources = flattenAssetList(assetManager.resolve(config.getConcat()));
         if (empty(sources)) die("operate: no sources");
 
         // create output object
         final JAsset output = json2asset(op.getCreates());
+        if (output.hasDest() && output.destExists()) {
+            log.info("operate: dest exists, not re-creating: "+output.getDest());
+            return;
+        }
 
         // if any format settings are missing, use settings from first source
-        output.mergeFormat(sources[0].getFormat());
+        output.mergeFormat(sources.get(0).getFormat());
 
         // set the path, check if output asset already exists
         final JFileExtension formatType = output.getFormat().getFileExtension();
-        final File outfile = assetManager.assetPath(op, sources, formatType);
+        final File outfile = output.hasDest()
+                ? new File(output.getDest())
+                : assetManager.assetPath(op, sources, formatType);
         if (outfile.exists()) {
             log.info("operate: outfile exists, not re-creating: "+abs(outfile));
             return;
         }
+        if (!outfile.getParentFile().canWrite()) die("operate: cannot write file (parent directory not writeable): "+abs(outfile));
+
         output.setPath(abs(outfile));
 
         final Map<String, Object> ctx = new HashMap<>();

src/main/java/jvcl/op/SplitOperation.java

@@ -30,7 +30,7 @@ import static org.cobbzilla.util.system.CommandShell.execScript;
 public class SplitOperation implements JOperator {
 
     public static final String SPLIT_TEMPLATE
-            = "{{ffmpeg}} -i {{{source.path}}} -ss {{startSeconds}} -t {{endSeconds}} {{{output.path}}}";
+            = "{{ffmpeg}} -i {{{source.path}}} -ss {{startSeconds}} -t {{interval}} {{{output.path}}}";
 
     @Override public void operate(JOperation op, Toolbox toolbox, AssetManager assetManager) {
         final SplitConfig config = json(json(op.getPerform()), SplitConfig.class);
@@ -83,7 +83,7 @@ public class SplitOperation implements JOperator {
 
             ctx.put("output", slice);
             ctx.put("startSeconds", i);
-            ctx.put("endSeconds", i.add(incr));
+            ctx.put("interval", incr);
             final String script = HandlebarsUtil.apply(toolbox.getHandlebars(), SPLIT_TEMPLATE, ctx);
 
             log.debug("operate: running script: "+script);

src/main/java/jvcl/service/AssetManager.java

@@ -3,19 +3,24 @@ package jvcl.service;
 import jvcl.model.JAsset;
 import jvcl.model.JFileExtension;
 import jvcl.model.JOperation;
+import org.cobbzilla.util.handlebars.HandlebarsUtil;
 
 import java.io.File;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import static java.lang.Integer.parseInt;
 import static jvcl.model.JAsset.NULL_ASSET;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
+import static org.cobbzilla.util.string.StringUtil.isOnlyDigits;
 
 public class AssetManager {
 
     public static final String OUTFILE_PREFIX = AssetManager.class.getSimpleName() + ".output.";
+    public static final String RANGE_SEP = "..";
 
     private final Toolbox toolbox;
     private final File scratchDir;
@@ -40,6 +45,10 @@ public class AssetManager {
         return assetPath(op, sources, formatType, null);
     }
 
+    public File assetPath(JOperation op, List<JAsset> sources, JFileExtension formatType) {
+        return assetPath(op, sources.toArray(JAsset[]::new), formatType, null);
+    }
+
     public File assetPath(JOperation op, JAsset[] sources, JFileExtension formatType, Object[] args) {
         return new File(scratchDir, OUTFILE_PREFIX
                 + op.hash(sources, args)
@@ -93,8 +102,72 @@ public class AssetManager {
     }
 
     public JAsset resolve(String name) {
-        final JAsset asset = assets.get(name);
+        final JAsset asset;
+        if (name.contains("[") && name.contains("]")) {
+            final int openPos = name.indexOf("[");
+            final int closePos = name.indexOf("]");
+            final String assetName = name.substring(0, openPos);
+            asset = assets.get(assetName);
+            if (asset == null) return die("resolve("+name+"): asset not found: "+assetName);
+            if (!asset.hasList()) return die("resolve("+name+"): "+assetName+" is not a list asset");
+            final JAsset[] list = asset.getList();
+
+            final String indexExpr = name.substring(openPos+1, closePos);
+            if (indexExpr.contains(RANGE_SEP)) {
+                final int dotPos = indexExpr.indexOf(RANGE_SEP);
+                final String from = indexExpr.substring(0, dotPos);
+                final String to = indexExpr.substring(dotPos+RANGE_SEP.length());
+                final int fromIndex = parseIndexExpression(from, list, JIndexType.from);
+                final int toIndex = parseIndexExpression(to, list, JIndexType.to);
+                if (toIndex < fromIndex) return die("parseIndexExpression("+indexExpr+"): 'to' index ("+toIndex+") < 'from' index ("+fromIndex+")");
+                final int len = toIndex - fromIndex;
+                final JAsset[] subList = new JAsset[len];
+                System.arraycopy(list, fromIndex, subList, 0, len);
+                return new JAsset(asset).setList(subList);
+
+            } else {
+                // single element
+                final int index = parseIndexExpression(indexExpr, list, JIndexType.single);
+                if (index < 0 || index >= list.length) return die("parseIndexExpression("+indexExpr+"): index out of range: "+index);
+                return list[index];
+            }
+
+        } else {
+            asset = assets.get(name);
+        }
         return asset == null ? die("resolve("+name+")") : asset;
     }
 
+    private int parseIndexExpression(String indexExpr, JAsset[] list, JIndexType type) {
+        if (empty(indexExpr)) {
+            switch (type) {
+                case from: return 0;
+                case to: return list.length;
+                case single: return die("parseIndexExpression(): no expression provided!");
+                default: return die("parseIndexExpression(): invalid type: "+type);
+            }
+        }
+        final int index;
+        if (isOnlyDigits(indexExpr)) {
+            try {
+                index = parseInt(indexExpr);
+            } catch (Exception e) {
+                return die("parseIndexExpression("+indexExpr+"): not an integer: "+indexExpr);
+            }
+
+        } else {
+            // not all digits, evaluate as handlebars expression
+            final Map<String, Object> ctx = new HashMap<>(assets);
+            final String indexString = HandlebarsUtil.apply(toolbox.getHandlebars(), "{{{" + indexExpr + "}}}", ctx);
+            if (empty(indexString)) return die("parseIndexExpression("+indexExpr+"): index expression resolved to empty string: "+indexExpr);
+            try {
+                index = parseInt(indexString);
+            } catch (Exception e) {
+                return die("parseIndexExpression("+indexExpr+"): index expression did not evaluate to an integer: "+indexString);
+            }
+        }
+        if (index < 0 || index > list.length) return die("parseIndexExpression("+indexExpr+"): '"+type+"' index out of range: "+index);
+        return index;
+    }
+
 }

src/main/java/jvcl/service/JIndexType.java

@@ -0,0 +1,11 @@
+package jvcl.service;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+public enum JIndexType {
+
+    single, from, to;
+
+    @JsonCreator public static JIndexType fromString (String v) { return valueOf(v.toLowerCase()); }
+
+}

src/main/java/jvcl/service/Toolbox.java

@@ -70,7 +70,12 @@ public class Toolbox {
         if (!infoFile.exists() || infoFile.length() == 0) {
             return die("getInfo: info file was not created or was empty: "+infoPath);
         }
-        return infoCache.computeIfAbsent(infoPath, p -> json(FileUtil.toStringOrDie(infoFile), JMediaInfo.class, FULL_MAPPER_ALLOW_UNKNOWN_FIELDS));
+        return infoCache.computeIfAbsent(infoPath, p -> {
+            try {
+                return json(FileUtil.toStringOrDie(infoFile), JMediaInfo.class, FULL_MAPPER_ALLOW_UNKNOWN_FIELDS);
+            } catch (Exception e) {
+                return die("getInfo: "+shortError(e), e);
+            }
+        });
     }
-
 }

src/test/java/javicle/test/BasicTest.java

@@ -1,21 +1,25 @@
 package javicle.test;
 
 import jvcl.main.Jvcl;
-import jvcl.main.JvclOptions;
 import lombok.Cleanup;
 import org.junit.Test;
 
 import java.io.File;
 
+import static jvcl.main.JvclOptions.LONGOPT_SPEC;
 import static org.cobbzilla.util.io.FileUtil.abs;
 import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream;
 import static org.cobbzilla.util.io.StreamUtil.stream2file;
 
 public class BasicTest {
 
-    @Test public void testSplitAndConcat () throws Exception {
-        @Cleanup("delete") final File specFile = stream2file(loadResourceAsStream("tests/test_split.json"));
-        Jvcl.main(new String[]{JvclOptions.LONGOPT_SPEC, abs(specFile)});
+    @Test public void testSplit() { runSpec("tests/test_split.json"); }
+
+    @Test public void testConcat() { runSpec("tests/test_concat.json"); }
+
+    private void runSpec(String specPath) {
+        @Cleanup("delete") final File specFile = stream2file(loadResourceAsStream(specPath));
+        Jvcl.main(new String[]{LONGOPT_SPEC, abs(specFile)});
     }
 
 }

src/test/resources/tests/test_concat.json

@@ -1,13 +1,16 @@
 {
   "assets": [
-    { "name": "vid1_splits" }
+    { "name": "vid1_splits", "path": "src/test/resources/outputs/vid1_splits_*.mp4" }
   ],
   "operations": [
     {
       "operation": "concat",
-      "creates": "combined_vid",
+      "creates": {
+        "name": "combined_vid",
+        "dest": "src/test/resources/outputs/combined.mp4"
+      },
       "perform": {
-        "concat": ["vid1_splits[1..{{vid1_splits.length}}"]
+        "concat": ["vid1_splits[1..]"]
       }
     }
   ]

utils/cobbzilla-utils

@@ -1 +1 @@
-Subproject commit bcd403f6fdf14985cd99dcbeee257683abd47aaf
+Subproject commit 9f3bc574b0e9fc99b41b381116ff37a9b8844eaf