improve handling of files with missing audio or video track

README.md

@@ -90,7 +90,7 @@ goes to new files.
 # Requirements
  * Java 11
  * Maven 3
- * ffmpeg
+ * ffmpeg (`HEAD` is required for `overlay` and `merge-audio` operations)
  * mediainfo
 
 These programs should executable (given your `PATH`): `javac`, `java`, `mvn`,

docs/jvc_js.md

@@ -82,6 +82,12 @@ An array of the audio tracks in a video.
 #### `videoTracks`
 An array of the video tracks in a video.
 
+#### `audioTrack`
+The track number of the first audio track.
+
+#### `videoTrack`
+The track number of the first video track.
+
 #### `hasAudio`
 A boolean, true if the asset has any audio tracks, false otherwise
 

src/main/java/jvc/model/JAsset.java

@@ -47,7 +47,9 @@ public class JAsset implements JsObjectView {
 
     public JAsset(JAsset other, File file) { this(other); setPath(abs(file)); }
 
-    @Getter @Setter private String name;
+    @Setter private String name;
+    public String getName () { return empty(name) ? basename(path) : name; }
+
     @Getter @Setter private String path;
 
     public boolean hasPath() { return !empty(path); }
@@ -159,9 +161,11 @@ public class JAsset implements JsObjectView {
 
     public BigDecimal width() { return hasInfo() ? getInfo().width() : null; }
     @JsonIgnore public BigDecimal getWidth () { return width(); }
+    public boolean hasWidth () { return width() != null; }
 
     public BigDecimal height() { return hasInfo() ? getInfo().height() : null; }
     @JsonIgnore public BigDecimal getHeight () { return height(); }
+    public boolean hasHeight () { return height() != null; }
 
     public int numTracks(JTrackType type) { return hasInfo() ? getInfo().numTracks(type) : 0; }
 

src/main/java/jvc/model/js/JAssetJs.java

@@ -26,6 +26,10 @@ public class JAssetJs {
     @Getter public JTrackJs[] tracks = EMPTY_TRACKS;
     @Getter public JTrackJs[] videoTracks = EMPTY_TRACKS;
     @Getter public JTrackJs[] audioTracks = EMPTY_TRACKS;
+
+    @Getter public final Integer videoTrack;
+    @Getter public final Integer audioTrack;
+
     @Getter public JAssetJs[] assets = EMPTY_ASSETS;
     @Getter public final boolean hasAudio;
     @Getter public final boolean hasVideo;
@@ -44,10 +48,13 @@ public class JAssetJs {
 
         this.aspectRatio = asset.aspectRatio() == null ? Double.NaN : asset.aspectRatio().doubleValue();
         this.channelLayout = asset.hasChannelLayout() ? asset.channelLayout() : null;
-        this.samplingRate = asset.hasSamplingRate() ? asset.samplingRate().intValue() : 0;
+        this.samplingRate = asset.hasSamplingRate() ? asset.samplingRate().intValue() : null;
 
+        Integer vTrack = null;
+        Integer aTrack = null;
         if (asset.hasInfo()) {
             final JMediaInfo info = asset.getInfo();
+            int trackCount = 0;
             for (JTrack track : info.getMedia().getTrack()) {
 
                 final JTrackJs trackJs = new JTrackJs(track.type().name());
@@ -59,13 +66,18 @@ public class JAssetJs {
                 switch (track.type()) {
                     case audio:
                         audioTracks = ArrayUtil.append(audioTracks, trackJs);
+                        if (aTrack == null) aTrack = trackCount;
                         break;
                     case video:
                         videoTracks = ArrayUtil.append(videoTracks, trackJs);
+                        if (vTrack == null) vTrack = trackCount;
                         break;
                 }
+                trackCount++;
             }
         }
+        this.videoTrack = vTrack;
+        this.audioTrack = aTrack;
 
         if (asset.hasListAssets()) {
             final JAsset[] list = asset.getList();

src/main/java/jvc/model/operation/JMultiSourceOperation.java

@@ -6,6 +6,7 @@ import jvc.service.AssetManager;
 import jvc.service.Toolbox;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.experimental.Accessors;
 
 import java.util.List;
 
@@ -14,6 +15,7 @@ import static jvc.model.JAsset.json2asset;
 import static org.cobbzilla.util.daemon.ZillaRuntime.die;
 import static org.cobbzilla.util.daemon.ZillaRuntime.empty;
 
+@Accessors(chain=true)
 public abstract class JMultiSourceOperation extends JOperation {
 
     @Getter @Setter private String[] sources;

src/main/java/jvc/operation/ConcatOperation.java

@@ -1,7 +1,18 @@
 package jvc.operation;
 
 import jvc.model.operation.JMultiSourceOperation;
+import lombok.Getter;
+import lombok.Setter;
+import lombok.experimental.Accessors;
 import lombok.extern.slf4j.Slf4j;
 
-@Slf4j
-public class ConcatOperation extends JMultiSourceOperation {}
+@Slf4j @Accessors(chain=true)
+public class ConcatOperation extends JMultiSourceOperation {
+
+    @Getter @Setter private Boolean audioOnly;
+    public boolean audioOnly () { return audioOnly != null && audioOnly; }
+
+    @Getter @Setter private Boolean videoOnly;
+    public boolean videoOnly () { return videoOnly != null && videoOnly; }
+
+}

src/main/java/jvc/operation/exec/ConcatExec.java

@@ -2,6 +2,7 @@ package jvc.operation.exec;
 
 import jvc.model.JAsset;
 import jvc.model.JStreamType;
+import jvc.model.js.JAssetJs;
 import jvc.model.operation.JMultiOperationContext;
 import jvc.operation.ConcatOperation;
 import jvc.service.AssetManager;
@@ -23,13 +24,31 @@ public class ConcatExec extends ExecBase<ConcatOperation> {
             = "{{ffmpeg}} {{#each sources}} -i {{{this.path}}}{{/each}} "
 
             // filter: list inputs
-            + "-filter_complex \"{{#each sources}}[{{@index}}:v] [{{@index}}:a] {{/each}} "
+            + "-filter_complex \""
+
+            // create null sources for missing audio/video streams
+            + "{{#each sources}}"
+              + "{{#if anyVideo}}{{#unless hasVideo}} nullsrc=s={{firstWidth}}x{{firstHeight}}:duration={{duration}} [null_video_{{@index}}]; {{/unless}}{{/if}}"
+              + "{{#if anyAudio}}{{#unless hasAudio}} anullsrc=channel_layout={{firstChannelLayout}}:sample_rate={{firstSamplingRate}}:duration={{duration}} [null_audio_{{@index}}]; {{/unless}}{{/if}}"
+            + "{{/each}} "
+
+            + "{{#each sources}}"
+              + "{{#if anyVideo}}{{#if hasVideo}}[{{@index}}:{{videoTrack}}]{{else}}[null_video_{{@index}}]{{/if}}{{/if}} "
+              + "{{#if anyAudio}}{{#if hasAudio}}[{{@index}}:{{audioTrack}}]{{else}}[null_audio_{{@index}}]{{/if}}{{/if}} "
+            + "{{/each}} "
 
             // filter: concat filter them together
-            + "concat=n={{sources.length}}:v=1:a=1 [v] [a]\" "
+            + "concat=n={{sources.length}}"
+            + ":v={{#if anyVideo}}1{{else}}0{{/if}}"
+            + ":a={{#if anyAudio}}1{{else}}0{{/if}} "
+            + "{{#if anyVideo}}[v]{{/if}} "
+            + "{{#if anyAudio}}[a]{{/if}}"
+            + "\" "
 
             // output combined result
-            + "-map \"[v]\" -map \"[a]\" -y {{{output.path}}}";
+            + "{{#if anyVideo}}-map \"[v]\"{{else}}-vn{{/if}} "
+            + "{{#if anyAudio}}-map \"[a]\"{{else}}-an{{/if}} "
+            + "-y {{{output.path}}}";
 
     @Override public Map<String, Object> operate(ConcatOperation op, Toolbox toolbox, AssetManager assetManager) {
 
@@ -39,13 +58,45 @@ public class ConcatExec extends ExecBase<ConcatOperation> {
         final JStreamType streamType = opCtx.streamType;
 
         final File defaultOutfile = assetManager.assetPath(op, sources, streamType);
-        final File path = resolveOutputPath(output, defaultOutfile);
+        final File path = resolveOutputPath(assetManager, output, defaultOutfile);
         if (path == null) return null;
         output.setPath(abs(path));
 
         final Map<String, Object> ctx = new HashMap<>();
         ctx.put("ffmpeg", toolbox.getFfmpeg());
         ctx.put("sources", sources);
+
+        final boolean anyVideo = !op.audioOnly() && sources.stream().anyMatch(JAsset::hasHeight);
+        final boolean anyAudio = !op.videoOnly() && sources.stream().anyMatch(JAsset::hasChannelLayout);
+        ctx.put("anyVideo", anyVideo);
+        ctx.put("anyAudio", anyAudio);
+
+        if (anyVideo) {
+            ctx.put("firstWidth", sources.stream()
+                    .map(s -> ((JAssetJs) s.toJs()))
+                    .filter(a -> a.hasVideo && a.width != null)
+                    .map(a -> a.width)
+                    .findFirst().orElse(null));
+            ctx.put("firstHeight", sources.stream()
+                    .map(s -> ((JAssetJs) s.toJs()))
+                    .filter(a -> a.hasVideo && a.height != null)
+                    .map(a -> a.height)
+                    .findFirst().orElse(null));
+        }
+
+        if (anyAudio) {
+            ctx.put("firstChannelLayout", sources.stream()
+                    .map(s -> ((JAssetJs) s.toJs()))
+                    .filter(a -> a.hasAudio && a.channelLayout != null)
+                    .map(a -> a.channelLayout)
+                    .findFirst().orElse(null));
+            ctx.put("firstSamplingRate", sources.stream()
+                    .map(s -> ((JAssetJs) s.toJs()))
+                    .filter(a -> a.hasAudio && a.samplingRate != null)
+                    .map(a -> a.samplingRate)
+                    .findFirst().orElse(null));
+        }
+
         ctx.put("output", output);
         final String script = renderScript(toolbox, ctx, CONCAT_RECODE_TEMPLATE_1);
 

src/main/java/jvc/operation/exec/ExecBase.java

@@ -42,10 +42,11 @@ public abstract class ExecBase<OP extends JOperation> {
         return HandlebarsUtil.apply(toolbox.getHandlebars(), template, jsContext(ctx));
     }
 
-    protected File resolveOutputPath(JAsset output, File defaultOutfile) {
+    protected File resolveOutputPath(AssetManager assetManager, JAsset output, File defaultOutfile) {
         if (output.hasDest()) {
             if (output.destExists() && !output.destIsDirectory()) {
                 log.info("resolveOutputPath: dest exists: " + output.getDest());
+                assetManager.addOperationAsset(output.setPath(output.getDest()));
                 return null;
             } else if (output.destIsDirectory()) {
                 return new File(output.destDirectory(), basename(abs(defaultOutfile)));
@@ -115,6 +116,7 @@ public abstract class ExecBase<OP extends JOperation> {
         final String scriptOutput = exec(script, op.isNoExec());
         log.debug("createSilence: command output: "+scriptOutput);
 
+        assetManager.addOperationAsset(silence);
         return silence;
     }
 

src/main/java/jvc/operation/exec/KenBurnsExec.java

@@ -52,7 +52,7 @@ public class KenBurnsExec extends ExecBase<KenBurnsOperation> {
         final JStreamType streamType = opCtx.streamType;
 
         final File defaultOutfile = assetManager.assetPath(op, source, streamType);
-        final File path = resolveOutputPath(output, defaultOutfile);
+        final File path = resolveOutputPath(assetManager, output, defaultOutfile);
         if (path == null) return null;
         output.setPath(abs(path));
 

src/main/java/jvc/operation/exec/MergeAudioExec.java

@@ -1,46 +1,38 @@
 package jvc.operation.exec;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import jvc.model.JAsset;
 import jvc.model.JStreamType;
+import jvc.model.operation.JOperation;
 import jvc.model.operation.JSingleOperationContext;
+import jvc.operation.ConcatOperation;
 import jvc.operation.MergeAudioOperation;
 import jvc.service.AssetManager;
 import jvc.service.Toolbox;
-import lombok.Cleanup;
 import lombok.extern.slf4j.Slf4j;
-import org.cobbzilla.util.io.TempDir;
 import org.cobbzilla.util.javascript.JsEngine;
 
-import java.io.File;
 import java.math.BigDecimal;
-import java.util.HashMap;
 import java.util.Map;
 
 import static java.math.BigDecimal.ZERO;
-import static org.cobbzilla.util.daemon.ZillaRuntime.die;
-import static org.cobbzilla.util.io.FileUtil.*;
+import static org.cobbzilla.util.io.FileUtil.abs;
+import static org.cobbzilla.util.json.JsonUtil.json;
 
 @Slf4j
 public class MergeAudioExec extends SingleOrMultiSourceExecBase<MergeAudioOperation> {
 
-    public static final String PAD_WITH_SILENCE_TEMPLATE
-            = "cd {{{tempDir}}} && {{{ffmpeg}}} -f concat -i {{{playlist.path}}} -codec copy -y {{{padded}}}";
-
     public static final String MERGE_AUDIO_TEMPLATE
             = "{{{ffmpeg}}} -i {{{source.path}}} -itsoffset {{start}} -i {{audio.path}} "
-            // if source has no audio, define a null audio input source
-            + "{{#unless source.hasAudio}}"
-              + "-i anullsrc=channel_layout={{audio.channelLayout}}:sample_rate={{audio.samplingRate}} "
-            + "{{/unless}}"
 
             + "-filter_complex \""
             + "{{#if source.hasAudio}}"
               // source has audio -- mix with insertion
-              + "[0:a][1:a] amix=inputs=2 [merged]"
+              + "[0:a][1:{{audio.audioTrack}}] amix=inputs=2 [merged]"
             + "{{else}}"
               // source has no audio -- mix null source with insertion
-              + "[1:a] aresample=ocl={{audio.channelLayout}}:osr={{audio.samplingRate}} [1a]; "
-              + "[2:a][1a] amix=inputs=2 [merged]"
+              + "anullsrc=channel_layout={{audio.channelLayout}}:sample_rate={{audio.samplingRate}}:duration={{source.duration}} [silence]; "
+              + "[silence][1:{{audio.audioTrack}}] amix=inputs=2 [merged]"
             + "{{/if}}"
             + "\" "
             + "-map 0:v -map \"[merged]\" -c:v copy "
@@ -70,47 +62,24 @@ public class MergeAudioExec extends SingleOrMultiSourceExecBase<MergeAudioOperat
                                     AssetManager assetManager,
                                     JAsset audio,
                                     JAsset silence) {
-        final Map<String, Object> ctx = new HashMap<>();
-        ctx.put("ffmpeg", toolbox.getFfmpeg());
-
         final JStreamType streamType = audio.audioExtension();
-        final JAsset padded = new JAsset().setPath(abs(assetManager.assetPath(op, audio, streamType)));
-        final String paddedName = basename(padded.getPath());
-        ctx.put("padded", paddedName);
-
-        // create a temp dir for concat, it really likes to have everything in the same directory
-        @Cleanup("delete") final TempDir tempDir = new TempDir(assetManager.getScratchDir());
-        ctx.put("tempDir", abs(tempDir));
-
-        final String silenceName = basename(silence.getPath());
-        final String audioName = basename(audio.getPath());
-
-        // write playlist
-        final File playlistFile = new File(tempDir, "playlist.txt");
-        toFileOrDie(playlistFile, "file "+ silenceName +"\nfile "+ audioName);
-
-        // copy audio and silence assets to temp dir
-        copyFile(new File(silence.getPath()), new File(tempDir, silenceName));
-        copyFile(new File(audio.getPath()), new File(tempDir, audioName));
-
-        final JAsset playlist = new JAsset().setPath(abs(playlistFile));
-        ctx.put("playlist", playlist);
-
-        final String script = renderScript(toolbox, ctx, PAD_WITH_SILENCE_TEMPLATE);
-
-        log.debug("padWithSilence: running script: "+script);
-        final String scriptOutput = exec(script, op.isNoExec());
-
-        final File outputFile = new File(tempDir, paddedName);
-        if (!outputFile.exists()) return die("padWithSilence: output file not found: "+abs(outputFile));
-        copyFile(outputFile, new File(abs(padded.getPath())));
-
-        log.debug("padWithSilence: command output: "+scriptOutput);
+        final JAsset padded = new JAsset()
+                .setPath(abs(assetManager.assetPath(op, audio, streamType)));
+
+        final JOperation concat = new ConcatOperation()
+                .setAudioOnly(true)
+                .setSources(new String[]{silence.getName(), audio.getName()})
+                .setCreates(json(json(padded), JsonNode.class))
+                .setOperation("concat")
+                .setExecIndex(op.getExecIndex())
+                .setNoExec(op.isNoExec());
+        final Map<String, Object> concatCtx
+                = concat.getExec(getSpec()).operate(concat, toolbox, assetManager);
 
         // initialize metadata
-        padded.init(assetManager, toolbox);
+        final JAsset result = assetManager.resolve(padded.getName());
 
-        return padded;
+        return result;
     }
 
 }

src/main/java/jvc/operation/exec/OverlayExec.java

@@ -2,6 +2,7 @@ package jvc.operation.exec;
 
 import jvc.model.JAsset;
 import jvc.model.JStreamType;
+import jvc.model.js.JAssetJs;
 import jvc.model.operation.JSingleOperationContext;
 import jvc.operation.OverlayOperation;
 import jvc.service.AssetManager;
@@ -29,21 +30,31 @@ public class OverlayExec extends ExecBase<OverlayOperation> {
             + "[1:v] setpts=PTS-STARTPTS"
             + "{{#exists width}}, scale={{width}}x{{height}}{{/exists}}"
             + " [1v]; "
-            + "[0:v][1v] overlay={{{overlayFilterConfig}}} [v]; "
-
-            + "{{#if source.hasAudio}}"
-              // source has audio -- mix with overlay
-              + "[1:a] setpts=PTS-STARTPTS{{#if hasOverlayStart}}-({{overlayStart}}/TB){{/if}} [1a]; "
-              + "[0:a][1a] amix=inputs=2 [merged]"
-            + "{{else}}"
-              // source has no audio -- mix null source with overlay
-              + "anullsrc=channel_layout={{overlay.channelLayout}}:sample_rate={{overlay.samplingRate}}:duration={{source.duration}} [silence]; "
-              // + "[1:a] setpts=PTS-STARTPTS{{#if hasOverlayStart}}-({{overlayStart}}/TB){{/if}} [1a]; "
-              + "[silence][1:a] amix=inputs=2 [merged]"
+            + "[0:v][1v] overlay={{{overlayFilterConfig}}} [v] "
+
+            + "{{#if hasAudio}}"
+              + "{{#if source.hasAudio}}"
+                + "{{#if overlay.hasAudio}}"
+                  // source and overlay both audio -- mix them
+                  + "; [1:a] setpts=PTS-STARTPTS{{#if hasOverlayStart}}-({{overlayStart}}/TB){{/if}} [1a] "
+                  + "; [0:a][1a] amix=inputs=2 [merged]"
+                + "{{else}}"
+                  // source has audio but overlay has none, use source audio
+                  + "; anullsrc=channel_layout={{source.channelLayout}}:sample_rate={{source.samplingRate}}:duration={{source.duration}} [silence] "
+                  + "; [silence][0:a] amix=inputs=2 [merged]"
+                + "{{/if}}"
+              + "{{else}}"
+                + "{{#if overlay.hasAudio}}"
+                  // source has no audio -- mix null source with overlay
+                  + "; anullsrc=channel_layout={{overlay.channelLayout}}:sample_rate={{overlay.samplingRate}}:duration={{source.duration}} [silence] "
+                  + "; [silence][1:a] amix=inputs=2 [merged]"
+                + "{{/if}}"
+              + "{{/if}}"
             + "{{/if}}"
 
             + "\" "
-            + "-map \"[v]\" -map \"[merged]\" "
+            + "-map \"[v]\" "
+            + "{{#if hasAudio}} -map \"[merged]\"{{else}}-an{{/if}} "
             + " -y {{{output.path}}}";
 
     @Override public Map<String, Object> operate(OverlayOperation op, Toolbox toolbox, AssetManager assetManager) {
@@ -57,7 +68,7 @@ public class OverlayExec extends ExecBase<OverlayOperation> {
         final JAsset overlaySource = assetManager.resolve(overlay.getSource());
 
         final File defaultOutfile = assetManager.assetPath(op, source, streamType);
-        final File path = resolveOutputPath(output, defaultOutfile);
+        final File path = resolveOutputPath(assetManager, output, defaultOutfile);
         if (path == null) return null;
         output.setPath(abs(path));
 
@@ -103,6 +114,7 @@ public class OverlayExec extends ExecBase<OverlayOperation> {
         ctx.put("overlayStart", startTime.doubleValue());
         ctx.put("hasOverlayStart", !startTime.equals(ZERO));
         ctx.put("overlayEnd", endTime.doubleValue());
+        ctx.put("hasAudio", ((JAssetJs) source.toJs()).hasAudio || ((JAssetJs) overlaySource.toJs()).hasAudio);
         b.append("enable=between(t\\,")
                 .append(startTime.doubleValue())
                 .append("\\,")

src/main/java/jvc/operation/exec/SingleOrMultiSourceExecBase.java

@@ -61,7 +61,7 @@ public abstract class SingleOrMultiSourceExecBase<OP extends JSingleSourceOperat
             }
         } else {
             final File defaultOutfile = assetManager.assetPath(op, source, streamType);
-            final File path = resolveOutputPath(output, defaultOutfile);
+            final File path = resolveOutputPath(assetManager, output, defaultOutfile);
             if (path == null) return null;
             output.setPath(abs(path));
             process(ctx, op, source, output, output, toolbox, assetManager);