From 49ebdb81ff26003dcaf784e3f009d8739dfecdf8 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 28 Dec 2020 05:13:19 -0500 Subject: [PATCH] improve handling of files with missing audio or video track --- README.md | 2 +- docs/jvc_js.md | 6 ++ src/main/java/jvc/model/JAsset.java | 6 +- src/main/java/jvc/model/js/JAssetJs.java | 14 +++- .../operation/JMultiSourceOperation.java | 2 + .../java/jvc/operation/ConcatOperation.java | 15 +++- .../java/jvc/operation/exec/ConcatExec.java | 59 ++++++++++++++- .../java/jvc/operation/exec/ExecBase.java | 4 +- .../java/jvc/operation/exec/KenBurnsExec.java | 2 +- .../jvc/operation/exec/MergeAudioExec.java | 75 ++++++------------- .../java/jvc/operation/exec/OverlayExec.java | 38 ++++++---- .../exec/SingleOrMultiSourceExecBase.java | 2 +- 12 files changed, 147 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 8d7032f..8bff453 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ goes to new files. # Requirements * Java 11 * Maven 3 - * ffmpeg + * ffmpeg (`HEAD` is required for `overlay` and `merge-audio` operations) * mediainfo These programs should executable (given your `PATH`): `javac`, `java`, `mvn`, diff --git a/docs/jvc_js.md b/docs/jvc_js.md index e1fc4fb..79fcef2 100644 --- a/docs/jvc_js.md +++ b/docs/jvc_js.md @@ -82,6 +82,12 @@ An array of the audio tracks in a video. #### `videoTracks` An array of the video tracks in a video. +#### `audioTrack` +The track number of the first audio track. + +#### `videoTrack` +The track number of the first video track. + #### `hasAudio` A boolean, true if the asset has any audio tracks, false otherwise diff --git a/src/main/java/jvc/model/JAsset.java b/src/main/java/jvc/model/JAsset.java index 2037274..e540b97 100644 --- a/src/main/java/jvc/model/JAsset.java +++ b/src/main/java/jvc/model/JAsset.java @@ -47,7 +47,9 @@ public class JAsset implements JsObjectView { public JAsset(JAsset other, File file) { this(other); setPath(abs(file)); } - @Getter @Setter private String name; + @Setter private String name; + public String getName () { return empty(name) ? basename(path) : name; } + @Getter @Setter private String path; public boolean hasPath() { return !empty(path); } @@ -159,9 +161,11 @@ public class JAsset implements JsObjectView { public BigDecimal width() { return hasInfo() ? getInfo().width() : null; } @JsonIgnore public BigDecimal getWidth () { return width(); } + public boolean hasWidth () { return width() != null; } public BigDecimal height() { return hasInfo() ? getInfo().height() : null; } @JsonIgnore public BigDecimal getHeight () { return height(); } + public boolean hasHeight () { return height() != null; } public int numTracks(JTrackType type) { return hasInfo() ? getInfo().numTracks(type) : 0; } diff --git a/src/main/java/jvc/model/js/JAssetJs.java b/src/main/java/jvc/model/js/JAssetJs.java index 2efc8e1..6606516 100644 --- a/src/main/java/jvc/model/js/JAssetJs.java +++ b/src/main/java/jvc/model/js/JAssetJs.java @@ -26,6 +26,10 @@ public class JAssetJs { @Getter public JTrackJs[] tracks = EMPTY_TRACKS; @Getter public JTrackJs[] videoTracks = EMPTY_TRACKS; @Getter public JTrackJs[] audioTracks = EMPTY_TRACKS; + + @Getter public final Integer videoTrack; + @Getter public final Integer audioTrack; + @Getter public JAssetJs[] assets = EMPTY_ASSETS; @Getter public final boolean hasAudio; @Getter public final boolean hasVideo; @@ -44,10 +48,13 @@ public class JAssetJs { this.aspectRatio = asset.aspectRatio() == null ? Double.NaN : asset.aspectRatio().doubleValue(); this.channelLayout = asset.hasChannelLayout() ? asset.channelLayout() : null; - this.samplingRate = asset.hasSamplingRate() ? asset.samplingRate().intValue() : 0; + this.samplingRate = asset.hasSamplingRate() ? asset.samplingRate().intValue() : null; + Integer vTrack = null; + Integer aTrack = null; if (asset.hasInfo()) { final JMediaInfo info = asset.getInfo(); + int trackCount = 0; for (JTrack track : info.getMedia().getTrack()) { final JTrackJs trackJs = new JTrackJs(track.type().name()); @@ -59,13 +66,18 @@ public class JAssetJs { switch (track.type()) { case audio: audioTracks = ArrayUtil.append(audioTracks, trackJs); + if (aTrack == null) aTrack = trackCount; break; case video: videoTracks = ArrayUtil.append(videoTracks, trackJs); + if (vTrack == null) vTrack = trackCount; break; } + trackCount++; } } + this.videoTrack = vTrack; + this.audioTrack = aTrack; if (asset.hasListAssets()) { final JAsset[] list = asset.getList(); diff --git a/src/main/java/jvc/model/operation/JMultiSourceOperation.java b/src/main/java/jvc/model/operation/JMultiSourceOperation.java index 71e9b03..a2bd586 100644 --- a/src/main/java/jvc/model/operation/JMultiSourceOperation.java +++ b/src/main/java/jvc/model/operation/JMultiSourceOperation.java @@ -6,6 +6,7 @@ import jvc.service.AssetManager; import jvc.service.Toolbox; import lombok.Getter; import lombok.Setter; +import lombok.experimental.Accessors; import java.util.List; @@ -14,6 +15,7 @@ import static jvc.model.JAsset.json2asset; import static org.cobbzilla.util.daemon.ZillaRuntime.die; import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +@Accessors(chain=true) public abstract class JMultiSourceOperation extends JOperation { @Getter @Setter private String[] sources; diff --git a/src/main/java/jvc/operation/ConcatOperation.java b/src/main/java/jvc/operation/ConcatOperation.java index acba033..3be34e5 100644 --- a/src/main/java/jvc/operation/ConcatOperation.java +++ b/src/main/java/jvc/operation/ConcatOperation.java @@ -1,7 +1,18 @@ package jvc.operation; import jvc.model.operation.JMultiSourceOperation; +import lombok.Getter; +import lombok.Setter; +import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; -@Slf4j -public class ConcatOperation extends JMultiSourceOperation {} +@Slf4j @Accessors(chain=true) +public class ConcatOperation extends JMultiSourceOperation { + + @Getter @Setter private Boolean audioOnly; + public boolean audioOnly () { return audioOnly != null && audioOnly; } + + @Getter @Setter private Boolean videoOnly; + public boolean videoOnly () { return videoOnly != null && videoOnly; } + +} diff --git a/src/main/java/jvc/operation/exec/ConcatExec.java b/src/main/java/jvc/operation/exec/ConcatExec.java index d3ec3a9..91d86c9 100644 --- a/src/main/java/jvc/operation/exec/ConcatExec.java +++ b/src/main/java/jvc/operation/exec/ConcatExec.java @@ -2,6 +2,7 @@ package jvc.operation.exec; import jvc.model.JAsset; import jvc.model.JStreamType; +import jvc.model.js.JAssetJs; import jvc.model.operation.JMultiOperationContext; import jvc.operation.ConcatOperation; import jvc.service.AssetManager; @@ -23,13 +24,31 @@ public class ConcatExec extends ExecBase { = "{{ffmpeg}} {{#each sources}} -i {{{this.path}}}{{/each}} " // filter: list inputs - + "-filter_complex \"{{#each sources}}[{{@index}}:v] [{{@index}}:a] {{/each}} " + + "-filter_complex \"" + + // create null sources for missing audio/video streams + + "{{#each sources}}" + + "{{#if anyVideo}}{{#unless hasVideo}} nullsrc=s={{firstWidth}}x{{firstHeight}}:duration={{duration}} [null_video_{{@index}}]; {{/unless}}{{/if}}" + + "{{#if anyAudio}}{{#unless hasAudio}} anullsrc=channel_layout={{firstChannelLayout}}:sample_rate={{firstSamplingRate}}:duration={{duration}} [null_audio_{{@index}}]; {{/unless}}{{/if}}" + + "{{/each}} " + + + "{{#each sources}}" + + "{{#if anyVideo}}{{#if hasVideo}}[{{@index}}:{{videoTrack}}]{{else}}[null_video_{{@index}}]{{/if}}{{/if}} " + + "{{#if anyAudio}}{{#if hasAudio}}[{{@index}}:{{audioTrack}}]{{else}}[null_audio_{{@index}}]{{/if}}{{/if}} " + + "{{/each}} " // filter: concat filter them together - + "concat=n={{sources.length}}:v=1:a=1 [v] [a]\" " + + "concat=n={{sources.length}}" + + ":v={{#if anyVideo}}1{{else}}0{{/if}}" + + ":a={{#if anyAudio}}1{{else}}0{{/if}} " + + "{{#if anyVideo}}[v]{{/if}} " + + "{{#if anyAudio}}[a]{{/if}}" + + "\" " // output combined result - + "-map \"[v]\" -map \"[a]\" -y {{{output.path}}}"; + + "{{#if anyVideo}}-map \"[v]\"{{else}}-vn{{/if}} " + + "{{#if anyAudio}}-map \"[a]\"{{else}}-an{{/if}} " + + "-y {{{output.path}}}"; @Override public Map operate(ConcatOperation op, Toolbox toolbox, AssetManager assetManager) { @@ -39,13 +58,45 @@ public class ConcatExec extends ExecBase { final JStreamType streamType = opCtx.streamType; final File defaultOutfile = assetManager.assetPath(op, sources, streamType); - final File path = resolveOutputPath(output, defaultOutfile); + final File path = resolveOutputPath(assetManager, output, defaultOutfile); if (path == null) return null; output.setPath(abs(path)); final Map ctx = new HashMap<>(); ctx.put("ffmpeg", toolbox.getFfmpeg()); ctx.put("sources", sources); + + final boolean anyVideo = !op.audioOnly() && sources.stream().anyMatch(JAsset::hasHeight); + final boolean anyAudio = !op.videoOnly() && sources.stream().anyMatch(JAsset::hasChannelLayout); + ctx.put("anyVideo", anyVideo); + ctx.put("anyAudio", anyAudio); + + if (anyVideo) { + ctx.put("firstWidth", sources.stream() + .map(s -> ((JAssetJs) s.toJs())) + .filter(a -> a.hasVideo && a.width != null) + .map(a -> a.width) + .findFirst().orElse(null)); + ctx.put("firstHeight", sources.stream() + .map(s -> ((JAssetJs) s.toJs())) + .filter(a -> a.hasVideo && a.height != null) + .map(a -> a.height) + .findFirst().orElse(null)); + } + + if (anyAudio) { + ctx.put("firstChannelLayout", sources.stream() + .map(s -> ((JAssetJs) s.toJs())) + .filter(a -> a.hasAudio && a.channelLayout != null) + .map(a -> a.channelLayout) + .findFirst().orElse(null)); + ctx.put("firstSamplingRate", sources.stream() + .map(s -> ((JAssetJs) s.toJs())) + .filter(a -> a.hasAudio && a.samplingRate != null) + .map(a -> a.samplingRate) + .findFirst().orElse(null)); + } + ctx.put("output", output); final String script = renderScript(toolbox, ctx, CONCAT_RECODE_TEMPLATE_1); diff --git a/src/main/java/jvc/operation/exec/ExecBase.java b/src/main/java/jvc/operation/exec/ExecBase.java index 5abc948..5e9ee13 100644 --- a/src/main/java/jvc/operation/exec/ExecBase.java +++ b/src/main/java/jvc/operation/exec/ExecBase.java @@ -42,10 +42,11 @@ public abstract class ExecBase { return HandlebarsUtil.apply(toolbox.getHandlebars(), template, jsContext(ctx)); } - protected File resolveOutputPath(JAsset output, File defaultOutfile) { + protected File resolveOutputPath(AssetManager assetManager, JAsset output, File defaultOutfile) { if (output.hasDest()) { if (output.destExists() && !output.destIsDirectory()) { log.info("resolveOutputPath: dest exists: " + output.getDest()); + assetManager.addOperationAsset(output.setPath(output.getDest())); return null; } else if (output.destIsDirectory()) { return new File(output.destDirectory(), basename(abs(defaultOutfile))); @@ -115,6 +116,7 @@ public abstract class ExecBase { final String scriptOutput = exec(script, op.isNoExec()); log.debug("createSilence: command output: "+scriptOutput); + assetManager.addOperationAsset(silence); return silence; } diff --git a/src/main/java/jvc/operation/exec/KenBurnsExec.java b/src/main/java/jvc/operation/exec/KenBurnsExec.java index 4e97f01..a7a9638 100644 --- a/src/main/java/jvc/operation/exec/KenBurnsExec.java +++ b/src/main/java/jvc/operation/exec/KenBurnsExec.java @@ -52,7 +52,7 @@ public class KenBurnsExec extends ExecBase { final JStreamType streamType = opCtx.streamType; final File defaultOutfile = assetManager.assetPath(op, source, streamType); - final File path = resolveOutputPath(output, defaultOutfile); + final File path = resolveOutputPath(assetManager, output, defaultOutfile); if (path == null) return null; output.setPath(abs(path)); diff --git a/src/main/java/jvc/operation/exec/MergeAudioExec.java b/src/main/java/jvc/operation/exec/MergeAudioExec.java index 7313111..e241728 100644 --- a/src/main/java/jvc/operation/exec/MergeAudioExec.java +++ b/src/main/java/jvc/operation/exec/MergeAudioExec.java @@ -1,46 +1,38 @@ package jvc.operation.exec; +import com.fasterxml.jackson.databind.JsonNode; import jvc.model.JAsset; import jvc.model.JStreamType; +import jvc.model.operation.JOperation; import jvc.model.operation.JSingleOperationContext; +import jvc.operation.ConcatOperation; import jvc.operation.MergeAudioOperation; import jvc.service.AssetManager; import jvc.service.Toolbox; -import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.cobbzilla.util.io.TempDir; import org.cobbzilla.util.javascript.JsEngine; -import java.io.File; import java.math.BigDecimal; -import java.util.HashMap; import java.util.Map; import static java.math.BigDecimal.ZERO; -import static org.cobbzilla.util.daemon.ZillaRuntime.die; -import static org.cobbzilla.util.io.FileUtil.*; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.json.JsonUtil.json; @Slf4j public class MergeAudioExec extends SingleOrMultiSourceExecBase { - public static final String PAD_WITH_SILENCE_TEMPLATE - = "cd {{{tempDir}}} && {{{ffmpeg}}} -f concat -i {{{playlist.path}}} -codec copy -y {{{padded}}}"; - public static final String MERGE_AUDIO_TEMPLATE = "{{{ffmpeg}}} -i {{{source.path}}} -itsoffset {{start}} -i {{audio.path}} " - // if source has no audio, define a null audio input source - + "{{#unless source.hasAudio}}" - + "-i anullsrc=channel_layout={{audio.channelLayout}}:sample_rate={{audio.samplingRate}} " - + "{{/unless}}" + "-filter_complex \"" + "{{#if source.hasAudio}}" // source has audio -- mix with insertion - + "[0:a][1:a] amix=inputs=2 [merged]" + + "[0:a][1:{{audio.audioTrack}}] amix=inputs=2 [merged]" + "{{else}}" // source has no audio -- mix null source with insertion - + "[1:a] aresample=ocl={{audio.channelLayout}}:osr={{audio.samplingRate}} [1a]; " - + "[2:a][1a] amix=inputs=2 [merged]" + + "anullsrc=channel_layout={{audio.channelLayout}}:sample_rate={{audio.samplingRate}}:duration={{source.duration}} [silence]; " + + "[silence][1:{{audio.audioTrack}}] amix=inputs=2 [merged]" + "{{/if}}" + "\" " + "-map 0:v -map \"[merged]\" -c:v copy " @@ -70,47 +62,24 @@ public class MergeAudioExec extends SingleOrMultiSourceExecBase ctx = new HashMap<>(); - ctx.put("ffmpeg", toolbox.getFfmpeg()); - final JStreamType streamType = audio.audioExtension(); - final JAsset padded = new JAsset().setPath(abs(assetManager.assetPath(op, audio, streamType))); - final String paddedName = basename(padded.getPath()); - ctx.put("padded", paddedName); - - // create a temp dir for concat, it really likes to have everything in the same directory - @Cleanup("delete") final TempDir tempDir = new TempDir(assetManager.getScratchDir()); - ctx.put("tempDir", abs(tempDir)); - - final String silenceName = basename(silence.getPath()); - final String audioName = basename(audio.getPath()); - - // write playlist - final File playlistFile = new File(tempDir, "playlist.txt"); - toFileOrDie(playlistFile, "file "+ silenceName +"\nfile "+ audioName); - - // copy audio and silence assets to temp dir - copyFile(new File(silence.getPath()), new File(tempDir, silenceName)); - copyFile(new File(audio.getPath()), new File(tempDir, audioName)); - - final JAsset playlist = new JAsset().setPath(abs(playlistFile)); - ctx.put("playlist", playlist); - - final String script = renderScript(toolbox, ctx, PAD_WITH_SILENCE_TEMPLATE); - - log.debug("padWithSilence: running script: "+script); - final String scriptOutput = exec(script, op.isNoExec()); - - final File outputFile = new File(tempDir, paddedName); - if (!outputFile.exists()) return die("padWithSilence: output file not found: "+abs(outputFile)); - copyFile(outputFile, new File(abs(padded.getPath()))); - - log.debug("padWithSilence: command output: "+scriptOutput); + final JAsset padded = new JAsset() + .setPath(abs(assetManager.assetPath(op, audio, streamType))); + + final JOperation concat = new ConcatOperation() + .setAudioOnly(true) + .setSources(new String[]{silence.getName(), audio.getName()}) + .setCreates(json(json(padded), JsonNode.class)) + .setOperation("concat") + .setExecIndex(op.getExecIndex()) + .setNoExec(op.isNoExec()); + final Map concatCtx + = concat.getExec(getSpec()).operate(concat, toolbox, assetManager); // initialize metadata - padded.init(assetManager, toolbox); + final JAsset result = assetManager.resolve(padded.getName()); - return padded; + return result; } } diff --git a/src/main/java/jvc/operation/exec/OverlayExec.java b/src/main/java/jvc/operation/exec/OverlayExec.java index 3e96386..eb4d09e 100644 --- a/src/main/java/jvc/operation/exec/OverlayExec.java +++ b/src/main/java/jvc/operation/exec/OverlayExec.java @@ -2,6 +2,7 @@ package jvc.operation.exec; import jvc.model.JAsset; import jvc.model.JStreamType; +import jvc.model.js.JAssetJs; import jvc.model.operation.JSingleOperationContext; import jvc.operation.OverlayOperation; import jvc.service.AssetManager; @@ -29,21 +30,31 @@ public class OverlayExec extends ExecBase { + "[1:v] setpts=PTS-STARTPTS" + "{{#exists width}}, scale={{width}}x{{height}}{{/exists}}" + " [1v]; " - + "[0:v][1v] overlay={{{overlayFilterConfig}}} [v]; " - - + "{{#if source.hasAudio}}" - // source has audio -- mix with overlay - + "[1:a] setpts=PTS-STARTPTS{{#if hasOverlayStart}}-({{overlayStart}}/TB){{/if}} [1a]; " - + "[0:a][1a] amix=inputs=2 [merged]" - + "{{else}}" - // source has no audio -- mix null source with overlay - + "anullsrc=channel_layout={{overlay.channelLayout}}:sample_rate={{overlay.samplingRate}}:duration={{source.duration}} [silence]; " - // + "[1:a] setpts=PTS-STARTPTS{{#if hasOverlayStart}}-({{overlayStart}}/TB){{/if}} [1a]; " - + "[silence][1:a] amix=inputs=2 [merged]" + + "[0:v][1v] overlay={{{overlayFilterConfig}}} [v] " + + + "{{#if hasAudio}}" + + "{{#if source.hasAudio}}" + + "{{#if overlay.hasAudio}}" + // source and overlay both audio -- mix them + + "; [1:a] setpts=PTS-STARTPTS{{#if hasOverlayStart}}-({{overlayStart}}/TB){{/if}} [1a] " + + "; [0:a][1a] amix=inputs=2 [merged]" + + "{{else}}" + // source has audio but overlay has none, use source audio + + "; anullsrc=channel_layout={{source.channelLayout}}:sample_rate={{source.samplingRate}}:duration={{source.duration}} [silence] " + + "; [silence][0:a] amix=inputs=2 [merged]" + + "{{/if}}" + + "{{else}}" + + "{{#if overlay.hasAudio}}" + // source has no audio -- mix null source with overlay + + "; anullsrc=channel_layout={{overlay.channelLayout}}:sample_rate={{overlay.samplingRate}}:duration={{source.duration}} [silence] " + + "; [silence][1:a] amix=inputs=2 [merged]" + + "{{/if}}" + + "{{/if}}" + "{{/if}}" + "\" " - + "-map \"[v]\" -map \"[merged]\" " + + "-map \"[v]\" " + + "{{#if hasAudio}} -map \"[merged]\"{{else}}-an{{/if}} " + " -y {{{output.path}}}"; @Override public Map operate(OverlayOperation op, Toolbox toolbox, AssetManager assetManager) { @@ -57,7 +68,7 @@ public class OverlayExec extends ExecBase { final JAsset overlaySource = assetManager.resolve(overlay.getSource()); final File defaultOutfile = assetManager.assetPath(op, source, streamType); - final File path = resolveOutputPath(output, defaultOutfile); + final File path = resolveOutputPath(assetManager, output, defaultOutfile); if (path == null) return null; output.setPath(abs(path)); @@ -103,6 +114,7 @@ public class OverlayExec extends ExecBase { ctx.put("overlayStart", startTime.doubleValue()); ctx.put("hasOverlayStart", !startTime.equals(ZERO)); ctx.put("overlayEnd", endTime.doubleValue()); + ctx.put("hasAudio", ((JAssetJs) source.toJs()).hasAudio || ((JAssetJs) overlaySource.toJs()).hasAudio); b.append("enable=between(t\\,") .append(startTime.doubleValue()) .append("\\,") diff --git a/src/main/java/jvc/operation/exec/SingleOrMultiSourceExecBase.java b/src/main/java/jvc/operation/exec/SingleOrMultiSourceExecBase.java index ee8568b..c3d9d63 100644 --- a/src/main/java/jvc/operation/exec/SingleOrMultiSourceExecBase.java +++ b/src/main/java/jvc/operation/exec/SingleOrMultiSourceExecBase.java @@ -61,7 +61,7 @@ public abstract class SingleOrMultiSourceExecBase