commit 2d347781b672716674abb79c4b8545ab83f22d88 Author: Jonathan Cobb Date: Fri Dec 11 16:21:34 2020 -0500 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc4a95e --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.iml +.idea +target +tmp +logs +dependency-reduced-pom.xml +*~ +*.log +*.mp4 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3505368 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "utils/cobbzilla-utils"] + path = utils/cobbzilla-utils + url = git@git.bubblev.org:bubblev/cobbzilla-utils.git diff --git a/README.md b/README.md new file mode 100644 index 0000000..99bb639 --- /dev/null +++ b/README.md @@ -0,0 +1,109 @@ +# Javicle - a JSON Video Composition Language + +Javicle is not a replacement for Final Cut Pro or even iMovie. + +Javicle might be right for you if your video composition and manipulation needs are relatively simple +and you enjoy doing things with the command line and some JSON config instead of a GUI. + +This also give you the ability to track more of your workflow in source control - if you commit all +the original assets and the .jvcl files that describe how to create the output assets, you don't need +to save/archive the output assets anywhere. + +Note, for those who want truly 100% re-creatable builds, you would also need to record the versions +of the various tools used (ffmpeg, etc) and reuse those same versions when recreating a build. This is +generally overkill though, since the options we use on the different tools have been stable for a while +and I see a low likelihood of a significant change in behavior, or a bug being introduced. + +In JVCL there are two main concepts: assets and operations. + +Assets are the inputs - generally image, audio and video files. Assets have a name and a path. +The path can be a file or a URL. + +Operations are transformations to perform on the inputs. +An operation can produce a new intermediate asset. +Intermediate assets have names, and special paths that indicate how to reconstruct them from their assets, such that if you have the path of an intermediate asset, you can recreate its content, assuming you supply the same input assets. + +## Operations + +### split +Split an audio/video asset into multiple assets + +### concat +Concatenate audio/video assets together into one asset + +### trim +Trim audio/video - crop from beginning, end, or both + +### overlay +Overlay one video file onto another + +### ken-burns +For transforming still images into video via a fade-pan (aka Ken Burns) effect + +### letterbox +Transform a video in one size to another size using black letterboxes on the sides or top/bottom. Handy for embedding mobile videos into other screen formats + +### split-silence +Split an audio file according to silence + +## Example +```json +{ + "assets": [ + {"name": "vid1", "path": "/tmp/path/to/video1.mp4"}, + {"name": "vid2", "path": "/tmp/path/to/video2.mp4"} + ], + "operations": [ + { + "operation": "split", // name of the operation + "creates": "vid1_split_%", // assets it creates, the '%' will be replaced with a counter + "perform": { + "split": "vid1", // split this source asset + "interval": "10s" // split every ten seconds + } + }, + { + "operation": "concat", // name of the operation + "creates": "recombined_vid1", // assets it creates, the '%' will be replaced with a counter + "perform": { + "concat": ["vid1_split"] // recombine all split assets + } + }, + { + "operation": "concat", // name of the operation + "creates": "combined_vid", // asset it creates, can be referenced later + "perform": { + "concat": ["vid1", "vid2"] // operation-specific: this says, concatenate these named assets + } + }, + { + "operation": "concat", // name of the operation + "creates": "combined_vid", // the asset it creates, can be referenced later + "perform": { + "concat": ["vid1", "vid2"] // operation-specific: this says, concatenate these named assets + } + }, + { + "operation": "overlay", // name of the operation + "creates": "overlay1", // asset it creates + "perform": { + "source": "combined_vid1", // main video asset + "overlay": "vid1", // overlay this video on the main video + + "start": "vid1.end_ts", // when (on the main video timeline) to start the overlay. default is 0 (beginning) + "duration": "vid1.duration", // how long to play the overlay. default is to play the entire overlay asset + + "width": 400, // how wide the overlay will be, in pixels. default is "overlay.width" + "height": 300, // how tall the overlay will be, in pixels. default is "overlay.height" + + "x": "source.width / 2", // horizontal overlay position. default is 0 + "y": "source.height / 2", // vertical overlay position. default is 0 + + "out": "1080p", // this is a shortcut to the two lines below, and is the preferred way of specifying the output resolution + "out_width": 1920, // output width in pixels. default is source width + "out_height": 1024 // output height in pixes. default is source height + } + } + ] +} +``` \ No newline at end of file diff --git a/bin/jvcl b/bin/jvcl new file mode 100755 index 0000000..a32430b --- /dev/null +++ b/bin/jvcl @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Run JVCL on a spec +# +# Usage: +# +# jvcl [-f spec-file] [-t temp-dir] +# +# If the `-f` option is omitted and no spec-file is provided, then jvcl will try to read +# a spec from stdin. +# +# Every run of JVCL will create a temp directory. You can either specify the name of the directory +# with `-t` (it will be created if it does not exist), or let jvcl create a temp directory for you. +# +# Note: this command will build the jvcl jar file if needed. The first time you run it, it might +# take a long time to start up. +# +function die () { + echo 1>&2 "${1}" + exit 1 +} + +JVCL_DIR="$(cd "$(dirname "${0}")"/.. && pwd)" +JVCL_JAR="$(find "${JVCL_DIR}"/target -type f -name "jvcl-*.jar" | head -1)" +if [[ -z "${JVCL_JAR}" ]] ; then + mvn -DskipTests=true clean package || die "Error building JVCL jar" + JVCL_JAR="$(find "${JVCL_DIR}"/target -type f -name "jvcl-*.jar" | head -1)" + if [[ -z "${JVCL_JAR}" ]] ; then + die "No JVCL jar file found after successful build" + fi +fi + +java -cp "${JVCL_JAR}" jvcl.main.Jvcl "${@}" diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..a006f76 --- /dev/null +++ b/pom.xml @@ -0,0 +1,171 @@ + + + + + 4.0.0 + + org.cobbzilla + javicle + javicle + 1.0.0-SNAPSHOT + jar + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + jitpack.io + https://jitpack.io + + + + + 2.0.23 + 19.2.0 + 2.11.2 + 1.3 + 4.4.13 + 4.5.13 + 4.5.13 + 2.10.8 + 1.7.30 + 1.2.3 + 4.2.0 + 4.13.1 + + + + + uberjar + + + + org.apache.maven.plugins + maven-shade-plugin + 2.1 + + zilla-utils + + + + package + + shade + + + + + jvcl.main.Jvcl + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + + org.cobbzilla + cobbzilla-utils + 2.0.1 + + + org.apache.poi + * + + + fr.opensagres.xdocreport + * + + + com.codeborne + * + + + jtidy + * + + + xalan + * + + + net.sf.saxon + * + + + org.apache.ant + * + + + com.opencsv + * + + + org.quartz-scheduler + * + + + org.apache.pdfbox + * + + + org.atteo + * + + + io.github.bonigarcia + * + + + + + + junit + junit + ${junit.version} + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + 11 + 11 + true + + + + + + diff --git a/src/main/java/jvcl/main/Jvcl.java b/src/main/java/jvcl/main/Jvcl.java new file mode 100644 index 0000000..43a0ce8 --- /dev/null +++ b/src/main/java/jvcl/main/Jvcl.java @@ -0,0 +1,44 @@ +package jvcl.main; + + +import jvcl.model.JSpec; +import jvcl.service.AssetManager; +import jvcl.service.OperationEngine; +import jvcl.service.Toolbox; +import org.cobbzilla.util.main.BaseMain; + +import java.util.Arrays; + +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.json.JsonUtil.json; + +public class Jvcl extends BaseMain { + + public static void main (String[] args) { main(Jvcl.class, args); } + + @Override protected void run() throws Exception { + final JvclOptions options = getOptions(); + final JSpec spec = options.getSpec(); + + if (empty(spec.getAssets())) { + err(">>> jvcl: no assets defined in spec"); + return; + } + if (empty(spec.getOperations())) { + err(">>> jvcl: no operations defined in spec"); + return; + } + + final Toolbox toolbox = Toolbox.DEFAULT_TOOLBOX; + + final AssetManager assetManager = new AssetManager(toolbox, getOptions().getScratchDir()); + Arrays.stream(spec.getAssets()).forEach(assetManager::defineAsset); + + final OperationEngine opEngine = new OperationEngine(toolbox, assetManager); + Arrays.stream(spec.getOperations()).forEach(opEngine::perform); + + err(">>> jvcl: completed " + spec.getOperations().length + " operations"); + out(json(assetManager.getAssets())); + } + +} diff --git a/src/main/java/jvcl/main/JvclOptions.java b/src/main/java/jvcl/main/JvclOptions.java new file mode 100644 index 0000000..aba0053 --- /dev/null +++ b/src/main/java/jvcl/main/JvclOptions.java @@ -0,0 +1,43 @@ +package jvcl.main; + +import jvcl.model.JSpec; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.main.BaseMainOptions; +import org.kohsuke.args4j.Option; + +import java.io.File; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.readStdin; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.FileUtil.toStringOrDie; +import static org.cobbzilla.util.json.JsonUtil.FULL_MAPPER_ALLOW_COMMENTS; +import static org.cobbzilla.util.json.JsonUtil.json; + +@Slf4j +public class JvclOptions extends BaseMainOptions { + + public static final String USAGE_SPEC = "Spec file to run. Default is to read from stdin."; + public static final String OPT_SPEC = "-f"; + public static final String LONGOPT_SPEC = "--file"; + @Option(name=OPT_SPEC, aliases=LONGOPT_SPEC, usage=USAGE_SPEC) + @Getter @Setter private File specFile; + + public JSpec getSpec() { + if (specFile != null && !specFile.getName().equals("-")) { + if (!specFile.exists()) return die("File not found: "+abs(specFile)); + return json(toStringOrDie(specFile), JSpec.class, FULL_MAPPER_ALLOW_COMMENTS); + } + log.info("reading JVCL spec from stdin..."); + return json(readStdin(), JSpec.class); + } + + public static final String USAGE_SCRATCH_DIR = "Scratch directory. Default is to create a temp directory under /tmp"; + public static final String OPT_SCRATCH_DIR = "-t"; + public static final String LONGOPT_SCRATCH_DIR = "--temp-dir"; + @Option(name=OPT_SCRATCH_DIR, aliases=LONGOPT_SCRATCH_DIR, usage=USAGE_SCRATCH_DIR) + @Getter @Setter private File scratchDir; + +} diff --git a/src/main/java/jvcl/model/JArtifact.java b/src/main/java/jvcl/model/JArtifact.java new file mode 100644 index 0000000..15e2057 --- /dev/null +++ b/src/main/java/jvcl/model/JArtifact.java @@ -0,0 +1,10 @@ +package jvcl.model; + +import lombok.Getter; +import lombok.Setter; + +public class JArtifact { + + @Getter @Setter private String name; + @Getter @Setter private String file; +} diff --git a/src/main/java/jvcl/model/JAsset.java b/src/main/java/jvcl/model/JAsset.java new file mode 100644 index 0000000..bac540c --- /dev/null +++ b/src/main/java/jvcl/model/JAsset.java @@ -0,0 +1,135 @@ +package jvcl.model; + +import com.fasterxml.jackson.databind.JsonNode; +import jvcl.model.info.JMediaInfo; +import jvcl.service.AssetManager; +import jvcl.service.Toolbox; +import lombok.*; +import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.IOUtils; +import org.cobbzilla.util.collection.ArrayUtil; +import org.cobbzilla.util.http.HttpUtil; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; + +import static org.cobbzilla.util.daemon.ZillaRuntime.*; +import static org.cobbzilla.util.http.HttpSchemes.isHttpOrHttps; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@NoArgsConstructor @AllArgsConstructor @Accessors(chain=true) @Slf4j +public class JAsset { + + public static final JAsset NULL_ASSET = new JAsset().setName("~null asset~").setPath("/dev/null"); + public static final String PREFIX_CLASSPATH = "classpath:"; + + public JAsset(JAsset other) { copy(this, other); } + + @Getter @Setter private String name; + @Getter @Setter private String path; + + // an asset can specify where its file should live + // if the file already exists, it is used and not overwritten + @Getter @Setter private String dest; + public boolean hasDest() { return !empty(dest); } + public boolean destExists() { return new File(dest).exists(); } + + // if path was not a file, it got resolved to a file + // the original value of 'path' is stored here + @Getter @Setter private String originalPath; + + @Getter @Setter private JAsset[] list; + public boolean hasList () { return list != null; } + public void addAsset(JAsset slice) { list = ArrayUtil.append(list, slice); } + + @Getter @Setter private JFormat format; + public boolean hasFormat() { return format != null; } + + @Getter private JMediaInfo info; + public JAsset setInfo (JMediaInfo info) { + this.info = info; + setFormat(info.getFormat()); + return this; + } + public boolean hasInfo() { return info != null; } + + public static JAsset json2asset(JsonNode node) { + if (node.isObject()) return json(node, JAsset.class); + if (node.isTextual()) return new JAsset().setName(node.textValue()); + return die("json2asset: node was neither a JSON object nor a string: "+json(node)); + } + + public void mergeFormat(JFormat format) { + if (format == null) { + log.warn("mergeFormat: cannot merge null"); + return; + } + if (this.format == null) { + this.format = new JFormat(format); + } else { + this.format.merge(format); + } + } + + public BigDecimal duration() { return getInfo().duration(); } + + public JAsset init(AssetManager assetManager, Toolbox toolbox) { + initPath(assetManager); + setInfo(toolbox.getInfo(this)); + return this; + } + + private JAsset initPath(AssetManager assetManager) { + final String path = getPath(); + if (empty(path)) return die("initPath: no path!"); + + // if dest already exists, use that + if (hasDest() && destExists()) { + setOriginalPath(path); + setPath(getDest()); + return this; + } + + final File sourcePath = hasDest() ? new File(getDest()) : assetManager.sourcePath(getName()); + if (path.startsWith(PREFIX_CLASSPATH)) { + // it's a classpath resource + final String resource = path.substring(PREFIX_CLASSPATH.length()); + try { + @Cleanup final InputStream in = loadResourceAsStream(resource); + @Cleanup final OutputStream out = new FileOutputStream(sourcePath); + IOUtils.copyLarge(in, out); + setOriginalPath(path); + setPath(abs(sourcePath)); + } catch (Exception e) { + return die("initPath: error loading classpath resource "+resource+" : "+shortError(e)); + } + + } else if (isHttpOrHttps(path)) { + // it's a URL + try { + @Cleanup final InputStream in = HttpUtil.get(path); + @Cleanup final OutputStream out = new FileOutputStream(sourcePath); + IOUtils.copyLarge(in, out); + setOriginalPath(path); + setPath(abs(sourcePath)); + } catch (Exception e) { + return die("initPath: error loading URL resource "+path+" : "+shortError(e)); + } + + } else { + // must be a file + final File f = new File(path); + if (!f.exists()) return die("initPath: file path does not exist: "+path); + if (!f.canRead()) return die("initPath: file path is not readable: "+path); + } + return this; + } + +} diff --git a/src/main/java/jvcl/model/JFileExtension.java b/src/main/java/jvcl/model/JFileExtension.java new file mode 100644 index 0000000..9aef8d4 --- /dev/null +++ b/src/main/java/jvcl/model/JFileExtension.java @@ -0,0 +1,23 @@ +package jvcl.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public enum JFileExtension { + + mp4 (".mp4"), + mkv (".mkv"), + raw (".yuv"); + + @JsonCreator public static JFileExtension fromString(String v) { return valueOf(v.toLowerCase()); } + + public static boolean isValid(String v) { + try { fromString(v); return true; } catch (Exception ignored) {} + return false; + } + + private final String ext; + public String ext() { return ext; } + +} diff --git a/src/main/java/jvcl/model/JFormat.java b/src/main/java/jvcl/model/JFormat.java new file mode 100644 index 0000000..38f3e3b --- /dev/null +++ b/src/main/java/jvcl/model/JFormat.java @@ -0,0 +1,69 @@ +package jvcl.model; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import java.util.Arrays; +import java.util.Optional; + +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.reflect.ReflectionUtil.copy; + +@NoArgsConstructor @Accessors(chain=true) +public class JFormat { + + @Getter @Setter private Integer height; + public boolean hasHeight () { return height != null; } + + @Getter @Setter private Integer width; + public boolean hasWidth () { return width != null; } + + @Getter @Setter private JFileExtension fileExtension; + public boolean hasFileExtension() { return fileExtension != null; } + + public JFormat(JFormat format) { copy(this, format); } + + public void merge(JFormat other) { + if (!hasHeight()) setHeight(other.getHeight()); + if (!hasWidth()) setWidth(other.getWidth()); + if (!hasFileExtension()) setFileExtension(other.getFileExtension()); + } + + public static JFormat getFormat(JsonNode formatNode, JAsset[] sources) { + if (formatNode == null) { + // no format supplied, use format from first source + return new JFormat().setFileExtension(sources[0].getFormat().getFileExtension()); + } + if (formatNode.isObject()) { + return json(formatNode, JFormat.class); + + } else if (formatNode.isTextual()) { + final JFileExtension formatType; + final String formatTypeString = formatNode.textValue(); + if (!empty(formatTypeString)) { + // is the format the name of an input? + final Optional asset = Arrays.stream(sources).filter(s -> s.getName().equals(formatTypeString)).findFirst(); + if (asset.isEmpty()) { + // not the name of an asset, must be the name of a format + formatType = JFileExtension.valueOf(formatTypeString); + } else { + // it's the name of an asset, use that asset's format + formatType = asset.get().getFormat().getFileExtension(); + } + return new JFormat().setFileExtension(formatType); + } else { + // is the format a valid format type? + if (JFileExtension.isValid(formatTypeString)) return new JFormat().setFileExtension(JFileExtension.valueOf(formatTypeString)); + return die("getFormat: invalid format type: "+formatTypeString); + } + } else { + return die("getFormat: invalid format node: "+json(formatNode)); + } + } + +} diff --git a/src/main/java/jvcl/model/JOperation.java b/src/main/java/jvcl/model/JOperation.java new file mode 100644 index 0000000..95508e2 --- /dev/null +++ b/src/main/java/jvcl/model/JOperation.java @@ -0,0 +1,25 @@ +package jvcl.model; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +import static org.cobbzilla.util.daemon.ZillaRuntime.hashOf; +import static org.cobbzilla.util.json.JsonUtil.json; + +@NoArgsConstructor @Accessors(chain=true) +public class JOperation { + + @Getter @Setter private JOperationType operation; + @Getter @Setter private JsonNode creates; + @Getter @Setter private JsonNode perform; + + public String hash(JAsset[] sources) { return hash(sources, null); } + + public String hash(JAsset[] sources, Object[] args) { + return hashOf(getOperation(), json(creates), json(perform), json(sources), args); + } + +} diff --git a/src/main/java/jvcl/model/JOperationType.java b/src/main/java/jvcl/model/JOperationType.java new file mode 100644 index 0000000..b04a754 --- /dev/null +++ b/src/main/java/jvcl/model/JOperationType.java @@ -0,0 +1,31 @@ +package jvcl.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import jvcl.op.ConcatOperation; +import jvcl.op.SplitOperation; +import jvcl.op.TrimOperation; +import jvcl.service.AssetManager; +import jvcl.service.JOperator; +import jvcl.service.Toolbox; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public enum JOperationType { + + concat (new ConcatOperation()), + split (new SplitOperation()), + trim (new TrimOperation()), + overlay (null), + ken_burns (null), + letterbox (null), + split_silence (null); + + @JsonCreator public static JOperationType fromString(String v) { return valueOf(v.toLowerCase().replace("-", "_")); } + + private final JOperator operator; + + public void perform(JOperation op, Toolbox toolbox, AssetManager assetManager) { + operator.operate(op, toolbox, assetManager); + } + +} diff --git a/src/main/java/jvcl/model/JSpec.java b/src/main/java/jvcl/model/JSpec.java new file mode 100644 index 0000000..a74fcbe --- /dev/null +++ b/src/main/java/jvcl/model/JSpec.java @@ -0,0 +1,16 @@ +package jvcl.model; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.Accessors; + +@NoArgsConstructor @Accessors(chain=true) +public class JSpec { + + @Getter @Setter private JAsset[] assets; + @Getter @Setter private JOperation[] operations; + @Getter @Setter private JArtifact[] artifacts; + +} diff --git a/src/main/java/jvcl/model/info/JMedia.java b/src/main/java/jvcl/model/info/JMedia.java new file mode 100644 index 0000000..3d8f9a2 --- /dev/null +++ b/src/main/java/jvcl/model/info/JMedia.java @@ -0,0 +1,11 @@ +package jvcl.model.info; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; + +public class JMedia { + + @JsonProperty("@ref") @Getter @Setter private String ref; + @Getter @Setter private JTrack[] track; +} diff --git a/src/main/java/jvcl/model/info/JMediaInfo.java b/src/main/java/jvcl/model/info/JMediaInfo.java new file mode 100644 index 0000000..152ab12 --- /dev/null +++ b/src/main/java/jvcl/model/info/JMediaInfo.java @@ -0,0 +1,69 @@ +package jvcl.model.info; + +import jvcl.model.JFormat; +import jvcl.model.JFileExtension; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.math.BigDecimal; + +import static org.cobbzilla.util.daemon.ZillaRuntime.big; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +@Slf4j +public class JMediaInfo { + + @Getter @Setter private JMedia media; + @Getter(lazy=true) private final JFormat format = initFormat(); + public boolean hasFormat () { return getFormat() != null; } + + private JFormat initFormat () { + if (media == null || empty(media.getTrack())) return null; + JTrack general = null; + JTrack video = null; + JTrack audio = null; + for (int i=0; i ctx = new HashMap<>(); + ctx.put("ffmpeg", toolbox.getFfmpeg()); + ctx.put("sources", sources); + ctx.put("output", output); + final String script = HandlebarsUtil.apply(toolbox.getHandlebars(), CONCAT_RECODE_TEMPLATE_1, ctx); + + log.debug("operate: running script: "+script); + final String scriptOutput = execScript(script); + log.debug("operate: command output: "+scriptOutput); + assetManager.addOperationAsset(output); + } + + @NoArgsConstructor + private static class ConcatConfig { + @Getter @Setter private String[] concat; + } + +} diff --git a/src/main/java/jvcl/op/SplitOperation.java b/src/main/java/jvcl/op/SplitOperation.java new file mode 100644 index 0000000..b65801a --- /dev/null +++ b/src/main/java/jvcl/op/SplitOperation.java @@ -0,0 +1,92 @@ +package jvcl.op; + +import jvcl.model.JAsset; +import jvcl.model.JFileExtension; +import jvcl.model.JOperation; +import jvcl.service.AssetManager; +import jvcl.service.JOperator; +import jvcl.service.Toolbox; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.cobbzilla.util.handlebars.HandlebarsUtil; + +import java.io.File; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.HashMap; +import java.util.Map; + +import static jvcl.model.JAsset.json2asset; +import static org.cobbzilla.util.daemon.ZillaRuntime.big; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.json.JsonUtil.json; +import static org.cobbzilla.util.system.CommandShell.execScript; +import static org.cobbzilla.util.time.TimeUtil.parseDuration; + +@Slf4j +public class SplitOperation implements JOperator { + + public static final String SPLIT_TEMPLATE + = "{{ffmpeg}} -i {{{source.path}}} -ss {{startSeconds}} -t {{endSeconds}} {{{output.path}}}"; + + @Override public void operate(JOperation op, Toolbox toolbox, AssetManager assetManager) { + final SplitConfig config = json(json(op.getPerform()), SplitConfig.class); + + final JAsset source = assetManager.resolve(config.getSplit()); + + // create output object + final JAsset output = json2asset(op.getCreates()); + + // if any format settings are missing, use settings from source + output.mergeFormat(source.getFormat()); + + // get format type + final JFileExtension formatType = output.getFormat().getFileExtension(); + + final Map ctx = new HashMap<>(); + ctx.put("ffmpeg", toolbox.getFfmpeg()); + ctx.put("source", source); + final BigDecimal incr = config.getIntervalIncr(); + for (BigDecimal i = config.getStartTime(); + i.compareTo(config.getEndTime(source)) < 0; + i = i.add(incr)) { + + final File outfile = assetManager.assetPath(op, source, formatType, new Object[]{i, incr}); + if (outfile.exists()) { + log.info("operate: outfile exists, not re-creating: "+abs(outfile)); + return; + } + final JAsset slice = new JAsset(output); + slice.setPath(abs(outfile)); + + ctx.put("output", slice); + ctx.put("startSeconds", i); + ctx.put("endSeconds", i.add(incr)); + final String script = HandlebarsUtil.apply(toolbox.getHandlebars(), SPLIT_TEMPLATE, ctx); + log.debug("operate: running script: "+script); + final String scriptOutput = execScript(script); + log.debug("operate: command output: "+scriptOutput); + output.addAsset(slice); + } + assetManager.addOperationAsset(output); + } + + @NoArgsConstructor + private static class SplitConfig { + + @Getter @Setter private String split; + + @Getter @Setter private String interval; + public BigDecimal getIntervalIncr() { return big(parseDuration(interval)).divide(big(1000), RoundingMode.UNNECESSARY); } + + @Getter @Setter private String start; + public BigDecimal getStartTime() { return empty(start) ? BigDecimal.ZERO : big(start); } + + @Getter @Setter private String end; + public BigDecimal getEndTime(JAsset source) { return empty(end) ? source.duration() : big(end); } + } + +} diff --git a/src/main/java/jvcl/op/TrimOperation.java b/src/main/java/jvcl/op/TrimOperation.java new file mode 100644 index 0000000..f76952b --- /dev/null +++ b/src/main/java/jvcl/op/TrimOperation.java @@ -0,0 +1,13 @@ +package jvcl.op; + +import jvcl.model.JOperation; +import jvcl.service.AssetManager; +import jvcl.service.JOperator; +import jvcl.service.Toolbox; + +public class TrimOperation implements JOperator { + + @Override public void operate(JOperation op, Toolbox toolbox, AssetManager assetManager) { + } + +} diff --git a/src/main/java/jvcl/service/AssetManager.java b/src/main/java/jvcl/service/AssetManager.java new file mode 100644 index 0000000..c3f1409 --- /dev/null +++ b/src/main/java/jvcl/service/AssetManager.java @@ -0,0 +1,92 @@ +package jvcl.service; + +import jvcl.model.JAsset; +import jvcl.model.JFileExtension; +import jvcl.model.JOperation; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static jvcl.model.JAsset.NULL_ASSET; +import static org.cobbzilla.util.daemon.ZillaRuntime.die; +import static org.cobbzilla.util.daemon.ZillaRuntime.empty; + +public class AssetManager { + + public static final String OUTFILE_PREFIX = AssetManager.class.getSimpleName() + ".output."; + + private final Toolbox toolbox; + private final File scratchDir; + private final Map assets = new ConcurrentHashMap<>(); + + public AssetManager(Toolbox toolbox, File scratchDir) { + this.toolbox = toolbox; + this.scratchDir = scratchDir; + } + + public File sourcePath(String name) { return new File(scratchDir, OUTFILE_PREFIX + "source." + name); } + + public File assetPath(JOperation op, JAsset source, JFileExtension formatType) { + return assetPath(op, source, formatType, null); + } + + public File assetPath(JOperation op, JAsset source, JFileExtension formatType, Object[] args) { + return assetPath(op, new JAsset[]{source}, formatType, args); + } + + public File assetPath(JOperation op, JAsset[] sources, JFileExtension formatType) { + return assetPath(op, sources, formatType, null); + } + + public File assetPath(JOperation op, JAsset[] sources, JFileExtension formatType, Object[] args) { + return new File(scratchDir, OUTFILE_PREFIX + + op.hash(sources, args) + + (!empty(args) ? "_" + args[0] + (args.length > 1 ? "_" + args[1] : "") : "") + + formatType.ext()); + } + + public Map getAssets () { + final Map map = new HashMap<>(); + for (Map.Entry entry : assets.entrySet()) { + map.put(entry.getKey(), entry.getValue()); + } + return map; + } + + private String checkName(JAsset asset) { + final String name = asset.getName(); + if (assets.containsKey(name)) die("defineAsset: name already defined: "+ name); + return name; + } + + public void defineAsset(JAsset asset) { + final String name = checkName(asset); + assets.put(name, asset.init(this, toolbox)); + } + + public void addOperationAsset(JAsset asset) { + if (asset == null || asset == NULL_ASSET) return; + if (asset.hasList()) { + for (JAsset a : asset.getList()) addOperationAsset(a); + } else { + final String name = checkName(asset); + assets.put(name, asset.init(this, toolbox)); + } + } + + public JAsset[] resolve(String[] assets) { + final JAsset[] resolved = new JAsset[assets.length]; + for (int i=0; i infoCache = new ConcurrentHashMap<>(); + + public JMediaInfo getInfo(JAsset asset) { + final File infoFile = new File(replaceExt(asset.getPath(), ".json")); + final String infoPath = abs(infoFile); + if (!infoFile.exists() || infoFile.length() == 0) { + execScript(getMediainfo() + " --Output=JSON " + abs(asset.getPath())+" > "+infoPath); + } + if (!infoFile.exists() || infoFile.length() == 0) { + return die("getInfo: info file was not created or was empty: "+infoPath); + } + return infoCache.computeIfAbsent(infoPath, p -> json(FileUtil.toStringOrDie(infoFile), JMediaInfo.class, FULL_MAPPER_ALLOW_UNKNOWN_FIELDS)); + } + +} diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml new file mode 100644 index 0000000..ae77a9f --- /dev/null +++ b/src/main/resources/logback.xml @@ -0,0 +1,28 @@ + + + + + + true + + + + System.err + + [%d{yyyy-MM-dd HH:mm:ss.SSS}] [%level] [%thread] %logger{10} [%file:%line] %msg%n + + + + + + + + + + + + + + + + diff --git a/src/test/java/javicle/test/SplitTest.java b/src/test/java/javicle/test/SplitTest.java new file mode 100644 index 0000000..0952173 --- /dev/null +++ b/src/test/java/javicle/test/SplitTest.java @@ -0,0 +1,21 @@ +package javicle.test; + +import jvcl.main.Jvcl; +import jvcl.main.JvclOptions; +import lombok.Cleanup; +import org.junit.Test; + +import java.io.File; + +import static org.cobbzilla.util.io.FileUtil.abs; +import static org.cobbzilla.util.io.StreamUtil.loadResourceAsStream; +import static org.cobbzilla.util.io.StreamUtil.stream2file; + +public class SplitTest { + + @Test public void testSplit () throws Exception { + @Cleanup("delete") final File specFile = stream2file(loadResourceAsStream("tests/test_split.json")); + Jvcl.main(new String[]{JvclOptions.LONGOPT_SPEC, abs(specFile)}); + } + +} diff --git a/src/test/resources/sources/.gitignore b/src/test/resources/sources/.gitignore new file mode 100644 index 0000000..327c2b4 --- /dev/null +++ b/src/test/resources/sources/.gitignore @@ -0,0 +1,2 @@ +*.mp4 +*.json diff --git a/src/test/resources/tests/test_split.json b/src/test/resources/tests/test_split.json new file mode 100644 index 0000000..27ee9bc --- /dev/null +++ b/src/test/resources/tests/test_split.json @@ -0,0 +1,24 @@ +{ + "assets": [ + { + "name": "vid1", + "path": "https://archive.org/download/gov.archives.arc.1257628/gov.archives.arc.1257628_512kb.mp4", + "dest": "src/test/resources/sources/gov.archives.arc.1257628_512kb.mp4" + }, + { + "name": "vid2", + "path": "https://archive.org/download/gov.archives.arc.49442/gov.archives.arc.49442_512kb.mp4", + "dest": "src/test/resources/sources/gov.archives.arc.49442_512kb.mp4" + } + ], + "operations": [ + { + "operation": "split", // name of the operation + "creates": "vid1_split_%", // assets it creates, the '%' will be replaced with a counter + "perform": { + "split": "vid1", // split this source asset + "interval": "10s" // split every ten seconds + } + } + ] +} diff --git a/utils/cobbzilla-utils b/utils/cobbzilla-utils new file mode 160000 index 0000000..bcd403f --- /dev/null +++ b/utils/cobbzilla-utils @@ -0,0 +1 @@ +Subproject commit bcd403f6fdf14985cd99dcbeee257683abd47aaf